diff --git a/Dockerfile b/Dockerfile index 3e30da5..78b0c77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,4 +24,8 @@ RUN uv sync --frozen COPY . . -ENV PATH="/opt/airflow/project/.venv/bin:${PATH}" \ No newline at end of file +ENV PATH="/opt/airflow/project/.venv/bin:${PATH}" + + +# Airflow DAG folder +ENV AIRFLOW__CORE__DAGS_FOLDER=/opt/airflow/dags \ No newline at end of file diff --git a/Pipeline_config.yml b/Pipeline_config.yml index fc5e27a..64903c8 100644 --- a/Pipeline_config.yml +++ b/Pipeline_config.yml @@ -2,4 +2,3 @@ pipeline: error_message: null run_date: null status: null - last_successful_run_date: '2026-06-22' diff --git a/db_con/.env b/db_con/.env index f02885b..1af46fe 100644 --- a/db_con/.env +++ b/db_con/.env @@ -9,4 +9,4 @@ CH_HOST=172.188.12.194 CH_PORT=8123 CH_USER=default CH_PASS=dipanshu_k -CH_DB=kelloggs_1 \ No newline at end of file +CH_DB=kelloggs_z \ No newline at end of file diff --git a/elt_pipeline_custom_dates.yml b/elt_pipeline_custom_dates.yml new file mode 100644 index 0000000..26c97fe --- /dev/null +++ b/elt_pipeline_custom_dates.yml @@ -0,0 +1,5 @@ +pipeline: + start_date: '2026-06-01' + end_date: '2026-06-20' + flag: N + Note: ' In flag (yes-:Y and no-:N) ' \ No newline at end of file diff --git a/failed_Pipeline_dates_config.yml b/failed_Pipeline_dates_config.yml new file mode 100644 index 0000000..a1bb2a3 --- /dev/null +++ b/failed_Pipeline_dates_config.yml @@ -0,0 +1,3 @@ +- pipeline_trigeered_on_date: '2026-06-23' + failed_run_date: none + attempt: none diff --git a/logs/etl_20260623.log b/logs/etl_20260623.log new file mode 100644 index 0000000..b8d6b17 --- /dev/null +++ b/logs/etl_20260623.log @@ -0,0 +1,2187 @@ +2026-06-23 11:59:53 | INFO | ================================================================================ +2026-06-23 11:59:53 | INFO | Hello from data-move Python data pipeline! +2026-06-23 11:59:53 | INFO | Pipeline Run Date: 2023-01-01 +2026-06-23 11:59:53 | INFO | Connecting to databases... +2026-06-23 11:59:55 | INFO | +2026-06-23 11:59:58 | INFO | +2026-06-23 11:59:59 | INFO | Database connections established +2026-06-23 11:59:59 | INFO | Collecting MIDs for: 2023-01-01 +2026-06-23 11:59:59 | INFO | Found 0 MIDs +2026-06-23 12:00:00 | INFO | ================================================================================ +2026-06-23 12:00:00 | INFO | Processing Table-:SOS_OneApp | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 12:00:00 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-23 12:00:00 | WARNING | No MIDs — nothing to fetch. +2026-06-23 12:00:00 | INFO | Fetched total row -: 0 from sql server for table-:SOS_OneApp ...........!!! +2026-06-23 12:00:00 | WARNING | SOS_OneApp returned no rows +2026-06-23 12:00:00 | INFO | ================================================================================ +2026-06-23 12:00:00 | INFO | Processing Table-:OQaD | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 12:00:00 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-23 12:00:01 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-23 12:00:09 | INFO | Fetched 0 total empid and visitdate fetched for OQAD from SQL Server +2026-06-23 12:00:09 | WARNING | Table 'OQaD' does not exist. +2026-06-23 12:00:09 | WARNING | OQaD Matched df in OQaD returned no rows +2026-06-23 12:00:09 | INFO | Fetched 1 matched empids fetched for OQAD +2026-06-23 12:00:09 | INFO | Exists: True +2026-06-23 12:00:09 | INFO | Path: D:\z\airflow-project\src\sql\fact\OQaD.sql +2026-06-23 12:00:09 | INFO | Fetching data for 1 EMPIDs +2026-06-23 12:00:09 | INFO | Fetching OQaD data for run_date=2023-01-01 +2026-06-23 12:00:10 | INFO | fn name is fetch_OQad ------Fetched 0 rows +2026-06-23 12:00:10 | INFO | Fetched 0 rows from SQL Server +2026-06-23 12:00:10 | INFO | Fetched total row -: 0 from sql server for table-:OQaD ...........!!! +2026-06-23 12:00:10 | WARNING | OQaD returned no rows +2026-06-23 12:00:10 | INFO | ================================================================================ +2026-06-23 12:00:10 | INFO | Processing Table-:Survey | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 12:00:10 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-23 12:00:10 | WARNING | No MIDs — nothing to fetch. +2026-06-23 12:00:10 | INFO | Fetched total row -: 0 from sql server for table-:Survey ...........!!! +2026-06-23 12:00:10 | WARNING | Survey returned no rows +2026-06-23 12:00:10 | INFO | ================================================================================ +2026-06-23 12:00:10 | INFO | Processing Table-:additional_visibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 12:00:10 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-23 12:00:10 | WARNING | No MIDs — nothing to fetch. +2026-06-23 12:00:10 | INFO | Fetched total row -: 0 from sql server for table-:additional_visibility ...........!!! +2026-06-23 12:00:10 | WARNING | additional_visibility returned no rows +2026-06-23 12:00:10 | INFO | ================================================================================ +2026-06-23 12:00:10 | INFO | Processing Table-:Coverage | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 12:00:10 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-23 12:00:10 | WARNING | No MIDs — nothing to fetch. +2026-06-23 12:00:10 | INFO | Fetched total row -: 0 from sql server for table-:Coverage ...........!!! +2026-06-23 12:00:10 | WARNING | Coverage returned no rows +2026-06-23 12:00:10 | INFO | ================================================================================ +2026-06-23 12:00:10 | INFO | Processing Table-:Login | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 12:00:10 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-23 12:00:10 | INFO | Start Fetching data for these 0 MIDs or based on this date 2023-01-01 +2026-06-23 12:00:10 | INFO | Fetching data for 0 MIDs +2026-06-23 12:00:12 | INFO | Fetched 0 rows from SQL Server +2026-06-23 12:00:12 | INFO | Fetched total row -: 0 from sql server for table-:Login ...........!!! +2026-06-23 12:00:12 | WARNING | Login returned no rows +2026-06-23 12:00:12 | INFO | ================================================================================ +2026-06-23 12:00:12 | INFO | Processing Table-:Stock_Details | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 12:00:12 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-23 12:00:12 | WARNING | No MIDs — nothing to fetch. +2026-06-23 12:00:12 | INFO | Fetched total row -: 0 from sql server for table-:Stock_Details ...........!!! +2026-06-23 12:00:12 | WARNING | Stock_Details returned no rows +2026-06-23 12:00:12 | INFO | ================================================================================ +2026-06-23 12:00:12 | INFO | Processing Table-:Attendance | Table type -:FACT | fetcht by-:run_date | operation-:DELETE+INSERT +2026-06-23 12:00:12 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-23 12:00:12 | INFO | Fetching Attendance data from 2022-12-17 to 2023-01-01 +2026-06-23 12:00:16 | INFO | Fetched 5,481 attendance rows for 347 employees +2026-06-23 12:00:16 | INFO | Fetched total row -: 5481 from sql server for table-:Attendance ...........!!! +2026-06-23 12:00:16 | INFO | Fetched 5481 rows +2026-06-23 12:00:16 | INFO | Creating table Attendance +2026-06-23 12:00:16 | INFO | Table ready: Attendance +2026-06-23 12:00:16 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 12:00:17 | INFO | Attendance: inserted 5,481 rows into ClickHouse +2026-06-23 12:00:17 | INFO | Attendance loaded successfully (5481 rows) +2026-06-23 12:00:17 | INFO | ================================================================================ +2026-06-23 12:00:17 | INFO | Processing Table-:Journey_Plan | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 12:00:17 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-23 12:00:17 | WARNING | No MIDs — nothing to fetch. +2026-06-23 12:00:17 | INFO | Fetched total row -: 0 from sql server for table-:Journey_Plan ...........!!! +2026-06-23 12:00:17 | WARNING | Journey_Plan returned no rows +2026-06-23 12:00:17 | INFO | ================================================================================ +2026-06-23 12:00:17 | INFO | Processing Table-:Web_Logins | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 12:00:17 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-23 12:00:17 | WARNING | No MIDs — nothing to fetch. +2026-06-23 12:00:17 | INFO | Fetched total row -: 0 from sql server for table-:Web_Logins ...........!!! +2026-06-23 12:00:17 | WARNING | Web_Logins returned no rows +2026-06-23 12:00:17 | INFO | ================================================================================ +2026-06-23 12:00:17 | INFO | Processing Table-:Promotion | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 12:00:17 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-23 12:00:17 | WARNING | No MIDs — nothing to fetch. +2026-06-23 12:00:17 | INFO | Fetched total row -: 0 from sql server for table-:Promotion ...........!!! +2026-06-23 12:00:17 | WARNING | Promotion returned no rows +2026-06-23 12:00:17 | INFO | ================================================================================ +2026-06-23 12:00:17 | INFO | Processing Table-:PaidVisibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 12:00:17 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-23 12:00:17 | WARNING | No MIDs — nothing to fetch. +2026-06-23 12:00:17 | INFO | Fetched total row -: 0 from sql server for table-:PaidVisibility ...........!!! +2026-06-23 12:00:17 | WARNING | PaidVisibility returned no rows +2026-06-23 12:00:17 | INFO | ================================================================================ +2026-06-23 12:00:17 | INFO | Processing Table-:Store_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 12:00:17 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-23 12:00:17 | INFO | Fetching data from sql server for Master table...... +2026-06-23 12:00:17 | INFO | Fetching in progress .... +2026-06-23 12:00:20 | INFO | Fetched 5,998 rows from SQL Server +2026-06-23 12:00:20 | INFO | Fetched total row -: 5998 from sql server for table-:Store_Master ...........!!! +2026-06-23 12:00:20 | INFO | Fetched 5998 rows +2026-06-23 12:00:20 | INFO | Creating table Store_Master +2026-06-23 12:00:21 | INFO | Table ready: Store_Master +2026-06-23 12:00:21 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 12:00:21 | INFO | Store_Master: inserted 5,998 rows into ClickHouse +2026-06-23 12:00:21 | INFO | Store_Master loaded successfully (5998 rows) +2026-06-23 12:00:21 | INFO | ================================================================================ +2026-06-23 12:00:21 | INFO | Processing Table-:SKU_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 12:00:21 | INFO | Fetching Data from sql server for table-: SKU_Master .............. +2026-06-23 12:00:21 | INFO | Fetching data from sql server for Master table...... +2026-06-23 12:00:21 | INFO | Fetching in progress .... +2026-06-23 12:00:21 | INFO | Fetched 160 rows from SQL Server +2026-06-23 12:00:21 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! +2026-06-23 12:00:21 | INFO | Fetched 160 rows +2026-06-23 12:00:21 | INFO | Creating table SKU_Master +2026-06-23 12:00:22 | INFO | Table ready: SKU_Master +2026-06-23 12:00:22 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 12:00:22 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-23 12:00:22 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-23 12:00:22 | INFO | ================================================================================ +2026-06-23 12:00:22 | INFO | Processing Table-:display_master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 12:00:22 | INFO | Fetching Data from sql server for table-: display_master .............. +2026-06-23 12:00:22 | INFO | Fetching data from sql server for Master table...... +2026-06-23 12:00:22 | INFO | Fetching in progress .... +2026-06-23 12:00:22 | INFO | Fetched 135 rows from SQL Server +2026-06-23 12:00:22 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! +2026-06-23 12:00:22 | INFO | Fetched 135 rows +2026-06-23 12:00:22 | INFO | Creating table display_master +2026-06-23 12:00:22 | INFO | Table ready: display_master +2026-06-23 12:00:22 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 12:00:22 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-23 12:00:22 | INFO | display_master loaded successfully (135 rows) +2026-06-23 12:00:22 | INFO | ================================================================================ +2026-06-23 12:00:22 | INFO | Processing Table-:Employee_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 12:00:22 | INFO | Fetching Data from sql server for table-: Employee_Master .............. +2026-06-23 12:00:22 | INFO | Fetching data from sql server for Master table...... +2026-06-23 12:00:22 | INFO | Fetching in progress .... +2026-06-23 12:00:23 | INFO | Fetched 2,289 rows from SQL Server +2026-06-23 12:00:23 | INFO | Fetched total row -: 2289 from sql server for table-:Employee_Master ...........!!! +2026-06-23 12:00:23 | INFO | Fetched 2289 rows +2026-06-23 12:00:23 | INFO | Creating table Employee_Master +2026-06-23 12:00:23 | INFO | Table ready: Employee_Master +2026-06-23 12:00:23 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 12:00:24 | INFO | Employee_Master: inserted 2,289 rows into ClickHouse +2026-06-23 12:00:24 | INFO | Employee_Master loaded successfully (2289 rows) +2026-06-23 12:00:24 | INFO | ================================================================================ +2026-06-23 12:00:24 | INFO | Processing Table-:coverage_remarks | Table type -:DIMENSION | fetcht by-:reason_id | operation-:DELETE+INSERT +2026-06-23 12:00:24 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. +2026-06-23 12:00:24 | WARNING | Table 'coverage_remarks' does not exist. During collecting reason_ids +2026-06-23 12:00:24 | INFO | Fetching data from sql server for Master table...... +2026-06-23 12:00:24 | INFO | Fetching in progress .... +2026-06-23 12:00:24 | INFO | Fetched 29 rows from SQL Server +2026-06-23 12:00:24 | INFO | Fetched 29 rows from SQL Server +2026-06-23 12:00:24 | INFO | Fetched total row -: 29 from sql server for table-:coverage_remarks ...........!!! +2026-06-23 12:00:24 | INFO | Fetched 29 rows +2026-06-23 12:00:25 | INFO | Creating table coverage_remarks +2026-06-23 12:00:25 | INFO | Table ready: coverage_remarks +2026-06-23 12:00:25 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 12:00:25 | INFO | coverage_remarks: inserted 29 rows into ClickHouse +2026-06-23 12:00:25 | INFO | coverage_remarks loaded successfully (29 rows) +2026-06-23 12:00:25 | INFO | ================================================================================ +2026-06-23 12:00:25 | INFO | Processing Table-:mapping_storevisibility | Table type -:BRIDGE | fetcht by-:run_date | operation-:ONLY_INSERT +2026-06-23 12:00:25 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-23 12:00:25 | WARNING | Table 'mapping_storevisibility' does not exist. During collecting store_ids +2026-06-23 12:00:25 | INFO | Fetching data from sql server for Master table...... +2026-06-23 12:00:25 | INFO | Fetching in progress .... +2026-06-23 12:00:25 | INFO | Fetched 0 rows from SQL Server +2026-06-23 12:00:25 | INFO | Fetched 0 rows from SQL Server +2026-06-23 12:00:25 | INFO | Fetched total row -: 0 from sql server for table-:mapping_storevisibility ...........!!! +2026-06-23 12:00:25 | WARNING | mapping_storevisibility returned no rows +2026-06-23 12:00:25 | INFO | ================================================================================ +2026-06-23 12:00:25 | INFO | Processing Table-:Master_VisibilityReason | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-23 12:00:25 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-23 12:00:25 | INFO | Fetching data from sql server for Master table...... +2026-06-23 12:00:25 | INFO | Fetching in progress .... +2026-06-23 12:00:25 | INFO | Fetched 17 rows from SQL Server +2026-06-23 12:00:25 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-23 12:00:25 | INFO | Fetched 17 rows +2026-06-23 12:00:26 | INFO | Creating table Master_VisibilityReason +2026-06-23 12:00:26 | INFO | Table ready: Master_VisibilityReason +2026-06-23 12:00:26 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 12:00:26 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-23 12:00:26 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-23 12:00:26 | INFO | ================================================================================ +2026-06-23 12:00:26 | INFO | Processing Table-:Master_VisibilityDefinition | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-23 12:00:26 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. +2026-06-23 12:00:26 | INFO | Fetching data from sql server for Master table...... +2026-06-23 12:00:26 | INFO | Fetching in progress .... +2026-06-23 12:00:26 | INFO | Fetched 861 rows from SQL Server +2026-06-23 12:00:26 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! +2026-06-23 12:00:26 | INFO | Fetched 861 rows +2026-06-23 12:00:26 | INFO | Creating table Master_VisibilityDefinition +2026-06-23 12:00:26 | INFO | Table ready: Master_VisibilityDefinition +2026-06-23 12:00:26 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 12:00:26 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-23 12:00:26 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-23 12:00:26 | INFO | ================================================================================ +2026-06-23 12:00:26 | INFO | Processing Table-:Master_Salesterritorylayer | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-23 12:00:26 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. +2026-06-23 12:00:26 | INFO | Fetching data from sql server for Master table...... +2026-06-23 12:00:26 | INFO | Fetching in progress .... +2026-06-23 12:00:27 | INFO | Fetched 33 rows from SQL Server +2026-06-23 12:00:27 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! +2026-06-23 12:00:27 | INFO | Fetched 33 rows +2026-06-23 12:00:27 | INFO | Creating table Master_Salesterritorylayer +2026-06-23 12:00:27 | INFO | Table ready: Master_Salesterritorylayer +2026-06-23 12:00:27 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 12:00:27 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-23 12:00:27 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-23 12:00:27 | INFO | ================================================================================ +2026-06-23 12:00:27 | INFO | Pipeline Completed Successfully +2026-06-23 12:00:27 | INFO | ================================================================================ +2026-06-23 12:00:27 | INFO | Pipeline completed successfully. last_successful_run_date=2026-06-23 +2026-06-23 16:01:09 | INFO | Pipeline Start Date: 2026-06-22 +2026-06-23 16:01:09 | INFO | ================================================================================ +2026-06-23 16:01:09 | INFO | Hello from data-move Python data pipeline! +2026-06-23 16:01:09 | INFO | Pipeline Run Date: 2026-06-22 +2026-06-23 16:01:09 | INFO | Connecting to databases... +2026-06-23 16:01:10 | INFO | +2026-06-23 16:01:13 | INFO | +2026-06-23 16:01:13 | INFO | Database connections established +2026-06-23 16:01:13 | INFO | Collecting MIDs for: 2026-06-22 +2026-06-23 16:01:14 | INFO | Found 882 MIDs +2026-06-23 16:01:14 | INFO | ================================================================================ +2026-06-23 16:01:14 | INFO | Processing Table-:SOS_OneApp | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:01:14 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-23 16:01:14 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:01:14 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:01:14 | INFO | Fetching data for 882 MIDs +2026-06-23 16:01:25 | INFO | Fetched 3,888 rows from SQL Server +2026-06-23 16:01:25 | INFO | Fetched total row -: 3888 from sql server for table-:SOS_OneApp ...........!!! +2026-06-23 16:01:25 | INFO | Fetched 3888 rows +2026-06-23 16:01:26 | INFO | Creating table SOS_OneApp +2026-06-23 16:01:26 | INFO | Table ready: SOS_OneApp +2026-06-23 16:01:26 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:01:26 | INFO | SOS_OneApp: inserted 3,888 rows into ClickHouse +2026-06-23 16:01:26 | INFO | SOS_OneApp loaded successfully (3888 rows) +2026-06-23 16:01:26 | INFO | ================================================================================ +2026-06-23 16:01:26 | INFO | Processing Table-:OQaD | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:01:26 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-23 16:01:27 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-23 16:02:05 | INFO | Fetched 291,599 total empid and visitdate fetched for OQAD from SQL Server +2026-06-23 16:02:05 | WARNING | Table 'OQaD' does not exist. +2026-06-23 16:02:05 | WARNING | OQaD Matched df in OQaD returned no rows +2026-06-23 16:02:05 | INFO | Fetched 1 matched empids fetched for OQAD +2026-06-23 16:02:05 | INFO | Exists: True +2026-06-23 16:02:05 | INFO | Path: D:\z\airflow-project\src\sql\fact\OQaD.sql +2026-06-23 16:02:05 | INFO | Fetching data for 1 EMPIDs +2026-06-23 16:02:05 | INFO | Fetching OQaD data for run_date=2026-06-22 +2026-06-23 16:08:39 | INFO | Pipeline Start Date: 2026-06-22 +2026-06-23 16:08:39 | INFO | ================================================================================ +2026-06-23 16:08:39 | INFO | Hello from data-move Python data pipeline! +2026-06-23 16:08:39 | INFO | Pipeline Run Date: 2026-06-22 +2026-06-23 16:08:39 | INFO | Connecting to databases... +2026-06-23 16:08:40 | INFO | +2026-06-23 16:08:41 | INFO | +2026-06-23 16:08:41 | INFO | Database connections established +2026-06-23 16:08:41 | INFO | Collecting MIDs for: 2026-06-22 +2026-06-23 16:08:42 | INFO | Found 882 MIDs +2026-06-23 16:08:42 | INFO | ================================================================================ +2026-06-23 16:08:42 | INFO | Processing Table-:SOS_OneApp | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:08:42 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-23 16:08:42 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:08:42 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:08:42 | INFO | Fetching data for 882 MIDs +2026-06-23 16:08:52 | INFO | Fetched 3,888 rows from SQL Server +2026-06-23 16:08:52 | INFO | Fetched total row -: 3888 from sql server for table-:SOS_OneApp ...........!!! +2026-06-23 16:08:52 | INFO | Fetched 3888 rows +2026-06-23 16:08:52 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-23 16:08:52 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:08:52 | INFO | SOS_OneApp: inserted 3,888 rows into ClickHouse +2026-06-23 16:08:52 | INFO | SOS_OneApp loaded successfully (3888 rows) +2026-06-23 16:08:52 | INFO | ================================================================================ +2026-06-23 16:08:52 | INFO | Processing Table-:Survey | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:08:52 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-23 16:08:52 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:08:52 | INFO | Fetching data for 882 MIDs +2026-06-23 16:08:53 | INFO | Fetched 152 rows from SQL Server +2026-06-23 16:08:53 | INFO | Fetched total row -: 152 from sql server for table-:Survey ...........!!! +2026-06-23 16:08:53 | INFO | Fetched 152 rows +2026-06-23 16:08:53 | INFO | Creating table Survey +2026-06-23 16:08:53 | INFO | Table ready: Survey +2026-06-23 16:08:53 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:08:53 | INFO | Survey: inserted 152 rows into ClickHouse +2026-06-23 16:08:53 | INFO | Survey loaded successfully (152 rows) +2026-06-23 16:08:53 | INFO | ================================================================================ +2026-06-23 16:08:53 | INFO | Processing Table-:additional_visibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:08:53 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-23 16:08:53 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:08:53 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:08:53 | INFO | Fetching data for 882 MIDs +2026-06-23 16:08:54 | INFO | Fetched 2,136 rows from SQL Server +2026-06-23 16:08:54 | INFO | Fetched total row -: 2136 from sql server for table-:additional_visibility ...........!!! +2026-06-23 16:08:54 | INFO | Fetched 2136 rows +2026-06-23 16:08:54 | INFO | Creating table additional_visibility +2026-06-23 16:08:54 | INFO | Table ready: additional_visibility +2026-06-23 16:08:54 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:08:54 | INFO | additional_visibility: inserted 2,136 rows into ClickHouse +2026-06-23 16:08:54 | INFO | additional_visibility loaded successfully (2136 rows) +2026-06-23 16:08:54 | INFO | ================================================================================ +2026-06-23 16:08:54 | INFO | Processing Table-:Coverage | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:08:54 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-23 16:08:54 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:08:54 | INFO | Fetching data for 882 MIDs +2026-06-23 16:09:06 | INFO | Fetched 825 rows from SQL Server +2026-06-23 16:09:06 | INFO | Fetched total row -: 825 from sql server for table-:Coverage ...........!!! +2026-06-23 16:09:06 | INFO | Fetched 825 rows +2026-06-23 16:09:06 | INFO | Creating table Coverage +2026-06-23 16:09:06 | INFO | Table ready: Coverage +2026-06-23 16:09:06 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:09:07 | INFO | Coverage: inserted 825 rows into ClickHouse +2026-06-23 16:09:07 | INFO | Coverage loaded successfully (825 rows) +2026-06-23 16:09:07 | INFO | ================================================================================ +2026-06-23 16:09:07 | INFO | Processing Table-:Login | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:09:07 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-23 16:09:07 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:09:07 | INFO | Fetching data for 882 MIDs +2026-06-23 16:09:09 | INFO | Fetched 501 rows from SQL Server +2026-06-23 16:09:09 | INFO | Fetched total row -: 501 from sql server for table-:Login ...........!!! +2026-06-23 16:09:09 | INFO | Fetched 501 rows +2026-06-23 16:09:09 | INFO | Creating table Login +2026-06-23 16:09:09 | INFO | Table ready: Login +2026-06-23 16:09:09 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:09:09 | INFO | Login: inserted 501 rows into ClickHouse +2026-06-23 16:09:09 | INFO | Login loaded successfully (501 rows) +2026-06-23 16:09:09 | INFO | ================================================================================ +2026-06-23 16:09:09 | INFO | Processing Table-:Stock_Details | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:09:09 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-23 16:09:09 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:09:09 | INFO | Fetching data for 882 MIDs +2026-06-23 16:09:13 | INFO | Fetched 44,061 rows from SQL Server +2026-06-23 16:09:13 | INFO | Fetched total row -: 44061 from sql server for table-:Stock_Details ...........!!! +2026-06-23 16:09:13 | INFO | Fetched 44061 rows +2026-06-23 16:09:13 | INFO | Creating table Stock_Details +2026-06-23 16:09:13 | INFO | Table ready: Stock_Details +2026-06-23 16:09:13 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:09:14 | INFO | Stock_Details: inserted 44,061 rows into ClickHouse +2026-06-23 16:09:14 | INFO | Stock_Details loaded successfully (44061 rows) +2026-06-23 16:09:14 | INFO | ================================================================================ +2026-06-23 16:09:14 | INFO | Processing Table-:Attendance | Table type -:FACT | fetcht by-:run_date | operation-:DELETE+INSERT +2026-06-23 16:09:14 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-23 16:09:14 | INFO | Fetching Attendance data from 2026-06-07 to 2026-06-22 +2026-06-23 16:09:29 | INFO | Fetched 121,144 attendance rows for 542 employees +2026-06-23 16:09:29 | INFO | Fetched total row -: 121144 from sql server for table-:Attendance ...........!!! +2026-06-23 16:09:29 | INFO | Fetched 121144 rows +2026-06-23 16:09:29 | INFO | Truncated table Attendance +2026-06-23 16:09:29 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:09:34 | INFO | Attendance: inserted 121,144 rows into ClickHouse +2026-06-23 16:09:34 | INFO | Attendance loaded successfully (121144 rows) +2026-06-23 16:09:34 | INFO | ================================================================================ +2026-06-23 16:09:34 | INFO | Processing Table-:Journey_Plan | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:09:34 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-23 16:09:34 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:09:34 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:09:34 | INFO | Fetching data for 882 MIDs +2026-06-23 16:09:35 | INFO | Fetched 21,737 rows from SQL Server +2026-06-23 16:09:35 | INFO | Fetched total row -: 21737 from sql server for table-:Journey_Plan ...........!!! +2026-06-23 16:09:35 | INFO | Fetched 21737 rows +2026-06-23 16:09:35 | INFO | Creating table Journey_Plan +2026-06-23 16:09:36 | INFO | Table ready: Journey_Plan +2026-06-23 16:09:36 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:09:36 | INFO | Journey_Plan: inserted 21,737 rows into ClickHouse +2026-06-23 16:09:36 | INFO | Journey_Plan loaded successfully (21737 rows) +2026-06-23 16:09:36 | INFO | ================================================================================ +2026-06-23 16:09:36 | INFO | Processing Table-:Web_Logins | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:09:36 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-23 16:09:36 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:09:36 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:09:36 | INFO | Fetching data for 882 MIDs +2026-06-23 16:09:37 | INFO | Fetched 214 rows from SQL Server +2026-06-23 16:09:37 | INFO | Fetched total row -: 214 from sql server for table-:Web_Logins ...........!!! +2026-06-23 16:09:37 | INFO | Fetched 214 rows +2026-06-23 16:09:37 | INFO | Creating table Web_Logins +2026-06-23 16:09:37 | INFO | Table ready: Web_Logins +2026-06-23 16:09:37 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:09:37 | INFO | Web_Logins: inserted 214 rows into ClickHouse +2026-06-23 16:09:37 | INFO | Web_Logins loaded successfully (214 rows) +2026-06-23 16:09:37 | INFO | ================================================================================ +2026-06-23 16:09:37 | INFO | Processing Table-:Promotion | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:09:37 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-23 16:09:37 | INFO | Fetching data for 882 MIDs +2026-06-23 16:09:40 | INFO | Fetched 8,835 rows from SQL Server +2026-06-23 16:09:40 | INFO | Fetched total row -: 8835 from sql server for table-:Promotion ...........!!! +2026-06-23 16:09:40 | INFO | Fetched 8835 rows +2026-06-23 16:09:40 | INFO | Creating table Promotion +2026-06-23 16:09:40 | INFO | Table ready: Promotion +2026-06-23 16:09:40 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:09:41 | INFO | Promotion: inserted 8,835 rows into ClickHouse +2026-06-23 16:09:41 | INFO | Promotion loaded successfully (8835 rows) +2026-06-23 16:09:41 | INFO | ================================================================================ +2026-06-23 16:09:41 | INFO | Processing Table-:PaidVisibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:09:41 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-23 16:09:41 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:09:41 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:09:41 | INFO | Fetching data for 882 MIDs +2026-06-23 16:09:43 | INFO | Fetched 1,029 rows from SQL Server +2026-06-23 16:09:43 | INFO | Fetched total row -: 1029 from sql server for table-:PaidVisibility ...........!!! +2026-06-23 16:09:43 | INFO | Fetched 1029 rows +2026-06-23 16:09:43 | INFO | Creating table PaidVisibility +2026-06-23 16:09:43 | INFO | Table ready: PaidVisibility +2026-06-23 16:09:43 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:09:43 | INFO | PaidVisibility: inserted 1,029 rows into ClickHouse +2026-06-23 16:09:43 | INFO | PaidVisibility loaded successfully (1029 rows) +2026-06-23 16:09:43 | INFO | ================================================================================ +2026-06-23 16:09:43 | INFO | Processing Table-:Store_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:09:43 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-23 16:09:43 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:09:43 | INFO | Fetching in progress .... +2026-06-23 16:09:45 | INFO | Fetched 5,998 rows from SQL Server +2026-06-23 16:09:45 | INFO | Fetched total row -: 5998 from sql server for table-:Store_Master ...........!!! +2026-06-23 16:09:45 | INFO | Fetched 5998 rows +2026-06-23 16:09:45 | INFO | Truncated table Store_Master +2026-06-23 16:09:45 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:09:48 | INFO | Store_Master: inserted 5,998 rows into ClickHouse +2026-06-23 16:09:48 | INFO | Store_Master loaded successfully (5998 rows) +2026-06-23 16:09:48 | INFO | ================================================================================ +2026-06-23 16:09:48 | INFO | Processing Table-:SKU_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:09:48 | INFO | Fetching Data from sql server for table-: SKU_Master .............. +2026-06-23 16:09:48 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:09:48 | INFO | Fetching in progress .... +2026-06-23 16:09:48 | INFO | Fetched 160 rows from SQL Server +2026-06-23 16:09:48 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! +2026-06-23 16:09:48 | INFO | Fetched 160 rows +2026-06-23 16:09:49 | INFO | Truncated table SKU_Master +2026-06-23 16:09:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:09:49 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-23 16:09:49 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-23 16:09:49 | INFO | ================================================================================ +2026-06-23 16:09:49 | INFO | Processing Table-:display_master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:09:49 | INFO | Fetching Data from sql server for table-: display_master .............. +2026-06-23 16:09:49 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:09:49 | INFO | Fetching in progress .... +2026-06-23 16:09:49 | INFO | Fetched 135 rows from SQL Server +2026-06-23 16:09:49 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! +2026-06-23 16:09:49 | INFO | Fetched 135 rows +2026-06-23 16:09:49 | INFO | Truncated table display_master +2026-06-23 16:09:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:09:49 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-23 16:09:49 | INFO | display_master loaded successfully (135 rows) +2026-06-23 16:09:49 | INFO | ================================================================================ +2026-06-23 16:09:49 | INFO | Processing Table-:Employee_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:09:49 | INFO | Fetching Data from sql server for table-: Employee_Master .............. +2026-06-23 16:09:49 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:09:49 | INFO | Fetching in progress .... +2026-06-23 16:09:50 | INFO | Fetched 2,289 rows from SQL Server +2026-06-23 16:09:50 | INFO | Fetched total row -: 2289 from sql server for table-:Employee_Master ...........!!! +2026-06-23 16:09:50 | INFO | Fetched 2289 rows +2026-06-23 16:09:50 | INFO | Truncated table Employee_Master +2026-06-23 16:09:50 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:09:51 | INFO | Employee_Master: inserted 2,289 rows into ClickHouse +2026-06-23 16:09:51 | INFO | Employee_Master loaded successfully (2289 rows) +2026-06-23 16:09:51 | INFO | ================================================================================ +2026-06-23 16:09:51 | INFO | Processing Table-:coverage_remarks | Table type -:DIMENSION | fetcht by-:reason_id | operation-:DELETE+INSERT +2026-06-23 16:09:51 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. +2026-06-23 16:09:52 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:09:52 | INFO | Fetching in progress .... +2026-06-23 16:09:52 | INFO | Fetched 0 rows from SQL Server +2026-06-23 16:09:52 | INFO | Fetched 0 rows from SQL Server +2026-06-23 16:09:52 | INFO | Fetched total row -: 0 from sql server for table-:coverage_remarks ...........!!! +2026-06-23 16:09:52 | WARNING | coverage_remarks returned no rows +2026-06-23 16:09:52 | INFO | ================================================================================ +2026-06-23 16:09:52 | INFO | Processing Table-:mapping_storevisibility | Table type -:BRIDGE | fetcht by-:none | operation-:DELETE+INSERT +2026-06-23 16:09:52 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-23 16:09:52 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:09:52 | ERROR | Failed processing table mapping_storevisibility +Traceback (most recent call last): + File "D:\z\airflow-project\main2.py", line 194, in elt + df = fetch_data(sql_engine ,table_name,table_type) + File "D:\z\airflow-project\src\dim.py", line 38, in fetch_data + with open(sql_file, "r", encoding="utf-8") as f: + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'src\\sql\\dim\\mapping_storevisibility.sql' +2026-06-23 16:09:52 | WARNING | Pipeline failed. Retry 1/3. Error: [Errno 2] No such file or directory: 'src\\sql\\dim\\mapping_storevisibility.sql' +2026-06-23 16:09:57 | INFO | ================================================================================ +2026-06-23 16:09:57 | INFO | Hello from data-move Python data pipeline! +2026-06-23 16:09:57 | INFO | Pipeline Run Date: 2026-06-22 +2026-06-23 16:09:57 | INFO | Connecting to databases... +2026-06-23 16:09:57 | INFO | +2026-06-23 16:09:58 | INFO | +2026-06-23 16:09:59 | INFO | Database connections established +2026-06-23 16:09:59 | INFO | Collecting MIDs for: 2026-06-22 +2026-06-23 16:09:59 | INFO | Found 882 MIDs +2026-06-23 16:09:59 | INFO | ================================================================================ +2026-06-23 16:09:59 | INFO | Processing Table-:SOS_OneApp | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:09:59 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-23 16:09:59 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:09:59 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:09:59 | INFO | Fetching data for 882 MIDs +2026-06-23 16:12:37 | INFO | Pipeline Start Date: 2026-06-22 +2026-06-23 16:12:37 | INFO | ================================================================================ +2026-06-23 16:12:37 | INFO | Hello from data-move Python data pipeline! +2026-06-23 16:12:37 | INFO | Pipeline Run Date: 2026-06-22 +2026-06-23 16:12:37 | INFO | Connecting to databases... +2026-06-23 16:12:38 | INFO | +2026-06-23 16:12:39 | INFO | +2026-06-23 16:12:39 | INFO | Database connections established +2026-06-23 16:12:39 | INFO | Collecting MIDs for: 2026-06-22 +2026-06-23 16:12:40 | INFO | Found 882 MIDs +2026-06-23 16:12:40 | INFO | ================================================================================ +2026-06-23 16:12:40 | INFO | Processing Table-:SOS_OneApp | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:12:40 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-23 16:12:40 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:12:40 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:12:40 | INFO | Fetching data for 882 MIDs +2026-06-23 16:12:45 | INFO | Fetched 3,888 rows from SQL Server +2026-06-23 16:12:45 | INFO | Fetched total row -: 3888 from sql server for table-:SOS_OneApp ...........!!! +2026-06-23 16:12:45 | INFO | Fetched 3888 rows +2026-06-23 16:12:45 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-23 16:12:45 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:12:45 | INFO | SOS_OneApp: inserted 3,888 rows into ClickHouse +2026-06-23 16:12:45 | INFO | SOS_OneApp loaded successfully (3888 rows) +2026-06-23 16:12:45 | INFO | ================================================================================ +2026-06-23 16:12:46 | INFO | Processing Table-:Survey | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:12:46 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-23 16:12:46 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:12:46 | INFO | Fetching data for 882 MIDs +2026-06-23 16:12:46 | INFO | Fetched 152 rows from SQL Server +2026-06-23 16:12:46 | INFO | Fetched total row -: 152 from sql server for table-:Survey ...........!!! +2026-06-23 16:12:46 | INFO | Fetched 152 rows +2026-06-23 16:12:46 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-23 16:12:46 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:12:46 | INFO | Survey: inserted 152 rows into ClickHouse +2026-06-23 16:12:46 | INFO | Survey loaded successfully (152 rows) +2026-06-23 16:12:46 | INFO | ================================================================================ +2026-06-23 16:12:46 | INFO | Processing Table-:additional_visibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:12:46 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-23 16:12:46 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:12:46 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:12:46 | INFO | Fetching data for 882 MIDs +2026-06-23 16:12:47 | INFO | Fetched 2,136 rows from SQL Server +2026-06-23 16:12:47 | INFO | Fetched total row -: 2136 from sql server for table-:additional_visibility ...........!!! +2026-06-23 16:12:47 | INFO | Fetched 2136 rows +2026-06-23 16:12:47 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-23 16:12:47 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:12:47 | INFO | additional_visibility: inserted 2,136 rows into ClickHouse +2026-06-23 16:12:47 | INFO | additional_visibility loaded successfully (2136 rows) +2026-06-23 16:12:47 | INFO | ================================================================================ +2026-06-23 16:12:47 | INFO | Processing Table-:Coverage | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:12:47 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-23 16:12:47 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:12:47 | INFO | Fetching data for 882 MIDs +2026-06-23 16:12:58 | INFO | Fetched 825 rows from SQL Server +2026-06-23 16:12:58 | INFO | Fetched total row -: 825 from sql server for table-:Coverage ...........!!! +2026-06-23 16:12:58 | INFO | Fetched 825 rows +2026-06-23 16:12:58 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-23 16:12:58 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:12:58 | INFO | Coverage: inserted 825 rows into ClickHouse +2026-06-23 16:12:58 | INFO | Coverage loaded successfully (825 rows) +2026-06-23 16:12:58 | INFO | ================================================================================ +2026-06-23 16:12:58 | INFO | Processing Table-:Login | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:12:58 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-23 16:12:58 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:12:58 | INFO | Fetching data for 882 MIDs +2026-06-23 16:13:00 | INFO | Fetched 501 rows from SQL Server +2026-06-23 16:13:00 | INFO | Fetched total row -: 501 from sql server for table-:Login ...........!!! +2026-06-23 16:13:00 | INFO | Fetched 501 rows +2026-06-23 16:13:00 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-23 16:13:01 | INFO | No delete logic required for Login +2026-06-23 16:13:01 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:13:01 | INFO | Login: inserted 501 rows into ClickHouse +2026-06-23 16:13:01 | INFO | Login loaded successfully (501 rows) +2026-06-23 16:13:01 | INFO | ================================================================================ +2026-06-23 16:13:01 | INFO | Processing Table-:Stock_Details | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:13:01 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-23 16:13:01 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:13:01 | INFO | Fetching data for 882 MIDs +2026-06-23 16:13:05 | INFO | Fetched 44,061 rows from SQL Server +2026-06-23 16:13:05 | INFO | Fetched total row -: 44061 from sql server for table-:Stock_Details ...........!!! +2026-06-23 16:13:05 | INFO | Fetched 44061 rows +2026-06-23 16:13:05 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-23 16:13:05 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:13:05 | INFO | Stock_Details: inserted 44,061 rows into ClickHouse +2026-06-23 16:13:05 | INFO | Stock_Details loaded successfully (44061 rows) +2026-06-23 16:13:05 | INFO | ================================================================================ +2026-06-23 16:13:05 | INFO | Processing Table-:Attendance | Table type -:FACT | fetcht by-:run_date | operation-:DELETE+INSERT +2026-06-23 16:13:05 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-23 16:13:05 | INFO | Fetching Attendance data from 2026-06-07 to 2026-06-22 +2026-06-23 16:13:24 | INFO | Fetched 121,144 attendance rows for 542 employees +2026-06-23 16:13:24 | INFO | Fetched total row -: 121144 from sql server for table-:Attendance ...........!!! +2026-06-23 16:13:24 | INFO | Fetched 121144 rows +2026-06-23 16:13:25 | INFO | Truncated table Attendance +2026-06-23 16:13:25 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:13:27 | INFO | Attendance: inserted 121,144 rows into ClickHouse +2026-06-23 16:13:27 | INFO | Attendance loaded successfully (121144 rows) +2026-06-23 16:13:27 | INFO | ================================================================================ +2026-06-23 16:13:27 | INFO | Processing Table-:Journey_Plan | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:13:27 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-23 16:13:27 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:13:27 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:13:27 | INFO | Fetching data for 882 MIDs +2026-06-23 16:13:29 | INFO | Fetched 21,737 rows from SQL Server +2026-06-23 16:13:29 | INFO | Fetched total row -: 21737 from sql server for table-:Journey_Plan ...........!!! +2026-06-23 16:13:29 | INFO | Fetched 21737 rows +2026-06-23 16:13:29 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-23 16:13:29 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:13:30 | INFO | Journey_Plan: inserted 21,737 rows into ClickHouse +2026-06-23 16:13:30 | INFO | Journey_Plan loaded successfully (21737 rows) +2026-06-23 16:13:30 | INFO | ================================================================================ +2026-06-23 16:13:30 | INFO | Processing Table-:Web_Logins | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:13:30 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-23 16:13:30 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:13:30 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:13:30 | INFO | Fetching data for 882 MIDs +2026-06-23 16:13:30 | INFO | Fetched 214 rows from SQL Server +2026-06-23 16:13:30 | INFO | Fetched total row -: 214 from sql server for table-:Web_Logins ...........!!! +2026-06-23 16:13:30 | INFO | Fetched 214 rows +2026-06-23 16:13:30 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ +2026-06-23 16:13:30 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:13:31 | INFO | Web_Logins: inserted 214 rows into ClickHouse +2026-06-23 16:13:31 | INFO | Web_Logins loaded successfully (214 rows) +2026-06-23 16:13:31 | INFO | ================================================================================ +2026-06-23 16:13:31 | INFO | Processing Table-:Promotion | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:13:31 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-23 16:13:31 | INFO | Fetching data for 882 MIDs +2026-06-23 16:13:39 | INFO | Fetched 8,835 rows from SQL Server +2026-06-23 16:13:39 | INFO | Fetched total row -: 8835 from sql server for table-:Promotion ...........!!! +2026-06-23 16:13:39 | INFO | Fetched 8835 rows +2026-06-23 16:13:39 | INFO | _ _ _ _ Deleting Data from ClickHouse for Promotion _ _ _ _ +2026-06-23 16:13:39 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:13:39 | INFO | Promotion: inserted 8,835 rows into ClickHouse +2026-06-23 16:13:39 | INFO | Promotion loaded successfully (8835 rows) +2026-06-23 16:13:39 | INFO | ================================================================================ +2026-06-23 16:13:39 | INFO | Processing Table-:PaidVisibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:13:39 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-23 16:13:39 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:13:39 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:13:39 | INFO | Fetching data for 882 MIDs +2026-06-23 16:13:41 | INFO | Fetched 1,029 rows from SQL Server +2026-06-23 16:13:41 | INFO | Fetched total row -: 1029 from sql server for table-:PaidVisibility ...........!!! +2026-06-23 16:13:41 | INFO | Fetched 1029 rows +2026-06-23 16:13:42 | INFO | _ _ _ _ Deleting Data from ClickHouse for PaidVisibility _ _ _ _ +2026-06-23 16:13:42 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:13:42 | INFO | PaidVisibility: inserted 1,029 rows into ClickHouse +2026-06-23 16:13:42 | INFO | PaidVisibility loaded successfully (1029 rows) +2026-06-23 16:13:42 | INFO | ================================================================================ +2026-06-23 16:13:42 | INFO | Processing Table-:Store_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:13:42 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-23 16:13:42 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:13:42 | INFO | Fetching in progress .... +2026-06-23 16:13:45 | INFO | Fetched 5,998 rows from SQL Server +2026-06-23 16:13:45 | INFO | Fetched total row -: 5998 from sql server for table-:Store_Master ...........!!! +2026-06-23 16:13:45 | INFO | Fetched 5998 rows +2026-06-23 16:13:46 | INFO | Truncated table Store_Master +2026-06-23 16:13:46 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:13:46 | INFO | Store_Master: inserted 5,998 rows into ClickHouse +2026-06-23 16:13:46 | INFO | Store_Master loaded successfully (5998 rows) +2026-06-23 16:13:46 | INFO | ================================================================================ +2026-06-23 16:13:46 | INFO | Processing Table-:SKU_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:13:46 | INFO | Fetching Data from sql server for table-: SKU_Master .............. +2026-06-23 16:13:46 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:13:46 | INFO | Fetching in progress .... +2026-06-23 16:13:46 | INFO | Fetched 160 rows from SQL Server +2026-06-23 16:13:46 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! +2026-06-23 16:13:46 | INFO | Fetched 160 rows +2026-06-23 16:13:46 | INFO | Truncated table SKU_Master +2026-06-23 16:13:46 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:13:47 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-23 16:13:47 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-23 16:13:47 | INFO | ================================================================================ +2026-06-23 16:13:47 | INFO | Processing Table-:display_master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:13:47 | INFO | Fetching Data from sql server for table-: display_master .............. +2026-06-23 16:13:47 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:13:47 | INFO | Fetching in progress .... +2026-06-23 16:13:47 | INFO | Fetched 135 rows from SQL Server +2026-06-23 16:13:47 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! +2026-06-23 16:13:47 | INFO | Fetched 135 rows +2026-06-23 16:13:47 | INFO | Truncated table display_master +2026-06-23 16:13:47 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:13:47 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-23 16:13:47 | INFO | display_master loaded successfully (135 rows) +2026-06-23 16:13:47 | INFO | ================================================================================ +2026-06-23 16:13:47 | INFO | Processing Table-:Employee_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:13:47 | INFO | Fetching Data from sql server for table-: Employee_Master .............. +2026-06-23 16:13:47 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:13:47 | INFO | Fetching in progress .... +2026-06-23 16:13:48 | INFO | Fetched 2,289 rows from SQL Server +2026-06-23 16:13:48 | INFO | Fetched total row -: 2289 from sql server for table-:Employee_Master ...........!!! +2026-06-23 16:13:48 | INFO | Fetched 2289 rows +2026-06-23 16:13:48 | INFO | Truncated table Employee_Master +2026-06-23 16:13:48 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:13:48 | INFO | Employee_Master: inserted 2,289 rows into ClickHouse +2026-06-23 16:13:48 | INFO | Employee_Master loaded successfully (2289 rows) +2026-06-23 16:13:48 | INFO | ================================================================================ +2026-06-23 16:13:48 | INFO | Processing Table-:coverage_remarks | Table type -:DIMENSION | fetcht by-:reason_id | operation-:DELETE+INSERT +2026-06-23 16:13:48 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. +2026-06-23 16:13:49 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:13:49 | INFO | Fetching in progress .... +2026-06-23 16:13:49 | INFO | Fetched 0 rows from SQL Server +2026-06-23 16:13:49 | INFO | Fetched 0 rows from SQL Server +2026-06-23 16:13:49 | INFO | Fetched total row -: 0 from sql server for table-:coverage_remarks ...........!!! +2026-06-23 16:13:49 | WARNING | coverage_remarks returned no rows +2026-06-23 16:13:49 | INFO | ================================================================================ +2026-06-23 16:13:49 | INFO | Processing Table-:mapping_storevisibility | Table type -:BRIDGE | fetcht by-:run_date | operation-:DELETE+INSERT +2026-06-23 16:13:49 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-23 16:13:49 | INFO | Fetching data from sql server for BRIDGE table...... +2026-06-23 16:13:49 | ERROR | Failed processing table mapping_storevisibility +Traceback (most recent call last): + File "D:\z\airflow-project\main2.py", line 192, in elt + df=fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\z\airflow-project\src\bridge.py", line 34, in fetch_mapping_storevisibility + sql = sql_template.format( ) +KeyError: 'store_id_list' +2026-06-23 16:13:49 | WARNING | Pipeline failed. Retry 1/3. Error: 'store_id_list' +2026-06-23 16:13:54 | INFO | ================================================================================ +2026-06-23 16:13:54 | INFO | Hello from data-move Python data pipeline! +2026-06-23 16:13:54 | INFO | Pipeline Run Date: 2026-06-22 +2026-06-23 16:13:54 | INFO | Connecting to databases... +2026-06-23 16:13:54 | INFO | +2026-06-23 16:13:55 | INFO | +2026-06-23 16:13:55 | INFO | Database connections established +2026-06-23 16:13:55 | INFO | Collecting MIDs for: 2026-06-22 +2026-06-23 16:13:55 | INFO | Found 882 MIDs +2026-06-23 16:13:56 | INFO | ================================================================================ +2026-06-23 16:13:56 | INFO | Processing Table-:SOS_OneApp | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:13:56 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-23 16:13:56 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:13:56 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:13:56 | INFO | Fetching data for 882 MIDs +2026-06-23 16:14:03 | INFO | Fetched 3,888 rows from SQL Server +2026-06-23 16:14:03 | INFO | Fetched total row -: 3888 from sql server for table-:SOS_OneApp ...........!!! +2026-06-23 16:14:03 | INFO | Fetched 3888 rows +2026-06-23 16:14:03 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-23 16:14:04 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:14:04 | INFO | SOS_OneApp: inserted 3,888 rows into ClickHouse +2026-06-23 16:14:04 | INFO | SOS_OneApp loaded successfully (3888 rows) +2026-06-23 16:14:04 | INFO | ================================================================================ +2026-06-23 16:14:04 | INFO | Processing Table-:Survey | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:14:04 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-23 16:14:04 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:14:04 | INFO | Fetching data for 882 MIDs +2026-06-23 16:14:04 | INFO | Fetched 152 rows from SQL Server +2026-06-23 16:14:04 | INFO | Fetched total row -: 152 from sql server for table-:Survey ...........!!! +2026-06-23 16:14:04 | INFO | Fetched 152 rows +2026-06-23 16:14:04 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-23 16:14:04 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:14:04 | INFO | Survey: inserted 152 rows into ClickHouse +2026-06-23 16:14:04 | INFO | Survey loaded successfully (152 rows) +2026-06-23 16:14:04 | INFO | ================================================================================ +2026-06-23 16:14:04 | INFO | Processing Table-:additional_visibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:14:04 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-23 16:14:04 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:14:04 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:14:04 | INFO | Fetching data for 882 MIDs +2026-06-23 16:14:08 | INFO | Fetched 2,136 rows from SQL Server +2026-06-23 16:14:08 | INFO | Fetched total row -: 2136 from sql server for table-:additional_visibility ...........!!! +2026-06-23 16:14:08 | INFO | Fetched 2136 rows +2026-06-23 16:14:08 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-23 16:14:08 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:14:08 | INFO | additional_visibility: inserted 2,136 rows into ClickHouse +2026-06-23 16:14:08 | INFO | additional_visibility loaded successfully (2136 rows) +2026-06-23 16:14:08 | INFO | ================================================================================ +2026-06-23 16:14:08 | INFO | Processing Table-:Coverage | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:14:08 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-23 16:14:08 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:14:08 | INFO | Fetching data for 882 MIDs +2026-06-23 16:14:18 | INFO | Fetched 825 rows from SQL Server +2026-06-23 16:14:18 | INFO | Fetched total row -: 825 from sql server for table-:Coverage ...........!!! +2026-06-23 16:14:18 | INFO | Fetched 825 rows +2026-06-23 16:14:18 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-23 16:14:18 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:14:19 | INFO | Coverage: inserted 825 rows into ClickHouse +2026-06-23 16:14:19 | INFO | Coverage loaded successfully (825 rows) +2026-06-23 16:14:19 | INFO | ================================================================================ +2026-06-23 16:14:19 | INFO | Processing Table-:Login | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:14:19 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-23 16:14:19 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:14:19 | INFO | Fetching data for 882 MIDs +2026-06-23 16:14:20 | INFO | Fetched 501 rows from SQL Server +2026-06-23 16:14:20 | INFO | Fetched total row -: 501 from sql server for table-:Login ...........!!! +2026-06-23 16:14:20 | INFO | Fetched 501 rows +2026-06-23 16:14:20 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-23 16:14:20 | INFO | No delete logic required for Login +2026-06-23 16:14:20 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:14:20 | INFO | Login: inserted 501 rows into ClickHouse +2026-06-23 16:14:20 | INFO | Login loaded successfully (501 rows) +2026-06-23 16:14:20 | INFO | ================================================================================ +2026-06-23 16:14:20 | INFO | Processing Table-:Stock_Details | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:14:20 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-23 16:14:20 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:14:20 | INFO | Fetching data for 882 MIDs +2026-06-23 16:14:36 | INFO | Fetched 44,061 rows from SQL Server +2026-06-23 16:14:36 | INFO | Fetched total row -: 44061 from sql server for table-:Stock_Details ...........!!! +2026-06-23 16:14:36 | INFO | Fetched 44061 rows +2026-06-23 16:14:36 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-23 16:14:36 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:14:37 | INFO | Stock_Details: inserted 44,061 rows into ClickHouse +2026-06-23 16:14:37 | INFO | Stock_Details loaded successfully (44061 rows) +2026-06-23 16:14:37 | INFO | ================================================================================ +2026-06-23 16:14:37 | INFO | Processing Table-:Attendance | Table type -:FACT | fetcht by-:run_date | operation-:DELETE+INSERT +2026-06-23 16:14:37 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-23 16:14:37 | INFO | Fetching Attendance data from 2026-06-07 to 2026-06-22 +2026-06-23 16:16:13 | INFO | Fetched 121,144 attendance rows for 542 employees +2026-06-23 16:16:13 | INFO | Fetched total row -: 121144 from sql server for table-:Attendance ...........!!! +2026-06-23 16:16:13 | INFO | Fetched 121144 rows +2026-06-23 16:16:13 | INFO | Truncated table Attendance +2026-06-23 16:16:13 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:16:14 | INFO | Attendance: inserted 121,144 rows into ClickHouse +2026-06-23 16:16:14 | INFO | Attendance loaded successfully (121144 rows) +2026-06-23 16:16:14 | INFO | ================================================================================ +2026-06-23 16:16:14 | INFO | Processing Table-:Journey_Plan | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:16:14 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-23 16:16:14 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:16:14 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:16:14 | INFO | Fetching data for 882 MIDs +2026-06-23 16:16:18 | INFO | Fetched 21,737 rows from SQL Server +2026-06-23 16:16:18 | INFO | Fetched total row -: 21737 from sql server for table-:Journey_Plan ...........!!! +2026-06-23 16:16:18 | INFO | Fetched 21737 rows +2026-06-23 16:16:18 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-23 16:16:18 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:16:19 | INFO | Journey_Plan: inserted 21,737 rows into ClickHouse +2026-06-23 16:16:19 | INFO | Journey_Plan loaded successfully (21737 rows) +2026-06-23 16:16:19 | INFO | ================================================================================ +2026-06-23 16:16:19 | INFO | Processing Table-:Web_Logins | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:16:19 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-23 16:16:19 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:16:19 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:16:19 | INFO | Fetching data for 882 MIDs +2026-06-23 16:16:20 | INFO | Fetched 214 rows from SQL Server +2026-06-23 16:16:20 | INFO | Fetched total row -: 214 from sql server for table-:Web_Logins ...........!!! +2026-06-23 16:16:20 | INFO | Fetched 214 rows +2026-06-23 16:16:20 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ +2026-06-23 16:16:20 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:16:20 | INFO | Web_Logins: inserted 214 rows into ClickHouse +2026-06-23 16:16:20 | INFO | Web_Logins loaded successfully (214 rows) +2026-06-23 16:16:20 | INFO | ================================================================================ +2026-06-23 16:16:20 | INFO | Processing Table-:Promotion | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:16:20 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-23 16:16:20 | INFO | Fetching data for 882 MIDs +2026-06-23 16:16:47 | INFO | Fetched 8,835 rows from SQL Server +2026-06-23 16:16:47 | INFO | Fetched total row -: 8835 from sql server for table-:Promotion ...........!!! +2026-06-23 16:16:47 | INFO | Fetched 8835 rows +2026-06-23 16:16:47 | INFO | _ _ _ _ Deleting Data from ClickHouse for Promotion _ _ _ _ +2026-06-23 16:16:47 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:16:48 | INFO | Promotion: inserted 8,835 rows into ClickHouse +2026-06-23 16:16:48 | INFO | Promotion loaded successfully (8835 rows) +2026-06-23 16:16:48 | INFO | ================================================================================ +2026-06-23 16:16:48 | INFO | Processing Table-:PaidVisibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:16:48 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-23 16:16:48 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:16:48 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:16:48 | INFO | Fetching data for 882 MIDs +2026-06-23 16:16:56 | INFO | Fetched 1,029 rows from SQL Server +2026-06-23 16:16:56 | INFO | Fetched total row -: 1029 from sql server for table-:PaidVisibility ...........!!! +2026-06-23 16:16:56 | INFO | Fetched 1029 rows +2026-06-23 16:16:56 | INFO | _ _ _ _ Deleting Data from ClickHouse for PaidVisibility _ _ _ _ +2026-06-23 16:16:57 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:16:57 | INFO | PaidVisibility: inserted 1,029 rows into ClickHouse +2026-06-23 16:16:57 | INFO | PaidVisibility loaded successfully (1029 rows) +2026-06-23 16:16:57 | INFO | ================================================================================ +2026-06-23 16:16:57 | INFO | Processing Table-:Store_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:16:57 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-23 16:16:57 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:16:57 | INFO | Fetching in progress .... +2026-06-23 16:17:15 | INFO | Fetched 5,998 rows from SQL Server +2026-06-23 16:17:15 | INFO | Fetched total row -: 5998 from sql server for table-:Store_Master ...........!!! +2026-06-23 16:17:15 | INFO | Fetched 5998 rows +2026-06-23 16:17:15 | INFO | Truncated table Store_Master +2026-06-23 16:17:15 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:17:17 | INFO | Store_Master: inserted 5,998 rows into ClickHouse +2026-06-23 16:17:17 | INFO | Store_Master loaded successfully (5998 rows) +2026-06-23 16:17:17 | INFO | ================================================================================ +2026-06-23 16:17:17 | INFO | Processing Table-:SKU_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:17:17 | INFO | Fetching Data from sql server for table-: SKU_Master .............. +2026-06-23 16:17:17 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:17:17 | INFO | Fetching in progress .... +2026-06-23 16:17:18 | INFO | Fetched 160 rows from SQL Server +2026-06-23 16:17:18 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! +2026-06-23 16:17:18 | INFO | Fetched 160 rows +2026-06-23 16:17:18 | INFO | Truncated table SKU_Master +2026-06-23 16:17:18 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:17:19 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-23 16:17:19 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-23 16:17:19 | INFO | ================================================================================ +2026-06-23 16:17:19 | INFO | Processing Table-:display_master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:17:19 | INFO | Fetching Data from sql server for table-: display_master .............. +2026-06-23 16:17:19 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:17:19 | INFO | Fetching in progress .... +2026-06-23 16:17:19 | INFO | Fetched 135 rows from SQL Server +2026-06-23 16:17:19 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! +2026-06-23 16:17:19 | INFO | Fetched 135 rows +2026-06-23 16:17:19 | INFO | Truncated table display_master +2026-06-23 16:17:19 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:17:19 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-23 16:17:19 | INFO | display_master loaded successfully (135 rows) +2026-06-23 16:17:19 | INFO | ================================================================================ +2026-06-23 16:17:19 | INFO | Processing Table-:Employee_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:17:19 | INFO | Fetching Data from sql server for table-: Employee_Master .............. +2026-06-23 16:17:19 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:17:19 | INFO | Fetching in progress .... +2026-06-23 16:17:24 | INFO | Fetched 2,289 rows from SQL Server +2026-06-23 16:17:24 | INFO | Fetched total row -: 2289 from sql server for table-:Employee_Master ...........!!! +2026-06-23 16:17:24 | INFO | Fetched 2289 rows +2026-06-23 16:17:25 | INFO | Truncated table Employee_Master +2026-06-23 16:17:25 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:17:25 | INFO | Employee_Master: inserted 2,289 rows into ClickHouse +2026-06-23 16:17:25 | INFO | Employee_Master loaded successfully (2289 rows) +2026-06-23 16:17:25 | INFO | ================================================================================ +2026-06-23 16:17:25 | INFO | Processing Table-:coverage_remarks | Table type -:DIMENSION | fetcht by-:reason_id | operation-:DELETE+INSERT +2026-06-23 16:17:25 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. +2026-06-23 16:17:26 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:17:26 | INFO | Fetching in progress .... +2026-06-23 16:17:26 | INFO | Fetched 0 rows from SQL Server +2026-06-23 16:17:26 | INFO | Fetched 0 rows from SQL Server +2026-06-23 16:17:26 | INFO | Fetched total row -: 0 from sql server for table-:coverage_remarks ...........!!! +2026-06-23 16:17:26 | WARNING | coverage_remarks returned no rows +2026-06-23 16:17:26 | INFO | ================================================================================ +2026-06-23 16:17:26 | INFO | Processing Table-:mapping_storevisibility | Table type -:BRIDGE | fetcht by-:run_date | operation-:DELETE+INSERT +2026-06-23 16:17:26 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-23 16:17:26 | INFO | Fetching data from sql server for BRIDGE table...... +2026-06-23 16:17:26 | ERROR | Failed processing table mapping_storevisibility +Traceback (most recent call last): + File "D:\z\airflow-project\main2.py", line 192, in elt + df=fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\z\airflow-project\src\bridge.py", line 34, in fetch_mapping_storevisibility + sql = sql_template.format( ) +KeyError: 'store_id_list' +2026-06-23 16:17:26 | WARNING | Pipeline failed. Retry 2/3. Error: 'store_id_list' +2026-06-23 16:17:31 | INFO | ================================================================================ +2026-06-23 16:17:31 | INFO | Hello from data-move Python data pipeline! +2026-06-23 16:17:31 | INFO | Pipeline Run Date: 2026-06-22 +2026-06-23 16:17:31 | INFO | Connecting to databases... +2026-06-23 16:17:31 | INFO | +2026-06-23 16:17:32 | INFO | +2026-06-23 16:17:32 | INFO | Database connections established +2026-06-23 16:17:32 | INFO | Collecting MIDs for: 2026-06-22 +2026-06-23 16:17:32 | INFO | Found 882 MIDs +2026-06-23 16:17:32 | INFO | ================================================================================ +2026-06-23 16:17:32 | INFO | Processing Table-:SOS_OneApp | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:17:32 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-23 16:17:32 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:17:32 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:17:32 | INFO | Fetching data for 882 MIDs +2026-06-23 16:17:43 | INFO | Fetched 3,888 rows from SQL Server +2026-06-23 16:17:43 | INFO | Fetched total row -: 3888 from sql server for table-:SOS_OneApp ...........!!! +2026-06-23 16:17:43 | INFO | Fetched 3888 rows +2026-06-23 16:17:43 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-23 16:17:43 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:17:44 | INFO | SOS_OneApp: inserted 3,888 rows into ClickHouse +2026-06-23 16:17:44 | INFO | SOS_OneApp loaded successfully (3888 rows) +2026-06-23 16:17:44 | INFO | ================================================================================ +2026-06-23 16:17:44 | INFO | Processing Table-:Survey | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:17:44 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-23 16:17:44 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:17:44 | INFO | Fetching data for 882 MIDs +2026-06-23 16:17:44 | INFO | Fetched 152 rows from SQL Server +2026-06-23 16:17:44 | INFO | Fetched total row -: 152 from sql server for table-:Survey ...........!!! +2026-06-23 16:17:44 | INFO | Fetched 152 rows +2026-06-23 16:17:44 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-23 16:17:44 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:17:44 | INFO | Survey: inserted 152 rows into ClickHouse +2026-06-23 16:17:44 | INFO | Survey loaded successfully (152 rows) +2026-06-23 16:17:44 | INFO | ================================================================================ +2026-06-23 16:17:44 | INFO | Processing Table-:additional_visibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:17:44 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-23 16:17:44 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:17:44 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:17:44 | INFO | Fetching data for 882 MIDs +2026-06-23 16:17:47 | INFO | Fetched 2,136 rows from SQL Server +2026-06-23 16:17:47 | INFO | Fetched total row -: 2136 from sql server for table-:additional_visibility ...........!!! +2026-06-23 16:17:47 | INFO | Fetched 2136 rows +2026-06-23 16:17:47 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-23 16:17:48 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:17:48 | INFO | additional_visibility: inserted 2,136 rows into ClickHouse +2026-06-23 16:17:48 | INFO | additional_visibility loaded successfully (2136 rows) +2026-06-23 16:17:48 | INFO | ================================================================================ +2026-06-23 16:17:48 | INFO | Processing Table-:Coverage | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:17:48 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-23 16:17:48 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:17:48 | INFO | Fetching data for 882 MIDs +2026-06-23 16:17:58 | INFO | Fetched 825 rows from SQL Server +2026-06-23 16:17:58 | INFO | Fetched total row -: 825 from sql server for table-:Coverage ...........!!! +2026-06-23 16:17:58 | INFO | Fetched 825 rows +2026-06-23 16:17:58 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-23 16:17:58 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:17:58 | INFO | Coverage: inserted 825 rows into ClickHouse +2026-06-23 16:17:58 | INFO | Coverage loaded successfully (825 rows) +2026-06-23 16:17:58 | INFO | ================================================================================ +2026-06-23 16:17:58 | INFO | Processing Table-:Login | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:17:58 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-23 16:17:58 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:17:58 | INFO | Fetching data for 882 MIDs +2026-06-23 16:18:01 | INFO | Fetched 501 rows from SQL Server +2026-06-23 16:18:01 | INFO | Fetched total row -: 501 from sql server for table-:Login ...........!!! +2026-06-23 16:18:01 | INFO | Fetched 501 rows +2026-06-23 16:18:01 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-23 16:18:01 | INFO | No delete logic required for Login +2026-06-23 16:18:01 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:18:01 | INFO | Login: inserted 501 rows into ClickHouse +2026-06-23 16:18:01 | INFO | Login loaded successfully (501 rows) +2026-06-23 16:18:01 | INFO | ================================================================================ +2026-06-23 16:18:01 | INFO | Processing Table-:Stock_Details | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:18:01 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-23 16:18:01 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:18:01 | INFO | Fetching data for 882 MIDs +2026-06-23 16:18:56 | INFO | Fetched 44,061 rows from SQL Server +2026-06-23 16:18:56 | INFO | Fetched total row -: 44061 from sql server for table-:Stock_Details ...........!!! +2026-06-23 16:18:56 | INFO | Fetched 44061 rows +2026-06-23 16:18:57 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-23 16:18:57 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:18:59 | INFO | Stock_Details: inserted 44,061 rows into ClickHouse +2026-06-23 16:18:59 | INFO | Stock_Details loaded successfully (44061 rows) +2026-06-23 16:18:59 | INFO | ================================================================================ +2026-06-23 16:18:59 | INFO | Processing Table-:Attendance | Table type -:FACT | fetcht by-:run_date | operation-:DELETE+INSERT +2026-06-23 16:18:59 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-23 16:18:59 | INFO | Fetching Attendance data from 2026-06-07 to 2026-06-22 +2026-06-23 16:20:19 | INFO | Fetched 121,144 attendance rows for 542 employees +2026-06-23 16:20:19 | INFO | Fetched total row -: 121144 from sql server for table-:Attendance ...........!!! +2026-06-23 16:20:19 | INFO | Fetched 121144 rows +2026-06-23 16:20:20 | INFO | Truncated table Attendance +2026-06-23 16:20:20 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:20:25 | INFO | Attendance: inserted 121,144 rows into ClickHouse +2026-06-23 16:20:25 | INFO | Attendance loaded successfully (121144 rows) +2026-06-23 16:20:25 | INFO | ================================================================================ +2026-06-23 16:20:25 | INFO | Processing Table-:Journey_Plan | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:20:25 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-23 16:20:25 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:20:25 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:20:25 | INFO | Fetching data for 882 MIDs +2026-06-23 16:20:29 | INFO | Fetched 21,737 rows from SQL Server +2026-06-23 16:20:29 | INFO | Fetched total row -: 21737 from sql server for table-:Journey_Plan ...........!!! +2026-06-23 16:20:29 | INFO | Fetched 21737 rows +2026-06-23 16:20:29 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-23 16:20:29 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:20:30 | INFO | Journey_Plan: inserted 21,737 rows into ClickHouse +2026-06-23 16:20:30 | INFO | Journey_Plan loaded successfully (21737 rows) +2026-06-23 16:20:30 | INFO | ================================================================================ +2026-06-23 16:20:30 | INFO | Processing Table-:Web_Logins | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:20:30 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-23 16:20:30 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:20:30 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:20:30 | INFO | Fetching data for 882 MIDs +2026-06-23 16:20:30 | INFO | Fetched 214 rows from SQL Server +2026-06-23 16:20:30 | INFO | Fetched total row -: 214 from sql server for table-:Web_Logins ...........!!! +2026-06-23 16:20:30 | INFO | Fetched 214 rows +2026-06-23 16:20:30 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ +2026-06-23 16:20:30 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:20:30 | INFO | Web_Logins: inserted 214 rows into ClickHouse +2026-06-23 16:20:30 | INFO | Web_Logins loaded successfully (214 rows) +2026-06-23 16:20:30 | INFO | ================================================================================ +2026-06-23 16:20:30 | INFO | Processing Table-:Promotion | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:20:30 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-23 16:20:30 | INFO | Fetching data for 882 MIDs +2026-06-23 16:20:45 | INFO | Fetched 8,835 rows from SQL Server +2026-06-23 16:20:45 | INFO | Fetched total row -: 8835 from sql server for table-:Promotion ...........!!! +2026-06-23 16:20:45 | INFO | Fetched 8835 rows +2026-06-23 16:20:45 | INFO | _ _ _ _ Deleting Data from ClickHouse for Promotion _ _ _ _ +2026-06-23 16:20:45 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:20:46 | INFO | Promotion: inserted 8,835 rows into ClickHouse +2026-06-23 16:20:46 | INFO | Promotion loaded successfully (8835 rows) +2026-06-23 16:20:46 | INFO | ================================================================================ +2026-06-23 16:20:46 | INFO | Processing Table-:PaidVisibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:20:46 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-23 16:20:46 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 16:20:46 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 16:20:46 | INFO | Fetching data for 882 MIDs +2026-06-23 16:20:50 | INFO | Fetched 1,029 rows from SQL Server +2026-06-23 16:20:50 | INFO | Fetched total row -: 1029 from sql server for table-:PaidVisibility ...........!!! +2026-06-23 16:20:50 | INFO | Fetched 1029 rows +2026-06-23 16:20:50 | INFO | _ _ _ _ Deleting Data from ClickHouse for PaidVisibility _ _ _ _ +2026-06-23 16:20:50 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:20:50 | INFO | PaidVisibility: inserted 1,029 rows into ClickHouse +2026-06-23 16:20:50 | INFO | PaidVisibility loaded successfully (1029 rows) +2026-06-23 16:20:50 | INFO | ================================================================================ +2026-06-23 16:20:50 | INFO | Processing Table-:Store_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:20:50 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-23 16:20:50 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:20:50 | INFO | Fetching in progress .... +2026-06-23 16:21:04 | INFO | Fetched 5,998 rows from SQL Server +2026-06-23 16:21:04 | INFO | Fetched total row -: 5998 from sql server for table-:Store_Master ...........!!! +2026-06-23 16:21:04 | INFO | Fetched 5998 rows +2026-06-23 16:21:04 | INFO | Truncated table Store_Master +2026-06-23 16:21:04 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:21:05 | INFO | Store_Master: inserted 5,998 rows into ClickHouse +2026-06-23 16:21:05 | INFO | Store_Master loaded successfully (5998 rows) +2026-06-23 16:21:05 | INFO | ================================================================================ +2026-06-23 16:21:05 | INFO | Processing Table-:SKU_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:21:05 | INFO | Fetching Data from sql server for table-: SKU_Master .............. +2026-06-23 16:21:05 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:21:05 | INFO | Fetching in progress .... +2026-06-23 16:21:05 | INFO | Fetched 160 rows from SQL Server +2026-06-23 16:21:05 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! +2026-06-23 16:21:05 | INFO | Fetched 160 rows +2026-06-23 16:21:05 | INFO | Truncated table SKU_Master +2026-06-23 16:21:05 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:21:05 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-23 16:21:05 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-23 16:21:05 | INFO | ================================================================================ +2026-06-23 16:21:05 | INFO | Processing Table-:display_master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:21:05 | INFO | Fetching Data from sql server for table-: display_master .............. +2026-06-23 16:21:05 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:21:05 | INFO | Fetching in progress .... +2026-06-23 16:21:06 | INFO | Fetched 135 rows from SQL Server +2026-06-23 16:21:06 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! +2026-06-23 16:21:06 | INFO | Fetched 135 rows +2026-06-23 16:21:06 | INFO | Truncated table display_master +2026-06-23 16:21:06 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:21:06 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-23 16:21:06 | INFO | display_master loaded successfully (135 rows) +2026-06-23 16:21:06 | INFO | ================================================================================ +2026-06-23 16:21:06 | INFO | Processing Table-:Employee_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:21:06 | INFO | Fetching Data from sql server for table-: Employee_Master .............. +2026-06-23 16:21:06 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:21:06 | INFO | Fetching in progress .... +2026-06-23 16:21:11 | INFO | Fetched 2,289 rows from SQL Server +2026-06-23 16:21:11 | INFO | Fetched total row -: 2289 from sql server for table-:Employee_Master ...........!!! +2026-06-23 16:21:11 | INFO | Fetched 2289 rows +2026-06-23 16:21:11 | INFO | Truncated table Employee_Master +2026-06-23 16:21:11 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:21:12 | INFO | Employee_Master: inserted 2,289 rows into ClickHouse +2026-06-23 16:21:12 | INFO | Employee_Master loaded successfully (2289 rows) +2026-06-23 16:21:12 | INFO | ================================================================================ +2026-06-23 16:21:12 | INFO | Processing Table-:coverage_remarks | Table type -:DIMENSION | fetcht by-:reason_id | operation-:DELETE+INSERT +2026-06-23 16:21:12 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. +2026-06-23 16:21:12 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:21:12 | INFO | Fetching in progress .... +2026-06-23 16:21:12 | INFO | Fetched 0 rows from SQL Server +2026-06-23 16:21:12 | INFO | Fetched 0 rows from SQL Server +2026-06-23 16:21:12 | INFO | Fetched total row -: 0 from sql server for table-:coverage_remarks ...........!!! +2026-06-23 16:21:12 | WARNING | coverage_remarks returned no rows +2026-06-23 16:21:12 | INFO | ================================================================================ +2026-06-23 16:21:12 | INFO | Processing Table-:mapping_storevisibility | Table type -:BRIDGE | fetcht by-:run_date | operation-:DELETE+INSERT +2026-06-23 16:21:12 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-23 16:21:12 | INFO | Fetching data from sql server for BRIDGE table...... +2026-06-23 16:21:12 | INFO | Fetching in progress .... +2026-06-23 16:21:13 | INFO | Fetched 805 rows from SQL Server +2026-06-23 16:21:13 | INFO | Fetched total row -: 805 from sql server for table-:mapping_storevisibility ...........!!! +2026-06-23 16:21:13 | INFO | Fetched 805 rows +2026-06-23 16:21:13 | INFO | Creating table mapping_storevisibility +2026-06-23 16:21:13 | INFO | Table ready: mapping_storevisibility +2026-06-23 16:21:13 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:21:13 | INFO | mapping_storevisibility: inserted 805 rows into ClickHouse +2026-06-23 16:21:13 | INFO | mapping_storevisibility loaded successfully (805 rows) +2026-06-23 16:21:13 | INFO | ================================================================================ +2026-06-23 16:21:13 | INFO | Processing Table-:Master_VisibilityReason | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-23 16:21:13 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-23 16:21:13 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:21:13 | INFO | Fetching in progress .... +2026-06-23 16:21:13 | INFO | Fetched 17 rows from SQL Server +2026-06-23 16:21:13 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-23 16:21:13 | INFO | Fetched 17 rows +2026-06-23 16:21:13 | INFO | Truncated table Master_VisibilityReason +2026-06-23 16:21:13 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:21:13 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-23 16:21:13 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-23 16:21:13 | INFO | ================================================================================ +2026-06-23 16:21:13 | INFO | Processing Table-:Master_VisibilityDefinition | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-23 16:21:13 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. +2026-06-23 16:21:13 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:21:13 | INFO | Fetching in progress .... +2026-06-23 16:21:14 | INFO | Fetched 861 rows from SQL Server +2026-06-23 16:21:14 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! +2026-06-23 16:21:14 | INFO | Fetched 861 rows +2026-06-23 16:21:14 | INFO | Truncated table Master_VisibilityDefinition +2026-06-23 16:21:14 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:21:15 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-23 16:21:15 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-23 16:21:15 | INFO | ================================================================================ +2026-06-23 16:21:15 | INFO | Processing Table-:Master_Salesterritorylayer | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-23 16:21:15 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. +2026-06-23 16:21:15 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:21:15 | INFO | Fetching in progress .... +2026-06-23 16:21:15 | INFO | Fetched 33 rows from SQL Server +2026-06-23 16:21:15 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! +2026-06-23 16:21:15 | INFO | Fetched 33 rows +2026-06-23 16:21:15 | INFO | Truncated table Master_Salesterritorylayer +2026-06-23 16:21:15 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:21:15 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-23 16:21:15 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-23 16:21:15 | INFO | ================================================================================ +2026-06-23 16:21:15 | INFO | Processing Table-:OQaD | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:21:15 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-23 16:21:15 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-23 16:21:45 | INFO | Fetched 291,599 total empid and visitdate fetched for OQAD from SQL Server +2026-06-23 16:21:45 | WARNING | Table 'OQaD' does not exist. +2026-06-23 16:21:45 | WARNING | OQaD Matched df in OQaD returned no rows +2026-06-23 16:21:45 | INFO | Fetched 1 matched empids fetched for OQAD +2026-06-23 16:21:45 | INFO | Exists: True +2026-06-23 16:21:45 | INFO | Path: D:\z\airflow-project\src\sql\fact\OQaD.sql +2026-06-23 16:21:45 | INFO | Fetching data for 1 EMPIDs +2026-06-23 16:21:45 | INFO | Fetching OQaD data for run_date=2026-06-22 +2026-06-23 16:27:23 | INFO | fn name is fetch_OQad ------Fetched 291222 rows +2026-06-23 16:27:23 | INFO | Fetched 291,222 rows from SQL Server +2026-06-23 16:27:23 | INFO | Fetched total row -: 291222 from sql server for table-:OQaD ...........!!! +2026-06-23 16:27:23 | INFO | Fetched 291222 rows +2026-06-23 16:27:23 | INFO | Creating table OQaD +2026-06-23 16:27:23 | INFO | Table ready: OQaD +2026-06-23 16:27:23 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:27:26 | INFO | OQaD: inserted 291,222 rows into ClickHouse +2026-06-23 16:27:26 | INFO | OQaD loaded successfully (291222 rows) +2026-06-23 16:27:26 | INFO | ================================================================================ +2026-06-23 16:27:26 | INFO | Pipeline Completed Successfully +2026-06-23 16:27:26 | INFO | ================================================================================ +2026-06-23 16:27:26 | INFO | Pipeline completed successfully. last_successful_run_date=2026-06-23 +2026-06-23 16:27:26 | INFO | ================================================================================ +2026-06-23 16:27:26 | INFO | Hello from data-move Python data pipeline! +2026-06-23 16:27:26 | INFO | Pipeline Run Date: 2026-06-23 +2026-06-23 16:27:26 | INFO | Connecting to databases... +2026-06-23 16:27:26 | INFO | +2026-06-23 16:27:27 | INFO | +2026-06-23 16:27:27 | INFO | Database connections established +2026-06-23 16:27:27 | INFO | Collecting MIDs for: 2026-06-23 +2026-06-23 16:27:27 | INFO | Found 857 MIDs +2026-06-23 16:27:28 | INFO | ================================================================================ +2026-06-23 16:27:28 | INFO | Processing Table-:SOS_OneApp | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:27:28 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-23 16:27:28 | INFO | Start Fetching data for these 857 MIDs +2026-06-23 16:27:28 | INFO | Start Fetching data for these 857 MIDs or based on this date 2026-06-23 +2026-06-23 16:27:28 | INFO | Fetching data for 857 MIDs +2026-06-23 16:27:30 | INFO | Fetched 1,748 rows from SQL Server +2026-06-23 16:27:30 | INFO | Fetched total row -: 1748 from sql server for table-:SOS_OneApp ...........!!! +2026-06-23 16:27:30 | INFO | Fetched 1748 rows +2026-06-23 16:27:31 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-23 16:27:31 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:27:31 | INFO | SOS_OneApp: inserted 1,748 rows into ClickHouse +2026-06-23 16:27:31 | INFO | SOS_OneApp loaded successfully (1748 rows) +2026-06-23 16:27:31 | INFO | ================================================================================ +2026-06-23 16:27:31 | INFO | Processing Table-:Survey | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:27:31 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-23 16:27:31 | INFO | Start Fetching data for these 857 MIDs or based on this date 2026-06-23 +2026-06-23 16:27:31 | INFO | Fetching data for 857 MIDs +2026-06-23 16:27:31 | INFO | Fetched 55 rows from SQL Server +2026-06-23 16:27:31 | INFO | Fetched total row -: 55 from sql server for table-:Survey ...........!!! +2026-06-23 16:27:31 | INFO | Fetched 55 rows +2026-06-23 16:27:31 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-23 16:27:31 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:27:31 | INFO | Survey: inserted 55 rows into ClickHouse +2026-06-23 16:27:31 | INFO | Survey loaded successfully (55 rows) +2026-06-23 16:27:31 | INFO | ================================================================================ +2026-06-23 16:27:31 | INFO | Processing Table-:additional_visibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:27:31 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-23 16:27:31 | INFO | Start Fetching data for these 857 MIDs +2026-06-23 16:27:31 | INFO | Start Fetching data for these 857 MIDs or based on this date 2026-06-23 +2026-06-23 16:27:31 | INFO | Fetching data for 857 MIDs +2026-06-23 16:27:32 | INFO | Fetched 642 rows from SQL Server +2026-06-23 16:27:32 | INFO | Fetched total row -: 642 from sql server for table-:additional_visibility ...........!!! +2026-06-23 16:27:32 | INFO | Fetched 642 rows +2026-06-23 16:27:32 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-23 16:27:32 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:27:33 | INFO | additional_visibility: inserted 642 rows into ClickHouse +2026-06-23 16:27:33 | INFO | additional_visibility loaded successfully (642 rows) +2026-06-23 16:27:33 | INFO | ================================================================================ +2026-06-23 16:27:33 | INFO | Processing Table-:Coverage | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:27:33 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-23 16:27:33 | INFO | Start Fetching data for these 857 MIDs or based on this date 2026-06-23 +2026-06-23 16:27:33 | INFO | Fetching data for 857 MIDs +2026-06-23 16:27:33 | INFO | Fetched 800 rows from SQL Server +2026-06-23 16:27:33 | INFO | Fetched total row -: 800 from sql server for table-:Coverage ...........!!! +2026-06-23 16:27:33 | INFO | Fetched 800 rows +2026-06-23 16:27:33 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-23 16:27:34 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:27:34 | INFO | Coverage: inserted 800 rows into ClickHouse +2026-06-23 16:27:34 | INFO | Coverage loaded successfully (800 rows) +2026-06-23 16:27:34 | INFO | ================================================================================ +2026-06-23 16:27:34 | INFO | Processing Table-:Login | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:27:34 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-23 16:27:34 | INFO | Start Fetching data for these 857 MIDs or based on this date 2026-06-23 +2026-06-23 16:27:34 | INFO | Fetching data for 857 MIDs +2026-06-23 16:27:35 | INFO | Fetched 451 rows from SQL Server +2026-06-23 16:27:35 | INFO | Fetched total row -: 451 from sql server for table-:Login ...........!!! +2026-06-23 16:27:35 | INFO | Fetched 451 rows +2026-06-23 16:27:35 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-23 16:27:35 | INFO | No delete logic required for Login +2026-06-23 16:27:35 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:27:35 | INFO | Login: inserted 451 rows into ClickHouse +2026-06-23 16:27:35 | INFO | Login loaded successfully (451 rows) +2026-06-23 16:27:35 | INFO | ================================================================================ +2026-06-23 16:27:35 | INFO | Processing Table-:Stock_Details | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:27:35 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-23 16:27:35 | INFO | Start Fetching data for these 857 MIDs or based on this date 2026-06-23 +2026-06-23 16:27:35 | INFO | Fetching data for 857 MIDs +2026-06-23 16:27:42 | INFO | Fetched 19,243 rows from SQL Server +2026-06-23 16:27:42 | INFO | Fetched total row -: 19243 from sql server for table-:Stock_Details ...........!!! +2026-06-23 16:27:42 | INFO | Fetched 19243 rows +2026-06-23 16:27:43 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-23 16:27:43 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:27:43 | INFO | Stock_Details: inserted 19,243 rows into ClickHouse +2026-06-23 16:27:43 | INFO | Stock_Details loaded successfully (19243 rows) +2026-06-23 16:27:43 | INFO | ================================================================================ +2026-06-23 16:27:43 | INFO | Processing Table-:Attendance | Table type -:FACT | fetcht by-:run_date | operation-:DELETE+INSERT +2026-06-23 16:27:43 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-23 16:27:43 | INFO | Fetching Attendance data from 2026-06-08 to 2026-06-23 +2026-06-23 16:29:09 | INFO | Fetched 121,962 attendance rows for 542 employees +2026-06-23 16:29:09 | INFO | Fetched total row -: 121962 from sql server for table-:Attendance ...........!!! +2026-06-23 16:29:09 | INFO | Fetched 121962 rows +2026-06-23 16:29:10 | INFO | Truncated table Attendance +2026-06-23 16:29:10 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:29:11 | INFO | Attendance: inserted 121,962 rows into ClickHouse +2026-06-23 16:29:11 | INFO | Attendance loaded successfully (121962 rows) +2026-06-23 16:29:11 | INFO | ================================================================================ +2026-06-23 16:29:11 | INFO | Processing Table-:Journey_Plan | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:29:11 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-23 16:29:11 | INFO | Start Fetching data for these 857 MIDs +2026-06-23 16:29:11 | INFO | Start Fetching data for these 857 MIDs or based on this date 2026-06-23 +2026-06-23 16:29:11 | INFO | Fetching data for 857 MIDs +2026-06-23 16:29:15 | INFO | Fetched 21,737 rows from SQL Server +2026-06-23 16:29:15 | INFO | Fetched total row -: 21737 from sql server for table-:Journey_Plan ...........!!! +2026-06-23 16:29:15 | INFO | Fetched 21737 rows +2026-06-23 16:29:15 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-23 16:29:15 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:29:16 | INFO | Journey_Plan: inserted 21,737 rows into ClickHouse +2026-06-23 16:29:16 | INFO | Journey_Plan loaded successfully (21737 rows) +2026-06-23 16:29:16 | INFO | ================================================================================ +2026-06-23 16:29:16 | INFO | Processing Table-:Web_Logins | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:29:16 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-23 16:29:16 | INFO | Start Fetching data for these 857 MIDs +2026-06-23 16:29:16 | INFO | Start Fetching data for these 857 MIDs or based on this date 2026-06-23 +2026-06-23 16:29:16 | INFO | Fetching data for 857 MIDs +2026-06-23 16:29:16 | INFO | Fetched 109 rows from SQL Server +2026-06-23 16:29:16 | INFO | Fetched total row -: 109 from sql server for table-:Web_Logins ...........!!! +2026-06-23 16:29:16 | INFO | Fetched 109 rows +2026-06-23 16:29:16 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ +2026-06-23 16:29:16 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:29:16 | INFO | Web_Logins: inserted 109 rows into ClickHouse +2026-06-23 16:29:16 | INFO | Web_Logins loaded successfully (109 rows) +2026-06-23 16:29:16 | INFO | ================================================================================ +2026-06-23 16:29:16 | INFO | Processing Table-:Promotion | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:29:16 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-23 16:29:16 | INFO | Fetching data for 857 MIDs +2026-06-23 16:29:22 | INFO | Fetched 3,938 rows from SQL Server +2026-06-23 16:29:22 | INFO | Fetched total row -: 3938 from sql server for table-:Promotion ...........!!! +2026-06-23 16:29:22 | INFO | Fetched 3938 rows +2026-06-23 16:29:22 | INFO | _ _ _ _ Deleting Data from ClickHouse for Promotion _ _ _ _ +2026-06-23 16:29:22 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:29:23 | INFO | Promotion: inserted 3,938 rows into ClickHouse +2026-06-23 16:29:23 | INFO | Promotion loaded successfully (3938 rows) +2026-06-23 16:29:23 | INFO | ================================================================================ +2026-06-23 16:29:23 | INFO | Processing Table-:PaidVisibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 16:29:23 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-23 16:29:23 | INFO | Start Fetching data for these 857 MIDs +2026-06-23 16:29:23 | INFO | Start Fetching data for these 857 MIDs or based on this date 2026-06-23 +2026-06-23 16:29:23 | INFO | Fetching data for 857 MIDs +2026-06-23 16:29:25 | INFO | Fetched 236 rows from SQL Server +2026-06-23 16:29:25 | INFO | Fetched total row -: 236 from sql server for table-:PaidVisibility ...........!!! +2026-06-23 16:29:25 | INFO | Fetched 236 rows +2026-06-23 16:29:25 | INFO | _ _ _ _ Deleting Data from ClickHouse for PaidVisibility _ _ _ _ +2026-06-23 16:29:25 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:29:25 | INFO | PaidVisibility: inserted 236 rows into ClickHouse +2026-06-23 16:29:25 | INFO | PaidVisibility loaded successfully (236 rows) +2026-06-23 16:29:25 | INFO | ================================================================================ +2026-06-23 16:29:25 | INFO | Processing Table-:Store_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:29:25 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-23 16:29:25 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:29:25 | INFO | Fetching in progress .... +2026-06-23 16:29:38 | INFO | Fetched 5,998 rows from SQL Server +2026-06-23 16:29:38 | INFO | Fetched total row -: 5998 from sql server for table-:Store_Master ...........!!! +2026-06-23 16:29:38 | INFO | Fetched 5998 rows +2026-06-23 16:29:38 | INFO | Truncated table Store_Master +2026-06-23 16:29:38 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:29:39 | INFO | Store_Master: inserted 5,998 rows into ClickHouse +2026-06-23 16:29:39 | INFO | Store_Master loaded successfully (5998 rows) +2026-06-23 16:29:39 | INFO | ================================================================================ +2026-06-23 16:29:39 | INFO | Processing Table-:SKU_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:29:39 | INFO | Fetching Data from sql server for table-: SKU_Master .............. +2026-06-23 16:29:39 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:29:39 | INFO | Fetching in progress .... +2026-06-23 16:29:39 | INFO | Fetched 160 rows from SQL Server +2026-06-23 16:29:39 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! +2026-06-23 16:29:39 | INFO | Fetched 160 rows +2026-06-23 16:29:39 | INFO | Truncated table SKU_Master +2026-06-23 16:29:39 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:29:40 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-23 16:29:40 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-23 16:29:40 | INFO | ================================================================================ +2026-06-23 16:29:40 | INFO | Processing Table-:display_master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:29:40 | INFO | Fetching Data from sql server for table-: display_master .............. +2026-06-23 16:29:40 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:29:40 | INFO | Fetching in progress .... +2026-06-23 16:29:40 | INFO | Fetched 135 rows from SQL Server +2026-06-23 16:29:40 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! +2026-06-23 16:29:40 | INFO | Fetched 135 rows +2026-06-23 16:29:40 | INFO | Truncated table display_master +2026-06-23 16:29:40 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:29:40 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-23 16:29:40 | INFO | display_master loaded successfully (135 rows) +2026-06-23 16:29:40 | INFO | ================================================================================ +2026-06-23 16:29:40 | INFO | Processing Table-:Employee_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 16:29:40 | INFO | Fetching Data from sql server for table-: Employee_Master .............. +2026-06-23 16:29:40 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:29:40 | INFO | Fetching in progress .... +2026-06-23 16:29:42 | INFO | Fetched 2,289 rows from SQL Server +2026-06-23 16:29:42 | INFO | Fetched total row -: 2289 from sql server for table-:Employee_Master ...........!!! +2026-06-23 16:29:42 | INFO | Fetched 2289 rows +2026-06-23 16:29:42 | INFO | Truncated table Employee_Master +2026-06-23 16:29:42 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:29:43 | INFO | Employee_Master: inserted 2,289 rows into ClickHouse +2026-06-23 16:29:43 | INFO | Employee_Master loaded successfully (2289 rows) +2026-06-23 16:29:43 | INFO | ================================================================================ +2026-06-23 16:29:43 | INFO | Processing Table-:coverage_remarks | Table type -:DIMENSION | fetcht by-:reason_id | operation-:DELETE+INSERT +2026-06-23 16:29:43 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. +2026-06-23 16:29:43 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:29:43 | INFO | Fetching in progress .... +2026-06-23 16:29:43 | INFO | Fetched 0 rows from SQL Server +2026-06-23 16:29:43 | INFO | Fetched 0 rows from SQL Server +2026-06-23 16:29:43 | INFO | Fetched total row -: 0 from sql server for table-:coverage_remarks ...........!!! +2026-06-23 16:29:43 | WARNING | coverage_remarks returned no rows +2026-06-23 16:29:43 | INFO | ================================================================================ +2026-06-23 16:29:43 | INFO | Processing Table-:mapping_storevisibility | Table type -:BRIDGE | fetcht by-:run_date | operation-:DELETE+INSERT +2026-06-23 16:29:43 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-23 16:29:43 | INFO | Fetching data from sql server for BRIDGE table...... +2026-06-23 16:29:43 | INFO | Fetching in progress .... +2026-06-23 16:29:43 | INFO | Fetched 805 rows from SQL Server +2026-06-23 16:29:43 | INFO | Fetched total row -: 805 from sql server for table-:mapping_storevisibility ...........!!! +2026-06-23 16:29:43 | INFO | Fetched 805 rows +2026-06-23 16:29:44 | INFO | Truncated table mapping_storevisibility +2026-06-23 16:29:44 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:29:44 | INFO | mapping_storevisibility: inserted 805 rows into ClickHouse +2026-06-23 16:29:44 | INFO | mapping_storevisibility loaded successfully (805 rows) +2026-06-23 16:29:44 | INFO | ================================================================================ +2026-06-23 16:29:44 | INFO | Processing Table-:Master_VisibilityReason | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-23 16:29:44 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-23 16:29:44 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:29:44 | INFO | Fetching in progress .... +2026-06-23 16:29:44 | INFO | Fetched 17 rows from SQL Server +2026-06-23 16:29:44 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-23 16:29:44 | INFO | Fetched 17 rows +2026-06-23 16:29:44 | INFO | Truncated table Master_VisibilityReason +2026-06-23 16:29:44 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:29:44 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-23 16:29:44 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-23 16:29:44 | INFO | ================================================================================ +2026-06-23 16:29:44 | INFO | Processing Table-:Master_VisibilityDefinition | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-23 16:29:44 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. +2026-06-23 16:29:44 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:29:44 | INFO | Fetching in progress .... +2026-06-23 16:29:44 | INFO | Fetched 861 rows from SQL Server +2026-06-23 16:29:44 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! +2026-06-23 16:29:44 | INFO | Fetched 861 rows +2026-06-23 16:29:44 | INFO | Truncated table Master_VisibilityDefinition +2026-06-23 16:29:44 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:29:45 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-23 16:29:45 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-23 16:29:45 | INFO | ================================================================================ +2026-06-23 16:29:45 | INFO | Processing Table-:Master_Salesterritorylayer | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-23 16:29:45 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. +2026-06-23 16:29:45 | INFO | Fetching data from sql server for Master table...... +2026-06-23 16:29:45 | INFO | Fetching in progress .... +2026-06-23 16:29:45 | INFO | Fetched 33 rows from SQL Server +2026-06-23 16:29:45 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! +2026-06-23 16:29:45 | INFO | Fetched 33 rows +2026-06-23 16:29:45 | INFO | Truncated table Master_Salesterritorylayer +2026-06-23 16:29:45 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 16:29:45 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-23 16:29:45 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-23 16:29:45 | INFO | ================================================================================ +2026-06-23 16:29:45 | INFO | Processing Table-:OQaD | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 16:29:45 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-23 16:29:45 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-23 16:30:04 | INFO | Fetched 291,600 total empid and visitdate fetched for OQAD from SQL Server +2026-06-23 16:30:05 | INFO | Fetched 291,165 matched empids fetched for OQAD +2026-06-23 16:30:05 | INFO | Exists: True +2026-06-23 16:30:05 | INFO | Path: D:\z\airflow-project\src\sql\fact\OQaD.sql +2026-06-23 16:30:05 | INFO | Fetching data for 291,165 EMPIDs +2026-06-23 16:30:05 | INFO | Fetching OQaD data for run_date=2026-06-23 +2026-06-23 16:30:53 | INFO | fn name is fetch_OQad ------Fetched 0 rows +2026-06-23 16:30:53 | INFO | Fetched 0 rows from SQL Server +2026-06-23 16:30:53 | INFO | Fetched total row -: 0 from sql server for table-:OQaD ...........!!! +2026-06-23 16:30:53 | WARNING | OQaD returned no rows +2026-06-23 16:30:53 | INFO | ================================================================================ +2026-06-23 16:30:53 | INFO | Pipeline Completed Successfully +2026-06-23 16:30:53 | INFO | ================================================================================ +2026-06-23 16:30:53 | INFO | Pipeline completed successfully. last_successful_run_date=2026-06-23 +2026-06-23 17:07:25 | INFO | Pipeline Start Date: 2026-06-22 +2026-06-23 17:07:25 | INFO | ================================================================================ +2026-06-23 17:07:25 | INFO | Hello from data-move Python data pipeline! +2026-06-23 17:07:25 | INFO | Pipeline Run Date: 2026-06-22 +2026-06-23 17:07:25 | INFO | Connecting to databases... +2026-06-23 17:07:25 | INFO | +2026-06-23 17:07:27 | INFO | +2026-06-23 17:07:28 | INFO | Database connections established +2026-06-23 17:07:28 | INFO | Collecting MIDs for: 2026-06-22 +2026-06-23 17:07:28 | INFO | Found 882 MIDs +2026-06-23 17:07:29 | INFO | ================================================================================ +2026-06-23 17:07:29 | INFO | Processing Table-:SOS_OneApp | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 17:07:29 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-23 17:07:29 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 17:07:29 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 17:07:29 | INFO | Fetching data for 882 MIDs +2026-06-23 17:07:34 | INFO | Fetched 3,888 rows from SQL Server +2026-06-23 17:07:34 | INFO | Fetched total row -: 3888 from sql server for table-:SOS_OneApp ...........!!! +2026-06-23 17:07:34 | INFO | Fetched 3888 rows +2026-06-23 17:07:34 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-23 17:07:34 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:07:34 | INFO | SOS_OneApp: inserted 3,888 rows into ClickHouse +2026-06-23 17:07:34 | INFO | SOS_OneApp loaded successfully (3888 rows) +2026-06-23 17:07:34 | INFO | ================================================================================ +2026-06-23 17:07:34 | INFO | Processing Table-:Survey | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 17:07:34 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-23 17:07:34 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 17:07:34 | INFO | Fetching data for 882 MIDs +2026-06-23 17:07:35 | INFO | Fetched 152 rows from SQL Server +2026-06-23 17:07:35 | INFO | Fetched total row -: 152 from sql server for table-:Survey ...........!!! +2026-06-23 17:07:35 | INFO | Fetched 152 rows +2026-06-23 17:07:35 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-23 17:07:35 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:07:35 | INFO | Survey: inserted 152 rows into ClickHouse +2026-06-23 17:07:35 | INFO | Survey loaded successfully (152 rows) +2026-06-23 17:07:35 | INFO | ================================================================================ +2026-06-23 17:07:35 | INFO | Processing Table-:additional_visibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 17:07:35 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-23 17:07:35 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 17:07:35 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 17:07:35 | INFO | Fetching data for 882 MIDs +2026-06-23 17:07:36 | INFO | Fetched 2,136 rows from SQL Server +2026-06-23 17:07:36 | INFO | Fetched total row -: 2136 from sql server for table-:additional_visibility ...........!!! +2026-06-23 17:07:36 | INFO | Fetched 2136 rows +2026-06-23 17:07:36 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-23 17:07:36 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:07:36 | INFO | additional_visibility: inserted 2,136 rows into ClickHouse +2026-06-23 17:07:36 | INFO | additional_visibility loaded successfully (2136 rows) +2026-06-23 17:07:36 | INFO | ================================================================================ +2026-06-23 17:07:36 | INFO | Processing Table-:Coverage | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 17:07:36 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-23 17:07:36 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 17:07:36 | INFO | Fetching data for 882 MIDs +2026-06-23 17:07:47 | INFO | Fetched 825 rows from SQL Server +2026-06-23 17:07:47 | INFO | Fetched total row -: 825 from sql server for table-:Coverage ...........!!! +2026-06-23 17:07:47 | INFO | Fetched 825 rows +2026-06-23 17:07:47 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-23 17:07:47 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:07:48 | INFO | Coverage: inserted 825 rows into ClickHouse +2026-06-23 17:07:48 | INFO | Coverage loaded successfully (825 rows) +2026-06-23 17:07:48 | INFO | ================================================================================ +2026-06-23 17:07:48 | INFO | Processing Table-:Login | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 17:07:48 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-23 17:07:48 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 17:07:48 | INFO | Fetching data for 882 MIDs +2026-06-23 17:07:50 | INFO | Fetched 501 rows from SQL Server +2026-06-23 17:07:50 | INFO | Fetched total row -: 501 from sql server for table-:Login ...........!!! +2026-06-23 17:07:50 | INFO | Fetched 501 rows +2026-06-23 17:07:50 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-23 17:07:50 | INFO | No delete logic required for Login +2026-06-23 17:07:50 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:07:50 | INFO | Login: inserted 501 rows into ClickHouse +2026-06-23 17:07:50 | INFO | Login loaded successfully (501 rows) +2026-06-23 17:07:50 | INFO | ================================================================================ +2026-06-23 17:07:50 | INFO | Processing Table-:Stock_Details | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 17:07:50 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-23 17:07:50 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 17:07:50 | INFO | Fetching data for 882 MIDs +2026-06-23 17:07:53 | INFO | Fetched 44,061 rows from SQL Server +2026-06-23 17:07:53 | INFO | Fetched total row -: 44061 from sql server for table-:Stock_Details ...........!!! +2026-06-23 17:07:53 | INFO | Fetched 44061 rows +2026-06-23 17:07:54 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-23 17:07:54 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:07:54 | INFO | Stock_Details: inserted 44,061 rows into ClickHouse +2026-06-23 17:07:54 | INFO | Stock_Details loaded successfully (44061 rows) +2026-06-23 17:07:54 | INFO | ================================================================================ +2026-06-23 17:07:54 | INFO | Processing Table-:Attendance | Table type -:FACT | fetcht by-:run_date | operation-:DELETE+INSERT +2026-06-23 17:07:54 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-23 17:07:54 | INFO | Fetching Attendance data from 2026-06-07 to 2026-06-22 +2026-06-23 17:08:03 | INFO | Fetched 121,132 attendance rows for 541 employees +2026-06-23 17:08:03 | INFO | Fetched total row -: 121132 from sql server for table-:Attendance ...........!!! +2026-06-23 17:08:03 | INFO | Fetched 121132 rows +2026-06-23 17:08:03 | INFO | Truncated table Attendance +2026-06-23 17:08:03 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:08:04 | INFO | Attendance: inserted 121,132 rows into ClickHouse +2026-06-23 17:08:04 | INFO | Attendance loaded successfully (121132 rows) +2026-06-23 17:08:04 | INFO | ================================================================================ +2026-06-23 17:08:04 | INFO | Processing Table-:Journey_Plan | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 17:08:04 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-23 17:08:04 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 17:08:04 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 17:08:04 | INFO | Fetching data for 882 MIDs +2026-06-23 17:08:05 | INFO | Fetched 21,739 rows from SQL Server +2026-06-23 17:08:05 | INFO | Fetched total row -: 21739 from sql server for table-:Journey_Plan ...........!!! +2026-06-23 17:08:05 | INFO | Fetched 21739 rows +2026-06-23 17:08:05 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-23 17:08:05 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:08:06 | INFO | Journey_Plan: inserted 21,739 rows into ClickHouse +2026-06-23 17:08:06 | INFO | Journey_Plan loaded successfully (21739 rows) +2026-06-23 17:08:06 | INFO | ================================================================================ +2026-06-23 17:08:06 | INFO | Processing Table-:Web_Logins | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 17:08:06 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-23 17:08:06 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 17:08:06 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 17:08:06 | INFO | Fetching data for 882 MIDs +2026-06-23 17:08:06 | INFO | Fetched 214 rows from SQL Server +2026-06-23 17:08:06 | INFO | Fetched total row -: 214 from sql server for table-:Web_Logins ...........!!! +2026-06-23 17:08:06 | INFO | Fetched 214 rows +2026-06-23 17:08:06 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ +2026-06-23 17:08:06 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:08:06 | INFO | Web_Logins: inserted 214 rows into ClickHouse +2026-06-23 17:08:06 | INFO | Web_Logins loaded successfully (214 rows) +2026-06-23 17:08:06 | INFO | ================================================================================ +2026-06-23 17:08:06 | INFO | Processing Table-:Promotion | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 17:08:06 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-23 17:08:06 | INFO | Fetching data for 882 MIDs +2026-06-23 17:08:09 | INFO | Fetched 8,835 rows from SQL Server +2026-06-23 17:08:09 | INFO | Fetched total row -: 8835 from sql server for table-:Promotion ...........!!! +2026-06-23 17:08:09 | INFO | Fetched 8835 rows +2026-06-23 17:08:09 | INFO | _ _ _ _ Deleting Data from ClickHouse for Promotion _ _ _ _ +2026-06-23 17:08:09 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:08:10 | INFO | Promotion: inserted 8,835 rows into ClickHouse +2026-06-23 17:08:10 | INFO | Promotion loaded successfully (8835 rows) +2026-06-23 17:08:10 | INFO | ================================================================================ +2026-06-23 17:08:10 | INFO | Processing Table-:PaidVisibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 17:08:10 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-23 17:08:10 | INFO | Start Fetching data for these 882 MIDs +2026-06-23 17:08:10 | INFO | Start Fetching data for these 882 MIDs or based on this date 2026-06-22 +2026-06-23 17:08:10 | INFO | Fetching data for 882 MIDs +2026-06-23 17:08:11 | INFO | Fetched 1,029 rows from SQL Server +2026-06-23 17:08:11 | INFO | Fetched total row -: 1029 from sql server for table-:PaidVisibility ...........!!! +2026-06-23 17:08:11 | INFO | Fetched 1029 rows +2026-06-23 17:08:11 | INFO | _ _ _ _ Deleting Data from ClickHouse for PaidVisibility _ _ _ _ +2026-06-23 17:08:11 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:08:11 | INFO | PaidVisibility: inserted 1,029 rows into ClickHouse +2026-06-23 17:08:11 | INFO | PaidVisibility loaded successfully (1029 rows) +2026-06-23 17:08:11 | INFO | ================================================================================ +2026-06-23 17:08:11 | INFO | Processing Table-:Store_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 17:08:11 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-23 17:08:11 | INFO | Fetching data from sql server for Master table...... +2026-06-23 17:08:11 | INFO | Fetching in progress .... +2026-06-23 17:08:17 | INFO | Fetched 5,998 rows from SQL Server +2026-06-23 17:08:17 | INFO | Fetched total row -: 5998 from sql server for table-:Store_Master ...........!!! +2026-06-23 17:08:17 | INFO | Fetched 5998 rows +2026-06-23 17:08:17 | INFO | Truncated table Store_Master +2026-06-23 17:08:17 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:08:18 | INFO | Store_Master: inserted 5,998 rows into ClickHouse +2026-06-23 17:08:18 | INFO | Store_Master loaded successfully (5998 rows) +2026-06-23 17:08:18 | INFO | ================================================================================ +2026-06-23 17:08:18 | INFO | Processing Table-:SKU_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 17:08:18 | INFO | Fetching Data from sql server for table-: SKU_Master .............. +2026-06-23 17:08:18 | INFO | Fetching data from sql server for Master table...... +2026-06-23 17:08:18 | INFO | Fetching in progress .... +2026-06-23 17:08:18 | INFO | Fetched 160 rows from SQL Server +2026-06-23 17:08:18 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! +2026-06-23 17:08:18 | INFO | Fetched 160 rows +2026-06-23 17:08:18 | INFO | Truncated table SKU_Master +2026-06-23 17:08:18 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:08:18 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-23 17:08:18 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-23 17:08:18 | INFO | ================================================================================ +2026-06-23 17:08:18 | INFO | Processing Table-:display_master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 17:08:18 | INFO | Fetching Data from sql server for table-: display_master .............. +2026-06-23 17:08:18 | INFO | Fetching data from sql server for Master table...... +2026-06-23 17:08:18 | INFO | Fetching in progress .... +2026-06-23 17:08:18 | INFO | Fetched 135 rows from SQL Server +2026-06-23 17:08:18 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! +2026-06-23 17:08:18 | INFO | Fetched 135 rows +2026-06-23 17:08:18 | INFO | Truncated table display_master +2026-06-23 17:08:18 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:08:19 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-23 17:08:19 | INFO | display_master loaded successfully (135 rows) +2026-06-23 17:08:19 | INFO | ================================================================================ +2026-06-23 17:08:19 | INFO | Processing Table-:Employee_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 17:08:19 | INFO | Fetching Data from sql server for table-: Employee_Master .............. +2026-06-23 17:08:19 | INFO | Fetching data from sql server for Master table...... +2026-06-23 17:08:19 | INFO | Fetching in progress .... +2026-06-23 17:08:20 | INFO | Fetched 2,289 rows from SQL Server +2026-06-23 17:08:20 | INFO | Fetched total row -: 2289 from sql server for table-:Employee_Master ...........!!! +2026-06-23 17:08:20 | INFO | Fetched 2289 rows +2026-06-23 17:08:20 | INFO | Truncated table Employee_Master +2026-06-23 17:08:20 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:08:20 | INFO | Employee_Master: inserted 2,289 rows into ClickHouse +2026-06-23 17:08:20 | INFO | Employee_Master loaded successfully (2289 rows) +2026-06-23 17:08:20 | INFO | ================================================================================ +2026-06-23 17:08:20 | INFO | Processing Table-:coverage_remarks | Table type -:DIMENSION | fetcht by-:reason_id | operation-:DELETE+INSERT +2026-06-23 17:08:20 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. +2026-06-23 17:08:21 | INFO | Fetching data from sql server for Master table...... +2026-06-23 17:08:21 | INFO | Fetching in progress .... +2026-06-23 17:08:21 | INFO | Fetched 0 rows from SQL Server +2026-06-23 17:08:21 | INFO | Fetched 0 rows from SQL Server +2026-06-23 17:08:21 | INFO | Fetched total row -: 0 from sql server for table-:coverage_remarks ...........!!! +2026-06-23 17:08:21 | WARNING | coverage_remarks returned no rows +2026-06-23 17:08:21 | INFO | ================================================================================ +2026-06-23 17:08:21 | INFO | Processing Table-:mapping_storevisibility | Table type -:BRIDGE | fetcht by-:run_date | operation-:DELETE+INSERT +2026-06-23 17:08:21 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-23 17:08:21 | INFO | Fetching data from sql server for BRIDGE table...... +2026-06-23 17:08:21 | INFO | Fetching in progress .... +2026-06-23 17:08:21 | INFO | Fetched 805 rows from SQL Server +2026-06-23 17:08:21 | INFO | Fetched total row -: 805 from sql server for table-:mapping_storevisibility ...........!!! +2026-06-23 17:08:21 | INFO | Fetched 805 rows +2026-06-23 17:08:22 | INFO | Truncated table mapping_storevisibility +2026-06-23 17:08:22 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:08:22 | INFO | mapping_storevisibility: inserted 805 rows into ClickHouse +2026-06-23 17:08:22 | INFO | mapping_storevisibility loaded successfully (805 rows) +2026-06-23 17:08:22 | INFO | ================================================================================ +2026-06-23 17:08:22 | INFO | Processing Table-:Master_VisibilityReason | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-23 17:08:22 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-23 17:08:22 | INFO | Fetching data from sql server for Master table...... +2026-06-23 17:08:22 | INFO | Fetching in progress .... +2026-06-23 17:08:22 | INFO | Fetched 17 rows from SQL Server +2026-06-23 17:08:22 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-23 17:08:22 | INFO | Fetched 17 rows +2026-06-23 17:08:22 | INFO | Truncated table Master_VisibilityReason +2026-06-23 17:08:22 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:08:22 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-23 17:08:22 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-23 17:08:22 | INFO | ================================================================================ +2026-06-23 17:08:22 | INFO | Processing Table-:Master_VisibilityDefinition | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-23 17:08:22 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. +2026-06-23 17:08:22 | INFO | Fetching data from sql server for Master table...... +2026-06-23 17:08:22 | INFO | Fetching in progress .... +2026-06-23 17:08:22 | INFO | Fetched 861 rows from SQL Server +2026-06-23 17:08:22 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! +2026-06-23 17:08:22 | INFO | Fetched 861 rows +2026-06-23 17:08:22 | INFO | Truncated table Master_VisibilityDefinition +2026-06-23 17:08:22 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:08:22 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-23 17:08:22 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-23 17:08:22 | INFO | ================================================================================ +2026-06-23 17:08:22 | INFO | Processing Table-:Master_Salesterritorylayer | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-23 17:08:22 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. +2026-06-23 17:08:22 | INFO | Fetching data from sql server for Master table...... +2026-06-23 17:08:22 | INFO | Fetching in progress .... +2026-06-23 17:08:23 | INFO | Fetched 33 rows from SQL Server +2026-06-23 17:08:23 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! +2026-06-23 17:08:23 | INFO | Fetched 33 rows +2026-06-23 17:08:23 | INFO | Truncated table Master_Salesterritorylayer +2026-06-23 17:08:23 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:08:23 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-23 17:08:23 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-23 17:08:23 | INFO | ================================================================================ +2026-06-23 17:08:23 | INFO | Processing Table-:OQaD | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 17:08:23 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-23 17:08:23 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-23 17:08:35 | INFO | Fetched 291,607 total empid and visitdate fetched for OQAD from SQL Server +2026-06-23 17:08:36 | INFO | Fetched 291,165 matched empids fetched for OQAD +2026-06-23 17:08:36 | INFO | Exists: True +2026-06-23 17:08:36 | INFO | Path: D:\z\airflow-project\src\sql\fact\OQaD.sql +2026-06-23 17:08:36 | INFO | Fetching data for 291,165 EMPIDs +2026-06-23 17:08:36 | INFO | Fetching OQaD data for run_date=2026-06-22 +2026-06-23 17:09:14 | INFO | fn name is fetch_OQad ------Fetched 49 rows +2026-06-23 17:09:14 | INFO | Fetched 49 rows from SQL Server +2026-06-23 17:09:14 | INFO | Fetched total row -: 49 from sql server for table-:OQaD ...........!!! +2026-06-23 17:09:14 | INFO | Fetched 49 rows +2026-06-23 17:09:14 | INFO | _ _ _ _ Deleting Data from ClickHouse for OQaD _ _ _ _ +2026-06-23 17:09:15 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:09:15 | INFO | OQaD: inserted 49 rows into ClickHouse +2026-06-23 17:09:15 | INFO | OQaD loaded successfully (49 rows) +2026-06-23 17:09:15 | INFO | ================================================================================ +2026-06-23 17:09:15 | INFO | Pipeline Completed Successfully +2026-06-23 17:09:15 | INFO | ================================================================================ +2026-06-23 17:09:15 | INFO | Pipeline completed successfully. last_successful_run_date=2026-06-23 +2026-06-23 17:09:15 | INFO | ================================================================================ +2026-06-23 17:09:15 | INFO | Hello from data-move Python data pipeline! +2026-06-23 17:09:15 | INFO | Pipeline Run Date: 2026-06-23 +2026-06-23 17:09:15 | INFO | Connecting to databases... +2026-06-23 17:09:15 | INFO | +2026-06-23 17:09:19 | INFO | +2026-06-23 17:09:20 | INFO | Database connections established +2026-06-23 17:09:20 | INFO | Collecting MIDs for: 2026-06-23 +2026-06-23 17:09:20 | INFO | Found 860 MIDs +2026-06-23 17:09:21 | INFO | ================================================================================ +2026-06-23 17:09:21 | INFO | Processing Table-:SOS_OneApp | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 17:09:21 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-23 17:09:21 | INFO | Start Fetching data for these 860 MIDs +2026-06-23 17:09:21 | INFO | Start Fetching data for these 860 MIDs or based on this date 2026-06-23 +2026-06-23 17:09:21 | INFO | Fetching data for 860 MIDs +2026-06-23 17:09:29 | INFO | Fetched 1,965 rows from SQL Server +2026-06-23 17:09:29 | INFO | Fetched total row -: 1965 from sql server for table-:SOS_OneApp ...........!!! +2026-06-23 17:09:29 | INFO | Fetched 1965 rows +2026-06-23 17:09:29 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-23 17:09:29 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:09:29 | INFO | SOS_OneApp: inserted 1,965 rows into ClickHouse +2026-06-23 17:09:29 | INFO | SOS_OneApp loaded successfully (1965 rows) +2026-06-23 17:09:29 | INFO | ================================================================================ +2026-06-23 17:09:29 | INFO | Processing Table-:Survey | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 17:09:29 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-23 17:09:29 | INFO | Start Fetching data for these 860 MIDs or based on this date 2026-06-23 +2026-06-23 17:09:29 | INFO | Fetching data for 860 MIDs +2026-06-23 17:09:30 | INFO | Fetched 60 rows from SQL Server +2026-06-23 17:09:30 | INFO | Fetched total row -: 60 from sql server for table-:Survey ...........!!! +2026-06-23 17:09:30 | INFO | Fetched 60 rows +2026-06-23 17:09:30 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-23 17:09:30 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:09:30 | INFO | Survey: inserted 60 rows into ClickHouse +2026-06-23 17:09:30 | INFO | Survey loaded successfully (60 rows) +2026-06-23 17:09:30 | INFO | ================================================================================ +2026-06-23 17:09:30 | INFO | Processing Table-:additional_visibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 17:09:30 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-23 17:09:30 | INFO | Start Fetching data for these 860 MIDs +2026-06-23 17:09:30 | INFO | Start Fetching data for these 860 MIDs or based on this date 2026-06-23 +2026-06-23 17:09:30 | INFO | Fetching data for 860 MIDs +2026-06-23 17:09:31 | INFO | Fetched 752 rows from SQL Server +2026-06-23 17:09:31 | INFO | Fetched total row -: 752 from sql server for table-:additional_visibility ...........!!! +2026-06-23 17:09:31 | INFO | Fetched 752 rows +2026-06-23 17:09:32 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-23 17:09:32 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:09:32 | INFO | additional_visibility: inserted 752 rows into ClickHouse +2026-06-23 17:09:32 | INFO | additional_visibility loaded successfully (752 rows) +2026-06-23 17:09:32 | INFO | ================================================================================ +2026-06-23 17:09:32 | INFO | Processing Table-:Coverage | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 17:09:32 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-23 17:09:32 | INFO | Start Fetching data for these 860 MIDs or based on this date 2026-06-23 +2026-06-23 17:09:32 | INFO | Fetching data for 860 MIDs +2026-06-23 17:09:33 | INFO | Fetched 803 rows from SQL Server +2026-06-23 17:09:33 | INFO | Fetched total row -: 803 from sql server for table-:Coverage ...........!!! +2026-06-23 17:09:33 | INFO | Fetched 803 rows +2026-06-23 17:09:33 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-23 17:09:33 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:09:33 | INFO | Coverage: inserted 803 rows into ClickHouse +2026-06-23 17:09:33 | INFO | Coverage loaded successfully (803 rows) +2026-06-23 17:09:33 | INFO | ================================================================================ +2026-06-23 17:09:33 | INFO | Processing Table-:Login | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 17:09:33 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-23 17:09:33 | INFO | Start Fetching data for these 860 MIDs or based on this date 2026-06-23 +2026-06-23 17:09:33 | INFO | Fetching data for 860 MIDs +2026-06-23 17:09:35 | INFO | Fetched 457 rows from SQL Server +2026-06-23 17:09:35 | INFO | Fetched total row -: 457 from sql server for table-:Login ...........!!! +2026-06-23 17:09:35 | INFO | Fetched 457 rows +2026-06-23 17:09:35 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-23 17:09:35 | INFO | No delete logic required for Login +2026-06-23 17:09:35 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:09:35 | INFO | Login: inserted 457 rows into ClickHouse +2026-06-23 17:09:35 | INFO | Login loaded successfully (457 rows) +2026-06-23 17:09:35 | INFO | ================================================================================ +2026-06-23 17:09:35 | INFO | Processing Table-:Stock_Details | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 17:09:35 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-23 17:09:35 | INFO | Start Fetching data for these 860 MIDs or based on this date 2026-06-23 +2026-06-23 17:09:35 | INFO | Fetching data for 860 MIDs +2026-06-23 17:09:59 | INFO | Fetched 21,429 rows from SQL Server +2026-06-23 17:09:59 | INFO | Fetched total row -: 21429 from sql server for table-:Stock_Details ...........!!! +2026-06-23 17:09:59 | INFO | Fetched 21429 rows +2026-06-23 17:09:59 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-23 17:09:59 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:10:00 | INFO | Stock_Details: inserted 21,429 rows into ClickHouse +2026-06-23 17:10:00 | INFO | Stock_Details loaded successfully (21429 rows) +2026-06-23 17:10:00 | INFO | ================================================================================ +2026-06-23 17:10:00 | INFO | Processing Table-:Attendance | Table type -:FACT | fetcht by-:run_date | operation-:DELETE+INSERT +2026-06-23 17:10:00 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-23 17:10:00 | INFO | Fetching Attendance data from 2026-06-08 to 2026-06-23 +2026-06-23 17:12:41 | INFO | Fetched 121,974 attendance rows for 541 employees +2026-06-23 17:12:41 | INFO | Fetched total row -: 121974 from sql server for table-:Attendance ...........!!! +2026-06-23 17:12:41 | INFO | Fetched 121974 rows +2026-06-23 17:12:42 | INFO | Truncated table Attendance +2026-06-23 17:12:42 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:12:43 | INFO | Attendance: inserted 121,974 rows into ClickHouse +2026-06-23 17:12:43 | INFO | Attendance loaded successfully (121974 rows) +2026-06-23 17:12:43 | INFO | ================================================================================ +2026-06-23 17:12:43 | INFO | Processing Table-:Journey_Plan | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 17:12:43 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-23 17:12:43 | INFO | Start Fetching data for these 860 MIDs +2026-06-23 17:12:43 | INFO | Start Fetching data for these 860 MIDs or based on this date 2026-06-23 +2026-06-23 17:12:43 | INFO | Fetching data for 860 MIDs +2026-06-23 17:12:46 | INFO | Fetched 21,739 rows from SQL Server +2026-06-23 17:12:46 | INFO | Fetched total row -: 21739 from sql server for table-:Journey_Plan ...........!!! +2026-06-23 17:12:46 | INFO | Fetched 21739 rows +2026-06-23 17:12:46 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-23 17:12:46 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:12:46 | INFO | Journey_Plan: inserted 21,739 rows into ClickHouse +2026-06-23 17:12:46 | INFO | Journey_Plan loaded successfully (21739 rows) +2026-06-23 17:12:46 | INFO | ================================================================================ +2026-06-23 17:12:46 | INFO | Processing Table-:Web_Logins | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 17:12:46 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-23 17:12:46 | INFO | Start Fetching data for these 860 MIDs +2026-06-23 17:12:46 | INFO | Start Fetching data for these 860 MIDs or based on this date 2026-06-23 +2026-06-23 17:12:46 | INFO | Fetching data for 860 MIDs +2026-06-23 17:12:47 | INFO | Fetched 109 rows from SQL Server +2026-06-23 17:12:47 | INFO | Fetched total row -: 109 from sql server for table-:Web_Logins ...........!!! +2026-06-23 17:12:47 | INFO | Fetched 109 rows +2026-06-23 17:12:47 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ +2026-06-23 17:12:47 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:12:47 | INFO | Web_Logins: inserted 109 rows into ClickHouse +2026-06-23 17:12:47 | INFO | Web_Logins loaded successfully (109 rows) +2026-06-23 17:12:47 | INFO | ================================================================================ +2026-06-23 17:12:47 | INFO | Processing Table-:Promotion | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 17:12:47 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-23 17:12:47 | INFO | Fetching data for 860 MIDs +2026-06-23 17:12:58 | INFO | Fetched 4,595 rows from SQL Server +2026-06-23 17:12:58 | INFO | Fetched total row -: 4595 from sql server for table-:Promotion ...........!!! +2026-06-23 17:12:58 | INFO | Fetched 4595 rows +2026-06-23 17:12:58 | INFO | _ _ _ _ Deleting Data from ClickHouse for Promotion _ _ _ _ +2026-06-23 17:12:58 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:12:58 | INFO | Promotion: inserted 4,595 rows into ClickHouse +2026-06-23 17:12:58 | INFO | Promotion loaded successfully (4595 rows) +2026-06-23 17:12:58 | INFO | ================================================================================ +2026-06-23 17:12:58 | INFO | Processing Table-:PaidVisibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 17:12:58 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-23 17:12:58 | INFO | Start Fetching data for these 860 MIDs +2026-06-23 17:12:58 | INFO | Start Fetching data for these 860 MIDs or based on this date 2026-06-23 +2026-06-23 17:12:58 | INFO | Fetching data for 860 MIDs +2026-06-23 17:13:01 | INFO | Fetched 284 rows from SQL Server +2026-06-23 17:13:01 | INFO | Fetched total row -: 284 from sql server for table-:PaidVisibility ...........!!! +2026-06-23 17:13:01 | INFO | Fetched 284 rows +2026-06-23 17:13:01 | INFO | _ _ _ _ Deleting Data from ClickHouse for PaidVisibility _ _ _ _ +2026-06-23 17:13:01 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:13:01 | INFO | PaidVisibility: inserted 284 rows into ClickHouse +2026-06-23 17:13:01 | INFO | PaidVisibility loaded successfully (284 rows) +2026-06-23 17:13:01 | INFO | ================================================================================ +2026-06-23 17:13:01 | INFO | Processing Table-:Store_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 17:13:01 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-23 17:13:01 | INFO | Fetching data from sql server for Master table...... +2026-06-23 17:13:01 | INFO | Fetching in progress .... +2026-06-23 17:13:13 | INFO | Fetched 5,998 rows from SQL Server +2026-06-23 17:13:13 | INFO | Fetched total row -: 5998 from sql server for table-:Store_Master ...........!!! +2026-06-23 17:13:13 | INFO | Fetched 5998 rows +2026-06-23 17:13:13 | INFO | Truncated table Store_Master +2026-06-23 17:13:13 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:13:14 | INFO | Store_Master: inserted 5,998 rows into ClickHouse +2026-06-23 17:13:14 | INFO | Store_Master loaded successfully (5998 rows) +2026-06-23 17:13:14 | INFO | ================================================================================ +2026-06-23 17:13:14 | INFO | Processing Table-:SKU_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 17:13:14 | INFO | Fetching Data from sql server for table-: SKU_Master .............. +2026-06-23 17:13:14 | INFO | Fetching data from sql server for Master table...... +2026-06-23 17:13:14 | INFO | Fetching in progress .... +2026-06-23 17:13:14 | INFO | Fetched 160 rows from SQL Server +2026-06-23 17:13:14 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! +2026-06-23 17:13:14 | INFO | Fetched 160 rows +2026-06-23 17:13:14 | INFO | Truncated table SKU_Master +2026-06-23 17:13:14 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:13:15 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-23 17:13:15 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-23 17:13:15 | INFO | ================================================================================ +2026-06-23 17:13:15 | INFO | Processing Table-:display_master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 17:13:15 | INFO | Fetching Data from sql server for table-: display_master .............. +2026-06-23 17:13:15 | INFO | Fetching data from sql server for Master table...... +2026-06-23 17:13:15 | INFO | Fetching in progress .... +2026-06-23 17:13:15 | INFO | Fetched 135 rows from SQL Server +2026-06-23 17:13:15 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! +2026-06-23 17:13:15 | INFO | Fetched 135 rows +2026-06-23 17:13:15 | INFO | Truncated table display_master +2026-06-23 17:13:15 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:13:16 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-23 17:13:16 | INFO | display_master loaded successfully (135 rows) +2026-06-23 17:13:16 | INFO | ================================================================================ +2026-06-23 17:13:16 | INFO | Processing Table-:Employee_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-23 17:13:16 | INFO | Fetching Data from sql server for table-: Employee_Master .............. +2026-06-23 17:13:16 | INFO | Fetching data from sql server for Master table...... +2026-06-23 17:13:16 | INFO | Fetching in progress .... +2026-06-23 17:13:22 | INFO | Fetched 2,289 rows from SQL Server +2026-06-23 17:13:22 | INFO | Fetched total row -: 2289 from sql server for table-:Employee_Master ...........!!! +2026-06-23 17:13:22 | INFO | Fetched 2289 rows +2026-06-23 17:13:22 | INFO | Truncated table Employee_Master +2026-06-23 17:13:22 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:13:22 | INFO | Employee_Master: inserted 2,289 rows into ClickHouse +2026-06-23 17:13:22 | INFO | Employee_Master loaded successfully (2289 rows) +2026-06-23 17:13:22 | INFO | ================================================================================ +2026-06-23 17:13:22 | INFO | Processing Table-:coverage_remarks | Table type -:DIMENSION | fetcht by-:reason_id | operation-:DELETE+INSERT +2026-06-23 17:13:22 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. +2026-06-23 17:13:23 | INFO | Fetching data from sql server for Master table...... +2026-06-23 17:13:23 | INFO | Fetching in progress .... +2026-06-23 17:13:23 | INFO | Fetched 0 rows from SQL Server +2026-06-23 17:13:23 | INFO | Fetched 0 rows from SQL Server +2026-06-23 17:13:23 | INFO | Fetched total row -: 0 from sql server for table-:coverage_remarks ...........!!! +2026-06-23 17:13:23 | WARNING | coverage_remarks returned no rows +2026-06-23 17:13:23 | INFO | ================================================================================ +2026-06-23 17:13:23 | INFO | Processing Table-:mapping_storevisibility | Table type -:BRIDGE | fetcht by-:run_date | operation-:DELETE+INSERT +2026-06-23 17:13:23 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-23 17:13:23 | INFO | Fetching data from sql server for BRIDGE table...... +2026-06-23 17:13:23 | INFO | Fetching in progress .... +2026-06-23 17:13:24 | INFO | Fetched 805 rows from SQL Server +2026-06-23 17:13:24 | INFO | Fetched total row -: 805 from sql server for table-:mapping_storevisibility ...........!!! +2026-06-23 17:13:24 | INFO | Fetched 805 rows +2026-06-23 17:13:24 | INFO | Truncated table mapping_storevisibility +2026-06-23 17:13:24 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:13:24 | INFO | mapping_storevisibility: inserted 805 rows into ClickHouse +2026-06-23 17:13:24 | INFO | mapping_storevisibility loaded successfully (805 rows) +2026-06-23 17:13:24 | INFO | ================================================================================ +2026-06-23 17:13:24 | INFO | Processing Table-:Master_VisibilityReason | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-23 17:13:24 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-23 17:13:24 | INFO | Fetching data from sql server for Master table...... +2026-06-23 17:13:24 | INFO | Fetching in progress .... +2026-06-23 17:13:24 | INFO | Fetched 17 rows from SQL Server +2026-06-23 17:13:24 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-23 17:13:24 | INFO | Fetched 17 rows +2026-06-23 17:13:24 | INFO | Truncated table Master_VisibilityReason +2026-06-23 17:13:24 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:13:24 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-23 17:13:24 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-23 17:13:24 | INFO | ================================================================================ +2026-06-23 17:13:24 | INFO | Processing Table-:Master_VisibilityDefinition | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-23 17:13:24 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. +2026-06-23 17:13:24 | INFO | Fetching data from sql server for Master table...... +2026-06-23 17:13:24 | INFO | Fetching in progress .... +2026-06-23 17:13:25 | INFO | Fetched 861 rows from SQL Server +2026-06-23 17:13:25 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! +2026-06-23 17:13:25 | INFO | Fetched 861 rows +2026-06-23 17:13:25 | INFO | Truncated table Master_VisibilityDefinition +2026-06-23 17:13:25 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:13:25 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-23 17:13:25 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-23 17:13:25 | INFO | ================================================================================ +2026-06-23 17:13:25 | INFO | Processing Table-:Master_Salesterritorylayer | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-23 17:13:25 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. +2026-06-23 17:13:25 | INFO | Fetching data from sql server for Master table...... +2026-06-23 17:13:25 | INFO | Fetching in progress .... +2026-06-23 17:13:26 | INFO | Fetched 33 rows from SQL Server +2026-06-23 17:13:26 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! +2026-06-23 17:13:26 | INFO | Fetched 33 rows +2026-06-23 17:13:26 | INFO | Truncated table Master_Salesterritorylayer +2026-06-23 17:13:26 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:13:26 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-23 17:13:26 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-23 17:13:26 | INFO | ================================================================================ +2026-06-23 17:13:26 | INFO | Processing Table-:OQaD | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 17:13:26 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-23 17:13:26 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-23 17:13:56 | INFO | Fetched 291,607 total empid and visitdate fetched for OQAD from SQL Server +2026-06-23 17:13:57 | INFO | Fetched 290,674 matched empids fetched for OQAD +2026-06-23 17:13:58 | INFO | Exists: True +2026-06-23 17:13:58 | INFO | Path: D:\z\airflow-project\src\sql\fact\OQaD.sql +2026-06-23 17:13:58 | INFO | Fetching data for 290,674 EMPIDs +2026-06-23 17:13:58 | INFO | Fetching OQaD data for run_date=2026-06-23 +2026-06-23 17:14:34 | INFO | fn name is fetch_OQad ------Fetched 2 rows +2026-06-23 17:14:34 | INFO | Fetched 2 rows from SQL Server +2026-06-23 17:14:34 | INFO | Fetched total row -: 2 from sql server for table-:OQaD ...........!!! +2026-06-23 17:14:34 | INFO | Fetched 2 rows +2026-06-23 17:14:34 | INFO | _ _ _ _ Deleting Data from ClickHouse for OQaD _ _ _ _ +2026-06-23 17:14:35 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 17:14:35 | INFO | OQaD: inserted 2 rows into ClickHouse +2026-06-23 17:14:35 | INFO | OQaD loaded successfully (2 rows) +2026-06-23 17:14:35 | INFO | ================================================================================ +2026-06-23 17:14:35 | INFO | Pipeline Completed Successfully +2026-06-23 17:14:35 | INFO | ================================================================================ +2026-06-23 17:14:35 | INFO | Pipeline completed successfully. last_successful_run_date=2026-06-23 +2026-06-23 18:06:23 | INFO | Pipeline Start Date: 2026-06-22 +2026-06-23 18:06:23 | INFO | ================================================================================ +2026-06-23 18:06:23 | INFO | Hello from data-move Python data pipeline! +2026-06-23 18:06:23 | INFO | Pipeline Run Date: 2026-06-22 +2026-06-23 18:06:23 | INFO | Connecting to databases... +2026-06-23 18:06:24 | INFO | +2026-06-23 18:06:26 | INFO | +2026-06-23 18:06:26 | INFO | Database connections established +2026-06-23 18:06:26 | INFO | Collecting MIDs for: 2026-06-22 +2026-06-23 18:06:27 | INFO | Found 882 MIDs +2026-06-23 18:06:27 | INFO | ================================================================================ +2026-06-23 18:06:27 | INFO | Processing Table-:OQaD | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 18:06:27 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-23 18:06:28 | INFO | Fetching quiz empids for run_date=2026-06-22 +2026-06-23 18:06:28 | INFO | Fetched 491 (EmpId, VisitDate) pairs from SQL Server +2026-06-23 18:06:29 | INFO | Fetched 290725 existing (EmpId, VisitDate) pairs from ClickHouse +2026-06-23 18:06:29 | INFO | Found 489 NEW empids to fetch for OQaD +2026-06-23 18:06:29 | INFO | Loading SQL from: D:\z\airflow-project\src\sql\fact\OQaD.sql (exists=True) +2026-06-23 18:06:29 | INFO | Fetching full OQaD data for 489 empids, run_date=2026-06-22 +2026-06-23 18:06:33 | INFO | Fetched 2 rows from SQL Server for table=OQaD +2026-06-23 18:06:33 | INFO | fetch_OQaD complete — returning 2 rows +2026-06-23 18:06:33 | INFO | Fetched total row -: 2 from sql server for table-:OQaD ...........!!! +2026-06-23 18:06:33 | INFO | Fetched 2 rows +2026-06-23 18:06:33 | INFO | _ _ _ _ Deleting Data from ClickHouse for OQaD _ _ _ _ +2026-06-23 18:06:33 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 18:06:33 | INFO | OQaD: inserted 2 rows into ClickHouse +2026-06-23 18:06:33 | INFO | OQaD loaded successfully (2 rows) +2026-06-23 18:06:33 | INFO | ================================================================================ +2026-06-23 18:06:33 | INFO | Processing Table-:Promotion | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 18:06:33 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-23 18:06:33 | INFO | Fetching data for 882 MIDs +2026-06-23 18:06:37 | INFO | Fetched 8,835 rows from SQL Server +2026-06-23 18:06:37 | INFO | Fetched total row -: 8835 from sql server for table-:Promotion ...........!!! +2026-06-23 18:06:37 | INFO | Fetched 8835 rows +2026-06-23 18:06:37 | INFO | _ _ _ _ Deleting Data from ClickHouse for Promotion _ _ _ _ +2026-06-23 18:06:37 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 18:06:37 | INFO | Promotion: inserted 8,835 rows into ClickHouse +2026-06-23 18:06:37 | INFO | Promotion loaded successfully (8835 rows) +2026-06-23 18:06:37 | INFO | ================================================================================ +2026-06-23 18:06:37 | INFO | Pipeline Completed Successfully +2026-06-23 18:06:37 | INFO | ================================================================================ +2026-06-23 18:06:37 | INFO | Pipeline completed successfully. last_successful_run_date=2026-06-23 +2026-06-23 18:06:37 | INFO | ================================================================================ +2026-06-23 18:06:37 | INFO | Hello from data-move Python data pipeline! +2026-06-23 18:06:37 | INFO | Pipeline Run Date: 2026-06-23 +2026-06-23 18:06:37 | INFO | Connecting to databases... +2026-06-23 18:06:38 | INFO | +2026-06-23 18:06:38 | INFO | +2026-06-23 18:06:38 | INFO | Database connections established +2026-06-23 18:06:38 | INFO | Collecting MIDs for: 2026-06-23 +2026-06-23 18:06:39 | INFO | Found 860 MIDs +2026-06-23 18:06:39 | INFO | ================================================================================ +2026-06-23 18:06:39 | INFO | Processing Table-:OQaD | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 18:06:39 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-23 18:06:39 | INFO | Fetching quiz empids for run_date=2026-06-23 +2026-06-23 18:06:40 | INFO | Fetched 447 (EmpId, VisitDate) pairs from SQL Server +2026-06-23 18:06:41 | INFO | Fetched 290725 existing (EmpId, VisitDate) pairs from ClickHouse +2026-06-23 18:06:41 | INFO | Found 446 NEW empids to fetch for OQaD +2026-06-23 18:06:41 | INFO | Loading SQL from: D:\z\airflow-project\src\sql\fact\OQaD.sql (exists=True) +2026-06-23 18:06:41 | INFO | Fetching full OQaD data for 446 empids, run_date=2026-06-23 +2026-06-23 18:06:44 | INFO | Fetched 1 rows from SQL Server for table=OQaD +2026-06-23 18:06:44 | INFO | fetch_OQaD complete — returning 1 rows +2026-06-23 18:06:44 | INFO | Fetched total row -: 1 from sql server for table-:OQaD ...........!!! +2026-06-23 18:06:44 | INFO | Fetched 1 rows +2026-06-23 18:06:44 | INFO | _ _ _ _ Deleting Data from ClickHouse for OQaD _ _ _ _ +2026-06-23 18:06:44 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 18:06:44 | INFO | OQaD: inserted 1 rows into ClickHouse +2026-06-23 18:06:44 | INFO | OQaD loaded successfully (1 rows) +2026-06-23 18:06:44 | INFO | ================================================================================ +2026-06-23 18:06:44 | INFO | Processing Table-:Promotion | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-23 18:06:44 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-23 18:06:44 | INFO | Fetching data for 860 MIDs +2026-06-23 18:06:46 | INFO | Fetched 5,668 rows from SQL Server +2026-06-23 18:06:46 | INFO | Fetched total row -: 5668 from sql server for table-:Promotion ...........!!! +2026-06-23 18:06:46 | INFO | Fetched 5668 rows +2026-06-23 18:06:46 | INFO | _ _ _ _ Deleting Data from ClickHouse for Promotion _ _ _ _ +2026-06-23 18:06:46 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 18:06:47 | INFO | Promotion: inserted 5,668 rows into ClickHouse +2026-06-23 18:06:47 | INFO | Promotion loaded successfully (5668 rows) +2026-06-23 18:06:47 | INFO | ================================================================================ +2026-06-23 18:06:47 | INFO | Pipeline Completed Successfully +2026-06-23 18:06:47 | INFO | ================================================================================ +2026-06-23 18:06:47 | INFO | Pipeline completed successfully. last_successful_run_date=2026-06-23 +2026-06-23 18:14:23 | INFO | Pipeline Start Date: 2026-06-22 +2026-06-23 18:14:23 | INFO | ================================================================================ +2026-06-23 18:14:23 | INFO | Hello from data-move Python data pipeline! +2026-06-23 18:14:23 | INFO | Pipeline Run Date: 2026-06-22 +2026-06-23 18:14:23 | INFO | Connecting to databases... +2026-06-23 18:14:24 | INFO | +2026-06-23 18:14:25 | INFO | +2026-06-23 18:14:26 | INFO | Database connections established +2026-06-23 18:14:26 | INFO | Collecting MIDs for: 2026-06-22 +2026-06-23 18:14:26 | INFO | Found 882 MIDs +2026-06-23 18:14:27 | INFO | ================================================================================ +2026-06-23 18:14:27 | INFO | Processing Table-:OQaD | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 18:14:27 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-23 18:14:27 | INFO | Fetching quiz empids for run_date=2026-06-22 +2026-06-23 18:14:28 | INFO | Fetched 491 (EmpId, VisitDate) pairs from SQL Server +2026-06-23 18:14:29 | INFO | Fetched 290725 existing (EmpId, VisitDate) pairs from ClickHouse +2026-06-23 18:14:29 | INFO | Found 489 NEW empids to fetch for OQaD +2026-06-23 18:14:29 | INFO | Loading SQL from: D:\z\airflow-project\src\sql\fact\OQaD.sql (exists=True) +2026-06-23 18:14:29 | INFO | Fetching full OQaD data for 489 empids, run_date=2026-06-22 +2026-06-23 18:14:32 | INFO | Fetched 2 rows from SQL Server for table=OQaD +2026-06-23 18:14:32 | INFO | fetch_OQaD complete — returning 2 rows +2026-06-23 18:14:32 | INFO | Fetched total row -: 2 from sql server for table-:OQaD ...........!!! +2026-06-23 18:14:32 | INFO | Fetched 2 rows +2026-06-23 18:14:32 | INFO | _ _ _ _ Deleting Data from ClickHouse for OQaD _ _ _ _ +2026-06-23 18:14:32 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 18:14:32 | INFO | OQaD: inserted 2 rows into ClickHouse +2026-06-23 18:14:32 | INFO | OQaD loaded successfully (2 rows) +2026-06-23 18:14:32 | INFO | ================================================================================ +2026-06-23 18:14:32 | INFO | Pipeline Completed Successfully +2026-06-23 18:14:32 | INFO | ================================================================================ +2026-06-23 18:14:32 | INFO | Pipeline completed successfully. last_successful_run_date=2026-06-23 +2026-06-23 18:14:32 | INFO | ================================================================================ +2026-06-23 18:14:32 | INFO | Hello from data-move Python data pipeline! +2026-06-23 18:14:32 | INFO | Pipeline Run Date: 2026-06-23 +2026-06-23 18:14:32 | INFO | Connecting to databases... +2026-06-23 18:14:33 | INFO | +2026-06-23 18:14:33 | INFO | +2026-06-23 18:14:34 | INFO | Database connections established +2026-06-23 18:14:34 | INFO | Collecting MIDs for: 2026-06-23 +2026-06-23 18:14:34 | INFO | Found 860 MIDs +2026-06-23 18:14:34 | INFO | ================================================================================ +2026-06-23 18:14:34 | INFO | Processing Table-:OQaD | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 18:14:34 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-23 18:14:34 | INFO | Fetching quiz empids for run_date=2026-06-23 +2026-06-23 18:14:35 | INFO | Fetched 447 (EmpId, VisitDate) pairs from SQL Server +2026-06-23 18:14:36 | INFO | Fetched 290725 existing (EmpId, VisitDate) pairs from ClickHouse +2026-06-23 18:14:36 | INFO | Found 446 NEW empids to fetch for OQaD +2026-06-23 18:14:36 | INFO | Loading SQL from: D:\z\airflow-project\src\sql\fact\OQaD.sql (exists=True) +2026-06-23 18:14:36 | INFO | Fetching full OQaD data for 446 empids, run_date=2026-06-23 +2026-06-23 18:14:39 | INFO | Fetched 1 rows from SQL Server for table=OQaD +2026-06-23 18:14:39 | INFO | fetch_OQaD complete — returning 1 rows +2026-06-23 18:14:39 | INFO | Fetched total row -: 1 from sql server for table-:OQaD ...........!!! +2026-06-23 18:14:39 | INFO | Fetched 1 rows +2026-06-23 18:14:39 | INFO | _ _ _ _ Deleting Data from ClickHouse for OQaD _ _ _ _ +2026-06-23 18:14:39 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 18:14:39 | INFO | OQaD: inserted 1 rows into ClickHouse +2026-06-23 18:14:39 | INFO | OQaD loaded successfully (1 rows) +2026-06-23 18:14:39 | INFO | ================================================================================ +2026-06-23 18:14:39 | INFO | Pipeline Completed Successfully +2026-06-23 18:14:39 | INFO | ================================================================================ +2026-06-23 18:14:39 | INFO | Pipeline completed successfully. last_successful_run_date=2026-06-23 +2026-06-23 18:18:24 | INFO | Pipeline Start Date: 2026-06-22 +2026-06-23 18:18:24 | INFO | ================================================================================ +2026-06-23 18:18:24 | INFO | Hello from data-move Python data pipeline! +2026-06-23 18:18:24 | INFO | Pipeline Run Date: 2026-06-22 +2026-06-23 18:18:24 | INFO | Connecting to databases... +2026-06-23 18:18:25 | INFO | +2026-06-23 18:18:26 | INFO | +2026-06-23 18:18:26 | INFO | Database connections established +2026-06-23 18:18:26 | INFO | Collecting MIDs for: 2026-06-22 +2026-06-23 18:18:27 | INFO | Found 882 MIDs +2026-06-23 18:18:27 | INFO | ================================================================================ +2026-06-23 18:18:27 | INFO | Processing Table-:OQaD | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-23 18:18:27 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-23 18:18:27 | INFO | Fetching quiz empids for run_date=2026-06-22 +2026-06-23 18:18:28 | INFO | Fetched 491 (EmpId, VisitDate) pairs from SQL Server +2026-06-23 18:18:29 | INFO | Fetched 290725 existing (EmpId, VisitDate) pairs from ClickHouse +2026-06-23 18:18:29 | INFO | Found 489 NEW empids to fetch for OQaD +2026-06-23 18:18:29 | INFO | Loading SQL from: D:\z\airflow-project\src\sql\fact\OQaD.sql (exists=True) +2026-06-23 18:18:29 | INFO | Fetching full OQaD data for 489 empids, run_date=2026-06-22 +2026-06-23 18:18:30 | INFO | Fetched 2 rows from SQL Server for table=OQaD +2026-06-23 18:18:30 | INFO | fetch_OQaD complete — returning 2 rows +2026-06-23 18:18:30 | INFO | Fetched total row -: 2 from sql server for table-:OQaD ...........!!! +2026-06-23 18:18:30 | INFO | Fetched 2 rows +2026-06-23 18:18:30 | INFO | _ _ _ _ Deleting Data from ClickHouse for OQaD _ _ _ _ +2026-06-23 18:18:31 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-23 18:18:31 | INFO | OQaD: inserted 2 rows into ClickHouse +2026-06-23 18:18:31 | INFO | OQaD loaded successfully (2 rows) +2026-06-23 18:18:31 | INFO | ================================================================================ +2026-06-23 18:18:31 | INFO | Pipeline Completed Successfully +2026-06-23 18:18:31 | INFO | ================================================================================ +2026-06-23 18:18:31 | INFO | Pipeline completed successfully. pipeline_trigeered_on_date=2026-06-23last_successful_run_date=2026-06-22 diff --git a/main2.py b/main2.py index 87439c4..57b5c0b 100644 --- a/main2.py +++ b/main2.py @@ -52,6 +52,45 @@ from src.dim import * # Helpers # ========================================================== +def get_dates_from_yaml(filename: str): + with open(filename, "r") as file: + data = yaml.safe_load(file) + + start_date = date.fromisoformat( + str(data["pipeline"]["start_date"]) + ) + + end_date = date.fromisoformat( + str(data["pipeline"]["end_date"]) + ) + flag=str(data["pipeline"]["flag"]) + + return start_date, end_date , flag + + +def write_table_to_yaml( + data: dict, + run_date: date, + filename: str | None = None +): + """Write table data to a YAML file.""" + + if filename is None: + filename = f"elt_pipeline_{run_date}.yml" + + with open(filename, "w") as file: + yaml.dump( + data, + file, + default_flow_style=False, + sort_keys=False + ) + + print(f"Table written to {filename}") + + + + def table_exists( client, table_name: str, @@ -68,7 +107,7 @@ def table_exists( # Main # ========================================================== -def main(): +def elt(run_date : date): log.info("=" * 80) log.info("Hello from data-move Python data pipeline!") @@ -77,13 +116,7 @@ def main(): # Run Date # ------------------------------------------------------ - if len(sys.argv) > 1: - run_date = datetime.strptime( - sys.argv[1], - "%Y-%m-%d", - ).date() - else: - run_date = date.today() - timedelta(days=1) + log.info( "Pipeline Run Date: %s", @@ -126,7 +159,7 @@ def main(): # ------------------------------------------------------ with open( - "t.yml", + "y.yml", "r", ) as file: @@ -211,8 +244,6 @@ def main(): table_name, ) - elif operation =="ONLY_INSERT" : - continue else: delete_existing_data( @@ -253,8 +284,8 @@ def main(): log.info("=" * 80) -if __name__ == "__main__": - +def main() : + config_file = Path("Pipeline_config.yml") if not config_file.exists(): @@ -273,41 +304,97 @@ if __name__ == "__main__": config = yaml.safe_load(f) - - for attempt in range(3): - try: - main() - with open("Pipeline_config.yml", "r") as f: - config = yaml.safe_load(f) + p_start_date, p_end_date , flag= get_dates_from_yaml("elt_pipeline_custom_dates.yml") + if flag =="Y" : + start_date=p_start_date + end_date=p_end_date - config["pipeline"]["last_successful_run_date"] = str(date.today()) + elif len(sys.argv) > 1: + start_date = datetime.strptime( + sys.argv[1], + "%Y-%m-%d", + ).date() + end_date=start_date + timedelta(days=1) + else: + start_date = date.today() - timedelta(days=1) + end_date=start_date - with open("Pipeline_config.yml", "w") as f: - yaml.safe_dump(config, f, sort_keys=False) + log.info( + "Pipeline Start Date: %s", + start_date, + ) + + + failed_dates=[] + successful_dates=[] + filename_successful = "successful_Pipeline_dates_config.yml" + filename_failed = "failed_Pipeline_dates_config.yml" + + while start_date <=end_date: + run_date = start_date - log.info( - f"Pipeline completed successfully. " - f"last_successful_run_date={date.today()}" + for attempt in range(3): + try: + elt(run_date) + + successful_dates.append({ + 'pipeline_trigeered_on_date': str(date.today()), + 'last_successful_run_date': run_date, + }) + + log.info( + f"Pipeline completed successfully. " + f"pipeline_trigeered_on_date={date.today()}" + f"last_successful_run_date={run_date}" + ) + + break + + except Exception as e: + + + + failed_dates.append({ + 'pipeline_trigeered_on_date': str(date.today()), + 'failed_run_date': run_date, + "attempt" : attempt + }) + + + if attempt == 2: + raise + + log.warning( + f"Pipeline failed. Retry {attempt + 1}/3. Error: {e}" + ) + + sleep(5) + + + start_date=start_date + timedelta(days=1) + + + + with open(filename_successful, "w") as f: + yaml.dump( + successful_dates, + f, + default_flow_style=False, + sort_keys=False, ) + if len(failed_dates) == 0 : + failed_dates.append({ + 'pipeline_trigeered_on_date': str(date.today()), + 'failed_run_date': "none", + "attempt" : "none" + }) + with open(filename_failed, "w") as f: + yaml.dump(failed_dates, + f, default_flow_style=False, + sort_keys=False) + - break +if __name__ == "__main__": - except Exception as e: - - with open("Pipeline_config.yml", "r") as f: - config = yaml.safe_load(f) - - config["pipeline"]["run_date"] = str(date.today()) - - with open("Pipeline_config.yml", "w") as f: - yaml.safe_dump(config, f, sort_keys=False) - - if attempt == 2: - raise - - log.warning( - f"Pipeline failed. Retry {attempt + 1}/3. Error: {e}" - ) - - sleep(5) \ No newline at end of file + main() \ No newline at end of file diff --git a/src/OQAD.py b/src/OQAD.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/bridge.py b/src/bridge.py index 8edb255..1169486 100644 --- a/src/bridge.py +++ b/src/bridge.py @@ -14,97 +14,125 @@ from db_con.connection import ( ) - - - def fetch_mapping_storevisibility( sql_engine: Engine, table_name: str, - table_type: str, - mids: list[int], - run_date: date + table_type: str, + mids: list[int], + run_date: date ) -> pl.DataFrame: - - run_date = run_date + timedelta(days=1) - client= get_clickhouse_client() - def table_exists( - client, - table_name: str, - ) -> bool: - return bool( - client.command( - f"EXISTS TABLE {table_name}" - ) - ) - def get_reason_ids_mapping_storevisibility( - client, - run_date: date, - table_name: str = "mapping_storevisibility", - ) -> list[int] : - if not table_exists(client, table_name): - log.warning(f"Table '{table_name}' does not exist. During collecting store_ids") - return [0] - - query = f""" - SELECT DISTINCT StoreId - FROM mapping_storevisibility - WHERE toDate(Fromdate) <= toDate('{run_date + timedelta(days= 1)}') - AND toDate(Todate) >= toDate('{run_date + timedelta(days= 1)}') - AND project_Id = '40148' + log.info(f"Fetching data from sql server for {table_type} table......") - """ + sql_file = Path("src") / "sql" / f"bridge" / f"{table_name}.sql" - # ClickHouse -> PyArrow -> Polars - arrow_table = client.query_arrow(query) + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() - df= pl.from_arrow(arrow_table) - list=df["reason_id"].to_list() - return list + sql = sql_template.format( ) - def fetch_data( - engine: Engine, - table_name: str, - table_type: str, - run_date: date, - store_id: list[int] -) -> pl.DataFrame: - log.info(f"Fetching data from sql server for Master table......") + log.info(f"Fetching in progress .... ") - store_id_list = ",".join(str(sid) for sid in store_id) + df = pl.read_database( + query=sql, + connection=sql_engine + ) - sql_file = Path("src") / "sql" / f"bridge" / f"{table_name}.sql" - - with open(sql_file, "r", encoding="utf-8") as f: - sql_template = f.read() - - sql = sql_template.format( - store_id_list=store_id_list, - run_date=run_date.strftime("%Y-%m-%d") - - ) - - log.info(f"Fetching in progress .... ") - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info(f"Fetched {len(df):,} rows from SQL Server") - - return df - - store_id=get_reason_ids_mapping_storevisibility(client, run_date, "mapping_storevisibility") - df=fetch_data(engine=sql_engine, - table_name=table_name, - table_type=table_type, - run_date=run_date, - store_id=store_id, - ) log.info(f"Fetched {len(df):,} rows from SQL Server") return df + +# def fetch_mapping_storevisibility( +# sql_engine: Engine, +# table_name: str, +# table_type: str, +# mids: list[int], +# run_date: date +# ) -> pl.DataFrame: + +# run_date = run_date + timedelta(days=1) +# client= get_clickhouse_client() +# def table_exists( +# client, +# table_name: str, +# ) -> bool: + +# return bool( +# client.command( +# f"EXISTS TABLE {table_name}" +# ) +# ) +# def get_reason_ids_mapping_storevisibility( +# client, +# run_date: date, +# table_name: str = "mapping_storevisibility", +# ) -> list[int] : + +# if not table_exists(client, table_name): +# log.warning(f"Table '{table_name}' does not exist. During collecting store_ids") +# return [0] + + +# query = f""" +# SELECT DISTINCT StoreId +# FROM mapping_storevisibility +# WHERE toDate(Fromdate) <= toDate('{run_date + timedelta(days= 1)}') +# AND toDate(Todate) >= toDate('{run_date + timedelta(days= 1)}') +# AND project_Id = '40148' + +# """ + +# # ClickHouse -> PyArrow -> Polars +# arrow_table = client.query_arrow(query) + +# df= pl.from_arrow(arrow_table) +# list=df["reason_id"].to_list() +# return list + +# def fetch_data( +# engine: Engine, +# table_name: str, +# table_type: str, +# run_date: date, +# store_id: list[int] +# ) -> pl.DataFrame: +# log.info(f"Fetching data from sql server for Master table......") + +# store_id_list = ",".join(str(sid) for sid in store_id) + +# sql_file = Path("src") / "sql" / f"bridge" / f"{table_name}.sql" + +# with open(sql_file, "r", encoding="utf-8") as f: +# sql_template = f.read() + +# sql = sql_template.format( +# store_id_list=store_id_list, +# run_date=run_date.strftime("%Y-%m-%d") + +# ) + +# log.info(f"Fetching in progress .... ") + +# df = pl.read_database( +# query=sql, +# connection=engine +# ) + +# log.info(f"Fetched {len(df):,} rows from SQL Server") + +# return df + +# store_id=get_reason_ids_mapping_storevisibility(client, run_date, "mapping_storevisibility") +# df=fetch_data(engine=sql_engine, +# table_name=table_name, +# table_type=table_type, +# run_date=run_date, +# store_id=store_id, +# ) +# log.info(f"Fetched {len(df):,} rows from SQL Server") + +# return df + diff --git a/src/fact.py b/src/fact.py index dcf2f21..d875571 100644 --- a/src/fact.py +++ b/src/fact.py @@ -134,6 +134,7 @@ def fetch_additional_visibility( engine: Engine, return df + def fetch_OQaD( sql_engine: Engine, table_name: str, @@ -142,166 +143,335 @@ def fetch_OQaD( run_date: date ) -> pl.DataFrame: + # ───────────────────────────────────────────── + # INNER HELPERS (defined once, used below) + # ───────────────────────────────────────────── - client= get_clickhouse_client() - def table_exists( - client, - table_name: str, - ) -> bool: + client = get_clickhouse_client() - return bool( - client.command( - f"EXISTS TABLE {table_name}" - ) - ) - + # ── Does a ClickHouse table exist? ──────────── + def table_exists(client, table_name: str) -> bool: - - def fetch_quiz_empids(engine: Engine, run_date : date) -> pl.DataFrame: + return bool(client.command(f"EXISTS TABLE {table_name}")) - sql_template = f""" - WITH MID_TABLE_COV1 AS + # ── STEP 1: Who submitted yesterday in SQL Server? ─── + def fetch_quiz_empids(engine: Engine, run_date: date) -> pl.DataFrame: + + + # Format date ONCE safely — avoids f-string injection bugs + run_date_str = run_date.strftime("%Y-%m-%d") + next_date_str = (run_date + timedelta(days=1)).strftime("%Y-%m-%d") + + sql = f""" + WITH MID_TABLE_COV1 AS ( - SELECT EmpId, VisitDate - FROM OneApp_KelloggsMT.dbo.T_OQAD - WHERE CreateDate >= {run_date} - AND CreateDate < DATEADD(DAY,1,'{run_date}') + -- Records CREATED yesterday + SELECT EmpId, CAST(VisitDate AS DATE) AS VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CreateDate >= '{run_date_str}' + AND CreateDate < '{next_date_str}' UNION ALL - SELECT EmpId, VisitDate - FROM OneApp_KelloggsMT.dbo.T_OQAD - WHERE UpdateDate >= {run_date} - AND UpdateDate < DATEADD(DAY,1, '{run_date}') + -- Records UPDATED yesterday (different rows, safe to UNION ALL) + SELECT EmpId, CAST(VisitDate AS DATE) AS VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE UpdateDate >= '{run_date_str}' + AND UpdateDate < '{next_date_str}' ), - QUIZ AS - ( - SELECT Distinct E.EmpId as empid - , CONVERT(date,DQ.VisitDate) AS visitdate - FROM OneApp_KelloggsMT.dbo.T_OQAD DQ INNER JOIN - OneApp_KelloggsMT.dbo.vw_Employee_Detail E ON DQ.EmpId = E.EmpId inner join - OneApp_KelloggsMT.dbo.Master_OQAD_Question QU on DQ.QuestionId= qu.QuestionId inner join - OneApp_KelloggsMT.dbo.Master_OQAD_Category qc on qu.QuestionCategoryId= qc.QuestionCategoryId - where e.EmpName not like 'test%' and e.RightId in (6) - and (E.ResignDate is null or E.ResignDate>=''+CONVERT(VARCHAR,'{run_date}')+'') AND E.EmpName NOT LIKE '%TEST%' - AND DQ.EmpId IN (SELECT EmpId FROM MID_TABLE_COV1 A WHERE - DQ.EmpId=A.EmpId AND CONVERT(date,VisitDate)=CONVERT(date,A.VisitDate) ) - ) select * from quiz - """ - sql = sql_template.format( - run_date=run_date.strftime("%Y-%m-%d") - ) - - log.info(f"Fetching quiz_empids data for EMPID and Visitid") - - df = pl.read_database( - query=sql, - connection=engine - ) - - - log.info(f"Fetched {len(df):,} total empid and visitdate fetched for OQAD from SQL Server") - - return df - - - def get_empids_clickhouse_OQAD( - client, - table_name: str = "OQaD", - ) -> pl.DataFrame: - - if not table_exists(client, table_name): - log.warning(f"Table '{table_name}' does not exist.") - return pl.DataFrame( - schema={ - "empid": pl.Int64, - "visitdate": pl.Date, - } - ) - - query = f""" + ( SELECT DISTINCT - employee_id AS empid, - visit_date AS visitdate - FROM {table_name} - """ + E.EmpId AS empid, + CAST(DQ.VisitDate AS DATE) AS visitdate + FROM OneApp_KelloggsMT.dbo.T_OQAD DQ + INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E + ON DQ.EmpId = E.EmpId + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU + ON DQ.QuestionId = QU.QuestionId + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC + ON QU.QuestionCategoryId = QC.QuestionCategoryId + WHERE E.EmpName NOT LIKE '%TEST%' -- exclude test employees + AND E.RightId = 6 -- only field reps + AND ( + E.ResignDate IS NULL + OR CAST(E.ResignDate AS DATE) >= '{run_date_str}' + ) + AND EXISTS ( -- ✅ EXISTS beats IN for large sets + SELECT 1 + FROM MID_TABLE_COV1 A + WHERE A.EmpId = DQ.EmpId + AND A.VisitDate = CAST(DQ.VisitDate AS DATE) + ) + ) + SELECT * FROM QUIZ + """ - # ClickHouse -> PyArrow -> Polars - arrow_table = client.query_arrow(query) + log.info("Fetching quiz empids for run_date=%s", run_date_str) + df = pl.read_database(query=sql, connection=engine) + log.info("Fetched %s (EmpId, VisitDate) pairs from SQL Server", len(df)) + return df - return pl.from_arrow(arrow_table) - + # ── STEP 2: Who do we ALREADY have in ClickHouse? ─── + def get_empids_clickhouse_OQAD( + client, + table_name: str = "OQaD", + ) -> pl.DataFrame: - qf=fetch_quiz_empids(sql_engine,run_date) - db_df = get_empids_clickhouse_OQAD(client) + if not table_exists(client, table_name): + log.warning("Table '%s' does not exist in ClickHouse.", table_name) + return pl.DataFrame(schema={"empid": pl.Int64, "visitdate": pl.Date}) - matched = qf.join( - db_df, + query = f""" + SELECT DISTINCT + employee_id AS empid, + visit_date AS visitdate + FROM {table_name} + """ + + arrow_table = client.query_arrow(query) + df = pl.from_arrow(arrow_table) + log.info("Fetched %s existing (EmpId, VisitDate) pairs from ClickHouse", len(df)) + return df + + # ── STEP 3: Who is NEW? (in SQL Server but NOT yet in ClickHouse) ─── + def find_new_empids( + sql_df: pl.DataFrame, + ch_df: pl.DataFrame, + ) -> list[int]: + + + new_df = sql_df.join( + ch_df, on=["empid", "visitdate"], - how="inner", + how="anti", # ✅ anti = keep rows NOT found in ch_df ) + + if new_df.is_empty(): + log.warning("No new EmpIds found for table=%s — nothing to fetch.", table_name) + return [0] # sentinel value — the .sql WHERE will return 0 rows safely + + empids = new_df["empid"].unique().to_list() + log.info("Found %s NEW empids to fetch for %s", len(empids), table_name) + return empids + + # ── STEP 4: Fetch full quiz data for new empids ─── + def fetch_data( + engine: Engine, + table_name: str, + table_type: str, + empids: list[int], + run_date: date, + ) -> pl.DataFrame: + + + run_date_str = run_date.strftime("%Y-%m-%d") + empid_list = ", ".join(str(e) for e in empids) # "101, 102, 103" + + sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql" + log.info("Loading SQL from: %s (exists=%s)", sql_file.resolve(), sql_file.exists()) + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + empid_list=empid_list, + run_date=run_date_str, + ) + + log.info("Fetching full OQaD data for %s empids, run_date=%s", len(empids), run_date_str) + df = pl.read_database(query=sql, connection=engine) + log.info("Fetched %s rows from SQL Server for table=%s", len(df), table_name) + return df + + # ───────────────────────────────────────────── + # MAIN FLOW (the 4 steps, clearly sequenced) + # ───────────────────────────────────────────── + + qf = fetch_quiz_empids(sql_engine, run_date) # Step 1 + db_df = get_empids_clickhouse_OQAD(client, table_name) # Step 2 + empids = find_new_empids(qf, db_df) # Step 3 + + df = fetch_data( # Step 4 + engine=sql_engine, + table_name=table_name, + table_type=table_type, + empids=empids, + run_date=run_date, + ) + + log.info("fetch_OQaD complete — returning %s rows", len(df)) + return df + +# def fetch_OQaD( +# sql_engine: Engine, +# table_name: str, +# table_type: str, +# mids: list[int], +# run_date: date +# ) -> pl.DataFrame: + + +# client= get_clickhouse_client() +# def table_exists( +# client, +# table_name: str, +# ) -> bool: + +# return bool( +# client.command( +# f"EXISTS TABLE {table_name}" +# ) +# ) - if matched.is_empty(): - empids=[0] - log.warning( - "%s Matched df in OQaD returned no rows", - table_name, - ) - - else: - empids=matched["empid"].to_list() + +# def fetch_quiz_empids(engine: Engine, run_date : date) -> pl.DataFrame: - - log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ") +# sql_template = f""" +# WITH MID_TABLE_COV1 AS +# ( +# SELECT EmpId, VisitDate +# FROM OneApp_KelloggsMT.dbo.T_OQAD +# WHERE CreateDate >= {run_date} +# AND CreateDate < DATEADD(DAY,1,'{run_date}') - def fetch_data( - engine: Engine, - table_name: str, - table_type: str, - empids: list[int], - run_date: date - ) -> pl.DataFrame: +# UNION - empid_list = ",".join(str(empid) for empid in empids) +# SELECT EmpId, VisitDate +# FROM OneApp_KelloggsMT.dbo.T_OQAD +# WHERE UpdateDate >= {run_date} +# AND UpdateDate < DATEADD(DAY,1, '{run_date}') +# ), + +# QUIZ AS +# ( +# SELECT Distinct E.EmpId as empid +# , CONVERT(date,DQ.VisitDate) AS visitdate +# FROM OneApp_KelloggsMT.dbo.T_OQAD DQ INNER JOIN +# OneApp_KelloggsMT.dbo.vw_Employee_Detail E ON DQ.EmpId = E.EmpId inner join +# OneApp_KelloggsMT.dbo.Master_OQAD_Question QU on DQ.QuestionId= qu.QuestionId inner join +# OneApp_KelloggsMT.dbo.Master_OQAD_Category qc on qu.QuestionCategoryId= qc.QuestionCategoryId +# where e.EmpName not like 'test%' and e.RightId in (6) +# and (E.ResignDate is null or E.ResignDate>=''+CONVERT(VARCHAR,'{run_date}')+'') AND E.EmpName NOT LIKE '%TEST%' +# AND DQ.EmpId IN (SELECT EmpId FROM MID_TABLE_COV1 A WHERE +# DQ.EmpId=A.EmpId AND CONVERT(date,VisitDate)=CONVERT(date,A.VisitDate) ) +# ) select * from quiz +# """ +# sql = sql_template.format( +# run_date=run_date.strftime("%Y-%m-%d") +# ) + +# log.info(f"Fetching quiz_empids data for EMPID and Visitid") + +# df = pl.read_database( +# query=sql, +# connection=engine +# ) - sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql" +# log.info(f"Fetched {len(df):,} total empid and visitdate fetched for OQAD from SQL Server") - log.info(f"Exists: {sql_file.exists()}") - log.info(f"Path: {sql_file.resolve()}") +# return df - with open(sql_file, "r", encoding="utf-8") as f: - sql_template = f.read() - sql = sql_template.format( - empid_list=empid_list, - run_date=run_date.strftime("%Y-%m-%d") - ) +# def get_empids_clickhouse_OQAD( +# client, +# table_name: str = "OQaD", +# ) -> pl.DataFrame: - log.info(f"Fetching data for {len(empids):,} EMPIDs") +# if not table_exists(client, table_name): +# log.warning(f"Table '{table_name}' does not exist.") +# return pl.DataFrame( +# schema={ +# "empid": pl.Int64, +# "visitdate": pl.Date, +# } +# ) - log.info("Fetching OQaD data for run_date=%s", run_date) +# query = f""" +# SELECT DISTINCT +# employee_id AS empid, +# visit_date AS visitdate +# FROM {table_name} +# """ - df = pl.read_database( - query=sql, - connection=engine, - ) +# # ClickHouse -> PyArrow -> Polars +# arrow_table = client.query_arrow(query) - log.info("fn name is fetch_OQad ------Fetched %s rows", len(df)) +# return pl.from_arrow(arrow_table) + - return df - df=fetch_data( engine=sql_engine, - table_name=table_name, - table_type=table_type, - empids=empids, - run_date=run_date - ) - log.info(f"Fetched {len(df):,} rows from SQL Server") - return df +# qf=fetch_quiz_empids(sql_engine,run_date) +# db_df = get_empids_clickhouse_OQAD(client) + +# matched = qf.join( +# db_df, +# on=["empid", "visitdate"], +# how="inner", +# ) + +# if matched.is_empty(): + +# empids=[0] +# log.warning( +# "%s Matched df in OQaD returned no rows", +# table_name, +# ) + +# else: +# empids=matched["empid"].to_list() + + +# log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ") + +# def fetch_data( +# engine: Engine, +# table_name: str, +# table_type: str, +# empids: list[int], +# run_date: date +# ) -> pl.DataFrame: + +# empid_list = ",".join(str(empid) for empid in empids) + + +# sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql" + +# log.info(f"Exists: {sql_file.exists()}") +# log.info(f"Path: {sql_file.resolve()}") + +# with open(sql_file, "r", encoding="utf-8") as f: +# sql_template = f.read() + +# sql = sql_template.format( +# empid_list=empid_list, +# run_date=run_date.strftime("%Y-%m-%d") +# ) + +# log.info(f"Fetching data for {len(empids):,} EMPIDs") + +# log.info("Fetching OQaD data for run_date=%s", run_date) + +# df = pl.read_database( +# query=sql, +# connection=engine, +# ) + +# log.info("fn name is fetch_OQad ------Fetched %s rows", len(df)) + +# return df +# df=fetch_data( engine=sql_engine, +# table_name=table_name, +# table_type=table_type, +# empids=empids, +# run_date=run_date +# ) +# log.info(f"Fetched {len(df):,} rows from SQL Server") + +# return df # def fetch_OQaD( diff --git a/src/sql/bridge/mapping_storevisibility.sql b/src/sql/bridge/mapping_storevisibility.sql index af35f60..1ae8fe4 100644 --- a/src/sql/bridge/mapping_storevisibility.sql +++ b/src/sql/bridge/mapping_storevisibility.sql @@ -1,11 +1,10 @@ + + + with mapping_storevisibility (Project_Id,StoreId,VisibilityDefinitionid,Fromdate,Todate,CreateDate,CreateBy) AS ( select DISTINCT '40148' as Project_Id,StoreId,VisibilityDefinitionid,Fromdate,Todate,getdate(),'SP-Pius' -FROM OneApp_KelloggsMT.dbo.mapping_storevisibility z WHERE -convert(date,FROMDATE,101)<=convert(Date,getdate(),101) AND CONVERT(DATE,ToDate,101)>=convert(Date,getdate(),101) -AND z.VisibilityDefinitionid IN -(SELECT DISTINCT VisibilityDefinitionid FROM OneApp_KelloggsMT.dbo.MASTER_VISIBILITYDEFINITION WHERE MENUID=22 ) -AND z.StoreId NOT IN ({store_id_list}) +FROM OneApp_KelloggsMT.dbo.mapping_storevisibility ) select * from mapping_storevisibility \ No newline at end of file diff --git a/src/sql/fact/OQaD.sql b/src/sql/fact/OQaD.sql index c2cbeca..bf6feff 100644 --- a/src/sql/fact/OQaD.sql +++ b/src/sql/fact/OQaD.sql @@ -1,19 +1,19 @@ WITH MID_TABLE_COV1 AS ( - SELECT EmpId, VisitDate - FROM OneApp_KelloggsMT.dbo.T_OQAD - WHERE CreateDate >= {run_date} - AND CreateDate < DATEADD(DAY,1,'{run_date}') + + SELECT EmpId, CAST(VisitDate AS DATE) AS VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CreateDate >= '{run_date}' + AND CreateDate < DATEADD(DAY, 1, '{run_date}') UNION ALL - SELECT EmpId, VisitDate - FROM OneApp_KelloggsMT.dbo.T_OQAD - WHERE UpdateDate >= {run_date} - AND UpdateDate < DATEADD(DAY,1, '{run_date}') + SELECT EmpId, CAST(VisitDate AS DATE) AS VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE UpdateDate >= '{run_date}' + AND UpdateDate < DATEADD(DAY, 1, '{run_date}') ), - QUIZ AS ( SELECT DISTINCT @@ -25,57 +25,52 @@ QUIZ AS E.CityName, E.StateName, E.RegionName, - CAST(DQ.VisitDate AS DATE) AS VisitDate, + CAST(DQ.VisitDate AS DATE) AS VisitDate, DQ.QuestionId, DQ.AnswerId, QC.QuestionCategoryId, QC.QuestionCategory - FROM OneApp_KelloggsMT.dbo.T_OQAD DQ - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E - ON DQ.EmpId = E.EmpId + FROM OneApp_KelloggsMT.dbo.T_OQAD DQ + INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E + ON DQ.EmpId = E.EmpId INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU - ON DQ.QuestionId = QU.QuestionId + ON DQ.QuestionId = QU.QuestionId INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC - ON QU.QuestionCategoryId = QC.QuestionCategoryId - WHERE E.EmpName NOT LIKE '%TEST%' - AND E.RightId = 6 - AND ( - E.ResignDate IS NULL - OR CAST(E.ResignDate AS DATE) >= '{run_date}' - ) - AND EXISTS - ( - SELECT 1 - FROM MID_TABLE_COV1 A - WHERE A.EmpId = DQ.EmpId - AND CAST(A.VisitDate AS DATE) = CAST(DQ.VisitDate AS DATE) - ) + ON QU.QuestionCategoryId = QC.QuestionCategoryId + WHERE E.EmpName NOT LIKE '%TEST%' + AND E.RightId = 6 + AND (E.ResignDate IS NULL OR CAST(E.ResignDate AS DATE) >= '{run_date}') + AND EXISTS ( + SELECT 1 + FROM MID_TABLE_COV1 A + WHERE A.EmpId = DQ.EmpId + AND A.VisitDate = CAST(DQ.VisitDate AS DATE) + ) + -- ✅ Exclude EmpIds already loaded into ClickHouse + AND E.EmpId NOT IN ({empid_list}) ) SELECT - 40148 AS project_id, - Q.EmpId AS employee_id, - 0 AS process_id, - Q.VisitDate AS visit_date, - Q.QuestionCategoryId AS question_category_id, - Q.QuestionCategory AS question_category, - QM.QuestionId AS question_id, - QM.Question AS question, - ISNULL(QA.AnswerId,0) AS answer_id, - ISNULL(QA.Answer,'') AS answer, - CASE - WHEN QA.AnswerId IS NULL THEN 'Not Answer' - WHEN QA.RightAnswer = 1 THEN 'Y' - WHEN QA.RightAnswer IS NULL THEN 'Not Answer' - ELSE 'N' - END AS correct_answer, - GETDATE() AS update_date, - 'SP-Pius' AS update_by -FROM QUIZ Q + 40148 AS project_id, + Q.EmpId AS employee_id, + 0 AS process_id, + Q.VisitDate AS visit_date, + Q.QuestionCategoryId AS question_category_id, + Q.QuestionCategory AS question_category, + QM.QuestionId AS question_id, + QM.Question AS question, + ISNULL(QA.AnswerId, 0) AS answer_id, + ISNULL(QA.Answer, '') AS answer, + CASE + WHEN QA.AnswerId IS NULL THEN 'Not Answer' + WHEN QA.RightAnswer = 1 THEN 'Y' + WHEN QA.RightAnswer IS NULL THEN 'Not Answer' + ELSE 'N' + END AS correct_answer, + GETDATE() AS update_date, + 'SP-Pius' AS update_by +FROM QUIZ Q INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM - ON Q.QuestionId = QM.QuestionId -LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA - ON Q.AnswerId = QA.AnswerId -where Q.EmpId not in ({empid_list}) - - + ON Q.QuestionId = QM.QuestionId +LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA + ON Q.AnswerId = QA.AnswerId \ No newline at end of file diff --git a/successful_Pipeline_dates_config.yml b/successful_Pipeline_dates_config.yml new file mode 100644 index 0000000..278d2a6 --- /dev/null +++ b/successful_Pipeline_dates_config.yml @@ -0,0 +1,2 @@ +- pipeline_trigeered_on_date: '2026-06-23' + last_successful_run_date: 2026-06-22 diff --git a/t.yml b/t.yml index 285375d..01144ff 100644 --- a/t.yml +++ b/t.yml @@ -4,11 +4,6 @@ tables: operation: INSERT fetch_by: mids - - name: OQaD - type: FACT - operation: INSERT - fetch_by: run_date - - name: Survey type: FACT operation: INSERT @@ -87,7 +82,7 @@ tables: - name: mapping_storevisibility type: BRIDGE - operation: ONLY_INSERT + operation: DELETE+INSERT fetch_by: run_date - name: Master_VisibilityReason @@ -104,4 +99,10 @@ tables: - name: Master_Salesterritorylayer type: DIMENSION operation: DELETE+INSERT - fetch_by: none \ No newline at end of file + fetch_by: none + + + - name: OQaD + type: FACT + operation: INSERT + fetch_by: run_date \ No newline at end of file diff --git a/y.yml b/y.yml index 737c148..81a0b71 100644 --- a/y.yml +++ b/y.yml @@ -4,10 +4,10 @@ tables: # operation: INSERT # fetch_by: mids - # # - name: OQaD - # # type: FACT - # # operation: INSERT - # # fetch_by: run_date + - name: OQaD + type: FACT + operation: INSERT + fetch_by: run_date # - name: additional_visibility # type: FACT @@ -102,8 +102,8 @@ tables: # operation: DELETE+INSERT # fetch_by: none - - name: Promotion - type: FACT - operation: INSERT - fetch_by: mids + # - name: Promotion + # type: FACT + # operation: INSERT + # fetch_by: mids