diff --git a/clickhouse_task/delete_task.py b/clickhouse_task/delete_task.py index 13c5cea..2f289df 100644 --- a/clickhouse_task/delete_task.py +++ b/clickhouse_task/delete_task.py @@ -1,6 +1,7 @@ from __future__ import annotations import polars as pl +from datetime import date, datetime, timedelta from log import log @@ -91,9 +92,10 @@ def delete_existing_data( client, table_name, f""" - toMonth(visit_date) = {run_date.month} - AND toYear(visit_date) = {run_date.year} - """, + project_id = 40148 + AND toMonth(visit_date) = toMonth(toDate('2026-06-18')) + AND toYear(visit_date) = toYear(toDate('2026-06-18')) + """, ) return @@ -142,7 +144,7 @@ def delete_existing_data( client, table_name, f""" - toDate(visit_date) BETWEEN toDate('{run_date - }') AND toDate('{run_date}') + toDate(visit_date) BETWEEN toDate('{run_date - timedelta(days=15) }') AND toDate('{run_date}') = """, ) diff --git a/logs/etl_20260619.log b/logs/etl_20260619.log new file mode 100644 index 0000000..49f298f --- /dev/null +++ b/logs/etl_20260619.log @@ -0,0 +1,1295 @@ +2026-06-19 10:14:16 | INFO | ================================================================================ +2026-06-19 10:14:16 | INFO | Hello from data-move Python data pipeline! +2026-06-19 10:14:16 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 10:14:16 | INFO | Connecting to databases... +2026-06-19 10:14:18 | INFO | +2026-06-19 10:14:22 | INFO | +2026-06-19 10:14:23 | INFO | Database connections established +2026-06-19 10:14:23 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 10:14:24 | INFO | Found 859 MIDs +2026-06-19 10:14:25 | INFO | ================================================================================ +2026-06-19 10:14:25 | INFO | Processing Table: Attendance | Table type is -: FACT | Based on -run_date and operation is used -DELETE+INSERT +2026-06-19 10:14:25 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-19 10:14:25 | ERROR | Failed processing table Attendance +Traceback (most recent call last): + File "D:\data_move\main2.py", line 181, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 80, in get_dataframe + return fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\data_move\src\fact.py", line 550, in fetch_Attendance + sql = sql_template.format( + star_date=start_date.strftime("%Y-%m-%d"), + run_date=run_date.strftime("%Y-%m-%d") + ) +KeyError: 'start_date' +2026-06-19 10:14:55 | INFO | ================================================================================ +2026-06-19 10:14:55 | INFO | Hello from data-move Python data pipeline! +2026-06-19 10:14:55 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 10:14:55 | INFO | Connecting to databases... +2026-06-19 10:14:56 | INFO | +2026-06-19 10:14:57 | INFO | +2026-06-19 10:14:58 | INFO | Database connections established +2026-06-19 10:14:58 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 10:14:58 | INFO | Found 859 MIDs +2026-06-19 10:14:58 | INFO | ================================================================================ +2026-06-19 10:14:58 | INFO | Processing Table: Attendance | Table type is -: FACT | Based on -run_date and operation is used -DELETE+INSERT +2026-06-19 10:14:58 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-19 10:14:58 | ERROR | Failed processing table Attendance +Traceback (most recent call last): + File "D:\data_move\main2.py", line 181, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 80, in get_dataframe + return fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\data_move\src\fact.py", line 550, in fetch_Attendance + sql = sql_template.format( + star_date=start_date.strftime("%Y-%m-%d"), + run_date=run_date.strftime("%Y-%m-%d") + ) +KeyError: 'start_date' +2026-06-19 10:15:38 | INFO | ================================================================================ +2026-06-19 10:15:38 | INFO | Hello from data-move Python data pipeline! +2026-06-19 10:15:38 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 10:15:38 | INFO | Connecting to databases... +2026-06-19 10:15:39 | INFO | +2026-06-19 10:15:41 | INFO | +2026-06-19 10:15:41 | INFO | Database connections established +2026-06-19 10:15:41 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 10:15:42 | INFO | Found 859 MIDs +2026-06-19 10:15:42 | INFO | ================================================================================ +2026-06-19 10:15:42 | INFO | Processing Table: Attendance | Table type is -: FACT | Based on -run_date and operation is used -DELETE+INSERT +2026-06-19 10:15:42 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-19 10:15:42 | INFO | Fetching Attendance data from 2026-06-03 to 2026-06-18 +2026-06-19 10:15:51 | INFO | Fetched 121,124 attendance rows for 540 employees +2026-06-19 10:15:51 | INFO | Fetched total row -: 121124 from sql server for table-:Attendance ...........!!! +2026-06-19 10:15:51 | INFO | Fetched 121124 rows +2026-06-19 10:15:51 | INFO | Creating table Attendance +2026-06-19 10:15:51 | INFO | Table ready: Attendance +2026-06-19 10:15:51 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 10:15:53 | INFO | Attendance: inserted 121,124 rows into ClickHouse +2026-06-19 10:15:53 | INFO | Attendance loaded successfully (121124 rows) +2026-06-19 10:15:53 | INFO | ================================================================================ +2026-06-19 10:15:53 | INFO | Pipeline Completed Successfully +2026-06-19 10:15:53 | INFO | ================================================================================ +2026-06-19 10:15:59 | INFO | ================================================================================ +2026-06-19 10:15:59 | INFO | Hello from data-move Python data pipeline! +2026-06-19 10:15:59 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 10:15:59 | INFO | Connecting to databases... +2026-06-19 10:16:00 | INFO | +2026-06-19 10:16:01 | INFO | +2026-06-19 10:16:02 | INFO | Database connections established +2026-06-19 10:16:02 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 10:16:02 | INFO | Found 859 MIDs +2026-06-19 10:16:02 | INFO | ================================================================================ +2026-06-19 10:16:02 | INFO | Processing Table: Attendance | Table type is -: FACT | Based on -run_date and operation is used -DELETE+INSERT +2026-06-19 10:16:02 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-19 10:16:02 | INFO | Fetching Attendance data from 2026-06-03 to 2026-06-18 +2026-06-19 10:16:17 | INFO | Fetched 121,124 attendance rows for 540 employees +2026-06-19 10:16:17 | INFO | Fetched total row -: 121124 from sql server for table-:Attendance ...........!!! +2026-06-19 10:16:17 | INFO | Fetched 121124 rows +2026-06-19 10:16:18 | INFO | Truncated table Attendance +2026-06-19 10:16:18 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 10:16:19 | INFO | Attendance: inserted 121,124 rows into ClickHouse +2026-06-19 10:16:19 | INFO | Attendance loaded successfully (121124 rows) +2026-06-19 10:16:19 | INFO | ================================================================================ +2026-06-19 10:16:19 | INFO | Pipeline Completed Successfully +2026-06-19 10:16:19 | INFO | ================================================================================ +2026-06-19 11:17:09 | INFO | ================================================================================ +2026-06-19 11:17:09 | INFO | Hello from data-move Python data pipeline! +2026-06-19 11:17:09 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 11:17:09 | INFO | Connecting to databases... +2026-06-19 11:17:11 | INFO | +2026-06-19 11:17:12 | INFO | +2026-06-19 11:17:13 | INFO | Database connections established +2026-06-19 11:17:13 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 11:17:14 | INFO | Found 859 MIDs +2026-06-19 11:17:14 | INFO | ================================================================================ +2026-06-19 11:17:14 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-19 11:17:14 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-19 11:17:14 | INFO | Start Fetching data for these 859 MIDs +2026-06-19 11:17:14 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 +2026-06-19 11:17:14 | INFO | Fetching data for 859 MIDs +2026-06-19 11:17:16 | INFO | Fetched 21,439 rows from SQL Server +2026-06-19 11:17:16 | INFO | Fetched total row -: 21439 from sql server for table-:Journey_Plan ...........!!! +2026-06-19 11:17:16 | INFO | Fetched 21439 rows +2026-06-19 11:17:16 | INFO | Creating table Journey_Plan +2026-06-19 11:17:16 | INFO | Table ready: Journey_Plan +2026-06-19 11:17:16 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 11:17:17 | INFO | Journey_Plan: inserted 21,439 rows into ClickHouse +2026-06-19 11:17:17 | INFO | Journey_Plan loaded successfully (21439 rows) +2026-06-19 11:17:17 | INFO | ================================================================================ +2026-06-19 11:17:17 | INFO | Pipeline Completed Successfully +2026-06-19 11:17:17 | INFO | ================================================================================ +2026-06-19 11:17:27 | INFO | ================================================================================ +2026-06-19 11:17:27 | INFO | Hello from data-move Python data pipeline! +2026-06-19 11:17:27 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 11:17:27 | INFO | Connecting to databases... +2026-06-19 11:17:28 | INFO | +2026-06-19 11:17:30 | INFO | +2026-06-19 11:17:30 | INFO | Database connections established +2026-06-19 11:17:30 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 11:17:31 | INFO | Found 859 MIDs +2026-06-19 11:17:31 | INFO | ================================================================================ +2026-06-19 11:17:31 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-19 11:17:31 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-19 11:17:31 | INFO | Start Fetching data for these 859 MIDs +2026-06-19 11:17:31 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 +2026-06-19 11:17:31 | INFO | Fetching data for 859 MIDs +2026-06-19 11:17:33 | INFO | Fetched 21,439 rows from SQL Server +2026-06-19 11:17:33 | INFO | Fetched total row -: 21439 from sql server for table-:Journey_Plan ...........!!! +2026-06-19 11:17:33 | INFO | Fetched 21439 rows +2026-06-19 11:17:33 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-19 11:17:33 | ERROR | Failed processing table Journey_Plan +Traceback (most recent call last): + File "D:\data_move\main2.py", line 243, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 91, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + ...<4 lines>... + """, + ^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 44, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 43, server response: Code: 43. DB::Exception: Illegal type String of argument of function toMonth. Should be Date, Date32, DateTime or DateTime64: While processing isZeroOrNull((toMonth(visit_date) = 6) AND (toYear(visit_date) = 2026)). (ILLEGAL_TYPE_OF_ARGUMENT) (for url http://172.188.12.194:8123) +2026-06-19 11:21:40 | INFO | ================================================================================ +2026-06-19 11:21:40 | INFO | Hello from data-move Python data pipeline! +2026-06-19 11:21:40 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 11:21:40 | INFO | Connecting to databases... +2026-06-19 11:21:41 | INFO | +2026-06-19 11:21:43 | INFO | +2026-06-19 11:21:43 | INFO | Database connections established +2026-06-19 11:21:43 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 11:21:44 | INFO | Found 859 MIDs +2026-06-19 11:21:44 | INFO | ================================================================================ +2026-06-19 11:21:44 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-19 11:21:44 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-19 11:21:44 | INFO | Start Fetching data for these 859 MIDs +2026-06-19 11:21:44 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 +2026-06-19 11:21:44 | INFO | Fetching data for 859 MIDs +2026-06-19 11:21:46 | INFO | Fetched 21,440 rows from SQL Server +2026-06-19 11:21:46 | INFO | Fetched total row -: 21440 from sql server for table-:Journey_Plan ...........!!! +2026-06-19 11:21:46 | INFO | Fetched 21440 rows +2026-06-19 11:21:46 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-19 11:21:46 | ERROR | Failed processing table Journey_Plan +Traceback (most recent call last): + File "D:\data_move\main2.py", line 243, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 91, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + ...<5 lines>... + """, + ^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 44, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'Visit_Date' while processing: 'isZeroOrNull((project_id = 40148) AND (toMonth(Visit_Date) = toMonth(toDate('2026-06-18'))) AND (toYear(Visit_Date) = toYear(toDate('2026-06-18')))), _block_offset, _part_granule_offset, _disk_name, _sample_factor, _partition_id, _part_index, _part, UpdateDate, _block_number, CreateDate, CreateBy, _part_offset, _part_starting_offset, employee_id, _part_data_version, process_id, _database, _distance, UpdateBy, _table, _part_uuid, visit_date, store_id, project_id', required columns: 'project_id' 'Visit_Date' '_block_offset' 'UpdateBy' '_table' '_part_uuid' '_part_granule_offset' '_part_index' '_part' '_sample_factor' '_partition_id' '_disk_name' 'UpdateDate' '_block_number' 'CreateDate' 'CreateBy' '_part_data_version' '_part_starting_offset' 'employee_id' '_database' '_distance' '_part_offset' 'visit_date' 'process_id' 'store_id', maybe you meant: 'project_id', 'visit_date', 'UpdateBy', 'UpdateDate', 'CreateDate', 'CreateBy', 'employee_id', 'process_id' or 'store_id'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) +2026-06-19 11:22:52 | INFO | ================================================================================ +2026-06-19 11:22:52 | INFO | Hello from data-move Python data pipeline! +2026-06-19 11:22:52 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 11:22:52 | INFO | Connecting to databases... +2026-06-19 11:22:53 | INFO | +2026-06-19 11:22:55 | INFO | +2026-06-19 11:22:56 | INFO | Database connections established +2026-06-19 11:22:56 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 11:22:56 | INFO | Found 859 MIDs +2026-06-19 11:22:57 | INFO | ================================================================================ +2026-06-19 11:22:57 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-19 11:22:57 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-19 11:22:57 | INFO | Start Fetching data for these 859 MIDs +2026-06-19 11:22:57 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 +2026-06-19 11:22:57 | INFO | Fetching data for 859 MIDs +2026-06-19 11:22:58 | INFO | Fetched 21,441 rows from SQL Server +2026-06-19 11:22:58 | INFO | Fetched total row -: 21441 from sql server for table-:Journey_Plan ...........!!! +2026-06-19 11:22:58 | INFO | Fetched 21441 rows +2026-06-19 11:22:58 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-19 11:22:58 | ERROR | Failed processing table Journey_Plan +Traceback (most recent call last): + File "D:\data_move\main2.py", line 243, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 91, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + ...<5 lines>... + """, + ^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 44, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 43, server response: Code: 43. DB::Exception: Illegal type String of argument of function toMonth. Should be Date, Date32, DateTime or DateTime64: While processing isZeroOrNull((project_id = 40148) AND (toMonth(visit_date) = toMonth(toDate('2026-06-18'))) AND (toYear(visit_date) = toYear(toDate('2026-06-18')))). (ILLEGAL_TYPE_OF_ARGUMENT) (for url http://172.188.12.194:8123) +2026-06-19 11:25:44 | INFO | ================================================================================ +2026-06-19 11:25:44 | INFO | Hello from data-move Python data pipeline! +2026-06-19 11:25:44 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 11:25:44 | INFO | Connecting to databases... +2026-06-19 11:25:46 | INFO | +2026-06-19 11:25:47 | INFO | +2026-06-19 11:25:48 | INFO | Database connections established +2026-06-19 11:25:48 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 11:25:48 | INFO | Found 859 MIDs +2026-06-19 11:25:49 | INFO | ================================================================================ +2026-06-19 11:25:49 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-19 11:25:49 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-19 11:25:49 | INFO | Start Fetching data for these 859 MIDs +2026-06-19 11:25:49 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 +2026-06-19 11:25:49 | INFO | Fetching data for 859 MIDs +2026-06-19 11:25:50 | INFO | Fetched 21,441 rows from SQL Server +2026-06-19 11:25:50 | INFO | Fetched total row -: 21441 from sql server for table-:Journey_Plan ...........!!! +2026-06-19 11:25:50 | INFO | Fetched 21441 rows +2026-06-19 11:25:50 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-19 11:25:51 | ERROR | Failed processing table Journey_Plan +Traceback (most recent call last): + File "D:\data_move\main2.py", line 243, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 91, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + ...<5 lines>... + """, + ^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 44, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 43, server response: Code: 43. DB::Exception: Illegal type String of argument of function toMonth. Should be Date, Date32, DateTime or DateTime64: While processing isZeroOrNull((project_id = 40148) AND (toMonth(visit_date) = toMonth(toDate('2026-06-18'))) AND (toYear(visit_date) = toYear(toDate('2026-06-18')))). (ILLEGAL_TYPE_OF_ARGUMENT) (for url http://172.188.12.194:8123) +2026-06-19 11:27:56 | INFO | ================================================================================ +2026-06-19 11:27:56 | INFO | Hello from data-move Python data pipeline! +2026-06-19 11:27:56 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 11:27:56 | INFO | Connecting to databases... +2026-06-19 11:27:57 | INFO | +2026-06-19 11:27:59 | INFO | +2026-06-19 11:28:00 | INFO | Database connections established +2026-06-19 11:28:00 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 11:28:00 | INFO | Found 859 MIDs +2026-06-19 11:28:00 | INFO | ================================================================================ +2026-06-19 11:28:00 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-19 11:28:00 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-19 11:28:00 | INFO | Start Fetching data for these 859 MIDs +2026-06-19 11:28:00 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 +2026-06-19 11:28:00 | INFO | Fetching data for 859 MIDs +2026-06-19 11:28:02 | INFO | Fetched 21,441 rows from SQL Server +2026-06-19 11:28:02 | INFO | Fetched total row -: 21441 from sql server for table-:Journey_Plan ...........!!! +2026-06-19 11:28:02 | INFO | Fetched 21441 rows +2026-06-19 11:28:02 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-19 11:28:02 | ERROR | Failed processing table Journey_Plan +Traceback (most recent call last): + File "D:\data_move\main2.py", line 243, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 91, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + ...<5 lines>... + """, + ^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 44, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 43, server response: Code: 43. DB::Exception: Illegal type String of argument of function toMonth. Should be Date, Date32, DateTime or DateTime64: While processing isZeroOrNull((project_id = 40148) AND (toMonth(visit_date) = toMonth(toDate('2026-06-18'))) AND (toYear(visit_date) = toYear(toDate('2026-06-18')))). (ILLEGAL_TYPE_OF_ARGUMENT) (for url http://172.188.12.194:8123) +2026-06-19 11:28:09 | INFO | ================================================================================ +2026-06-19 11:28:09 | INFO | Hello from data-move Python data pipeline! +2026-06-19 11:28:09 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 11:28:09 | INFO | Connecting to databases... +2026-06-19 11:28:10 | INFO | +2026-06-19 11:28:11 | INFO | +2026-06-19 11:28:13 | INFO | Database connections established +2026-06-19 11:28:13 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 11:28:13 | INFO | Found 859 MIDs +2026-06-19 11:28:13 | INFO | ================================================================================ +2026-06-19 11:28:13 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-19 11:28:13 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-19 11:28:13 | INFO | Start Fetching data for these 859 MIDs +2026-06-19 11:28:13 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 +2026-06-19 11:28:13 | INFO | Fetching data for 859 MIDs +2026-06-19 11:28:14 | INFO | Fetched 21,441 rows from SQL Server +2026-06-19 11:28:14 | INFO | Fetched total row -: 21441 from sql server for table-:Journey_Plan ...........!!! +2026-06-19 11:28:14 | INFO | Fetched 21441 rows +2026-06-19 11:28:15 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-19 11:28:15 | ERROR | Failed processing table Journey_Plan +Traceback (most recent call last): + File "D:\data_move\main2.py", line 243, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 91, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + ...<5 lines>... + """, + ^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 44, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 43, server response: Code: 43. DB::Exception: Illegal type String of argument of function toMonth. Should be Date, Date32, DateTime or DateTime64: While processing isZeroOrNull((project_id = 40148) AND (toMonth(visit_date) = toMonth(toDate('2026-06-18'))) AND (toYear(visit_date) = toYear(toDate('2026-06-18')))). (ILLEGAL_TYPE_OF_ARGUMENT) (for url http://172.188.12.194:8123) +2026-06-19 11:33:41 | INFO | ================================================================================ +2026-06-19 11:33:41 | INFO | Hello from data-move Python data pipeline! +2026-06-19 11:33:41 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 11:33:41 | INFO | Connecting to databases... +2026-06-19 11:33:43 | INFO | +2026-06-19 11:33:44 | INFO | +2026-06-19 11:33:44 | INFO | Database connections established +2026-06-19 11:33:44 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 11:33:45 | INFO | Found 859 MIDs +2026-06-19 11:33:46 | INFO | ================================================================================ +2026-06-19 11:33:46 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-19 11:33:46 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-19 11:33:46 | INFO | Start Fetching data for these 859 MIDs +2026-06-19 11:33:46 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 +2026-06-19 11:33:46 | INFO | Fetching data for 859 MIDs +2026-06-19 11:33:48 | INFO | Fetched 21,442 rows from SQL Server +2026-06-19 11:33:48 | INFO | Fetched total row -: 21442 from sql server for table-:Journey_Plan ...........!!! +2026-06-19 11:33:48 | INFO | Fetched 21442 rows +2026-06-19 11:33:48 | INFO | Creating table Journey_Plan +2026-06-19 11:33:48 | INFO | Table ready: Journey_Plan +2026-06-19 11:33:48 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 11:33:49 | INFO | Journey_Plan: inserted 21,442 rows into ClickHouse +2026-06-19 11:33:49 | INFO | Journey_Plan loaded successfully (21442 rows) +2026-06-19 11:33:49 | INFO | ================================================================================ +2026-06-19 11:33:49 | INFO | Pipeline Completed Successfully +2026-06-19 11:33:49 | INFO | ================================================================================ +2026-06-19 11:33:53 | INFO | ================================================================================ +2026-06-19 11:33:53 | INFO | Hello from data-move Python data pipeline! +2026-06-19 11:33:53 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 11:33:53 | INFO | Connecting to databases... +2026-06-19 11:33:55 | INFO | +2026-06-19 11:33:56 | INFO | +2026-06-19 11:33:56 | INFO | Database connections established +2026-06-19 11:33:56 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 11:33:57 | INFO | Found 859 MIDs +2026-06-19 11:33:58 | INFO | ================================================================================ +2026-06-19 11:33:58 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-19 11:33:58 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-19 11:33:58 | INFO | Start Fetching data for these 859 MIDs +2026-06-19 11:33:58 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 +2026-06-19 11:33:58 | INFO | Fetching data for 859 MIDs +2026-06-19 11:33:59 | INFO | Fetched 21,442 rows from SQL Server +2026-06-19 11:33:59 | INFO | Fetched total row -: 21442 from sql server for table-:Journey_Plan ...........!!! +2026-06-19 11:33:59 | INFO | Fetched 21442 rows +2026-06-19 11:33:59 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-19 11:33:59 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 11:34:00 | INFO | Journey_Plan: inserted 21,442 rows into ClickHouse +2026-06-19 11:34:00 | INFO | Journey_Plan loaded successfully (21442 rows) +2026-06-19 11:34:00 | INFO | ================================================================================ +2026-06-19 11:34:00 | INFO | Pipeline Completed Successfully +2026-06-19 11:34:00 | INFO | ================================================================================ +2026-06-19 12:02:14 | INFO | ================================================================================ +2026-06-19 12:02:14 | INFO | Hello from data-move Python data pipeline! +2026-06-19 12:02:14 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 12:02:14 | INFO | Connecting to databases... +2026-06-19 12:02:15 | INFO | +2026-06-19 12:02:17 | INFO | +2026-06-19 12:02:17 | INFO | Database connections established +2026-06-19 12:02:17 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 12:02:18 | INFO | Found 859 MIDs +2026-06-19 12:02:19 | INFO | ================================================================================ +2026-06-19 12:02:19 | INFO | Processing Table: PaidVisibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-19 12:02:19 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-19 12:02:19 | INFO | Start Fetching data for these 859 MIDs +2026-06-19 12:02:19 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 +2026-06-19 12:02:19 | INFO | Fetching data for 859 MIDs +2026-06-19 12:02:23 | INFO | Fetched 937 rows from SQL Server +2026-06-19 12:02:23 | INFO | Fetched total row -: 937 from sql server for table-:PaidVisibility ...........!!! +2026-06-19 12:02:23 | INFO | Fetched 937 rows +2026-06-19 12:02:23 | INFO | Creating table PaidVisibility +2026-06-19 12:02:23 | INFO | Table ready: PaidVisibility +2026-06-19 12:02:23 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 12:02:24 | INFO | PaidVisibility: inserted 937 rows into ClickHouse +2026-06-19 12:02:24 | INFO | PaidVisibility loaded successfully (937 rows) +2026-06-19 12:02:24 | INFO | ================================================================================ +2026-06-19 12:02:24 | INFO | Pipeline Completed Successfully +2026-06-19 12:02:24 | INFO | ================================================================================ +2026-06-19 12:03:50 | INFO | ================================================================================ +2026-06-19 12:03:50 | INFO | Hello from data-move Python data pipeline! +2026-06-19 12:03:50 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 12:03:50 | INFO | Connecting to databases... +2026-06-19 12:03:58 | INFO | +2026-06-19 12:04:00 | INFO | +2026-06-19 12:04:01 | INFO | Database connections established +2026-06-19 12:04:01 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 12:04:01 | INFO | Found 859 MIDs +2026-06-19 12:04:02 | INFO | ================================================================================ +2026-06-19 12:04:02 | INFO | Processing Table: PaidVisibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-19 12:04:02 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-19 12:04:02 | INFO | Start Fetching data for these 859 MIDs +2026-06-19 12:04:02 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 +2026-06-19 12:04:02 | INFO | Fetching data for 859 MIDs +2026-06-19 12:04:05 | INFO | Fetched 937 rows from SQL Server +2026-06-19 12:04:05 | INFO | Fetched total row -: 937 from sql server for table-:PaidVisibility ...........!!! +2026-06-19 12:04:05 | INFO | Fetched 937 rows +2026-06-19 12:04:05 | INFO | Creating table PaidVisibility +2026-06-19 12:04:05 | INFO | Table ready: PaidVisibility +2026-06-19 12:04:05 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 12:04:05 | INFO | PaidVisibility: inserted 937 rows into ClickHouse +2026-06-19 12:04:05 | INFO | PaidVisibility loaded successfully (937 rows) +2026-06-19 12:04:05 | INFO | ================================================================================ +2026-06-19 12:04:05 | INFO | Pipeline Completed Successfully +2026-06-19 12:04:05 | INFO | ================================================================================ +2026-06-19 12:04:52 | INFO | ================================================================================ +2026-06-19 12:04:52 | INFO | Hello from data-move Python data pipeline! +2026-06-19 12:04:52 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 12:04:52 | INFO | Connecting to databases... +2026-06-19 12:04:53 | INFO | +2026-06-19 12:04:54 | INFO | +2026-06-19 12:04:55 | INFO | Database connections established +2026-06-19 12:04:55 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 12:04:56 | INFO | Found 859 MIDs +2026-06-19 12:04:57 | INFO | ================================================================================ +2026-06-19 12:04:57 | INFO | Processing Table: PaidVisibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-19 12:04:57 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-19 12:04:57 | INFO | Start Fetching data for these 859 MIDs +2026-06-19 12:04:57 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 +2026-06-19 12:04:57 | INFO | Fetching data for 859 MIDs +2026-06-19 12:04:59 | INFO | Fetched 937 rows from SQL Server +2026-06-19 12:04:59 | INFO | Fetched total row -: 937 from sql server for table-:PaidVisibility ...........!!! +2026-06-19 12:04:59 | INFO | Fetched 937 rows +2026-06-19 12:04:59 | INFO | _ _ _ _ Deleting Data from ClickHouse for PaidVisibility _ _ _ _ +2026-06-19 12:04:59 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 12:04:59 | INFO | PaidVisibility: inserted 937 rows into ClickHouse +2026-06-19 12:04:59 | INFO | PaidVisibility loaded successfully (937 rows) +2026-06-19 12:04:59 | INFO | ================================================================================ +2026-06-19 12:04:59 | INFO | Pipeline Completed Successfully +2026-06-19 12:04:59 | INFO | ================================================================================ +2026-06-19 12:21:12 | INFO | ================================================================================ +2026-06-19 12:21:12 | INFO | Hello from data-move Python data pipeline! +2026-06-19 12:21:12 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 12:21:12 | INFO | Connecting to databases... +2026-06-19 12:21:14 | INFO | +2026-06-19 12:21:15 | INFO | +2026-06-19 12:21:16 | INFO | Database connections established +2026-06-19 12:21:16 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 12:21:16 | INFO | Found 859 MIDs +2026-06-19 12:21:17 | INFO | ================================================================================ +2026-06-19 12:21:17 | INFO | Processing Table: Web_Logins | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-19 12:21:17 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-19 12:21:17 | ERROR | Failed processing table Web_Logins +Traceback (most recent call last): + File "D:\data_move\main2.py", line 181, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 77, in get_dataframe + fn = globals()[fn_name] + ~~~~~~~~~^^^^^^^^^ +KeyError: 'fetch_Web_Logins' +2026-06-19 12:22:20 | INFO | ================================================================================ +2026-06-19 12:22:20 | INFO | Hello from data-move Python data pipeline! +2026-06-19 12:22:20 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 12:22:20 | INFO | Connecting to databases... +2026-06-19 12:22:22 | INFO | +2026-06-19 12:22:24 | INFO | +2026-06-19 12:22:25 | INFO | Database connections established +2026-06-19 12:22:25 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 12:22:25 | INFO | Found 859 MIDs +2026-06-19 12:22:26 | INFO | ================================================================================ +2026-06-19 12:22:26 | INFO | Processing Table: Web_Logins | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-19 12:22:26 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-19 12:22:26 | INFO | Start Fetching data for these 859 MIDs +2026-06-19 12:22:26 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 +2026-06-19 12:22:26 | INFO | Fetching data for 859 MIDs +2026-06-19 12:22:27 | INFO | Fetched 136 rows from SQL Server +2026-06-19 12:22:27 | INFO | Fetched total row -: 136 from sql server for table-:Web_Logins ...........!!! +2026-06-19 12:22:27 | INFO | Fetched 136 rows +2026-06-19 12:22:27 | INFO | Creating table Web_Logins +2026-06-19 12:22:27 | INFO | Table ready: Web_Logins +2026-06-19 12:22:27 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 12:22:27 | INFO | Web_Logins: inserted 136 rows into ClickHouse +2026-06-19 12:22:27 | INFO | Web_Logins loaded successfully (136 rows) +2026-06-19 12:22:27 | INFO | ================================================================================ +2026-06-19 12:22:27 | INFO | Pipeline Completed Successfully +2026-06-19 12:22:27 | INFO | ================================================================================ +2026-06-19 12:26:29 | INFO | ================================================================================ +2026-06-19 12:26:29 | INFO | Hello from data-move Python data pipeline! +2026-06-19 12:26:29 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 12:26:29 | INFO | Connecting to databases... +2026-06-19 12:26:31 | INFO | +2026-06-19 12:26:33 | INFO | +2026-06-19 12:26:34 | INFO | Database connections established +2026-06-19 12:26:34 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 12:26:34 | INFO | Found 859 MIDs +2026-06-19 12:26:35 | INFO | ================================================================================ +2026-06-19 12:26:35 | INFO | Processing Table: Web_Logins | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-19 12:26:35 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-19 12:26:35 | INFO | Start Fetching data for these 859 MIDs +2026-06-19 12:26:35 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 +2026-06-19 12:26:35 | INFO | Fetching data for 859 MIDs +2026-06-19 12:26:36 | INFO | Fetched 136 rows from SQL Server +2026-06-19 12:26:36 | INFO | Fetched total row -: 136 from sql server for table-:Web_Logins ...........!!! +2026-06-19 12:26:36 | INFO | Fetched 136 rows +2026-06-19 12:26:36 | INFO | Creating table Web_Logins +2026-06-19 12:26:36 | INFO | Table ready: Web_Logins +2026-06-19 12:26:36 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 12:26:36 | INFO | Web_Logins: inserted 136 rows into ClickHouse +2026-06-19 12:26:36 | INFO | Web_Logins loaded successfully (136 rows) +2026-06-19 12:26:36 | INFO | ================================================================================ +2026-06-19 12:26:36 | INFO | Pipeline Completed Successfully +2026-06-19 12:26:36 | INFO | ================================================================================ +2026-06-19 12:26:54 | INFO | ================================================================================ +2026-06-19 12:26:54 | INFO | Hello from data-move Python data pipeline! +2026-06-19 12:26:54 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 12:26:54 | INFO | Connecting to databases... +2026-06-19 12:26:55 | INFO | +2026-06-19 12:26:56 | INFO | +2026-06-19 12:26:57 | INFO | Database connections established +2026-06-19 12:26:57 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 12:26:57 | INFO | Found 859 MIDs +2026-06-19 12:26:57 | INFO | ================================================================================ +2026-06-19 12:26:57 | INFO | Processing Table: Web_Logins | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-19 12:26:57 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-19 12:26:57 | INFO | Start Fetching data for these 859 MIDs +2026-06-19 12:26:57 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 +2026-06-19 12:26:57 | INFO | Fetching data for 859 MIDs +2026-06-19 12:26:58 | INFO | Fetched 136 rows from SQL Server +2026-06-19 12:26:58 | INFO | Fetched total row -: 136 from sql server for table-:Web_Logins ...........!!! +2026-06-19 12:26:58 | INFO | Fetched 136 rows +2026-06-19 12:26:58 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ +2026-06-19 12:26:58 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 12:26:58 | INFO | Web_Logins: inserted 136 rows into ClickHouse +2026-06-19 12:26:58 | INFO | Web_Logins loaded successfully (136 rows) +2026-06-19 12:26:58 | INFO | ================================================================================ +2026-06-19 12:26:58 | INFO | Pipeline Completed Successfully +2026-06-19 12:26:58 | INFO | ================================================================================ +2026-06-19 12:27:05 | INFO | ================================================================================ +2026-06-19 12:27:05 | INFO | Hello from data-move Python data pipeline! +2026-06-19 12:27:05 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 12:27:05 | INFO | Connecting to databases... +2026-06-19 12:27:07 | INFO | +2026-06-19 12:27:09 | INFO | +2026-06-19 12:27:10 | INFO | Database connections established +2026-06-19 12:27:10 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 12:27:10 | INFO | Found 859 MIDs +2026-06-19 12:27:10 | INFO | ================================================================================ +2026-06-19 12:27:10 | INFO | Processing Table: Web_Logins | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-19 12:27:10 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-19 12:27:10 | INFO | Start Fetching data for these 859 MIDs +2026-06-19 12:27:10 | INFO | Start Fetching data for these 859 MIDs or based on this date 2026-06-18 +2026-06-19 12:27:10 | INFO | Fetching data for 859 MIDs +2026-06-19 12:27:11 | INFO | Fetched 136 rows from SQL Server +2026-06-19 12:27:11 | INFO | Fetched total row -: 136 from sql server for table-:Web_Logins ...........!!! +2026-06-19 12:27:11 | INFO | Fetched 136 rows +2026-06-19 12:27:11 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ +2026-06-19 12:27:11 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 12:27:11 | INFO | Web_Logins: inserted 136 rows into ClickHouse +2026-06-19 12:27:11 | INFO | Web_Logins loaded successfully (136 rows) +2026-06-19 12:27:11 | INFO | ================================================================================ +2026-06-19 12:27:11 | INFO | Pipeline Completed Successfully +2026-06-19 12:27:11 | INFO | ================================================================================ +2026-06-19 12:53:21 | INFO | ================================================================================ +2026-06-19 12:53:21 | INFO | Hello from data-move Python data pipeline! +2026-06-19 12:53:21 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 12:53:21 | INFO | Connecting to databases... +2026-06-19 12:53:22 | INFO | +2026-06-19 12:53:24 | INFO | +2026-06-19 12:53:24 | INFO | Database connections established +2026-06-19 12:53:24 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 12:53:25 | INFO | Found 859 MIDs +2026-06-19 12:53:25 | INFO | ================================================================================ +2026-06-19 12:53:25 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-19 12:53:25 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 12:53:25 | ERROR | Failed processing table Store_Master +Traceback (most recent call last): + File "D:\data_move\main2.py", line 181, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 80, in get_dataframe + return fn(sql_engine, table_name , table_type, mids, run_date) +TypeError: fetch_Store_Master() takes 1 positional argument but 5 were given +2026-06-19 12:57:22 | INFO | ================================================================================ +2026-06-19 12:57:22 | INFO | Hello from data-move Python data pipeline! +2026-06-19 12:57:22 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 12:57:22 | INFO | Connecting to databases... +2026-06-19 12:57:24 | INFO | +2026-06-19 12:57:27 | INFO | +2026-06-19 12:57:27 | INFO | Database connections established +2026-06-19 12:57:27 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 12:57:28 | INFO | Found 859 MIDs +2026-06-19 12:57:29 | INFO | ================================================================================ +2026-06-19 12:57:29 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-19 12:57:29 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 12:57:29 | ERROR | Failed processing table Store_Master +Traceback (most recent call last): + File "D:\data_move\main2.py", line 181, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 77, in get_dataframe + fn = globals()[fn_name] + ~~~~~~~~~^^^^^^^^^ +KeyError: 'fetch_Store_Master' +2026-06-19 13:00:35 | INFO | ================================================================================ +2026-06-19 13:00:35 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:00:35 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:00:35 | INFO | Connecting to databases... +2026-06-19 13:00:36 | INFO | +2026-06-19 13:00:38 | INFO | +2026-06-19 13:00:39 | INFO | Database connections established +2026-06-19 13:00:39 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:00:39 | INFO | Found 859 MIDs +2026-06-19 13:00:41 | INFO | ================================================================================ +2026-06-19 13:00:41 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-19 13:00:41 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:00:41 | ERROR | Failed processing table Store_Master +Traceback (most recent call last): + File "D:\data_move\main2.py", line 181, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 77, in get_dataframe + fn = globals()[fn_name] + ~~~~~~~~~^^^^^^^^^ +KeyError: 'fetch_Store_Master' +2026-06-19 13:01:45 | INFO | ================================================================================ +2026-06-19 13:01:45 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:01:45 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:01:45 | INFO | Connecting to databases... +2026-06-19 13:01:46 | INFO | +2026-06-19 13:01:48 | INFO | +2026-06-19 13:01:49 | INFO | Database connections established +2026-06-19 13:01:49 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:01:50 | INFO | Found 859 MIDs +2026-06-19 13:01:51 | INFO | ================================================================================ +2026-06-19 13:01:51 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-19 13:01:51 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:01:51 | ERROR | Failed processing table Store_Master +Traceback (most recent call last): + File "D:\data_move\main2.py", line 181, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 77, in get_dataframe + fn = globals()[fn_name] + ~~~~~~~~~^^^^^^^^^ +KeyError: 'fetch_Store_Master' +2026-06-19 13:03:00 | INFO | ================================================================================ +2026-06-19 13:03:00 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:03:00 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:03:00 | INFO | Connecting to databases... +2026-06-19 13:03:01 | INFO | +2026-06-19 13:03:03 | INFO | +2026-06-19 13:03:03 | INFO | Database connections established +2026-06-19 13:03:03 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:03:04 | INFO | Found 859 MIDs +2026-06-19 13:03:05 | INFO | ================================================================================ +2026-06-19 13:03:05 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 13:03:05 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:03:05 | ERROR | Failed processing table Store_Master +Traceback (most recent call last): + File "D:\data_move\main2.py", line 182, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 80, in get_dataframe + fn = globals()[fn_name] + ~~~~~~~~~^^^^^^^^^ +KeyError: 'fetch_Store_Master' +2026-06-19 13:03:18 | INFO | ================================================================================ +2026-06-19 13:03:18 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:03:18 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:03:18 | INFO | Connecting to databases... +2026-06-19 13:03:19 | INFO | +2026-06-19 13:03:21 | INFO | +2026-06-19 13:03:22 | INFO | Database connections established +2026-06-19 13:03:22 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:03:22 | INFO | Found 859 MIDs +2026-06-19 13:03:23 | INFO | ================================================================================ +2026-06-19 13:03:23 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 13:03:23 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:03:23 | ERROR | Failed processing table Store_Master +Traceback (most recent call last): + File "D:\data_move\main2.py", line 182, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 80, in get_dataframe + fn = globals()[fn_name] + ~~~~~~~~~^^^^^^^^^ +KeyError: 'fetch_Store_Master' +2026-06-19 13:07:20 | INFO | ================================================================================ +2026-06-19 13:07:20 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:07:20 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:07:20 | INFO | Connecting to databases... +2026-06-19 13:07:22 | INFO | +2026-06-19 13:07:23 | INFO | +2026-06-19 13:07:24 | INFO | Database connections established +2026-06-19 13:07:24 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:07:25 | INFO | Found 859 MIDs +2026-06-19 13:07:26 | INFO | ================================================================================ +2026-06-19 13:07:26 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 13:07:26 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:07:26 | ERROR | Failed processing table Store_Master +Traceback (most recent call last): + File "D:\data_move\main2.py", line 182, in main + df = get_dataframe( + sql_engine, + ...<4 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 80, in get_dataframe + fn = globals()[fn_name] + ~~~~~~~~~^^^^^^^^^ +KeyError: 'fetch_Store_Master' +2026-06-19 13:13:46 | INFO | ================================================================================ +2026-06-19 13:13:46 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:13:46 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:13:46 | INFO | Connecting to databases... +2026-06-19 13:13:47 | INFO | +2026-06-19 13:13:48 | INFO | +2026-06-19 13:13:49 | INFO | Database connections established +2026-06-19 13:13:49 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:13:49 | INFO | Found 859 MIDs +2026-06-19 13:13:50 | INFO | ================================================================================ +2026-06-19 13:13:50 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 13:13:50 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:13:50 | ERROR | Failed processing table Store_Master +Traceback (most recent call last): + File "D:\data_move\main2.py", line 184, in main + fn = globals()[fn_name] + ~~~~~~~~~^^^^^^^^^ +KeyError: 'fetch_Store_Master' +2026-06-19 13:17:37 | INFO | ================================================================================ +2026-06-19 13:17:37 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:17:37 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:17:37 | INFO | Connecting to databases... +2026-06-19 13:17:38 | INFO | +2026-06-19 13:17:39 | INFO | +2026-06-19 13:17:40 | INFO | Database connections established +2026-06-19 13:17:40 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:17:40 | INFO | Found 859 MIDs +2026-06-19 13:17:41 | INFO | ================================================================================ +2026-06-19 13:17:41 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 13:17:41 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:17:41 | ERROR | Failed processing table Store_Master +Traceback (most recent call last): + File "D:\data_move\main2.py", line 184, in main + fn = globals()[fn_name] + ~~~~~~~~~^^^^^^^^^ +KeyError: 'fetch_Store_Master' +2026-06-19 13:18:26 | INFO | ================================================================================ +2026-06-19 13:18:26 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:18:26 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:18:26 | INFO | Connecting to databases... +2026-06-19 13:18:27 | INFO | +2026-06-19 13:18:29 | INFO | +2026-06-19 13:18:29 | INFO | Database connections established +2026-06-19 13:18:29 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:18:30 | INFO | Found 859 MIDs +2026-06-19 13:18:30 | INFO | ================================================================================ +2026-06-19 13:18:30 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 13:18:30 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:18:30 | INFO | ================================================================================ +2026-06-19 13:18:30 | INFO | Pipeline Completed Successfully +2026-06-19 13:18:30 | INFO | ================================================================================ +2026-06-19 13:19:45 | INFO | ================================================================================ +2026-06-19 13:19:45 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:19:45 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:19:45 | INFO | Connecting to databases... +2026-06-19 13:19:46 | INFO | +2026-06-19 13:19:48 | INFO | +2026-06-19 13:19:48 | INFO | Database connections established +2026-06-19 13:19:48 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:19:49 | INFO | Found 859 MIDs +2026-06-19 13:19:49 | INFO | ================================================================================ +2026-06-19 13:19:49 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 13:19:49 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:19:49 | INFO | ================================================================================ +2026-06-19 13:19:49 | INFO | Pipeline Completed Successfully +2026-06-19 13:19:49 | INFO | ================================================================================ +2026-06-19 13:21:52 | INFO | ================================================================================ +2026-06-19 13:21:52 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:21:52 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:21:52 | INFO | Connecting to databases... +2026-06-19 13:21:54 | INFO | +2026-06-19 13:21:56 | INFO | +2026-06-19 13:21:56 | INFO | Database connections established +2026-06-19 13:21:56 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:21:57 | INFO | Found 859 MIDs +2026-06-19 13:21:57 | INFO | ================================================================================ +2026-06-19 13:21:57 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 13:21:57 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:21:57 | INFO | ================================================================================ +2026-06-19 13:21:57 | INFO | Pipeline Completed Successfully +2026-06-19 13:21:57 | INFO | ================================================================================ +2026-06-19 13:24:49 | INFO | ================================================================================ +2026-06-19 13:24:49 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:24:49 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:24:49 | INFO | Connecting to databases... +2026-06-19 13:24:50 | INFO | +2026-06-19 13:24:51 | INFO | +2026-06-19 13:24:52 | INFO | Database connections established +2026-06-19 13:24:52 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:24:52 | INFO | Found 859 MIDs +2026-06-19 13:24:53 | INFO | ================================================================================ +2026-06-19 13:24:53 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 13:24:53 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:24:53 | INFO | ================================================================================ +2026-06-19 13:24:53 | INFO | Pipeline Completed Successfully +2026-06-19 13:24:53 | INFO | ================================================================================ +2026-06-19 13:26:17 | INFO | ================================================================================ +2026-06-19 13:26:17 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:26:17 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:26:17 | INFO | Connecting to databases... +2026-06-19 13:26:18 | INFO | +2026-06-19 13:26:20 | INFO | +2026-06-19 13:26:21 | INFO | Database connections established +2026-06-19 13:26:21 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:26:21 | INFO | Found 859 MIDs +2026-06-19 13:26:21 | INFO | ================================================================================ +2026-06-19 13:26:21 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 13:26:21 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:26:21 | INFO | ================================================================================ +2026-06-19 13:26:21 | INFO | Pipeline Completed Successfully +2026-06-19 13:26:21 | INFO | ================================================================================ +2026-06-19 13:27:00 | INFO | ================================================================================ +2026-06-19 13:27:00 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:27:00 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:27:00 | INFO | Connecting to databases... +2026-06-19 13:27:01 | INFO | +2026-06-19 13:27:03 | INFO | +2026-06-19 13:27:03 | INFO | Database connections established +2026-06-19 13:27:03 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:27:04 | INFO | Found 859 MIDs +2026-06-19 13:27:04 | INFO | ================================================================================ +2026-06-19 13:27:04 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 13:27:04 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:27:04 | INFO | ================================================================================ +2026-06-19 13:27:04 | INFO | Pipeline Completed Successfully +2026-06-19 13:27:04 | INFO | ================================================================================ +2026-06-19 13:28:58 | INFO | ================================================================================ +2026-06-19 13:28:58 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:28:58 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:28:58 | INFO | Connecting to databases... +2026-06-19 13:28:59 | INFO | +2026-06-19 13:29:01 | INFO | +2026-06-19 13:29:01 | INFO | Database connections established +2026-06-19 13:29:01 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:29:02 | INFO | Found 859 MIDs +2026-06-19 13:29:02 | INFO | ================================================================================ +2026-06-19 13:29:02 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 13:29:02 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:29:02 | INFO | Fetching data from sql server for Master table...... +2026-06-19 13:29:02 | ERROR | Failed processing table Store_Master +Traceback (most recent call last): + File "D:\data_move\main2.py", line 188, in main + df = fetch_data(sql_engine ,table_name,table_type) + File "D:\data_move\src\dim.py", line 38, in fetch_data + with open(sql_file, "r", encoding="utf-8") as f: + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'src\\sql\\dimension\\Store_Master.sql' +2026-06-19 13:29:53 | INFO | ================================================================================ +2026-06-19 13:29:53 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:29:53 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:29:53 | INFO | Connecting to databases... +2026-06-19 13:29:55 | INFO | +2026-06-19 13:29:56 | INFO | +2026-06-19 13:29:57 | INFO | Database connections established +2026-06-19 13:29:57 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:29:57 | INFO | Found 859 MIDs +2026-06-19 13:29:57 | INFO | ================================================================================ +2026-06-19 13:29:57 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 13:29:57 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:29:57 | INFO | Fetching data from sql server for Master table...... +2026-06-19 13:29:57 | INFO | Fetching in progress .... +2026-06-19 13:30:00 | INFO | Fetched 5,991 rows from SQL Server +2026-06-19 13:30:00 | INFO | Fetched total row -: 5991 from sql server for table-:Store_Master ...........!!! +2026-06-19 13:30:00 | INFO | Fetched 5991 rows +2026-06-19 13:30:00 | INFO | Creating table Store_Master +2026-06-19 13:30:01 | INFO | Table ready: Store_Master +2026-06-19 13:30:01 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 13:30:01 | INFO | Store_Master: inserted 5,991 rows into ClickHouse +2026-06-19 13:30:01 | INFO | Store_Master loaded successfully (5991 rows) +2026-06-19 13:30:01 | INFO | ================================================================================ +2026-06-19 13:30:01 | INFO | Pipeline Completed Successfully +2026-06-19 13:30:01 | INFO | ================================================================================ +2026-06-19 13:30:48 | INFO | ================================================================================ +2026-06-19 13:30:48 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:30:48 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:30:48 | INFO | Connecting to databases... +2026-06-19 13:30:49 | INFO | +2026-06-19 13:30:50 | INFO | +2026-06-19 13:30:51 | INFO | Database connections established +2026-06-19 13:30:51 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:30:51 | INFO | Found 859 MIDs +2026-06-19 13:30:52 | INFO | ================================================================================ +2026-06-19 13:30:52 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 13:30:52 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:30:52 | ERROR | Failed processing table Store_Master +Traceback (most recent call last): + File "D:\data_move\main2.py", line 181, in main + fn = globals()[fn_name] + ~~~~~~~~~^^^^^^^^^ +KeyError: 'fetch_Store_Master' +2026-06-19 13:32:15 | INFO | ================================================================================ +2026-06-19 13:32:15 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:32:15 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:32:15 | INFO | Connecting to databases... +2026-06-19 13:32:16 | INFO | +2026-06-19 13:32:18 | INFO | +2026-06-19 13:32:19 | INFO | Database connections established +2026-06-19 13:32:19 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:32:19 | INFO | Found 859 MIDs +2026-06-19 13:32:20 | INFO | ================================================================================ +2026-06-19 13:32:20 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 13:32:20 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:32:20 | INFO | Fetching data from sql server for Master table...... +2026-06-19 13:32:20 | INFO | Fetching in progress .... +2026-06-19 13:32:26 | INFO | Fetched 5,991 rows from SQL Server +2026-06-19 13:32:26 | INFO | Fetched total row -: 5991 from sql server for table-:Store_Master ...........!!! +2026-06-19 13:32:26 | INFO | Fetched 5991 rows +2026-06-19 13:32:26 | INFO | Truncated table Store_Master +2026-06-19 13:32:26 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 13:32:29 | INFO | Store_Master: inserted 5,991 rows into ClickHouse +2026-06-19 13:32:29 | INFO | Store_Master loaded successfully (5991 rows) +2026-06-19 13:32:29 | INFO | ================================================================================ +2026-06-19 13:32:29 | INFO | Pipeline Completed Successfully +2026-06-19 13:32:29 | INFO | ================================================================================ +2026-06-19 13:32:34 | INFO | ================================================================================ +2026-06-19 13:32:34 | INFO | Hello from data-move Python data pipeline! +2026-06-19 13:32:34 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 13:32:34 | INFO | Connecting to databases... +2026-06-19 13:32:35 | INFO | +2026-06-19 13:32:37 | INFO | +2026-06-19 13:32:38 | INFO | Database connections established +2026-06-19 13:32:38 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 13:32:38 | INFO | Found 859 MIDs +2026-06-19 13:32:38 | INFO | ================================================================================ +2026-06-19 13:32:38 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 13:32:38 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-19 13:32:38 | INFO | Fetching data from sql server for Master table...... +2026-06-19 13:32:38 | INFO | Fetching in progress .... +2026-06-19 13:32:43 | INFO | Fetched 5,991 rows from SQL Server +2026-06-19 13:32:43 | INFO | Fetched total row -: 5991 from sql server for table-:Store_Master ...........!!! +2026-06-19 13:32:43 | INFO | Fetched 5991 rows +2026-06-19 13:32:43 | INFO | Truncated table Store_Master +2026-06-19 13:32:43 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 13:32:44 | INFO | Store_Master: inserted 5,991 rows into ClickHouse +2026-06-19 13:32:44 | INFO | Store_Master loaded successfully (5991 rows) +2026-06-19 13:32:44 | INFO | ================================================================================ +2026-06-19 13:32:44 | INFO | Pipeline Completed Successfully +2026-06-19 13:32:44 | INFO | ================================================================================ +2026-06-19 15:28:51 | INFO | ================================================================================ +2026-06-19 15:28:51 | INFO | Hello from data-move Python data pipeline! +2026-06-19 15:28:51 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 15:28:51 | INFO | Connecting to databases... +2026-06-19 15:28:53 | INFO | +2026-06-19 15:28:54 | INFO | +2026-06-19 15:28:55 | INFO | Database connections established +2026-06-19 15:28:55 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 15:28:55 | INFO | Found 859 MIDs +2026-06-19 15:28:56 | INFO | ================================================================================ +2026-06-19 15:28:56 | INFO | Processing Table: coverage_remarks | Table type is -: DIMENSION | Based on -reason_id and operation is used -DELETE+INSERT +2026-06-19 15:28:56 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. +2026-06-19 15:28:57 | WARNING | Table 'coverage_remarks' does not exist. +2026-06-19 15:28:57 | INFO | Fetching data from sql server for Master table...... +2026-06-19 15:28:57 | INFO | Fetching in progress .... +2026-06-19 15:28:57 | INFO | Fetched 29 rows from SQL Server +2026-06-19 15:28:57 | INFO | Fetched 29 rows from SQL Server +2026-06-19 15:28:57 | INFO | Fetched total row -: 29 from sql server for table-:coverage_remarks ...........!!! +2026-06-19 15:28:57 | INFO | Fetched 29 rows +2026-06-19 15:28:57 | INFO | Creating table coverage_remarks +2026-06-19 15:28:57 | INFO | Table ready: coverage_remarks +2026-06-19 15:28:57 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 15:28:57 | INFO | coverage_remarks: inserted 29 rows into ClickHouse +2026-06-19 15:28:57 | INFO | coverage_remarks loaded successfully (29 rows) +2026-06-19 15:28:57 | INFO | ================================================================================ +2026-06-19 15:28:57 | INFO | Pipeline Completed Successfully +2026-06-19 15:28:57 | INFO | ================================================================================ +2026-06-19 15:29:04 | INFO | ================================================================================ +2026-06-19 15:29:04 | INFO | Hello from data-move Python data pipeline! +2026-06-19 15:29:04 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 15:29:04 | INFO | Connecting to databases... +2026-06-19 15:29:05 | INFO | +2026-06-19 15:29:07 | INFO | +2026-06-19 15:29:07 | INFO | Database connections established +2026-06-19 15:29:07 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 15:29:08 | INFO | Found 859 MIDs +2026-06-19 15:29:08 | INFO | ================================================================================ +2026-06-19 15:29:08 | INFO | Processing Table: coverage_remarks | Table type is -: DIMENSION | Based on -reason_id and operation is used -DELETE+INSERT +2026-06-19 15:29:08 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. +2026-06-19 15:29:09 | INFO | Fetching data from sql server for Master table...... +2026-06-19 15:29:09 | INFO | Fetching in progress .... +2026-06-19 15:29:09 | INFO | Fetched 0 rows from SQL Server +2026-06-19 15:29:09 | INFO | Fetched 0 rows from SQL Server +2026-06-19 15:29:09 | INFO | Fetched total row -: 0 from sql server for table-:coverage_remarks ...........!!! +2026-06-19 15:29:09 | WARNING | coverage_remarks returned no rows +2026-06-19 15:29:09 | INFO | ================================================================================ +2026-06-19 15:29:09 | INFO | Pipeline Completed Successfully +2026-06-19 15:29:09 | INFO | ================================================================================ +2026-06-19 15:51:47 | INFO | ================================================================================ +2026-06-19 15:51:47 | INFO | Hello from data-move Python data pipeline! +2026-06-19 15:51:47 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 15:51:47 | INFO | Connecting to databases... +2026-06-19 15:51:49 | INFO | +2026-06-19 15:51:51 | INFO | +2026-06-19 15:51:52 | INFO | Database connections established +2026-06-19 15:51:52 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 15:51:53 | INFO | Found 859 MIDs +2026-06-19 15:51:54 | INFO | ================================================================================ +2026-06-19 15:51:54 | INFO | Processing Table: SKU_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 15:51:54 | INFO | Fetching Data from sql server for table-: SKU_Master .............. +2026-06-19 15:51:54 | INFO | Fetching data from sql server for Master table...... +2026-06-19 15:51:54 | INFO | Fetching in progress .... +2026-06-19 15:51:55 | INFO | Fetched 160 rows from SQL Server +2026-06-19 15:51:55 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! +2026-06-19 15:51:55 | INFO | Fetched 160 rows +2026-06-19 15:51:55 | INFO | Creating table SKU_Master +2026-06-19 15:51:55 | INFO | Table ready: SKU_Master +2026-06-19 15:51:55 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 15:51:55 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-19 15:51:55 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-19 15:51:55 | INFO | ================================================================================ +2026-06-19 15:51:55 | INFO | Pipeline Completed Successfully +2026-06-19 15:51:55 | INFO | ================================================================================ +2026-06-19 15:52:22 | INFO | ================================================================================ +2026-06-19 15:52:22 | INFO | Hello from data-move Python data pipeline! +2026-06-19 15:52:22 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 15:52:22 | INFO | Connecting to databases... +2026-06-19 15:52:23 | INFO | +2026-06-19 15:52:25 | INFO | +2026-06-19 15:52:28 | INFO | Database connections established +2026-06-19 15:52:28 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 15:52:28 | INFO | Found 859 MIDs +2026-06-19 15:52:29 | INFO | ================================================================================ +2026-06-19 15:52:29 | INFO | Processing Table: SKU_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 15:52:29 | INFO | Fetching Data from sql server for table-: SKU_Master .............. +2026-06-19 15:52:29 | INFO | Fetching data from sql server for Master table...... +2026-06-19 15:52:29 | INFO | Fetching in progress .... +2026-06-19 15:52:29 | INFO | Fetched 160 rows from SQL Server +2026-06-19 15:52:29 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! +2026-06-19 15:52:29 | INFO | Fetched 160 rows +2026-06-19 15:52:30 | INFO | Truncated table SKU_Master +2026-06-19 15:52:30 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 15:52:30 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-19 15:52:30 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-19 15:52:30 | INFO | ================================================================================ +2026-06-19 15:52:30 | INFO | Pipeline Completed Successfully +2026-06-19 15:52:30 | INFO | ================================================================================ +2026-06-19 15:56:07 | INFO | ================================================================================ +2026-06-19 15:56:07 | INFO | Hello from data-move Python data pipeline! +2026-06-19 15:56:07 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 15:56:07 | INFO | Connecting to databases... +2026-06-19 15:56:08 | INFO | +2026-06-19 15:56:09 | INFO | +2026-06-19 15:56:10 | INFO | Database connections established +2026-06-19 15:56:10 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 15:56:11 | INFO | Found 859 MIDs +2026-06-19 15:56:12 | INFO | ================================================================================ +2026-06-19 15:56:12 | INFO | Processing Table: display_master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 15:56:12 | INFO | Fetching Data from sql server for table-: display_master .............. +2026-06-19 15:56:12 | INFO | Fetching data from sql server for Master table...... +2026-06-19 15:56:12 | INFO | Fetching in progress .... +2026-06-19 15:56:12 | INFO | Fetched 135 rows from SQL Server +2026-06-19 15:56:12 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! +2026-06-19 15:56:12 | INFO | Fetched 135 rows +2026-06-19 15:56:12 | INFO | Creating table display_master +2026-06-19 15:56:12 | INFO | Table ready: display_master +2026-06-19 15:56:12 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 15:56:12 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-19 15:56:12 | INFO | display_master loaded successfully (135 rows) +2026-06-19 15:56:12 | INFO | ================================================================================ +2026-06-19 15:56:12 | INFO | Pipeline Completed Successfully +2026-06-19 15:56:12 | INFO | ================================================================================ +2026-06-19 15:56:19 | INFO | ================================================================================ +2026-06-19 15:56:19 | INFO | Hello from data-move Python data pipeline! +2026-06-19 15:56:19 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 15:56:19 | INFO | Connecting to databases... +2026-06-19 15:56:20 | INFO | +2026-06-19 15:56:22 | INFO | +2026-06-19 15:56:23 | INFO | Database connections established +2026-06-19 15:56:23 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 15:56:24 | INFO | Found 859 MIDs +2026-06-19 15:56:24 | INFO | ================================================================================ +2026-06-19 15:56:24 | INFO | Processing Table: display_master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 15:56:24 | INFO | Fetching Data from sql server for table-: display_master .............. +2026-06-19 15:56:24 | INFO | Fetching data from sql server for Master table...... +2026-06-19 15:56:24 | INFO | Fetching in progress .... +2026-06-19 15:56:24 | INFO | Fetched 135 rows from SQL Server +2026-06-19 15:56:24 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! +2026-06-19 15:56:24 | INFO | Fetched 135 rows +2026-06-19 15:56:25 | INFO | Truncated table display_master +2026-06-19 15:56:25 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 15:56:25 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-19 15:56:25 | INFO | display_master loaded successfully (135 rows) +2026-06-19 15:56:25 | INFO | ================================================================================ +2026-06-19 15:56:25 | INFO | Pipeline Completed Successfully +2026-06-19 15:56:25 | INFO | ================================================================================ +2026-06-19 16:00:13 | INFO | ================================================================================ +2026-06-19 16:00:13 | INFO | Hello from data-move Python data pipeline! +2026-06-19 16:00:13 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 16:00:13 | INFO | Connecting to databases... +2026-06-19 16:00:15 | INFO | +2026-06-19 16:00:17 | INFO | +2026-06-19 16:00:18 | INFO | Database connections established +2026-06-19 16:00:18 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 16:00:19 | INFO | Found 859 MIDs +2026-06-19 16:00:20 | INFO | ================================================================================ +2026-06-19 16:00:20 | INFO | Processing Table: Employee_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 16:00:20 | INFO | Fetching Data from sql server for table-: Employee_Master .............. +2026-06-19 16:00:20 | INFO | Fetching data from sql server for Master table...... +2026-06-19 16:00:20 | INFO | Fetching in progress .... +2026-06-19 16:00:23 | INFO | Fetched 2,287 rows from SQL Server +2026-06-19 16:00:23 | INFO | Fetched total row -: 2287 from sql server for table-:Employee_Master ...........!!! +2026-06-19 16:00:23 | INFO | Fetched 2287 rows +2026-06-19 16:00:23 | INFO | Creating table Employee_Master +2026-06-19 16:00:23 | INFO | Table ready: Employee_Master +2026-06-19 16:00:23 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 16:00:24 | INFO | Employee_Master: inserted 2,287 rows into ClickHouse +2026-06-19 16:00:24 | INFO | Employee_Master loaded successfully (2287 rows) +2026-06-19 16:00:24 | INFO | ================================================================================ +2026-06-19 16:00:24 | INFO | Pipeline Completed Successfully +2026-06-19 16:00:24 | INFO | ================================================================================ +2026-06-19 16:00:33 | INFO | ================================================================================ +2026-06-19 16:00:33 | INFO | Hello from data-move Python data pipeline! +2026-06-19 16:00:33 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 16:00:33 | INFO | Connecting to databases... +2026-06-19 16:00:34 | INFO | +2026-06-19 16:00:36 | INFO | +2026-06-19 16:00:36 | INFO | Database connections established +2026-06-19 16:00:36 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 16:00:37 | INFO | Found 859 MIDs +2026-06-19 16:00:37 | INFO | ================================================================================ +2026-06-19 16:00:37 | INFO | Processing Table: Employee_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-19 16:00:37 | INFO | Fetching Data from sql server for table-: Employee_Master .............. +2026-06-19 16:00:37 | INFO | Fetching data from sql server for Master table...... +2026-06-19 16:00:37 | INFO | Fetching in progress .... +2026-06-19 16:00:38 | INFO | Fetched 2,287 rows from SQL Server +2026-06-19 16:00:38 | INFO | Fetched total row -: 2287 from sql server for table-:Employee_Master ...........!!! +2026-06-19 16:00:38 | INFO | Fetched 2287 rows +2026-06-19 16:00:38 | INFO | Truncated table Employee_Master +2026-06-19 16:00:38 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-19 16:00:39 | INFO | Employee_Master: inserted 2,287 rows into ClickHouse +2026-06-19 16:00:39 | INFO | Employee_Master loaded successfully (2287 rows) +2026-06-19 16:00:39 | INFO | ================================================================================ +2026-06-19 16:00:39 | INFO | Pipeline Completed Successfully +2026-06-19 16:00:39 | INFO | ================================================================================ diff --git a/main2.py b/main2.py index 237ccf8..1571d79 100644 --- a/main2.py +++ b/main2.py @@ -43,9 +43,9 @@ from mids import ( MID_TABLE_COV1, ) -from masters.dimensions import * from masters.bridge import * from src.fact import * +from src.dim import * # ========================================================== @@ -64,29 +64,6 @@ def table_exists( ) -def get_dataframe( - sql_engine, - fn_name: str, - fetch_by: str, - table_name: str, - table_type: str , - mids, - run_date, -): - - fn = globals()[fn_name] - - if fetch_by == "mids" or "run_date": - return fn(sql_engine, table_name , table_type, mids, run_date) - - - - return fn(sql_engine ,table_name,table_type) - - - - - # ========================================================== # Main # ========================================================== @@ -158,9 +135,7 @@ def main(): # ------------------------------------------------------ # Process Tables # ------------------------------------------------------ - for table in config["tables"]: - table_name = table["name"] operation = table["operation"] fetch_by = table["fetch_by"] @@ -177,16 +152,14 @@ def main(): log.info(f"Fetching Data from sql server for table-: {table_name} ..............") - fn_name = f"fetch_{table_name}" - df = get_dataframe( - sql_engine, - fn_name=fn_name, - fetch_by=fetch_by, - table_name=table_name, - table_type=table_type, - mids=mids, - run_date=run_date, - ) + fetch_list=["mids" ,"run_date", "reason_id"] + if fetch_by in fetch_list : + fn_name = f"fetch_{table_name}" + fn = globals()[fn_name] + df=fn(sql_engine, table_name , table_type, mids, run_date) + else: + df = fetch_data(sql_engine ,table_name,table_type) + log.info(f"Fetched total row -: {len(df)} from sql server for table-:{table_name} ...........!!!") if df.is_empty(): diff --git a/src/bridge.py b/src/bridge.py index e69de29..efbcbe9 100644 --- a/src/bridge.py +++ b/src/bridge.py @@ -0,0 +1,142 @@ +from pathlib import Path +import polars as pl +from sqlalchemy import Engine +from datetime import date , timedelta +from log import log + + + + +from db_con.connection import ( + build_sql_server_engine, + build_clickhouse_engine, + get_clickhouse_client, +) + + + + + +def fetch_mapping_storevisibility( + sql_engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + client= get_clickhouse_client() + def table_exists( + client, + table_name: str, + ) -> bool: + + return bool( + client.command( + f"EXISTS TABLE {table_name}" + ) + ) + def get_reason_ids_mapping_storevisibility( + client, + table_name: str = "coverage_remarks", + ) -> list[int] : + + if not table_exists(client, table_name): + log.warning(f"Table '{table_name}' does not exist. During collecting reason_ids") + return [0] + + + query = f""" + SELECT DISTINCT StoreId + FROM mapping_storevisibility + WHERE toDate(Fromdate) <= {run_date} + AND toDate(Todate) >= {run_date} + AND a.Project_Id = '40148' + + """ + + # ClickHouse -> PyArrow -> Polars + arrow_table = client.query_arrow(query) + + df= pl.from_arrow(arrow_table) + list=df["reason_id"].to_list() + return list + + def fetch_data( + engine: Engine, + table_name: str, + table_type: str, + reason_ids: list[int] +) -> pl.DataFrame: + log.info(f"Fetching data from sql server for Master table......") + + resaon_id_list = ",".join(str(rid) for rid in reason_ids) + + sql_file = Path("src") / "sql" / f"dim" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + resaon_id_list=resaon_id_list + + ) + + log.info(f"Fetching in progress .... ") + + df = pl.read_database( + query=sql, + connection=engine + ) + + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + store_id=get_reason_ids_mapping_storevisibility(client, "coverage_remarks") + df=fetch_data(engine=sql_engine, + table_name=table_name, + table_type=table_type, + store_id=store_id, + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + + sql = f""" + SELECT DISTINCT + 40148 AS project_id, + Z.StoreId AS store_id, + Z.VisibilityDefinitionId AS visibility_definition_id, + Z.FromDate AS from_date, + Z.ToDate AS to_date + + FROM OneApp_KelloggsMT.dbo.Mapping_StoreVisibility Z + + WHERE CAST(Z.FromDate AS DATE) <= '{run_date}' + AND CAST(Z.ToDate AS DATE) >= '{run_date}' + + AND Z.VisibilityDefinitionId IN + ( + SELECT DISTINCT VisibilityDefinitionId + FROM OneApp_KelloggsMT.dbo.Master_VisibilityDefinition + WHERE MenuId = 22 + ) + """ + + log.info( + f"Fetching Mapping Store Visibility for {run_date}" + ) + + df = pl.read_database( + query=sql, + connection=engine + ) + + log.info( + f"Fetched {len(df):,} Mapping Store Visibility records" + ) + + return df \ No newline at end of file diff --git a/src/dim.py b/src/dim.py index e69de29..0a1085a 100644 --- a/src/dim.py +++ b/src/dim.py @@ -0,0 +1,138 @@ +from pathlib import Path +import polars as pl +from sqlalchemy import Engine +from datetime import date , timedelta +from log import log + + + + +from db_con.connection import ( + build_sql_server_engine, + build_clickhouse_engine, + get_clickhouse_client, +) + + + + + + + + + + +def fetch_data( + engine: Engine, + table_name: str, + table_type: str + +) -> pl.DataFrame: + + + + log.info(f"Fetching data from sql server for Master table......") + + sql_file = Path("src") / "sql" / f"dim" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( ) + + log.info(f"Fetching in progress .... ") + + df = pl.read_database( + query=sql, + connection=engine + ) + + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + +def fetch_coverage_remarks( + sql_engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date + +) -> pl.DataFrame: + + + client= get_clickhouse_client() + def table_exists( + client, + table_name: str, + ) -> bool: + + return bool( + client.command( + f"EXISTS TABLE {table_name}" + ) + ) + def get_reason_ids_coverage_remarks( + client, + table_name: str = "coverage_remarks", + ) -> list[int] : + + if not table_exists(client, table_name): + log.warning(f"Table '{table_name}' does not exist. During collecting reason_ids") + return [0] + + + query = f""" + SELECT DISTINCT + reason_id + FROM {table_name} + """ + + # ClickHouse -> PyArrow -> Polars + arrow_table = client.query_arrow(query) + + df= pl.from_arrow(arrow_table) + list=df["reason_id"].to_list() + return list + + def fetch_data( + engine: Engine, + table_name: str, + table_type: str, + reason_ids: list[int] +) -> pl.DataFrame: + log.info(f"Fetching data from sql server for Master table......") + + resaon_id_list = ",".join(str(rid) for rid in reason_ids) + + sql_file = Path("src") / "sql" / f"dim" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + resaon_id_list=resaon_id_list + + ) + + log.info(f"Fetching in progress .... ") + + df = pl.read_database( + query=sql, + connection=engine + ) + + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + reason_ids=get_reason_ids_coverage_remarks(client, "coverage_remarks") + df=fetch_data(engine=sql_engine, + table_name=table_name, + table_type=table_type, + reason_ids=reason_ids, + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df diff --git a/src/fact.py b/src/fact.py index 125712a..cee6fae 100644 --- a/src/fact.py +++ b/src/fact.py @@ -26,40 +26,40 @@ from db_con.connection import ( -def fetch_data( - engine: Engine, - table_name: str, - table_type: str, - mids: list[int], - run_date: date -) -> pl.DataFrame: +# def fetch_data( +# engine: Engine, +# table_name: str, +# table_type: str, +# mids: list[int], +# run_date: date +# ) -> pl.DataFrame: - if not mids: - log.warning("No MIDs — nothing to fetch.") - return pl.DataFrame() +# if not mids: +# log.warning("No MIDs — nothing to fetch.") +# return pl.DataFrame() - mid_list = ",".join(str(mid) for mid in mids) +# mid_list = ",".join(str(mid) for mid in mids) - sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" +# sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" - with open(sql_file, "r", encoding="utf-8") as f: - sql_template = f.read() +# with open(sql_file, "r", encoding="utf-8") as f: +# sql_template = f.read() - sql = sql_template.format( - mid_list=mid_list, - run_date=run_date.strftime("%Y-%m-%d") - ) +# sql = sql_template.format( +# mid_list=mid_list, +# run_date=run_date.strftime("%Y-%m-%d") +# ) - log.info(f"Fetching data for {len(mids):,} MIDs") +# log.info(f"Fetching data for {len(mids):,} MIDs") - df = pl.read_database( - query=sql, - connection=engine - ) +# df = pl.read_database( +# query=sql, +# connection=engine +# ) - log.info(f"Fetched {len(df):,} rows from SQL Server") +# log.info(f"Fetched {len(df):,} rows from SQL Server") - return df +# return df @@ -548,7 +548,7 @@ def fetch_Attendance( sql_template = f.read() sql = sql_template.format( - star_date=start_date.strftime("%Y-%m-%d"), + start_date=start_date.strftime("%Y-%m-%d"), run_date=run_date.strftime("%Y-%m-%d") ) log.info( @@ -566,3 +566,111 @@ def fetch_Attendance( ) return df + + + +def fetch_Journey_Plan( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + + log.info(f" Start Fetching data for these {len(mids):} MIDs ") + mid_list = ",".join(str(mid) for mid in mids) + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + +def fetch_PaidVisibility( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + + log.info(f" Start Fetching data for these {len(mids):} MIDs ") + mid_list = ",".join(str(mid) for mid in mids) + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + +def fetch_Web_Logins( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + + log.info(f" Start Fetching data for these {len(mids):} MIDs ") + mid_list = ",".join(str(mid) for mid in mids) + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df \ No newline at end of file diff --git a/src/sql/bridge/mapping_storevisibility.sql b/src/sql/bridge/mapping_storevisibility.sql new file mode 100644 index 0000000..d0a6568 --- /dev/null +++ b/src/sql/bridge/mapping_storevisibility.sql @@ -0,0 +1,12 @@ + INSERT INTO mapping_storevisibility(Project_Id,StoreId,VisibilityDefinitionid,Fromdate,Todate,CreateDate,CreateBy) + + +select DISTINCT '40148' as Project_Id,StoreId,VisibilityDefinitionid,Fromdate,Todate,getdate(),'SP-Pius' +FROM OneApp_KelloggsMT.dbo.mapping_storevisibility z WHERE +convert(date,FROMDATE,101)<=convert(Date,getdate(),101) AND CONVERT(DATE,ToDate,101)>=convert(Date,getdate(),101) +AND z.VisibilityDefinitionid IN +(SELECT DISTINCT VisibilityDefinitionid FROM OneApp_KelloggsMT.dbo.MASTER_VISIBILITYDEFINITION WHERE MENUID=22 ) +AND z.StoreId NOT IN ( +SELECT distinct a.STOREID FROM Mapping_StoreVisibility a WHERE convert(date,a.FROMDATE,101)<=convert(Date,getdate(),101) +AND CONVERT(DATE,a.ToDate,101)>=convert(Date,getdate(),101) AND a.Project_Id='40148' +and a.StoreId=z.StoreId and a.VisibilityDefinitionid=z.VisibilityDefinitionId \ No newline at end of file diff --git a/src/sql/dim/Employee_Master.sql b/src/sql/dim/Employee_Master.sql new file mode 100644 index 0000000..7cd00b9 --- /dev/null +++ b/src/sql/dim/Employee_Master.sql @@ -0,0 +1,17 @@ + with Employee_Master (project_id,region_id,region,state_id,state,city_id, + city,employee_id,employee_name,gender,designation_id,designation,manager_id, + manager_name,employee_joining_date,employee_resign_date,position_code,employee_legacy_code,employee_role,EMPLOYEE_TYPE) + as ( +select '40148' as ProjectId, RegionId, + RegionName,StateId,StateName, CityId,CityName, + a.EmpId,EmpName,Gender,a.DesignationId,DesignationName,SupervisorId,SupervisorName,JoinDate,ResignDate,c.PositionCode ,EmpCode ,RIGHTNAME, + + case when RIGHTNAME in ('Client','Client HO') Then 'NON CPM' else 'CPM' END AS EMPLOYEE_TYPE +from OneApp_KelloggsMT.dbo.vw_Employee_Detail a Left join (Select distinct PositionId, EmpId from OneApp_KelloggsMT.dbo.Mapping_PositionUser b + Where DATEdiff(d,FromDate, GETDATE())>= 0 and DATEDIFF(d,ToDate,getdate())<=0) b on a.EmpId=b.EmpId + LEFT join (select PositionId,PositionCode from OneApp_KelloggsMT.dbo.Master_Position) c on b.PositionId=c.PositionId + where 1=1 + -- and EmpId not in ( + --select employee_id from [dbo].[Employee_Master] where project_id='40148') + ) +select * from Employee_Master \ No newline at end of file diff --git a/src/sql/dim/Master_Salesterritorylayer.sql b/src/sql/dim/Master_Salesterritorylayer.sql new file mode 100644 index 0000000..e69de29 diff --git a/src/sql/dim/Master_VisibilityDefinition.sql b/src/sql/dim/Master_VisibilityDefinition.sql new file mode 100644 index 0000000..e69de29 diff --git a/src/sql/dim/Master_VisibilityReason.sql b/src/sql/dim/Master_VisibilityReason.sql new file mode 100644 index 0000000..e69de29 diff --git a/src/sql/dim/SKU_Master.sql b/src/sql/dim/SKU_Master.sql new file mode 100644 index 0000000..18aa5be --- /dev/null +++ b/src/sql/dim/SKU_Master.sql @@ -0,0 +1,18 @@ + +with SKU_Master (project_id,category_id,category_code,Category_name,sub_category_id,sub_category_code,sub_category_name,brand_id, +brand_code,brand_name,sub_brand_id,sub_brand_code,sub_brand_name,product_id,product_name,product_code,mrp,flavour_id, +flavour,grammage,product_sequence,case_size,company_name,is_competitor,ptr,update_date,update_by) + as ( + + Select '40148' as ProjectId, cm.CategoryId, +cm.CategoryCode,cm.CategoryName,sca.SubCategoryId,sca.SubCategoryCode,sca.SubCategoryName,br.BrandId,br.BrandCode,br.BrandName +,sb.SubBrandId,sb.SubBrandCode,sb.SubBrandName,p.ProductId,p.ProductName,p.ProductCode,p.Mrp,fl.FlavourId, +fl.Flavour,p.Grammage,p.ProductSequence,p.CaseSize,mc.Company,mc.IsCompetitor,p.ptr,GETDATE(),'MB' +from OneApp_KelloggsMT.dbo.Master_Product p Right join +OneApp_KelloggsMT.dbo.Master_Flavour fl on p.FlavourId= Fl.FlavourId Right join +OneApp_KelloggsMT.dbo.Master_subbrand sb on p.SubBrandId= sb.SubBrandId Right join +OneApp_KelloggsMT.dbo.Master_Brand br on sb.BrandId = br.BrandId Right join +OneApp_KelloggsMT.dbo.Master_SubCategory sca on br.SubCategoryId= sca.SubCategoryId Right join +OneApp_KelloggsMT.dbo.Master_Category cm on sca.CategoryId= cm.CategoryId Right join +OneApp_KelloggsMT.dbo.Master_Company mc on mc.companyId=br.companyId ) +select * from SKU_Master diff --git a/src/sql/dim/Store_Master.sql b/src/sql/dim/Store_Master.sql new file mode 100644 index 0000000..a768d40 --- /dev/null +++ b/src/sql/dim/Store_Master.sql @@ -0,0 +1,14 @@ +with Store_Master (project_id, +region_id,region,state_id,state,city_id,city,cpm_city_id,channel_id,channel,distributor_id,distributor_name, keyaccount_id, keyaccount, +insight_store_id,client_store_code,latitude,longitude,store_category_id,store_category,store_type_id,store_type,store_classification_id,store_classification, StLayerFourId, +store_id,store_name,address) + AS ( +select '40148',RegionId,RegionName,StateId,StateName,CityId, +CityName,CityCode,ChannelId,ChannelName,DistributorId,Distributor,ChainId,ChainName,StoreUniqueCode,StoreCode,Latitude, +Longitude, StoreCategoryId,StoreCategory,StoreTypeId, +StoreType,StoreClassId,StoreClass,StLayerFourId,StoreId, +StoreName,Address from OneApp_KelloggsMT.dbo.vw_storedetail where 1=1 +--and storeid not in ( +--select store_id from Store_Master where project_id='40148') +) +select * from Store_Master \ No newline at end of file diff --git a/src/sql/dim/coverage_remarks.sql b/src/sql/dim/coverage_remarks.sql new file mode 100644 index 0000000..3eef46e --- /dev/null +++ b/src/sql/dim/coverage_remarks.sql @@ -0,0 +1,8 @@ + with coverage_remarks + (project_id,reason_id,reason_remarks) + as( + select + '40148' ,reasonid,reason from OneApp_KelloggsMT.dbo.master_nonworkingreason + where reasonid not in ( {resaon_id_list}) + ) + select * from coverage_remarks \ No newline at end of file diff --git a/src/sql/dim/display_master.sql b/src/sql/dim/display_master.sql new file mode 100644 index 0000000..cb0ea82 --- /dev/null +++ b/src/sql/dim/display_master.sql @@ -0,0 +1,8 @@ + with display_master (project_id,display_id,display_code,display_name,display_ref_url,created_date, + created_by) as ( + select '40148',displayid,displaycode,displayname,displayrefImage,getdate(),'Pius' + from OneApp_KelloggsMT.dbo.Master_display +-- and ProductId not in ( +--select product_id from [dbo].[SKU Master] where project_id='40148') + ) +select * from display_master \ No newline at end of file diff --git a/src/sql/fact/Attendance.sql b/src/sql/fact/Attendance.sql index 1fcbf56..ecf3b4f 100644 --- a/src/sql/fact/Attendance.sql +++ b/src/sql/fact/Attendance.sql @@ -6,7 +6,7 @@ with Emp_cte as , em1.Id as [Supervisor Id], isnull(em1.EmployeeName,'') as [Supervisor Name] , em.Id as [Employee Id], isnull(em.EmployeeName,'') as [Employee Name] , dm.DesignationId, dm.DesignationName as Designation - , convert(varchar,EM.ResignDate,101) as DOR, convert(varchar,EM.JoinDate,101) as DOJ, em.LegacyCode + , CAST(EM.ResignDate AS DATE) as DOR, CAST(EM.JoinDate AS DATE) as DOJ, em.LegacyCode --,ISNULL(ps.PositionCode ,'') PositionCode--, FromDate, ToDate from OneApp_KelloggsMT.dbo.AspNetUsers em inner join @@ -18,16 +18,16 @@ with Emp_cte as OneApp_KelloggsMT.dbo.Master_Designation dm on Em.DesignationId= dm.DesignationId where em.RightId in (6) and em.EmployeeName not like '%test%' and - (Em.ResignDate is null or Em.ResignDate>='{start_date}') and CONVERT(date,em.JoinDate,101)<=CONVERT(date,'{run_date}',101) + (Em.ResignDate is null or Em.ResignDate>=CAST('{start_date}' AS DATE)) and CONVERT(date,em.JoinDate,101)<=CONVERT(date,CAST('{run_date}' AS DATE),101) --and em.Id=2290 )a - cross join (select distinct DATE from DBO.GET_ALL_DAYS('{start_date}','{run_date}')) b + cross join (select distinct DATE from DBO.GET_ALL_DAYS(CAST('{start_date}' AS DATE),CAST('{run_date}' AS DATE))) b ) AS T1 LEFT JOIN (Select distinct EmpId, PositionId, Date,FromDate,ToDate from (Select distinct EmpId,PositionId,FromDate,ToDate from OneApp_KelloggsMT.dbo.Mapping_PositionUser - Where convert(Date,FromDate)<= CONVERT(date,'{run_date}',101) and convert(Date,ToDate)>=CONVERT(date,'{start_date}',101)) as m Inner Join - (Select Date from OneApp_KelloggsMT.dbo.Master_Calender Where 1=1 and Date between CONVERT(date,'{start_date}',101) and CONVERT(date,'{run_date}',101)) + Where convert(Date,FromDate)<= CONVERT(date,CAST('{run_date}' AS DATE),101) and convert(Date,ToDate)>=CONVERT(date,CAST('{start_date}' AS DATE),101)) as m Inner Join + (Select Date from OneApp_KelloggsMT.dbo.Master_Calender Where 1=1 and Date between CONVERT(date,CAST('{start_date}' AS DATE),101) and CONVERT(date,CAST('{run_date}' AS DATE),101)) cal on cal.Date >= m.FromDate AND cal.Date <= m.ToDate) pu on T1.[Employee Id]=pu.EmpId and T1.date= pu.date Left Join OneApp_KelloggsMT.dbo.Master_Position ps WITH (NOLOCK) ON pu.PositionId=ps.PositionId @@ -53,7 +53,7 @@ with Emp_cte as FROM ( - SELECT A.DATE,TT.[MERCHANDISER CD],TT.VisitDate,TT.DOJ1,TT.DOR1,TT.COL from DBO.GET_ALL_DAYS('{start_date}','{run_date}') a + SELECT A.DATE,TT.[MERCHANDISER CD],TT.VisitDate,TT.DOJ1,TT.DOR1,TT.COL from DBO.GET_ALL_DAYS(CAST('{start_date}' AS DATE),CAST('{run_date}' AS DATE)) a left join @@ -115,7 +115,7 @@ with Emp_cte as - WHERE EM.EMPLOYEENAME not like 'teststore%' AND JP.VisitDate BETWEEN '{start_date}' AND '{run_date}' + WHERE EM.EMPLOYEENAME not like 'teststore%' AND JP.VisitDate BETWEEN CAST('{start_date}' AS DATE) AND CAST('{run_date}' AS DATE) ) AS T1 ) AS T2 @@ -123,9 +123,12 @@ with Emp_cte as WHERE COL1=1 ) AS TT on a.date=tt.visitdate)A)b - on emp.[Employee Id]=b.[MERCHANDISER CD] and emp.DATE= b.DATE ) + on emp.[Employee Id]=b.[MERCHANDISER CD] and ( + emp.DOR IS NULL + OR emp.DATE >= emp.DOR + ) ) select * from Attendance where parinaam_attendance !='-' and - project_id=40148 and - CONVERT(date,visit_date,101) > CONVERT(date,date_of_join,101) + project_id=40148 + AND visit_date > date_of_join order by employee_id, visit_date \ No newline at end of file diff --git a/src/sql/fact/Journey_Plan.sql b/src/sql/fact/Journey_Plan.sql index e69de29..f777999 100644 --- a/src/sql/fact/Journey_Plan.sql +++ b/src/sql/fact/Journey_Plan.sql @@ -0,0 +1,10 @@ +with Journey_Plan + (project_id,store_id,employee_id,visit_date,process_id,CreateDate,CreateBy,UpdateDate, + UpdateBy) + AS ( + select + '40148' ,storeid,EmpId,cast(visitdate as DATE),Deviation,GETDATE(),'Pius',GETDATE(),'Pius' from OneApp_KelloggsMT.dbo.mapping_journeyplan + where MONTH(VisitDate) = MONTH(CAST('{run_date}' AS DATE)) AND Year(VisitDate)=Year(CAST('{run_date}' AS DATE)) + AND EmpId NOT IN ( SELECT ID FROM OneApp_KelloggsMT.dbo.ASPNETUSERS + WHERE USERNAME LIKE 'TEST%')) +select * from Journey_Plan; \ No newline at end of file diff --git a/src/sql/fact/Login.sql b/src/sql/fact/Login.sql index c0be767..9660fe7 100644 --- a/src/sql/fact/Login.sql +++ b/src/sql/fact/Login.sql @@ -12,7 +12,7 @@ with employee_detail AS INNER JOIN OneApp_KelloggsMT.dbo.Master_Region rm ON st.RegionId = rm.RegionId INNER JOIN OneApp_KelloggsMT.dbo.Master_Designation dm ON Em.DesignationId = dm.DesignationId WHERE em.RightId IN (6) - AND (Em.ResignDate IS NULL OR CONVERT(DATE, Em.ResignDate, 101) >= '{run_date}') + AND (Em.ResignDate IS NULL OR CONVERT(DATE, Em.ResignDate, 101) >= CAST('{start_date}' AS DATE)) AND em.EmployeeName NOT LIKE '%test%' ), Login ( @@ -71,7 +71,7 @@ Login ( ) AS [Last Store Out Time] FROM OneApp_KelloggsMT.dbo.T_DeviceLogin ud INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em ON ud.EmpId = Em.EmpId - WHERE CAST(ud.LoginDate AS DATE) = '{run_date}' -- fixed: direct date comparison + WHERE CAST(ud.LoginDate AS DATE) = CAST('{start_date}' AS DATE) -- fixed: direct date comparison ) AS T1 WHERE T1.col = 1 ) AS T ON Em.[Employee code] = T.[Employee Code] diff --git a/src/sql/fact/PaidVisibility.sql b/src/sql/fact/PaidVisibility.sql index e69de29..84324a7 100644 --- a/src/sql/fact/PaidVisibility.sql +++ b/src/sql/fact/PaidVisibility.sql @@ -0,0 +1,56 @@ + + with PaidVisibility(Mid, + project_id,store_id,employee_id,visit_date,supervisor_id,channel_id,chain_id,storetype_id,Menuid,MenuName, + Visibility_Id, + Visibility_Name, + Visibility_definition_id,Visibility_definition_name, + Visibility_deatils,Visibility_deatils_id,Visibility_value_name, + present,REASONID,reason,VisibilityQuestion,VisibilityAnswer,image1,image2,update_date,update_by) + AS ( + + Select sc.MID, '40148' Projectid,sc.StoreId,Em.EmpId,sc.VisitDate,Em.SupervisorId,sm.ChannelId,sm.ChainId,sm.StoreTypeId, 0 as menuid ,'' as menuname, + ts.visibilityid,mv.Visibilityname,msd.VisibilityDefinitionId, + MSD.VisibilityDefinitionName, case when isnull(TS.VisibilityTable,'')='Master_Category' then 'Category' + when isnull(TS.VisibilityTable,'')='Master_SubCategory' then 'SubCategory' + when isnull(TS.VisibilityTable,'')='Master_Brand' then 'Brand' + when isnull(TS.VisibilityTable,'')='Master_SubBrand' then 'SubBrand' + end as PaidVisibilityDeatils ,ts.visibilityvalue[Paid_visiValueID], + CASE when isnull(TS.VisibilityTable,'')='Master_Category' THEN (SELECT a.CategoryName from OneApp_KelloggsMT.dbo.Master_Category a where ts.VisibilityValue=a.CategoryId ) + when isnull(TS.VisibilityTable,'')='Master_SubCategory' THEN (SELECT a.SubCategoryName from OneApp_KelloggsMT.dbo.Master_SubCategory a where ts.VisibilityValue=a.SubCategoryId ) + when isnull(TS.VisibilityTable,'')='Master_Brand' THEN (SELECT a.BrandName from OneApp_KelloggsMT.dbo.Master_Brand a where ts.VisibilityValue=a.BrandId ) + when isnull(TS.VisibilityTable,'')='Master_SubBrand' THEN (SELECT a.SubBrandName from OneApp_KelloggsMT.dbo.Master_SubBrand a where ts.VisibilityValue=a.SubBrandId ) end as + [Paid_VisiValueName],case when ts.Present=0 then 'N' else 'Y' end Present, + CASE WHEN TS.PRESENT=0 THEN TS.VISIBILITYREASONID ELSE '' END AS VISIBILITYREASONID , + case when ts.Present=0 then isnull(mnp.VisibilityReason,'') else '' end Reason, + ISNULL( MVQ.VisibilityQuestionName,'') AS Question, ISNULL(TVQ.VisibilityAnswerName,'') AS Answer , + case when ts.present=0 then '' else + case when isnull(SHI.VisibilityImage1,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PaidVisibilityImages/'+SHI.VisibilityImage1 + end end as PaidVisibilityImg1 ,case when isnull(SHI.VisibilityImage2,'')='' then '' else 'https://kimt.parinaam.in/Upload/PaidVisibilityImages/'+SHI.VisibilityImage2 + end as PaidVisibilityImg2 ,GETDATE(),'SP-Pius' + + + from + OneApp_KelloggsMT.dbo.T_Visibility ts WITH (NOLOCK) Inner join + OneApp_KelloggsMT.dbo.T_StoreCoverage sc WITH (NOLOCK) on ts.mid= sc.mid Inner Join + OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join + OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId + inner join + OneApp_KelloggsMT.dbo.Master_VisibilityDefinition msd WITH (NOLOCK) on + msd.VisibilityDefinitionId=ts.VisibilityDefinitionId --and msd.menuid=ts.menuid + --inner join OneApp_KelloggsMT.dbo.Master_menu MM WITH (NOLOCK) ON ts.menuid=mm.menuid and msd.menuid=mm.menuid + left join OneApp_KelloggsMT.DBO. master_visibility mv WITH (NOLOCK) on ts.visibilityid=mv.visibilityid + + + LEFT join + OneApp_KelloggsMT.dbo.T_VisibilityImages SHI WITH (NOLOCK) ON ts.VId=SHI.VId + left join + OneApp_KelloggsMT.dbo.Master_VisibilityReason mnp WITH (NOLOCK) on ts.VisibilityReasonId=mnp.VisibilityReasonId + LEFT JOIN OneApp_KelloggsMT.dbo.T_VisibilityStock TVS WITH (NOLOCK) ON TS.VID=TVS.VID + LEFT JOIN OneApp_KelloggsMT.dbo.T_VisibilityQuestion TVQ WITH (NOLOCK) ON TS.Vid=TVQ.Vid + LEFT JOIN OneApp_KelloggsMT.dbo.Master_VisibilityQuestion MVQ WITH (NOLOCK) on TVQ.VisibilityQuestionId=MVQ.VisibilityQuestionId + + Where 1=1 and Em.EmpName not like 'test%' + --and sc.visitdate between '10/01/2023' and '10/31/2023' + AND sc.MID in ({mid_list}) + ) +select * from PaidVisibility \ No newline at end of file diff --git a/src/sql/fact/Promotion.sql b/src/sql/fact/Promotion.sql index e69de29..05b5d8e 100644 --- a/src/sql/fact/Promotion.sql +++ b/src/sql/fact/Promotion.sql @@ -0,0 +1,43 @@ +with Promotion(Mid, + project_id,store_id,employee_id,visit_date,supervisor_id,channel_id,chain_id,storetype_id,promo_definition_id,promo_definition_name, + promotion_deatils,promotion_deatils_id,promotion_value_name, + present,reason,PromoQuestion,PromoAnswer,image1,image2,update_date,update_by) + as ( + Select + sc.MID, '40148' Projectid,sc.StoreId,Em.EmpId,sc.VisitDate,Em.SupervisorId,sm.ChannelId,sm.ChainId,sm.StoreTypeId,msd.PromoDefinitionId, + MSD.PromoDefinitionName, case when isnull(TS.PromoTable,'')='Master_Category' then 'Category' + when isnull(TS.PromoTable,'')='Master_SubCategory' then 'SubCategory' + when isnull(TS.PromoTable,'')='Master_Brand' then 'Brand' + when isnull(TS.PromoTable,'')='Master_SubBrand' then 'SubBrand' + end as PromotionDeatils , + TS.PromoValue[PromotionDeatilsId], + CASE when isnull(TS.PromoTable,'')='Master_Category' THEN (SELECT a.CategoryName from OneApp_KelloggsMT.dbo.Master_Category a where ts.PromoValue=a.CategoryId ) + when isnull(TS.PromoTable,'')='Master_SubCategory' THEN (SELECT a.SubCategoryName from OneApp_KelloggsMT.dbo.Master_SubCategory a where ts.PromoValue=a.SubCategoryId ) + when isnull(TS.PromoTable,'')='Master_Brand' THEN (SELECT a.BrandName from OneApp_KelloggsMT.dbo.Master_Brand a where ts.PromoValue=a.BrandId ) + when isnull(TS.PromoTable,'')='Master_SubBrand' THEN (SELECT a.SubBrandName from OneApp_KelloggsMT.dbo.Master_SubBrand a where ts.PromoValue=a.SubBrandId ) end as + [PromoValueName],case when ts.Present=0 then 'N' else 'Y' end Present, + case when ts.Present=1 then '' else isnull(mnp.PromoReason,'') end as Reason, ISNULL(mpq.PromoQuestionName,'') AS Question, + ISNULL(tpq.PromoAnswerName,'') AS Answer , + case when isnull(SHI.PromoImage1,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PromotionImages/'+SHI.PromoImage1 + end as Image1 ,case when isnull(SHI.PromoImage2,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PromotionImages/'+SHI.PromoImage2 + end as Image2 ,GETDATE(),'SP-Pius' + + from + OneApp_KelloggsMT.dbo.T_Promotion ts Inner join + OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join + OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join + OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId + inner join + OneApp_KelloggsMT.dbo.Master_PromotionDefinition msd on + msd.PromoDefinitionId=ts.PromoDefinitionId + LEFT join + OneApp_KelloggsMT.dbo.T_PromotionImages SHI ON ts.PId=SHI.PId + left join + OneApp_KelloggsMT.dbo.Master_PromotionReason mnp on ts.PromoReasonId=mnp.PromoReasonId + left join OneApp_KelloggsMT.dbo.t_promotionquestion tpq on ts.PId=tpq.PId + left join OneApp_KelloggsMT.dbo.Master_PromotionQuestion mpq on tpq.PromoQuestionId=mpq.PromoQuestionId + + Where 1=1 and Em.EmpName not like 'test%' + AND sc.MID in ({mid_list}) + ) +select * from Promotion \ No newline at end of file diff --git a/src/sql/fact/Web_Logins.sql b/src/sql/fact/Web_Logins.sql index e69de29..8e83d87 100644 --- a/src/sql/fact/Web_Logins.sql +++ b/src/sql/fact/Web_Logins.sql @@ -0,0 +1,23 @@ + + with Web_Logins (project_id, + supervisor_id,supervisor_name,emp_id,employee_name,designation,date,time,activity_name,activity_type,right_name,CreateDate,CreateBy) + as ( + Select Distinct '40148' as ProjectId + , EM1.Id AS SupervisorId, EM1.EmployeeName AS Supervisor + , EM.Id AS EmployeeId , EM.EmployeeName as Employee + , DM.DesignationName as Designation + , cast( CONVERT(date,AL.Date) as Date) AS Date,CONVERT(NVARCHAR,AL.Date ,108) AS Time + , AL.Thread as ActivityName, AL.Level as ActivityType, R.RightName,GETDATE(),'Pius' + FROM OneApp_KelloggsMT.dbo.T_User_Activity_Log AL LEFT JOIN + OneApp_KelloggsMT.dbo.AspNetUsers EM ON AL.Logger=EM.USERNAME INNER JOIN + OneApp_KelloggsMT.dbo.AspNetUsers EM1 ON EM1.Id = EM.ManagerId INNER JOIN + OneApp_KelloggsMT.dbo.AspNetUsers EM2 ON EM2.Id = EM1.ManagerId INNER JOIN + OneApp_KelloggsMT.dbo.Master_Designation DM ON EM.DesignationId = DM.DesignationId INNER JOIN + OneApp_KelloggsMT.dbo.Master_City CM ON EM.CityId = CM.CityId INNER JOIN + OneApp_KelloggsMT.dbo.Master_State ST ON ST.StateId = cm.StateId INNER JOIN + OneApp_KelloggsMT.dbo.Master_Region RM ON RM.RegionId = ST.RegionId INNER JOIN + OneApp_KelloggsMT.dbo.Right_Master R ON EM.RightId=R.RightId + Where 1=1 and CONVERT(date,AL.Date,101) = CONVERT(DATE,CAST('{run_date}' AS DATE),101) + ) + select * from Web_Logins + ORDER BY employee_name \ No newline at end of file diff --git a/t.yml b/t.yml index 8170a18..285375d 100644 --- a/t.yml +++ b/t.yml @@ -62,32 +62,32 @@ tables: - name: Store_Master type: DIMENSION operation: DELETE+INSERT - fetch_by: none + fetch_by: master - name: SKU_Master type: DIMENSION operation: DELETE+INSERT - fetch_by: none + fetch_by: master - name: display_master type: DIMENSION operation: DELETE+INSERT - fetch_by: none + fetch_by: master - name: Employee_Master type: DIMENSION operation: DELETE+INSERT - fetch_by: none + fetch_by: master - name: coverage_remarks type: DIMENSION operation: DELETE+INSERT - fetch_by: none + fetch_by: reason_id - name: mapping_storevisibility type: BRIDGE - operation: DELETE+INSERT + operation: ONLY_INSERT fetch_by: run_date - name: Master_VisibilityReason diff --git a/y.yml b/y.yml index f6daf0e..f8e0634 100644 --- a/y.yml +++ b/y.yml @@ -1,35 +1,82 @@ tables: - - name: SOS_OneApp - type: FACT - operation: INSERT - fetch_by: mids + # - name: SOS_OneApp + # type: FACT + # operation: INSERT + # fetch_by: mids - # - name: OQaD + # # - name: OQaD + # # type: FACT + # # operation: INSERT + # # fetch_by: run_date + + # - name: additional_visibility + # type: FACT + # operation: INSERT + # fetch_by: mids + + # - name: Coverage + # type: FACT + # operation: INSERT + # fetch_by: mids + + # - name: Survey + # type: FACT + # operation: INSERT + # fetch_by: mids + + # - name: Login + # type: FACT + # operation: INSERT + # fetch_by: run_date + + # - name: Stock_Details + # type: FACT + # operation: INSERT + # fetch_by: mids + + # - name: Attendance + # type: FACT + # operation: DELETE+INSERT + # fetch_by: run_date + + # - name: Journey_Plan # type: FACT # operation: INSERT # fetch_by: run_date - - name: additional_visibility - type: FACT - operation: INSERT - fetch_by: mids + # - name: PaidVisibility + # type: FACT + # operation: INSERT + # fetch_by: mids - - name: Coverage - type: FACT - operation: INSERT - fetch_by: mids + # - name: Web_Logins + # type: FACT + # operation: INSERT + # fetch_by: run_date + + + # - name: Store_Master + # type: DIMENSION + # operation: DELETE+INSERT + # fetch_by: master - - name: Survey - type: FACT - operation: INSERT - fetch_by: mids + # - name: coverage_remarks + # type: DIMENSION + # operation: DELETE+INSERT + # fetch_by: reason_id - - name: Login - type: FACT - operation: INSERT - fetch_by: run_date + # - name: SKU_Master + # type: DIMENSION + # operation: DELETE+INSERT + # fetch_by: master + + # - name: display_master + # type: DIMENSION + # operation: DELETE+INSERT + # fetch_by: master + + - name: Employee_Master + type: DIMENSION + operation: DELETE+INSERT + fetch_by: master - - name: Stock_Details - type: FACT - operation: INSERT - fetch_by: mids \ No newline at end of file