From d60740ae48c98cb7035c94a616fa8e9309c69166 Mon Sep 17 00:00:00 2001 From: Ankit Malik Date: Thu, 18 Jun 2026 18:16:50 +0530 Subject: [PATCH] 18-06-2026 1st commit --- clickhouse_task/delete_task.py | 17 +- logs/etl_20260618.log | 2134 ++++++++++++++++++++++++++++++++ main2.py | 86 +- mids.py | 48 - src/fact.py | 527 +++++++- src/sql/fact/Attendance.sql | 131 ++ src/sql/fact/Coverage.sql | 262 ++-- src/sql/fact/Login.sql | 81 ++ src/sql/fact/SOS_OneApp.sql | 70 ++ src/sql/fact/Stock_Details.sql | 29 + y.yml | 30 +- 11 files changed, 3104 insertions(+), 311 deletions(-) create mode 100644 logs/etl_20260618.log diff --git a/clickhouse_task/delete_task.py b/clickhouse_task/delete_task.py index dde7e58..13c5cea 100644 --- a/clickhouse_task/delete_task.py +++ b/clickhouse_task/delete_task.py @@ -38,10 +38,7 @@ def delete_rows( WHERE {where_clause} """ - log.info( - "Deleting from %s", - table_name, - ) + log.info(f"_ _ _ _ Deleting Data from ClickHouse for {table_name} _ _ _ _") client.command(query) @@ -72,16 +69,14 @@ def delete_existing_data( "Stock_Details", } - if table_name in mid_tables and mids: + if table_name in mid_tables and mids : - mids_str = ",".join( - map(str, mids) - ) + mids_str = ",".join(str(mid) for mid in mids) delete_rows( client, table_name, - f"MID IN ({mids_str})", + f"Mid IN ({mids_str})", ) return @@ -147,8 +142,8 @@ def delete_existing_data( client, table_name, f""" - toDate(attendance_date) - = toDate('{run_date}') + toDate(visit_date) BETWEEN toDate('{run_date - }') AND toDate('{run_date}') + = """, ) diff --git a/logs/etl_20260618.log b/logs/etl_20260618.log new file mode 100644 index 0000000..6165602 --- /dev/null +++ b/logs/etl_20260618.log @@ -0,0 +1,2134 @@ +2026-06-18 11:27:30 | INFO | ================================================================================ +2026-06-18 11:27:30 | INFO | Hello from data-move Python data pipeline! +2026-06-18 11:27:30 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 11:27:30 | INFO | Connecting to databases... +2026-06-18 11:27:31 | INFO | +2026-06-18 11:27:33 | INFO | +2026-06-18 11:27:33 | INFO | Database connections established +2026-06-18 11:27:33 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 11:27:34 | INFO | Found 836 MIDs +2026-06-18 11:27:35 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-18 11:28:13 | INFO | Fetched 289,830 total empid and visitdate fetched for OQAD from SQL Server +2026-06-18 11:28:13 | INFO | Fetched 4,800 matched empids fetched for OQAD +2026-06-18 11:28:13 | INFO | ================================================================================ +2026-06-18 11:28:13 | INFO | Processing Table: Login +2026-06-18 11:28:13 | INFO | Fetching data for 836 MIDs +2026-06-18 11:28:15 | INFO | Fetched 475 rows from SQL Server +2026-06-18 11:28:15 | INFO | Fetched 475 rows +2026-06-18 11:28:16 | INFO | Deleting from Login +2026-06-18 11:28:16 | INFO | No delete logic required for Login +2026-06-18 11:28:16 | ERROR | Failed processing table Login +Traceback (most recent call last): + File "D:\data_move\main2.py", line 299, in main + load_to_clickhouse( + ~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + table_name=table_name, + ^^^^^^^^^^^^^^^^^^^^^^ + df=df, + ^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse + client.insert_arrow( + ~~~~~~~~~~~~~~~~~~~^ + table=table_name, + ^^^^^^^^^^^^^^^^^ + arrow_table=arrow_table, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow + return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings) + ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert + response = self._raw_request(insert_block, params, headers, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column project_id in table kelloggs_1.Login (3444490f-e300-409d-87dc-fc98fff9e514). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123) +2026-06-18 11:30:44 | INFO | ================================================================================ +2026-06-18 11:30:44 | INFO | Hello from data-move Python data pipeline! +2026-06-18 11:30:44 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 11:30:44 | INFO | Connecting to databases... +2026-06-18 11:30:45 | INFO | +2026-06-18 11:30:46 | INFO | +2026-06-18 11:30:47 | INFO | Database connections established +2026-06-18 11:30:47 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 11:30:48 | INFO | Found 836 MIDs +2026-06-18 11:30:51 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-18 11:32:29 | INFO | Fetched 289,833 total empid and visitdate fetched for OQAD from SQL Server +2026-06-18 11:32:30 | INFO | Fetched 4,800 matched empids fetched for OQAD +2026-06-18 11:32:30 | INFO | ================================================================================ +2026-06-18 11:32:30 | INFO | Processing Table: Login +2026-06-18 11:32:30 | INFO | Fetching data for 836 MIDs +2026-06-18 11:32:37 | INFO | Fetched 475 rows from SQL Server +2026-06-18 11:32:37 | INFO | Fetched 475 rows +2026-06-18 11:32:37 | INFO | Creating table Login +2026-06-18 11:32:37 | INFO | Table ready: Login +2026-06-18 11:32:38 | INFO | Login: inserted 475 rows into ClickHouse +2026-06-18 11:32:38 | INFO | Login loaded successfully (475 rows) +2026-06-18 11:32:38 | INFO | ================================================================================ +2026-06-18 11:32:38 | INFO | Pipeline Completed Successfully +2026-06-18 11:32:38 | INFO | ================================================================================ +2026-06-18 11:49:35 | INFO | ================================================================================ +2026-06-18 11:49:35 | INFO | Hello from data-move Python data pipeline! +2026-06-18 11:49:35 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 11:49:35 | INFO | Connecting to databases... +2026-06-18 11:49:37 | INFO | +2026-06-18 11:49:38 | INFO | +2026-06-18 11:49:39 | INFO | Database connections established +2026-06-18 11:49:39 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 11:49:40 | INFO | Found 836 MIDs +2026-06-18 11:49:40 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-18 11:49:59 | INFO | Fetched 289,843 total empid and visitdate fetched for OQAD from SQL Server +2026-06-18 11:50:00 | INFO | Fetched 4,800 matched empids fetched for OQAD +2026-06-18 11:50:00 | INFO | ================================================================================ +2026-06-18 11:50:00 | INFO | Processing Table: Login +2026-06-18 11:50:00 | INFO | Fetching data for 836 MIDs +2026-06-18 11:50:18 | INFO | Fetched 475 rows from SQL Server +2026-06-18 11:50:18 | INFO | Fetched 475 rows +2026-06-18 11:50:18 | INFO | Deleting from Login +2026-06-18 11:50:19 | INFO | No delete logic required for Login +2026-06-18 11:50:19 | INFO | Login: inserted 475 rows into ClickHouse +2026-06-18 11:50:19 | INFO | Login loaded successfully (475 rows) +2026-06-18 11:50:19 | INFO | ================================================================================ +2026-06-18 11:50:19 | INFO | Pipeline Completed Successfully +2026-06-18 11:50:19 | INFO | ================================================================================ +2026-06-18 11:51:32 | INFO | ================================================================================ +2026-06-18 11:51:32 | INFO | Hello from data-move Python data pipeline! +2026-06-18 11:51:32 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 11:51:32 | INFO | Connecting to databases... +2026-06-18 11:51:34 | INFO | +2026-06-18 11:51:35 | INFO | +2026-06-18 11:51:36 | INFO | Database connections established +2026-06-18 11:51:36 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 11:51:36 | INFO | Found 836 MIDs +2026-06-18 11:51:37 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-18 11:52:03 | INFO | Fetched 289,846 total empid and visitdate fetched for OQAD from SQL Server +2026-06-18 11:52:03 | INFO | Fetched 4,800 matched empids fetched for OQAD +2026-06-18 11:52:03 | INFO | ================================================================================ +2026-06-18 11:52:04 | INFO | Processing Table: Stock_Details +2026-06-18 11:52:04 | INFO | Fetching data for 836 MIDs +2026-06-18 11:52:23 | INFO | Fetched 39,313 rows from SQL Server +2026-06-18 11:52:23 | INFO | Fetched 39313 rows +2026-06-18 11:52:24 | INFO | Deleting from Stock_Details +2026-06-18 11:52:24 | ERROR | Failed processing table Stock_Details +Traceback (most recent call last): + File "D:\data_move\main2.py", line 299, in main + load_to_clickhouse( + ~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + table_name=table_name, + ^^^^^^^^^^^^^^^^^^^^^^ + df=df, + ^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse + client.insert_arrow( + ~~~~~~~~~~~~~~~~~~~^ + table=table_name, + ^^^^^^^^^^^^^^^^^ + arrow_table=arrow_table, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow + return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings) + ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert + response = self._raw_request(insert_block, params, headers, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column project_id in table kelloggs_1.Stock_Details (cf4b6e46-3078-4bfb-9aa3-39357da21740). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123) +2026-06-18 11:55:44 | INFO | ================================================================================ +2026-06-18 11:55:44 | INFO | Hello from data-move Python data pipeline! +2026-06-18 11:55:44 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 11:55:44 | INFO | Connecting to databases... +2026-06-18 11:55:45 | INFO | +2026-06-18 11:55:46 | INFO | +2026-06-18 11:55:46 | INFO | Database connections established +2026-06-18 11:55:46 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 11:55:47 | INFO | Found 836 MIDs +2026-06-18 11:55:47 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-18 11:56:11 | INFO | Fetched 289,856 total empid and visitdate fetched for OQAD from SQL Server +2026-06-18 11:56:11 | INFO | Fetched 4,800 matched empids fetched for OQAD +2026-06-18 11:56:11 | INFO | ================================================================================ +2026-06-18 11:56:11 | INFO | Processing Table: Stock_Details +2026-06-18 11:56:11 | INFO | Fetching data for 836 MIDs +2026-06-18 11:56:26 | INFO | Fetched 39,313 rows from SQL Server +2026-06-18 11:56:26 | INFO | Fetched 39313 rows +2026-06-18 11:56:26 | INFO | Creating table Stock_Details +2026-06-18 11:56:26 | INFO | Table ready: Stock_Details +2026-06-18 11:56:26 | INFO | inserting data into clickhouse db from sql server------------------ +2026-06-18 11:56:27 | INFO | Stock_Details: inserted 39,313 rows into ClickHouse +2026-06-18 11:56:27 | INFO | Stock_Details loaded successfully (39313 rows) +2026-06-18 11:56:27 | INFO | ================================================================================ +2026-06-18 11:56:27 | INFO | Pipeline Completed Successfully +2026-06-18 11:56:27 | INFO | ================================================================================ +2026-06-18 12:41:09 | INFO | ================================================================================ +2026-06-18 12:41:09 | INFO | Hello from data-move Python data pipeline! +2026-06-18 12:41:09 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 12:41:09 | INFO | Connecting to databases... +2026-06-18 12:41:11 | INFO | +2026-06-18 12:41:12 | INFO | +2026-06-18 12:41:13 | INFO | Database connections established +2026-06-18 12:41:13 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 12:41:14 | INFO | Found 836 MIDs +2026-06-18 12:41:15 | INFO | ================================================================================ +2026-06-18 12:41:15 | INFO | Processing Table: SOS_OneApp +2026-06-18 12:41:15 | INFO | Fetching Data from sql server for table-:SOS_OneApp--------------------------- +2026-06-18 12:41:15 | ERROR | Failed processing table SOS_OneApp +Traceback (most recent call last): + File "D:\data_move\main2.py", line 184, in main + df = get_dataframe( + engine=sql_engine, + ...<5 lines>... + run_date=run_date, + ) +TypeError: get_dataframe() got an unexpected keyword argument 'engine' +2026-06-18 12:43:11 | INFO | ================================================================================ +2026-06-18 12:43:11 | INFO | Hello from data-move Python data pipeline! +2026-06-18 12:43:11 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 12:43:11 | INFO | Connecting to databases... +2026-06-18 12:43:13 | INFO | +2026-06-18 12:43:14 | INFO | +2026-06-18 12:43:15 | INFO | Database connections established +2026-06-18 12:43:15 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 12:43:15 | INFO | Found 836 MIDs +2026-06-18 12:43:16 | INFO | ================================================================================ +2026-06-18 12:43:16 | INFO | Processing Table: SOS_OneApp +2026-06-18 12:43:16 | INFO | Fetching Data from sql server for table-:SOS_OneApp--------------------------- +2026-06-18 12:43:16 | INFO | Fetching data for 836 MIDs +2026-06-18 12:43:16 | ERROR | Failed processing table SOS_OneApp +Traceback (most recent call last): + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +pyodbc.Error: ('HY090', '[HY090] [Microsoft][ODBC Driver Manager] Invalid string or buffer length (0) (SQLExecDirectW)') + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "D:\data_move\main2.py", line 184, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 80, in get_dataframe + return fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\data_move\src\fact.py", line 92, in fetch_SOS_OneApp + df = pl.read_database( + query=sql, + connection=engine + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database + return cx.execute( + ~~~~~~~~~~^ + query=query, + ^^^^^^^^^^^^ + options=execute_options, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ).to_polars( + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute + result = cursor_execute(query, **options) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute + return meth( + self, + distilled_parameters, + execution_options or NO_OPTIONS, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection + return connection._execute_clauseelement( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + self, distilled_params, execution_options + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement + ret = self._execute_context( + dialect, + ...<8 lines>... + cache_hit=cache_hit, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context + return self._exec_single_context( + ~~~~~~~~~~~~~~~~~~~~~~~~~^ + dialect, context, statement, parameters + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context + self._handle_dbapi_exception( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + e, str_statement, effective_parameters, cursor, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +sqlalchemy.exc.DBAPIError: (pyodbc.Error) ('HY090', '[HY090] [Microsoft][ODBC Driver Manager] Invalid string or buffer length (0) (SQLExecDirectW)') +(Background on this error at: https://sqlalche.me/e/20/dbapi) +2026-06-18 12:47:32 | INFO | ================================================================================ +2026-06-18 12:47:32 | INFO | Hello from data-move Python data pipeline! +2026-06-18 12:47:32 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 12:47:32 | INFO | Connecting to databases... +2026-06-18 12:47:33 | INFO | +2026-06-18 12:47:35 | INFO | +2026-06-18 12:47:37 | INFO | Database connections established +2026-06-18 12:47:37 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 12:47:37 | INFO | Found 836 MIDs +2026-06-18 12:47:38 | INFO | ================================================================================ +2026-06-18 12:47:38 | INFO | Processing Table: SOS_OneApp +2026-06-18 12:47:38 | INFO | Fetching Data from sql server for table-:SOS_OneApp--------------------------- +2026-06-18 12:47:38 | INFO | Fetching data for 836 MIDs +2026-06-18 12:47:38 | ERROR | Failed processing table SOS_OneApp +Traceback (most recent call last): + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +pyodbc.Error: ('HY090', '[HY090] [Microsoft][ODBC Driver Manager] Invalid string or buffer length (0) (SQLExecDirectW)') + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "D:\data_move\main2.py", line 184, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 80, in get_dataframe + return fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\data_move\src\fact.py", line 92, in fetch_SOS_OneApp + df = pl.read_database( + query=sql, + connection=engine + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database + return cx.execute( + ~~~~~~~~~~^ + query=query, + ^^^^^^^^^^^^ + options=execute_options, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ).to_polars( + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute + result = cursor_execute(query, **options) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute + return meth( + self, + distilled_parameters, + execution_options or NO_OPTIONS, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection + return connection._execute_clauseelement( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + self, distilled_params, execution_options + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement + ret = self._execute_context( + dialect, + ...<8 lines>... + cache_hit=cache_hit, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context + return self._exec_single_context( + ~~~~~~~~~~~~~~~~~~~~~~~~~^ + dialect, context, statement, parameters + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context + self._handle_dbapi_exception( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + e, str_statement, effective_parameters, cursor, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +sqlalchemy.exc.DBAPIError: (pyodbc.Error) ('HY090', '[HY090] [Microsoft][ODBC Driver Manager] Invalid string or buffer length (0) (SQLExecDirectW)') +(Background on this error at: https://sqlalche.me/e/20/dbapi) +2026-06-18 12:54:11 | INFO | ================================================================================ +2026-06-18 12:54:11 | INFO | Hello from data-move Python data pipeline! +2026-06-18 12:54:11 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 12:54:11 | INFO | Connecting to databases... +2026-06-18 12:54:12 | INFO | +2026-06-18 12:54:14 | INFO | +2026-06-18 12:54:15 | INFO | Database connections established +2026-06-18 12:54:15 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 12:54:15 | INFO | Found 836 MIDs +2026-06-18 12:54:16 | INFO | ================================================================================ +2026-06-18 12:54:16 | INFO | Processing Table: SOS_OneApp +2026-06-18 12:54:16 | INFO | Fetching Data from sql server for table-:SOS_OneApp--------------------------- +2026-06-18 12:54:16 | INFO | Fetching data for 836 MIDs +2026-06-18 12:54:16 | ERROR | Failed processing table SOS_OneApp +Traceback (most recent call last): + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +pyodbc.Error: ('HY090', '[HY090] [Microsoft][ODBC Driver Manager] Invalid string or buffer length (0) (SQLExecDirectW)') + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "D:\data_move\main2.py", line 184, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 80, in get_dataframe + return fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\data_move\src\fact.py", line 92, in fetch_SOS_OneApp + df = pl.read_database( + query=sql, + connection=engine + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database + return cx.execute( + ~~~~~~~~~~^ + query=query, + ^^^^^^^^^^^^ + options=execute_options, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ).to_polars( + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute + result = cursor_execute(query, **options) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute + return meth( + self, + distilled_parameters, + execution_options or NO_OPTIONS, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection + return connection._execute_clauseelement( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + self, distilled_params, execution_options + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement + ret = self._execute_context( + dialect, + ...<8 lines>... + cache_hit=cache_hit, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context + return self._exec_single_context( + ~~~~~~~~~~~~~~~~~~~~~~~~~^ + dialect, context, statement, parameters + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context + self._handle_dbapi_exception( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + e, str_statement, effective_parameters, cursor, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +sqlalchemy.exc.DBAPIError: (pyodbc.Error) ('HY090', '[HY090] [Microsoft][ODBC Driver Manager] Invalid string or buffer length (0) (SQLExecDirectW)') +(Background on this error at: https://sqlalche.me/e/20/dbapi) +2026-06-18 12:57:59 | INFO | ================================================================================ +2026-06-18 12:57:59 | INFO | Hello from data-move Python data pipeline! +2026-06-18 12:57:59 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 12:57:59 | INFO | Connecting to databases... +2026-06-18 12:58:00 | INFO | +2026-06-18 12:58:01 | INFO | +2026-06-18 12:58:02 | INFO | Database connections established +2026-06-18 12:58:02 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 12:58:02 | INFO | Found 836 MIDs +2026-06-18 12:58:03 | INFO | ================================================================================ +2026-06-18 12:58:03 | INFO | Processing Table: OQaD +2026-06-18 12:58:03 | INFO | Fetching Data from sql server for table-:OQaD--------------------------- +2026-06-18 12:58:03 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-18 12:58:22 | INFO | Fetched 289,901 total empid and visitdate fetched for OQAD from SQL Server +2026-06-18 12:58:23 | INFO | Fetched 4,800 matched empids fetched for OQAD +2026-06-18 12:58:23 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "D:\data_move\main2.py", line 184, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 80, in get_dataframe + return fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\data_move\src\fact.py", line 284, in fetch_OQaD + df=fetch_OQaD( engine=sql_engine, + table_name=table_name, + ...<2 lines>... + run_date=run_date + ) +TypeError: fetch_OQaD() got an unexpected keyword argument 'engine' +2026-06-18 12:59:46 | INFO | ================================================================================ +2026-06-18 12:59:46 | INFO | Hello from data-move Python data pipeline! +2026-06-18 12:59:46 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 12:59:46 | INFO | Connecting to databases... +2026-06-18 12:59:46 | INFO | +2026-06-18 12:59:48 | INFO | +2026-06-18 12:59:49 | INFO | Database connections established +2026-06-18 12:59:49 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 12:59:49 | INFO | Found 836 MIDs +2026-06-18 12:59:50 | INFO | ================================================================================ +2026-06-18 12:59:50 | INFO | Processing Table: OQaD +2026-06-18 12:59:50 | INFO | Fetching Data from sql server for table-:OQaD--------------------------- +2026-06-18 12:59:50 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-18 13:00:25 | INFO | Fetched 289,901 total empid and visitdate fetched for OQAD from SQL Server +2026-06-18 13:00:25 | INFO | Fetched 4,800 matched empids fetched for OQAD +2026-06-18 13:00:25 | INFO | Exists: True +2026-06-18 13:00:25 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql +2026-06-18 13:00:25 | INFO | Fetching data for 4,800 EMPIDs +2026-06-18 13:00:25 | INFO | Fetching OQaD data for run_date=2026-06-17 +2026-06-18 13:00:40 | INFO | fn name is fetch_OQad ------Fetched 2341 rows +2026-06-18 13:00:40 | INFO | Fetched 2,341 rows from SQL Server +2026-06-18 13:00:40 | INFO | Fetched total row -: 2341 from sql server for table-:OQaD ...........!!! +2026-06-18 13:00:40 | INFO | Fetched 2341 rows +2026-06-18 13:00:41 | INFO | Deleting from OQaD +2026-06-18 13:00:41 | INFO | inserting data into clickhouse db from sql server------------------ +2026-06-18 13:00:41 | INFO | OQaD: inserted 2,341 rows into ClickHouse +2026-06-18 13:00:41 | INFO | OQaD loaded successfully (2341 rows) +2026-06-18 13:00:41 | INFO | ================================================================================ +2026-06-18 13:00:41 | INFO | Processing Table: additional_visibility +2026-06-18 13:00:41 | INFO | Fetching Data from sql server for table-:additional_visibility--------------------------- +2026-06-18 13:00:41 | INFO | Fetching data for 836 MIDs +2026-06-18 13:00:43 | INFO | Fetched 1,885 rows from SQL Server +2026-06-18 13:00:43 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! +2026-06-18 13:00:43 | INFO | Fetched 1885 rows +2026-06-18 13:00:43 | INFO | Deleting from additional_visibility +2026-06-18 13:00:44 | ERROR | Failed processing table additional_visibility +Traceback (most recent call last): + File "D:\data_move\main2.py", line 246, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 81, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + table_name, + ^^^^^^^^^^^ + f"MID IN ({mids_str})", + ^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'MID' while processing: 'isZeroOrNull(MID IN (1060493, 1060752, 1060791, 1060816, 1061143, 1061234, 1061334, 1061336, 1061343, 1061345, 1061352, 1061354, 1061359, 1061361, 1061368, 1061370, 1061377, 1061379, 1061384, 1061386, 1061393, 1061395, 1061402, 1061404, 1061409, 1061411, 1061418, 1061420, 1061427, 1061429, 1061434, 1061436, 1061438, 1061443, 1061445, 1061452, 1061454, 1061459, 1061461, 1061463, 1061468, 1061470, 1061477, 1061479, 1061484, 1061486, 1061488, 1061493, 1061495, 1061502, 1061504, 1061511, 1061513, 1061518, 1061520, 1061527, 1061529, 1061536, 1061538, 1061543, 1061545, 1061552, 1061554, 1061561, 1061563, 1061568, 1061570, 1061577, 1061579, 1061586, 1061588, 1061593, 1061595, 1061602, 1061604, 1061611, 1061613, 1061618, 1061620, 1061627, 1061629, 1061636, 1061638, 1061643, 1061645, 1061652, 1061654, 1061661, 1061663, 1061668, 1061670, 1061677, 1061679, 1061686, 1061688, 1061693, 1061695, 1061697, 1061702, 1061704, 1061711, 1061713, 1061718, 1061720, 1061722, 1061727, 1061729, 1061736, 1061738, 1061745, 1061747, 1061752, 1061754, 1061761, 1061763, 1061770, 1061772, 1061777, 1061779, 1061786, 1061788, 1061795, 1061797, 1061802, 1061804, 1061811, 1061813, 1061820, 1061822, 1061827, 1061829, 1061836, 1061838, 1061845, 1061847, 1061852, 1061854, 1061861, 1061863, 1061870, 1061872, 1061877, 1061879, 1061886, 1061888, 1061895, 1061897, 1061902, 1061904, 1061911, 1061913, 1061920, 1061922, 1061927, 1061929, 1061931, 1061936, 1061938, 1061945, 1061947, 1061952, 1061954, 1061956, 1061961, 1061963, 1061970, 1061972, 1061977, 1061979, 1061981, 1061986, 1061988, 1061995, 1061997, 1062004, 1062006, 1062011, 1062013, 1062020, 1062022, 1062029, 1062031, 1062036, 1062038, 1062045, 1062047, 1062054, 1062056, 1062061, 1062063, 1062070, 1062072, 1062079, 1062081, 1062086, 1062088, 1062095, 1062097, 1062104, 1062106, 1062111, 1062113, 1062120, 1062122, 1062129, 1062131, 1062136, 1062138, 1062145, 1062147, 1060399, 1060526, 1060783, 1061335, 1061337, 1061342, 1061344, 1061346, 1061351, 1061353, 1061360, 1061362, 1061367, 1061369, 1061371, 1061376, 1061378, 1061385, 1061387, 1061394, 1061396, 1061401, 1061403, 1061410, 1061412, 1061419, 1061421, 1061426, 1061428, 1061435, 1061437, 1061444, 1061446, 1061451, 1061453, 1061460, 1061462, 1061469, 1061471, 1061476, 1061478, 1061485, 1061487, 1061494, 1061496, 1061501, 1061503, 1061510, 1061512, 1061519, 1061521, 1061526, 1061528, 1061535, 1061537, 1061544, 1061546, 1061551, 1061553, 1061555, 1061560, 1061562, 1061569, 1061571, 1061576, 1061578, 1061580, 1061585, 1061587, 1061594, 1061596, 1061601, 1061603, 1061605, 1061610, 1061612, 1061619, 1061621, 1061628, 1061630, 1061635, 1061637, 1061644, 1061646, 1061653, 1061655, 1061660, 1061662, 1061669, 1061671, 1061678, 1061680, 1061685, 1061687, 1061694, 1061696, 1061703, 1061705, 1061710, 1061712, 1061719, 1061721, 1061728, 1061730, 1061735, 1061737, 1061744, 1061746, 1061753, 1061755, 1061760, 1061762, 1061769, 1061771, 1061778, 1061780, 1061785, 1061787, 1061794, 1061796, 1061803, 1061805, 1061810, 1061812, 1061814, 1061819, 1061821, 1061828, 1061830, 1061835, 1061837, 1061839, 1061844, 1061846, 1061853, 1061855, 1061862, 1061864, 1061869, 1061871, 1061878, 1061880, 1061887, 1061889, 1061894, 1061896, 1061903, 1061905, 1061912, 1061914, 1061919, 1061921, 1061928, 1061930, 1061937, 1061939, 1061944, 1061946, 1061953, 1061955, 1061962, 1061964, 1061969, 1061971, 1061978, 1061980, 1061987, 1061989, 1061994, 1061996, 1062003, 1062005, 1062012, 1062014, 1062019, 1062021, 1062028, 1062030, 1062037, 1062039, 1062044, 1062046, 1062048, 1062053, 1062055, 1062062, 1062064, 1062069, 1062071, 1062073, 1062078, 1062080, 1062087, 1062089, 1062094, 1062096, 1062098, 1062103, 1062105, 1062112, 1062114, 1062121, 1062123, 1062128, 1062130, 1062137, 1062139, 1062146, 1060060, 1060781, 1061233, 1061331, 1061332, 1061333, 1061347, 1061348, 1061349, 1061350, 1061363, 1061364, 1061365, 1061366, 1061380, 1061381, 1061382, 1061383, 1061397, 1061398, 1061399, 1061400, 1061413, 1061414, 1061415, 1061416, 1061417, 1061430, 1061431, 1061432, 1061433, 1061447, 1061448, 1061449, 1061450, 1061464, 1061465, 1061466, 1061467, 1061480, 1061481, 1061482, 1061483, 1061497, 1061498, 1061499, 1061500, 1061514, 1061515, 1061516, 1061517, 1061530, 1061531, 1061532, 1061533, 1061534, 1061547, 1061548, 1061549, 1061550, 1061564, 1061565, 1061566, 1061567, 1061581, 1061582, 1061583, 1061584, 1061597, 1061598, 1061599, 1061600, 1061614, 1061615, 1061616, 1061617, 1061631, 1061632, 1061633, 1061634, 1061647, 1061648, 1061649, 1061650, 1061651, 1061664, 1061665, 1061666, 1061667, 1061681, 1061682, 1061683, 1061684, 1061698, 1061699, 1061700, 1061701, 1061714, 1061715, 1061716, 1061717, 1061731, 1061732, 1061733, 1061734, 1061748, 1061749, 1061750, 1061751, 1061764, 1061765, 1061766, 1061767, 1061768, 1061781, 1061782, 1061783, 1061784, 1061798, 1061799, 1061800, 1061801, 1061815, 1061816, 1061817, 1061818, 1061831, 1061832, 1061833, 1061834, 1061848, 1061849, 1061850, 1061851, 1061865, 1061866, 1061867, 1061868, 1061881, 1061882, 1061883, 1061884, 1061885, 1061898, 1061899, 1061900, 1061901, 1061915, 1061916, 1061917, 1061918, 1061932, 1061933, 1061934, 1061935, 1061948, 1061949, 1061950, 1061951, 1061965, 1061966, 1061967, 1061968, 1061982, 1061983, 1061984, 1061985, 1061998, 1061999, 1062000, 1062001, 1062002, 1062015, 1062016, 1062017, 1062018, 1062032, 1062033, 1062034, 1062035, 1062049, 1062050, 1062051, 1062052, 1062065, 1062066, 1062067, 1062068, 1062082, 1062083, 1062084, 1062085, 1062099, 1062100, 1062101, 1062102, 1062115, 1062116, 1062117, 1062118, 1062119, 1062132, 1062133, 1062134, 1062135, 1060487, 1060555, 1060637, 1060656, 1061071, 1061238, 1061256, 1061338, 1061339, 1061340, 1061341, 1061355, 1061356, 1061357, 1061358, 1061372, 1061373, 1061374, 1061375, 1061388, 1061389, 1061390, 1061391, 1061392, 1061405, 1061406, 1061407, 1061408, 1061422, 1061423, 1061424, 1061425, 1061439, 1061440, 1061441, 1061442, 1061455, 1061456, 1061457, 1061458, 1061472, 1061473, 1061474, 1061475, 1061489, 1061490, 1061491, 1061492, 1061505, 1061506, 1061507, 1061508, 1061509, 1061522, 1061523, 1061524, 1061525, 1061539, 1061540, 1061541, 1061542, 1061556, 1061557, 1061558, 1061559, 1061572, 1061573, 1061574, 1061575, 1061589, 1061590, 1061591, 1061592, 1061606, 1061607, 1061608, 1061609, 1061622, 1061623, 1061624, 1061625, 1061626, 1061639, 1061640, 1061641, 1061642, 1061656, 1061657, 1061658, 1061659, 1061672, 1061673, 1061674, 1061675, 1061676, 1061689, 1061690, 1061691, 1061692, 1061706, 1061707, 1061708, 1061709, 1061723, 1061724, 1061725, 1061726, 1061739, 1061740, 1061741, 1061742, 1061743, 1061756, 1061757, 1061758, 1061759, 1061773, 1061774, 1061775, 1061776, 1061789, 1061790, 1061791, 1061792, 1061793, 1061806, 1061807, 1061808, 1061809, 1061823, 1061824, 1061825, 1061826, 1061840, 1061841, 1061842, 1061843, 1061856, 1061857, 1061858, 1061859, 1061860, 1061873, 1061874, 1061875, 1061876, 1061890, 1061891, 1061892, 1061893, 1061906, 1061907, 1061908, 1061909, 1061910, 1061923, 1061924, 1061925, 1061926, 1061940, 1061941, 1061942, 1061943, 1061957, 1061958, 1061959, 1061960, 1061973, 1061974, 1061975, 1061976, 1061990, 1061991, 1061992, 1061993, 1062007, 1062008, 1062009, 1062010, 1062023, 1062024, 1062025, 1062026, 1062027, 1062040, 1062041, 1062042, 1062043, 1062057, 1062058, 1062059, 1062060, 1062074, 1062075, 1062076, 1062077, 1062090, 1062091, 1062092, 1062093, 1062107, 1062108, 1062109, 1062110, 1062124, 1062125, 1062126, 1062127, 1062140, 1062141, 1062142, 1062143, 1062144)), _block_offset, _block_number, _table, _database, _distance, _disk_name, _part_granule_offset, _partition_id, _sample_factor, brand_id, _part_uuid, _part_index, _part, created_by, storetype_id, created_date, camera_allowed, image_url, display_id, Remarks, is_present, _part_data_version, _part_starting_offset, visit_date, Mid, channel_id, store_id, chain_id, _part_offset, emp_id, project_id', required columns: 'MID' '_part_granule_offset' '_block_number' '_table' '_block_offset' '_database' '_distance' 'Remarks' 'is_present' '_disk_name' '_partition_id' '_sample_factor' 'display_id' 'brand_id' '_part_uuid' '_part_index' '_part' 'created_by' 'created_date' 'storetype_id' 'camera_allowed' 'image_url' '_part_data_version' '_part_starting_offset' 'visit_date' 'Mid' 'channel_id' 'store_id' 'chain_id' '_part_offset' 'emp_id' 'project_id', maybe you meant: 'Mid', 'Remarks', 'is_present', 'display_id', 'brand_id', 'created_by', 'created_date', 'storetype_id', 'camera_allowed', 'image_url', 'visit_date', 'channel_id', 'store_id', 'chain_id', 'emp_id' or 'project_id'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) +2026-06-18 13:05:16 | INFO | ================================================================================ +2026-06-18 13:05:16 | INFO | Hello from data-move Python data pipeline! +2026-06-18 13:05:16 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 13:05:16 | INFO | Connecting to databases... +2026-06-18 13:05:17 | INFO | +2026-06-18 13:05:19 | INFO | +2026-06-18 13:05:20 | INFO | Database connections established +2026-06-18 13:05:20 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 13:05:21 | INFO | Found 836 MIDs +2026-06-18 13:05:22 | INFO | ================================================================================ +2026-06-18 13:05:22 | INFO | Processing Table: additional_visibility Table type is -: FACT Based on mids and operation is used INSERT +2026-06-18 13:05:22 | INFO | Fetching Data from sql server for table-:additional_visibility--------------------------- +2026-06-18 13:05:22 | INFO | Fetching data for 836 MIDs +2026-06-18 13:05:25 | INFO | Fetched 1,885 rows from SQL Server +2026-06-18 13:05:25 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! +2026-06-18 13:05:25 | INFO | Fetched 1885 rows +2026-06-18 13:05:25 | INFO | Deleting from additional_visibility +2026-06-18 13:05:25 | ERROR | Failed processing table additional_visibility +Traceback (most recent call last): + File "D:\data_move\main2.py", line 243, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 81, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + table_name, + ^^^^^^^^^^^ + f"MID IN ({mids_str})", + ^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'MID' while processing: 'isZeroOrNull(MID IN (1060060, 1060781, 1061233, 1061331, 1061332, 1061333, 1061347, 1061348, 1061349, 1061350, 1061363, 1061364, 1061365, 1061366, 1061380, 1061381, 1061382, 1061383, 1061397, 1061398, 1061399, 1061400, 1061413, 1061414, 1061415, 1061416, 1061417, 1061430, 1061431, 1061432, 1061433, 1061447, 1061448, 1061449, 1061450, 1061464, 1061465, 1061466, 1061467, 1061480, 1061481, 1061482, 1061483, 1061497, 1061498, 1061499, 1061500, 1061514, 1061515, 1061516, 1061517, 1061530, 1061531, 1061532, 1061533, 1061534, 1061547, 1061548, 1061549, 1061550, 1061564, 1061565, 1061566, 1061567, 1061581, 1061582, 1061583, 1061584, 1061597, 1061598, 1061599, 1061600, 1061614, 1061615, 1061616, 1061617, 1061631, 1061632, 1061633, 1061634, 1061647, 1061648, 1061649, 1061650, 1061651, 1061664, 1061665, 1061666, 1061667, 1061681, 1061682, 1061683, 1061684, 1061698, 1061699, 1061700, 1061701, 1061714, 1061715, 1061716, 1061717, 1061731, 1061732, 1061733, 1061734, 1061748, 1061749, 1061750, 1061751, 1061764, 1061765, 1061766, 1061767, 1061768, 1061781, 1061782, 1061783, 1061784, 1061798, 1061799, 1061800, 1061801, 1061815, 1061816, 1061817, 1061818, 1061831, 1061832, 1061833, 1061834, 1061848, 1061849, 1061850, 1061851, 1061865, 1061866, 1061867, 1061868, 1061881, 1061882, 1061883, 1061884, 1061885, 1061898, 1061899, 1061900, 1061901, 1061915, 1061916, 1061917, 1061918, 1061932, 1061933, 1061934, 1061935, 1061948, 1061949, 1061950, 1061951, 1061965, 1061966, 1061967, 1061968, 1061982, 1061983, 1061984, 1061985, 1061998, 1061999, 1062000, 1062001, 1062002, 1062015, 1062016, 1062017, 1062018, 1062032, 1062033, 1062034, 1062035, 1062049, 1062050, 1062051, 1062052, 1062065, 1062066, 1062067, 1062068, 1062082, 1062083, 1062084, 1062085, 1062099, 1062100, 1062101, 1062102, 1062115, 1062116, 1062117, 1062118, 1062119, 1062132, 1062133, 1062134, 1062135, 1060399, 1060526, 1060783, 1061335, 1061337, 1061342, 1061344, 1061346, 1061351, 1061353, 1061360, 1061362, 1061367, 1061369, 1061371, 1061376, 1061378, 1061385, 1061387, 1061394, 1061396, 1061401, 1061403, 1061410, 1061412, 1061419, 1061421, 1061426, 1061428, 1061435, 1061437, 1061444, 1061446, 1061451, 1061453, 1061460, 1061462, 1061469, 1061471, 1061476, 1061478, 1061485, 1061487, 1061494, 1061496, 1061501, 1061503, 1061510, 1061512, 1061519, 1061521, 1061526, 1061528, 1061535, 1061537, 1061544, 1061546, 1061551, 1061553, 1061555, 1061560, 1061562, 1061569, 1061571, 1061576, 1061578, 1061580, 1061585, 1061587, 1061594, 1061596, 1061601, 1061603, 1061605, 1061610, 1061612, 1061619, 1061621, 1061628, 1061630, 1061635, 1061637, 1061644, 1061646, 1061653, 1061655, 1061660, 1061662, 1061669, 1061671, 1061678, 1061680, 1061685, 1061687, 1061694, 1061696, 1061703, 1061705, 1061710, 1061712, 1061719, 1061721, 1061728, 1061730, 1061735, 1061737, 1061744, 1061746, 1061753, 1061755, 1061760, 1061762, 1061769, 1061771, 1061778, 1061780, 1061785, 1061787, 1061794, 1061796, 1061803, 1061805, 1061810, 1061812, 1061814, 1061819, 1061821, 1061828, 1061830, 1061835, 1061837, 1061839, 1061844, 1061846, 1061853, 1061855, 1061862, 1061864, 1061869, 1061871, 1061878, 1061880, 1061887, 1061889, 1061894, 1061896, 1061903, 1061905, 1061912, 1061914, 1061919, 1061921, 1061928, 1061930, 1061937, 1061939, 1061944, 1061946, 1061953, 1061955, 1061962, 1061964, 1061969, 1061971, 1061978, 1061980, 1061987, 1061989, 1061994, 1061996, 1062003, 1062005, 1062012, 1062014, 1062019, 1062021, 1062028, 1062030, 1062037, 1062039, 1062044, 1062046, 1062048, 1062053, 1062055, 1062062, 1062064, 1062069, 1062071, 1062073, 1062078, 1062080, 1062087, 1062089, 1062094, 1062096, 1062098, 1062103, 1062105, 1062112, 1062114, 1062121, 1062123, 1062128, 1062130, 1062137, 1062139, 1062146, 1060493, 1060752, 1060791, 1060816, 1061143, 1061234, 1061334, 1061336, 1061343, 1061345, 1061352, 1061354, 1061359, 1061361, 1061368, 1061370, 1061377, 1061379, 1061384, 1061386, 1061393, 1061395, 1061402, 1061404, 1061409, 1061411, 1061418, 1061420, 1061427, 1061429, 1061434, 1061436, 1061438, 1061443, 1061445, 1061452, 1061454, 1061459, 1061461, 1061463, 1061468, 1061470, 1061477, 1061479, 1061484, 1061486, 1061488, 1061493, 1061495, 1061502, 1061504, 1061511, 1061513, 1061518, 1061520, 1061527, 1061529, 1061536, 1061538, 1061543, 1061545, 1061552, 1061554, 1061561, 1061563, 1061568, 1061570, 1061577, 1061579, 1061586, 1061588, 1061593, 1061595, 1061602, 1061604, 1061611, 1061613, 1061618, 1061620, 1061627, 1061629, 1061636, 1061638, 1061643, 1061645, 1061652, 1061654, 1061661, 1061663, 1061668, 1061670, 1061677, 1061679, 1061686, 1061688, 1061693, 1061695, 1061697, 1061702, 1061704, 1061711, 1061713, 1061718, 1061720, 1061722, 1061727, 1061729, 1061736, 1061738, 1061745, 1061747, 1061752, 1061754, 1061761, 1061763, 1061770, 1061772, 1061777, 1061779, 1061786, 1061788, 1061795, 1061797, 1061802, 1061804, 1061811, 1061813, 1061820, 1061822, 1061827, 1061829, 1061836, 1061838, 1061845, 1061847, 1061852, 1061854, 1061861, 1061863, 1061870, 1061872, 1061877, 1061879, 1061886, 1061888, 1061895, 1061897, 1061902, 1061904, 1061911, 1061913, 1061920, 1061922, 1061927, 1061929, 1061931, 1061936, 1061938, 1061945, 1061947, 1061952, 1061954, 1061956, 1061961, 1061963, 1061970, 1061972, 1061977, 1061979, 1061981, 1061986, 1061988, 1061995, 1061997, 1062004, 1062006, 1062011, 1062013, 1062020, 1062022, 1062029, 1062031, 1062036, 1062038, 1062045, 1062047, 1062054, 1062056, 1062061, 1062063, 1062070, 1062072, 1062079, 1062081, 1062086, 1062088, 1062095, 1062097, 1062104, 1062106, 1062111, 1062113, 1062120, 1062122, 1062129, 1062131, 1062136, 1062138, 1062145, 1062147, 1060487, 1060555, 1060637, 1060656, 1061071, 1061238, 1061256, 1061338, 1061339, 1061340, 1061341, 1061355, 1061356, 1061357, 1061358, 1061372, 1061373, 1061374, 1061375, 1061388, 1061389, 1061390, 1061391, 1061392, 1061405, 1061406, 1061407, 1061408, 1061422, 1061423, 1061424, 1061425, 1061439, 1061440, 1061441, 1061442, 1061455, 1061456, 1061457, 1061458, 1061472, 1061473, 1061474, 1061475, 1061489, 1061490, 1061491, 1061492, 1061505, 1061506, 1061507, 1061508, 1061509, 1061522, 1061523, 1061524, 1061525, 1061539, 1061540, 1061541, 1061542, 1061556, 1061557, 1061558, 1061559, 1061572, 1061573, 1061574, 1061575, 1061589, 1061590, 1061591, 1061592, 1061606, 1061607, 1061608, 1061609, 1061622, 1061623, 1061624, 1061625, 1061626, 1061639, 1061640, 1061641, 1061642, 1061656, 1061657, 1061658, 1061659, 1061672, 1061673, 1061674, 1061675, 1061676, 1061689, 1061690, 1061691, 1061692, 1061706, 1061707, 1061708, 1061709, 1061723, 1061724, 1061725, 1061726, 1061739, 1061740, 1061741, 1061742, 1061743, 1061756, 1061757, 1061758, 1061759, 1061773, 1061774, 1061775, 1061776, 1061789, 1061790, 1061791, 1061792, 1061793, 1061806, 1061807, 1061808, 1061809, 1061823, 1061824, 1061825, 1061826, 1061840, 1061841, 1061842, 1061843, 1061856, 1061857, 1061858, 1061859, 1061860, 1061873, 1061874, 1061875, 1061876, 1061890, 1061891, 1061892, 1061893, 1061906, 1061907, 1061908, 1061909, 1061910, 1061923, 1061924, 1061925, 1061926, 1061940, 1061941, 1061942, 1061943, 1061957, 1061958, 1061959, 1061960, 1061973, 1061974, 1061975, 1061976, 1061990, 1061991, 1061992, 1061993, 1062007, 1062008, 1062009, 1062010, 1062023, 1062024, 1062025, 1062026, 1062027, 1062040, 1062041, 1062042, 1062043, 1062057, 1062058, 1062059, 1062060, 1062074, 1062075, 1062076, 1062077, 1062090, 1062091, 1062092, 1062093, 1062107, 1062108, 1062109, 1062110, 1062124, 1062125, 1062126, 1062127, 1062140, 1062141, 1062142, 1062143, 1062144)), _block_offset, _block_number, _table, _database, _distance, _disk_name, _part_granule_offset, _partition_id, _sample_factor, brand_id, _part_uuid, _part_index, _part, created_by, storetype_id, created_date, camera_allowed, image_url, display_id, Remarks, is_present, _part_data_version, _part_starting_offset, visit_date, Mid, channel_id, store_id, chain_id, _part_offset, emp_id, project_id', required columns: 'MID' '_part_granule_offset' '_block_number' '_table' '_block_offset' '_database' '_distance' 'Remarks' 'is_present' '_disk_name' '_partition_id' '_sample_factor' 'display_id' 'brand_id' '_part_uuid' '_part_index' '_part' 'created_by' 'created_date' 'storetype_id' 'camera_allowed' 'image_url' '_part_data_version' '_part_starting_offset' 'visit_date' 'Mid' 'channel_id' 'store_id' 'chain_id' '_part_offset' 'emp_id' 'project_id', maybe you meant: 'Mid', 'Remarks', 'is_present', 'display_id', 'brand_id', 'created_by', 'created_date', 'storetype_id', 'camera_allowed', 'image_url', 'visit_date', 'channel_id', 'store_id', 'chain_id', 'emp_id' or 'project_id'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) +2026-06-18 13:11:42 | INFO | ================================================================================ +2026-06-18 13:11:42 | INFO | Hello from data-move Python data pipeline! +2026-06-18 13:11:42 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 13:11:42 | INFO | Connecting to databases... +2026-06-18 13:11:43 | INFO | +2026-06-18 13:11:45 | INFO | +2026-06-18 13:11:45 | INFO | Database connections established +2026-06-18 13:11:45 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 13:11:46 | INFO | Found 836 MIDs +2026-06-18 13:11:47 | INFO | ================================================================================ +2026-06-18 13:11:47 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:11:47 | INFO | Fetching Data from sql server for table-:additional_visibility--------------------------- +2026-06-18 13:11:47 | INFO | Fetching data for 836 MIDs +2026-06-18 13:11:51 | INFO | Fetched 1,885 rows from SQL Server +2026-06-18 13:11:51 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! +2026-06-18 13:11:51 | INFO | Fetched 1885 rows +2026-06-18 13:11:51 | INFO | Deleting from additional_visibility +2026-06-18 13:11:51 | ERROR | Failed processing table additional_visibility +Traceback (most recent call last): + File "D:\data_move\main2.py", line 243, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 81, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + table_name, + ^^^^^^^^^^^ + f"MID IN ({mids_str})", + ^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'MID' while processing: 'isZeroOrNull(MID IN (1060060, 1060781, 1061233, 1061331, 1061332, 1061333, 1061347, 1061348, 1061349, 1061350, 1061363, 1061364, 1061365, 1061366, 1061380, 1061381, 1061382, 1061383, 1061397, 1061398, 1061399, 1061400, 1061413, 1061414, 1061415, 1061416, 1061417, 1061430, 1061431, 1061432, 1061433, 1061447, 1061448, 1061449, 1061450, 1061464, 1061465, 1061466, 1061467, 1061480, 1061481, 1061482, 1061483, 1061497, 1061498, 1061499, 1061500, 1061514, 1061515, 1061516, 1061517, 1061530, 1061531, 1061532, 1061533, 1061534, 1061547, 1061548, 1061549, 1061550, 1061564, 1061565, 1061566, 1061567, 1061581, 1061582, 1061583, 1061584, 1061597, 1061598, 1061599, 1061600, 1061614, 1061615, 1061616, 1061617, 1061631, 1061632, 1061633, 1061634, 1061647, 1061648, 1061649, 1061650, 1061651, 1061664, 1061665, 1061666, 1061667, 1061681, 1061682, 1061683, 1061684, 1061698, 1061699, 1061700, 1061701, 1061714, 1061715, 1061716, 1061717, 1061731, 1061732, 1061733, 1061734, 1061748, 1061749, 1061750, 1061751, 1061764, 1061765, 1061766, 1061767, 1061768, 1061781, 1061782, 1061783, 1061784, 1061798, 1061799, 1061800, 1061801, 1061815, 1061816, 1061817, 1061818, 1061831, 1061832, 1061833, 1061834, 1061848, 1061849, 1061850, 1061851, 1061865, 1061866, 1061867, 1061868, 1061881, 1061882, 1061883, 1061884, 1061885, 1061898, 1061899, 1061900, 1061901, 1061915, 1061916, 1061917, 1061918, 1061932, 1061933, 1061934, 1061935, 1061948, 1061949, 1061950, 1061951, 1061965, 1061966, 1061967, 1061968, 1061982, 1061983, 1061984, 1061985, 1061998, 1061999, 1062000, 1062001, 1062002, 1062015, 1062016, 1062017, 1062018, 1062032, 1062033, 1062034, 1062035, 1062049, 1062050, 1062051, 1062052, 1062065, 1062066, 1062067, 1062068, 1062082, 1062083, 1062084, 1062085, 1062099, 1062100, 1062101, 1062102, 1062115, 1062116, 1062117, 1062118, 1062119, 1062132, 1062133, 1062134, 1062135, 1060399, 1060526, 1060783, 1061335, 1061337, 1061342, 1061344, 1061346, 1061351, 1061353, 1061360, 1061362, 1061367, 1061369, 1061371, 1061376, 1061378, 1061385, 1061387, 1061394, 1061396, 1061401, 1061403, 1061410, 1061412, 1061419, 1061421, 1061426, 1061428, 1061435, 1061437, 1061444, 1061446, 1061451, 1061453, 1061460, 1061462, 1061469, 1061471, 1061476, 1061478, 1061485, 1061487, 1061494, 1061496, 1061501, 1061503, 1061510, 1061512, 1061519, 1061521, 1061526, 1061528, 1061535, 1061537, 1061544, 1061546, 1061551, 1061553, 1061555, 1061560, 1061562, 1061569, 1061571, 1061576, 1061578, 1061580, 1061585, 1061587, 1061594, 1061596, 1061601, 1061603, 1061605, 1061610, 1061612, 1061619, 1061621, 1061628, 1061630, 1061635, 1061637, 1061644, 1061646, 1061653, 1061655, 1061660, 1061662, 1061669, 1061671, 1061678, 1061680, 1061685, 1061687, 1061694, 1061696, 1061703, 1061705, 1061710, 1061712, 1061719, 1061721, 1061728, 1061730, 1061735, 1061737, 1061744, 1061746, 1061753, 1061755, 1061760, 1061762, 1061769, 1061771, 1061778, 1061780, 1061785, 1061787, 1061794, 1061796, 1061803, 1061805, 1061810, 1061812, 1061814, 1061819, 1061821, 1061828, 1061830, 1061835, 1061837, 1061839, 1061844, 1061846, 1061853, 1061855, 1061862, 1061864, 1061869, 1061871, 1061878, 1061880, 1061887, 1061889, 1061894, 1061896, 1061903, 1061905, 1061912, 1061914, 1061919, 1061921, 1061928, 1061930, 1061937, 1061939, 1061944, 1061946, 1061953, 1061955, 1061962, 1061964, 1061969, 1061971, 1061978, 1061980, 1061987, 1061989, 1061994, 1061996, 1062003, 1062005, 1062012, 1062014, 1062019, 1062021, 1062028, 1062030, 1062037, 1062039, 1062044, 1062046, 1062048, 1062053, 1062055, 1062062, 1062064, 1062069, 1062071, 1062073, 1062078, 1062080, 1062087, 1062089, 1062094, 1062096, 1062098, 1062103, 1062105, 1062112, 1062114, 1062121, 1062123, 1062128, 1062130, 1062137, 1062139, 1062146, 1060487, 1060555, 1060637, 1060656, 1061071, 1061238, 1061256, 1061338, 1061339, 1061340, 1061341, 1061355, 1061356, 1061357, 1061358, 1061372, 1061373, 1061374, 1061375, 1061388, 1061389, 1061390, 1061391, 1061392, 1061405, 1061406, 1061407, 1061408, 1061422, 1061423, 1061424, 1061425, 1061439, 1061440, 1061441, 1061442, 1061455, 1061456, 1061457, 1061458, 1061472, 1061473, 1061474, 1061475, 1061489, 1061490, 1061491, 1061492, 1061505, 1061506, 1061507, 1061508, 1061509, 1061522, 1061523, 1061524, 1061525, 1061539, 1061540, 1061541, 1061542, 1061556, 1061557, 1061558, 1061559, 1061572, 1061573, 1061574, 1061575, 1061589, 1061590, 1061591, 1061592, 1061606, 1061607, 1061608, 1061609, 1061622, 1061623, 1061624, 1061625, 1061626, 1061639, 1061640, 1061641, 1061642, 1061656, 1061657, 1061658, 1061659, 1061672, 1061673, 1061674, 1061675, 1061676, 1061689, 1061690, 1061691, 1061692, 1061706, 1061707, 1061708, 1061709, 1061723, 1061724, 1061725, 1061726, 1061739, 1061740, 1061741, 1061742, 1061743, 1061756, 1061757, 1061758, 1061759, 1061773, 1061774, 1061775, 1061776, 1061789, 1061790, 1061791, 1061792, 1061793, 1061806, 1061807, 1061808, 1061809, 1061823, 1061824, 1061825, 1061826, 1061840, 1061841, 1061842, 1061843, 1061856, 1061857, 1061858, 1061859, 1061860, 1061873, 1061874, 1061875, 1061876, 1061890, 1061891, 1061892, 1061893, 1061906, 1061907, 1061908, 1061909, 1061910, 1061923, 1061924, 1061925, 1061926, 1061940, 1061941, 1061942, 1061943, 1061957, 1061958, 1061959, 1061960, 1061973, 1061974, 1061975, 1061976, 1061990, 1061991, 1061992, 1061993, 1062007, 1062008, 1062009, 1062010, 1062023, 1062024, 1062025, 1062026, 1062027, 1062040, 1062041, 1062042, 1062043, 1062057, 1062058, 1062059, 1062060, 1062074, 1062075, 1062076, 1062077, 1062090, 1062091, 1062092, 1062093, 1062107, 1062108, 1062109, 1062110, 1062124, 1062125, 1062126, 1062127, 1062140, 1062141, 1062142, 1062143, 1062144, 1060493, 1060752, 1060791, 1060816, 1061143, 1061234, 1061334, 1061336, 1061343, 1061345, 1061352, 1061354, 1061359, 1061361, 1061368, 1061370, 1061377, 1061379, 1061384, 1061386, 1061393, 1061395, 1061402, 1061404, 1061409, 1061411, 1061418, 1061420, 1061427, 1061429, 1061434, 1061436, 1061438, 1061443, 1061445, 1061452, 1061454, 1061459, 1061461, 1061463, 1061468, 1061470, 1061477, 1061479, 1061484, 1061486, 1061488, 1061493, 1061495, 1061502, 1061504, 1061511, 1061513, 1061518, 1061520, 1061527, 1061529, 1061536, 1061538, 1061543, 1061545, 1061552, 1061554, 1061561, 1061563, 1061568, 1061570, 1061577, 1061579, 1061586, 1061588, 1061593, 1061595, 1061602, 1061604, 1061611, 1061613, 1061618, 1061620, 1061627, 1061629, 1061636, 1061638, 1061643, 1061645, 1061652, 1061654, 1061661, 1061663, 1061668, 1061670, 1061677, 1061679, 1061686, 1061688, 1061693, 1061695, 1061697, 1061702, 1061704, 1061711, 1061713, 1061718, 1061720, 1061722, 1061727, 1061729, 1061736, 1061738, 1061745, 1061747, 1061752, 1061754, 1061761, 1061763, 1061770, 1061772, 1061777, 1061779, 1061786, 1061788, 1061795, 1061797, 1061802, 1061804, 1061811, 1061813, 1061820, 1061822, 1061827, 1061829, 1061836, 1061838, 1061845, 1061847, 1061852, 1061854, 1061861, 1061863, 1061870, 1061872, 1061877, 1061879, 1061886, 1061888, 1061895, 1061897, 1061902, 1061904, 1061911, 1061913, 1061920, 1061922, 1061927, 1061929, 1061931, 1061936, 1061938, 1061945, 1061947, 1061952, 1061954, 1061956, 1061961, 1061963, 1061970, 1061972, 1061977, 1061979, 1061981, 1061986, 1061988, 1061995, 1061997, 1062004, 1062006, 1062011, 1062013, 1062020, 1062022, 1062029, 1062031, 1062036, 1062038, 1062045, 1062047, 1062054, 1062056, 1062061, 1062063, 1062070, 1062072, 1062079, 1062081, 1062086, 1062088, 1062095, 1062097, 1062104, 1062106, 1062111, 1062113, 1062120, 1062122, 1062129, 1062131, 1062136, 1062138, 1062145, 1062147)), _block_offset, _block_number, _table, _database, _distance, _disk_name, _part_granule_offset, _partition_id, _sample_factor, brand_id, _part_uuid, _part_index, _part, created_by, storetype_id, created_date, camera_allowed, image_url, display_id, Remarks, is_present, _part_data_version, _part_starting_offset, visit_date, Mid, channel_id, store_id, chain_id, _part_offset, emp_id, project_id', required columns: 'MID' '_part_granule_offset' '_block_number' '_table' '_block_offset' '_database' '_distance' 'Remarks' 'is_present' '_disk_name' '_partition_id' '_sample_factor' 'display_id' 'brand_id' '_part_uuid' '_part_index' '_part' 'created_by' 'created_date' 'storetype_id' 'camera_allowed' 'image_url' '_part_data_version' '_part_starting_offset' 'visit_date' 'Mid' 'channel_id' 'store_id' 'chain_id' '_part_offset' 'emp_id' 'project_id', maybe you meant: 'Mid', 'Remarks', 'is_present', 'display_id', 'brand_id', 'created_by', 'created_date', 'storetype_id', 'camera_allowed', 'image_url', 'visit_date', 'channel_id', 'store_id', 'chain_id', 'emp_id' or 'project_id'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) +2026-06-18 13:15:48 | INFO | ================================================================================ +2026-06-18 13:15:48 | INFO | Hello from data-move Python data pipeline! +2026-06-18 13:15:48 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 13:15:48 | INFO | Connecting to databases... +2026-06-18 13:15:49 | INFO | +2026-06-18 13:15:51 | INFO | +2026-06-18 13:15:51 | INFO | Database connections established +2026-06-18 13:15:51 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 13:15:52 | INFO | Found 836 MIDs +2026-06-18 13:15:52 | INFO | ================================================================================ +2026-06-18 13:15:52 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:15:52 | INFO | Fetching Data from sql server for table-:additional_visibility--------------------------- +2026-06-18 13:15:52 | INFO | Fetching data for 836 MIDs +2026-06-18 13:15:54 | INFO | Fetched 1,885 rows from SQL Server +2026-06-18 13:15:54 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! +2026-06-18 13:15:54 | INFO | Fetched 1885 rows +2026-06-18 13:15:54 | INFO | Deleting from additional_visibility +2026-06-18 13:15:54 | INFO | inserting data into clickhouse db from sql server------------------ +2026-06-18 13:15:54 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse +2026-06-18 13:15:54 | INFO | additional_visibility loaded successfully (1885 rows) +2026-06-18 13:15:54 | INFO | ================================================================================ +2026-06-18 13:15:54 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:15:54 | INFO | Fetching Data from sql server for table-:Coverage--------------------------- +2026-06-18 13:15:54 | INFO | Fetching data for 836 MIDs +2026-06-18 13:15:54 | ERROR | Failed processing table Coverage +Traceback (most recent call last): + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'as'. (156) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ','. (102)") + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "D:\data_move\main2.py", line 181, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 80, in get_dataframe + return fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\data_move\src\fact.py", line 363, in fetch_Coverage + df = pl.read_database( + query=sql, + connection=engine + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database + return cx.execute( + ~~~~~~~~~~^ + query=query, + ^^^^^^^^^^^^ + options=execute_options, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ).to_polars( + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute + result = cursor_execute(query, **options) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute + return meth( + self, + distilled_parameters, + execution_options or NO_OPTIONS, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection + return connection._execute_clauseelement( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + self, distilled_params, execution_options + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement + ret = self._execute_context( + dialect, + ...<8 lines>... + cache_hit=cache_hit, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context + return self._exec_single_context( + ~~~~~~~~~~~~~~~~~~~~~~~~~^ + dialect, context, statement, parameters + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context + self._handle_dbapi_exception( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + e, str_statement, effective_parameters, cursor, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'as'. (156) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ','. (102)") +[SQL: +Executed as +(select JP.mid, CASE WHEN isnull(B.MID,'')<>'' then 'Y' +ELSE 'N' END AS COVERED, +CASE WHEN isnull(C.MID,'')<>'' then 'Y' +ELSE 'N' END AS EXECUTED,isnull(NR.REASON,'') AS REASON + +from OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) left join +(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) +inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B + ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) + AND JP.ReasonId in (0,1,3,9,10,19,20))b +on JP.MID=b.MID + +left join +(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) +inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B + ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) + AND JP.ReasonId in (0,19))c +on JP.MID=c.MID +left join +OneApp_KelloggsMT.dbo.Master_NonWorkingReason nr on JP.ReasonId=nr.ReasonId +Where 1=1 AND JP.MID in (1060487,1060555,1060637,1060656,1061071,1061238,1061256,1061338,1061339,1061340,1061341,1061355,1061356,1061357,1061358,1061372,1061373,1061374,1061375,1061388,1061389,1061390,1061391,1061392,1061405,1061406,1061407,1061408,1061422,1061423,1061424,1061425,1061439,1061440,1061441,1061442,1061455,1061456,1061457,1061458,1061472,1061473,1061474,1061475,1061489,1061490,1061491,1061492,1061505,1061506,1061507,1061508,1061509,1061522,1061523,1061524,1061525,1061539,1061540,1061541,1061542,1061556,1061557,1061558,1061559,1061572,1061573,1061574,1061575,1061589,1061590,1061591,1061592,1061606,1061607,1061608,1061609,1061622,1061623,1061624,1061625,1061626,1061639,1061640,1061641,1061642,1061656,1061657,1061658,1061659,1061672,1061673,1061674,1061675,1061676,1061689,1061690,1061691,1061692,1061706,1061707,1061708,1061709,1061723,1061724,1061725,1061726,1061739,1061740,1061741,1061742,1061743,1061756,1061757,1061758,1061759,1061773,1061774,1061775,1061776,1061789,1061790,1061791,1061792,1061793,1061806,1061807,1061808,1061809,1061823,1061824,1061825,1061826,1061840,1061841,1061842,1061843,1061856,1061857,1061858,1061859,1061860,1061873,1061874,1061875,1061876,1061890,1061891,1061892,1061893,1061906,1061907,1061908,1061909,1061910,1061923,1061924,1061925,1061926,1061940,1061941,1061942,1061943,1061957,1061958,1061959,1061960,1061973,1061974,1061975,1061976,1061990,1061991,1061992,1061993,1062007,1062008,1062009,1062010,1062023,1062024,1062025,1062026,1062027,1062040,1062041,1062042,1062043,1062057,1062058,1062059,1062060,1062074,1062075,1062076,1062077,1062090,1062091,1062092,1062093,1062107,1062108,1062109,1062110,1062124,1062125,1062126,1062127,1062140,1062141,1062142,1062143,1062144,1060493,1060752,1060791,1060816,1061143,1061234,1061334,1061336,1061343,1061345,1061352,1061354,1061359,1061361,1061368,1061370,1061377,1061379,1061384,1061386,1061393,1061395,1061402,1061404,1061409,1061411,1061418,1061420,1061427,1061429,1061434,1061436,1061438,1061443,1061445,1061452,1061454,1061459,1061461,1061463,1061468,1061470,1061477,1061479,1061484,1061486,1061488,1061493,1061495,1061502,1061504,1061511,1061513,1061518,1061520,1061527,1061529,1061536,1061538,1061543,1061545,1061552,1061554,1061561,1061563,1061568,1061570,1061577,1061579,1061586,1061588,1061593,1061595,1061602,1061604,1061611,1061613,1061618,1061620,1061627,1061629,1061636,1061638,1061643,1061645,1061652,1061654,1061661,1061663,1061668,1061670,1061677,1061679,1061686,1061688,1061693,1061695,1061697,1061702,1061704,1061711,1061713,1061718,1061720,1061722,1061727,1061729,1061736,1061738,1061745,1061747,1061752,1061754,1061761,1061763,1061770,1061772,1061777,1061779,1061786,1061788,1061795,1061797,1061802,1061804,1061811,1061813,1061820,1061822,1061827,1061829,1061836,1061838,1061845,1061847,1061852,1061854,1061861,1061863,1061870,1061872,1061877,1061879,1061886,1061888,1061895,1061897,1061902,1061904,1061911,1061913,1061920,1061922,1061927,1061929,1061931,1061936,1061938,1061945,1061947,1061952,1061954,1061956,1061961,1061963,1061970,1061972,1061977,1061979,1061981,1061986,1061988,1061995,1061997,1062004,1062006,1062011,1062013,1062020,1062022,1062029,1062031,1062036,1062038,1062045,1062047,1062054,1062056,1062061,1062063,1062070,1062072,1062079,1062081,1062086,1062088,1062095,1062097,1062104,1062106,1062111,1062113,1062120,1062122,1062129,1062131,1062136,1062138,1062145,1062147,1060060,1060781,1061233,1061331,1061332,1061333,1061347,1061348,1061349,1061350,1061363,1061364,1061365,1061366,1061380,1061381,1061382,1061383,1061397,1061398,1061399,1061400,1061413,1061414,1061415,1061416,1061417,1061430,1061431,1061432,1061433,1061447,1061448,1061449,1061450,1061464,1061465,1061466,1061467,1061480,1061481,1061482,1061483,1061497,1061498,1061499,1061500,1061514,1061515,1061516,1061517,1061530,1061531,1061532,1061533,1061534,1061547,1061548,1061549,1061550,1061564,1061565,1061566,1061567,1061581,1061582,1061583,1061584,1061597,1061598,1061599,1061600,1061614,1061615,1061616,1061617,1061631,1061632,1061633,1061634,1061647,1061648,1061649,1061650,1061651,1061664,1061665,1061666,1061667,1061681,1061682,1061683,1061684,1061698,1061699,1061700,1061701,1061714,1061715,1061716,1061717,1061731,1061732,1061733,1061734,1061748,1061749,1061750,1061751,1061764,1061765,1061766,1061767,1061768,1061781,1061782,1061783,1061784,1061798,1061799,1061800,1061801,1061815,1061816,1061817,1061818,1061831,1061832,1061833,1061834,1061848,1061849,1061850,1061851,1061865,1061866,1061867,1061868,1061881,1061882,1061883,1061884,1061885,1061898,1061899,1061900,1061901,1061915,1061916,1061917,1061918,1061932,1061933,1061934,1061935,1061948,1061949,1061950,1061951,1061965,1061966,1061967,1061968,1061982,1061983,1061984,1061985,1061998,1061999,1062000,1062001,1062002,1062015,1062016,1062017,1062018,1062032,1062033,1062034,1062035,1062049,1062050,1062051,1062052,1062065,1062066,1062067,1062068,1062082,1062083,1062084,1062085,1062099,1062100,1062101,1062102,1062115,1062116,1062117,1062118,1062119,1062132,1062133,1062134,1062135,1060399,1060526,1060783,1061335,1061337,1061342,1061344,1061346,1061351,1061353,1061360,1061362,1061367,1061369,1061371,1061376,1061378,1061385,1061387,1061394,1061396,1061401,1061403,1061410,1061412,1061419,1061421,1061426,1061428,1061435,1061437,1061444,1061446,1061451,1061453,1061460,1061462,1061469,1061471,1061476,1061478,1061485,1061487,1061494,1061496,1061501,1061503,1061510,1061512,1061519,1061521,1061526,1061528,1061535,1061537,1061544,1061546,1061551,1061553,1061555,1061560,1061562,1061569,1061571,1061576,1061578,1061580,1061585,1061587,1061594,1061596,1061601,1061603,1061605,1061610,1061612,1061619,1061621,1061628,1061630,1061635,1061637,1061644,1061646,1061653,1061655,1061660,1061662,1061669,1061671,1061678,1061680,1061685,1061687,1061694,1061696,1061703,1061705,1061710,1061712,1061719,1061721,1061728,1061730,1061735,1061737,1061744,1061746,1061753,1061755,1061760,1061762,1061769,1061771,1061778,1061780,1061785,1061787,1061794,1061796,1061803,1061805,1061810,1061812,1061814,1061819,1061821,1061828,1061830,1061835,1061837,1061839,1061844,1061846,1061853,1061855,1061862,1061864,1061869,1061871,1061878,1061880,1061887,1061889,1061894,1061896,1061903,1061905,1061912,1061914,1061919,1061921,1061928,1061930,1061937,1061939,1061944,1061946,1061953,1061955,1061962,1061964,1061969,1061971,1061978,1061980,1061987,1061989,1061994,1061996,1062003,1062005,1062012,1062014,1062019,1062021,1062028,1062030,1062037,1062039,1062044,1062046,1062048,1062053,1062055,1062062,1062064,1062069,1062071,1062073,1062078,1062080,1062087,1062089,1062094,1062096,1062098,1062103,1062105,1062112,1062114,1062121,1062123,1062128,1062130,1062137,1062139,1062146) + +) , + Coverage + (project_id,MID, +store_id,employee_id,visit_date,in_time,out_time,duration_in_minutes, +is_covered,is_executed,reason_remarks,storetype_id, +supervisor_id,coverage_type,distance_in_meters,reasonId,camera_allow,update_date, +update_by,Unique_Id) + +AS ( + +Select '40148' as ProjectId,JP.MID, sm.StoreId,JP.EmpId as EmpId, + Convert(VARCHAR,Jp.VisitDate) AS VisitDate +, Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, + Convert(Datetime,InTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, + Convert(Datetime,InTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage Where EmpId = Jp.EmpId And StoreId = Jp.StoreId + And VisitDate = Jp.VisitDate),'') As [In Time] + , Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, + Convert(Datetime,OutTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, + Convert(Datetime,OutTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage + Where EmpId = Jp.EmpId And StoreId = Jp.StoreId + And Convert(Date,VisitDate) = Jp.VisitDate),'') As [Out Time] + , + Isnull((Select Top 1 Case When convert(varchar,outtime) like '%00:00:00.00%' THEN 0 + Else Case When outtime0 Then 'Y' +Else 'N' End As Covered +, isnull(Exe.Executed,'') as Executed + +,Isnull((Select Top 1 Case +When D.ReasonId Is Null Or D.ReasonId =0 Then '' Else Reason+' - ' + +Case When Replace(Replace(D.Remark,Char(13),''''),Char(10),' ')<>'' Then Replace(Replace(D.Remark,Char(13),''),Char(10),' ') +Else Replace(Replace(D.Remark,Char(13),''),Char(10),' ') End End As Reason +From OneApp_KelloggsMT.dbo.T_StoreCoverage D +Inner Join OneApp_KelloggsMT.dbo.Master_NonWorkingReason N On N.ReasonId = D.ReasonId +Where D.Isdel = 0 +And D.EmpId = Jp.EmpId And D.StoreId = Jp.StoreId And D.VisitDate = Jp.VisitDate),'''') As [Detailed Remarks], +sm.StoreTypeId,Em.SupervisorId, +Case When jp.Deviation=0 Then 'Planned' When jp.Deviation=1 Then 'Adhoc' +When jp.Deviation=2 Then 'Beat Plan' When jp.Deviation=3 Then 'Non Merchandised' +When jp.Deviation=4 Then 'Add New Store' When jp.Deviation=5 Then 'Non Program' else '' End as [PJP Status] +,Isnull((Select Top 1 Case When (Sc.Latitude=0.00000000 Or Sc.Longitude Is Null) Then 0 Else Case When (sm.Latitude=0.00000000 Or sm.Latitude Is Null) Then 0 + Else Case When (sm.Longitude=0.00000000 Or sm.Longitude Is Null) Then 0 +Else SQRT(POWER(69.1 * ( Sc.Latitude - Sm.Latitude),2) + POWER(69.1 * ( Sm.Longitude - Sc.Longitude ) * COS(Sc.Latitude / 57.3), 2))*1000 + End End End As [Distance In Mtr] From OneApp_KelloggsMT.dbo.T_StoreCoverage Sc Where Sc.Isdel=0 And Sc.EmpId= Jp.EmpId +And Sc.StoreId= Jp.StoreId And Sc.VisitDate= Jp.VisitDate),'') As [Distance In Mtr] +,ISNULL(CAST (JP.ReasonId AS VARCHAR),'0'),sm.CameraAllow, + +GETDATE(),'SP-Pius' , + + CAST('40148' AS VARCHAR) + '_' + CAST(SM.storeid AS VARCHAR) + + '_' + CAST(EM.EMPID AS VARCHAR) + + + +FROM OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) Inner Join +Executed Exe on Jp.MID= Exe.MID Inner Join +OneApp_KelloggsMT.dbo.vw_StoreDetail sm on Jp.StoreId= sm.StoreId Inner Join +OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on JP.EmpId= Em.EmpId +left join +OneApp_KelloggsMT.dbo.T_StoreCoveragePositionPivot pv with (nolock) +on jp.mid=pv.MID +Where 1=1 and em.UserName not like 'test%' AND JP.MID in (1060487,1060555,1060637,1060656,1061071,1061238,1061256,1061338,1061339,1061340,1061341,1061355,1061356,1061357,1061358,1061372,1061373,1061374,1061375,1061388,1061389,1061390,1061391,1061392,1061405,1061406,1061407,1061408,1061422,1061423,1061424,1061425,1061439,1061440,1061441,1061442,1061455,1061456,1061457,1061458,1061472,1061473,1061474,1061475,1061489,1061490,1061491,1061492,1061505,1061506,1061507,1061508,1061509,1061522,1061523,1061524,1061525,1061539,1061540,1061541,1061542,1061556,1061557,1061558,1061559,1061572,1061573,1061574,1061575,1061589,1061590,1061591,1061592,1061606,1061607,1061608,1061609,1061622,1061623,1061624,1061625,1061626,1061639,1061640,1061641,1061642,1061656,1061657,1061658,1061659,1061672,1061673,1061674,1061675,1061676,1061689,1061690,1061691,1061692,1061706,1061707,1061708,1061709,1061723,1061724,1061725,1061726,1061739,1061740,1061741,1061742,1061743,1061756,1061757,1061758,1061759,1061773,1061774,1061775,1061776,1061789,1061790,1061791,1061792,1061793,1061806,1061807,1061808,1061809,1061823,1061824,1061825,1061826,1061840,1061841,1061842,1061843,1061856,1061857,1061858,1061859,1061860,1061873,1061874,1061875,1061876,1061890,1061891,1061892,1061893,1061906,1061907,1061908,1061909,1061910,1061923,1061924,1061925,1061926,1061940,1061941,1061942,1061943,1061957,1061958,1061959,1061960,1061973,1061974,1061975,1061976,1061990,1061991,1061992,1061993,1062007,1062008,1062009,1062010,1062023,1062024,1062025,1062026,1062027,1062040,1062041,1062042,1062043,1062057,1062058,1062059,1062060,1062074,1062075,1062076,1062077,1062090,1062091,1062092,1062093,1062107,1062108,1062109,1062110,1062124,1062125,1062126,1062127,1062140,1062141,1062142,1062143,1062144,1060493,1060752,1060791,1060816,1061143,1061234,1061334,1061336,1061343,1061345,1061352,1061354,1061359,1061361,1061368,1061370,1061377,1061379,1061384,1061386,1061393,1061395,1061402,1061404,1061409,1061411,1061418,1061420,1061427,1061429,1061434,1061436,1061438,1061443,1061445,1061452,1061454,1061459,1061461,1061463,1061468,1061470,1061477,1061479,1061484,1061486,1061488,1061493,1061495,1061502,1061504,1061511,1061513,1061518,1061520,1061527,1061529,1061536,1061538,1061543,1061545,1061552,1061554,1061561,1061563,1061568,1061570,1061577,1061579,1061586,1061588,1061593,1061595,1061602,1061604,1061611,1061613,1061618,1061620,1061627,1061629,1061636,1061638,1061643,1061645,1061652,1061654,1061661,1061663,1061668,1061670,1061677,1061679,1061686,1061688,1061693,1061695,1061697,1061702,1061704,1061711,1061713,1061718,1061720,1061722,1061727,1061729,1061736,1061738,1061745,1061747,1061752,1061754,1061761,1061763,1061770,1061772,1061777,1061779,1061786,1061788,1061795,1061797,1061802,1061804,1061811,1061813,1061820,1061822,1061827,1061829,1061836,1061838,1061845,1061847,1061852,1061854,1061861,1061863,1061870,1061872,1061877,1061879,1061886,1061888,1061895,1061897,1061902,1061904,1061911,1061913,1061920,1061922,1061927,1061929,1061931,1061936,1061938,1061945,1061947,1061952,1061954,1061956,1061961,1061963,1061970,1061972,1061977,1061979,1061981,1061986,1061988,1061995,1061997,1062004,1062006,1062011,1062013,1062020,1062022,1062029,1062031,1062036,1062038,1062045,1062047,1062054,1062056,1062061,1062063,1062070,1062072,1062079,1062081,1062086,1062088,1062095,1062097,1062104,1062106,1062111,1062113,1062120,1062122,1062129,1062131,1062136,1062138,1062145,1062147,1060060,1060781,1061233,1061331,1061332,1061333,1061347,1061348,1061349,1061350,1061363,1061364,1061365,1061366,1061380,1061381,1061382,1061383,1061397,1061398,1061399,1061400,1061413,1061414,1061415,1061416,1061417,1061430,1061431,1061432,1061433,1061447,1061448,1061449,1061450,1061464,1061465,1061466,1061467,1061480,1061481,1061482,1061483,1061497,1061498,1061499,1061500,1061514,1061515,1061516,1061517,1061530,1061531,1061532,1061533,1061534,1061547,1061548,1061549,1061550,1061564,1061565,1061566,1061567,1061581,1061582,1061583,1061584,1061597,1061598,1061599,1061600,1061614,1061615,1061616,1061617,1061631,1061632,1061633,1061634,1061647,1061648,1061649,1061650,1061651,1061664,1061665,1061666,1061667,1061681,1061682,1061683,1061684,1061698,1061699,1061700,1061701,1061714,1061715,1061716,1061717,1061731,1061732,1061733,1061734,1061748,1061749,1061750,1061751,1061764,1061765,1061766,1061767,1061768,1061781,1061782,1061783,1061784,1061798,1061799,1061800,1061801,1061815,1061816,1061817,1061818,1061831,1061832,1061833,1061834,1061848,1061849,1061850,1061851,1061865,1061866,1061867,1061868,1061881,1061882,1061883,1061884,1061885,1061898,1061899,1061900,1061901,1061915,1061916,1061917,1061918,1061932,1061933,1061934,1061935,1061948,1061949,1061950,1061951,1061965,1061966,1061967,1061968,1061982,1061983,1061984,1061985,1061998,1061999,1062000,1062001,1062002,1062015,1062016,1062017,1062018,1062032,1062033,1062034,1062035,1062049,1062050,1062051,1062052,1062065,1062066,1062067,1062068,1062082,1062083,1062084,1062085,1062099,1062100,1062101,1062102,1062115,1062116,1062117,1062118,1062119,1062132,1062133,1062134,1062135,1060399,1060526,1060783,1061335,1061337,1061342,1061344,1061346,1061351,1061353,1061360,1061362,1061367,1061369,1061371,1061376,1061378,1061385,1061387,1061394,1061396,1061401,1061403,1061410,1061412,1061419,1061421,1061426,1061428,1061435,1061437,1061444,1061446,1061451,1061453,1061460,1061462,1061469,1061471,1061476,1061478,1061485,1061487,1061494,1061496,1061501,1061503,1061510,1061512,1061519,1061521,1061526,1061528,1061535,1061537,1061544,1061546,1061551,1061553,1061555,1061560,1061562,1061569,1061571,1061576,1061578,1061580,1061585,1061587,1061594,1061596,1061601,1061603,1061605,1061610,1061612,1061619,1061621,1061628,1061630,1061635,1061637,1061644,1061646,1061653,1061655,1061660,1061662,1061669,1061671,1061678,1061680,1061685,1061687,1061694,1061696,1061703,1061705,1061710,1061712,1061719,1061721,1061728,1061730,1061735,1061737,1061744,1061746,1061753,1061755,1061760,1061762,1061769,1061771,1061778,1061780,1061785,1061787,1061794,1061796,1061803,1061805,1061810,1061812,1061814,1061819,1061821,1061828,1061830,1061835,1061837,1061839,1061844,1061846,1061853,1061855,1061862,1061864,1061869,1061871,1061878,1061880,1061887,1061889,1061894,1061896,1061903,1061905,1061912,1061914,1061919,1061921,1061928,1061930,1061937,1061939,1061944,1061946,1061953,1061955,1061962,1061964,1061969,1061971,1061978,1061980,1061987,1061989,1061994,1061996,1062003,1062005,1062012,1062014,1062019,1062021,1062028,1062030,1062037,1062039,1062044,1062046,1062048,1062053,1062055,1062062,1062064,1062069,1062071,1062073,1062078,1062080,1062087,1062089,1062094,1062096,1062098,1062103,1062105,1062112,1062114,1062121,1062123,1062128,1062130,1062137,1062139,1062146 ) + +) +select * from Coverage] +(Background on this error at: https://sqlalche.me/e/20/f405) +2026-06-18 13:19:16 | INFO | ================================================================================ +2026-06-18 13:19:16 | INFO | Hello from data-move Python data pipeline! +2026-06-18 13:19:16 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 13:19:16 | INFO | Connecting to databases... +2026-06-18 13:19:17 | INFO | +2026-06-18 13:19:19 | INFO | +2026-06-18 13:19:20 | INFO | Database connections established +2026-06-18 13:19:20 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 13:19:21 | INFO | Found 836 MIDs +2026-06-18 13:19:21 | INFO | ================================================================================ +2026-06-18 13:19:21 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:19:21 | INFO | Fetching Data from sql server for table-:additional_visibility--------------------------- +2026-06-18 13:19:21 | INFO | Fetching data for 836 MIDs +2026-06-18 13:19:26 | INFO | Fetched 1,885 rows from SQL Server +2026-06-18 13:19:26 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! +2026-06-18 13:19:26 | INFO | Fetched 1885 rows +2026-06-18 13:19:26 | INFO | Deleting from additional_visibility +2026-06-18 13:19:26 | INFO | inserting data into clickhouse db from sql server------------------ +2026-06-18 13:19:26 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse +2026-06-18 13:19:26 | INFO | additional_visibility loaded successfully (1885 rows) +2026-06-18 13:19:26 | INFO | ================================================================================ +2026-06-18 13:19:26 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:19:26 | INFO | Fetching Data from sql server for table-:Coverage--------------------------- +2026-06-18 13:19:26 | INFO | Fetching data for 836 MIDs +2026-06-18 13:19:26 | ERROR | Failed processing table Coverage +Traceback (most recent call last): + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'as'. (156) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ','. (102)") + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "D:\data_move\main2.py", line 181, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 80, in get_dataframe + return fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\data_move\src\fact.py", line 363, in fetch_Coverage + df = pl.read_database( + query=sql, + connection=engine + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database + return cx.execute( + ~~~~~~~~~~^ + query=query, + ^^^^^^^^^^^^ + options=execute_options, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ).to_polars( + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute + result = cursor_execute(query, **options) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute + return meth( + self, + distilled_parameters, + execution_options or NO_OPTIONS, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection + return connection._execute_clauseelement( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + self, distilled_params, execution_options + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement + ret = self._execute_context( + dialect, + ...<8 lines>... + cache_hit=cache_hit, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context + return self._exec_single_context( + ~~~~~~~~~~~~~~~~~~~~~~~~~^ + dialect, context, statement, parameters + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context + self._handle_dbapi_exception( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + e, str_statement, effective_parameters, cursor, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'as'. (156) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ','. (102)") +[SQL: +Executed as +(select JP.mid, CASE WHEN isnull(B.MID,'')<>'' then 'Y' +ELSE 'N' END AS COVERED, +CASE WHEN isnull(C.MID,'')<>'' then 'Y' +ELSE 'N' END AS EXECUTED,isnull(NR.REASON,'') AS REASON + +from OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) left join +(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) +inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B + ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) + AND JP.ReasonId in (0,1,3,9,10,19,20))b +on JP.MID=b.MID + +left join +(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) +inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B + ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) + AND JP.ReasonId in (0,19))c +on JP.MID=c.MID +left join +OneApp_KelloggsMT.dbo.Master_NonWorkingReason nr on JP.ReasonId=nr.ReasonId +Where 1=1 AND JP.MID in (1060487,1060555,1060637,1060656,1061071,1061238,1061256,1061338,1061339,1061340,1061341,1061355,1061356,1061357,1061358,1061372,1061373,1061374,1061375,1061388,1061389,1061390,1061391,1061392,1061405,1061406,1061407,1061408,1061422,1061423,1061424,1061425,1061439,1061440,1061441,1061442,1061455,1061456,1061457,1061458,1061472,1061473,1061474,1061475,1061489,1061490,1061491,1061492,1061505,1061506,1061507,1061508,1061509,1061522,1061523,1061524,1061525,1061539,1061540,1061541,1061542,1061556,1061557,1061558,1061559,1061572,1061573,1061574,1061575,1061589,1061590,1061591,1061592,1061606,1061607,1061608,1061609,1061622,1061623,1061624,1061625,1061626,1061639,1061640,1061641,1061642,1061656,1061657,1061658,1061659,1061672,1061673,1061674,1061675,1061676,1061689,1061690,1061691,1061692,1061706,1061707,1061708,1061709,1061723,1061724,1061725,1061726,1061739,1061740,1061741,1061742,1061743,1061756,1061757,1061758,1061759,1061773,1061774,1061775,1061776,1061789,1061790,1061791,1061792,1061793,1061806,1061807,1061808,1061809,1061823,1061824,1061825,1061826,1061840,1061841,1061842,1061843,1061856,1061857,1061858,1061859,1061860,1061873,1061874,1061875,1061876,1061890,1061891,1061892,1061893,1061906,1061907,1061908,1061909,1061910,1061923,1061924,1061925,1061926,1061940,1061941,1061942,1061943,1061957,1061958,1061959,1061960,1061973,1061974,1061975,1061976,1061990,1061991,1061992,1061993,1062007,1062008,1062009,1062010,1062023,1062024,1062025,1062026,1062027,1062040,1062041,1062042,1062043,1062057,1062058,1062059,1062060,1062074,1062075,1062076,1062077,1062090,1062091,1062092,1062093,1062107,1062108,1062109,1062110,1062124,1062125,1062126,1062127,1062140,1062141,1062142,1062143,1062144,1060060,1060781,1061233,1061331,1061332,1061333,1061347,1061348,1061349,1061350,1061363,1061364,1061365,1061366,1061380,1061381,1061382,1061383,1061397,1061398,1061399,1061400,1061413,1061414,1061415,1061416,1061417,1061430,1061431,1061432,1061433,1061447,1061448,1061449,1061450,1061464,1061465,1061466,1061467,1061480,1061481,1061482,1061483,1061497,1061498,1061499,1061500,1061514,1061515,1061516,1061517,1061530,1061531,1061532,1061533,1061534,1061547,1061548,1061549,1061550,1061564,1061565,1061566,1061567,1061581,1061582,1061583,1061584,1061597,1061598,1061599,1061600,1061614,1061615,1061616,1061617,1061631,1061632,1061633,1061634,1061647,1061648,1061649,1061650,1061651,1061664,1061665,1061666,1061667,1061681,1061682,1061683,1061684,1061698,1061699,1061700,1061701,1061714,1061715,1061716,1061717,1061731,1061732,1061733,1061734,1061748,1061749,1061750,1061751,1061764,1061765,1061766,1061767,1061768,1061781,1061782,1061783,1061784,1061798,1061799,1061800,1061801,1061815,1061816,1061817,1061818,1061831,1061832,1061833,1061834,1061848,1061849,1061850,1061851,1061865,1061866,1061867,1061868,1061881,1061882,1061883,1061884,1061885,1061898,1061899,1061900,1061901,1061915,1061916,1061917,1061918,1061932,1061933,1061934,1061935,1061948,1061949,1061950,1061951,1061965,1061966,1061967,1061968,1061982,1061983,1061984,1061985,1061998,1061999,1062000,1062001,1062002,1062015,1062016,1062017,1062018,1062032,1062033,1062034,1062035,1062049,1062050,1062051,1062052,1062065,1062066,1062067,1062068,1062082,1062083,1062084,1062085,1062099,1062100,1062101,1062102,1062115,1062116,1062117,1062118,1062119,1062132,1062133,1062134,1062135,1060493,1060752,1060791,1060816,1061143,1061234,1061334,1061336,1061343,1061345,1061352,1061354,1061359,1061361,1061368,1061370,1061377,1061379,1061384,1061386,1061393,1061395,1061402,1061404,1061409,1061411,1061418,1061420,1061427,1061429,1061434,1061436,1061438,1061443,1061445,1061452,1061454,1061459,1061461,1061463,1061468,1061470,1061477,1061479,1061484,1061486,1061488,1061493,1061495,1061502,1061504,1061511,1061513,1061518,1061520,1061527,1061529,1061536,1061538,1061543,1061545,1061552,1061554,1061561,1061563,1061568,1061570,1061577,1061579,1061586,1061588,1061593,1061595,1061602,1061604,1061611,1061613,1061618,1061620,1061627,1061629,1061636,1061638,1061643,1061645,1061652,1061654,1061661,1061663,1061668,1061670,1061677,1061679,1061686,1061688,1061693,1061695,1061697,1061702,1061704,1061711,1061713,1061718,1061720,1061722,1061727,1061729,1061736,1061738,1061745,1061747,1061752,1061754,1061761,1061763,1061770,1061772,1061777,1061779,1061786,1061788,1061795,1061797,1061802,1061804,1061811,1061813,1061820,1061822,1061827,1061829,1061836,1061838,1061845,1061847,1061852,1061854,1061861,1061863,1061870,1061872,1061877,1061879,1061886,1061888,1061895,1061897,1061902,1061904,1061911,1061913,1061920,1061922,1061927,1061929,1061931,1061936,1061938,1061945,1061947,1061952,1061954,1061956,1061961,1061963,1061970,1061972,1061977,1061979,1061981,1061986,1061988,1061995,1061997,1062004,1062006,1062011,1062013,1062020,1062022,1062029,1062031,1062036,1062038,1062045,1062047,1062054,1062056,1062061,1062063,1062070,1062072,1062079,1062081,1062086,1062088,1062095,1062097,1062104,1062106,1062111,1062113,1062120,1062122,1062129,1062131,1062136,1062138,1062145,1062147,1060399,1060526,1060783,1061335,1061337,1061342,1061344,1061346,1061351,1061353,1061360,1061362,1061367,1061369,1061371,1061376,1061378,1061385,1061387,1061394,1061396,1061401,1061403,1061410,1061412,1061419,1061421,1061426,1061428,1061435,1061437,1061444,1061446,1061451,1061453,1061460,1061462,1061469,1061471,1061476,1061478,1061485,1061487,1061494,1061496,1061501,1061503,1061510,1061512,1061519,1061521,1061526,1061528,1061535,1061537,1061544,1061546,1061551,1061553,1061555,1061560,1061562,1061569,1061571,1061576,1061578,1061580,1061585,1061587,1061594,1061596,1061601,1061603,1061605,1061610,1061612,1061619,1061621,1061628,1061630,1061635,1061637,1061644,1061646,1061653,1061655,1061660,1061662,1061669,1061671,1061678,1061680,1061685,1061687,1061694,1061696,1061703,1061705,1061710,1061712,1061719,1061721,1061728,1061730,1061735,1061737,1061744,1061746,1061753,1061755,1061760,1061762,1061769,1061771,1061778,1061780,1061785,1061787,1061794,1061796,1061803,1061805,1061810,1061812,1061814,1061819,1061821,1061828,1061830,1061835,1061837,1061839,1061844,1061846,1061853,1061855,1061862,1061864,1061869,1061871,1061878,1061880,1061887,1061889,1061894,1061896,1061903,1061905,1061912,1061914,1061919,1061921,1061928,1061930,1061937,1061939,1061944,1061946,1061953,1061955,1061962,1061964,1061969,1061971,1061978,1061980,1061987,1061989,1061994,1061996,1062003,1062005,1062012,1062014,1062019,1062021,1062028,1062030,1062037,1062039,1062044,1062046,1062048,1062053,1062055,1062062,1062064,1062069,1062071,1062073,1062078,1062080,1062087,1062089,1062094,1062096,1062098,1062103,1062105,1062112,1062114,1062121,1062123,1062128,1062130,1062137,1062139,1062146) + +) , + Coverage + (project_id,MID, +store_id,employee_id,visit_date,in_time,out_time,duration_in_minutes, +is_covered,is_executed,reason_remarks,storetype_id, +supervisor_id,coverage_type,distance_in_meters,reasonId,camera_allow,update_date, +update_by,Unique_Id) + +AS ( + +Select '40148' as ProjectId,JP.MID, sm.StoreId,JP.EmpId as EmpId, + Convert(VARCHAR,Jp.VisitDate) AS VisitDate +, Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, + Convert(Datetime,InTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, + Convert(Datetime,InTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage Where EmpId = Jp.EmpId And StoreId = Jp.StoreId + And VisitDate = Jp.VisitDate),'') As [In Time] + , Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, + Convert(Datetime,OutTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, + Convert(Datetime,OutTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage + Where EmpId = Jp.EmpId And StoreId = Jp.StoreId + And Convert(Date,VisitDate) = Jp.VisitDate),'') As [Out Time] + , + Isnull((Select Top 1 Case When convert(varchar,outtime) like '%00:00:00.00%' THEN 0 + Else Case When outtime0 Then 'Y' +Else 'N' End As Covered +, isnull(Exe.Executed,'') as Executed + +,Isnull((Select Top 1 Case +When D.ReasonId Is Null Or D.ReasonId =0 Then '' Else Reason+' - ' + +Case When Replace(Replace(D.Remark,Char(13),''''),Char(10),' ')<>'' Then Replace(Replace(D.Remark,Char(13),''),Char(10),' ') +Else Replace(Replace(D.Remark,Char(13),''),Char(10),' ') End End As Reason +From OneApp_KelloggsMT.dbo.T_StoreCoverage D +Inner Join OneApp_KelloggsMT.dbo.Master_NonWorkingReason N On N.ReasonId = D.ReasonId +Where D.Isdel = 0 +And D.EmpId = Jp.EmpId And D.StoreId = Jp.StoreId And D.VisitDate = Jp.VisitDate),'''') As [Detailed Remarks], +sm.StoreTypeId,Em.SupervisorId, +Case When jp.Deviation=0 Then 'Planned' When jp.Deviation=1 Then 'Adhoc' +When jp.Deviation=2 Then 'Beat Plan' When jp.Deviation=3 Then 'Non Merchandised' +When jp.Deviation=4 Then 'Add New Store' When jp.Deviation=5 Then 'Non Program' else '' End as [PJP Status] +,Isnull((Select Top 1 Case When (Sc.Latitude=0.00000000 Or Sc.Longitude Is Null) Then 0 Else Case When (sm.Latitude=0.00000000 Or sm.Latitude Is Null) Then 0 + Else Case When (sm.Longitude=0.00000000 Or sm.Longitude Is Null) Then 0 +Else SQRT(POWER(69.1 * ( Sc.Latitude - Sm.Latitude),2) + POWER(69.1 * ( Sm.Longitude - Sc.Longitude ) * COS(Sc.Latitude / 57.3), 2))*1000 + End End End As [Distance In Mtr] From OneApp_KelloggsMT.dbo.T_StoreCoverage Sc Where Sc.Isdel=0 And Sc.EmpId= Jp.EmpId +And Sc.StoreId= Jp.StoreId And Sc.VisitDate= Jp.VisitDate),'') As [Distance In Mtr] +,ISNULL(CAST (JP.ReasonId AS VARCHAR),'0'),sm.CameraAllow, + +GETDATE(),'SP-Pius' , + + CAST('40148' AS VARCHAR) + '_' + CAST(SM.storeid AS VARCHAR) + + '_' + CAST(EM.EMPID AS VARCHAR) + + + +FROM OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) Inner Join +Executed Exe on Jp.MID= Exe.MID Inner Join +OneApp_KelloggsMT.dbo.vw_StoreDetail sm on Jp.StoreId= sm.StoreId Inner Join +OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on JP.EmpId= Em.EmpId +left join +OneApp_KelloggsMT.dbo.T_StoreCoveragePositionPivot pv with (nolock) +on jp.mid=pv.MID +Where 1=1 and em.UserName not like 'test%' AND JP.MID in (1060487,1060555,1060637,1060656,1061071,1061238,1061256,1061338,1061339,1061340,1061341,1061355,1061356,1061357,1061358,1061372,1061373,1061374,1061375,1061388,1061389,1061390,1061391,1061392,1061405,1061406,1061407,1061408,1061422,1061423,1061424,1061425,1061439,1061440,1061441,1061442,1061455,1061456,1061457,1061458,1061472,1061473,1061474,1061475,1061489,1061490,1061491,1061492,1061505,1061506,1061507,1061508,1061509,1061522,1061523,1061524,1061525,1061539,1061540,1061541,1061542,1061556,1061557,1061558,1061559,1061572,1061573,1061574,1061575,1061589,1061590,1061591,1061592,1061606,1061607,1061608,1061609,1061622,1061623,1061624,1061625,1061626,1061639,1061640,1061641,1061642,1061656,1061657,1061658,1061659,1061672,1061673,1061674,1061675,1061676,1061689,1061690,1061691,1061692,1061706,1061707,1061708,1061709,1061723,1061724,1061725,1061726,1061739,1061740,1061741,1061742,1061743,1061756,1061757,1061758,1061759,1061773,1061774,1061775,1061776,1061789,1061790,1061791,1061792,1061793,1061806,1061807,1061808,1061809,1061823,1061824,1061825,1061826,1061840,1061841,1061842,1061843,1061856,1061857,1061858,1061859,1061860,1061873,1061874,1061875,1061876,1061890,1061891,1061892,1061893,1061906,1061907,1061908,1061909,1061910,1061923,1061924,1061925,1061926,1061940,1061941,1061942,1061943,1061957,1061958,1061959,1061960,1061973,1061974,1061975,1061976,1061990,1061991,1061992,1061993,1062007,1062008,1062009,1062010,1062023,1062024,1062025,1062026,1062027,1062040,1062041,1062042,1062043,1062057,1062058,1062059,1062060,1062074,1062075,1062076,1062077,1062090,1062091,1062092,1062093,1062107,1062108,1062109,1062110,1062124,1062125,1062126,1062127,1062140,1062141,1062142,1062143,1062144,1060060,1060781,1061233,1061331,1061332,1061333,1061347,1061348,1061349,1061350,1061363,1061364,1061365,1061366,1061380,1061381,1061382,1061383,1061397,1061398,1061399,1061400,1061413,1061414,1061415,1061416,1061417,1061430,1061431,1061432,1061433,1061447,1061448,1061449,1061450,1061464,1061465,1061466,1061467,1061480,1061481,1061482,1061483,1061497,1061498,1061499,1061500,1061514,1061515,1061516,1061517,1061530,1061531,1061532,1061533,1061534,1061547,1061548,1061549,1061550,1061564,1061565,1061566,1061567,1061581,1061582,1061583,1061584,1061597,1061598,1061599,1061600,1061614,1061615,1061616,1061617,1061631,1061632,1061633,1061634,1061647,1061648,1061649,1061650,1061651,1061664,1061665,1061666,1061667,1061681,1061682,1061683,1061684,1061698,1061699,1061700,1061701,1061714,1061715,1061716,1061717,1061731,1061732,1061733,1061734,1061748,1061749,1061750,1061751,1061764,1061765,1061766,1061767,1061768,1061781,1061782,1061783,1061784,1061798,1061799,1061800,1061801,1061815,1061816,1061817,1061818,1061831,1061832,1061833,1061834,1061848,1061849,1061850,1061851,1061865,1061866,1061867,1061868,1061881,1061882,1061883,1061884,1061885,1061898,1061899,1061900,1061901,1061915,1061916,1061917,1061918,1061932,1061933,1061934,1061935,1061948,1061949,1061950,1061951,1061965,1061966,1061967,1061968,1061982,1061983,1061984,1061985,1061998,1061999,1062000,1062001,1062002,1062015,1062016,1062017,1062018,1062032,1062033,1062034,1062035,1062049,1062050,1062051,1062052,1062065,1062066,1062067,1062068,1062082,1062083,1062084,1062085,1062099,1062100,1062101,1062102,1062115,1062116,1062117,1062118,1062119,1062132,1062133,1062134,1062135,1060493,1060752,1060791,1060816,1061143,1061234,1061334,1061336,1061343,1061345,1061352,1061354,1061359,1061361,1061368,1061370,1061377,1061379,1061384,1061386,1061393,1061395,1061402,1061404,1061409,1061411,1061418,1061420,1061427,1061429,1061434,1061436,1061438,1061443,1061445,1061452,1061454,1061459,1061461,1061463,1061468,1061470,1061477,1061479,1061484,1061486,1061488,1061493,1061495,1061502,1061504,1061511,1061513,1061518,1061520,1061527,1061529,1061536,1061538,1061543,1061545,1061552,1061554,1061561,1061563,1061568,1061570,1061577,1061579,1061586,1061588,1061593,1061595,1061602,1061604,1061611,1061613,1061618,1061620,1061627,1061629,1061636,1061638,1061643,1061645,1061652,1061654,1061661,1061663,1061668,1061670,1061677,1061679,1061686,1061688,1061693,1061695,1061697,1061702,1061704,1061711,1061713,1061718,1061720,1061722,1061727,1061729,1061736,1061738,1061745,1061747,1061752,1061754,1061761,1061763,1061770,1061772,1061777,1061779,1061786,1061788,1061795,1061797,1061802,1061804,1061811,1061813,1061820,1061822,1061827,1061829,1061836,1061838,1061845,1061847,1061852,1061854,1061861,1061863,1061870,1061872,1061877,1061879,1061886,1061888,1061895,1061897,1061902,1061904,1061911,1061913,1061920,1061922,1061927,1061929,1061931,1061936,1061938,1061945,1061947,1061952,1061954,1061956,1061961,1061963,1061970,1061972,1061977,1061979,1061981,1061986,1061988,1061995,1061997,1062004,1062006,1062011,1062013,1062020,1062022,1062029,1062031,1062036,1062038,1062045,1062047,1062054,1062056,1062061,1062063,1062070,1062072,1062079,1062081,1062086,1062088,1062095,1062097,1062104,1062106,1062111,1062113,1062120,1062122,1062129,1062131,1062136,1062138,1062145,1062147,1060399,1060526,1060783,1061335,1061337,1061342,1061344,1061346,1061351,1061353,1061360,1061362,1061367,1061369,1061371,1061376,1061378,1061385,1061387,1061394,1061396,1061401,1061403,1061410,1061412,1061419,1061421,1061426,1061428,1061435,1061437,1061444,1061446,1061451,1061453,1061460,1061462,1061469,1061471,1061476,1061478,1061485,1061487,1061494,1061496,1061501,1061503,1061510,1061512,1061519,1061521,1061526,1061528,1061535,1061537,1061544,1061546,1061551,1061553,1061555,1061560,1061562,1061569,1061571,1061576,1061578,1061580,1061585,1061587,1061594,1061596,1061601,1061603,1061605,1061610,1061612,1061619,1061621,1061628,1061630,1061635,1061637,1061644,1061646,1061653,1061655,1061660,1061662,1061669,1061671,1061678,1061680,1061685,1061687,1061694,1061696,1061703,1061705,1061710,1061712,1061719,1061721,1061728,1061730,1061735,1061737,1061744,1061746,1061753,1061755,1061760,1061762,1061769,1061771,1061778,1061780,1061785,1061787,1061794,1061796,1061803,1061805,1061810,1061812,1061814,1061819,1061821,1061828,1061830,1061835,1061837,1061839,1061844,1061846,1061853,1061855,1061862,1061864,1061869,1061871,1061878,1061880,1061887,1061889,1061894,1061896,1061903,1061905,1061912,1061914,1061919,1061921,1061928,1061930,1061937,1061939,1061944,1061946,1061953,1061955,1061962,1061964,1061969,1061971,1061978,1061980,1061987,1061989,1061994,1061996,1062003,1062005,1062012,1062014,1062019,1062021,1062028,1062030,1062037,1062039,1062044,1062046,1062048,1062053,1062055,1062062,1062064,1062069,1062071,1062073,1062078,1062080,1062087,1062089,1062094,1062096,1062098,1062103,1062105,1062112,1062114,1062121,1062123,1062128,1062130,1062137,1062139,1062146 ) + +) +select * from Coverage] +(Background on this error at: https://sqlalche.me/e/20/f405) +2026-06-18 13:21:52 | INFO | ================================================================================ +2026-06-18 13:21:52 | INFO | Hello from data-move Python data pipeline! +2026-06-18 13:21:52 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 13:21:52 | INFO | Connecting to databases... +2026-06-18 13:21:53 | INFO | +2026-06-18 13:21:55 | INFO | +2026-06-18 13:21:56 | INFO | Database connections established +2026-06-18 13:21:56 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 13:21:56 | INFO | Found 836 MIDs +2026-06-18 13:21:57 | INFO | ================================================================================ +2026-06-18 13:21:57 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:21:57 | INFO | Fetching Data from sql server for table-:additional_visibility--------------------------- +2026-06-18 13:21:57 | INFO | Fetching data for 836 MIDs +2026-06-18 13:21:58 | INFO | Fetched 1,885 rows from SQL Server +2026-06-18 13:21:58 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! +2026-06-18 13:21:58 | INFO | Fetched 1885 rows +2026-06-18 13:21:58 | INFO | Creating table additional_visibility +2026-06-18 13:21:58 | INFO | Table ready: additional_visibility +2026-06-18 13:21:58 | INFO | inserting data into clickhouse db from sql server------------------ +2026-06-18 13:21:59 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse +2026-06-18 13:21:59 | INFO | additional_visibility loaded successfully (1885 rows) +2026-06-18 13:21:59 | INFO | ================================================================================ +2026-06-18 13:21:59 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:21:59 | INFO | Fetching Data from sql server for table-:Coverage--------------------------- +2026-06-18 13:21:59 | INFO | Fetching data for 836 MIDs +2026-06-18 13:21:59 | ERROR | Failed processing table Coverage +Traceback (most recent call last): + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'as'. (156) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ','. (102)") + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "D:\data_move\main2.py", line 181, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 80, in get_dataframe + return fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\data_move\src\fact.py", line 363, in fetch_Coverage + df = pl.read_database( + query=sql, + connection=engine + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database + return cx.execute( + ~~~~~~~~~~^ + query=query, + ^^^^^^^^^^^^ + options=execute_options, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ).to_polars( + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute + result = cursor_execute(query, **options) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute + return meth( + self, + distilled_parameters, + execution_options or NO_OPTIONS, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection + return connection._execute_clauseelement( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + self, distilled_params, execution_options + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement + ret = self._execute_context( + dialect, + ...<8 lines>... + cache_hit=cache_hit, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context + return self._exec_single_context( + ~~~~~~~~~~~~~~~~~~~~~~~~~^ + dialect, context, statement, parameters + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context + self._handle_dbapi_exception( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + e, str_statement, effective_parameters, cursor, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'as'. (156) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ','. (102)") +[SQL: +Executed as +(select JP.mid, CASE WHEN isnull(B.MID,'')<>'' then 'Y' +ELSE 'N' END AS COVERED, +CASE WHEN isnull(C.MID,'')<>'' then 'Y' +ELSE 'N' END AS EXECUTED,isnull(NR.REASON,'') AS REASON + +from OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) left join +(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) +inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B + ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) + AND JP.ReasonId in (0,1,3,9,10,19,20))b +on JP.MID=b.MID + +left join +(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) +inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B + ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) + AND JP.ReasonId in (0,19))c +on JP.MID=c.MID +left join +OneApp_KelloggsMT.dbo.Master_NonWorkingReason nr on JP.ReasonId=nr.ReasonId +Where 1=1 AND JP.MID in (1060060,1060781,1061233,1061331,1061332,1061333,1061347,1061348,1061349,1061350,1061363,1061364,1061365,1061366,1061380,1061381,1061382,1061383,1061397,1061398,1061399,1061400,1061413,1061414,1061415,1061416,1061417,1061430,1061431,1061432,1061433,1061447,1061448,1061449,1061450,1061464,1061465,1061466,1061467,1061480,1061481,1061482,1061483,1061497,1061498,1061499,1061500,1061514,1061515,1061516,1061517,1061530,1061531,1061532,1061533,1061534,1061547,1061548,1061549,1061550,1061564,1061565,1061566,1061567,1061581,1061582,1061583,1061584,1061597,1061598,1061599,1061600,1061614,1061615,1061616,1061617,1061631,1061632,1061633,1061634,1061647,1061648,1061649,1061650,1061651,1061664,1061665,1061666,1061667,1061681,1061682,1061683,1061684,1061698,1061699,1061700,1061701,1061714,1061715,1061716,1061717,1061731,1061732,1061733,1061734,1061748,1061749,1061750,1061751,1061764,1061765,1061766,1061767,1061768,1061781,1061782,1061783,1061784,1061798,1061799,1061800,1061801,1061815,1061816,1061817,1061818,1061831,1061832,1061833,1061834,1061848,1061849,1061850,1061851,1061865,1061866,1061867,1061868,1061881,1061882,1061883,1061884,1061885,1061898,1061899,1061900,1061901,1061915,1061916,1061917,1061918,1061932,1061933,1061934,1061935,1061948,1061949,1061950,1061951,1061965,1061966,1061967,1061968,1061982,1061983,1061984,1061985,1061998,1061999,1062000,1062001,1062002,1062015,1062016,1062017,1062018,1062032,1062033,1062034,1062035,1062049,1062050,1062051,1062052,1062065,1062066,1062067,1062068,1062082,1062083,1062084,1062085,1062099,1062100,1062101,1062102,1062115,1062116,1062117,1062118,1062119,1062132,1062133,1062134,1062135,1060493,1060752,1060791,1060816,1061143,1061234,1061334,1061336,1061343,1061345,1061352,1061354,1061359,1061361,1061368,1061370,1061377,1061379,1061384,1061386,1061393,1061395,1061402,1061404,1061409,1061411,1061418,1061420,1061427,1061429,1061434,1061436,1061438,1061443,1061445,1061452,1061454,1061459,1061461,1061463,1061468,1061470,1061477,1061479,1061484,1061486,1061488,1061493,1061495,1061502,1061504,1061511,1061513,1061518,1061520,1061527,1061529,1061536,1061538,1061543,1061545,1061552,1061554,1061561,1061563,1061568,1061570,1061577,1061579,1061586,1061588,1061593,1061595,1061602,1061604,1061611,1061613,1061618,1061620,1061627,1061629,1061636,1061638,1061643,1061645,1061652,1061654,1061661,1061663,1061668,1061670,1061677,1061679,1061686,1061688,1061693,1061695,1061697,1061702,1061704,1061711,1061713,1061718,1061720,1061722,1061727,1061729,1061736,1061738,1061745,1061747,1061752,1061754,1061761,1061763,1061770,1061772,1061777,1061779,1061786,1061788,1061795,1061797,1061802,1061804,1061811,1061813,1061820,1061822,1061827,1061829,1061836,1061838,1061845,1061847,1061852,1061854,1061861,1061863,1061870,1061872,1061877,1061879,1061886,1061888,1061895,1061897,1061902,1061904,1061911,1061913,1061920,1061922,1061927,1061929,1061931,1061936,1061938,1061945,1061947,1061952,1061954,1061956,1061961,1061963,1061970,1061972,1061977,1061979,1061981,1061986,1061988,1061995,1061997,1062004,1062006,1062011,1062013,1062020,1062022,1062029,1062031,1062036,1062038,1062045,1062047,1062054,1062056,1062061,1062063,1062070,1062072,1062079,1062081,1062086,1062088,1062095,1062097,1062104,1062106,1062111,1062113,1062120,1062122,1062129,1062131,1062136,1062138,1062145,1062147,1060487,1060555,1060637,1060656,1061071,1061238,1061256,1061338,1061339,1061340,1061341,1061355,1061356,1061357,1061358,1061372,1061373,1061374,1061375,1061388,1061389,1061390,1061391,1061392,1061405,1061406,1061407,1061408,1061422,1061423,1061424,1061425,1061439,1061440,1061441,1061442,1061455,1061456,1061457,1061458,1061472,1061473,1061474,1061475,1061489,1061490,1061491,1061492,1061505,1061506,1061507,1061508,1061509,1061522,1061523,1061524,1061525,1061539,1061540,1061541,1061542,1061556,1061557,1061558,1061559,1061572,1061573,1061574,1061575,1061589,1061590,1061591,1061592,1061606,1061607,1061608,1061609,1061622,1061623,1061624,1061625,1061626,1061639,1061640,1061641,1061642,1061656,1061657,1061658,1061659,1061672,1061673,1061674,1061675,1061676,1061689,1061690,1061691,1061692,1061706,1061707,1061708,1061709,1061723,1061724,1061725,1061726,1061739,1061740,1061741,1061742,1061743,1061756,1061757,1061758,1061759,1061773,1061774,1061775,1061776,1061789,1061790,1061791,1061792,1061793,1061806,1061807,1061808,1061809,1061823,1061824,1061825,1061826,1061840,1061841,1061842,1061843,1061856,1061857,1061858,1061859,1061860,1061873,1061874,1061875,1061876,1061890,1061891,1061892,1061893,1061906,1061907,1061908,1061909,1061910,1061923,1061924,1061925,1061926,1061940,1061941,1061942,1061943,1061957,1061958,1061959,1061960,1061973,1061974,1061975,1061976,1061990,1061991,1061992,1061993,1062007,1062008,1062009,1062010,1062023,1062024,1062025,1062026,1062027,1062040,1062041,1062042,1062043,1062057,1062058,1062059,1062060,1062074,1062075,1062076,1062077,1062090,1062091,1062092,1062093,1062107,1062108,1062109,1062110,1062124,1062125,1062126,1062127,1062140,1062141,1062142,1062143,1062144,1060399,1060526,1060783,1061335,1061337,1061342,1061344,1061346,1061351,1061353,1061360,1061362,1061367,1061369,1061371,1061376,1061378,1061385,1061387,1061394,1061396,1061401,1061403,1061410,1061412,1061419,1061421,1061426,1061428,1061435,1061437,1061444,1061446,1061451,1061453,1061460,1061462,1061469,1061471,1061476,1061478,1061485,1061487,1061494,1061496,1061501,1061503,1061510,1061512,1061519,1061521,1061526,1061528,1061535,1061537,1061544,1061546,1061551,1061553,1061555,1061560,1061562,1061569,1061571,1061576,1061578,1061580,1061585,1061587,1061594,1061596,1061601,1061603,1061605,1061610,1061612,1061619,1061621,1061628,1061630,1061635,1061637,1061644,1061646,1061653,1061655,1061660,1061662,1061669,1061671,1061678,1061680,1061685,1061687,1061694,1061696,1061703,1061705,1061710,1061712,1061719,1061721,1061728,1061730,1061735,1061737,1061744,1061746,1061753,1061755,1061760,1061762,1061769,1061771,1061778,1061780,1061785,1061787,1061794,1061796,1061803,1061805,1061810,1061812,1061814,1061819,1061821,1061828,1061830,1061835,1061837,1061839,1061844,1061846,1061853,1061855,1061862,1061864,1061869,1061871,1061878,1061880,1061887,1061889,1061894,1061896,1061903,1061905,1061912,1061914,1061919,1061921,1061928,1061930,1061937,1061939,1061944,1061946,1061953,1061955,1061962,1061964,1061969,1061971,1061978,1061980,1061987,1061989,1061994,1061996,1062003,1062005,1062012,1062014,1062019,1062021,1062028,1062030,1062037,1062039,1062044,1062046,1062048,1062053,1062055,1062062,1062064,1062069,1062071,1062073,1062078,1062080,1062087,1062089,1062094,1062096,1062098,1062103,1062105,1062112,1062114,1062121,1062123,1062128,1062130,1062137,1062139,1062146) + +) , + Coverage + (project_id,MID, +store_id,employee_id,visit_date,in_time,out_time,duration_in_minutes, +is_covered,is_executed,reason_remarks,storetype_id, +supervisor_id,coverage_type,distance_in_meters,reasonId,camera_allow,update_date, +update_by,Unique_Id) + +AS ( + +Select '40148' as ProjectId,JP.MID, sm.StoreId,JP.EmpId as EmpId, + Convert(VARCHAR,Jp.VisitDate) AS VisitDate +, Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, + Convert(Datetime,InTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, + Convert(Datetime,InTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage Where EmpId = Jp.EmpId And StoreId = Jp.StoreId + And VisitDate = Jp.VisitDate),'') As [In Time] + , Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, + Convert(Datetime,OutTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, + Convert(Datetime,OutTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage + Where EmpId = Jp.EmpId And StoreId = Jp.StoreId + And Convert(Date,VisitDate) = Jp.VisitDate),'') As [Out Time] + , + Isnull((Select Top 1 Case When convert(varchar,outtime) like '%00:00:00.00%' THEN 0 + Else Case When outtime0 Then 'Y' +Else 'N' End As Covered +, isnull(Exe.Executed,'') as Executed + +,Isnull((Select Top 1 Case +When D.ReasonId Is Null Or D.ReasonId =0 Then '' Else Reason+' - ' + +Case When Replace(Replace(D.Remark,Char(13),''''),Char(10),' ')<>'' Then Replace(Replace(D.Remark,Char(13),''),Char(10),' ') +Else Replace(Replace(D.Remark,Char(13),''),Char(10),' ') End End As Reason +From OneApp_KelloggsMT.dbo.T_StoreCoverage D +Inner Join OneApp_KelloggsMT.dbo.Master_NonWorkingReason N On N.ReasonId = D.ReasonId +Where D.Isdel = 0 +And D.EmpId = Jp.EmpId And D.StoreId = Jp.StoreId And D.VisitDate = Jp.VisitDate),'''') As [Detailed Remarks], +sm.StoreTypeId,Em.SupervisorId, +Case When jp.Deviation=0 Then 'Planned' When jp.Deviation=1 Then 'Adhoc' +When jp.Deviation=2 Then 'Beat Plan' When jp.Deviation=3 Then 'Non Merchandised' +When jp.Deviation=4 Then 'Add New Store' When jp.Deviation=5 Then 'Non Program' else '' End as [PJP Status] +,Isnull((Select Top 1 Case When (Sc.Latitude=0.00000000 Or Sc.Longitude Is Null) Then 0 Else Case When (sm.Latitude=0.00000000 Or sm.Latitude Is Null) Then 0 + Else Case When (sm.Longitude=0.00000000 Or sm.Longitude Is Null) Then 0 +Else SQRT(POWER(69.1 * ( Sc.Latitude - Sm.Latitude),2) + POWER(69.1 * ( Sm.Longitude - Sc.Longitude ) * COS(Sc.Latitude / 57.3), 2))*1000 + End End End As [Distance In Mtr] From OneApp_KelloggsMT.dbo.T_StoreCoverage Sc Where Sc.Isdel=0 And Sc.EmpId= Jp.EmpId +And Sc.StoreId= Jp.StoreId And Sc.VisitDate= Jp.VisitDate),'') As [Distance In Mtr] +,ISNULL(CAST (JP.ReasonId AS VARCHAR),'0'),sm.CameraAllow, + +GETDATE(),'SP-Pius' , + + CAST('40148' AS VARCHAR) + '_' + CAST(SM.storeid AS VARCHAR) + + '_' + CAST(EM.EMPID AS VARCHAR) + + + +FROM OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) Inner Join +Executed Exe on Jp.MID= Exe.MID Inner Join +OneApp_KelloggsMT.dbo.vw_StoreDetail sm on Jp.StoreId= sm.StoreId Inner Join +OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on JP.EmpId= Em.EmpId +left join +OneApp_KelloggsMT.dbo.T_StoreCoveragePositionPivot pv with (nolock) +on jp.mid=pv.MID +Where 1=1 and em.UserName not like 'test%' AND JP.MID in (1060060,1060781,1061233,1061331,1061332,1061333,1061347,1061348,1061349,1061350,1061363,1061364,1061365,1061366,1061380,1061381,1061382,1061383,1061397,1061398,1061399,1061400,1061413,1061414,1061415,1061416,1061417,1061430,1061431,1061432,1061433,1061447,1061448,1061449,1061450,1061464,1061465,1061466,1061467,1061480,1061481,1061482,1061483,1061497,1061498,1061499,1061500,1061514,1061515,1061516,1061517,1061530,1061531,1061532,1061533,1061534,1061547,1061548,1061549,1061550,1061564,1061565,1061566,1061567,1061581,1061582,1061583,1061584,1061597,1061598,1061599,1061600,1061614,1061615,1061616,1061617,1061631,1061632,1061633,1061634,1061647,1061648,1061649,1061650,1061651,1061664,1061665,1061666,1061667,1061681,1061682,1061683,1061684,1061698,1061699,1061700,1061701,1061714,1061715,1061716,1061717,1061731,1061732,1061733,1061734,1061748,1061749,1061750,1061751,1061764,1061765,1061766,1061767,1061768,1061781,1061782,1061783,1061784,1061798,1061799,1061800,1061801,1061815,1061816,1061817,1061818,1061831,1061832,1061833,1061834,1061848,1061849,1061850,1061851,1061865,1061866,1061867,1061868,1061881,1061882,1061883,1061884,1061885,1061898,1061899,1061900,1061901,1061915,1061916,1061917,1061918,1061932,1061933,1061934,1061935,1061948,1061949,1061950,1061951,1061965,1061966,1061967,1061968,1061982,1061983,1061984,1061985,1061998,1061999,1062000,1062001,1062002,1062015,1062016,1062017,1062018,1062032,1062033,1062034,1062035,1062049,1062050,1062051,1062052,1062065,1062066,1062067,1062068,1062082,1062083,1062084,1062085,1062099,1062100,1062101,1062102,1062115,1062116,1062117,1062118,1062119,1062132,1062133,1062134,1062135,1060493,1060752,1060791,1060816,1061143,1061234,1061334,1061336,1061343,1061345,1061352,1061354,1061359,1061361,1061368,1061370,1061377,1061379,1061384,1061386,1061393,1061395,1061402,1061404,1061409,1061411,1061418,1061420,1061427,1061429,1061434,1061436,1061438,1061443,1061445,1061452,1061454,1061459,1061461,1061463,1061468,1061470,1061477,1061479,1061484,1061486,1061488,1061493,1061495,1061502,1061504,1061511,1061513,1061518,1061520,1061527,1061529,1061536,1061538,1061543,1061545,1061552,1061554,1061561,1061563,1061568,1061570,1061577,1061579,1061586,1061588,1061593,1061595,1061602,1061604,1061611,1061613,1061618,1061620,1061627,1061629,1061636,1061638,1061643,1061645,1061652,1061654,1061661,1061663,1061668,1061670,1061677,1061679,1061686,1061688,1061693,1061695,1061697,1061702,1061704,1061711,1061713,1061718,1061720,1061722,1061727,1061729,1061736,1061738,1061745,1061747,1061752,1061754,1061761,1061763,1061770,1061772,1061777,1061779,1061786,1061788,1061795,1061797,1061802,1061804,1061811,1061813,1061820,1061822,1061827,1061829,1061836,1061838,1061845,1061847,1061852,1061854,1061861,1061863,1061870,1061872,1061877,1061879,1061886,1061888,1061895,1061897,1061902,1061904,1061911,1061913,1061920,1061922,1061927,1061929,1061931,1061936,1061938,1061945,1061947,1061952,1061954,1061956,1061961,1061963,1061970,1061972,1061977,1061979,1061981,1061986,1061988,1061995,1061997,1062004,1062006,1062011,1062013,1062020,1062022,1062029,1062031,1062036,1062038,1062045,1062047,1062054,1062056,1062061,1062063,1062070,1062072,1062079,1062081,1062086,1062088,1062095,1062097,1062104,1062106,1062111,1062113,1062120,1062122,1062129,1062131,1062136,1062138,1062145,1062147,1060487,1060555,1060637,1060656,1061071,1061238,1061256,1061338,1061339,1061340,1061341,1061355,1061356,1061357,1061358,1061372,1061373,1061374,1061375,1061388,1061389,1061390,1061391,1061392,1061405,1061406,1061407,1061408,1061422,1061423,1061424,1061425,1061439,1061440,1061441,1061442,1061455,1061456,1061457,1061458,1061472,1061473,1061474,1061475,1061489,1061490,1061491,1061492,1061505,1061506,1061507,1061508,1061509,1061522,1061523,1061524,1061525,1061539,1061540,1061541,1061542,1061556,1061557,1061558,1061559,1061572,1061573,1061574,1061575,1061589,1061590,1061591,1061592,1061606,1061607,1061608,1061609,1061622,1061623,1061624,1061625,1061626,1061639,1061640,1061641,1061642,1061656,1061657,1061658,1061659,1061672,1061673,1061674,1061675,1061676,1061689,1061690,1061691,1061692,1061706,1061707,1061708,1061709,1061723,1061724,1061725,1061726,1061739,1061740,1061741,1061742,1061743,1061756,1061757,1061758,1061759,1061773,1061774,1061775,1061776,1061789,1061790,1061791,1061792,1061793,1061806,1061807,1061808,1061809,1061823,1061824,1061825,1061826,1061840,1061841,1061842,1061843,1061856,1061857,1061858,1061859,1061860,1061873,1061874,1061875,1061876,1061890,1061891,1061892,1061893,1061906,1061907,1061908,1061909,1061910,1061923,1061924,1061925,1061926,1061940,1061941,1061942,1061943,1061957,1061958,1061959,1061960,1061973,1061974,1061975,1061976,1061990,1061991,1061992,1061993,1062007,1062008,1062009,1062010,1062023,1062024,1062025,1062026,1062027,1062040,1062041,1062042,1062043,1062057,1062058,1062059,1062060,1062074,1062075,1062076,1062077,1062090,1062091,1062092,1062093,1062107,1062108,1062109,1062110,1062124,1062125,1062126,1062127,1062140,1062141,1062142,1062143,1062144,1060399,1060526,1060783,1061335,1061337,1061342,1061344,1061346,1061351,1061353,1061360,1061362,1061367,1061369,1061371,1061376,1061378,1061385,1061387,1061394,1061396,1061401,1061403,1061410,1061412,1061419,1061421,1061426,1061428,1061435,1061437,1061444,1061446,1061451,1061453,1061460,1061462,1061469,1061471,1061476,1061478,1061485,1061487,1061494,1061496,1061501,1061503,1061510,1061512,1061519,1061521,1061526,1061528,1061535,1061537,1061544,1061546,1061551,1061553,1061555,1061560,1061562,1061569,1061571,1061576,1061578,1061580,1061585,1061587,1061594,1061596,1061601,1061603,1061605,1061610,1061612,1061619,1061621,1061628,1061630,1061635,1061637,1061644,1061646,1061653,1061655,1061660,1061662,1061669,1061671,1061678,1061680,1061685,1061687,1061694,1061696,1061703,1061705,1061710,1061712,1061719,1061721,1061728,1061730,1061735,1061737,1061744,1061746,1061753,1061755,1061760,1061762,1061769,1061771,1061778,1061780,1061785,1061787,1061794,1061796,1061803,1061805,1061810,1061812,1061814,1061819,1061821,1061828,1061830,1061835,1061837,1061839,1061844,1061846,1061853,1061855,1061862,1061864,1061869,1061871,1061878,1061880,1061887,1061889,1061894,1061896,1061903,1061905,1061912,1061914,1061919,1061921,1061928,1061930,1061937,1061939,1061944,1061946,1061953,1061955,1061962,1061964,1061969,1061971,1061978,1061980,1061987,1061989,1061994,1061996,1062003,1062005,1062012,1062014,1062019,1062021,1062028,1062030,1062037,1062039,1062044,1062046,1062048,1062053,1062055,1062062,1062064,1062069,1062071,1062073,1062078,1062080,1062087,1062089,1062094,1062096,1062098,1062103,1062105,1062112,1062114,1062121,1062123,1062128,1062130,1062137,1062139,1062146 ) + +) +select * from Coverage] +(Background on this error at: https://sqlalche.me/e/20/f405) +2026-06-18 13:26:03 | INFO | ================================================================================ +2026-06-18 13:26:03 | INFO | Hello from data-move Python data pipeline! +2026-06-18 13:26:03 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 13:26:03 | INFO | Connecting to databases... +2026-06-18 13:26:05 | INFO | +2026-06-18 13:26:07 | INFO | +2026-06-18 13:26:07 | INFO | Database connections established +2026-06-18 13:26:07 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 13:26:08 | INFO | Found 836 MIDs +2026-06-18 13:26:09 | INFO | ================================================================================ +2026-06-18 13:26:09 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:26:09 | INFO | Fetching Data from sql server for table-:additional_visibility--------------------------- +2026-06-18 13:26:09 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 13:26:09 | INFO | Fetching data for 836 MIDs +2026-06-18 13:26:12 | INFO | Fetched 1,885 rows from SQL Server +2026-06-18 13:26:12 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! +2026-06-18 13:26:12 | INFO | Fetched 1885 rows +2026-06-18 13:26:13 | INFO | Deleting from additional_visibility +2026-06-18 13:26:13 | INFO | inserting data into clickhouse db from sql server------------------ +2026-06-18 13:26:13 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse +2026-06-18 13:26:13 | INFO | additional_visibility loaded successfully (1885 rows) +2026-06-18 13:26:13 | INFO | ================================================================================ +2026-06-18 13:26:13 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:26:13 | INFO | Fetching Data from sql server for table-:Coverage--------------------------- +2026-06-18 13:26:13 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 13:26:13 | INFO | Fetching data for 836 MIDs +2026-06-18 13:26:14 | ERROR | Failed processing table Coverage +Traceback (most recent call last): + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'as'. (156) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ','. (102)") + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "D:\data_move\main2.py", line 181, in main + df = get_dataframe( + sql_engine, + ...<5 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 80, in get_dataframe + return fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\data_move\src\fact.py", line 364, in fetch_Coverage + df = pl.read_database( + query=sql, + connection=engine + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database + return cx.execute( + ~~~~~~~~~~^ + query=query, + ^^^^^^^^^^^^ + options=execute_options, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ).to_polars( + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute + result = cursor_execute(query, **options) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute + return meth( + self, + distilled_parameters, + execution_options or NO_OPTIONS, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection + return connection._execute_clauseelement( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + self, distilled_params, execution_options + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement + ret = self._execute_context( + dialect, + ...<8 lines>... + cache_hit=cache_hit, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context + return self._exec_single_context( + ~~~~~~~~~~~~~~~~~~~~~~~~~^ + dialect, context, statement, parameters + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context + self._handle_dbapi_exception( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + e, str_statement, effective_parameters, cursor, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'as'. (156) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ','. (102)") +[SQL: +Executed as +(select JP.mid, CASE WHEN isnull(B.MID,'')<>'' then 'Y' +ELSE 'N' END AS COVERED, +CASE WHEN isnull(C.MID,'')<>'' then 'Y' +ELSE 'N' END AS EXECUTED,isnull(NR.REASON,'') AS REASON + +from OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) left join +(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) +inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B + ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) + AND JP.ReasonId in (0,1,3,9,10,19,20))b +on JP.MID=b.MID + +left join +(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) +inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B + ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) + AND JP.ReasonId in (0,19))c +on JP.MID=c.MID +left join +OneApp_KelloggsMT.dbo.Master_NonWorkingReason nr on JP.ReasonId=nr.ReasonId +Where 1=1 AND JP.MID in (1060487,1060555,1060637,1060656,1061071,1061238,1061256,1061338,1061339,1061340,1061341,1061355,1061356,1061357,1061358,1061372,1061373,1061374,1061375,1061388,1061389,1061390,1061391,1061392,1061405,1061406,1061407,1061408,1061422,1061423,1061424,1061425,1061439,1061440,1061441,1061442,1061455,1061456,1061457,1061458,1061472,1061473,1061474,1061475,1061489,1061490,1061491,1061492,1061505,1061506,1061507,1061508,1061509,1061522,1061523,1061524,1061525,1061539,1061540,1061541,1061542,1061556,1061557,1061558,1061559,1061572,1061573,1061574,1061575,1061589,1061590,1061591,1061592,1061606,1061607,1061608,1061609,1061622,1061623,1061624,1061625,1061626,1061639,1061640,1061641,1061642,1061656,1061657,1061658,1061659,1061672,1061673,1061674,1061675,1061676,1061689,1061690,1061691,1061692,1061706,1061707,1061708,1061709,1061723,1061724,1061725,1061726,1061739,1061740,1061741,1061742,1061743,1061756,1061757,1061758,1061759,1061773,1061774,1061775,1061776,1061789,1061790,1061791,1061792,1061793,1061806,1061807,1061808,1061809,1061823,1061824,1061825,1061826,1061840,1061841,1061842,1061843,1061856,1061857,1061858,1061859,1061860,1061873,1061874,1061875,1061876,1061890,1061891,1061892,1061893,1061906,1061907,1061908,1061909,1061910,1061923,1061924,1061925,1061926,1061940,1061941,1061942,1061943,1061957,1061958,1061959,1061960,1061973,1061974,1061975,1061976,1061990,1061991,1061992,1061993,1062007,1062008,1062009,1062010,1062023,1062024,1062025,1062026,1062027,1062040,1062041,1062042,1062043,1062057,1062058,1062059,1062060,1062074,1062075,1062076,1062077,1062090,1062091,1062092,1062093,1062107,1062108,1062109,1062110,1062124,1062125,1062126,1062127,1062140,1062141,1062142,1062143,1062144,1060399,1060526,1060783,1061335,1061337,1061342,1061344,1061346,1061351,1061353,1061360,1061362,1061367,1061369,1061371,1061376,1061378,1061385,1061387,1061394,1061396,1061401,1061403,1061410,1061412,1061419,1061421,1061426,1061428,1061435,1061437,1061444,1061446,1061451,1061453,1061460,1061462,1061469,1061471,1061476,1061478,1061485,1061487,1061494,1061496,1061501,1061503,1061510,1061512,1061519,1061521,1061526,1061528,1061535,1061537,1061544,1061546,1061551,1061553,1061555,1061560,1061562,1061569,1061571,1061576,1061578,1061580,1061585,1061587,1061594,1061596,1061601,1061603,1061605,1061610,1061612,1061619,1061621,1061628,1061630,1061635,1061637,1061644,1061646,1061653,1061655,1061660,1061662,1061669,1061671,1061678,1061680,1061685,1061687,1061694,1061696,1061703,1061705,1061710,1061712,1061719,1061721,1061728,1061730,1061735,1061737,1061744,1061746,1061753,1061755,1061760,1061762,1061769,1061771,1061778,1061780,1061785,1061787,1061794,1061796,1061803,1061805,1061810,1061812,1061814,1061819,1061821,1061828,1061830,1061835,1061837,1061839,1061844,1061846,1061853,1061855,1061862,1061864,1061869,1061871,1061878,1061880,1061887,1061889,1061894,1061896,1061903,1061905,1061912,1061914,1061919,1061921,1061928,1061930,1061937,1061939,1061944,1061946,1061953,1061955,1061962,1061964,1061969,1061971,1061978,1061980,1061987,1061989,1061994,1061996,1062003,1062005,1062012,1062014,1062019,1062021,1062028,1062030,1062037,1062039,1062044,1062046,1062048,1062053,1062055,1062062,1062064,1062069,1062071,1062073,1062078,1062080,1062087,1062089,1062094,1062096,1062098,1062103,1062105,1062112,1062114,1062121,1062123,1062128,1062130,1062137,1062139,1062146,1060493,1060752,1060791,1060816,1061143,1061234,1061334,1061336,1061343,1061345,1061352,1061354,1061359,1061361,1061368,1061370,1061377,1061379,1061384,1061386,1061393,1061395,1061402,1061404,1061409,1061411,1061418,1061420,1061427,1061429,1061434,1061436,1061438,1061443,1061445,1061452,1061454,1061459,1061461,1061463,1061468,1061470,1061477,1061479,1061484,1061486,1061488,1061493,1061495,1061502,1061504,1061511,1061513,1061518,1061520,1061527,1061529,1061536,1061538,1061543,1061545,1061552,1061554,1061561,1061563,1061568,1061570,1061577,1061579,1061586,1061588,1061593,1061595,1061602,1061604,1061611,1061613,1061618,1061620,1061627,1061629,1061636,1061638,1061643,1061645,1061652,1061654,1061661,1061663,1061668,1061670,1061677,1061679,1061686,1061688,1061693,1061695,1061697,1061702,1061704,1061711,1061713,1061718,1061720,1061722,1061727,1061729,1061736,1061738,1061745,1061747,1061752,1061754,1061761,1061763,1061770,1061772,1061777,1061779,1061786,1061788,1061795,1061797,1061802,1061804,1061811,1061813,1061820,1061822,1061827,1061829,1061836,1061838,1061845,1061847,1061852,1061854,1061861,1061863,1061870,1061872,1061877,1061879,1061886,1061888,1061895,1061897,1061902,1061904,1061911,1061913,1061920,1061922,1061927,1061929,1061931,1061936,1061938,1061945,1061947,1061952,1061954,1061956,1061961,1061963,1061970,1061972,1061977,1061979,1061981,1061986,1061988,1061995,1061997,1062004,1062006,1062011,1062013,1062020,1062022,1062029,1062031,1062036,1062038,1062045,1062047,1062054,1062056,1062061,1062063,1062070,1062072,1062079,1062081,1062086,1062088,1062095,1062097,1062104,1062106,1062111,1062113,1062120,1062122,1062129,1062131,1062136,1062138,1062145,1062147,1060060,1060781,1061233,1061331,1061332,1061333,1061347,1061348,1061349,1061350,1061363,1061364,1061365,1061366,1061380,1061381,1061382,1061383,1061397,1061398,1061399,1061400,1061413,1061414,1061415,1061416,1061417,1061430,1061431,1061432,1061433,1061447,1061448,1061449,1061450,1061464,1061465,1061466,1061467,1061480,1061481,1061482,1061483,1061497,1061498,1061499,1061500,1061514,1061515,1061516,1061517,1061530,1061531,1061532,1061533,1061534,1061547,1061548,1061549,1061550,1061564,1061565,1061566,1061567,1061581,1061582,1061583,1061584,1061597,1061598,1061599,1061600,1061614,1061615,1061616,1061617,1061631,1061632,1061633,1061634,1061647,1061648,1061649,1061650,1061651,1061664,1061665,1061666,1061667,1061681,1061682,1061683,1061684,1061698,1061699,1061700,1061701,1061714,1061715,1061716,1061717,1061731,1061732,1061733,1061734,1061748,1061749,1061750,1061751,1061764,1061765,1061766,1061767,1061768,1061781,1061782,1061783,1061784,1061798,1061799,1061800,1061801,1061815,1061816,1061817,1061818,1061831,1061832,1061833,1061834,1061848,1061849,1061850,1061851,1061865,1061866,1061867,1061868,1061881,1061882,1061883,1061884,1061885,1061898,1061899,1061900,1061901,1061915,1061916,1061917,1061918,1061932,1061933,1061934,1061935,1061948,1061949,1061950,1061951,1061965,1061966,1061967,1061968,1061982,1061983,1061984,1061985,1061998,1061999,1062000,1062001,1062002,1062015,1062016,1062017,1062018,1062032,1062033,1062034,1062035,1062049,1062050,1062051,1062052,1062065,1062066,1062067,1062068,1062082,1062083,1062084,1062085,1062099,1062100,1062101,1062102,1062115,1062116,1062117,1062118,1062119,1062132,1062133,1062134,1062135) + +) , + Coverage + (project_id,MID, +store_id,employee_id,visit_date,in_time,out_time,duration_in_minutes, +is_covered,is_executed,reason_remarks,storetype_id, +supervisor_id,coverage_type,distance_in_meters,reasonId,camera_allow,update_date, +update_by,Unique_Id) + +AS ( + +Select '40148' as ProjectId,JP.MID, sm.StoreId,JP.EmpId as EmpId, + Convert(VARCHAR,Jp.VisitDate) AS VisitDate +, Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, + Convert(Datetime,InTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, + Convert(Datetime,InTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage Where EmpId = Jp.EmpId And StoreId = Jp.StoreId + And VisitDate = Jp.VisitDate),'') As [In Time] + , Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, + Convert(Datetime,OutTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, + Convert(Datetime,OutTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage + Where EmpId = Jp.EmpId And StoreId = Jp.StoreId + And Convert(Date,VisitDate) = Jp.VisitDate),'') As [Out Time] + , + Isnull((Select Top 1 Case When convert(varchar,outtime) like '%00:00:00.00%' THEN 0 + Else Case When outtime0 Then 'Y' +Else 'N' End As Covered +, isnull(Exe.Executed,'') as Executed + +,Isnull((Select Top 1 Case +When D.ReasonId Is Null Or D.ReasonId =0 Then '' Else Reason+' - ' + +Case When Replace(Replace(D.Remark,Char(13),''''),Char(10),' ')<>'' Then Replace(Replace(D.Remark,Char(13),''),Char(10),' ') +Else Replace(Replace(D.Remark,Char(13),''),Char(10),' ') End End As Reason +From OneApp_KelloggsMT.dbo.T_StoreCoverage D +Inner Join OneApp_KelloggsMT.dbo.Master_NonWorkingReason N On N.ReasonId = D.ReasonId +Where D.Isdel = 0 +And D.EmpId = Jp.EmpId And D.StoreId = Jp.StoreId And D.VisitDate = Jp.VisitDate),'''') As [Detailed Remarks], +sm.StoreTypeId,Em.SupervisorId, +Case When jp.Deviation=0 Then 'Planned' When jp.Deviation=1 Then 'Adhoc' +When jp.Deviation=2 Then 'Beat Plan' When jp.Deviation=3 Then 'Non Merchandised' +When jp.Deviation=4 Then 'Add New Store' When jp.Deviation=5 Then 'Non Program' else '' End as [PJP Status] +,Isnull((Select Top 1 Case When (Sc.Latitude=0.00000000 Or Sc.Longitude Is Null) Then 0 Else Case When (sm.Latitude=0.00000000 Or sm.Latitude Is Null) Then 0 + Else Case When (sm.Longitude=0.00000000 Or sm.Longitude Is Null) Then 0 +Else SQRT(POWER(69.1 * ( Sc.Latitude - Sm.Latitude),2) + POWER(69.1 * ( Sm.Longitude - Sc.Longitude ) * COS(Sc.Latitude / 57.3), 2))*1000 + End End End As [Distance In Mtr] From OneApp_KelloggsMT.dbo.T_StoreCoverage Sc Where Sc.Isdel=0 And Sc.EmpId= Jp.EmpId +And Sc.StoreId= Jp.StoreId And Sc.VisitDate= Jp.VisitDate),'') As [Distance In Mtr] +,ISNULL(CAST (JP.ReasonId AS VARCHAR),'0'),sm.CameraAllow, + +GETDATE(),'SP-Pius' , + + CAST('40148' AS VARCHAR) + '_' + CAST(SM.storeid AS VARCHAR) + + '_' + CAST(EM.EMPID AS VARCHAR) + + + +FROM OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) Inner Join +Executed Exe on Jp.MID= Exe.MID Inner Join +OneApp_KelloggsMT.dbo.vw_StoreDetail sm on Jp.StoreId= sm.StoreId Inner Join +OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on JP.EmpId= Em.EmpId +left join +OneApp_KelloggsMT.dbo.T_StoreCoveragePositionPivot pv with (nolock) +on jp.mid=pv.MID +Where 1=1 and em.UserName not like 'test%' AND JP.MID in (1060487,1060555,1060637,1060656,1061071,1061238,1061256,1061338,1061339,1061340,1061341,1061355,1061356,1061357,1061358,1061372,1061373,1061374,1061375,1061388,1061389,1061390,1061391,1061392,1061405,1061406,1061407,1061408,1061422,1061423,1061424,1061425,1061439,1061440,1061441,1061442,1061455,1061456,1061457,1061458,1061472,1061473,1061474,1061475,1061489,1061490,1061491,1061492,1061505,1061506,1061507,1061508,1061509,1061522,1061523,1061524,1061525,1061539,1061540,1061541,1061542,1061556,1061557,1061558,1061559,1061572,1061573,1061574,1061575,1061589,1061590,1061591,1061592,1061606,1061607,1061608,1061609,1061622,1061623,1061624,1061625,1061626,1061639,1061640,1061641,1061642,1061656,1061657,1061658,1061659,1061672,1061673,1061674,1061675,1061676,1061689,1061690,1061691,1061692,1061706,1061707,1061708,1061709,1061723,1061724,1061725,1061726,1061739,1061740,1061741,1061742,1061743,1061756,1061757,1061758,1061759,1061773,1061774,1061775,1061776,1061789,1061790,1061791,1061792,1061793,1061806,1061807,1061808,1061809,1061823,1061824,1061825,1061826,1061840,1061841,1061842,1061843,1061856,1061857,1061858,1061859,1061860,1061873,1061874,1061875,1061876,1061890,1061891,1061892,1061893,1061906,1061907,1061908,1061909,1061910,1061923,1061924,1061925,1061926,1061940,1061941,1061942,1061943,1061957,1061958,1061959,1061960,1061973,1061974,1061975,1061976,1061990,1061991,1061992,1061993,1062007,1062008,1062009,1062010,1062023,1062024,1062025,1062026,1062027,1062040,1062041,1062042,1062043,1062057,1062058,1062059,1062060,1062074,1062075,1062076,1062077,1062090,1062091,1062092,1062093,1062107,1062108,1062109,1062110,1062124,1062125,1062126,1062127,1062140,1062141,1062142,1062143,1062144,1060399,1060526,1060783,1061335,1061337,1061342,1061344,1061346,1061351,1061353,1061360,1061362,1061367,1061369,1061371,1061376,1061378,1061385,1061387,1061394,1061396,1061401,1061403,1061410,1061412,1061419,1061421,1061426,1061428,1061435,1061437,1061444,1061446,1061451,1061453,1061460,1061462,1061469,1061471,1061476,1061478,1061485,1061487,1061494,1061496,1061501,1061503,1061510,1061512,1061519,1061521,1061526,1061528,1061535,1061537,1061544,1061546,1061551,1061553,1061555,1061560,1061562,1061569,1061571,1061576,1061578,1061580,1061585,1061587,1061594,1061596,1061601,1061603,1061605,1061610,1061612,1061619,1061621,1061628,1061630,1061635,1061637,1061644,1061646,1061653,1061655,1061660,1061662,1061669,1061671,1061678,1061680,1061685,1061687,1061694,1061696,1061703,1061705,1061710,1061712,1061719,1061721,1061728,1061730,1061735,1061737,1061744,1061746,1061753,1061755,1061760,1061762,1061769,1061771,1061778,1061780,1061785,1061787,1061794,1061796,1061803,1061805,1061810,1061812,1061814,1061819,1061821,1061828,1061830,1061835,1061837,1061839,1061844,1061846,1061853,1061855,1061862,1061864,1061869,1061871,1061878,1061880,1061887,1061889,1061894,1061896,1061903,1061905,1061912,1061914,1061919,1061921,1061928,1061930,1061937,1061939,1061944,1061946,1061953,1061955,1061962,1061964,1061969,1061971,1061978,1061980,1061987,1061989,1061994,1061996,1062003,1062005,1062012,1062014,1062019,1062021,1062028,1062030,1062037,1062039,1062044,1062046,1062048,1062053,1062055,1062062,1062064,1062069,1062071,1062073,1062078,1062080,1062087,1062089,1062094,1062096,1062098,1062103,1062105,1062112,1062114,1062121,1062123,1062128,1062130,1062137,1062139,1062146,1060493,1060752,1060791,1060816,1061143,1061234,1061334,1061336,1061343,1061345,1061352,1061354,1061359,1061361,1061368,1061370,1061377,1061379,1061384,1061386,1061393,1061395,1061402,1061404,1061409,1061411,1061418,1061420,1061427,1061429,1061434,1061436,1061438,1061443,1061445,1061452,1061454,1061459,1061461,1061463,1061468,1061470,1061477,1061479,1061484,1061486,1061488,1061493,1061495,1061502,1061504,1061511,1061513,1061518,1061520,1061527,1061529,1061536,1061538,1061543,1061545,1061552,1061554,1061561,1061563,1061568,1061570,1061577,1061579,1061586,1061588,1061593,1061595,1061602,1061604,1061611,1061613,1061618,1061620,1061627,1061629,1061636,1061638,1061643,1061645,1061652,1061654,1061661,1061663,1061668,1061670,1061677,1061679,1061686,1061688,1061693,1061695,1061697,1061702,1061704,1061711,1061713,1061718,1061720,1061722,1061727,1061729,1061736,1061738,1061745,1061747,1061752,1061754,1061761,1061763,1061770,1061772,1061777,1061779,1061786,1061788,1061795,1061797,1061802,1061804,1061811,1061813,1061820,1061822,1061827,1061829,1061836,1061838,1061845,1061847,1061852,1061854,1061861,1061863,1061870,1061872,1061877,1061879,1061886,1061888,1061895,1061897,1061902,1061904,1061911,1061913,1061920,1061922,1061927,1061929,1061931,1061936,1061938,1061945,1061947,1061952,1061954,1061956,1061961,1061963,1061970,1061972,1061977,1061979,1061981,1061986,1061988,1061995,1061997,1062004,1062006,1062011,1062013,1062020,1062022,1062029,1062031,1062036,1062038,1062045,1062047,1062054,1062056,1062061,1062063,1062070,1062072,1062079,1062081,1062086,1062088,1062095,1062097,1062104,1062106,1062111,1062113,1062120,1062122,1062129,1062131,1062136,1062138,1062145,1062147,1060060,1060781,1061233,1061331,1061332,1061333,1061347,1061348,1061349,1061350,1061363,1061364,1061365,1061366,1061380,1061381,1061382,1061383,1061397,1061398,1061399,1061400,1061413,1061414,1061415,1061416,1061417,1061430,1061431,1061432,1061433,1061447,1061448,1061449,1061450,1061464,1061465,1061466,1061467,1061480,1061481,1061482,1061483,1061497,1061498,1061499,1061500,1061514,1061515,1061516,1061517,1061530,1061531,1061532,1061533,1061534,1061547,1061548,1061549,1061550,1061564,1061565,1061566,1061567,1061581,1061582,1061583,1061584,1061597,1061598,1061599,1061600,1061614,1061615,1061616,1061617,1061631,1061632,1061633,1061634,1061647,1061648,1061649,1061650,1061651,1061664,1061665,1061666,1061667,1061681,1061682,1061683,1061684,1061698,1061699,1061700,1061701,1061714,1061715,1061716,1061717,1061731,1061732,1061733,1061734,1061748,1061749,1061750,1061751,1061764,1061765,1061766,1061767,1061768,1061781,1061782,1061783,1061784,1061798,1061799,1061800,1061801,1061815,1061816,1061817,1061818,1061831,1061832,1061833,1061834,1061848,1061849,1061850,1061851,1061865,1061866,1061867,1061868,1061881,1061882,1061883,1061884,1061885,1061898,1061899,1061900,1061901,1061915,1061916,1061917,1061918,1061932,1061933,1061934,1061935,1061948,1061949,1061950,1061951,1061965,1061966,1061967,1061968,1061982,1061983,1061984,1061985,1061998,1061999,1062000,1062001,1062002,1062015,1062016,1062017,1062018,1062032,1062033,1062034,1062035,1062049,1062050,1062051,1062052,1062065,1062066,1062067,1062068,1062082,1062083,1062084,1062085,1062099,1062100,1062101,1062102,1062115,1062116,1062117,1062118,1062119,1062132,1062133,1062134,1062135 ) + +) +select * from Coverage] +(Background on this error at: https://sqlalche.me/e/20/f405) +2026-06-18 13:35:43 | INFO | ================================================================================ +2026-06-18 13:35:43 | INFO | Hello from data-move Python data pipeline! +2026-06-18 13:35:43 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 13:35:43 | INFO | Connecting to databases... +2026-06-18 13:35:44 | INFO | +2026-06-18 13:35:46 | INFO | +2026-06-18 13:35:46 | INFO | Database connections established +2026-06-18 13:35:46 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 13:35:47 | INFO | Found 836 MIDs +2026-06-18 13:35:48 | INFO | ================================================================================ +2026-06-18 13:35:48 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:35:48 | INFO | Fetching Data from sql server for table-:additional_visibility--------------------------- +2026-06-18 13:35:48 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 13:35:48 | INFO | Fetching data for 836 MIDs +2026-06-18 13:35:49 | INFO | Fetched 1,885 rows from SQL Server +2026-06-18 13:35:49 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! +2026-06-18 13:35:49 | INFO | Fetched 1885 rows +2026-06-18 13:35:49 | INFO | Deleting from additional_visibility +2026-06-18 13:35:49 | INFO | inserting data into clickhouse db from sql server------------------ +2026-06-18 13:35:49 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse +2026-06-18 13:35:49 | INFO | additional_visibility loaded successfully (1885 rows) +2026-06-18 13:35:49 | INFO | ================================================================================ +2026-06-18 13:35:49 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:35:49 | INFO | Fetching Data from sql server for table-:Coverage--------------------------- +2026-06-18 13:35:49 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 13:35:49 | INFO | Fetching data for 836 MIDs +2026-06-18 13:36:18 | INFO | Fetched 779 rows from SQL Server +2026-06-18 13:36:18 | INFO | Fetched total row -: 779 from sql server for table-:Coverage ...........!!! +2026-06-18 13:36:18 | INFO | Fetched 779 rows +2026-06-18 13:36:18 | INFO | Creating table Coverage +2026-06-18 13:36:18 | INFO | Table ready: Coverage +2026-06-18 13:36:19 | INFO | inserting data into clickhouse db from sql server------------------ +2026-06-18 13:36:19 | INFO | Coverage: inserted 779 rows into ClickHouse +2026-06-18 13:36:19 | INFO | Coverage loaded successfully (779 rows) +2026-06-18 13:36:19 | INFO | ================================================================================ +2026-06-18 13:36:19 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:36:19 | INFO | Fetching Data from sql server for table-:Survey--------------------------- +2026-06-18 13:36:19 | INFO | Fetching data for 836 MIDs +2026-06-18 13:36:20 | INFO | Fetched 141 rows from SQL Server +2026-06-18 13:36:20 | INFO | Fetched total row -: 141 from sql server for table-:Survey ...........!!! +2026-06-18 13:36:20 | INFO | Fetched 141 rows +2026-06-18 13:36:20 | INFO | Creating table Survey +2026-06-18 13:36:20 | INFO | Table ready: Survey +2026-06-18 13:36:20 | INFO | inserting data into clickhouse db from sql server------------------ +2026-06-18 13:36:20 | INFO | Survey: inserted 141 rows into ClickHouse +2026-06-18 13:36:20 | INFO | Survey loaded successfully (141 rows) +2026-06-18 13:36:20 | INFO | ================================================================================ +2026-06-18 13:36:20 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-18 13:36:20 | INFO | Fetching Data from sql server for table-:Login--------------------------- +2026-06-18 13:36:20 | INFO | Fetching data for 836 MIDs +2026-06-18 13:36:22 | INFO | Fetched 475 rows from SQL Server +2026-06-18 13:36:22 | INFO | Fetched total row -: 475 from sql server for table-:Login ...........!!! +2026-06-18 13:36:22 | INFO | Fetched 475 rows +2026-06-18 13:36:22 | INFO | Creating table Login +2026-06-18 13:36:23 | INFO | Table ready: Login +2026-06-18 13:36:23 | INFO | inserting data into clickhouse db from sql server------------------ +2026-06-18 13:36:23 | INFO | Login: inserted 475 rows into ClickHouse +2026-06-18 13:36:23 | INFO | Login loaded successfully (475 rows) +2026-06-18 13:36:23 | INFO | ================================================================================ +2026-06-18 13:36:23 | INFO | Processing Table: Stock_Details | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:36:23 | INFO | Fetching Data from sql server for table-:Stock_Details--------------------------- +2026-06-18 13:36:23 | INFO | Fetching data for 836 MIDs +2026-06-18 13:36:29 | INFO | Fetched 39,313 rows from SQL Server +2026-06-18 13:36:29 | INFO | Fetched total row -: 39313 from sql server for table-:Stock_Details ...........!!! +2026-06-18 13:36:29 | INFO | Fetched 39313 rows +2026-06-18 13:36:29 | INFO | Creating table Stock_Details +2026-06-18 13:36:30 | INFO | Table ready: Stock_Details +2026-06-18 13:36:30 | INFO | inserting data into clickhouse db from sql server------------------ +2026-06-18 13:36:30 | INFO | Stock_Details: inserted 39,313 rows into ClickHouse +2026-06-18 13:36:30 | INFO | Stock_Details loaded successfully (39313 rows) +2026-06-18 13:36:30 | INFO | ================================================================================ +2026-06-18 13:36:30 | INFO | Pipeline Completed Successfully +2026-06-18 13:36:30 | INFO | ================================================================================ +2026-06-18 13:37:48 | INFO | ================================================================================ +2026-06-18 13:37:48 | INFO | Hello from data-move Python data pipeline! +2026-06-18 13:37:48 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 13:37:48 | INFO | Connecting to databases... +2026-06-18 13:37:49 | INFO | +2026-06-18 13:37:51 | INFO | +2026-06-18 13:37:51 | INFO | Database connections established +2026-06-18 13:37:51 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 13:37:52 | INFO | Found 836 MIDs +2026-06-18 13:37:53 | INFO | ================================================================================ +2026-06-18 13:37:53 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:37:53 | INFO | Fetching Data from sql server for table-:additional_visibility .............. +2026-06-18 13:37:53 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 13:37:53 | INFO | Fetching data for 836 MIDs +2026-06-18 13:37:54 | INFO | Fetched 1,885 rows from SQL Server +2026-06-18 13:37:54 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! +2026-06-18 13:37:54 | INFO | Fetched 1885 rows +2026-06-18 13:37:54 | INFO | Deleting from additional_visibility +2026-06-18 13:37:54 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:37:54 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse +2026-06-18 13:37:54 | INFO | additional_visibility loaded successfully (1885 rows) +2026-06-18 13:37:54 | INFO | ================================================================================ +2026-06-18 13:37:54 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:37:54 | INFO | Fetching Data from sql server for table-:Coverage .............. +2026-06-18 13:37:54 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 13:37:54 | INFO | Fetching data for 836 MIDs +2026-06-18 13:38:17 | INFO | Fetched 779 rows from SQL Server +2026-06-18 13:38:17 | INFO | Fetched total row -: 779 from sql server for table-:Coverage ...........!!! +2026-06-18 13:38:17 | INFO | Fetched 779 rows +2026-06-18 13:38:17 | INFO | Deleting from Coverage +2026-06-18 13:38:17 | ERROR | Failed processing table Coverage +Traceback (most recent call last): + File "D:\data_move\main2.py", line 243, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 79, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + table_name, + ^^^^^^^^^^^ + f"Mid IN ({mids_str})", + ^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'Mid' while processing: 'isZeroOrNull(Mid IN (1060060, 1060781, 1061233, 1061331, 1061332, 1061333, 1061347, 1061348, 1061349, 1061350, 1061363, 1061364, 1061365, 1061366, 1061380, 1061381, 1061382, 1061383, 1061397, 1061398, 1061399, 1061400, 1061413, 1061414, 1061415, 1061416, 1061417, 1061430, 1061431, 1061432, 1061433, 1061447, 1061448, 1061449, 1061450, 1061464, 1061465, 1061466, 1061467, 1061480, 1061481, 1061482, 1061483, 1061497, 1061498, 1061499, 1061500, 1061514, 1061515, 1061516, 1061517, 1061530, 1061531, 1061532, 1061533, 1061534, 1061547, 1061548, 1061549, 1061550, 1061564, 1061565, 1061566, 1061567, 1061581, 1061582, 1061583, 1061584, 1061597, 1061598, 1061599, 1061600, 1061614, 1061615, 1061616, 1061617, 1061631, 1061632, 1061633, 1061634, 1061647, 1061648, 1061649, 1061650, 1061651, 1061664, 1061665, 1061666, 1061667, 1061681, 1061682, 1061683, 1061684, 1061698, 1061699, 1061700, 1061701, 1061714, 1061715, 1061716, 1061717, 1061731, 1061732, 1061733, 1061734, 1061748, 1061749, 1061750, 1061751, 1061764, 1061765, 1061766, 1061767, 1061768, 1061781, 1061782, 1061783, 1061784, 1061798, 1061799, 1061800, 1061801, 1061815, 1061816, 1061817, 1061818, 1061831, 1061832, 1061833, 1061834, 1061848, 1061849, 1061850, 1061851, 1061865, 1061866, 1061867, 1061868, 1061881, 1061882, 1061883, 1061884, 1061885, 1061898, 1061899, 1061900, 1061901, 1061915, 1061916, 1061917, 1061918, 1061932, 1061933, 1061934, 1061935, 1061948, 1061949, 1061950, 1061951, 1061965, 1061966, 1061967, 1061968, 1061982, 1061983, 1061984, 1061985, 1061998, 1061999, 1062000, 1062001, 1062002, 1062015, 1062016, 1062017, 1062018, 1062032, 1062033, 1062034, 1062035, 1062049, 1062050, 1062051, 1062052, 1062065, 1062066, 1062067, 1062068, 1062082, 1062083, 1062084, 1062085, 1062099, 1062100, 1062101, 1062102, 1062115, 1062116, 1062117, 1062118, 1062119, 1062132, 1062133, 1062134, 1062135, 1060487, 1060555, 1060637, 1060656, 1061071, 1061238, 1061256, 1061338, 1061339, 1061340, 1061341, 1061355, 1061356, 1061357, 1061358, 1061372, 1061373, 1061374, 1061375, 1061388, 1061389, 1061390, 1061391, 1061392, 1061405, 1061406, 1061407, 1061408, 1061422, 1061423, 1061424, 1061425, 1061439, 1061440, 1061441, 1061442, 1061455, 1061456, 1061457, 1061458, 1061472, 1061473, 1061474, 1061475, 1061489, 1061490, 1061491, 1061492, 1061505, 1061506, 1061507, 1061508, 1061509, 1061522, 1061523, 1061524, 1061525, 1061539, 1061540, 1061541, 1061542, 1061556, 1061557, 1061558, 1061559, 1061572, 1061573, 1061574, 1061575, 1061589, 1061590, 1061591, 1061592, 1061606, 1061607, 1061608, 1061609, 1061622, 1061623, 1061624, 1061625, 1061626, 1061639, 1061640, 1061641, 1061642, 1061656, 1061657, 1061658, 1061659, 1061672, 1061673, 1061674, 1061675, 1061676, 1061689, 1061690, 1061691, 1061692, 1061706, 1061707, 1061708, 1061709, 1061723, 1061724, 1061725, 1061726, 1061739, 1061740, 1061741, 1061742, 1061743, 1061756, 1061757, 1061758, 1061759, 1061773, 1061774, 1061775, 1061776, 1061789, 1061790, 1061791, 1061792, 1061793, 1061806, 1061807, 1061808, 1061809, 1061823, 1061824, 1061825, 1061826, 1061840, 1061841, 1061842, 1061843, 1061856, 1061857, 1061858, 1061859, 1061860, 1061873, 1061874, 1061875, 1061876, 1061890, 1061891, 1061892, 1061893, 1061906, 1061907, 1061908, 1061909, 1061910, 1061923, 1061924, 1061925, 1061926, 1061940, 1061941, 1061942, 1061943, 1061957, 1061958, 1061959, 1061960, 1061973, 1061974, 1061975, 1061976, 1061990, 1061991, 1061992, 1061993, 1062007, 1062008, 1062009, 1062010, 1062023, 1062024, 1062025, 1062026, 1062027, 1062040, 1062041, 1062042, 1062043, 1062057, 1062058, 1062059, 1062060, 1062074, 1062075, 1062076, 1062077, 1062090, 1062091, 1062092, 1062093, 1062107, 1062108, 1062109, 1062110, 1062124, 1062125, 1062126, 1062127, 1062140, 1062141, 1062142, 1062143, 1062144, 1060399, 1060526, 1060783, 1061335, 1061337, 1061342, 1061344, 1061346, 1061351, 1061353, 1061360, 1061362, 1061367, 1061369, 1061371, 1061376, 1061378, 1061385, 1061387, 1061394, 1061396, 1061401, 1061403, 1061410, 1061412, 1061419, 1061421, 1061426, 1061428, 1061435, 1061437, 1061444, 1061446, 1061451, 1061453, 1061460, 1061462, 1061469, 1061471, 1061476, 1061478, 1061485, 1061487, 1061494, 1061496, 1061501, 1061503, 1061510, 1061512, 1061519, 1061521, 1061526, 1061528, 1061535, 1061537, 1061544, 1061546, 1061551, 1061553, 1061555, 1061560, 1061562, 1061569, 1061571, 1061576, 1061578, 1061580, 1061585, 1061587, 1061594, 1061596, 1061601, 1061603, 1061605, 1061610, 1061612, 1061619, 1061621, 1061628, 1061630, 1061635, 1061637, 1061644, 1061646, 1061653, 1061655, 1061660, 1061662, 1061669, 1061671, 1061678, 1061680, 1061685, 1061687, 1061694, 1061696, 1061703, 1061705, 1061710, 1061712, 1061719, 1061721, 1061728, 1061730, 1061735, 1061737, 1061744, 1061746, 1061753, 1061755, 1061760, 1061762, 1061769, 1061771, 1061778, 1061780, 1061785, 1061787, 1061794, 1061796, 1061803, 1061805, 1061810, 1061812, 1061814, 1061819, 1061821, 1061828, 1061830, 1061835, 1061837, 1061839, 1061844, 1061846, 1061853, 1061855, 1061862, 1061864, 1061869, 1061871, 1061878, 1061880, 1061887, 1061889, 1061894, 1061896, 1061903, 1061905, 1061912, 1061914, 1061919, 1061921, 1061928, 1061930, 1061937, 1061939, 1061944, 1061946, 1061953, 1061955, 1061962, 1061964, 1061969, 1061971, 1061978, 1061980, 1061987, 1061989, 1061994, 1061996, 1062003, 1062005, 1062012, 1062014, 1062019, 1062021, 1062028, 1062030, 1062037, 1062039, 1062044, 1062046, 1062048, 1062053, 1062055, 1062062, 1062064, 1062069, 1062071, 1062073, 1062078, 1062080, 1062087, 1062089, 1062094, 1062096, 1062098, 1062103, 1062105, 1062112, 1062114, 1062121, 1062123, 1062128, 1062130, 1062137, 1062139, 1062146, 1060493, 1060752, 1060791, 1060816, 1061143, 1061234, 1061334, 1061336, 1061343, 1061345, 1061352, 1061354, 1061359, 1061361, 1061368, 1061370, 1061377, 1061379, 1061384, 1061386, 1061393, 1061395, 1061402, 1061404, 1061409, 1061411, 1061418, 1061420, 1061427, 1061429, 1061434, 1061436, 1061438, 1061443, 1061445, 1061452, 1061454, 1061459, 1061461, 1061463, 1061468, 1061470, 1061477, 1061479, 1061484, 1061486, 1061488, 1061493, 1061495, 1061502, 1061504, 1061511, 1061513, 1061518, 1061520, 1061527, 1061529, 1061536, 1061538, 1061543, 1061545, 1061552, 1061554, 1061561, 1061563, 1061568, 1061570, 1061577, 1061579, 1061586, 1061588, 1061593, 1061595, 1061602, 1061604, 1061611, 1061613, 1061618, 1061620, 1061627, 1061629, 1061636, 1061638, 1061643, 1061645, 1061652, 1061654, 1061661, 1061663, 1061668, 1061670, 1061677, 1061679, 1061686, 1061688, 1061693, 1061695, 1061697, 1061702, 1061704, 1061711, 1061713, 1061718, 1061720, 1061722, 1061727, 1061729, 1061736, 1061738, 1061745, 1061747, 1061752, 1061754, 1061761, 1061763, 1061770, 1061772, 1061777, 1061779, 1061786, 1061788, 1061795, 1061797, 1061802, 1061804, 1061811, 1061813, 1061820, 1061822, 1061827, 1061829, 1061836, 1061838, 1061845, 1061847, 1061852, 1061854, 1061861, 1061863, 1061870, 1061872, 1061877, 1061879, 1061886, 1061888, 1061895, 1061897, 1061902, 1061904, 1061911, 1061913, 1061920, 1061922, 1061927, 1061929, 1061931, 1061936, 1061938, 1061945, 1061947, 1061952, 1061954, 1061956, 1061961, 1061963, 1061970, 1061972, 1061977, 1061979, 1061981, 1061986, 1061988, 1061995, 1061997, 1062004, 1062006, 1062011, 1062013, 1062020, 1062022, 1062029, 1062031, 1062036, 1062038, 1062045, 1062047, 1062054, 1062056, 1062061, 1062063, 1062070, 1062072, 1062079, 1062081, 1062086, 1062088, 1062095, 1062097, 1062104, 1062106, 1062111, 1062113, 1062120, 1062122, 1062129, 1062131, 1062136, 1062138, 1062145, 1062147)), _block_offset, _block_number, _table, _disk_name, _part_granule_offset, _part_offset, _part_uuid, _part_index, is_executed, _part, camera_allow, update_by, update_date, reason_remarks, is_covered, duration_in_minutes, in_time, Unique_Id, _part_data_version, _part_starting_offset, employee_id, coverage_type, storetype_id, _database, _distance, out_time, supervisor_id, _sample_factor, _partition_id, reasonId, store_id, MID, project_id, distance_in_meters, visit_date', required columns: 'Mid' 'reason_remarks' '_block_number' '_disk_name' 'is_covered' '_part_index' '_part' 'is_executed' '_part_granule_offset' '_part_uuid' '_table' '_block_offset' '_part_offset' 'camera_allow' 'update_by' 'update_date' 'duration_in_minutes' 'Unique_Id' 'in_time' '_part_starting_offset' 'coverage_type' 'employee_id' '_part_data_version' 'storetype_id' '_database' '_distance' 'out_time' 'reasonId' '_partition_id' '_sample_factor' 'supervisor_id' 'store_id' 'MID' 'project_id' 'distance_in_meters' 'visit_date', maybe you meant: 'MID', 'reason_remarks', 'is_covered', 'is_executed', 'camera_allow', 'update_by', 'update_date', 'duration_in_minutes', 'Unique_Id', 'in_time', 'coverage_type', 'employee_id', 'storetype_id', 'out_time', 'reasonId', 'supervisor_id', 'store_id', 'project_id', 'distance_in_meters' or 'visit_date'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) +2026-06-18 13:43:13 | INFO | ================================================================================ +2026-06-18 13:43:13 | INFO | Hello from data-move Python data pipeline! +2026-06-18 13:43:13 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 13:43:13 | INFO | Connecting to databases... +2026-06-18 13:43:14 | INFO | +2026-06-18 13:43:16 | INFO | +2026-06-18 13:43:16 | INFO | Database connections established +2026-06-18 13:43:16 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 13:43:17 | INFO | Found 836 MIDs +2026-06-18 13:43:18 | INFO | ================================================================================ +2026-06-18 13:43:18 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:43:18 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-18 13:43:18 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 13:43:18 | INFO | Fetching data for 836 MIDs +2026-06-18 13:43:19 | INFO | Fetched 1,885 rows from SQL Server +2026-06-18 13:43:19 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! +2026-06-18 13:43:19 | INFO | Fetched 1885 rows +2026-06-18 13:43:19 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-18 13:43:19 | ERROR | Failed processing table additional_visibility +Traceback (most recent call last): + File "D:\data_move\main2.py", line 243, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 76, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + table_name, + ^^^^^^^^^^^ + f"Mid IN ({mids_str}) or MID IN ({mids_str})", + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 43, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'MID' while processing: 'isZeroOrNull((Mid IN (1060493, 1060752, 1060791, 1060816, 1061143, 1061234, 1061334, 1061336, 1061343, 1061345, 1061352, 1061354, 1061359, 1061361, 1061368, 1061370, 1061377, 1061379, 1061384, 1061386, 1061393, 1061395, 1061402, 1061404, 1061409, 1061411, 1061418, 1061420, 1061427, 1061429, 1061434, 1061436, 1061438, 1061443, 1061445, 1061452, 1061454, 1061459, 1061461, 1061463, 1061468, 1061470, 1061477, 1061479, 1061484, 1061486, 1061488, 1061493, 1061495, 1061502, 1061504, 1061511, 1061513, 1061518, 1061520, 1061527, 1061529, 1061536, 1061538, 1061543, 1061545, 1061552, 1061554, 1061561, 1061563, 1061568, 1061570, 1061577, 1061579, 1061586, 1061588, 1061593, 1061595, 1061602, 1061604, 1061611, 1061613, 1061618, 1061620, 1061627, 1061629, 1061636, 1061638, 1061643, 1061645, 1061652, 1061654, 1061661, 1061663, 1061668, 1061670, 1061677, 1061679, 1061686, 1061688, 1061693, 1061695, 1061697, 1061702, 1061704, 1061711, 1061713, 1061718, 1061720, 1061722, 1061727, 1061729, 1061736, 1061738, 1061745, 1061747, 1061752, 1061754, 1061761, 1061763, 1061770, 1061772, 1061777, 1061779, 1061786, 1061788, 1061795, 1061797, 1061802, 1061804, 1061811, 1061813, 1061820, 1061822, 1061827, 1061829, 1061836, 1061838, 1061845, 1061847, 1061852, 1061854, 1061861, 1061863, 1061870, 1061872, 1061877, 1061879, 1061886, 1061888, 1061895, 1061897, 1061902, 1061904, 1061911, 1061913, 1061920, 1061922, 1061927, 1061929, 1061931, 1061936, 1061938, 1061945, 1061947, 1061952, 1061954, 1061956, 1061961, 1061963, 1061970, 1061972, 1061977, 1061979, 1061981, 1061986, 1061988, 1061995, 1061997, 1062004, 1062006, 1062011, 1062013, 1062020, 1062022, 1062029, 1062031, 1062036, 1062038, 1062045, 1062047, 1062054, 1062056, 1062061, 1062063, 1062070, 1062072, 1062079, 1062081, 1062086, 1062088, 1062095, 1062097, 1062104, 1062106, 1062111, 1062113, 1062120, 1062122, 1062129, 1062131, 1062136, 1062138, 1062145, 1062147, 1060487, 1060555, 1060637, 1060656, 1061071, 1061238, 1061256, 1061338, 1061339, 1061340, 1061341, 1061355, 1061356, 1061357, 1061358, 1061372, 1061373, 1061374, 1061375, 1061388, 1061389, 1061390, 1061391, 1061392, 1061405, 1061406, 1061407, 1061408, 1061422, 1061423, 1061424, 1061425, 1061439, 1061440, 1061441, 1061442, 1061455, 1061456, 1061457, 1061458, 1061472, 1061473, 1061474, 1061475, 1061489, 1061490, 1061491, 1061492, 1061505, 1061506, 1061507, 1061508, 1061509, 1061522, 1061523, 1061524, 1061525, 1061539, 1061540, 1061541, 1061542, 1061556, 1061557, 1061558, 1061559, 1061572, 1061573, 1061574, 1061575, 1061589, 1061590, 1061591, 1061592, 1061606, 1061607, 1061608, 1061609, 1061622, 1061623, 1061624, 1061625, 1061626, 1061639, 1061640, 1061641, 1061642, 1061656, 1061657, 1061658, 1061659, 1061672, 1061673, 1061674, 1061675, 1061676, 1061689, 1061690, 1061691, 1061692, 1061706, 1061707, 1061708, 1061709, 1061723, 1061724, 1061725, 1061726, 1061739, 1061740, 1061741, 1061742, 1061743, 1061756, 1061757, 1061758, 1061759, 1061773, 1061774, 1061775, 1061776, 1061789, 1061790, 1061791, 1061792, 1061793, 1061806, 1061807, 1061808, 1061809, 1061823, 1061824, 1061825, 1061826, 1061840, 1061841, 1061842, 1061843, 1061856, 1061857, 1061858, 1061859, 1061860, 1061873, 1061874, 1061875, 1061876, 1061890, 1061891, 1061892, 1061893, 1061906, 1061907, 1061908, 1061909, 1061910, 1061923, 1061924, 1061925, 1061926, 1061940, 1061941, 1061942, 1061943, 1061957, 1061958, 1061959, 1061960, 1061973, 1061974, 1061975, 1061976, 1061990, 1061991, 1061992, 1061993, 1062007, 1062008, 1062009, 1062010, 1062023, 1062024, 1062025, 1062026, 1062027, 1062040, 1062041, 1062042, 1062043, 1062057, 1062058, 1062059, 1062060, 1062074, 1062075, 1062076, 1062077, 1062090, 1062091, 1062092, 1062093, 1062107, 1062108, 1062109, 1062110, 1062124, 1062125, 1062126, 1062127, 1062140, 1062141, 1062142, 1062143, 1062144, 1060399, 1060526, 1060783, 1061335, 1061337, 1061342, 1061344, 1061346, 1061351, 1061353, 1061360, 1061362, 1061367, 1061369, 1061371, 1061376, 1061378, 1061385, 1061387, 1061394, 1061396, 1061401, 1061403, 1061410, 1061412, 1061419, 1061421, 1061426, 1061428, 1061435, 1061437, 1061444, 1061446, 1061451, 1061453, 1061460, 1061462, 1061469, 1061471, 1061476, 1061478, 1061485, 1061487, 1061494, 1061496, 1061501, 1061503, 1061510, 1061512, 1061519, 1061521, 1061526, 1061528, 1061535, 1061537, 1061544, 1061546, 1061551, 1061553, 1061555, 1061560, 1061562, 1061569, 1061571, 1061576, 1061578, 1061580, 1061585, 1061587, 1061594, 1061596, 1061601, 1061603, 1061605, 1061610, 1061612, 1061619, 1061621, 1061628, 1061630, 1061635, 1061637, 1061644, 1061646, 1061653, 1061655, 1061660, 1061662, 1061669, 1061671, 1061678, 1061680, 1061685, 1061687, 1061694, 1061696, 1061703, 1061705, 1061710, 1061712, 1061719, 1061721, 1061728, 1061730, 1061735, 1061737, 1061744, 1061746, 1061753, 1061755, 1061760, 1061762, 1061769, 1061771, 1061778, 1061780, 1061785, 1061787, 1061794, 1061796, 1061803, 1061805, 1061810, 1061812, 1061814, 1061819, 1061821, 1061828, 1061830, 1061835, 1061837, 1061839, 1061844, 1061846, 1061853, 1061855, 1061862, 1061864, 1061869, 1061871, 1061878, 1061880, 1061887, 1061889, 1061894, 1061896, 1061903, 1061905, 1061912, 1061914, 1061919, 1061921, 1061928, 1061930, 1061937, 1061939, 1061944, 1061946, 1061953, 1061955, 1061962, 1061964, 1061969, 1061971, 1061978, 1061980, 1061987, 1061989, 1061994, 1061996, 1062003, 1062005, 1062012, 1062014, 1062019, 1062021, 1062028, 1062030, 1062037, 1062039, 1062044, 1062046, 1062048, 1062053, 1062055, 1062062, 1062064, 1062069, 1062071, 1062073, 1062078, 1062080, 1062087, 1062089, 1062094, 1062096, 1062098, 1062103, 1062105, 1062112, 1062114, 1062121, 1062123, 1062128, 1062130, 1062137, 1062139, 1062146, 1060060, 1060781, 1061233, 1061331, 1061332, 1061333, 1061347, 1061348, 1061349, 1061350, 1061363, 1061364, 1061365, 1061366, 1061380, 1061381, 1061382, 1061383, 1061397, 1061398, 1061399, 1061400, 1061413, 1061414, 1061415, 1061416, 1061417, 1061430, 1061431, 1061432, 1061433, 1061447, 1061448, 1061449, 1061450, 1061464, 1061465, 1061466, 1061467, 1061480, 1061481, 1061482, 1061483, 1061497, 1061498, 1061499, 1061500, 1061514, 1061515, 1061516, 1061517, 1061530, 1061531, 1061532, 1061533, 1061534, 1061547, 1061548, 1061549, 1061550, 1061564, 1061565, 1061566, 1061567, 1061581, 1061582, 1061583, 1061584, 1061597, 1061598, 1061599, 1061600, 1061614, 1061615, 1061616, 1061617, 1061631, 1061632, 1061633, 1061634, 1061647, 1061648, 1061649, 1061650, 1061651, 1061664, 1061665, 1061666, 1061667, 1061681, 1061682, 1061683, 1061684, 1061698, 1061699, 1061700, 1061701, 1061714, 1061715, 1061716, 1061717, 1061731, 1061732, 1061733, 1061734, 1061748, 1061749, 1061750, 1061751, 1061764, 1061765, 1061766, 1061767, 1061768, 1061781, 1061782, 1061783, 1061784, 1061798, 1061799, 1061800, 1061801, 1061815, 1061816, 1061817, 1061818, 1061831, 1061832, 1061833, 1061834, 1061848, 1061849, 1061850, 1061851, 1061865, 1061866, 1061867, 1061868, 1061881, 1061882, 1061883, 1061884, 1061885, 1061898, 1061899, 1061900, 1061901, 1061915, 1061916, 1061917, 1061918, 1061932, 1061933, 1061934, 1061935, 1061948, 1061949, 1061950, 1061951, 1061965, 1061966, 1061967, 1061968, 1061982, 1061983, 1061984, 1061985, 1061998, 1061999, 1062000, 1062001, 1062002, 1062015, 1062016, 1062017, 1062018, 1062032, 1062033, 1062034, 1062035, 1062049, 1062050, 1062051, 1062052, 1062065, 1062066, 1062067, 1062068, 1062082, 1062083, 1062084, 1062085, 1062099, 1062100, 1062101, 1062102, 1062115, 1062116, 1062117, 1062118, 1062119, 1062132, 1062133, 1062134, 1062135)) OR (MID IN (1060493, 1060752, 1060791, 1060816, 1061143, 1061234, 1061334, 1061336, 1061343, 1061345, 1061352, 1061354, 1061359, 1061361, 1061368, 1061370, 1061377, 1061379, 1061384, 1061386, 1061393, 1061395, 1061402, 1061404, 1061409, 1061411, 1061418, 1061420, 1061427, 1061429, 1061434, 1061436, 1061438, 1061443, 1061445, 1061452, 1061454, 1061459, 1061461, 1061463, 1061468, 1061470, 1061477, 1061479, 1061484, 1061486, 1061488, 1061493, 1061495, 1061502, 1061504, 1061511, 1061513, 1061518, 1061520, 1061527, 1061529, 1061536, 1061538, 1061543, 1061545, 1061552, 1061554, 1061561, 1061563, 1061568, 1061570, 1061577, 1061579, 1061586, 1061588, 1061593, 1061595, 1061602, 1061604, 1061611, 1061613, 1061618, 1061620, 1061627, 1061629, 1061636, 1061638, 1061643, 1061645, 1061652, 1061654, 1061661, 1061663, 1061668, 1061670, 1061677, 1061679, 1061686, 1061688, 1061693, 1061695, 1061697, 1061702, 1061704, 1061711, 1061713, 1061718, 1061720, 1061722, 1061727, 1061729, 1061736, 1061738, 1061745, 1061747, 1061752, 1061754, 1061761, 1061763, 1061770, 1061772, 1061777, 1061779, 1061786, 1061788, 1061795, 1061797, 1061802, 1061804, 1061811, 1061813, 1061820, 1061822, 1061827, 1061829, 1061836, 1061838, 1061845, 1061847, 1061852, 1061854, 1061861, 1061863, 1061870, 1061872, 1061877, 1061879, 1061886, 1061888, 1061895, 1061897, 1061902, 1061904, 1061911, 1061913, 1061920, 1061922, 1061927, 1061929, 1061931, 1061936, 1061938, 1061945, 1061947, 1061952, 1061954, 1061956, 1061961, 1061963, 1061970, 1061972, 1061977, 1061979, 1061981, 1061986, 1061988, 1061995, 1061997, 1062004, 1062006, 1062011, 1062013, 1062020, 1062022, 1062029, 1062031, 1062036, 1062038, 1062045, 1062047, 1062054, 1062056, 1062061, 1062063, 1062070, 1062072, 1062079, 1062081, 1062086, 1062088, 1062095, 1062097, 1062104, 1062106, 1062111, 1062113, 1062120, 1062122, 1062129, 1062131, 1062136, 1062138, 1062145, 1062147, 1060487, 1060555, 1060637, 1060656, 1061071, 1061238, 1061256, 1061338, 1061339, 1061340, 1061341, 1061355, 1061356, 1061357, 1061358, 1061372, 1061373, 1061374, 1061375, 1061388, 1061389, 1061390, 1061391, 1061392, 1061405, 1061406, 1061407, 1061408, 1061422, 1061423, 1061424, 1061425, 1061439, 1061440, 1061441, 1061442, 1061455, 1061456, 1061457, 1061458, 1061472, 1061473, 1061474, 1061475, 1061489, 1061490, 1061491, 1061492, 1061505, 1061506, 1061507, 1061508, 1061509, 1061522, 1061523, 1061524, 1061525, 1061539, 1061540, 1061541, 1061542, 1061556, 1061557, 1061558, 1061559, 1061572, 1061573, 1061574, 1061575, 1061589, 1061590, 1061591, 1061592, 1061606, 1061607, 1061608, 1061609, 1061622, 1061623, 1061624, 1061625, 1061626, 1061639, 1061640, 1061641, 1061642, 1061656, 1061657, 1061658, 1061659, 1061672, 1061673, 1061674, 1061675, 1061676, 1061689, 1061690, 1061691, 1061692, 1061706, 1061707, 1061708, 1061709, 1061723, 1061724, 1061725, 1061726, 1061739, 1061740, 1061741, 1061742, 1061743, 1061756, 1061757, 1061758, 1061759, 1061773, 1061774, 1061775, 1061776, 1061789, 1061790, 1061791, 1061792, 1061793, 1061806, 1061807, 1061808, 1061809, 1061823, 1061824, 1061825, 1061826, 1061840, 1061841, 1061842, 1061843, 1061856, 1061857, 1061858, 1061859, 1061860, 1061873, 1061874, 1061875, 1061876, 1061890, 1061891, 1061892, 1061893, 1061906, 1061907, 1061908, 1061909, 1061910, 1061923, 1061924, 1061925, 1061926, 1061940, 1061941, 1061942, 1061943, 1061957, 1061958, 1061959, 1061960, 1061973, 1061974, 1061975, 1061976, 1061990, 1061991, 1061992, 1061993, 1062007, 1062008, 1062009, 1062010, 1062023, 1062024, 1062025, 1062026, 1062027, 1062040, 1062041, 1062042, 1062043, 1062057, 1062058, 1062059, 1062060, 1062074, 1062075, 1062076, 1062077, 1062090, 1062091, 1062092, 1062093, 1062107, 1062108, 1062109, 1062110, 1062124, 1062125, 1062126, 1062127, 1062140, 1062141, 1062142, 1062143, 1062144, 1060399, 1060526, 1060783, 1061335, 1061337, 1061342, 1061344, 1061346, 1061351, 1061353, 1061360, 1061362, 1061367, 1061369, 1061371, 1061376, 1061378, 1061385, 1061387, 1061394, 1061396, 1061401, 1061403, 1061410, 1061412, 1061419, 1061421, 1061426, 1061428, 1061435, 1061437, 1061444, 1061446, 1061451, 1061453, 1061460, 1061462, 1061469, 1061471, 1061476, 1061478, 1061485, 1061487, 1061494, 1061496, 1061501, 1061503, 1061510, 1061512, 1061519, 1061521, 1061526, 1061528, 1061535, 1061537, 1061544, 1061546, 1061551, 1061553, 1061555, 1061560, 1061562, 1061569, 1061571, 1061576, 1061578, 1061580, 1061585, 1061587, 1061594, 1061596, 1061601, 1061603, 1061605, 1061610, 1061612, 1061619, 1061621, 1061628, 1061630, 1061635, 1061637, 1061644, 1061646, 1061653, 1061655, 1061660, 1061662, 1061669, 1061671, 1061678, 1061680, 1061685, 1061687, 1061694, 1061696, 1061703, 1061705, 1061710, 1061712, 1061719, 1061721, 1061728, 1061730, 1061735, 1061737, 1061744, 1061746, 1061753, 1061755, 1061760, 1061762, 1061769, 1061771, 1061778, 1061780, 1061785, 1061787, 1061794, 1061796, 1061803, 1061805, 1061810, 1061812, 1061814, 1061819, 1061821, 1061828, 1061830, 1061835, 1061837, 1061839, 1061844, 1061846, 1061853, 1061855, 1061862, 1061864, 1061869, 1061871, 1061878, 1061880, 1061887, 1061889, 1061894, 1061896, 1061903, 1061905, 1061912, 1061914, 1061919, 1061921, 1061928, 1061930, 1061937, 1061939, 1061944, 1061946, 1061953, 1061955, 1061962, 1061964, 1061969, 1061971, 1061978, 1061980, 1061987, 1061989, 1061994, 1061996, 1062003, 1062005, 1062012, 1062014, 1062019, 1062021, 1062028, 1062030, 1062037, 1062039, 1062044, 1062046, 1062048, 1062053, 1062055, 1062062, 1062064, 1062069, 1062071, 1062073, 1062078, 1062080, 1062087, 1062089, 1062094, 1062096, 1062098, 1062103, 1062105, 1062112, 1062114, 1062121, 1062123, 1062128, 1062130, 1062137, 1062139, 1062146, 1060060, 1060781, 1061233, 1061331, 1061332, 1061333, 1061347, 1061348, 1061349, 1061350, 1061363, 1061364, 1061365, 1061366, 1061380, 1061381, 1061382, 1061383, 1061397, 1061398, 1061399, 1061400, 1061413, 1061414, 1061415, 1061416, 1061417, 1061430, 1061431, 1061432, 1061433, 1061447, 1061448, 1061449, 1061450, 1061464, 1061465, 1061466, 1061467, 1061480, 1061481, 1061482, 1061483, 1061497, 1061498, 1061499, 1061500, 1061514, 1061515, 1061516, 1061517, 1061530, 1061531, 1061532, 1061533, 1061534, 1061547, 1061548, 1061549, 1061550, 1061564, 1061565, 1061566, 1061567, 1061581, 1061582, 1061583, 1061584, 1061597, 1061598, 1061599, 1061600, 1061614, 1061615, 1061616, 1061617, 1061631, 1061632, 1061633, 1061634, 1061647, 1061648, 1061649, 1061650, 1061651, 1061664, 1061665, 1061666, 1061667, 1061681, 1061682, 1061683, 1061684, 1061698, 1061699, 1061700, 1061701, 1061714, 1061715, 1061716, 1061717, 1061731, 1061732, 1061733, 1061734, 1061748, 1061749, 1061750, 1061751, 1061764, 1061765, 1061766, 1061767, 1061768, 1061781, 1061782, 1061783, 1061784, 1061798, 1061799, 1061800, 1061801, 1061815, 1061816, 1061817, 1061818, 1061831, 1061832, 1061833, 1061834, 1061848, 1061849, 1061850, 1061851, 1061865, 1061866, 1061867, 1061868, 1061881, 1061882, 1061883, 1061884, 1061885, 1061898, 1061899, 1061900, 1061901, 1061915, 1061916, 1061917, 1061918, 1061932, 1061933, 1061934, 1061935, 1061948, 1061949, 1061950, 1061951, 1061965, 1061966, 1061967, 1061968, 1061982, 1061983, 1061984, 1061985, 1061998, 1061999, 1062000, 1062001, 1062002, 1062015, 1062016, 1062017, 1062018, 1062032, 1062033, 1062034, 1062035, 1062049, 1062050, 1062051, 1062052, 1062065, 1062066, 1062067, 1062068, 1062082, 1062083, 1062084, 1062085, 1062099, 1062100, 1062101, 1062102, 1062115, 1062116, 1062117, 1062118, 1062119, 1062132, 1062133, 1062134, 1062135))), _block_offset, _block_number, _table, _database, _distance, _disk_name, _part_granule_offset, _partition_id, _sample_factor, brand_id, _part_uuid, _part_index, _part, created_by, storetype_id, created_date, camera_allowed, image_url, display_id, Remarks, is_present, _part_data_version, _part_starting_offset, visit_date, Mid, channel_id, store_id, chain_id, _part_offset, emp_id, project_id', required columns: 'Mid' 'MID' '_part_granule_offset' '_block_number' '_table' '_block_offset' '_part_uuid' 'brand_id' '_database' '_distance' 'Remarks' 'is_present' '_disk_name' '_partition_id' '_sample_factor' 'display_id' '_part_index' '_part' 'created_by' 'storetype_id' 'created_date' 'camera_allowed' 'image_url' '_part_data_version' '_part_starting_offset' 'visit_date' 'channel_id' 'store_id' 'chain_id' '_part_offset' 'emp_id' 'project_id', maybe you meant: 'Mid', 'brand_id', 'Remarks', 'is_present', 'display_id', 'created_by', 'storetype_id', 'created_date', 'camera_allowed', 'image_url', 'visit_date', 'channel_id', 'store_id', 'chain_id', 'emp_id' or 'project_id'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) +2026-06-18 13:46:45 | INFO | ================================================================================ +2026-06-18 13:46:45 | INFO | Hello from data-move Python data pipeline! +2026-06-18 13:46:45 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 13:46:45 | INFO | Connecting to databases... +2026-06-18 13:46:45 | INFO | +2026-06-18 13:46:48 | INFO | +2026-06-18 13:46:48 | INFO | Database connections established +2026-06-18 13:46:48 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 13:46:49 | INFO | Found 836 MIDs +2026-06-18 13:46:49 | INFO | ================================================================================ +2026-06-18 13:46:49 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:46:49 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-18 13:46:49 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 13:46:49 | INFO | Fetching data for 836 MIDs +2026-06-18 13:46:53 | INFO | Fetched 1,885 rows from SQL Server +2026-06-18 13:46:53 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! +2026-06-18 13:46:53 | INFO | Fetched 1885 rows +2026-06-18 13:46:53 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-18 13:46:53 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:46:54 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse +2026-06-18 13:46:54 | INFO | additional_visibility loaded successfully (1885 rows) +2026-06-18 13:46:54 | INFO | ================================================================================ +2026-06-18 13:46:54 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:46:54 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-18 13:46:54 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 13:46:54 | INFO | Fetching data for 836 MIDs +2026-06-18 13:47:11 | INFO | Fetched 779 rows from SQL Server +2026-06-18 13:47:11 | INFO | Fetched total row -: 779 from sql server for table-:Coverage ...........!!! +2026-06-18 13:47:11 | INFO | Fetched 779 rows +2026-06-18 13:47:11 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-18 13:47:11 | ERROR | Failed processing table Coverage +Traceback (most recent call last): + File "D:\data_move\main2.py", line 243, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 76, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + table_name, + ^^^^^^^^^^^ + f"Mid IN ({mids_str})", + ^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 43, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'Mid' while processing: 'isZeroOrNull(Mid IN (1060493, 1060752, 1060791, 1060816, 1061143, 1061234, 1061334, 1061336, 1061343, 1061345, 1061352, 1061354, 1061359, 1061361, 1061368, 1061370, 1061377, 1061379, 1061384, 1061386, 1061393, 1061395, 1061402, 1061404, 1061409, 1061411, 1061418, 1061420, 1061427, 1061429, 1061434, 1061436, 1061438, 1061443, 1061445, 1061452, 1061454, 1061459, 1061461, 1061463, 1061468, 1061470, 1061477, 1061479, 1061484, 1061486, 1061488, 1061493, 1061495, 1061502, 1061504, 1061511, 1061513, 1061518, 1061520, 1061527, 1061529, 1061536, 1061538, 1061543, 1061545, 1061552, 1061554, 1061561, 1061563, 1061568, 1061570, 1061577, 1061579, 1061586, 1061588, 1061593, 1061595, 1061602, 1061604, 1061611, 1061613, 1061618, 1061620, 1061627, 1061629, 1061636, 1061638, 1061643, 1061645, 1061652, 1061654, 1061661, 1061663, 1061668, 1061670, 1061677, 1061679, 1061686, 1061688, 1061693, 1061695, 1061697, 1061702, 1061704, 1061711, 1061713, 1061718, 1061720, 1061722, 1061727, 1061729, 1061736, 1061738, 1061745, 1061747, 1061752, 1061754, 1061761, 1061763, 1061770, 1061772, 1061777, 1061779, 1061786, 1061788, 1061795, 1061797, 1061802, 1061804, 1061811, 1061813, 1061820, 1061822, 1061827, 1061829, 1061836, 1061838, 1061845, 1061847, 1061852, 1061854, 1061861, 1061863, 1061870, 1061872, 1061877, 1061879, 1061886, 1061888, 1061895, 1061897, 1061902, 1061904, 1061911, 1061913, 1061920, 1061922, 1061927, 1061929, 1061931, 1061936, 1061938, 1061945, 1061947, 1061952, 1061954, 1061956, 1061961, 1061963, 1061970, 1061972, 1061977, 1061979, 1061981, 1061986, 1061988, 1061995, 1061997, 1062004, 1062006, 1062011, 1062013, 1062020, 1062022, 1062029, 1062031, 1062036, 1062038, 1062045, 1062047, 1062054, 1062056, 1062061, 1062063, 1062070, 1062072, 1062079, 1062081, 1062086, 1062088, 1062095, 1062097, 1062104, 1062106, 1062111, 1062113, 1062120, 1062122, 1062129, 1062131, 1062136, 1062138, 1062145, 1062147, 1060060, 1060781, 1061233, 1061331, 1061332, 1061333, 1061347, 1061348, 1061349, 1061350, 1061363, 1061364, 1061365, 1061366, 1061380, 1061381, 1061382, 1061383, 1061397, 1061398, 1061399, 1061400, 1061413, 1061414, 1061415, 1061416, 1061417, 1061430, 1061431, 1061432, 1061433, 1061447, 1061448, 1061449, 1061450, 1061464, 1061465, 1061466, 1061467, 1061480, 1061481, 1061482, 1061483, 1061497, 1061498, 1061499, 1061500, 1061514, 1061515, 1061516, 1061517, 1061530, 1061531, 1061532, 1061533, 1061534, 1061547, 1061548, 1061549, 1061550, 1061564, 1061565, 1061566, 1061567, 1061581, 1061582, 1061583, 1061584, 1061597, 1061598, 1061599, 1061600, 1061614, 1061615, 1061616, 1061617, 1061631, 1061632, 1061633, 1061634, 1061647, 1061648, 1061649, 1061650, 1061651, 1061664, 1061665, 1061666, 1061667, 1061681, 1061682, 1061683, 1061684, 1061698, 1061699, 1061700, 1061701, 1061714, 1061715, 1061716, 1061717, 1061731, 1061732, 1061733, 1061734, 1061748, 1061749, 1061750, 1061751, 1061764, 1061765, 1061766, 1061767, 1061768, 1061781, 1061782, 1061783, 1061784, 1061798, 1061799, 1061800, 1061801, 1061815, 1061816, 1061817, 1061818, 1061831, 1061832, 1061833, 1061834, 1061848, 1061849, 1061850, 1061851, 1061865, 1061866, 1061867, 1061868, 1061881, 1061882, 1061883, 1061884, 1061885, 1061898, 1061899, 1061900, 1061901, 1061915, 1061916, 1061917, 1061918, 1061932, 1061933, 1061934, 1061935, 1061948, 1061949, 1061950, 1061951, 1061965, 1061966, 1061967, 1061968, 1061982, 1061983, 1061984, 1061985, 1061998, 1061999, 1062000, 1062001, 1062002, 1062015, 1062016, 1062017, 1062018, 1062032, 1062033, 1062034, 1062035, 1062049, 1062050, 1062051, 1062052, 1062065, 1062066, 1062067, 1062068, 1062082, 1062083, 1062084, 1062085, 1062099, 1062100, 1062101, 1062102, 1062115, 1062116, 1062117, 1062118, 1062119, 1062132, 1062133, 1062134, 1062135, 1060487, 1060555, 1060637, 1060656, 1061071, 1061238, 1061256, 1061338, 1061339, 1061340, 1061341, 1061355, 1061356, 1061357, 1061358, 1061372, 1061373, 1061374, 1061375, 1061388, 1061389, 1061390, 1061391, 1061392, 1061405, 1061406, 1061407, 1061408, 1061422, 1061423, 1061424, 1061425, 1061439, 1061440, 1061441, 1061442, 1061455, 1061456, 1061457, 1061458, 1061472, 1061473, 1061474, 1061475, 1061489, 1061490, 1061491, 1061492, 1061505, 1061506, 1061507, 1061508, 1061509, 1061522, 1061523, 1061524, 1061525, 1061539, 1061540, 1061541, 1061542, 1061556, 1061557, 1061558, 1061559, 1061572, 1061573, 1061574, 1061575, 1061589, 1061590, 1061591, 1061592, 1061606, 1061607, 1061608, 1061609, 1061622, 1061623, 1061624, 1061625, 1061626, 1061639, 1061640, 1061641, 1061642, 1061656, 1061657, 1061658, 1061659, 1061672, 1061673, 1061674, 1061675, 1061676, 1061689, 1061690, 1061691, 1061692, 1061706, 1061707, 1061708, 1061709, 1061723, 1061724, 1061725, 1061726, 1061739, 1061740, 1061741, 1061742, 1061743, 1061756, 1061757, 1061758, 1061759, 1061773, 1061774, 1061775, 1061776, 1061789, 1061790, 1061791, 1061792, 1061793, 1061806, 1061807, 1061808, 1061809, 1061823, 1061824, 1061825, 1061826, 1061840, 1061841, 1061842, 1061843, 1061856, 1061857, 1061858, 1061859, 1061860, 1061873, 1061874, 1061875, 1061876, 1061890, 1061891, 1061892, 1061893, 1061906, 1061907, 1061908, 1061909, 1061910, 1061923, 1061924, 1061925, 1061926, 1061940, 1061941, 1061942, 1061943, 1061957, 1061958, 1061959, 1061960, 1061973, 1061974, 1061975, 1061976, 1061990, 1061991, 1061992, 1061993, 1062007, 1062008, 1062009, 1062010, 1062023, 1062024, 1062025, 1062026, 1062027, 1062040, 1062041, 1062042, 1062043, 1062057, 1062058, 1062059, 1062060, 1062074, 1062075, 1062076, 1062077, 1062090, 1062091, 1062092, 1062093, 1062107, 1062108, 1062109, 1062110, 1062124, 1062125, 1062126, 1062127, 1062140, 1062141, 1062142, 1062143, 1062144, 1060399, 1060526, 1060783, 1061335, 1061337, 1061342, 1061344, 1061346, 1061351, 1061353, 1061360, 1061362, 1061367, 1061369, 1061371, 1061376, 1061378, 1061385, 1061387, 1061394, 1061396, 1061401, 1061403, 1061410, 1061412, 1061419, 1061421, 1061426, 1061428, 1061435, 1061437, 1061444, 1061446, 1061451, 1061453, 1061460, 1061462, 1061469, 1061471, 1061476, 1061478, 1061485, 1061487, 1061494, 1061496, 1061501, 1061503, 1061510, 1061512, 1061519, 1061521, 1061526, 1061528, 1061535, 1061537, 1061544, 1061546, 1061551, 1061553, 1061555, 1061560, 1061562, 1061569, 1061571, 1061576, 1061578, 1061580, 1061585, 1061587, 1061594, 1061596, 1061601, 1061603, 1061605, 1061610, 1061612, 1061619, 1061621, 1061628, 1061630, 1061635, 1061637, 1061644, 1061646, 1061653, 1061655, 1061660, 1061662, 1061669, 1061671, 1061678, 1061680, 1061685, 1061687, 1061694, 1061696, 1061703, 1061705, 1061710, 1061712, 1061719, 1061721, 1061728, 1061730, 1061735, 1061737, 1061744, 1061746, 1061753, 1061755, 1061760, 1061762, 1061769, 1061771, 1061778, 1061780, 1061785, 1061787, 1061794, 1061796, 1061803, 1061805, 1061810, 1061812, 1061814, 1061819, 1061821, 1061828, 1061830, 1061835, 1061837, 1061839, 1061844, 1061846, 1061853, 1061855, 1061862, 1061864, 1061869, 1061871, 1061878, 1061880, 1061887, 1061889, 1061894, 1061896, 1061903, 1061905, 1061912, 1061914, 1061919, 1061921, 1061928, 1061930, 1061937, 1061939, 1061944, 1061946, 1061953, 1061955, 1061962, 1061964, 1061969, 1061971, 1061978, 1061980, 1061987, 1061989, 1061994, 1061996, 1062003, 1062005, 1062012, 1062014, 1062019, 1062021, 1062028, 1062030, 1062037, 1062039, 1062044, 1062046, 1062048, 1062053, 1062055, 1062062, 1062064, 1062069, 1062071, 1062073, 1062078, 1062080, 1062087, 1062089, 1062094, 1062096, 1062098, 1062103, 1062105, 1062112, 1062114, 1062121, 1062123, 1062128, 1062130, 1062137, 1062139, 1062146)), _block_offset, _block_number, _table, _disk_name, _part_granule_offset, _part_offset, _part_uuid, _part_index, is_executed, _part, camera_allow, update_by, update_date, reason_remarks, is_covered, duration_in_minutes, in_time, Unique_Id, _part_data_version, _part_starting_offset, employee_id, coverage_type, storetype_id, _database, _distance, out_time, supervisor_id, _sample_factor, _partition_id, reasonId, store_id, MID, project_id, distance_in_meters, visit_date', required columns: 'Mid' 'reason_remarks' '_block_number' '_disk_name' 'is_covered' '_part_index' '_part' 'is_executed' '_part_granule_offset' '_part_uuid' '_table' '_block_offset' '_part_offset' 'camera_allow' 'update_by' 'update_date' 'duration_in_minutes' 'Unique_Id' 'in_time' '_part_starting_offset' 'coverage_type' 'employee_id' '_part_data_version' 'storetype_id' '_database' '_distance' 'out_time' 'reasonId' '_partition_id' '_sample_factor' 'supervisor_id' 'store_id' 'MID' 'project_id' 'distance_in_meters' 'visit_date', maybe you meant: 'MID', 'reason_remarks', 'is_covered', 'is_executed', 'camera_allow', 'update_by', 'update_date', 'duration_in_minutes', 'Unique_Id', 'in_time', 'coverage_type', 'employee_id', 'storetype_id', 'out_time', 'reasonId', 'supervisor_id', 'store_id', 'project_id', 'distance_in_meters' or 'visit_date'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) +2026-06-18 13:50:13 | INFO | ================================================================================ +2026-06-18 13:50:13 | INFO | Hello from data-move Python data pipeline! +2026-06-18 13:50:13 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 13:50:13 | INFO | Connecting to databases... +2026-06-18 13:50:14 | INFO | +2026-06-18 13:50:15 | INFO | +2026-06-18 13:50:16 | INFO | Database connections established +2026-06-18 13:50:16 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 13:50:17 | INFO | Found 836 MIDs +2026-06-18 13:50:19 | INFO | ================================================================================ +2026-06-18 13:50:19 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:50:19 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-18 13:50:19 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 13:50:19 | INFO | Fetching data for 836 MIDs +2026-06-18 13:50:22 | INFO | Fetched 1,885 rows from SQL Server +2026-06-18 13:50:22 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! +2026-06-18 13:50:22 | INFO | Fetched 1885 rows +2026-06-18 13:50:22 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-18 13:50:22 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:50:22 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse +2026-06-18 13:50:22 | INFO | additional_visibility loaded successfully (1885 rows) +2026-06-18 13:50:22 | INFO | ================================================================================ +2026-06-18 13:50:22 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:50:22 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-18 13:50:22 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 13:50:22 | INFO | Fetching data for 836 MIDs +2026-06-18 13:50:49 | INFO | Fetched 779 rows from SQL Server +2026-06-18 13:50:49 | INFO | Fetched total row -: 779 from sql server for table-:Coverage ...........!!! +2026-06-18 13:50:49 | INFO | Fetched 779 rows +2026-06-18 13:50:50 | INFO | Creating table Coverage +2026-06-18 13:50:50 | INFO | Table ready: Coverage +2026-06-18 13:50:50 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:50:50 | INFO | Coverage: inserted 779 rows into ClickHouse +2026-06-18 13:50:50 | INFO | Coverage loaded successfully (779 rows) +2026-06-18 13:50:50 | INFO | ================================================================================ +2026-06-18 13:50:50 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:50:50 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-18 13:50:50 | INFO | Fetching data for 836 MIDs +2026-06-18 13:50:51 | INFO | Fetched 141 rows from SQL Server +2026-06-18 13:50:51 | INFO | Fetched total row -: 141 from sql server for table-:Survey ...........!!! +2026-06-18 13:50:51 | INFO | Fetched 141 rows +2026-06-18 13:50:51 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-18 13:50:51 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:50:51 | INFO | Survey: inserted 141 rows into ClickHouse +2026-06-18 13:50:51 | INFO | Survey loaded successfully (141 rows) +2026-06-18 13:50:51 | INFO | ================================================================================ +2026-06-18 13:50:51 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-18 13:50:51 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-18 13:50:51 | INFO | Fetching data for 836 MIDs +2026-06-18 13:50:56 | INFO | Fetched 475 rows from SQL Server +2026-06-18 13:50:56 | INFO | Fetched total row -: 475 from sql server for table-:Login ...........!!! +2026-06-18 13:50:56 | INFO | Fetched 475 rows +2026-06-18 13:50:56 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-18 13:50:56 | INFO | No delete logic required for Login +2026-06-18 13:50:56 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:50:56 | INFO | Login: inserted 475 rows into ClickHouse +2026-06-18 13:50:56 | INFO | Login loaded successfully (475 rows) +2026-06-18 13:50:56 | INFO | ================================================================================ +2026-06-18 13:50:56 | INFO | Processing Table: Stock_Details | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:50:56 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-18 13:50:56 | INFO | Fetching data for 836 MIDs +2026-06-18 13:51:15 | INFO | Fetched 39,313 rows from SQL Server +2026-06-18 13:51:15 | INFO | Fetched total row -: 39313 from sql server for table-:Stock_Details ...........!!! +2026-06-18 13:51:15 | INFO | Fetched 39313 rows +2026-06-18 13:51:15 | INFO | Creating table Stock_Details +2026-06-18 13:51:15 | INFO | Table ready: Stock_Details +2026-06-18 13:51:15 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:51:16 | INFO | Stock_Details: inserted 39,313 rows into ClickHouse +2026-06-18 13:51:16 | INFO | Stock_Details loaded successfully (39313 rows) +2026-06-18 13:51:16 | INFO | ================================================================================ +2026-06-18 13:51:16 | INFO | Pipeline Completed Successfully +2026-06-18 13:51:16 | INFO | ================================================================================ +2026-06-18 13:51:30 | INFO | ================================================================================ +2026-06-18 13:51:30 | INFO | Hello from data-move Python data pipeline! +2026-06-18 13:51:30 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 13:51:30 | INFO | Connecting to databases... +2026-06-18 13:51:31 | INFO | +2026-06-18 13:51:33 | INFO | +2026-06-18 13:51:33 | INFO | Database connections established +2026-06-18 13:51:33 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 13:51:34 | INFO | Found 836 MIDs +2026-06-18 13:51:35 | INFO | ================================================================================ +2026-06-18 13:51:35 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:51:35 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-18 13:51:35 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 13:51:35 | INFO | Fetching data for 836 MIDs +2026-06-18 13:51:36 | INFO | Fetched 1,885 rows from SQL Server +2026-06-18 13:51:36 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! +2026-06-18 13:51:36 | INFO | Fetched 1885 rows +2026-06-18 13:51:36 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-18 13:51:36 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:51:36 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse +2026-06-18 13:51:36 | INFO | additional_visibility loaded successfully (1885 rows) +2026-06-18 13:51:36 | INFO | ================================================================================ +2026-06-18 13:51:36 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:51:36 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-18 13:51:36 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 13:51:36 | INFO | Fetching data for 836 MIDs +2026-06-18 13:51:57 | INFO | Fetched 779 rows from SQL Server +2026-06-18 13:51:57 | INFO | Fetched total row -: 779 from sql server for table-:Coverage ...........!!! +2026-06-18 13:51:57 | INFO | Fetched 779 rows +2026-06-18 13:51:57 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-18 13:51:57 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:51:57 | INFO | Coverage: inserted 779 rows into ClickHouse +2026-06-18 13:51:57 | INFO | Coverage loaded successfully (779 rows) +2026-06-18 13:51:57 | INFO | ================================================================================ +2026-06-18 13:51:57 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:51:57 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-18 13:51:57 | INFO | Fetching data for 836 MIDs +2026-06-18 13:51:58 | INFO | Fetched 141 rows from SQL Server +2026-06-18 13:51:58 | INFO | Fetched total row -: 141 from sql server for table-:Survey ...........!!! +2026-06-18 13:51:58 | INFO | Fetched 141 rows +2026-06-18 13:51:58 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-18 13:51:58 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:51:58 | INFO | Survey: inserted 141 rows into ClickHouse +2026-06-18 13:51:58 | INFO | Survey loaded successfully (141 rows) +2026-06-18 13:51:58 | INFO | ================================================================================ +2026-06-18 13:51:58 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-18 13:51:58 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-18 13:51:58 | INFO | Fetching data for 836 MIDs +2026-06-18 13:52:00 | INFO | Fetched 475 rows from SQL Server +2026-06-18 13:52:00 | INFO | Fetched total row -: 475 from sql server for table-:Login ...........!!! +2026-06-18 13:52:00 | INFO | Fetched 475 rows +2026-06-18 13:52:00 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-18 13:52:00 | INFO | No delete logic required for Login +2026-06-18 13:52:00 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:52:00 | INFO | Login: inserted 475 rows into ClickHouse +2026-06-18 13:52:00 | INFO | Login loaded successfully (475 rows) +2026-06-18 13:52:00 | INFO | ================================================================================ +2026-06-18 13:52:00 | INFO | Processing Table: Stock_Details | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:52:00 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-18 13:52:00 | INFO | Fetching data for 836 MIDs +2026-06-18 13:52:05 | INFO | Fetched 39,313 rows from SQL Server +2026-06-18 13:52:05 | INFO | Fetched total row -: 39313 from sql server for table-:Stock_Details ...........!!! +2026-06-18 13:52:05 | INFO | Fetched 39313 rows +2026-06-18 13:52:05 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-18 13:52:05 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:52:05 | INFO | Stock_Details: inserted 39,313 rows into ClickHouse +2026-06-18 13:52:05 | INFO | Stock_Details loaded successfully (39313 rows) +2026-06-18 13:52:05 | INFO | ================================================================================ +2026-06-18 13:52:05 | INFO | Pipeline Completed Successfully +2026-06-18 13:52:05 | INFO | ================================================================================ +2026-06-18 13:52:12 | INFO | ================================================================================ +2026-06-18 13:52:12 | INFO | Hello from data-move Python data pipeline! +2026-06-18 13:52:12 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 13:52:12 | INFO | Connecting to databases... +2026-06-18 13:52:13 | INFO | +2026-06-18 13:52:15 | INFO | +2026-06-18 13:52:16 | INFO | Database connections established +2026-06-18 13:52:16 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 13:52:16 | INFO | Found 836 MIDs +2026-06-18 13:52:17 | INFO | ================================================================================ +2026-06-18 13:52:17 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:52:17 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-18 13:52:17 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 13:52:17 | INFO | Fetching data for 836 MIDs +2026-06-18 13:52:17 | INFO | Fetched 1,885 rows from SQL Server +2026-06-18 13:52:17 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! +2026-06-18 13:52:17 | INFO | Fetched 1885 rows +2026-06-18 13:52:17 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-18 13:52:17 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:52:17 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse +2026-06-18 13:52:17 | INFO | additional_visibility loaded successfully (1885 rows) +2026-06-18 13:52:17 | INFO | ================================================================================ +2026-06-18 13:52:17 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:52:17 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-18 13:52:17 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 13:52:17 | INFO | Fetching data for 836 MIDs +2026-06-18 13:52:29 | INFO | Fetched 779 rows from SQL Server +2026-06-18 13:52:29 | INFO | Fetched total row -: 779 from sql server for table-:Coverage ...........!!! +2026-06-18 13:52:29 | INFO | Fetched 779 rows +2026-06-18 13:52:29 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-18 13:52:30 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:52:30 | INFO | Coverage: inserted 779 rows into ClickHouse +2026-06-18 13:52:30 | INFO | Coverage loaded successfully (779 rows) +2026-06-18 13:52:30 | INFO | ================================================================================ +2026-06-18 13:52:30 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:52:30 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-18 13:52:30 | INFO | Fetching data for 836 MIDs +2026-06-18 13:52:31 | INFO | Fetched 141 rows from SQL Server +2026-06-18 13:52:31 | INFO | Fetched total row -: 141 from sql server for table-:Survey ...........!!! +2026-06-18 13:52:31 | INFO | Fetched 141 rows +2026-06-18 13:52:31 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-18 13:52:31 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:52:31 | INFO | Survey: inserted 141 rows into ClickHouse +2026-06-18 13:52:31 | INFO | Survey loaded successfully (141 rows) +2026-06-18 13:52:31 | INFO | ================================================================================ +2026-06-18 13:52:31 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-18 13:52:31 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-18 13:52:31 | INFO | Fetching data for 836 MIDs +2026-06-18 13:52:33 | INFO | Fetched 475 rows from SQL Server +2026-06-18 13:52:33 | INFO | Fetched total row -: 475 from sql server for table-:Login ...........!!! +2026-06-18 13:52:33 | INFO | Fetched 475 rows +2026-06-18 13:52:33 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-18 13:52:33 | INFO | No delete logic required for Login +2026-06-18 13:52:33 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:52:33 | INFO | Login: inserted 475 rows into ClickHouse +2026-06-18 13:52:33 | INFO | Login loaded successfully (475 rows) +2026-06-18 13:52:33 | INFO | ================================================================================ +2026-06-18 13:52:33 | INFO | Processing Table: Stock_Details | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 13:52:33 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-18 13:52:33 | INFO | Fetching data for 836 MIDs +2026-06-18 13:52:43 | INFO | Fetched 39,313 rows from SQL Server +2026-06-18 13:52:43 | INFO | Fetched total row -: 39313 from sql server for table-:Stock_Details ...........!!! +2026-06-18 13:52:43 | INFO | Fetched 39313 rows +2026-06-18 13:52:43 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-18 13:52:43 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 13:52:45 | INFO | Stock_Details: inserted 39,313 rows into ClickHouse +2026-06-18 13:52:45 | INFO | Stock_Details loaded successfully (39313 rows) +2026-06-18 13:52:45 | INFO | ================================================================================ +2026-06-18 13:52:45 | INFO | Pipeline Completed Successfully +2026-06-18 13:52:45 | INFO | ================================================================================ +2026-06-18 15:49:07 | INFO | ================================================================================ +2026-06-18 15:49:07 | INFO | Hello from data-move Python data pipeline! +2026-06-18 15:49:07 | INFO | Pipeline Run Date: 2026-06-17 +2026-06-18 15:49:07 | INFO | Connecting to databases... +2026-06-18 15:49:12 | INFO | +2026-06-18 15:49:13 | INFO | +2026-06-18 15:49:13 | INFO | Database connections established +2026-06-18 15:49:13 | INFO | Collecting MIDs for: 2026-06-17 +2026-06-18 15:49:13 | INFO | Found 836 MIDs +2026-06-18 15:49:14 | INFO | ================================================================================ +2026-06-18 15:49:14 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 15:49:14 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-18 15:49:14 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 15:49:14 | INFO | Fetching data for 836 MIDs +2026-06-18 15:49:19 | INFO | Fetched 3,407 rows from SQL Server +2026-06-18 15:49:19 | INFO | Fetched total row -: 3407 from sql server for table-:SOS_OneApp ...........!!! +2026-06-18 15:49:19 | INFO | Fetched 3407 rows +2026-06-18 15:49:19 | INFO | Creating table SOS_OneApp +2026-06-18 15:49:19 | INFO | Table ready: SOS_OneApp +2026-06-18 15:49:19 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 15:49:19 | INFO | SOS_OneApp: inserted 3,407 rows into ClickHouse +2026-06-18 15:49:19 | INFO | SOS_OneApp loaded successfully (3407 rows) +2026-06-18 15:49:19 | INFO | ================================================================================ +2026-06-18 15:49:19 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 15:49:19 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-18 15:49:19 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 15:49:19 | INFO | Fetching data for 836 MIDs +2026-06-18 15:49:20 | INFO | Fetched 1,885 rows from SQL Server +2026-06-18 15:49:20 | INFO | Fetched total row -: 1885 from sql server for table-:additional_visibility ...........!!! +2026-06-18 15:49:20 | INFO | Fetched 1885 rows +2026-06-18 15:49:20 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-18 15:49:20 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 15:49:21 | INFO | additional_visibility: inserted 1,885 rows into ClickHouse +2026-06-18 15:49:21 | INFO | additional_visibility loaded successfully (1885 rows) +2026-06-18 15:49:21 | INFO | ================================================================================ +2026-06-18 15:49:21 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 15:49:21 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-18 15:49:21 | INFO | Start Fetching data for these 836 MIDs +2026-06-18 15:49:21 | INFO | Fetching data for 836 MIDs +2026-06-18 15:49:33 | INFO | Fetched 779 rows from SQL Server +2026-06-18 15:49:33 | INFO | Fetched total row -: 779 from sql server for table-:Coverage ...........!!! +2026-06-18 15:49:33 | INFO | Fetched 779 rows +2026-06-18 15:49:34 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-18 15:49:34 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 15:49:35 | INFO | Coverage: inserted 779 rows into ClickHouse +2026-06-18 15:49:35 | INFO | Coverage loaded successfully (779 rows) +2026-06-18 15:49:35 | INFO | ================================================================================ +2026-06-18 15:49:35 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 15:49:35 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-18 15:49:35 | INFO | Fetching data for 836 MIDs +2026-06-18 15:49:35 | INFO | Fetched 141 rows from SQL Server +2026-06-18 15:49:35 | INFO | Fetched total row -: 141 from sql server for table-:Survey ...........!!! +2026-06-18 15:49:35 | INFO | Fetched 141 rows +2026-06-18 15:49:36 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-18 15:49:36 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 15:49:37 | INFO | Survey: inserted 141 rows into ClickHouse +2026-06-18 15:49:37 | INFO | Survey loaded successfully (141 rows) +2026-06-18 15:49:37 | INFO | ================================================================================ +2026-06-18 15:49:37 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-18 15:49:37 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-18 15:49:37 | INFO | Fetching data for 836 MIDs +2026-06-18 15:49:39 | INFO | Fetched 475 rows from SQL Server +2026-06-18 15:49:39 | INFO | Fetched total row -: 475 from sql server for table-:Login ...........!!! +2026-06-18 15:49:39 | INFO | Fetched 475 rows +2026-06-18 15:49:39 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-18 15:49:39 | INFO | No delete logic required for Login +2026-06-18 15:49:39 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 15:49:39 | INFO | Login: inserted 475 rows into ClickHouse +2026-06-18 15:49:39 | INFO | Login loaded successfully (475 rows) +2026-06-18 15:49:39 | INFO | ================================================================================ +2026-06-18 15:49:39 | INFO | Processing Table: Stock_Details | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-18 15:49:39 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-18 15:49:39 | INFO | Fetching data for 836 MIDs +2026-06-18 15:50:23 | INFO | Fetched 39,313 rows from SQL Server +2026-06-18 15:50:23 | INFO | Fetched total row -: 39313 from sql server for table-:Stock_Details ...........!!! +2026-06-18 15:50:23 | INFO | Fetched 39313 rows +2026-06-18 15:50:23 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-18 15:50:24 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-18 15:50:24 | INFO | Stock_Details: inserted 39,313 rows into ClickHouse +2026-06-18 15:50:24 | INFO | Stock_Details loaded successfully (39313 rows) +2026-06-18 15:50:24 | INFO | ================================================================================ +2026-06-18 15:50:24 | INFO | Pipeline Completed Successfully +2026-06-18 15:50:24 | INFO | ================================================================================ diff --git a/main2.py b/main2.py index e0460b0..237ccf8 100644 --- a/main2.py +++ b/main2.py @@ -65,51 +65,27 @@ def table_exists( def get_dataframe( + sql_engine, fn_name: str, fetch_by: str, - sql_engine, + table_name: str, + table_type: str , mids, run_date, ): fn = globals()[fn_name] - if fetch_by == "mids": - return fn(sql_engine, mids) + if fetch_by == "mids" or "run_date": + return fn(sql_engine, table_name , table_type, mids, run_date) - if fetch_by == "run_date": - return fn(sql_engine, run_date) - return fn(sql_engine) + + return fn(sql_engine ,table_name,table_type) -def get_empids_clickhouse_OQAD( - client, - table_name: str = "OQaD", -) -> pl.DataFrame: - - if not table_exists(client, table_name): - log.warning(f"Table '{table_name}' does not exist.") - return pl.DataFrame( - schema={ - "empid": pl.Int64, - "visitdate": pl.Date, - } - ) - - query = f""" - SELECT DISTINCT - employee_id AS empid, - visit_date AS visitdate - FROM {table_name} - """ - - # ClickHouse -> PyArrow -> Polars - arrow_table = client.query_arrow(query) - - return pl.from_arrow(arrow_table) # ========================================================== # Main @@ -166,17 +142,7 @@ def main(): sql_engine, run_date, ) - qf=fetch_quiz_empids(sql_engine,run_date) - db_df = get_empids_clickhouse_OQAD(client) - matched = qf.join( - db_df, - on=["empid", "visitdate"], - how="inner", -) - empids=matched["empid"].to_list() - - log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ") # ------------------------------------------------------ # Config @@ -201,37 +167,27 @@ def main(): table_type=table["type"] log.info("=" * 80) - log.info( - "Processing Table: %s", - table_name, - ) + log.info(f"Processing Table: {table_name} | Table type is -: {table_type} | Based on -{fetch_by} and operation is used -{operation} " ) try: # ------------------------------------------ # Fetch Data # ------------------------------------------ - if table_name =="OQaD": - - - df=fetch_OQaD( engine=sql_engine, - table_name=table_name, - table_type=table_type, - empids=empids, - run_date=run_date - ) - - else: - - df = fetch_data( - engine=sql_engine, - table_name=table_name, - table_type=table_type, - mids=mids, - run_date=run_date, - ) + log.info(f"Fetching Data from sql server for table-: {table_name} ..............") + fn_name = f"fetch_{table_name}" + df = get_dataframe( + sql_engine, + fn_name=fn_name, + fetch_by=fetch_by, + table_name=table_name, + table_type=table_type, + mids=mids, + run_date=run_date, + ) + log.info(f"Fetched total row -: {len(df)} from sql server for table-:{table_name} ...........!!!") if df.is_empty(): @@ -295,7 +251,7 @@ def main(): # ------------------------------------------ # Load Data # ------------------------------------------ - + log.info("_ _ _ _Inserting data into clickhouse db from sql server_ _ _ _") load_to_clickhouse( client=client, table_name=table_name, diff --git a/mids.py b/mids.py index df81fd8..417b658 100644 --- a/mids.py +++ b/mids.py @@ -72,51 +72,3 @@ def MID_TABLE_COV1( - -def fetch_quiz_empids(engine: Engine, run_date : date) -> pl.DataFrame: - - sql_template = f""" - WITH MID_TABLE_COV1 AS -( - SELECT EmpId, VisitDate - FROM OneApp_KelloggsMT.dbo.T_OQAD - WHERE CreateDate >= {run_date} - AND CreateDate < DATEADD(DAY,1,'{run_date}') - - UNION ALL - - SELECT EmpId, VisitDate - FROM OneApp_KelloggsMT.dbo.T_OQAD - WHERE UpdateDate >= {run_date} - AND UpdateDate < DATEADD(DAY,1, '{run_date}') -), - -QUIZ AS -( -SELECT Distinct E.EmpId as empid -, CONVERT(date,DQ.VisitDate) AS visitdate -FROM OneApp_KelloggsMT.dbo.T_OQAD DQ INNER JOIN -OneApp_KelloggsMT.dbo.vw_Employee_Detail E ON DQ.EmpId = E.EmpId inner join -OneApp_KelloggsMT.dbo.Master_OQAD_Question QU on DQ.QuestionId= qu.QuestionId inner join -OneApp_KelloggsMT.dbo.Master_OQAD_Category qc on qu.QuestionCategoryId= qc.QuestionCategoryId -where e.EmpName not like 'test%' and e.RightId in (6) -and (E.ResignDate is null or E.ResignDate>=''+CONVERT(VARCHAR,'{run_date}')+'') AND E.EmpName NOT LIKE '%TEST%' -AND DQ.EmpId IN (SELECT EmpId FROM MID_TABLE_COV1 A WHERE -DQ.EmpId=A.EmpId AND CONVERT(date,VisitDate)=CONVERT(date,A.VisitDate) ) -) select * from quiz -""" - sql = sql_template.format( - run_date=run_date.strftime("%Y-%m-%d") - ) - - log.info(f"Fetching quiz_empids data for EMPID and Visitid") - - df = pl.read_database( - query=sql, - connection=engine - ) - - - log.info(f"Fetched {len(df):,} total empid and visitdate fetched for OQAD from SQL Server") - - return df diff --git a/src/fact.py b/src/fact.py index 97fdced..125712a 100644 --- a/src/fact.py +++ b/src/fact.py @@ -1,10 +1,31 @@ from pathlib import Path import polars as pl from sqlalchemy import Engine -from datetime import date +from datetime import date , timedelta from log import log + + +from db_con.connection import ( + build_sql_server_engine, + build_clickhouse_engine, + get_clickhouse_client, +) + + + + + + + + + + + + + + def fetch_data( engine: Engine, table_name: str, @@ -42,40 +63,506 @@ def fetch_data( - -def fetch_OQaD( - engine: Engine, +def fetch_SOS_OneApp( engine: Engine, table_name: str, table_type: str, - empids: list[int], + mids: list[int], run_date: date ) -> pl.DataFrame: - - empid_list = ",".join(str(empid) for empid in empids) - - sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql" - - log.info(f"Exists: {sql_file.exists()}") - log.info(f"Path: {sql_file.resolve()}") + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + log.info(f" Start Fetching data for these {len(mids):} MIDs ") + mid_list = ",".join(str(mid) for mid in mids) + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" with open(sql_file, "r", encoding="utf-8") as f: sql_template = f.read() sql = sql_template.format( - empid_list=empid_list, + mid_list=mid_list, run_date=run_date.strftime("%Y-%m-%d") ) - - log.info(f"Fetching data for {len(empids):,} EMPIDs") - - log.info("Fetching OQaD data for run_date=%s", run_date) + log.info(f"Fetching data for {len(mids):,} MIDs") df = pl.read_database( query=sql, - connection=engine, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + +def fetch_additional_visibility( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + + log.info(f" Start Fetching data for these {len(mids):} MIDs ") + mid_list = ",".join(str(mid) for mid in mids) + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + +def fetch_OQaD( + sql_engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + client= get_clickhouse_client() + def table_exists( + client, + table_name: str, + ) -> bool: + + return bool( + client.command( + f"EXISTS TABLE {table_name}" + ) + ) + + + + def fetch_quiz_empids(engine: Engine, run_date : date) -> pl.DataFrame: + + sql_template = f""" + WITH MID_TABLE_COV1 AS + ( + SELECT EmpId, VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CreateDate >= {run_date} + AND CreateDate < DATEADD(DAY,1,'{run_date}') + + UNION ALL + + SELECT EmpId, VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE UpdateDate >= {run_date} + AND UpdateDate < DATEADD(DAY,1, '{run_date}') + ), + + QUIZ AS + ( + SELECT Distinct E.EmpId as empid + , CONVERT(date,DQ.VisitDate) AS visitdate + FROM OneApp_KelloggsMT.dbo.T_OQAD DQ INNER JOIN + OneApp_KelloggsMT.dbo.vw_Employee_Detail E ON DQ.EmpId = E.EmpId inner join + OneApp_KelloggsMT.dbo.Master_OQAD_Question QU on DQ.QuestionId= qu.QuestionId inner join + OneApp_KelloggsMT.dbo.Master_OQAD_Category qc on qu.QuestionCategoryId= qc.QuestionCategoryId + where e.EmpName not like 'test%' and e.RightId in (6) + and (E.ResignDate is null or E.ResignDate>=''+CONVERT(VARCHAR,'{run_date}')+'') AND E.EmpName NOT LIKE '%TEST%' + AND DQ.EmpId IN (SELECT EmpId FROM MID_TABLE_COV1 A WHERE + DQ.EmpId=A.EmpId AND CONVERT(date,VisitDate)=CONVERT(date,A.VisitDate) ) + ) select * from quiz + """ + sql = sql_template.format( + run_date=run_date.strftime("%Y-%m-%d") + ) + + log.info(f"Fetching quiz_empids data for EMPID and Visitid") + + df = pl.read_database( + query=sql, + connection=engine + ) + + + log.info(f"Fetched {len(df):,} total empid and visitdate fetched for OQAD from SQL Server") + + return df + + + def get_empids_clickhouse_OQAD( + client, + table_name: str = "OQaD", + ) -> pl.DataFrame: + + if not table_exists(client, table_name): + log.warning(f"Table '{table_name}' does not exist.") + return pl.DataFrame( + schema={ + "empid": pl.Int64, + "visitdate": pl.Date, + } + ) + + query = f""" + SELECT DISTINCT + employee_id AS empid, + visit_date AS visitdate + FROM {table_name} + """ + + # ClickHouse -> PyArrow -> Polars + arrow_table = client.query_arrow(query) + + return pl.from_arrow(arrow_table) + + + + qf=fetch_quiz_empids(sql_engine,run_date) + db_df = get_empids_clickhouse_OQAD(client) + + matched = qf.join( + db_df, + on=["empid", "visitdate"], + how="inner", + ) + empids=matched["empid"].to_list() + + log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ") + + def fetch_data( + engine: Engine, + table_name: str, + table_type: str, + empids: list[int], + run_date: date + ) -> pl.DataFrame: + + empid_list = ",".join(str(empid) for empid in empids) + + + sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql" + + log.info(f"Exists: {sql_file.exists()}") + log.info(f"Path: {sql_file.resolve()}") + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + empid_list=empid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + + log.info(f"Fetching data for {len(empids):,} EMPIDs") + + log.info("Fetching OQaD data for run_date=%s", run_date) + + df = pl.read_database( + query=sql, + connection=engine, + ) + + log.info("fn name is fetch_OQad ------Fetched %s rows", len(df)) + + return df + df=fetch_data( engine=sql_engine, + table_name=table_name, + table_type=table_type, + empids=empids, + run_date=run_date + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + +# def fetch_OQaD( +# engine: Engine, +# table_name: str, +# table_type: str, +# empids: list[int], +# run_date: date +# ) -> pl.DataFrame: + +# empid_list = ",".join(str(empid) for empid in empids) + + +# sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql" + +# log.info(f"Exists: {sql_file.exists()}") +# log.info(f"Path: {sql_file.resolve()}") + +# with open(sql_file, "r", encoding="utf-8") as f: +# sql_template = f.read() + +# sql = sql_template.format( +# empid_list=empid_list, +# run_date=run_date.strftime("%Y-%m-%d") +# ) + +# log.info(f"Fetching data for {len(empids):,} EMPIDs") + +# log.info("Fetching OQaD data for run_date=%s", run_date) + +# df = pl.read_database( +# query=sql, +# connection=engine, +# ) + +# log.info("fn name is fetch_OQad ------Fetched %s rows", len(df)) + +# return df + + + + + + +def fetch_Coverage( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + mid_list = ",".join(str(mid) for mid in mids) + + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + + +def fetch_Survey( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + + mid_list = ",".join(str(mid) for mid in mids) + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + + + + +def fetch_Login( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + + mid_list = ",".join(str(mid) for mid in mids) + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + + + + + +def fetch_Stock_Details( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + + mid_list = ",".join(str(mid) for mid in mids) + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + + + +# def fetch_Coverage( engine: Engine, +# table_name: str, +# table_type: str, +# mids: list[int], +# run_date: date +# ) -> pl.DataFrame: + + + +# if not mids: +# log.warning("No MIDs — nothing to fetch.") +# return pl.DataFrame() + +# mid_list = ",".join(str(mid) for mid in mids) +# log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") +# sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + +# with open(sql_file, "r", encoding="utf-8") as f: +# sql_template = f.read() + +# sql = sql_template.format( +# mid_list=mid_list, +# run_date=run_date.strftime("%Y-%m-%d") +# ) +# log.info(f"Fetching data for {len(mids):,} MIDs") + +# df = pl.read_database( +# query=sql, +# connection=engine +# ) +# log.info(f"Fetched {len(df):,} rows from SQL Server") + +# return df + + + +def fetch_Attendance( + engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date | None = None, + days_back: int = 15 +) -> pl.DataFrame: + """ + Fetch attendance source data. + + Default: + end_date = yesterday + start_date = yesterday - 15 days + """ + + if run_date is None: + run_date = date.today() - timedelta(days=1) + + start_date = run_date - timedelta(days=days_back) + + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + star_date=start_date.strftime("%Y-%m-%d"), + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info( + f"Fetching Attendance data from {start_date} to {run_date}" ) - log.info("fn name is fetch_OQad ------Fetched %s rows", len(df)) + df = pl.read_database( + query=sql, + connection=engine + ) - return df \ No newline at end of file + log.info( + f"Fetched {len(df):,} attendance rows " + f"for {df['employee_id'].n_unique():,} employees" + ) + + return df diff --git a/src/sql/fact/Attendance.sql b/src/sql/fact/Attendance.sql index e69de29..1fcbf56 100644 --- a/src/sql/fact/Attendance.sql +++ b/src/sql/fact/Attendance.sql @@ -0,0 +1,131 @@ +with Emp_cte as + (Select T1.*, ISNULL(ps.PositionCode ,'') PositionCode,FromDate,ToDate from( + select a.*,b.date from + (Select rm.RegionId, rm.RegionName as Region, st.StateId, st.StateName , cm.CityId, cm.CityName + , em2.Id as [Area Manager Id], em2.EmployeeName AS [Area Manager] + , em1.Id as [Supervisor Id], isnull(em1.EmployeeName,'') as [Supervisor Name] + , em.Id as [Employee Id], isnull(em.EmployeeName,'') as [Employee Name] + , dm.DesignationId, dm.DesignationName as Designation + , convert(varchar,EM.ResignDate,101) as DOR, convert(varchar,EM.JoinDate,101) as DOJ, em.LegacyCode + --,ISNULL(ps.PositionCode ,'') PositionCode--, FromDate, ToDate + from + OneApp_KelloggsMT.dbo.AspNetUsers em inner join + OneApp_KelloggsMT.dbo.AspNetUsers em1 on em1.Id = em.ManagerId inner join + OneApp_KelloggsMT.dbo.AspNetUsers em2 on em2.Id = em1.ManagerId inner join + OneApp_KelloggsMT.dbo.Master_City cm on em.CityId= cm.CityId inner join + OneApp_KelloggsMT.dbo.Master_State st on cm.StateId= st.StateId inner join + OneApp_KelloggsMT.dbo.Master_Region rm on st.RegionId= rm.RegionId inner join + OneApp_KelloggsMT.dbo.Master_Designation dm on Em.DesignationId= dm.DesignationId + + where em.RightId in (6) and em.EmployeeName not like '%test%' and + (Em.ResignDate is null or Em.ResignDate>='{start_date}') and CONVERT(date,em.JoinDate,101)<=CONVERT(date,'{run_date}',101) + --and em.Id=2290 + + )a + cross join (select distinct DATE from DBO.GET_ALL_DAYS('{start_date}','{run_date}')) b + ) AS T1 LEFT JOIN + (Select distinct EmpId, PositionId, Date,FromDate,ToDate from + (Select distinct EmpId,PositionId,FromDate,ToDate from OneApp_KelloggsMT.dbo.Mapping_PositionUser + Where convert(Date,FromDate)<= CONVERT(date,'{run_date}',101) and convert(Date,ToDate)>=CONVERT(date,'{start_date}',101)) as m Inner Join + (Select Date from OneApp_KelloggsMT.dbo.Master_Calender Where 1=1 and Date between CONVERT(date,'{start_date}',101) and CONVERT(date,'{run_date}',101)) + cal on cal.Date >= m.FromDate AND cal.Date <= m.ToDate) pu + on T1.[Employee Id]=pu.EmpId and T1.date= pu.date Left Join + OneApp_KelloggsMT.dbo.Master_Position ps WITH (NOLOCK) ON pu.PositionId=ps.PositionId + ) , + Attendance + (project_id, + employee_id,position_code,legacy_code,supervisor_id,date_of_join,date_of_resign,visit_date,parinaam_attendance, + update_date,update_by,Unique_Id) + AS ( + + select '40148', emp.[Employee Id], Case When emp.DATE <= ToDate and emp.DATE>= FromDate + then emp.PositionCode Else '' End as PositionCode ,emp.LegacyCode, emp.[Supervisor Id] ,emp.DOJ,emp.DOR, + emp.DATE, case when emp.DATE>=emp.DOR then 'R' WHEN emp.DATE< emp.DOJ THEN '-' + WHEN b.COL IS NULL AND (emp.DATE<=emp.DOR OR emp.DOR IS NULL )AND (emp.DATE>=emp.DOJ OR emp.DOJ IS NULL) THEN 'W' ELSE b.COL + END AS Attendance,GETDATE(),'SP-Pius',CAST('40148' AS VARCHAR) + '_' + CAST(emp.[Employee Id] AS VARCHAR) + from Emp_cte as emp + + left join + + + (select a.* + + + FROM + ( + SELECT A.DATE,TT.[MERCHANDISER CD],TT.VisitDate,TT.DOJ1,TT.DOR1,TT.COL from DBO.GET_ALL_DAYS('{start_date}','{run_date}') a + left join + + + ( + + SELECT distinct [EMPLOYEE ID] AS [MERCHANDISER CD], VISITDATE + , CASE WHEN COL=1 THEN 'H' WHEN COL=2 THEN 'L' WHEN COL=4 THEN 'M' WHEN COL=5 THEN 'CO' WHEN COL=6 THEN 'WO' + WHEN COL=7 THEN 'P' WHEN COL=10 THEN 'EH' WHEN COL=8 THEN 'HD' WHEN COL IS NULL THEN 'A' END AS COL + , DOJ AS DOJ1, DOR AS DOR1 + FROM + ( + SELECT DOJ, DOR, Region, [AREA MANAGER], SUPERVISOR_CD, [SUPERVISOR], City, [EMPLOYEE ID], MERCHANDISER, VISITDATE + , REMARKS, COL, ROW_NUMBER()OVER (PARTITION BY [EMPLOYEE ID], VISITDATE ORDER BY COL DESC,Intime, VISITDATE, ROW ) AS COL1 + FROM + ( + SELECT DOJ, DOR, Region, [AREA MANAGER], SUPERVISOR_CD, [SUPERVISOR], City, [EMPLOYEE ID], MERCHANDISER, InTime, VISITDATE + , REMARKS, COL, ROW_NUMBER()OVER (PARTITION BY [EMPLOYEE ID], VISITDATE ORDER BY COL) AS ROW + FROM + ( + SELECT RM.RegionName as Region, em2.EmployeeName AS [Area Manager] + , em1.Id as Supervisor_cd, em1.EmployeeName AS [Supervisor] + , CM.CityName as City, EM.Id as [Employee Id], em.EmployeeName AS Merchandiser + ,JP.VisitDate as VisitDate, sc.InTime + ,ISNULL((SELECT top 1 case + when d.ReasonId IS NULL OR d.ReasonId =0 then '' else REASON+' - ' + + case when replace(replace(D.REMARK,CHAR(13),''),CHAR(10),' ')<>'' then replace(replace(D.REMARK,CHAR(13),''),CHAR(10),' ') + else replace(replace(D.REMARK,CHAR(13),''),CHAR(10),' ') end end as REASON FROM OneApp_KelloggsMT.dbo.T_StoreCoverage D + INNER JOIN OneApp_KelloggsMT.dbo.Master_NonWorkingReason N ON N.ReasonId = D.ReasonId + WHERE D.ISDEL = 0 + AND D.EmpId = JP.EmpId AND D.StoreId = JP.StoreId AND d.VisitDate = JP.VisitDate),'') AS Remarks + , CASE WHEN (Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId = SC.EmpId AND JP.StoreId = SC.StoreId AND JP.VisitDate = SC.VisitDate + AND SC.ReasonId=11) >=1 THEN 1 + ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate + AND SC.ReasonId=2) >=1 THEN 2 + ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate + AND SC.ReasonId=6) >=1 THEN 4 + ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate + AND SC.ReasonId=12) >=1 THEN 5 + ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate + AND SC.ReasonId=13) >=1 THEN 6 + ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate + AND SC.ReasonId=7) >=1 THEN 8 + ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate + AND SC.ReasonId=26) >=1 THEN 10 + ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate + AND SC.ReasonId IN(0,1,3,4,5,8,9,10,14,15,17,18,19,20,21,22,24)) >=1 THEN 7 + END END END END END END END END AS COL, em.ResignDate as DOR, em.JoinDate as DOJ + + FROM OneApp_KelloggsMT.dbo.Mapping_JourneyPlan JP LEFT OUTER JOIN + OneApp_KelloggsMT.dbo.T_StoreCoverage SC ON JP.StoreId= SC.StoreId AND JP.EmpId= SC.EmpId AND JP.VisitDate= SC.VisitDate INNER JOIN + OneApp_KelloggsMT.dbo.Master_Store SM ON JP.StoreId= SM.StoreId INNER JOIN + OneApp_KelloggsMT.dbo.AspNetUsers em ON JP.EmpId= em.Id inner join + OneApp_KelloggsMT.dbo.AspNetUsers em1 on em1.Id = em.ManagerId inner join + OneApp_KelloggsMT.dbo.AspNetUsers em2 on em2.Id = em1.ManagerId inner join + OneApp_KelloggsMT.dbo.Master_City cm on em.CityId= cm.CityId inner join + OneApp_KelloggsMT.dbo.Master_State st on cm.StateId= st.StateId inner join + OneApp_KelloggsMT.dbo.Master_Region rm on st.RegionId= rm.RegionId --Left Join + + + + + WHERE EM.EMPLOYEENAME not like 'teststore%' AND JP.VisitDate BETWEEN '{start_date}' AND '{run_date}' + + ) AS T1 + ) AS T2 + ) AS T + WHERE COL1=1 + ) AS TT + on a.date=tt.visitdate)A)b + on emp.[Employee Id]=b.[MERCHANDISER CD] and emp.DATE= b.DATE ) + select * from Attendance where + parinaam_attendance !='-' and + project_id=40148 and + CONVERT(date,visit_date,101) > CONVERT(date,date_of_join,101) + order by employee_id, visit_date \ No newline at end of file diff --git a/src/sql/fact/Coverage.sql b/src/sql/fact/Coverage.sql index 35b7b75..bd3f337 100644 --- a/src/sql/fact/Coverage.sql +++ b/src/sql/fact/Coverage.sql @@ -1,164 +1,108 @@ -WITH SOS_BASE AS -( - SELECT - sm.CountryName, - sc.MID, - sm.RegionName, - sm.StateName, - sm.CityName, - Em.SupervisorName, - em.EmpId, - Em.EmpName AS EmployeeName, - Em.DesignationName AS Designation, - sm.StoreId, - CONVERT(varchar, sc.VisitDate, 101) AS VisitDate, - sm.StoreCode, - sm.StoreName, - sm.Address, - sm.StoreTypeid, - sm.ChannelId, - sm.ChainName, - MSD.SOSDefinitionName, +with Executed as +(select JP.mid, CASE WHEN isnull(B.MID,'')<>'' then 'Y' +ELSE 'N' END AS COVERED, +CASE WHEN isnull(C.MID,'')<>'' then 'Y' +ELSE 'N' END AS EXECUTED,isnull(NR.REASON,'') AS REASON + +from OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) left join +(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) +inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B + ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) + AND JP.ReasonId in (0,1,3,9,10,19,20))b +on JP.MID=b.MID + +left join +(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) +inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B + ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) + AND JP.ReasonId in (0,19))c +on JP.MID=c.MID +left join +OneApp_KelloggsMT.dbo.Master_NonWorkingReason nr on JP.ReasonId=nr.ReasonId +Where 1=1 AND JP.MID in ({mid_list}) + +) , + Coverage + (project_id,Mid, +store_id,employee_id,visit_date,in_time,out_time,duration_in_minutes, +is_covered,is_executed,reason_remarks,storetype_id, +supervisor_id,coverage_type,distance_in_meters,reasonId,camera_allow,update_date, +update_by,Unique_Id) - CASE - WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Category' THEN 'Category' - WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubCategory' THEN 'SubCategory' - WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Brand' THEN 'Brand' - WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubBrand' THEN 'SubBrand' - END AS SOSHeaderDeatils, +AS ( + +Select '40148' as ProjectId,JP.MID, sm.StoreId,JP.EmpId as EmpId, + Convert(VARCHAR,Jp.VisitDate) AS VisitDate +, Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, + Convert(Datetime,InTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, + Convert(Datetime,InTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage Where EmpId = Jp.EmpId And StoreId = Jp.StoreId + And VisitDate = Jp.VisitDate),'') As [In Time] + , Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, + Convert(Datetime,OutTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, + Convert(Datetime,OutTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage + Where EmpId = Jp.EmpId And StoreId = Jp.StoreId + And Convert(Date,VisitDate) = Jp.VisitDate),'') As [Out Time] + , + Isnull((Select Top 1 Case When convert(varchar,outtime) like '%00:00:00.00%' THEN 0 + Else Case When outtime0 Then 'Y' +Else 'N' End As Covered +, isnull(Exe.Executed,'') as Executed + +,Isnull((Select Top 1 Case +When D.ReasonId Is Null Or D.ReasonId =0 Then '' Else Reason+' - ' + +Case When Replace(Replace(D.Remark,Char(13),''''),Char(10),' ')<>'' Then Replace(Replace(D.Remark,Char(13),''),Char(10),' ') +Else Replace(Replace(D.Remark,Char(13),''),Char(10),' ') End End As Reason +From OneApp_KelloggsMT.dbo.T_StoreCoverage D +Inner Join OneApp_KelloggsMT.dbo.Master_NonWorkingReason N On N.ReasonId = D.ReasonId +Where D.Isdel = 0 +And D.EmpId = Jp.EmpId And D.StoreId = Jp.StoreId And D.VisitDate = Jp.VisitDate),'''') As [Detailed Remarks], +sm.StoreTypeId,Em.SupervisorId, +Case When jp.Deviation=0 Then 'Planned' When jp.Deviation=1 Then 'Adhoc' +When jp.Deviation=2 Then 'Beat Plan' When jp.Deviation=3 Then 'Non Merchandised' +When jp.Deviation=4 Then 'Add New Store' When jp.Deviation=5 Then 'Non Program' else '' End as [PJP Status] +,Isnull((Select Top 1 Case When (Sc.Latitude=0.00000000 Or Sc.Longitude Is Null) Then 0 Else Case When (sm.Latitude=0.00000000 Or sm.Latitude Is Null) Then 0 + Else Case When (sm.Longitude=0.00000000 Or sm.Longitude Is Null) Then 0 +Else SQRT(POWER(69.1 * ( Sc.Latitude - Sm.Latitude),2) + POWER(69.1 * ( Sm.Longitude - Sc.Longitude ) * COS(Sc.Latitude / 57.3), 2))*1000 + End End End As [Distance In Mtr] From OneApp_KelloggsMT.dbo.T_StoreCoverage Sc Where Sc.Isdel=0 And Sc.EmpId= Jp.EmpId +And Sc.StoreId= Jp.StoreId And Sc.VisitDate= Jp.VisitDate),'') As [Distance In Mtr] +,ISNULL(CAST (JP.ReasonId AS VARCHAR),'0'),sm.CameraAllow, - TS.SOSHeaderName, - TS.SOSHeaderValue AS SOSHeaderID, +GETDATE(),'SP-Pius' , - 'Header_Image' AS HDR1, + CAST('40148' AS VARCHAR) + '_' + CAST(SM.storeid AS VARCHAR) + + '_' + CAST(EM.EMPID AS VARCHAR) + + + +FROM OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) Inner Join +Executed Exe on Jp.MID= Exe.MID Inner Join +OneApp_KelloggsMT.dbo.vw_StoreDetail sm on Jp.StoreId= sm.StoreId Inner Join +OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on JP.EmpId= Em.EmpId +left join +OneApp_KelloggsMT.dbo.T_StoreCoveragePositionPivot pv with (nolock) +on jp.mid=pv.MID +Where 1=1 and em.UserName not like 'test%' AND JP.MID in ({mid_list} ) - CASE - WHEN ISNULL(TSC.SOSChildTable,'')='Master_Category' THEN 'Category' - WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubCategory' THEN 'SubCategory' - WHEN ISNULL(TSC.SOSChildTable,'')='Master_Brand' THEN 'Brand' - WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubBrand' THEN 'SubBrand' - END AS SOSChildDeatils, - - TSC.SOSChildName, - TSC.SOSChildValue AS SOSChildID, - TSC.ChildTotalFacing, - TS.SOSHeaderFacing, - TSC.ChildSelfFacing, - - ( - SELECT TOP 1 SOSTarget - FROM OneApp_KelloggsMT.dbo.Mapping_StoreShareOfShelfTarget a - WHERE a.SOSDefinitionId = MSD.SOSDefinitionId - AND a.StoreId = sm.StoreId - AND a.FromDate <= sc.VisitDate - AND a.ToDate >= sc.VisitDate - ) AS SOSTarget, - - CASE - WHEN ISNULL(SHI.SOSHeaderImage,'') = '' - THEN '' - ELSE - 'https://kimt1.parinaam.in/Upload/SOSImages/' - + SHI.SOSHeaderImage - END AS SOSHeaderImg - - FROM OneApp_KelloggsMT.dbo.T_ShareOfShelfHeader ts - - INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage sc - ON ts.MID = sc.MID - - INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail sm - ON sc.StoreId = sm.StoreId - - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em - ON sc.EmpId = Em.EmpId - - INNER JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChild tsc - ON ts.SOSId = tsc.SOSId - - INNER JOIN OneApp_KelloggsMT.dbo.Master_ShareOfShelfDefinition msd - ON msd.SOSDefinitionId = tsc.SOSDefinitionId - - LEFT JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfHeaderImages SHI - ON ts.SOSId = SHI.SOSId - AND ts.SOSHeaderValue = SHI.SOSHeaderValue - - LEFT JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChildImages SCI - ON tsc.SOSId = SCI.SOSId - AND tsc.SOSChildValue = SCI.SOSChildValue - - WHERE Em.EmpName NOT LIKE 'test%' - AND sc.MID IN ({mid_list}) -), - -SOS_PIVOT AS -( - SELECT * - FROM SOS_BASE - PIVOT - ( - MIN(SOSHeaderImg) - FOR HDR1 IN ([Header_Image]) - ) pvt -) - -SELECT - '40148' AS ProjectId, - MID, - EmpId AS employee_id, - StoreId AS store_id, - VisitDate AS visit_date, - StoreTypeid AS storetype_id, - ChannelId AS channel_id, - SOSDefinitionName, - SOSHeaderDeatils, - SOSHeaderName, - SOSHeaderID, - SOSChildDeatils, - SOSChildName, - SOSChildID, - SOSHeaderFacing, - ChildTotalFacing, - ChildSelfFacing, - SOSTarget, - GETDATE() AS update_date, - 'SP-Pius' AS update_by - -FROM SOS_PIVOT - -GROUP BY - CountryName, - MID, - RegionName, - StateName, - CityName, - SupervisorName, - EmpId, - EmployeeName, - Designation, - StoreId, - VisitDate, - StoreCode, - StoreName, - Address, - StoreTypeid, - ChannelId, - ChainName, - SOSDefinitionName, - SOSHeaderDeatils, - SOSHeaderName, - SOSHeaderID, - SOSChildDeatils, - SOSChildName, - SOSChildID, - ChildTotalFacing, - SOSHeaderFacing, - ChildSelfFacing, - SOSTarget - -ORDER BY - RegionName, - StateName, - CityName, - VisitDate; \ No newline at end of file +) +select * from Coverage \ No newline at end of file diff --git a/src/sql/fact/Login.sql b/src/sql/fact/Login.sql index e69de29..c0be767 100644 --- a/src/sql/fact/Login.sql +++ b/src/sql/fact/Login.sql @@ -0,0 +1,81 @@ +with employee_detail AS +( + SELECT rm.RegionId, rm.RegionName, st.StateId, st.StateName, + cm.CityId, cm.CityName, + em1.Id AS [Supervisor Code], ISNULL(em1.EmployeeName, '') AS Supervisor, + em.Id AS [Employee code], ISNULL(em.EmployeeName, '') AS [Employee Name], + dm.DesignationId, dm.DesignationName AS Designation + FROM OneApp_KelloggsMT.dbo.AspNetUsers em + INNER JOIN OneApp_KelloggsMT.dbo.AspNetUsers em1 ON em1.Id = em.ManagerId + INNER JOIN OneApp_KelloggsMT.dbo.Master_City cm ON em.CityId = cm.CityId + INNER JOIN OneApp_KelloggsMT.dbo.Master_State st ON cm.StateId = st.StateId + INNER JOIN OneApp_KelloggsMT.dbo.Master_Region rm ON st.RegionId = rm.RegionId + INNER JOIN OneApp_KelloggsMT.dbo.Master_Designation dm ON Em.DesignationId = dm.DesignationId + WHERE em.RightId IN (6) + AND (Em.ResignDate IS NULL OR CONVERT(DATE, Em.ResignDate, 101) >= '{run_date}') + AND em.EmployeeName NOT LIKE '%test%' +), +Login ( + project_id, + employee_id, login_date, login_time, process_id, + first_store_in_time, last_store_out_time, update_date, update_by, + Unique_Id +) AS ( + SELECT '40148' AS ProjectId, + Em.[Employee Code], + [Login Date], + [Login Time], + '0', + [1st Store In Time], + [Last Store Out Time], + GETDATE(), + 'SP-Pius', + CAST(40148 AS VARCHAR) + '_' + CAST(Em.[Employee Code] AS VARCHAR) + FROM employee_detail Em + LEFT OUTER JOIN + ( + SELECT SupervisorId AS [Supervisor Code], + SupervisorName, + EmpId AS [Employee Code], + EmpName, + DesignationName, + [Login Date], + [Login Time], + [1st Store In Time], + [Last Store Out Time], + AppVersion AS [App Version] + FROM ( + SELECT Em.SupervisorId, + Em.SupervisorName, + Em.EmpId, + Em.EmpName, + Em.DesignationName, + CONVERT(VARCHAR, ud.LoginDate, 101) AS [Login Date], + CONVERT(VARCHAR, ud.InTime, 108) AS [Login Time], + ud.AppVersion, + ROW_NUMBER() OVER ( + PARTITION BY ud.Empid, CONVERT(VARCHAR, ud.LoginDate, 101) + ORDER BY ud.LoginDate ASC + ) AS col, + (SELECT MIN(CONVERT(NVARCHAR, sc.InTime, 108)) + FROM OneApp_KelloggsMT.dbo.T_StoreCoverage sc + WHERE sc.Isdel = 0 + AND sc.EmpId = ud.EmpId + AND CONVERT(VARCHAR, sc.VisitDate, 101) = CONVERT(VARCHAR, ud.LoginDate, 101) + ) AS [1st Store In Time], + (SELECT MAX(CONVERT(NVARCHAR, OutTime, 108)) + FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC + WHERE SC.Isdel = 0 + AND SC.EmpId = UD.EmpId + AND CONVERT(VARCHAR, SC.VisitDate, 101) = CONVERT(VARCHAR, UD.LoginDate, 101) + ) AS [Last Store Out Time] + FROM OneApp_KelloggsMT.dbo.T_DeviceLogin ud + INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em ON ud.EmpId = Em.EmpId + WHERE CAST(ud.LoginDate AS DATE) = '{run_date}' -- fixed: direct date comparison + ) AS T1 + WHERE T1.col = 1 + ) AS T ON Em.[Employee code] = T.[Employee Code] + WHERE T.[Login Date] IS NOT NULL +) + +SELECT * FROM Login; \ No newline at end of file diff --git a/src/sql/fact/SOS_OneApp.sql b/src/sql/fact/SOS_OneApp.sql index e69de29..df248e2 100644 --- a/src/sql/fact/SOS_OneApp.sql +++ b/src/sql/fact/SOS_OneApp.sql @@ -0,0 +1,70 @@ +with SOS_OneApp ( project_id,Mid, +employee_id,store_id,visit_date,storetype_id,channel_id,SOSDefinitionName,SOSHeaderDeatils,SOSHeaderName,SOSHeaderID,SOSChildDeatils,SOSChildName,SOSChildID, +SOSHeaderFacing,ChildTotalFacing,ChildSelfFacing,SOSTarget,update_date,update_by) + as ( +Select '40148' AS ProjectId,MID, +EmpId +,StoreId, +VisitDate,StoreTypeid,ChannelId , +--CameraAllow, +SOSDefinitionName,SOSHeaderDeatils,SOSHeaderName,[SOSHeaderID] ,SOSChildDeatils , SOSChildName,SOSChildID,SOSHeaderFacing,ChildTotalFacing, +ChildSelfFacing ,SOSTarget,getdate(),'SP-Pius' from + ( +Select sm.CountryName,SC.MID, sm.RegionName, sm.StateName, sm.CityName, Em.SupervisorName, em.EmpId as EmpId +, Em.EmpName as EmployeeName, Em.DesignationName as Designation, sm.StoreId,convert(varchar,sc.VisitDate,101)VisitDate, sm.StoreCode, sm.StoreName, +sm.Address, sm.StoreTypeid ,sm.ChannelId,sm.ChainName, +MSD.SOSDefinitionName, case when isnull(TS.SOSHeaderTable,'')='Master_Category' then 'Category' +when isnull(TS.SOSHeaderTable,'')='Master_SubCategory' then 'SubCategory' +when isnull(TS.SOSHeaderTable,'')='Master_Brand' then 'Brand' +when isnull(TS.SOSHeaderTable,'')='Master_SubBrand' then 'SubBrand' +end as SOSHeaderDeatils , +TS.SOSHeaderName,TS.SOSHeaderValue[SOSHeaderID], 'Header'+'_'+'Image' as HDR1, +case when isnull(tsc.SOSChildTable,'')='Master_Category' then 'Category' +when isnull(tsc.SOSChildTable,'')='Master_SubCategory' then 'SubCategory' +when isnull(tsc.SOSChildTable,'')='Master_Brand' then 'Brand' +when isnull(tsc.SOSChildTable,'')='Master_SubBrand' then 'SubBrand' +end as SOSChildDeatils , +TSC.SOSChildName,TSC.SOSChildValue[SOSChildID],TSC.ChildTotalFacing,ts.SOSHeaderFacing,TSC.ChildSelfFacing +, +(select top 1 SOSTarget from OneApp_KelloggsMT.dbo.Mapping_StoreShareOfShelfTarget a where a.SOSDefinitionId=msd.SOSDefinitionId and +a.Storeid=sm.StoreId and a.fromdate<=sc.visitdate and a.todate>=sc.visitdate ) as SOSTarget, +case when isnull(SHI.SOSHeaderImage,'')='' then '' else 'https://kimt1.parinaam.in/Upload/SOSImages/'+SHI.SOSHeaderImage +end as SOSHeaderImg +--,case when isnull(SCI.SOSChildImage,'')='' then '' else 'https://di1.parinaam.in/Upload/SOSImages/'+SCI.SOSChildImage +--end as SOSChildImage + + from +OneApp_KelloggsMT.dbo.T_ShareOfShelfHeader ts Inner join +OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join +OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join +OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId +inner join +OneApp_KelloggsMT.dbo.T_ShareOfShelfChild tsc on ts.sosid=tsc.SOSId +inner join +OneApp_KelloggsMT.dbo.Master_ShareOfShelfDefinition msd on +msd.SOSDefinitionId=tsc.SOSDefinitionId + +LEFT join +OneApp_KelloggsMT.dbo.T_ShareOfShelfHeaderImages SHI ON ts.SOSId=SHI.SOSId AND TS.SOSHeaderValue=SHI.SOSHeaderValue +LEFT JOIN +OneApp_KelloggsMT.dbo.T_ShareOfShelfChildImages SCI ON TSC.SOSId=SCI.SOSId AND TSC.SOSChildValue=SCI.SOSChildValue +Where 1=1 AND EM.EMPNAME NOT LIKE 'test%' AND sc.mid IN ({mid_list}) +-- AND sc.VisitDate BETWEEN '09/18/2023' AND '09/21/2023' +--and sc.StoreId not in ( +--select store_id from SOS_OneApp a where a.project_id=40148 and a.store_id=sc.StoreId +--and a.employee_id=sc.EmpId and convert(date,sc.VisitDate,101)=convert(date,a.visit_date,101) ) + +) as A + +Pivot +( +min(SOSHeaderImg) +for HDR1 in([Header_Image]) +) as pvt1 + +GROUP BY CountryName,MID, RegionName, StateName, CityName, SupervisorName, EmpId +, EmployeeName, Designation, StoreId,VisitDate,StoreCode, StoreName, +Address, StoreTypeid ,ChannelId,ChainName, +SOSDefinitionName,SOSHeaderDeatils,SOSHeaderName,[SOSHeaderID] ,SOSChildDeatils , +SOSChildName,SOSChildID,ChildTotalFacing,SOSHeaderFacing,ChildSelfFacing,SOSTarget ) + select * from SOS_OneApp \ No newline at end of file diff --git a/src/sql/fact/Stock_Details.sql b/src/sql/fact/Stock_Details.sql index e69de29..0cb5b0a 100644 --- a/src/sql/fact/Stock_Details.sql +++ b/src/sql/fact/Stock_Details.sql @@ -0,0 +1,29 @@ +with Stock_Details (project_id, +Mid,supervisor_id,employee_id,store_id, +visitdate,storetype_id,store_category_id,product_id,MSL, MBQ,Stock_Qty, +damagedstock,loststock,expirystock,skuavailability,update_date,update_by,StockType) + +AS ( + +Select '40148' AS ProjectId,sc.MID,Em.SupervisorId,Em.EmpId,sm.StoreId,sc.VisitDate,sc.StoreTypeId,sm.StoreCategoryId, +vp.ProductId,ts.MSL,mp.MBQ, ISNULL(ts.OpeningStock,0)+ISNULL(TS.middaystock,0),0,0,0, +Case WHEN ISNULL(ts.OpeningStock,0)+ISNULL(TS.middaystock,0)>=1THEN 'Y' ELSE 'N' END AS +skuavailability, GETDATE(),'SP-Pius','Parinaam' + from +OneApp_KelloggsMT.dbo.T_Stock ts Inner join +OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join +OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join +OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId +inner join OneApp_KelloggsMT.dbo.vw_product vp on ts.ProductId=vp.ProductId inner JOIN +(SELECT MBQ,ProductId,StateId,ChainId,StoreTypeId,StoreCategoryId,StoreClassId FROM OneApp_KelloggsMT.dbo.Mapping_ProductAssortment where FromDate<='{run_date}' and ToDate>='{run_date}') mp on mp.StateId=sm.StateId and +mp.ChainId=sm.ChainId +and mp.StoreTypeId=sm.StoreTypeId +and mp.StoreCategoryId=sm.StoreCategoryId +and mp.StoreClassId=sm.StoreClassId +and mp.ProductId=ts.ProductId + + +Where Em.EmpName not like 'test%' AND sc.MID in ({mid_list}) + +) +select * from Stock_Details \ No newline at end of file diff --git a/y.yml b/y.yml index 49ec53a..f6daf0e 100644 --- a/y.yml +++ b/y.yml @@ -1,8 +1,8 @@ tables: - # - name: SOS_OneApp - # type: FACT - # operation: INSERT - # fetch_by: mids + - name: SOS_OneApp + type: FACT + operation: INSERT + fetch_by: mids # - name: OQaD # type: FACT @@ -14,8 +14,22 @@ tables: operation: INSERT fetch_by: mids + - name: Coverage + type: FACT + operation: INSERT + fetch_by: mids - # - name: Survey - # type: FACT - # operation: INSERT - # fetch_by: mids \ No newline at end of file + - name: Survey + type: FACT + operation: INSERT + fetch_by: mids + + - name: Login + type: FACT + operation: INSERT + fetch_by: run_date + + - name: Stock_Details + type: FACT + operation: INSERT + fetch_by: mids \ No newline at end of file