ankitm/data_pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

17-06-2026 1st commit

Browse Source
This commit is contained in:
Ankit Malik
2026-06-17 15:40:13 +05:30
parent 3337c62dd7
commit 9cd4135d7b
9 changed files with 999 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
logs/etl_20260616.log
+71
View File
@@ -0,0 +1,71 @@
2026-06-16 17:56:17 | INFO | ================================================================================
2026-06-16 17:56:17 | INFO | Hello from data-move Python data pipeline!
2026-06-16 17:56:17 | INFO | Pipeline Run Date: 2026-06-15
2026-06-16 17:56:17 | INFO | Connecting to databases...
2026-06-16 17:56:19 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000017B644A81B0>
2026-06-16 17:56:23 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000017B6548EFD0>
2026-06-16 17:56:24 | INFO | Database connections established
2026-06-16 17:56:24 | INFO | Collecting MIDs for: 2026-06-15
2026-06-16 17:56:24 | INFO | Found 849 MIDs
2026-06-16 17:57:33 | INFO | ================================================================================
2026-06-16 17:57:33 | INFO | Hello from data-move Python data pipeline!
2026-06-16 17:57:33 | INFO | Pipeline Run Date: 2026-06-15
2026-06-16 17:57:33 | INFO | Connecting to databases...
2026-06-16 17:57:34 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DCA21AB750>
2026-06-16 17:57:36 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DCA3371590>
2026-06-16 17:57:36 | INFO | Database connections established
2026-06-16 17:57:36 | INFO | Collecting MIDs for: 2026-06-15
2026-06-16 17:57:37 | INFO | Found 849 MIDs
2026-06-16 17:58:52 | INFO | ================================================================================
2026-06-16 17:58:52 | INFO | Hello from data-move Python data pipeline!
2026-06-16 17:58:52 | INFO | Pipeline Run Date: 2026-06-15
2026-06-16 17:58:52 | INFO | Connecting to databases...
2026-06-16 17:58:53 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E5AB2CF750>
2026-06-16 17:58:54 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E5AC458910>
2026-06-16 17:58:55 | INFO | Database connections established
2026-06-16 17:58:55 | INFO | Collecting MIDs for: 2026-06-15
2026-06-16 17:58:55 | INFO | Found 849 MIDs
2026-06-16 18:00:31 | INFO | ================================================================================
2026-06-16 18:00:31 | INFO | Hello from data-move Python data pipeline!
2026-06-16 18:00:31 | INFO | Pipeline Run Date: 2026-06-15
2026-06-16 18:00:31 | INFO | Connecting to databases...
2026-06-16 18:00:38 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E8C854F750>
2026-06-16 18:00:40 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E8C9688910>
2026-06-16 18:00:40 | INFO | Database connections established
2026-06-16 18:00:40 | INFO | Collecting MIDs for: 2026-06-15
2026-06-16 18:00:42 | INFO | Found 849 MIDs
2026-06-16 18:00:44 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-16 18:02:58 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
2026-06-16 18:07:47 | INFO | ================================================================================
2026-06-16 18:07:47 | INFO | Hello from data-move Python data pipeline!
2026-06-16 18:07:47 | INFO | Pipeline Run Date: 2026-06-15
2026-06-16 18:07:47 | INFO | Connecting to databases...
2026-06-16 18:07:48 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DF4BB2F750>
2026-06-16 18:07:50 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DF4CCE8A50>
2026-06-16 18:07:50 | INFO | Database connections established
2026-06-16 18:07:50 | INFO | Collecting MIDs for: 2026-06-15
2026-06-16 18:07:50 | INFO | Found 849 MIDs
2026-06-16 18:07:51 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-16 18:08:09 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
2026-06-16 18:08:58 | INFO | ================================================================================
2026-06-16 18:08:58 | INFO | Hello from data-move Python data pipeline!
2026-06-16 18:08:58 | INFO | Pipeline Run Date: 2026-06-15
2026-06-16 18:08:58 | INFO | Connecting to databases...
2026-06-16 18:08:59 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001639269F750>
2026-06-16 18:09:01 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000016393858910>
2026-06-16 18:09:01 | INFO | Database connections established
2026-06-16 18:09:01 | INFO | Collecting MIDs for: 2026-06-15
2026-06-16 18:09:02 | INFO | Found 849 MIDs
2026-06-16 18:09:02 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-16 18:09:19 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
2026-06-16 18:11:21 | INFO | ================================================================================
2026-06-16 18:11:21 | INFO | Hello from data-move Python data pipeline!
2026-06-16 18:11:21 | INFO | Pipeline Run Date: 2026-06-15
2026-06-16 18:11:21 | INFO | Connecting to databases...
2026-06-16 18:11:23 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002240B7AF750>
2026-06-16 18:11:24 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002240C978910>
2026-06-16 18:11:24 | INFO | Database connections established
2026-06-16 18:11:24 | INFO | Collecting MIDs for: 2026-06-15
2026-06-16 18:11:25 | INFO | Found 849 MIDs
2026-06-16 18:11:25 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-16 18:13:05 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
logs/etl_20260617.log
+688
View File
@@ -0,0 +1,688 @@
2026-06-17 10:17:16 | INFO | ================================================================================
2026-06-17 10:17:16 | INFO | Hello from data-move Python data pipeline!
2026-06-17 10:17:16 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 10:17:16 | INFO | Connecting to databases...
2026-06-17 10:17:18 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000019A9A2CF750>
2026-06-17 10:17:21 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000019A9B778910>
2026-06-17 10:17:22 | INFO | Database connections established
2026-06-17 10:17:22 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 10:17:22 | INFO | Found 863 MIDs
2026-06-17 10:17:23 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 10:17:44 | INFO | Fetched 290,984 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 10:43:05 | INFO | ================================================================================
2026-06-17 10:43:05 | INFO | Hello from data-move Python data pipeline!
2026-06-17 10:43:05 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 10:43:05 | INFO | Connecting to databases...
2026-06-17 10:43:07 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002CE210FF750>
2026-06-17 10:43:08 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002CE2253CA50>
2026-06-17 10:43:09 | INFO | Database connections established
2026-06-17 10:43:09 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 10:43:10 | INFO | Found 863 MIDs
2026-06-17 10:43:11 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 10:43:55 | INFO | Fetched 291,001 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 10:45:51 | INFO | ================================================================================
2026-06-17 10:45:51 | INFO | Hello from data-move Python data pipeline!
2026-06-17 10:45:51 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 10:45:51 | INFO | Connecting to databases...
2026-06-17 10:45:52 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000214EC45F750>
2026-06-17 10:45:53 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000214ED908910>
2026-06-17 10:45:54 | INFO | Database connections established
2026-06-17 10:45:54 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 10:45:55 | INFO | Found 863 MIDs
2026-06-17 10:45:55 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 10:46:11 | INFO | Fetched 291,005 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 10:46:11 | INFO | ================================================================================
2026-06-17 10:46:11 | INFO | Processing Table: OQaD
2026-06-17 10:46:11 | ERROR | Failed processing table OQaD
Traceback (most recent call last):
File "D:\data_move\main2.py", line 212, in main
df=fetch_OQaD( engine=sql_engine,
table_name=table_name,
...<2 lines>...
run_date=run_date
)
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
with open(sql_file, "r", encoding="utf-8") as f:
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
2026-06-17 10:48:44 | INFO | ================================================================================
2026-06-17 10:48:44 | INFO | Hello from data-move Python data pipeline!
2026-06-17 10:48:44 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 10:48:44 | INFO | Connecting to databases...
2026-06-17 10:48:45 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000021A270378B0>
2026-06-17 10:48:46 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000021A28538910>
2026-06-17 10:48:47 | INFO | Database connections established
2026-06-17 10:48:47 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 10:48:48 | INFO | Found 863 MIDs
2026-06-17 10:48:48 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 10:49:00 | INFO | Fetched 291,005 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 10:49:01 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 10:49:01 | INFO | ================================================================================
2026-06-17 10:49:01 | INFO | Processing Table: OQaD
2026-06-17 10:49:01 | ERROR | Failed processing table OQaD
Traceback (most recent call last):
File "D:\data_move\main2.py", line 215, in main
df=fetch_OQaD( engine=sql_engine,
table_name=table_name,
...<2 lines>...
run_date=run_date
)
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
with open(sql_file, "r", encoding="utf-8") as f:
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
2026-06-17 10:53:50 | INFO | ================================================================================
2026-06-17 10:53:50 | INFO | Hello from data-move Python data pipeline!
2026-06-17 10:53:50 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 10:53:50 | INFO | Connecting to databases...
2026-06-17 10:53:51 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001D1D6DB78B0>
2026-06-17 10:53:52 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001D1D81F8910>
2026-06-17 10:53:53 | INFO | Database connections established
2026-06-17 10:53:53 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 10:53:54 | INFO | Found 863 MIDs
2026-06-17 10:53:54 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 10:54:08 | INFO | Fetched 291,006 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 10:54:09 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 10:54:09 | INFO | ================================================================================
2026-06-17 10:54:09 | INFO | Processing Table: OQaD
2026-06-17 10:54:09 | ERROR | Failed processing table OQaD
Traceback (most recent call last):
File "D:\data_move\main2.py", line 217, in main
df=fetch_OQaD( engine=sql_engine,
table_name=table_name,
...<2 lines>...
run_date=run_date
)
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
with open(sql_file, "r", encoding="utf-8") as f:
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
2026-06-17 10:57:28 | INFO | ================================================================================
2026-06-17 10:57:28 | INFO | Hello from data-move Python data pipeline!
2026-06-17 10:57:28 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 10:57:28 | INFO | Connecting to databases...
2026-06-17 10:57:30 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000197AB2778B0>
2026-06-17 10:57:31 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000197AC768910>
2026-06-17 10:57:32 | INFO | Database connections established
2026-06-17 10:57:32 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 10:57:32 | INFO | Found 863 MIDs
2026-06-17 10:57:33 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 10:57:45 | INFO | Fetched 291,010 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 10:57:46 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 10:57:46 | INFO | ================================================================================
2026-06-17 10:57:46 | INFO | Processing Table: OQaD
2026-06-17 10:57:46 | ERROR | Failed processing table OQaD
Traceback (most recent call last):
File "D:\data_move\main2.py", line 217, in main
df=fetch_OQaD( engine=sql_engine,
table_name=table_name,
...<2 lines>...
run_date=run_date
)
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
with open(sql_file, "r", encoding="utf-8") as f:
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
2026-06-17 10:59:57 | INFO | ================================================================================
2026-06-17 10:59:57 | INFO | Hello from data-move Python data pipeline!
2026-06-17 10:59:57 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 10:59:57 | INFO | Connecting to databases...
2026-06-17 10:59:58 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001A32C2078B0>
2026-06-17 11:00:00 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001A32D648910>
2026-06-17 11:00:00 | INFO | Database connections established
2026-06-17 11:00:00 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 11:00:01 | INFO | Found 863 MIDs
2026-06-17 11:00:02 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 11:00:24 | INFO | Fetched 291,011 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 11:00:25 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 11:00:25 | INFO | ================================================================================
2026-06-17 11:00:25 | INFO | Processing Table: OQaD
2026-06-17 11:00:25 | ERROR | Failed processing table OQaD
Traceback (most recent call last):
File "D:\data_move\main2.py", line 217, in main
df=fetch_OQaD( engine=sql_engine,
table_name=table_name,
...<2 lines>...
run_date=run_date
)
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
with open(sql_file, "r", encoding="utf-8") as f:
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\fact\\OQaD.sql'
2026-06-17 12:09:34 | INFO | ================================================================================
2026-06-17 12:09:34 | INFO | Hello from data-move Python data pipeline!
2026-06-17 12:09:34 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 12:09:34 | INFO | Connecting to databases...
2026-06-17 12:09:37 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000274B86078B0>
2026-06-17 12:09:39 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000274B9A18910>
2026-06-17 12:09:39 | INFO | Database connections established
2026-06-17 12:09:39 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 12:09:41 | INFO | Found 863 MIDs
2026-06-17 12:09:43 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 12:10:48 | INFO | Fetched 291,093 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 12:10:48 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 12:10:48 | INFO | ================================================================================
2026-06-17 12:10:48 | INFO | Processing Table: OQaD
2026-06-17 12:10:48 | INFO | Exists: False
2026-06-17 12:10:48 | INFO | Path: D:\data_move\sql\fact\OQaD.sql
2026-06-17 12:10:48 | ERROR | Failed processing table OQaD
Traceback (most recent call last):
File "D:\data_move\main2.py", line 217, in main
df=fetch_OQaD( engine=sql_engine,
table_name=table_name,
...<2 lines>...
run_date=run_date
)
File "D:\data_move\src\fact.py", line 62, in fetch_OQaD
with open(sql_file, "r", encoding="utf-8") as f:
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\fact\\OQaD.sql'
2026-06-17 12:11:47 | INFO | ================================================================================
2026-06-17 12:11:47 | INFO | Hello from data-move Python data pipeline!
2026-06-17 12:11:47 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 12:11:47 | INFO | Connecting to databases...
2026-06-17 12:11:48 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000241C8DF78B0>
2026-06-17 12:11:50 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000241CA238910>
2026-06-17 12:11:51 | INFO | Database connections established
2026-06-17 12:11:51 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 12:11:52 | INFO | Found 863 MIDs
2026-06-17 12:11:52 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 12:12:14 | INFO | Fetched 291,097 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 12:12:14 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 12:12:14 | INFO | ================================================================================
2026-06-17 12:12:14 | INFO | Processing Table: OQaD
2026-06-17 12:12:14 | INFO | Exists: True
2026-06-17 12:12:14 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
2026-06-17 12:12:14 | INFO | Fetching data for 893 EMPIDs
2026-06-17 12:12:14 | INFO | Fetching OQaD data for run_date=2026-06-16
2026-06-17 12:12:14 | ERROR | Failed processing table OQaD
Traceback (most recent call last):
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context
self.dialect.do_execute(
~~~~~~~~~~~~~~~~~~~~~~~^
cursor, str_statement, effective_parameters, context
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute
cursor.execute(statement, parameters)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near 'QUIZ'. (102) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'where'. (156)")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "D:\data_move\main2.py", line 217, in main
df=fetch_OQaD( engine=sql_engine,
table_name=table_name,
...<2 lines>...
run_date=run_date
)
File "D:\data_move\src\fact.py", line 74, in fetch_OQaD
df = pl.read_database(
query=sql,
connection=engine,
)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database
return cx.execute(
~~~~~~~~~~^
query=query,
^^^^^^^^^^^^
options=execute_options,
^^^^^^^^^^^^^^^^^^^^^^^^
).to_polars(
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute
result = cursor_execute(query, **options)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute
return meth(
self,
distilled_parameters,
execution_options or NO_OPTIONS,
)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection
return connection._execute_clauseelement(
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
self, distilled_params, execution_options
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement
ret = self._execute_context(
dialect,
...<8 lines>...
cache_hit=cache_hit,
)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context
return self._exec_single_context(
~~~~~~~~~~~~~~~~~~~~~~~~~^
dialect, context, statement, parameters
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context
self._handle_dbapi_exception(
~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
e, str_statement, effective_parameters, cursor, context
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context
self.dialect.do_execute(
~~~~~~~~~~~~~~~~~~~~~~~^
cursor, str_statement, effective_parameters, context
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute
cursor.execute(statement, parameters)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near 'QUIZ'. (102) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'where'. (156)")
[SQL: WITH MID_TABLE_COV1 AS
(
SELECT EmpId, VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE CreateDate >= 2026-06-16
AND CreateDate < DATEADD(DAY,1,'2026-06-16')
UNION ALL
SELECT EmpId, VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE UpdateDate >= 2026-06-16
AND UpdateDate < DATEADD(DAY,1, '2026-06-16')
),
OQaD
QUIZ AS
(
SELECT DISTINCT
E.EmpId,
E.EmpName,
E.SupervisorId,
E.SupervisorName,
E.DesignationName,
E.CityName,
E.StateName,
E.RegionName,
CAST(DQ.VisitDate AS DATE) AS VisitDate,
DQ.QuestionId,
DQ.AnswerId,
QC.QuestionCategoryId,
QC.QuestionCategory
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E
ON DQ.EmpId = E.EmpId
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU
ON DQ.QuestionId = QU.QuestionId
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC
ON QU.QuestionCategoryId = QC.QuestionCategoryId
WHERE E.EmpName NOT LIKE '%TEST%'
AND E.RightId = 6
AND (
E.ResignDate IS NULL
OR CAST(E.ResignDate AS DATE) >= '2026-06-16'
)
AND EXISTS
(
SELECT 1
FROM MID_TABLE_COV1 A
WHERE A.EmpId = DQ.EmpId
AND CAST(A.VisitDate AS DATE) = CAST(DQ.VisitDate AS DATE)
)
)
SELECT
40148 AS project_id,
Q.EmpId AS employee_id,
0 AS process_id,
Q.VisitDate AS visit_date,
Q.QuestionCategoryId AS question_category_id,
Q.QuestionCategory AS question_category,
QM.QuestionId AS question_id,
QM.Question AS question,
ISNULL(QA.AnswerId,0) AS answer_id,
ISNULL(QA.Answer,'') AS answer,
CASE
WHEN QA.AnswerId IS NULL THEN 'Not Answer'
WHEN QA.RightAnswer = 1 THEN 'Y'
WHEN QA.RightAnswer IS NULL THEN 'Not Answer'
ELSE 'N'
END AS correct_answer,
GETDATE() AS update_date,
'SP-Pius' AS update_by
FROM QUIZ Q
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM
ON Q.QuestionId = QM.QuestionId
LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA
ON Q.AnswerId = QA.AnswerId;
where Q.EmpId not in (3167,4724,4019,4902,4375,3352,3081,5244,3181,4810,5166,4895,4817,5173,5237,3608,4646,4126,3243,3948,3950,3599,3777,5235,5346,3606,3079,5175,5242,5064,4366,5071,5339,4812,4722,3169,5270,3326,3689,4062,3131,5054,5178,3371,4829,5185,4698,4836,3760,4665,5218,4451,4280,3269,3098,5327,4978,5261,3335,5090,5320,4214,4919,4570,3150,4221,3328,3112,3290,4950,4430,5104,3383,5213,4864,5199,5206,4439,3207,5033,4684,3979,5211,4774,3117,5102,5280,3309,4931,4753,5123,3155,5085,4914,5092,4387,5325,5154,4100,4805,3402,5187,3580,3409,3060,3962,4696,4831,3715,4710,4242,5282,5121,4933,5289,5282,3444,5104,3095,4933,4142,3437,3530,3088,3715,4840,5002,4831,5187,3361,3962,5113,3086,5106,5256,5085,4914,5092,3470,4387,5277,3309,5239,4712,4945,5246,5123,4774,3952,3603,4999,3727,3378,3207,5341,4992,5199,5334,4672,5206,3342,3520,3454,3171,4916,3105,4396,3112,3290,5265,3119,5227,3143,3150,5056,4529,3328,5313,5220,4693,5320,4615,3755,4444,3406,4978,4451,3746,4280,3399,5322,4617,4795,5218,3330,3159,4698,5144,5054,3473,3131,4726,4688,4743,3689,3812,3107,3169,5332,3347,5023,5022,4812,3380,3067,3950,4954,5132,3081,5296,5244,3140,4019,3492,4375,5210,3126,5289,3482,4847,5296,4954,4436,3380,5127,5305,3306,5291,5023,5120,4531,4346,5229,3107,3812,4353,3145,3152,3330,5322,4439,5144,4795,3363,4054,4232,3399,4239,3755,5279,4930,5213,3406,4864,5286,5220,4693,5101,5227,3985,3297,3636,4529,3420,5265,4916,3591,3335,4460,5343,4638,3342,5327,3520,3171,4467,3700,5336,4987,3515,4992,3760,5267,4562,3247,5082,4555,4733,5061,5239,3090,4712,5246,4904,5270,3059,4866,5277,4750,5106,3361,4968,5175,5310,3496,3608,4840,5248,4543,3615,4142,3437,3088,3444,3078,4902,3095,3567,4543,4450,3140,5319,4614,3496,4970,4443,5132,3916,3149,5310,3327,5312,3156,4968,4894,3394,5065,4538,4688,4866,3251,3090,3268,4671,4849,4904,4726,5082,4555,5191,4733,4562,3515,5300,4951,5115,3173,5293,4944,4460,4987,4638,4467,3591,3064,4526,3821,3232,3420,5056,5046,5008,3985,3636,5350,3109,5272,5101,5279,3358,4396,5172,5110,4054,4232,5281,3363,4999,3638,3603,3104,5027,5205,4856,4151,3083,4346,3764,3819,3415,4353,3470,3299,5167,3346,5113,3306,5331,5291,5160,5120,5127,4913,5091,4564,5262,5196,4847,5077,4666,4139,3612,3978,3078,5340,4139,3612,4991,5248,3170,5155,5333,4628,5077,4217,5084,4032,4913,4564,5257,5324,4381,5331,5250,5160,4134,4312,4141,3059,3764,3415,4305,5027,3220,3104,5321,4972,5307,5267,5136,5314,4787,3315,3322,4540,3835,3700,3486,4547,5172,5001,5008,3427,3379,5046,3372,3821,4526,3650,5124,5302,5286,3130,5115,4937,3132,5293,5117,3118,4239,5300,4951,4343,3897,5219,5191,4671,4849,5212,5229,5003,5181,4531,5065,5345,3538,4538,4189,4229,3531,5276,5305,4578,3106,3149,5312,5269,3113,4436,3099,4443,4970,3916,3567,3975,4153,4844,5210,5200,4844,3533,4666,5022,3184,5200,4402,4153,3704,3975,4922,3099,5262,4751,5091,3106,5269,5093,5276,4578,3346,5202,4675,4229,3531,3182,5195,5167,3203,5003,5205,5150,5328,3220,4979,4856,5157,4452,5335,3514,5219,3638,4343,4383,3289,4390,3118,3296,5281,5110,3438,3267,5074,5117,5252,3780,3431,5005,3372,3379,5176,3232,5316,4611,4967,3068,5309,3139,3495,3673,4540,3835,4891,3486,3488,3315,3322,5307,5136,3070,4877,4787,5314,5055,3939,3234,4072,5181,4305,3251,3374,5119,4134,3080,5297,4065,4894,3127,5250,4763,4941,5257,3298,4381,3327,3156,4687,4338,5221,4450,5155,3400,5333,5207,4628,5162,3742,4098,3742,4943,5121,3129,5207,5114,5292,4687,3122,4338,4696,3464,3113,4333,3298,3127,5031,4941,4763,5209,5119,3388,3566,3538,4065,3744,4072,3374,4753,3234,3939,5102,5088,5266,3070,5055,4877,3096,3979,3132,3488,3139,4841,5019,5302,3495,5124,5309,5012,4611,3369,3509,5176,5145,5323,5005,3153,4656,4371,3431,5254,5252,4547,3091,5074,4198,3782,3433,4383,4651,4829,4302,3953,5349,5321,3514,4972,5178,5150,5328,4979,5342,4452,5157,5304,3203,5195,3490,4312,3141,5126,4141,4675,4366,5057,5235,5064,3243,5093,4217,4922,3407,4751,4810,3170,4283,3756,3704,4974,3533,4991,3184,4974,3492,3143,4098,4283,3400,3756,3407,4710,5221,3067,3580,5057,3409,5196,3060,4805,4100,3402,5325,5297,3490,3141,5154,5126,3155,5304,4931,5109,5287,4117,5280,3473,4651,4302,3953,5033,5211,3782,4684,3433,3084,4891,3746,3383,4371,4022,5254,3673,5313,4207,3153,5316,3509,5145,5323,4430,4919,4570,3369,4221,3105,5090,4563,5012,3269,4841,3098,5019,3454,5261,3438,4672,3267,4836,3096,4665,5259,5088,3727,5266,3378,5185,4390,4397,3371,4062,4945,3528,3326,4722,3388,5256,3566,3744,5202,5242,3086,5071,5209,3777,4333,3606,3079,5002,3948,4126,3599,3464,5344,4817,5173,5351,4646,3122,3352,3530,5337,3181,3359,5166,5292,5114,4943)
-- select d.employee_id from OQaD d where project_id=40148 and d.employee_id=Q.EmpId and CONVERT(date,d.visit_date,101)=
-- CONVERT(date,q.VisitDate,101))
]
(Background on this error at: https://sqlalche.me/e/20/f405)
2026-06-17 12:19:35 | INFO | ================================================================================
2026-06-17 12:19:35 | INFO | Hello from data-move Python data pipeline!
2026-06-17 12:19:35 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 12:19:35 | INFO | Connecting to databases...
2026-06-17 12:19:37 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000210484378B0>
2026-06-17 12:19:38 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000021049928910>
2026-06-17 12:19:39 | INFO | Database connections established
2026-06-17 12:19:39 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 12:19:40 | INFO | Found 863 MIDs
2026-06-17 12:19:40 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 12:20:14 | INFO | Fetched 291,100 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 12:20:14 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 12:20:14 | INFO | ================================================================================
2026-06-17 12:20:14 | INFO | Processing Table: OQaD
2026-06-17 12:20:14 | INFO | Exists: True
2026-06-17 12:20:14 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
2026-06-17 12:20:14 | INFO | Fetching data for 893 EMPIDs
2026-06-17 12:20:14 | INFO | Fetching OQaD data for run_date=2026-06-16
2026-06-17 12:20:30 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
2026-06-17 12:20:30 | ERROR | Failed processing table OQaD
Traceback (most recent call last):
File "D:\data_move\main2.py", line 226, in main
df = get_dataframe(
fn_name=fn_name,
...<3 lines>...
run_date=run_date,
)
File "D:\data_move\main2.py", line 81, in get_dataframe
return fn(sql_engine, run_date)
TypeError: fetch_OQaD() missing 3 required positional arguments: 'table_type', 'empids', and 'run_date'
2026-06-17 12:38:43 | INFO | ================================================================================
2026-06-17 12:38:43 | INFO | Hello from data-move Python data pipeline!
2026-06-17 12:38:43 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 12:38:43 | INFO | Connecting to databases...
2026-06-17 12:38:45 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001C35AC2F750>
2026-06-17 12:38:46 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001C35BDF87D0>
2026-06-17 12:38:46 | INFO | Database connections established
2026-06-17 12:38:46 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 12:38:47 | INFO | Found 863 MIDs
2026-06-17 12:38:47 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 12:38:58 | INFO | Fetched 291,112 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 12:38:58 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 12:38:58 | INFO | ================================================================================
2026-06-17 12:38:58 | INFO | Processing Table: OQaD
2026-06-17 12:38:58 | INFO | Exists: True
2026-06-17 12:38:58 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
2026-06-17 12:38:58 | INFO | Fetching data for 893 EMPIDs
2026-06-17 12:38:58 | INFO | Fetching OQaD data for run_date=2026-06-16
2026-06-17 12:39:05 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
2026-06-17 12:39:05 | INFO | ================================================================================
2026-06-17 12:39:05 | INFO | Pipeline Completed Successfully
2026-06-17 12:39:05 | INFO | ================================================================================
2026-06-17 12:39:30 | INFO | ================================================================================
2026-06-17 12:39:30 | INFO | Hello from data-move Python data pipeline!
2026-06-17 12:39:30 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 12:39:30 | INFO | Connecting to databases...
2026-06-17 12:39:31 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000025D1633F750>
2026-06-17 12:39:32 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000025D175887D0>
2026-06-17 12:39:33 | INFO | Database connections established
2026-06-17 12:39:33 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 12:39:33 | INFO | Found 863 MIDs
2026-06-17 12:39:33 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 12:39:43 | INFO | Fetched 291,112 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 12:39:43 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 12:39:43 | INFO | ================================================================================
2026-06-17 12:39:43 | INFO | Processing Table: OQaD
2026-06-17 12:39:43 | INFO | Exists: True
2026-06-17 12:39:43 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
2026-06-17 12:39:43 | INFO | Fetching data for 893 EMPIDs
2026-06-17 12:39:43 | INFO | Fetching OQaD data for run_date=2026-06-16
2026-06-17 12:39:46 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
2026-06-17 12:39:46 | INFO | ================================================================================
2026-06-17 12:39:46 | INFO | Pipeline Completed Successfully
2026-06-17 12:39:46 | INFO | ================================================================================
2026-06-17 12:44:50 | INFO | ================================================================================
2026-06-17 12:44:50 | INFO | Hello from data-move Python data pipeline!
2026-06-17 12:44:50 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 12:44:50 | INFO | Connecting to databases...
2026-06-17 12:44:51 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E3647178B0>
2026-06-17 12:44:52 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E365BB8910>
2026-06-17 12:44:53 | INFO | Database connections established
2026-06-17 12:44:53 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 12:44:53 | INFO | Found 863 MIDs
2026-06-17 12:44:53 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 12:45:07 | INFO | Fetched 291,115 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 12:45:08 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 12:45:08 | INFO | ================================================================================
2026-06-17 12:45:08 | INFO | Processing Table: OQaD
2026-06-17 12:45:08 | INFO | Exists: True
2026-06-17 12:45:08 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
2026-06-17 12:45:08 | INFO | Fetching data for 893 EMPIDs
2026-06-17 12:45:08 | INFO | Fetching OQaD data for run_date=2026-06-16
2026-06-17 12:45:10 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
2026-06-17 12:45:10 | INFO | Fetched 4998 rows
2026-06-17 12:45:10 | INFO | Deleting from OQaD
2026-06-17 12:45:11 | INFO | OQaD: inserted 4,998 rows into ClickHouse
2026-06-17 12:45:11 | INFO | OQaD loaded successfully (4998 rows)
2026-06-17 12:45:11 | INFO | ================================================================================
2026-06-17 12:45:11 | INFO | Pipeline Completed Successfully
2026-06-17 12:45:11 | INFO | ================================================================================
2026-06-17 13:23:53 | INFO | ================================================================================
2026-06-17 13:23:53 | INFO | Hello from data-move Python data pipeline!
2026-06-17 13:23:53 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 13:23:53 | INFO | Connecting to databases...
2026-06-17 13:23:54 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000281306878B0>
2026-06-17 13:23:56 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000028131AD8910>
2026-06-17 13:23:56 | INFO | Database connections established
2026-06-17 13:23:56 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 13:23:57 | INFO | Found 863 MIDs
2026-06-17 13:23:58 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 13:24:15 | INFO | Fetched 291,128 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 13:24:15 | INFO | Fetched 4,819 matched empids fetched for OQAD
2026-06-17 13:24:15 | INFO | ================================================================================
2026-06-17 13:24:15 | INFO | Processing Table: OQaD
2026-06-17 13:24:15 | INFO | Exists: True
2026-06-17 13:24:15 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
2026-06-17 13:24:15 | INFO | Fetching data for 4,819 EMPIDs
2026-06-17 13:24:15 | INFO | Fetching OQaD data for run_date=2026-06-16
2026-06-17 13:24:22 | INFO | fn name is fetch_OQad ------Fetched 1072 rows
2026-06-17 13:24:22 | INFO | Fetched 1072 rows
2026-06-17 13:24:22 | INFO | Deleting from OQaD
2026-06-17 13:24:23 | INFO | OQaD: inserted 1,072 rows into ClickHouse
2026-06-17 13:24:23 | INFO | OQaD loaded successfully (1072 rows)
2026-06-17 13:24:23 | INFO | ================================================================================
2026-06-17 13:24:23 | INFO | Processing Table: Survey
2026-06-17 13:24:23 | INFO | Fetching data for 863 MIDs
2026-06-17 13:24:23 | INFO | Fetched 142 rows from SQL Server
2026-06-17 13:24:23 | INFO | Fetched 142 rows
2026-06-17 13:24:23 | INFO | Deleting from Survey
2026-06-17 13:24:24 | ERROR | Failed processing table Survey
Traceback (most recent call last):
File "D:\data_move\main2.py", line 299, in main
load_to_clickhouse(
~~~~~~~~~~~~~~~~~~^
client=client,
^^^^^^^^^^^^^^
table_name=table_name,
^^^^^^^^^^^^^^^^^^^^^^
df=df,
^^^^^^
)
^
File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse
client.insert_arrow(
~~~~~~~~~~~~~~~~~~~^
table=table_name,
^^^^^^^^^^^^^^^^^
arrow_table=arrow_table,
^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow
return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings)
~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert
response = self._raw_request(insert_block, params, headers, server_wait=False)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request
self._error_handler(response)
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler
raise err_type(err_str, code=code, name=name) from None
clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column Project_Id in table kelloggs_1.Survey (d8c44908-2ed7-4d29-bd10-8e70fa84dfba). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123)
2026-06-17 13:27:22 | INFO | ================================================================================
2026-06-17 13:27:22 | INFO | Hello from data-move Python data pipeline!
2026-06-17 13:27:22 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 13:27:22 | INFO | Connecting to databases...
2026-06-17 13:27:23 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000026D13D878B0>
2026-06-17 13:27:25 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000026D151D8910>
2026-06-17 13:27:25 | INFO | Database connections established
2026-06-17 13:27:25 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 13:27:26 | INFO | Found 863 MIDs
2026-06-17 13:27:26 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 13:27:37 | INFO | Fetched 291,128 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 13:27:38 | INFO | Fetched 4,814 matched empids fetched for OQAD
2026-06-17 13:27:38 | INFO | ================================================================================
2026-06-17 13:27:38 | INFO | Processing Table: OQaD
2026-06-17 13:27:38 | INFO | Exists: True
2026-06-17 13:27:38 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
2026-06-17 13:27:38 | INFO | Fetching data for 4,814 EMPIDs
2026-06-17 13:27:38 | INFO | Fetching OQaD data for run_date=2026-06-16
2026-06-17 13:27:44 | INFO | fn name is fetch_OQad ------Fetched 1072 rows
2026-06-17 13:27:44 | INFO | Fetched 1072 rows
2026-06-17 13:27:44 | INFO | Deleting from OQaD
2026-06-17 13:27:45 | INFO | OQaD: inserted 1,072 rows into ClickHouse
2026-06-17 13:27:45 | INFO | OQaD loaded successfully (1072 rows)
2026-06-17 13:27:45 | INFO | ================================================================================
2026-06-17 13:27:45 | INFO | Processing Table: Survey
2026-06-17 13:27:45 | INFO | Fetching data for 863 MIDs
2026-06-17 13:27:45 | INFO | Fetched 142 rows from SQL Server
2026-06-17 13:27:45 | INFO | Fetched 142 rows
2026-06-17 13:27:45 | INFO | Creating table Survey
2026-06-17 13:27:45 | INFO | Table ready: Survey
2026-06-17 13:27:45 | INFO | Survey: inserted 142 rows into ClickHouse
2026-06-17 13:27:45 | INFO | Survey loaded successfully (142 rows)
2026-06-17 13:27:45 | INFO | ================================================================================
2026-06-17 13:27:45 | INFO | Pipeline Completed Successfully
2026-06-17 13:27:45 | INFO | ================================================================================
2026-06-17 15:05:42 | INFO | ================================================================================
2026-06-17 15:05:42 | INFO | Hello from data-move Python data pipeline!
2026-06-17 15:05:42 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 15:05:42 | INFO | Connecting to databases...
2026-06-17 15:05:43 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002009FB778B0>
2026-06-17 15:05:45 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000200A0FB8910>
2026-06-17 15:05:45 | INFO | Database connections established
2026-06-17 15:05:45 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 15:05:45 | INFO | Found 863 MIDs
2026-06-17 15:05:46 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 15:05:57 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 15:05:57 | INFO | Fetched 4,814 matched empids fetched for OQAD
2026-06-17 15:05:57 | INFO | ================================================================================
2026-06-17 15:05:57 | INFO | Processing Table: additional_visibility
2026-06-17 15:05:57 | INFO | Fetching data for 863 MIDs
2026-06-17 15:05:58 | ERROR | Failed processing table additional_visibility
Traceback (most recent call last):
File "D:\data_move\main2.py", line 228, in main
df = fetch_data(
engine=sql_engine,
...<3 lines>...
run_date=run_date,
)
File "D:\data_move\src\fact.py", line 34, in fetch_data
df = pl.read_database(
query=sql,
connection=engine
)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 287, in read_database
).to_polars(
~~~~~~~~~^
batch_size=batch_size,
^^^^^^^^^^^^^^^^^^^^^^
...<2 lines>...
infer_schema_length=infer_schema_length,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 588, in to_polars
frame = frame_init(
batch_size=batch_size,
...<2 lines>...
infer_schema_length=infer_schema_length,
)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 285, in _from_rows
(d[0], d[1:]) for d in self.result.cursor.description
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AttributeError: 'NoneType' object has no attribute 'description'
2026-06-17 15:07:15 | INFO | ================================================================================
2026-06-17 15:07:15 | INFO | Hello from data-move Python data pipeline!
2026-06-17 15:07:15 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 15:07:15 | INFO | Connecting to databases...
2026-06-17 15:07:16 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001BD510978B0>
2026-06-17 15:07:17 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001BD52518910>
2026-06-17 15:07:18 | INFO | Database connections established
2026-06-17 15:07:18 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 15:07:18 | INFO | Found 863 MIDs
2026-06-17 15:07:18 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 15:07:31 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 15:07:31 | INFO | Fetched 4,814 matched empids fetched for OQAD
2026-06-17 15:07:31 | INFO | ================================================================================
2026-06-17 15:07:31 | INFO | Processing Table: additional_visibility
2026-06-17 15:07:31 | INFO | Fetching data for 863 MIDs
2026-06-17 15:07:32 | INFO | Fetched 1,773 rows from SQL Server
2026-06-17 15:07:32 | INFO | Fetched 1773 rows
2026-06-17 15:07:32 | INFO | Deleting from additional_visibility
2026-06-17 15:07:32 | ERROR | Failed processing table additional_visibility
Traceback (most recent call last):
File "D:\data_move\main2.py", line 299, in main
load_to_clickhouse(
~~~~~~~~~~~~~~~~~~^
client=client,
^^^^^^^^^^^^^^
table_name=table_name,
^^^^^^^^^^^^^^^^^^^^^^
df=df,
^^^^^^
)
^
File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse
client.insert_arrow(
~~~~~~~~~~~~~~~~~~~^
table=table_name,
^^^^^^^^^^^^^^^^^
arrow_table=arrow_table,
^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow
return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings)
~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert
response = self._raw_request(insert_block, params, headers, server_wait=False)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request
self._error_handler(response)
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler
raise err_type(err_str, code=code, name=name) from None
clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column project_id in table kelloggs_1.additional_visibility (9e03e625-028a-47db-8f60-e8ab93e907ea). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123)
2026-06-17 15:08:55 | INFO | ================================================================================
2026-06-17 15:08:55 | INFO | Hello from data-move Python data pipeline!
2026-06-17 15:08:55 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 15:08:55 | INFO | Connecting to databases...
2026-06-17 15:08:56 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001ED5A4C78B0>
2026-06-17 15:08:58 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001ED5B948910>
2026-06-17 15:08:58 | INFO | Database connections established
2026-06-17 15:08:58 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 15:08:59 | INFO | Found 863 MIDs
2026-06-17 15:08:59 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 15:09:12 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 15:09:12 | INFO | Fetched 4,814 matched empids fetched for OQAD
2026-06-17 15:09:12 | INFO | ================================================================================
2026-06-17 15:09:12 | INFO | Processing Table: additional_visibility
2026-06-17 15:09:12 | INFO | Fetching data for 863 MIDs
2026-06-17 15:09:14 | INFO | Fetched 1,773 rows from SQL Server
2026-06-17 15:09:14 | INFO | Fetched 1773 rows
2026-06-17 15:09:14 | INFO | Creating table additional_visibility
2026-06-17 15:09:14 | INFO | Table ready: additional_visibility
2026-06-17 15:09:15 | INFO | additional_visibility: inserted 1,773 rows into ClickHouse
2026-06-17 15:09:15 | INFO | additional_visibility loaded successfully (1773 rows)
2026-06-17 15:09:15 | INFO | ================================================================================
2026-06-17 15:09:15 | INFO | Pipeline Completed Successfully
2026-06-17 15:09:15 | INFO | ================================================================================
main2.py
+31 -16
View File
@@ -45,7 +45,7 @@ from mids import (
from masters.dimensions import * from masters.dimensions import *
from masters.bridge import * from masters.bridge import *
from kpi.facts import * from src.fact import *
# ========================================================== # ==========================================================
@@ -94,15 +94,15 @@ def get_empids_clickhouse_OQAD(
log.warning(f"Table '{table_name}' does not exist.") log.warning(f"Table '{table_name}' does not exist.")
return pl.DataFrame( return pl.DataFrame(
schema={ schema={
"EmpId": pl.Int64, "empid": pl.Int64,
"VisitDate": pl.Date, "visitdate": pl.Date,
} }
) )
query = f""" query = f"""
SELECT DISTINCT SELECT DISTINCT
employee_id AS EmpId, employee_id AS empid,
toDate(visit_date) AS VisitDate visit_date AS visitdate
FROM {table_name} FROM {table_name}
""" """
@@ -166,20 +166,24 @@ def main():
sql_engine, sql_engine,
run_date, run_date,
) )
qf=fetch_quiz_empids() qf=fetch_quiz_empids(sql_engine,run_date)
db_df = get_empids_clickhouse_OQAD(client) db_df = get_empids_clickhouse_OQAD(client)
matched = qf.join( matched = qf.join(
db_df, db_df,
on=["EmpId", "VisitDate"], on=["empid", "visitdate"],
how="inner", how="inner",
) )
empids=matched["empid"].to_list()
log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ")
# ------------------------------------------------------ # ------------------------------------------------------
# Config # Config
# ------------------------------------------------------ # ------------------------------------------------------
with open( with open(
"t.yml", "y.yml",
"r", "r",
) as file: ) as file:
@@ -207,16 +211,27 @@ def main():
# ------------------------------------------ # ------------------------------------------
# Fetch Data # Fetch Data
# ------------------------------------------ # ------------------------------------------
if table_name =="OQaD":
df=fetch_OQaD( engine=sql_engine,
table_name=table_name,
table_type=table_type,
empids=empids,
run_date=run_date
)
fn_name = f"fetch_{table_name}"
else:
df = get_dataframe( df = fetch_data(
fn_name=fn_name, engine=sql_engine,
fetch_by=fetch_by, table_name=table_name,
sql_engine=sql_engine, table_type=table_type,
mids=mids, mids=mids,
run_date=run_date, run_date=run_date,
) )
if df.is_empty(): if df.is_empty():
mids.py
+1 -1
View File
@@ -94,7 +94,7 @@ def fetch_quiz_empids(engine: Engine, run_date : date) -> pl.DataFrame:
QUIZ AS QUIZ AS
( (
SELECT Distinct E.EmpId as empid SELECT Distinct E.EmpId as empid
, CONVERT(VARCHAR,DQ.VisitDate) AS VisitDate , CONVERT(date,DQ.VisitDate) AS visitdate
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ INNER JOIN FROM OneApp_KelloggsMT.dbo.T_OQAD DQ INNER JOIN
OneApp_KelloggsMT.dbo.vw_Employee_Detail E ON DQ.EmpId = E.EmpId inner join OneApp_KelloggsMT.dbo.vw_Employee_Detail E ON DQ.EmpId = E.EmpId inner join
OneApp_KelloggsMT.dbo.Master_OQAD_Question QU on DQ.QuestionId= qu.QuestionId inner join OneApp_KelloggsMT.dbo.Master_OQAD_Question QU on DQ.QuestionId= qu.QuestionId inner join
src/fact.py
+7 -3
View File
@@ -19,7 +19,7 @@ def fetch_data(
mid_list = ",".join(str(mid) for mid in mids) mid_list = ",".join(str(mid) for mid in mids)
sql_file = Path("sql") / f"{table_type}" / f"{table_name}.sql" sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f: with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read() sql_template = f.read()
@@ -53,7 +53,11 @@ def fetch_OQaD(
empid_list = ",".join(str(empid) for empid in empids) empid_list = ",".join(str(empid) for empid in empids)
sql_file = Path("sql") / table_type / f"{table_name}.sql"
sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql"
log.info(f"Exists: {sql_file.exists()}")
log.info(f"Path: {sql_file.resolve()}")
with open(sql_file, "r", encoding="utf-8") as f: with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read() sql_template = f.read()
@@ -72,6 +76,6 @@ def fetch_OQaD(
connection=engine, connection=engine,
) )
log.info("Fetched %s rows", len(df)) log.info("fn name is fetch_OQad ------Fetched %s rows", len(df))
return df return df
src/sql/fact/OQaD.sql
+81
View File
@@ -0,0 +1,81 @@
WITH MID_TABLE_COV1 AS
(
SELECT EmpId, VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE CreateDate >= {run_date}
AND CreateDate < DATEADD(DAY,1,'{run_date}')
UNION ALL
SELECT EmpId, VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE UpdateDate >= {run_date}
AND UpdateDate < DATEADD(DAY,1, '{run_date}')
),
QUIZ AS
(
SELECT DISTINCT
E.EmpId,
E.EmpName,
E.SupervisorId,
E.SupervisorName,
E.DesignationName,
E.CityName,
E.StateName,
E.RegionName,
CAST(DQ.VisitDate AS DATE) AS VisitDate,
DQ.QuestionId,
DQ.AnswerId,
QC.QuestionCategoryId,
QC.QuestionCategory
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E
ON DQ.EmpId = E.EmpId
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU
ON DQ.QuestionId = QU.QuestionId
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC
ON QU.QuestionCategoryId = QC.QuestionCategoryId
WHERE E.EmpName NOT LIKE '%TEST%'
AND E.RightId = 6
AND (
E.ResignDate IS NULL
OR CAST(E.ResignDate AS DATE) >= '{run_date}'
)
AND EXISTS
(
SELECT 1
FROM MID_TABLE_COV1 A
WHERE A.EmpId = DQ.EmpId
AND CAST(A.VisitDate AS DATE) = CAST(DQ.VisitDate AS DATE)
)
)
SELECT
40148 AS project_id,
Q.EmpId AS employee_id,
0 AS process_id,
Q.VisitDate AS visit_date,
Q.QuestionCategoryId AS question_category_id,
Q.QuestionCategory AS question_category,
QM.QuestionId AS question_id,
QM.Question AS question,
ISNULL(QA.AnswerId,0) AS answer_id,
ISNULL(QA.Answer,'') AS answer,
CASE
WHEN QA.AnswerId IS NULL THEN 'Not Answer'
WHEN QA.RightAnswer = 1 THEN 'Y'
WHEN QA.RightAnswer IS NULL THEN 'Not Answer'
ELSE 'N'
END AS correct_answer,
GETDATE() AS update_date,
'SP-Pius' AS update_by
FROM QUIZ Q
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM
ON Q.QuestionId = QM.QuestionId
LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA
ON Q.AnswerId = QA.AnswerId
where Q.EmpId not in ({empid_list})
src/sql/fact/Survey.sql
+68
View File
@@ -0,0 +1,68 @@
WITH QUIZ AS
(
SELECT
SC.MID,
SC.StoreId,
EM.EmpId,
EM.EmpName,
EM.SupervisorId,
EM.SupervisorName,
EM.DesignationName,
CAST(SC.VisitDate AS DATE) AS VisitDate,
DQ.QuestionId,
QU.Question,
DQ.Answer,
QC.CategoryId,
QC.Category,
MSS.SubCategoryId,
MSS.SubCategory,
DQ.ImageUrl
FROM OneApp_KelloggsMT.dbo.T_SURVEY DQ
INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC
ON DQ.MID = SC.MID
INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveyQuestion QU
ON DQ.QuestionId = QU.QuestionId
INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveySubCategory MSS
ON QU.SubCategoryId = MSS.SubCategoryId
INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveyCategory QC
ON MSS.CategoryId = QC.CategoryId
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM
ON SC.EmpId = EM.EmpId
INNER JOIN OneApp_KelloggsMT.dbo.Master_Survey MS
ON DQ.SurveyId = MS.SurveyId
AND QU.SurveyId = MS.SurveyId
WHERE EM.EmpName NOT LIKE 'test%'
AND SC.MID IN ({mid_list})
)
SELECT
40148 AS Project_Id,
Q.MID AS Mid,
Q.SupervisorId AS SuperVisorId,
Q.EmpId AS EmpID,
SM.StoreId,
Q.VisitDate,
SM.StoreTypeId,
SM.ChainId,
Q.CategoryId,
Q.Category,
Q.SubCategoryId,
Q.SubCategory,
Q.QuestionId,
Q.Question,
Q.Answer,
GETDATE() AS CreateDate,
'SP-Pius' AS CreateBy
FROM QUIZ Q
INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM
ON Q.StoreId = SM.StoreId
src/sql/fact/additional_visibility.sql
+31
View File
@@ -0,0 +1,31 @@
WITH
additional_visibility (project_id,Mid,
emp_id,store_id,storetype_id,channel_id,chain_id,camera_allowed,visit_date,is_present,brand_id,
display_id,Remarks,image_url,created_date,created_by)
as (
SELECT '40148' AS ProjectId,ts.mid, em.EmpId as EmpId
, sm.StoreId,sm.storetypeid,sm.channelid,sm.chainid,case when sm.cameraallow=1 then 'Y' ELSE 'N' END AS
cameraallow , convert(varchar,sc.VisitDate,101)VisitDate,
case when ts.Present=0 then 'N' else 'Y' end Present,
isnull(MB.BrandId,'')[BrandId],
isnull(MD.DisplayId,'')[DisplayId],ISNULL(TS.REMARK,'')[Remark],
case when isnull(ts.ImageUrl,'')='' then '' else 'https://kimt1.parinaam.in/Upload/VisibilityImages/'+ts.ImageUrl
end as AdditionalImageUrl ,GETDATE(),'Pius'
from
OneApp_KelloggsMT.dbo.T_AdditionalVisibility ts Inner join
OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId
left join
OneApp_KelloggsMT.dbo.Master_Brand MB ON TS.BrandId=MB.BrandId
LEFT JOIN
OneApp_KelloggsMT.dbo.Master_Display MD ON TS.DisplayId=MD.DisplayId
Where Em.EmpName not like 'test%' and ts.Present=1 and sc.mid in ({mid_list})
)
select * from additional_visibility;
y.yml
+21
View File
@@ -0,0 +1,21 @@
tables:
# - name: SOS_OneApp
# type: FACT
# operation: INSERT
# fetch_by: mids
# - name: OQaD
# type: FACT
# operation: INSERT
# fetch_by: run_date
- name: additional_visibility
type: FACT
operation: INSERT
fetch_by: mids
# - name: Survey
# type: FACT
# operation: INSERT
# fetch_by: mids