diff --git a/logs/etl_20260616.log b/logs/etl_20260616.log new file mode 100644 index 0000000..9c7b84d --- /dev/null +++ b/logs/etl_20260616.log @@ -0,0 +1,71 @@ +2026-06-16 17:56:17 | INFO | ================================================================================ +2026-06-16 17:56:17 | INFO | Hello from data-move Python data pipeline! +2026-06-16 17:56:17 | INFO | Pipeline Run Date: 2026-06-15 +2026-06-16 17:56:17 | INFO | Connecting to databases... +2026-06-16 17:56:19 | INFO | +2026-06-16 17:56:23 | INFO | +2026-06-16 17:56:24 | INFO | Database connections established +2026-06-16 17:56:24 | INFO | Collecting MIDs for: 2026-06-15 +2026-06-16 17:56:24 | INFO | Found 849 MIDs +2026-06-16 17:57:33 | INFO | ================================================================================ +2026-06-16 17:57:33 | INFO | Hello from data-move Python data pipeline! +2026-06-16 17:57:33 | INFO | Pipeline Run Date: 2026-06-15 +2026-06-16 17:57:33 | INFO | Connecting to databases... +2026-06-16 17:57:34 | INFO | +2026-06-16 17:57:36 | INFO | +2026-06-16 17:57:36 | INFO | Database connections established +2026-06-16 17:57:36 | INFO | Collecting MIDs for: 2026-06-15 +2026-06-16 17:57:37 | INFO | Found 849 MIDs +2026-06-16 17:58:52 | INFO | ================================================================================ +2026-06-16 17:58:52 | INFO | Hello from data-move Python data pipeline! +2026-06-16 17:58:52 | INFO | Pipeline Run Date: 2026-06-15 +2026-06-16 17:58:52 | INFO | Connecting to databases... +2026-06-16 17:58:53 | INFO | +2026-06-16 17:58:54 | INFO | +2026-06-16 17:58:55 | INFO | Database connections established +2026-06-16 17:58:55 | INFO | Collecting MIDs for: 2026-06-15 +2026-06-16 17:58:55 | INFO | Found 849 MIDs +2026-06-16 18:00:31 | INFO | ================================================================================ +2026-06-16 18:00:31 | INFO | Hello from data-move Python data pipeline! +2026-06-16 18:00:31 | INFO | Pipeline Run Date: 2026-06-15 +2026-06-16 18:00:31 | INFO | Connecting to databases... +2026-06-16 18:00:38 | INFO | +2026-06-16 18:00:40 | INFO | +2026-06-16 18:00:40 | INFO | Database connections established +2026-06-16 18:00:40 | INFO | Collecting MIDs for: 2026-06-15 +2026-06-16 18:00:42 | INFO | Found 849 MIDs +2026-06-16 18:00:44 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-16 18:02:58 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server +2026-06-16 18:07:47 | INFO | ================================================================================ +2026-06-16 18:07:47 | INFO | Hello from data-move Python data pipeline! +2026-06-16 18:07:47 | INFO | Pipeline Run Date: 2026-06-15 +2026-06-16 18:07:47 | INFO | Connecting to databases... +2026-06-16 18:07:48 | INFO | +2026-06-16 18:07:50 | INFO | +2026-06-16 18:07:50 | INFO | Database connections established +2026-06-16 18:07:50 | INFO | Collecting MIDs for: 2026-06-15 +2026-06-16 18:07:50 | INFO | Found 849 MIDs +2026-06-16 18:07:51 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-16 18:08:09 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server +2026-06-16 18:08:58 | INFO | ================================================================================ +2026-06-16 18:08:58 | INFO | Hello from data-move Python data pipeline! +2026-06-16 18:08:58 | INFO | Pipeline Run Date: 2026-06-15 +2026-06-16 18:08:58 | INFO | Connecting to databases... +2026-06-16 18:08:59 | INFO | +2026-06-16 18:09:01 | INFO | +2026-06-16 18:09:01 | INFO | Database connections established +2026-06-16 18:09:01 | INFO | Collecting MIDs for: 2026-06-15 +2026-06-16 18:09:02 | INFO | Found 849 MIDs +2026-06-16 18:09:02 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-16 18:09:19 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server +2026-06-16 18:11:21 | INFO | ================================================================================ +2026-06-16 18:11:21 | INFO | Hello from data-move Python data pipeline! +2026-06-16 18:11:21 | INFO | Pipeline Run Date: 2026-06-15 +2026-06-16 18:11:21 | INFO | Connecting to databases... +2026-06-16 18:11:23 | INFO | +2026-06-16 18:11:24 | INFO | +2026-06-16 18:11:24 | INFO | Database connections established +2026-06-16 18:11:24 | INFO | Collecting MIDs for: 2026-06-15 +2026-06-16 18:11:25 | INFO | Found 849 MIDs +2026-06-16 18:11:25 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-16 18:13:05 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server diff --git a/logs/etl_20260617.log b/logs/etl_20260617.log new file mode 100644 index 0000000..0d18acb --- /dev/null +++ b/logs/etl_20260617.log @@ -0,0 +1,688 @@ +2026-06-17 10:17:16 | INFO | ================================================================================ +2026-06-17 10:17:16 | INFO | Hello from data-move Python data pipeline! +2026-06-17 10:17:16 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 10:17:16 | INFO | Connecting to databases... +2026-06-17 10:17:18 | INFO | +2026-06-17 10:17:21 | INFO | +2026-06-17 10:17:22 | INFO | Database connections established +2026-06-17 10:17:22 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 10:17:22 | INFO | Found 863 MIDs +2026-06-17 10:17:23 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 10:17:44 | INFO | Fetched 290,984 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 10:43:05 | INFO | ================================================================================ +2026-06-17 10:43:05 | INFO | Hello from data-move Python data pipeline! +2026-06-17 10:43:05 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 10:43:05 | INFO | Connecting to databases... +2026-06-17 10:43:07 | INFO | +2026-06-17 10:43:08 | INFO | +2026-06-17 10:43:09 | INFO | Database connections established +2026-06-17 10:43:09 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 10:43:10 | INFO | Found 863 MIDs +2026-06-17 10:43:11 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 10:43:55 | INFO | Fetched 291,001 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 10:45:51 | INFO | ================================================================================ +2026-06-17 10:45:51 | INFO | Hello from data-move Python data pipeline! +2026-06-17 10:45:51 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 10:45:51 | INFO | Connecting to databases... +2026-06-17 10:45:52 | INFO | +2026-06-17 10:45:53 | INFO | +2026-06-17 10:45:54 | INFO | Database connections established +2026-06-17 10:45:54 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 10:45:55 | INFO | Found 863 MIDs +2026-06-17 10:45:55 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 10:46:11 | INFO | Fetched 291,005 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 10:46:11 | INFO | ================================================================================ +2026-06-17 10:46:11 | INFO | Processing Table: OQaD +2026-06-17 10:46:11 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "D:\data_move\main2.py", line 212, in main + df=fetch_OQaD( engine=sql_engine, + table_name=table_name, + ...<2 lines>... + run_date=run_date + ) + File "D:\data_move\src\fact.py", line 58, in fetch_OQaD + with open(sql_file, "r", encoding="utf-8") as f: + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql' +2026-06-17 10:48:44 | INFO | ================================================================================ +2026-06-17 10:48:44 | INFO | Hello from data-move Python data pipeline! +2026-06-17 10:48:44 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 10:48:44 | INFO | Connecting to databases... +2026-06-17 10:48:45 | INFO | +2026-06-17 10:48:46 | INFO | +2026-06-17 10:48:47 | INFO | Database connections established +2026-06-17 10:48:47 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 10:48:48 | INFO | Found 863 MIDs +2026-06-17 10:48:48 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 10:49:00 | INFO | Fetched 291,005 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 10:49:01 | INFO | Fetched 893 matched empids fetched for OQAD +2026-06-17 10:49:01 | INFO | ================================================================================ +2026-06-17 10:49:01 | INFO | Processing Table: OQaD +2026-06-17 10:49:01 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "D:\data_move\main2.py", line 215, in main + df=fetch_OQaD( engine=sql_engine, + table_name=table_name, + ...<2 lines>... + run_date=run_date + ) + File "D:\data_move\src\fact.py", line 58, in fetch_OQaD + with open(sql_file, "r", encoding="utf-8") as f: + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql' +2026-06-17 10:53:50 | INFO | ================================================================================ +2026-06-17 10:53:50 | INFO | Hello from data-move Python data pipeline! +2026-06-17 10:53:50 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 10:53:50 | INFO | Connecting to databases... +2026-06-17 10:53:51 | INFO | +2026-06-17 10:53:52 | INFO | +2026-06-17 10:53:53 | INFO | Database connections established +2026-06-17 10:53:53 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 10:53:54 | INFO | Found 863 MIDs +2026-06-17 10:53:54 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 10:54:08 | INFO | Fetched 291,006 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 10:54:09 | INFO | Fetched 893 matched empids fetched for OQAD +2026-06-17 10:54:09 | INFO | ================================================================================ +2026-06-17 10:54:09 | INFO | Processing Table: OQaD +2026-06-17 10:54:09 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "D:\data_move\main2.py", line 217, in main + df=fetch_OQaD( engine=sql_engine, + table_name=table_name, + ...<2 lines>... + run_date=run_date + ) + File "D:\data_move\src\fact.py", line 58, in fetch_OQaD + with open(sql_file, "r", encoding="utf-8") as f: + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql' +2026-06-17 10:57:28 | INFO | ================================================================================ +2026-06-17 10:57:28 | INFO | Hello from data-move Python data pipeline! +2026-06-17 10:57:28 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 10:57:28 | INFO | Connecting to databases... +2026-06-17 10:57:30 | INFO | +2026-06-17 10:57:31 | INFO | +2026-06-17 10:57:32 | INFO | Database connections established +2026-06-17 10:57:32 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 10:57:32 | INFO | Found 863 MIDs +2026-06-17 10:57:33 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 10:57:45 | INFO | Fetched 291,010 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 10:57:46 | INFO | Fetched 893 matched empids fetched for OQAD +2026-06-17 10:57:46 | INFO | ================================================================================ +2026-06-17 10:57:46 | INFO | Processing Table: OQaD +2026-06-17 10:57:46 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "D:\data_move\main2.py", line 217, in main + df=fetch_OQaD( engine=sql_engine, + table_name=table_name, + ...<2 lines>... + run_date=run_date + ) + File "D:\data_move\src\fact.py", line 58, in fetch_OQaD + with open(sql_file, "r", encoding="utf-8") as f: + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql' +2026-06-17 10:59:57 | INFO | ================================================================================ +2026-06-17 10:59:57 | INFO | Hello from data-move Python data pipeline! +2026-06-17 10:59:57 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 10:59:57 | INFO | Connecting to databases... +2026-06-17 10:59:58 | INFO | +2026-06-17 11:00:00 | INFO | +2026-06-17 11:00:00 | INFO | Database connections established +2026-06-17 11:00:00 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 11:00:01 | INFO | Found 863 MIDs +2026-06-17 11:00:02 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 11:00:24 | INFO | Fetched 291,011 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 11:00:25 | INFO | Fetched 893 matched empids fetched for OQAD +2026-06-17 11:00:25 | INFO | ================================================================================ +2026-06-17 11:00:25 | INFO | Processing Table: OQaD +2026-06-17 11:00:25 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "D:\data_move\main2.py", line 217, in main + df=fetch_OQaD( engine=sql_engine, + table_name=table_name, + ...<2 lines>... + run_date=run_date + ) + File "D:\data_move\src\fact.py", line 58, in fetch_OQaD + with open(sql_file, "r", encoding="utf-8") as f: + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'sql\\fact\\OQaD.sql' +2026-06-17 12:09:34 | INFO | ================================================================================ +2026-06-17 12:09:34 | INFO | Hello from data-move Python data pipeline! +2026-06-17 12:09:34 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 12:09:34 | INFO | Connecting to databases... +2026-06-17 12:09:37 | INFO | +2026-06-17 12:09:39 | INFO | +2026-06-17 12:09:39 | INFO | Database connections established +2026-06-17 12:09:39 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 12:09:41 | INFO | Found 863 MIDs +2026-06-17 12:09:43 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 12:10:48 | INFO | Fetched 291,093 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 12:10:48 | INFO | Fetched 893 matched empids fetched for OQAD +2026-06-17 12:10:48 | INFO | ================================================================================ +2026-06-17 12:10:48 | INFO | Processing Table: OQaD +2026-06-17 12:10:48 | INFO | Exists: False +2026-06-17 12:10:48 | INFO | Path: D:\data_move\sql\fact\OQaD.sql +2026-06-17 12:10:48 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "D:\data_move\main2.py", line 217, in main + df=fetch_OQaD( engine=sql_engine, + table_name=table_name, + ...<2 lines>... + run_date=run_date + ) + File "D:\data_move\src\fact.py", line 62, in fetch_OQaD + with open(sql_file, "r", encoding="utf-8") as f: + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: 'sql\\fact\\OQaD.sql' +2026-06-17 12:11:47 | INFO | ================================================================================ +2026-06-17 12:11:47 | INFO | Hello from data-move Python data pipeline! +2026-06-17 12:11:47 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 12:11:47 | INFO | Connecting to databases... +2026-06-17 12:11:48 | INFO | +2026-06-17 12:11:50 | INFO | +2026-06-17 12:11:51 | INFO | Database connections established +2026-06-17 12:11:51 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 12:11:52 | INFO | Found 863 MIDs +2026-06-17 12:11:52 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 12:12:14 | INFO | Fetched 291,097 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 12:12:14 | INFO | Fetched 893 matched empids fetched for OQAD +2026-06-17 12:12:14 | INFO | ================================================================================ +2026-06-17 12:12:14 | INFO | Processing Table: OQaD +2026-06-17 12:12:14 | INFO | Exists: True +2026-06-17 12:12:14 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql +2026-06-17 12:12:14 | INFO | Fetching data for 893 EMPIDs +2026-06-17 12:12:14 | INFO | Fetching OQaD data for run_date=2026-06-16 +2026-06-17 12:12:14 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near 'QUIZ'. (102) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'where'. (156)") + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "D:\data_move\main2.py", line 217, in main + df=fetch_OQaD( engine=sql_engine, + table_name=table_name, + ...<2 lines>... + run_date=run_date + ) + File "D:\data_move\src\fact.py", line 74, in fetch_OQaD + df = pl.read_database( + query=sql, + connection=engine, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database + return cx.execute( + ~~~~~~~~~~^ + query=query, + ^^^^^^^^^^^^ + options=execute_options, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ).to_polars( + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute + result = cursor_execute(query, **options) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute + return meth( + self, + distilled_parameters, + execution_options or NO_OPTIONS, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection + return connection._execute_clauseelement( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + self, distilled_params, execution_options + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement + ret = self._execute_context( + dialect, + ...<8 lines>... + cache_hit=cache_hit, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context + return self._exec_single_context( + ~~~~~~~~~~~~~~~~~~~~~~~~~^ + dialect, context, statement, parameters + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context + self._handle_dbapi_exception( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + e, str_statement, effective_parameters, cursor, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near 'QUIZ'. (102) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'where'. (156)") +[SQL: WITH MID_TABLE_COV1 AS +( + SELECT EmpId, VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CreateDate >= 2026-06-16 + AND CreateDate < DATEADD(DAY,1,'2026-06-16') + + UNION ALL + + SELECT EmpId, VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE UpdateDate >= 2026-06-16 + AND UpdateDate < DATEADD(DAY,1, '2026-06-16') +), + +OQaD + + +QUIZ AS +( + SELECT DISTINCT + E.EmpId, + E.EmpName, + E.SupervisorId, + E.SupervisorName, + E.DesignationName, + E.CityName, + E.StateName, + E.RegionName, + CAST(DQ.VisitDate AS DATE) AS VisitDate, + DQ.QuestionId, + DQ.AnswerId, + QC.QuestionCategoryId, + QC.QuestionCategory + FROM OneApp_KelloggsMT.dbo.T_OQAD DQ + INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E + ON DQ.EmpId = E.EmpId + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU + ON DQ.QuestionId = QU.QuestionId + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC + ON QU.QuestionCategoryId = QC.QuestionCategoryId + WHERE E.EmpName NOT LIKE '%TEST%' + AND E.RightId = 6 + AND ( + E.ResignDate IS NULL + OR CAST(E.ResignDate AS DATE) >= '2026-06-16' + ) + AND EXISTS + ( + SELECT 1 + FROM MID_TABLE_COV1 A + WHERE A.EmpId = DQ.EmpId + AND CAST(A.VisitDate AS DATE) = CAST(DQ.VisitDate AS DATE) + ) +) + +SELECT + 40148 AS project_id, + Q.EmpId AS employee_id, + 0 AS process_id, + Q.VisitDate AS visit_date, + Q.QuestionCategoryId AS question_category_id, + Q.QuestionCategory AS question_category, + QM.QuestionId AS question_id, + QM.Question AS question, + ISNULL(QA.AnswerId,0) AS answer_id, + ISNULL(QA.Answer,'') AS answer, + CASE + WHEN QA.AnswerId IS NULL THEN 'Not Answer' + WHEN QA.RightAnswer = 1 THEN 'Y' + WHEN QA.RightAnswer IS NULL THEN 'Not Answer' + ELSE 'N' + END AS correct_answer, + GETDATE() AS update_date, + 'SP-Pius' AS update_by +FROM QUIZ Q +INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM + ON Q.QuestionId = QM.QuestionId +LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA + ON Q.AnswerId = QA.AnswerId; +where Q.EmpId not in (3167,4724,4019,4902,4375,3352,3081,5244,3181,4810,5166,4895,4817,5173,5237,3608,4646,4126,3243,3948,3950,3599,3777,5235,5346,3606,3079,5175,5242,5064,4366,5071,5339,4812,4722,3169,5270,3326,3689,4062,3131,5054,5178,3371,4829,5185,4698,4836,3760,4665,5218,4451,4280,3269,3098,5327,4978,5261,3335,5090,5320,4214,4919,4570,3150,4221,3328,3112,3290,4950,4430,5104,3383,5213,4864,5199,5206,4439,3207,5033,4684,3979,5211,4774,3117,5102,5280,3309,4931,4753,5123,3155,5085,4914,5092,4387,5325,5154,4100,4805,3402,5187,3580,3409,3060,3962,4696,4831,3715,4710,4242,5282,5121,4933,5289,5282,3444,5104,3095,4933,4142,3437,3530,3088,3715,4840,5002,4831,5187,3361,3962,5113,3086,5106,5256,5085,4914,5092,3470,4387,5277,3309,5239,4712,4945,5246,5123,4774,3952,3603,4999,3727,3378,3207,5341,4992,5199,5334,4672,5206,3342,3520,3454,3171,4916,3105,4396,3112,3290,5265,3119,5227,3143,3150,5056,4529,3328,5313,5220,4693,5320,4615,3755,4444,3406,4978,4451,3746,4280,3399,5322,4617,4795,5218,3330,3159,4698,5144,5054,3473,3131,4726,4688,4743,3689,3812,3107,3169,5332,3347,5023,5022,4812,3380,3067,3950,4954,5132,3081,5296,5244,3140,4019,3492,4375,5210,3126,5289,3482,4847,5296,4954,4436,3380,5127,5305,3306,5291,5023,5120,4531,4346,5229,3107,3812,4353,3145,3152,3330,5322,4439,5144,4795,3363,4054,4232,3399,4239,3755,5279,4930,5213,3406,4864,5286,5220,4693,5101,5227,3985,3297,3636,4529,3420,5265,4916,3591,3335,4460,5343,4638,3342,5327,3520,3171,4467,3700,5336,4987,3515,4992,3760,5267,4562,3247,5082,4555,4733,5061,5239,3090,4712,5246,4904,5270,3059,4866,5277,4750,5106,3361,4968,5175,5310,3496,3608,4840,5248,4543,3615,4142,3437,3088,3444,3078,4902,3095,3567,4543,4450,3140,5319,4614,3496,4970,4443,5132,3916,3149,5310,3327,5312,3156,4968,4894,3394,5065,4538,4688,4866,3251,3090,3268,4671,4849,4904,4726,5082,4555,5191,4733,4562,3515,5300,4951,5115,3173,5293,4944,4460,4987,4638,4467,3591,3064,4526,3821,3232,3420,5056,5046,5008,3985,3636,5350,3109,5272,5101,5279,3358,4396,5172,5110,4054,4232,5281,3363,4999,3638,3603,3104,5027,5205,4856,4151,3083,4346,3764,3819,3415,4353,3470,3299,5167,3346,5113,3306,5331,5291,5160,5120,5127,4913,5091,4564,5262,5196,4847,5077,4666,4139,3612,3978,3078,5340,4139,3612,4991,5248,3170,5155,5333,4628,5077,4217,5084,4032,4913,4564,5257,5324,4381,5331,5250,5160,4134,4312,4141,3059,3764,3415,4305,5027,3220,3104,5321,4972,5307,5267,5136,5314,4787,3315,3322,4540,3835,3700,3486,4547,5172,5001,5008,3427,3379,5046,3372,3821,4526,3650,5124,5302,5286,3130,5115,4937,3132,5293,5117,3118,4239,5300,4951,4343,3897,5219,5191,4671,4849,5212,5229,5003,5181,4531,5065,5345,3538,4538,4189,4229,3531,5276,5305,4578,3106,3149,5312,5269,3113,4436,3099,4443,4970,3916,3567,3975,4153,4844,5210,5200,4844,3533,4666,5022,3184,5200,4402,4153,3704,3975,4922,3099,5262,4751,5091,3106,5269,5093,5276,4578,3346,5202,4675,4229,3531,3182,5195,5167,3203,5003,5205,5150,5328,3220,4979,4856,5157,4452,5335,3514,5219,3638,4343,4383,3289,4390,3118,3296,5281,5110,3438,3267,5074,5117,5252,3780,3431,5005,3372,3379,5176,3232,5316,4611,4967,3068,5309,3139,3495,3673,4540,3835,4891,3486,3488,3315,3322,5307,5136,3070,4877,4787,5314,5055,3939,3234,4072,5181,4305,3251,3374,5119,4134,3080,5297,4065,4894,3127,5250,4763,4941,5257,3298,4381,3327,3156,4687,4338,5221,4450,5155,3400,5333,5207,4628,5162,3742,4098,3742,4943,5121,3129,5207,5114,5292,4687,3122,4338,4696,3464,3113,4333,3298,3127,5031,4941,4763,5209,5119,3388,3566,3538,4065,3744,4072,3374,4753,3234,3939,5102,5088,5266,3070,5055,4877,3096,3979,3132,3488,3139,4841,5019,5302,3495,5124,5309,5012,4611,3369,3509,5176,5145,5323,5005,3153,4656,4371,3431,5254,5252,4547,3091,5074,4198,3782,3433,4383,4651,4829,4302,3953,5349,5321,3514,4972,5178,5150,5328,4979,5342,4452,5157,5304,3203,5195,3490,4312,3141,5126,4141,4675,4366,5057,5235,5064,3243,5093,4217,4922,3407,4751,4810,3170,4283,3756,3704,4974,3533,4991,3184,4974,3492,3143,4098,4283,3400,3756,3407,4710,5221,3067,3580,5057,3409,5196,3060,4805,4100,3402,5325,5297,3490,3141,5154,5126,3155,5304,4931,5109,5287,4117,5280,3473,4651,4302,3953,5033,5211,3782,4684,3433,3084,4891,3746,3383,4371,4022,5254,3673,5313,4207,3153,5316,3509,5145,5323,4430,4919,4570,3369,4221,3105,5090,4563,5012,3269,4841,3098,5019,3454,5261,3438,4672,3267,4836,3096,4665,5259,5088,3727,5266,3378,5185,4390,4397,3371,4062,4945,3528,3326,4722,3388,5256,3566,3744,5202,5242,3086,5071,5209,3777,4333,3606,3079,5002,3948,4126,3599,3464,5344,4817,5173,5351,4646,3122,3352,3530,5337,3181,3359,5166,5292,5114,4943) + -- select d.employee_id from OQaD d where project_id=40148 and d.employee_id=Q.EmpId and CONVERT(date,d.visit_date,101)= + -- CONVERT(date,q.VisitDate,101)) + +] +(Background on this error at: https://sqlalche.me/e/20/f405) +2026-06-17 12:19:35 | INFO | ================================================================================ +2026-06-17 12:19:35 | INFO | Hello from data-move Python data pipeline! +2026-06-17 12:19:35 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 12:19:35 | INFO | Connecting to databases... +2026-06-17 12:19:37 | INFO | +2026-06-17 12:19:38 | INFO | +2026-06-17 12:19:39 | INFO | Database connections established +2026-06-17 12:19:39 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 12:19:40 | INFO | Found 863 MIDs +2026-06-17 12:19:40 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 12:20:14 | INFO | Fetched 291,100 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 12:20:14 | INFO | Fetched 893 matched empids fetched for OQAD +2026-06-17 12:20:14 | INFO | ================================================================================ +2026-06-17 12:20:14 | INFO | Processing Table: OQaD +2026-06-17 12:20:14 | INFO | Exists: True +2026-06-17 12:20:14 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql +2026-06-17 12:20:14 | INFO | Fetching data for 893 EMPIDs +2026-06-17 12:20:14 | INFO | Fetching OQaD data for run_date=2026-06-16 +2026-06-17 12:20:30 | INFO | fn name is fetch_OQad ------Fetched 4998 rows +2026-06-17 12:20:30 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "D:\data_move\main2.py", line 226, in main + df = get_dataframe( + fn_name=fn_name, + ...<3 lines>... + run_date=run_date, + ) + File "D:\data_move\main2.py", line 81, in get_dataframe + return fn(sql_engine, run_date) +TypeError: fetch_OQaD() missing 3 required positional arguments: 'table_type', 'empids', and 'run_date' +2026-06-17 12:38:43 | INFO | ================================================================================ +2026-06-17 12:38:43 | INFO | Hello from data-move Python data pipeline! +2026-06-17 12:38:43 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 12:38:43 | INFO | Connecting to databases... +2026-06-17 12:38:45 | INFO | +2026-06-17 12:38:46 | INFO | +2026-06-17 12:38:46 | INFO | Database connections established +2026-06-17 12:38:46 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 12:38:47 | INFO | Found 863 MIDs +2026-06-17 12:38:47 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 12:38:58 | INFO | Fetched 291,112 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 12:38:58 | INFO | Fetched 893 matched empids fetched for OQAD +2026-06-17 12:38:58 | INFO | ================================================================================ +2026-06-17 12:38:58 | INFO | Processing Table: OQaD +2026-06-17 12:38:58 | INFO | Exists: True +2026-06-17 12:38:58 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql +2026-06-17 12:38:58 | INFO | Fetching data for 893 EMPIDs +2026-06-17 12:38:58 | INFO | Fetching OQaD data for run_date=2026-06-16 +2026-06-17 12:39:05 | INFO | fn name is fetch_OQad ------Fetched 4998 rows +2026-06-17 12:39:05 | INFO | ================================================================================ +2026-06-17 12:39:05 | INFO | Pipeline Completed Successfully +2026-06-17 12:39:05 | INFO | ================================================================================ +2026-06-17 12:39:30 | INFO | ================================================================================ +2026-06-17 12:39:30 | INFO | Hello from data-move Python data pipeline! +2026-06-17 12:39:30 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 12:39:30 | INFO | Connecting to databases... +2026-06-17 12:39:31 | INFO | +2026-06-17 12:39:32 | INFO | +2026-06-17 12:39:33 | INFO | Database connections established +2026-06-17 12:39:33 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 12:39:33 | INFO | Found 863 MIDs +2026-06-17 12:39:33 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 12:39:43 | INFO | Fetched 291,112 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 12:39:43 | INFO | Fetched 893 matched empids fetched for OQAD +2026-06-17 12:39:43 | INFO | ================================================================================ +2026-06-17 12:39:43 | INFO | Processing Table: OQaD +2026-06-17 12:39:43 | INFO | Exists: True +2026-06-17 12:39:43 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql +2026-06-17 12:39:43 | INFO | Fetching data for 893 EMPIDs +2026-06-17 12:39:43 | INFO | Fetching OQaD data for run_date=2026-06-16 +2026-06-17 12:39:46 | INFO | fn name is fetch_OQad ------Fetched 4998 rows +2026-06-17 12:39:46 | INFO | ================================================================================ +2026-06-17 12:39:46 | INFO | Pipeline Completed Successfully +2026-06-17 12:39:46 | INFO | ================================================================================ +2026-06-17 12:44:50 | INFO | ================================================================================ +2026-06-17 12:44:50 | INFO | Hello from data-move Python data pipeline! +2026-06-17 12:44:50 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 12:44:50 | INFO | Connecting to databases... +2026-06-17 12:44:51 | INFO | +2026-06-17 12:44:52 | INFO | +2026-06-17 12:44:53 | INFO | Database connections established +2026-06-17 12:44:53 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 12:44:53 | INFO | Found 863 MIDs +2026-06-17 12:44:53 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 12:45:07 | INFO | Fetched 291,115 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 12:45:08 | INFO | Fetched 893 matched empids fetched for OQAD +2026-06-17 12:45:08 | INFO | ================================================================================ +2026-06-17 12:45:08 | INFO | Processing Table: OQaD +2026-06-17 12:45:08 | INFO | Exists: True +2026-06-17 12:45:08 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql +2026-06-17 12:45:08 | INFO | Fetching data for 893 EMPIDs +2026-06-17 12:45:08 | INFO | Fetching OQaD data for run_date=2026-06-16 +2026-06-17 12:45:10 | INFO | fn name is fetch_OQad ------Fetched 4998 rows +2026-06-17 12:45:10 | INFO | Fetched 4998 rows +2026-06-17 12:45:10 | INFO | Deleting from OQaD +2026-06-17 12:45:11 | INFO | OQaD: inserted 4,998 rows into ClickHouse +2026-06-17 12:45:11 | INFO | OQaD loaded successfully (4998 rows) +2026-06-17 12:45:11 | INFO | ================================================================================ +2026-06-17 12:45:11 | INFO | Pipeline Completed Successfully +2026-06-17 12:45:11 | INFO | ================================================================================ +2026-06-17 13:23:53 | INFO | ================================================================================ +2026-06-17 13:23:53 | INFO | Hello from data-move Python data pipeline! +2026-06-17 13:23:53 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 13:23:53 | INFO | Connecting to databases... +2026-06-17 13:23:54 | INFO | +2026-06-17 13:23:56 | INFO | +2026-06-17 13:23:56 | INFO | Database connections established +2026-06-17 13:23:56 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 13:23:57 | INFO | Found 863 MIDs +2026-06-17 13:23:58 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 13:24:15 | INFO | Fetched 291,128 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 13:24:15 | INFO | Fetched 4,819 matched empids fetched for OQAD +2026-06-17 13:24:15 | INFO | ================================================================================ +2026-06-17 13:24:15 | INFO | Processing Table: OQaD +2026-06-17 13:24:15 | INFO | Exists: True +2026-06-17 13:24:15 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql +2026-06-17 13:24:15 | INFO | Fetching data for 4,819 EMPIDs +2026-06-17 13:24:15 | INFO | Fetching OQaD data for run_date=2026-06-16 +2026-06-17 13:24:22 | INFO | fn name is fetch_OQad ------Fetched 1072 rows +2026-06-17 13:24:22 | INFO | Fetched 1072 rows +2026-06-17 13:24:22 | INFO | Deleting from OQaD +2026-06-17 13:24:23 | INFO | OQaD: inserted 1,072 rows into ClickHouse +2026-06-17 13:24:23 | INFO | OQaD loaded successfully (1072 rows) +2026-06-17 13:24:23 | INFO | ================================================================================ +2026-06-17 13:24:23 | INFO | Processing Table: Survey +2026-06-17 13:24:23 | INFO | Fetching data for 863 MIDs +2026-06-17 13:24:23 | INFO | Fetched 142 rows from SQL Server +2026-06-17 13:24:23 | INFO | Fetched 142 rows +2026-06-17 13:24:23 | INFO | Deleting from Survey +2026-06-17 13:24:24 | ERROR | Failed processing table Survey +Traceback (most recent call last): + File "D:\data_move\main2.py", line 299, in main + load_to_clickhouse( + ~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + table_name=table_name, + ^^^^^^^^^^^^^^^^^^^^^^ + df=df, + ^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse + client.insert_arrow( + ~~~~~~~~~~~~~~~~~~~^ + table=table_name, + ^^^^^^^^^^^^^^^^^ + arrow_table=arrow_table, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow + return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings) + ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert + response = self._raw_request(insert_block, params, headers, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column Project_Id in table kelloggs_1.Survey (d8c44908-2ed7-4d29-bd10-8e70fa84dfba). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123) +2026-06-17 13:27:22 | INFO | ================================================================================ +2026-06-17 13:27:22 | INFO | Hello from data-move Python data pipeline! +2026-06-17 13:27:22 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 13:27:22 | INFO | Connecting to databases... +2026-06-17 13:27:23 | INFO | +2026-06-17 13:27:25 | INFO | +2026-06-17 13:27:25 | INFO | Database connections established +2026-06-17 13:27:25 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 13:27:26 | INFO | Found 863 MIDs +2026-06-17 13:27:26 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 13:27:37 | INFO | Fetched 291,128 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 13:27:38 | INFO | Fetched 4,814 matched empids fetched for OQAD +2026-06-17 13:27:38 | INFO | ================================================================================ +2026-06-17 13:27:38 | INFO | Processing Table: OQaD +2026-06-17 13:27:38 | INFO | Exists: True +2026-06-17 13:27:38 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql +2026-06-17 13:27:38 | INFO | Fetching data for 4,814 EMPIDs +2026-06-17 13:27:38 | INFO | Fetching OQaD data for run_date=2026-06-16 +2026-06-17 13:27:44 | INFO | fn name is fetch_OQad ------Fetched 1072 rows +2026-06-17 13:27:44 | INFO | Fetched 1072 rows +2026-06-17 13:27:44 | INFO | Deleting from OQaD +2026-06-17 13:27:45 | INFO | OQaD: inserted 1,072 rows into ClickHouse +2026-06-17 13:27:45 | INFO | OQaD loaded successfully (1072 rows) +2026-06-17 13:27:45 | INFO | ================================================================================ +2026-06-17 13:27:45 | INFO | Processing Table: Survey +2026-06-17 13:27:45 | INFO | Fetching data for 863 MIDs +2026-06-17 13:27:45 | INFO | Fetched 142 rows from SQL Server +2026-06-17 13:27:45 | INFO | Fetched 142 rows +2026-06-17 13:27:45 | INFO | Creating table Survey +2026-06-17 13:27:45 | INFO | Table ready: Survey +2026-06-17 13:27:45 | INFO | Survey: inserted 142 rows into ClickHouse +2026-06-17 13:27:45 | INFO | Survey loaded successfully (142 rows) +2026-06-17 13:27:45 | INFO | ================================================================================ +2026-06-17 13:27:45 | INFO | Pipeline Completed Successfully +2026-06-17 13:27:45 | INFO | ================================================================================ +2026-06-17 15:05:42 | INFO | ================================================================================ +2026-06-17 15:05:42 | INFO | Hello from data-move Python data pipeline! +2026-06-17 15:05:42 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 15:05:42 | INFO | Connecting to databases... +2026-06-17 15:05:43 | INFO | +2026-06-17 15:05:45 | INFO | +2026-06-17 15:05:45 | INFO | Database connections established +2026-06-17 15:05:45 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 15:05:45 | INFO | Found 863 MIDs +2026-06-17 15:05:46 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 15:05:57 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 15:05:57 | INFO | Fetched 4,814 matched empids fetched for OQAD +2026-06-17 15:05:57 | INFO | ================================================================================ +2026-06-17 15:05:57 | INFO | Processing Table: additional_visibility +2026-06-17 15:05:57 | INFO | Fetching data for 863 MIDs +2026-06-17 15:05:58 | ERROR | Failed processing table additional_visibility +Traceback (most recent call last): + File "D:\data_move\main2.py", line 228, in main + df = fetch_data( + engine=sql_engine, + ...<3 lines>... + run_date=run_date, + ) + File "D:\data_move\src\fact.py", line 34, in fetch_data + df = pl.read_database( + query=sql, + connection=engine + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 287, in read_database + ).to_polars( + ~~~~~~~~~^ + batch_size=batch_size, + ^^^^^^^^^^^^^^^^^^^^^^ + ...<2 lines>... + infer_schema_length=infer_schema_length, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 588, in to_polars + frame = frame_init( + batch_size=batch_size, + ...<2 lines>... + infer_schema_length=infer_schema_length, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 285, in _from_rows + (d[0], d[1:]) for d in self.result.cursor.description + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +AttributeError: 'NoneType' object has no attribute 'description' +2026-06-17 15:07:15 | INFO | ================================================================================ +2026-06-17 15:07:15 | INFO | Hello from data-move Python data pipeline! +2026-06-17 15:07:15 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 15:07:15 | INFO | Connecting to databases... +2026-06-17 15:07:16 | INFO | +2026-06-17 15:07:17 | INFO | +2026-06-17 15:07:18 | INFO | Database connections established +2026-06-17 15:07:18 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 15:07:18 | INFO | Found 863 MIDs +2026-06-17 15:07:18 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 15:07:31 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 15:07:31 | INFO | Fetched 4,814 matched empids fetched for OQAD +2026-06-17 15:07:31 | INFO | ================================================================================ +2026-06-17 15:07:31 | INFO | Processing Table: additional_visibility +2026-06-17 15:07:31 | INFO | Fetching data for 863 MIDs +2026-06-17 15:07:32 | INFO | Fetched 1,773 rows from SQL Server +2026-06-17 15:07:32 | INFO | Fetched 1773 rows +2026-06-17 15:07:32 | INFO | Deleting from additional_visibility +2026-06-17 15:07:32 | ERROR | Failed processing table additional_visibility +Traceback (most recent call last): + File "D:\data_move\main2.py", line 299, in main + load_to_clickhouse( + ~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + table_name=table_name, + ^^^^^^^^^^^^^^^^^^^^^^ + df=df, + ^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse + client.insert_arrow( + ~~~~~~~~~~~~~~~~~~~^ + table=table_name, + ^^^^^^^^^^^^^^^^^ + arrow_table=arrow_table, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow + return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings) + ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert + response = self._raw_request(insert_block, params, headers, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column project_id in table kelloggs_1.additional_visibility (9e03e625-028a-47db-8f60-e8ab93e907ea). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123) +2026-06-17 15:08:55 | INFO | ================================================================================ +2026-06-17 15:08:55 | INFO | Hello from data-move Python data pipeline! +2026-06-17 15:08:55 | INFO | Pipeline Run Date: 2026-06-16 +2026-06-17 15:08:55 | INFO | Connecting to databases... +2026-06-17 15:08:56 | INFO | +2026-06-17 15:08:58 | INFO | +2026-06-17 15:08:58 | INFO | Database connections established +2026-06-17 15:08:58 | INFO | Collecting MIDs for: 2026-06-16 +2026-06-17 15:08:59 | INFO | Found 863 MIDs +2026-06-17 15:08:59 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-17 15:09:12 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server +2026-06-17 15:09:12 | INFO | Fetched 4,814 matched empids fetched for OQAD +2026-06-17 15:09:12 | INFO | ================================================================================ +2026-06-17 15:09:12 | INFO | Processing Table: additional_visibility +2026-06-17 15:09:12 | INFO | Fetching data for 863 MIDs +2026-06-17 15:09:14 | INFO | Fetched 1,773 rows from SQL Server +2026-06-17 15:09:14 | INFO | Fetched 1773 rows +2026-06-17 15:09:14 | INFO | Creating table additional_visibility +2026-06-17 15:09:14 | INFO | Table ready: additional_visibility +2026-06-17 15:09:15 | INFO | additional_visibility: inserted 1,773 rows into ClickHouse +2026-06-17 15:09:15 | INFO | additional_visibility loaded successfully (1773 rows) +2026-06-17 15:09:15 | INFO | ================================================================================ +2026-06-17 15:09:15 | INFO | Pipeline Completed Successfully +2026-06-17 15:09:15 | INFO | ================================================================================ diff --git a/main2.py b/main2.py index a661812..e0460b0 100644 --- a/main2.py +++ b/main2.py @@ -45,7 +45,7 @@ from mids import ( from masters.dimensions import * from masters.bridge import * -from kpi.facts import * +from src.fact import * # ========================================================== @@ -94,15 +94,15 @@ def get_empids_clickhouse_OQAD( log.warning(f"Table '{table_name}' does not exist.") return pl.DataFrame( schema={ - "EmpId": pl.Int64, - "VisitDate": pl.Date, + "empid": pl.Int64, + "visitdate": pl.Date, } ) query = f""" SELECT DISTINCT - employee_id AS EmpId, - toDate(visit_date) AS VisitDate + employee_id AS empid, + visit_date AS visitdate FROM {table_name} """ @@ -166,20 +166,24 @@ def main(): sql_engine, run_date, ) - qf=fetch_quiz_empids() + qf=fetch_quiz_empids(sql_engine,run_date) db_df = get_empids_clickhouse_OQAD(client) matched = qf.join( db_df, - on=["EmpId", "VisitDate"], + on=["empid", "visitdate"], how="inner", ) + empids=matched["empid"].to_list() + + log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ") + # ------------------------------------------------------ # Config # ------------------------------------------------------ with open( - "t.yml", + "y.yml", "r", ) as file: @@ -207,16 +211,27 @@ def main(): # ------------------------------------------ # Fetch Data # ------------------------------------------ + if table_name =="OQaD": + + + df=fetch_OQaD( engine=sql_engine, + table_name=table_name, + table_type=table_type, + empids=empids, + run_date=run_date + ) - fn_name = f"fetch_{table_name}" + + else: + - df = get_dataframe( - fn_name=fn_name, - fetch_by=fetch_by, - sql_engine=sql_engine, - mids=mids, - run_date=run_date, - ) + df = fetch_data( + engine=sql_engine, + table_name=table_name, + table_type=table_type, + mids=mids, + run_date=run_date, + ) if df.is_empty(): diff --git a/mids.py b/mids.py index b0deff6..df81fd8 100644 --- a/mids.py +++ b/mids.py @@ -94,7 +94,7 @@ def fetch_quiz_empids(engine: Engine, run_date : date) -> pl.DataFrame: QUIZ AS ( SELECT Distinct E.EmpId as empid -, CONVERT(VARCHAR,DQ.VisitDate) AS VisitDate +, CONVERT(date,DQ.VisitDate) AS visitdate FROM OneApp_KelloggsMT.dbo.T_OQAD DQ INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E ON DQ.EmpId = E.EmpId inner join OneApp_KelloggsMT.dbo.Master_OQAD_Question QU on DQ.QuestionId= qu.QuestionId inner join diff --git a/src/fact.py b/src/fact.py index 213c640..97fdced 100644 --- a/src/fact.py +++ b/src/fact.py @@ -19,7 +19,7 @@ def fetch_data( mid_list = ",".join(str(mid) for mid in mids) - sql_file = Path("sql") / f"{table_type}" / f"{table_name}.sql" + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" with open(sql_file, "r", encoding="utf-8") as f: sql_template = f.read() @@ -53,7 +53,11 @@ def fetch_OQaD( empid_list = ",".join(str(empid) for empid in empids) - sql_file = Path("sql") / table_type / f"{table_name}.sql" + + sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql" + + log.info(f"Exists: {sql_file.exists()}") + log.info(f"Path: {sql_file.resolve()}") with open(sql_file, "r", encoding="utf-8") as f: sql_template = f.read() @@ -72,6 +76,6 @@ def fetch_OQaD( connection=engine, ) - log.info("Fetched %s rows", len(df)) + log.info("fn name is fetch_OQad ------Fetched %s rows", len(df)) return df \ No newline at end of file diff --git a/src/sql/fact/OQaD.sql b/src/sql/fact/OQaD.sql index e69de29..c2cbeca 100644 --- a/src/sql/fact/OQaD.sql +++ b/src/sql/fact/OQaD.sql @@ -0,0 +1,81 @@ +WITH MID_TABLE_COV1 AS +( + SELECT EmpId, VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CreateDate >= {run_date} + AND CreateDate < DATEADD(DAY,1,'{run_date}') + + UNION ALL + + SELECT EmpId, VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE UpdateDate >= {run_date} + AND UpdateDate < DATEADD(DAY,1, '{run_date}') +), + + +QUIZ AS +( + SELECT DISTINCT + E.EmpId, + E.EmpName, + E.SupervisorId, + E.SupervisorName, + E.DesignationName, + E.CityName, + E.StateName, + E.RegionName, + CAST(DQ.VisitDate AS DATE) AS VisitDate, + DQ.QuestionId, + DQ.AnswerId, + QC.QuestionCategoryId, + QC.QuestionCategory + FROM OneApp_KelloggsMT.dbo.T_OQAD DQ + INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E + ON DQ.EmpId = E.EmpId + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU + ON DQ.QuestionId = QU.QuestionId + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC + ON QU.QuestionCategoryId = QC.QuestionCategoryId + WHERE E.EmpName NOT LIKE '%TEST%' + AND E.RightId = 6 + AND ( + E.ResignDate IS NULL + OR CAST(E.ResignDate AS DATE) >= '{run_date}' + ) + AND EXISTS + ( + SELECT 1 + FROM MID_TABLE_COV1 A + WHERE A.EmpId = DQ.EmpId + AND CAST(A.VisitDate AS DATE) = CAST(DQ.VisitDate AS DATE) + ) +) + +SELECT + 40148 AS project_id, + Q.EmpId AS employee_id, + 0 AS process_id, + Q.VisitDate AS visit_date, + Q.QuestionCategoryId AS question_category_id, + Q.QuestionCategory AS question_category, + QM.QuestionId AS question_id, + QM.Question AS question, + ISNULL(QA.AnswerId,0) AS answer_id, + ISNULL(QA.Answer,'') AS answer, + CASE + WHEN QA.AnswerId IS NULL THEN 'Not Answer' + WHEN QA.RightAnswer = 1 THEN 'Y' + WHEN QA.RightAnswer IS NULL THEN 'Not Answer' + ELSE 'N' + END AS correct_answer, + GETDATE() AS update_date, + 'SP-Pius' AS update_by +FROM QUIZ Q +INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM + ON Q.QuestionId = QM.QuestionId +LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA + ON Q.AnswerId = QA.AnswerId +where Q.EmpId not in ({empid_list}) + + diff --git a/src/sql/fact/Survey.sql b/src/sql/fact/Survey.sql index e69de29..674422d 100644 --- a/src/sql/fact/Survey.sql +++ b/src/sql/fact/Survey.sql @@ -0,0 +1,68 @@ +WITH QUIZ AS +( + SELECT + SC.MID, + SC.StoreId, + EM.EmpId, + EM.EmpName, + EM.SupervisorId, + EM.SupervisorName, + EM.DesignationName, + CAST(SC.VisitDate AS DATE) AS VisitDate, + DQ.QuestionId, + QU.Question, + DQ.Answer, + QC.CategoryId, + QC.Category, + MSS.SubCategoryId, + MSS.SubCategory, + DQ.ImageUrl + + FROM OneApp_KelloggsMT.dbo.T_SURVEY DQ + + INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC + ON DQ.MID = SC.MID + + INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveyQuestion QU + ON DQ.QuestionId = QU.QuestionId + + INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveySubCategory MSS + ON QU.SubCategoryId = MSS.SubCategoryId + + INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveyCategory QC + ON MSS.CategoryId = QC.CategoryId + + INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM + ON SC.EmpId = EM.EmpId + + INNER JOIN OneApp_KelloggsMT.dbo.Master_Survey MS + ON DQ.SurveyId = MS.SurveyId + AND QU.SurveyId = MS.SurveyId + + WHERE EM.EmpName NOT LIKE 'test%' + AND SC.MID IN ({mid_list}) +) + +SELECT + 40148 AS Project_Id, + Q.MID AS Mid, + Q.SupervisorId AS SuperVisorId, + Q.EmpId AS EmpID, + SM.StoreId, + Q.VisitDate, + SM.StoreTypeId, + SM.ChainId, + Q.CategoryId, + Q.Category, + Q.SubCategoryId, + Q.SubCategory, + Q.QuestionId, + Q.Question, + Q.Answer, + GETDATE() AS CreateDate, + 'SP-Pius' AS CreateBy + +FROM QUIZ Q + +INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM + ON Q.StoreId = SM.StoreId \ No newline at end of file diff --git a/src/sql/fact/additional_visibility.sql b/src/sql/fact/additional_visibility.sql index e69de29..ecb0250 100644 --- a/src/sql/fact/additional_visibility.sql +++ b/src/sql/fact/additional_visibility.sql @@ -0,0 +1,31 @@ + + +WITH + additional_visibility (project_id,Mid, + emp_id,store_id,storetype_id,channel_id,chain_id,camera_allowed,visit_date,is_present,brand_id, + display_id,Remarks,image_url,created_date,created_by) +as ( + + SELECT '40148' AS ProjectId,ts.mid, em.EmpId as EmpId + , sm.StoreId,sm.storetypeid,sm.channelid,sm.chainid,case when sm.cameraallow=1 then 'Y' ELSE 'N' END AS + cameraallow , convert(varchar,sc.VisitDate,101)VisitDate, + case when ts.Present=0 then 'N' else 'Y' end Present, + isnull(MB.BrandId,'')[BrandId], + isnull(MD.DisplayId,'')[DisplayId],ISNULL(TS.REMARK,'')[Remark], + case when isnull(ts.ImageUrl,'')='' then '' else 'https://kimt1.parinaam.in/Upload/VisibilityImages/'+ts.ImageUrl + end as AdditionalImageUrl ,GETDATE(),'Pius' + + from + OneApp_KelloggsMT.dbo.T_AdditionalVisibility ts Inner join + OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join + OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join + OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId + left join + OneApp_KelloggsMT.dbo.Master_Brand MB ON TS.BrandId=MB.BrandId + LEFT JOIN + OneApp_KelloggsMT.dbo.Master_Display MD ON TS.DisplayId=MD.DisplayId + Where Em.EmpName not like 'test%' and ts.Present=1 and sc.mid in ({mid_list}) + +) + +select * from additional_visibility; \ No newline at end of file diff --git a/y.yml b/y.yml new file mode 100644 index 0000000..49ec53a --- /dev/null +++ b/y.yml @@ -0,0 +1,21 @@ +tables: + # - name: SOS_OneApp + # type: FACT + # operation: INSERT + # fetch_by: mids + + # - name: OQaD + # type: FACT + # operation: INSERT + # fetch_by: run_date + + - name: additional_visibility + type: FACT + operation: INSERT + fetch_by: mids + + + # - name: Survey + # type: FACT + # operation: INSERT + # fetch_by: mids \ No newline at end of file