diff --git a/clickhouse_task/delete_task.py b/clickhouse_task/delete_task.py index ed92a9a..a9f6a99 100644 --- a/clickhouse_task/delete_task.py +++ b/clickhouse_task/delete_task.py @@ -1,31 +1,191 @@ -from db_con.connection import * +from __future__ import annotations -from log import * +import polars as pl + +from log import log -def truncate_table(client, table_name: str) -> None: +def truncate_table( + client, + table_name: str, +) -> None: """ - Truncate a ClickHouse table. + Full refresh tables. """ - query = f"TRUNCATE TABLE {table_name}" - print(f"Truncating table: {table_name}") - client.command(query) + client.command( + f"TRUNCATE TABLE {table_name}" + ) - log.info(f"Table {table_name} truncated successfully.") + log.info( + "Truncated table %s", + table_name, + ) - -def delete_rows(client, table_name: str, condition: str) -> None: +def delete_rows( + client, + table_name: str, + where_clause: str, +) -> None: """ - Delete rows from a ClickHouse table based on a condition. + Generic ClickHouse delete. """ + query = f""" ALTER TABLE {table_name} - DELETE WHERE {condition} + DELETE + WHERE {where_clause} """ - print(f"Deleting rows from {table_name} where {condition}") + log.info( + "Deleting from %s", + table_name, + ) + client.command(query) - log.info("Delete command submitted successfully.") \ No newline at end of file + +def delete_existing_data( + client, + table_name: str, + run_date, + mids: list[int], + emp_visit_df: pl.DataFrame, +) -> None: + """ + Incremental delete logic. + Matches the old SQL procedure. + """ + + # -------------------------------------------------- + # MID based tables + # -------------------------------------------------- + + mid_tables = { + "additional_visibility", + "Coverage", + "Survey", + "Promotion", + "PaidVisibility", + "SOS_OneApp", + "Stock_Details", + "Login", + "coverage_remarks", + } + + if table_name in mid_tables and mids: + + mids_str = ",".join( + map(str, mids) + ) + + delete_rows( + client, + table_name, + f"MID IN ({mids_str})", + ) + + return + + # -------------------------------------------------- + # Journey Plan + # -------------------------------------------------- + + if table_name == "Journey_Plan": + + delete_rows( + client, + table_name, + f""" + toMonth(visit_date) = {run_date.month} + AND toYear(visit_date) = {run_date.year} + """, + ) + + return + + # -------------------------------------------------- + # Logins + # -------------------------------------------------- + if table_name == "Login": + delete_rows( + client, + table_name, + f""" + project_id = 40148 + AND toDate(login_date) = toDate('{run_date}') + """ + ) + + + # -------------------------------------------------- + # Web Logins + # -------------------------------------------------- + + + + + + if table_name == "Web_Logins": + + delete_rows( + client, + table_name, + f""" + toDate(date) + = toDate('{run_date}') + """, + ) + + return + + # -------------------------------------------------- + # Attendance + # -------------------------------------------------- + + if table_name == "Attendance": + + delete_rows( + client, + table_name, + f""" + toDate(attendance_date) + = toDate('{run_date}') + """, + ) + + return + + # -------------------------------------------------- + # OQaD + # -------------------------------------------------- + + if ( + table_name == "OQaD" + and not emp_visit_df.is_empty() + ): + + conditions = [ + ( + f"(employee_id={row['EmpId']} " + f"AND toDate(visit_date)=" + f"toDate('{row['VisitDate']}'))" + ) + for row in emp_visit_df.iter_rows( + named=True + ) + ] + + delete_rows( + client, + table_name, + " OR ".join(conditions), + ) + + return + + log.info( + "No delete logic required for %s", + table_name, + ) \ No newline at end of file diff --git a/clickhouse_task/load_table.py b/clickhouse_task/load_table.py index c56ea6b..b7ace82 100644 --- a/clickhouse_task/load_table.py +++ b/clickhouse_task/load_table.py @@ -3,6 +3,7 @@ from sqlalchemy.engine import URL, Engine import os import clickhouse_connect import polars as pd +import pyarrow from log import log diff --git a/db_con/.env b/db_con/.env index fd3c943..f02885b 100644 --- a/db_con/.env +++ b/db_con/.env @@ -9,4 +9,4 @@ CH_HOST=172.188.12.194 CH_PORT=8123 CH_USER=default CH_PASS=dipanshu_k -CH_DB=kelloggs \ No newline at end of file +CH_DB=kelloggs_1 \ No newline at end of file diff --git a/kpi/facts.py b/kpi/facts.py index 45bfc9b..58b6581 100644 --- a/kpi/facts.py +++ b/kpi/facts.py @@ -140,23 +140,27 @@ def fetch_SOS_OneApp(engine: Engine, mids: list[int]) -> pl.DataFrame: return df - -def fetch_OQaD(engine: Engine, mids: list[int]) -> pl.DataFrame: - - if not mids: - log.warning("No MIDs — nothing to fetch.") - return pl.DataFrame() - - mid_list = ",".join(map(str, mids)) +def fetch_OQaD( + engine: Engine, + run_date: date, +) -> pl.DataFrame: sql = f""" WITH MID_TABLE_COV1 AS ( - SELECT DISTINCT + SELECT EmpId, CAST(VisitDate AS DATE) AS VisitDate - FROM OneApp_KelloggsMT.dbo.T_StoreCoverage - WHERE MID IN ({mid_list}) + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CAST(CreateDate AS DATE) = '{run_date}' + + UNION + + SELECT + EmpId, + CAST(VisitDate AS DATE) AS VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CAST(UpdateDate AS DATE) = '{run_date}' ), QUIZ AS @@ -180,12 +184,12 @@ def fetch_OQaD(engine: Engine, mids: list[int]) -> pl.DataFrame: INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC ON QU.QuestionCategoryId = QC.QuestionCategoryId - WHERE E.EmpName NOT LIKE 'test%' + WHERE E.RightId = 6 + AND E.EmpName NOT LIKE 'test%' AND E.EmpName NOT LIKE '%TEST%' - AND E.RightId = 6 AND ( E.ResignDate IS NULL - OR E.ResignDate >= DQ.VisitDate + OR E.ResignDate >= '{run_date}' ) AND EXISTS ( @@ -197,6 +201,7 @@ def fetch_OQaD(engine: Engine, mids: list[int]) -> pl.DataFrame: ) SELECT + 40148 AS project_id, Q.EmpId AS employee_id, 0 AS process_id, Q.VisitDate AS visit_date, @@ -204,16 +209,18 @@ def fetch_OQaD(engine: Engine, mids: list[int]) -> pl.DataFrame: Q.QuestionCategory AS question_category, QM.QuestionId AS question_id, QM.Question AS question, - - ISNULL(QA.AnswerId,0) AS answer_id, - ISNULL(QA.Answer,'') AS answer, + ISNULL(QA.AnswerId, 0) AS answer_id, + ISNULL(QA.Answer, '') AS answer, CASE WHEN QA.AnswerId IS NULL THEN 'Not Answer' WHEN QA.RightAnswer = 1 THEN 'Y' WHEN QA.RightAnswer IS NULL THEN 'Not Answer' ELSE 'N' - END AS correct_answer + END AS correct_answer, + + GETDATE() AS update_date, + 'SP-Pius' AS update_by FROM QUIZ Q @@ -223,24 +230,19 @@ def fetch_OQaD(engine: Engine, mids: list[int]) -> pl.DataFrame: LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA ON Q.AnswerId = QA.AnswerId """ - - log.info(f"Fetching OQaD data for {len(mids):,} MIDs") + + log.info("Fetching OQaD data for run_date=%s", run_date) df = pl.read_database( query=sql, - connection=engine + connection=engine, ) - - log.info(f"Fetched {len(df):,} rows") + log.info("Fetched %s rows", len(df)) return df - - - - def fetch_Survey(engine: Engine, mids: list[int]) -> pl.DataFrame: if not mids: log.warning("No MIDs — nothing to fetch.") diff --git a/logs/etl_20260612.log b/logs/etl_20260612.log index f9963e8..1c58d25 100644 --- a/logs/etl_20260612.log +++ b/logs/etl_20260612.log @@ -982,3 +982,43 @@ 2026-06-12 19:04:53 | INFO | Fetched 5,984 stores 2026-06-12 19:04:53 | INFO | Table Store_Master truncated successfully. 2026-06-12 19:04:53 | INFO | Truncate a ClickHouse table - Store_Master +2026-06-12 19:07:43 | INFO | ================================================================================ +2026-06-12 19:07:43 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 19:07:43 | INFO | Data-pipeline running Date is -:2026-06-11 +2026-06-12 19:07:43 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 19:07:44 | INFO | +2026-06-12 19:07:45 | INFO | +2026-06-12 19:07:46 | INFO | Both databases connected successfully +2026-06-12 19:07:46 | INFO | Collecting MIDs for: 2026-06-11 +2026-06-12 19:07:46 | INFO | Found 818 MIDs +2026-06-12 19:07:46 | INFO | ================================================================================ +2026-06-12 19:07:46 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT +2026-06-12 19:07:46 | INFO | Fetching data for 818 MIDs +2026-06-12 19:07:47 | INFO | Fetched 3,677 rows from SQL Server +2026-06-12 19:07:47 | INFO | Delete command submitted successfully. +2026-06-12 19:07:47 | INFO | SOS_OneApp: inserted 3,677 rows into ClickHouse +2026-06-12 19:07:47 | INFO | ================================================================================ +2026-06-12 19:07:47 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT +2026-06-12 19:07:47 | INFO | Fetching OQaD data for 818 MIDs +2026-06-12 19:07:50 | INFO | Fetched 464 rows +2026-06-12 19:09:36 | INFO | ================================================================================ +2026-06-12 19:09:36 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 19:09:36 | INFO | Data-pipeline running Date is -:2026-06-11 +2026-06-12 19:09:36 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 19:09:37 | INFO | +2026-06-12 19:09:38 | INFO | +2026-06-12 19:09:39 | INFO | Both databases connected successfully +2026-06-12 19:09:39 | INFO | Collecting MIDs for: 2026-06-11 +2026-06-12 19:09:39 | INFO | Found 818 MIDs +2026-06-12 19:09:39 | INFO | ================================================================================ +2026-06-12 19:09:39 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT +2026-06-12 19:09:39 | INFO | Fetching data for 818 MIDs +2026-06-12 19:09:40 | INFO | Fetched 3,677 rows from SQL Server +2026-06-12 19:09:40 | INFO | Deleting rows from SOS_OneApp +2026-06-12 19:09:40 | INFO | Delete command submitted successfully. +2026-06-12 19:09:41 | INFO | SOS_OneApp: inserted 3,677 rows into ClickHouse +2026-06-12 19:09:41 | INFO | ================================================================================ +2026-06-12 19:09:41 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT +2026-06-12 19:09:41 | INFO | Fetching OQaD data for 818 MIDs +2026-06-12 19:09:44 | INFO | Fetched 464 rows +2026-06-12 19:09:44 | INFO | Deleting rows from OQaD diff --git a/logs/etl_20260615.log b/logs/etl_20260615.log new file mode 100644 index 0000000..7b4d799 --- /dev/null +++ b/logs/etl_20260615.log @@ -0,0 +1,1103 @@ +2026-06-15 11:21:29 | INFO | ================================================================================ +2026-06-15 11:21:29 | INFO | Hello from data-move Python data pipeline ! +2026-06-15 11:21:29 | INFO | Pipeline Run Date : 2026-06-14 +2026-06-15 11:21:29 | INFO | Connecting to SQL Server and ClickHouse +2026-06-15 11:21:31 | INFO | +2026-06-15 11:21:34 | INFO | +2026-06-15 11:21:34 | INFO | Database connections established +2026-06-15 11:21:34 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 11:21:35 | INFO | Found 571 MIDs +2026-06-15 11:21:35 | INFO | Deleting data from additional_visibility +2026-06-15 11:27:46 | INFO | ================================================================================ +2026-06-15 11:27:46 | INFO | Hello from data-move Python data pipeline ! +2026-06-15 11:27:46 | INFO | Pipeline Run Date : 2026-06-14 +2026-06-15 11:27:46 | INFO | Connecting to SQL Server and ClickHouse +2026-06-15 11:27:46 | INFO | +2026-06-15 11:27:48 | INFO | +2026-06-15 11:27:48 | INFO | Database connections established +2026-06-15 11:27:48 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 11:27:49 | INFO | Found 571 MIDs +2026-06-15 11:27:49 | INFO | Deleting data from additional_visibility +2026-06-15 11:27:49 | INFO | Deleting data from Coverage +2026-06-15 11:27:49 | INFO | Deleting data from Survey +2026-06-15 11:27:49 | INFO | Deleting data from Promotion +2026-06-15 11:41:04 | INFO | ================================================================================ +2026-06-15 11:41:04 | INFO | Hello from data-move Python data pipeline! +2026-06-15 11:41:04 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 11:41:04 | INFO | Connecting to databases... +2026-06-15 11:41:05 | INFO | +2026-06-15 11:41:06 | INFO | +2026-06-15 11:41:07 | INFO | Database connections established +2026-06-15 11:41:07 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 11:41:07 | INFO | Found 571 MIDs +2026-06-15 11:41:07 | INFO | ================================================================================ +2026-06-15 11:41:07 | INFO | Processing Table: SOS_OneApp +2026-06-15 11:41:07 | INFO | Fetching data for 571 MIDs +2026-06-15 11:41:10 | INFO | Fetched 1,995 rows from SQL Server +2026-06-15 11:41:10 | INFO | Fetched 1995 rows +2026-06-15 11:41:10 | INFO | Deleting from SOS_OneApp +2026-06-15 11:41:10 | ERROR | Failed processing table SOS_OneApp +Traceback (most recent call last): + File "D:\data_move\test2.py", line 232, in main + load_to_clickhouse( + ~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + table_name=table_name, + ^^^^^^^^^^^^^^^^^^^^^^ + df=df, + ^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\load_table.py", line 22, in load_to_clickhouse + arrow_table = df.to_arrow() + File "d:\data_move\.venv\Lib\site-packages\polars\_utils\deprecation.py", line 128, in wrapper + return function(*args, **kwargs) + File "d:\data_move\.venv\Lib\site-packages\polars\dataframe\frame.py", line 1797, in to_arrow + record_batches = self._df.to_arrow(compat_level_py) +ModuleNotFoundError: No module named 'pyarrow' +2026-06-15 11:42:30 | INFO | ================================================================================ +2026-06-15 11:42:30 | INFO | Hello from data-move Python data pipeline! +2026-06-15 11:42:30 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 11:42:30 | INFO | Connecting to databases... +2026-06-15 11:42:31 | INFO | +2026-06-15 11:42:33 | INFO | +2026-06-15 11:42:33 | INFO | Database connections established +2026-06-15 11:42:33 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 11:42:34 | INFO | Found 571 MIDs +2026-06-15 11:42:34 | INFO | ================================================================================ +2026-06-15 11:42:34 | INFO | Processing Table: SOS_OneApp +2026-06-15 11:42:34 | INFO | Fetching data for 571 MIDs +2026-06-15 11:42:34 | INFO | Fetched 1,995 rows from SQL Server +2026-06-15 11:42:34 | INFO | Fetched 1995 rows +2026-06-15 11:42:35 | INFO | Deleting from SOS_OneApp +2026-06-15 11:42:35 | ERROR | Failed processing table SOS_OneApp +Traceback (most recent call last): + File "D:\data_move\test2.py", line 233, in main + load_to_clickhouse( + ~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + table_name=table_name, + ^^^^^^^^^^^^^^^^^^^^^^ + df=df, + ^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\load_table.py", line 22, in load_to_clickhouse + arrow_table = df.to_arrow() + File "d:\data_move\.venv\Lib\site-packages\polars\_utils\deprecation.py", line 128, in wrapper + return function(*args, **kwargs) + File "d:\data_move\.venv\Lib\site-packages\polars\dataframe\frame.py", line 1797, in to_arrow + record_batches = self._df.to_arrow(compat_level_py) +ModuleNotFoundError: No module named 'pyarrow' +2026-06-15 11:45:18 | INFO | ================================================================================ +2026-06-15 11:45:18 | INFO | Hello from data-move Python data pipeline! +2026-06-15 11:45:18 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 11:45:18 | INFO | Connecting to databases... +2026-06-15 11:45:20 | INFO | +2026-06-15 11:45:24 | INFO | +2026-06-15 11:45:24 | INFO | Database connections established +2026-06-15 11:45:24 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 11:45:25 | INFO | Found 571 MIDs +2026-06-15 11:45:25 | INFO | ================================================================================ +2026-06-15 11:45:25 | INFO | Processing Table: SOS_OneApp +2026-06-15 11:45:25 | INFO | Fetching data for 571 MIDs +2026-06-15 11:45:26 | INFO | Fetched 1,995 rows from SQL Server +2026-06-15 11:45:26 | INFO | Fetched 1995 rows +2026-06-15 11:45:26 | INFO | Deleting from SOS_OneApp +2026-06-15 11:45:26 | INFO | SOS_OneApp: inserted 1,995 rows into ClickHouse +2026-06-15 11:45:26 | INFO | SOS_OneApp loaded successfully (1995 rows) +2026-06-15 11:45:26 | INFO | ================================================================================ +2026-06-15 11:45:26 | INFO | Processing Table: OQaD +2026-06-15 11:45:26 | INFO | Fetching OQaD data for run_date=[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650] +2026-06-15 11:45:27 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +pyodbc.DataError: ('22007', '[22007] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Conversion failed when converting date and/or time from character string. (241) (SQLExecDirectW)') + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "D:\data_move\test2.py", line 177, in main + df = get_dataframe( + fn_name=fn_name, + ...<3 lines>... + run_date=run_date, + ) + File "D:\data_move\test2.py", line 78, in get_dataframe + return fn(sql_engine, mids) + File "D:\data_move\kpi\facts.py", line 236, in fetch_OQaD + df = pl.read_database( + query=sql, + connection=engine, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database + return cx.execute( + ~~~~~~~~~~^ + query=query, + ^^^^^^^^^^^^ + options=execute_options, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ).to_polars( + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute + result = cursor_execute(query, **options) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute + return meth( + self, + distilled_parameters, + execution_options or NO_OPTIONS, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection + return connection._execute_clauseelement( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + self, distilled_params, execution_options + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement + ret = self._execute_context( + dialect, + ...<8 lines>... + cache_hit=cache_hit, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context + return self._exec_single_context( + ~~~~~~~~~~~~~~~~~~~~~~~~~^ + dialect, context, statement, parameters + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context + self._handle_dbapi_exception( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + e, str_statement, effective_parameters, cursor, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +sqlalchemy.exc.DataError: (pyodbc.DataError) ('22007', '[22007] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Conversion failed when converting date and/or time from character string. (241) (SQLExecDirectW)') +[SQL: + WITH MID_TABLE_COV1 AS + ( + SELECT + EmpId, + CAST(VisitDate AS DATE) AS VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CAST(CreateDate AS DATE) = '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' + + UNION + + SELECT + EmpId, + CAST(VisitDate AS DATE) AS VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CAST(UpdateDate AS DATE) = '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' + ), + + QUIZ AS + ( + SELECT DISTINCT + E.EmpId, + CAST(DQ.VisitDate AS DATE) AS VisitDate, + DQ.QuestionId, + DQ.AnswerId, + QC.QuestionCategoryId, + QC.QuestionCategory + + FROM OneApp_KelloggsMT.dbo.T_OQAD DQ + + INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E + ON DQ.EmpId = E.EmpId + + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU + ON DQ.QuestionId = QU.QuestionId + + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC + ON QU.QuestionCategoryId = QC.QuestionCategoryId + + WHERE E.RightId = 6 + AND E.EmpName NOT LIKE 'test%' + AND E.EmpName NOT LIKE '%TEST%' + AND ( + E.ResignDate IS NULL + OR E.ResignDate >= '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' + ) + AND EXISTS + ( + SELECT 1 + FROM MID_TABLE_COV1 A + WHERE A.EmpId = DQ.EmpId + AND A.VisitDate = CAST(DQ.VisitDate AS DATE) + ) + ) + + SELECT + 40148 AS project_id, + Q.EmpId AS employee_id, + 0 AS process_id, + Q.VisitDate AS visit_date, + Q.QuestionCategoryId AS question_category_id, + Q.QuestionCategory AS question_category, + QM.QuestionId AS question_id, + QM.Question AS question, + ISNULL(QA.AnswerId, 0) AS answer_id, + ISNULL(QA.Answer, '') AS answer, + + CASE + WHEN QA.AnswerId IS NULL THEN 'Not Answer' + WHEN QA.RightAnswer = 1 THEN 'Y' + WHEN QA.RightAnswer IS NULL THEN 'Not Answer' + ELSE 'N' + END AS correct_answer, + + GETDATE() AS update_date, + 'SP-Pius' AS update_by + + FROM QUIZ Q + + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM + ON Q.QuestionId = QM.QuestionId + + LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA + ON Q.AnswerId = QA.AnswerId + ] +(Background on this error at: https://sqlalche.me/e/20/9h9h) +2026-06-15 11:47:45 | INFO | ================================================================================ +2026-06-15 11:47:45 | INFO | Hello from data-move Python data pipeline! +2026-06-15 11:47:45 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 11:47:45 | INFO | Connecting to databases... +2026-06-15 11:47:46 | INFO | +2026-06-15 11:47:48 | INFO | +2026-06-15 11:47:48 | INFO | Database connections established +2026-06-15 11:47:48 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 11:47:49 | INFO | Found 571 MIDs +2026-06-15 11:47:49 | INFO | ================================================================================ +2026-06-15 11:47:49 | INFO | Processing Table: SOS_OneApp +2026-06-15 11:47:49 | INFO | Fetching data for 571 MIDs +2026-06-15 11:47:49 | INFO | Fetched 1,995 rows from SQL Server +2026-06-15 11:47:49 | INFO | Fetched 1995 rows +2026-06-15 11:47:50 | INFO | Deleting from SOS_OneApp +2026-06-15 11:47:50 | INFO | SOS_OneApp: inserted 1,995 rows into ClickHouse +2026-06-15 11:47:50 | INFO | SOS_OneApp loaded successfully (1995 rows) +2026-06-15 11:47:50 | INFO | ================================================================================ +2026-06-15 11:47:50 | INFO | Processing Table: OQaD +2026-06-15 11:47:50 | INFO | Fetching OQaD data for run_date=[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650] +2026-06-15 11:47:50 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +pyodbc.DataError: ('22007', '[22007] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Conversion failed when converting date and/or time from character string. (241) (SQLExecDirectW)') + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "D:\data_move\test2.py", line 177, in main + df = get_dataframe( + fn_name=fn_name, + ...<3 lines>... + run_date=run_date, + ) + File "D:\data_move\test2.py", line 78, in get_dataframe + return fn(sql_engine, mids) + File "D:\data_move\kpi\facts.py", line 236, in fetch_OQaD + df = pl.read_database( + query=sql, + connection=engine, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database + return cx.execute( + ~~~~~~~~~~^ + query=query, + ^^^^^^^^^^^^ + options=execute_options, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ).to_polars( + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute + result = cursor_execute(query, **options) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute + return meth( + self, + distilled_parameters, + execution_options or NO_OPTIONS, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection + return connection._execute_clauseelement( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + self, distilled_params, execution_options + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement + ret = self._execute_context( + dialect, + ...<8 lines>... + cache_hit=cache_hit, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context + return self._exec_single_context( + ~~~~~~~~~~~~~~~~~~~~~~~~~^ + dialect, context, statement, parameters + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context + self._handle_dbapi_exception( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + e, str_statement, effective_parameters, cursor, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +sqlalchemy.exc.DataError: (pyodbc.DataError) ('22007', '[22007] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Conversion failed when converting date and/or time from character string. (241) (SQLExecDirectW)') +[SQL: + WITH MID_TABLE_COV1 AS + ( + SELECT + EmpId, + CAST(VisitDate AS DATE) AS VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CAST(CreateDate AS DATE) = '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' + + UNION + + SELECT + EmpId, + CAST(VisitDate AS DATE) AS VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CAST(UpdateDate AS DATE) = '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' + ), + + QUIZ AS + ( + SELECT DISTINCT + E.EmpId, + CAST(DQ.VisitDate AS DATE) AS VisitDate, + DQ.QuestionId, + DQ.AnswerId, + QC.QuestionCategoryId, + QC.QuestionCategory + + FROM OneApp_KelloggsMT.dbo.T_OQAD DQ + + INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E + ON DQ.EmpId = E.EmpId + + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU + ON DQ.QuestionId = QU.QuestionId + + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC + ON QU.QuestionCategoryId = QC.QuestionCategoryId + + WHERE E.RightId = 6 + AND E.EmpName NOT LIKE 'test%' + AND E.EmpName NOT LIKE '%TEST%' + AND ( + E.ResignDate IS NULL + OR E.ResignDate >= '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' + ) + AND EXISTS + ( + SELECT 1 + FROM MID_TABLE_COV1 A + WHERE A.EmpId = DQ.EmpId + AND A.VisitDate = CAST(DQ.VisitDate AS DATE) + ) + ) + + SELECT + 40148 AS project_id, + Q.EmpId AS employee_id, + 0 AS process_id, + Q.VisitDate AS visit_date, + Q.QuestionCategoryId AS question_category_id, + Q.QuestionCategory AS question_category, + QM.QuestionId AS question_id, + QM.Question AS question, + ISNULL(QA.AnswerId, 0) AS answer_id, + ISNULL(QA.Answer, '') AS answer, + + CASE + WHEN QA.AnswerId IS NULL THEN 'Not Answer' + WHEN QA.RightAnswer = 1 THEN 'Y' + WHEN QA.RightAnswer IS NULL THEN 'Not Answer' + ELSE 'N' + END AS correct_answer, + + GETDATE() AS update_date, + 'SP-Pius' AS update_by + + FROM QUIZ Q + + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM + ON Q.QuestionId = QM.QuestionId + + LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA + ON Q.AnswerId = QA.AnswerId + ] +(Background on this error at: https://sqlalche.me/e/20/9h9h) +2026-06-15 11:49:49 | INFO | ================================================================================ +2026-06-15 11:49:49 | INFO | Hello from data-move Python data pipeline! +2026-06-15 11:49:49 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 11:49:49 | INFO | Connecting to databases... +2026-06-15 11:49:50 | INFO | +2026-06-15 11:49:51 | INFO | +2026-06-15 11:49:52 | INFO | Database connections established +2026-06-15 11:49:52 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 11:49:52 | INFO | Found 571 MIDs +2026-06-15 11:49:52 | INFO | ================================================================================ +2026-06-15 11:49:52 | INFO | Processing Table: SOS_OneApp +2026-06-15 11:49:52 | INFO | Fetching data for 571 MIDs +2026-06-15 11:49:52 | INFO | Fetched 1,995 rows from SQL Server +2026-06-15 11:49:52 | INFO | Fetched 1995 rows +2026-06-15 11:49:52 | INFO | Deleting from SOS_OneApp +2026-06-15 11:49:53 | INFO | SOS_OneApp: inserted 1,995 rows into ClickHouse +2026-06-15 11:49:53 | INFO | SOS_OneApp loaded successfully (1995 rows) +2026-06-15 11:49:53 | INFO | ================================================================================ +2026-06-15 11:49:53 | INFO | Processing Table: OQaD +2026-06-15 11:49:53 | INFO | Fetching OQaD data for run_date=[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650] +2026-06-15 11:49:53 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +pyodbc.DataError: ('22007', '[22007] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Conversion failed when converting date and/or time from character string. (241) (SQLExecDirectW)') + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "D:\data_move\test2.py", line 177, in main + df = get_dataframe( + fn_name=fn_name, + ...<3 lines>... + run_date=run_date, + ) + File "D:\data_move\test2.py", line 78, in get_dataframe + return fn(sql_engine, mids) + File "D:\data_move\kpi\facts.py", line 236, in fetch_OQaD + df = pl.read_database( + query=sql, + connection=engine, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database + return cx.execute( + ~~~~~~~~~~^ + query=query, + ^^^^^^^^^^^^ + options=execute_options, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ).to_polars( + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute + result = cursor_execute(query, **options) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute + return meth( + self, + distilled_parameters, + execution_options or NO_OPTIONS, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection + return connection._execute_clauseelement( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + self, distilled_params, execution_options + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement + ret = self._execute_context( + dialect, + ...<8 lines>... + cache_hit=cache_hit, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context + return self._exec_single_context( + ~~~~~~~~~~~~~~~~~~~~~~~~~^ + dialect, context, statement, parameters + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context + self._handle_dbapi_exception( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + e, str_statement, effective_parameters, cursor, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +sqlalchemy.exc.DataError: (pyodbc.DataError) ('22007', '[22007] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Conversion failed when converting date and/or time from character string. (241) (SQLExecDirectW)') +[SQL: + WITH MID_TABLE_COV1 AS + ( + SELECT + EmpId, + CAST(VisitDate AS DATE) AS VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CAST(CreateDate AS DATE) = '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' + + UNION + + SELECT + EmpId, + CAST(VisitDate AS DATE) AS VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CAST(UpdateDate AS DATE) = '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' + ), + + QUIZ AS + ( + SELECT DISTINCT + E.EmpId, + CAST(DQ.VisitDate AS DATE) AS VisitDate, + DQ.QuestionId, + DQ.AnswerId, + QC.QuestionCategoryId, + QC.QuestionCategory + + FROM OneApp_KelloggsMT.dbo.T_OQAD DQ + + INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E + ON DQ.EmpId = E.EmpId + + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU + ON DQ.QuestionId = QU.QuestionId + + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC + ON QU.QuestionCategoryId = QC.QuestionCategoryId + + WHERE E.RightId = 6 + AND E.EmpName NOT LIKE 'test%' + AND E.EmpName NOT LIKE '%TEST%' + AND ( + E.ResignDate IS NULL + OR E.ResignDate >= '[1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650]' + ) + AND EXISTS + ( + SELECT 1 + FROM MID_TABLE_COV1 A + WHERE A.EmpId = DQ.EmpId + AND A.VisitDate = CAST(DQ.VisitDate AS DATE) + ) + ) + + SELECT + 40148 AS project_id, + Q.EmpId AS employee_id, + 0 AS process_id, + Q.VisitDate AS visit_date, + Q.QuestionCategoryId AS question_category_id, + Q.QuestionCategory AS question_category, + QM.QuestionId AS question_id, + QM.Question AS question, + ISNULL(QA.AnswerId, 0) AS answer_id, + ISNULL(QA.Answer, '') AS answer, + + CASE + WHEN QA.AnswerId IS NULL THEN 'Not Answer' + WHEN QA.RightAnswer = 1 THEN 'Y' + WHEN QA.RightAnswer IS NULL THEN 'Not Answer' + ELSE 'N' + END AS correct_answer, + + GETDATE() AS update_date, + 'SP-Pius' AS update_by + + FROM QUIZ Q + + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM + ON Q.QuestionId = QM.QuestionId + + LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA + ON Q.AnswerId = QA.AnswerId + ] +(Background on this error at: https://sqlalche.me/e/20/9h9h) +2026-06-15 11:51:58 | INFO | ================================================================================ +2026-06-15 11:51:58 | INFO | Hello from data-move Python data pipeline! +2026-06-15 11:51:58 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 11:51:58 | INFO | Connecting to databases... +2026-06-15 11:51:59 | INFO | +2026-06-15 11:52:01 | INFO | +2026-06-15 11:52:01 | INFO | Database connections established +2026-06-15 11:52:01 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 11:52:01 | INFO | Found 571 MIDs +2026-06-15 11:52:02 | INFO | ================================================================================ +2026-06-15 11:52:02 | INFO | Processing Table: SOS_OneApp +2026-06-15 11:52:02 | INFO | Fetching data for 571 MIDs +2026-06-15 11:52:02 | INFO | Fetched 1,995 rows from SQL Server +2026-06-15 11:52:02 | INFO | Fetched 1995 rows +2026-06-15 11:52:02 | INFO | Deleting from SOS_OneApp +2026-06-15 11:52:02 | INFO | SOS_OneApp: inserted 1,995 rows into ClickHouse +2026-06-15 11:52:02 | INFO | SOS_OneApp loaded successfully (1995 rows) +2026-06-15 11:52:02 | INFO | ================================================================================ +2026-06-15 11:52:02 | INFO | Processing Table: OQaD +2026-06-15 11:52:02 | INFO | Fetching OQaD data for run_date=2026-06-14 +2026-06-15 11:52:06 | INFO | Fetched 443 rows +2026-06-15 11:52:06 | INFO | Fetched 443 rows +2026-06-15 11:52:07 | INFO | Deleting from OQaD +2026-06-15 11:52:07 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "D:\data_move\test2.py", line 248, in main + load_to_clickhouse( + ~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + table_name=table_name, + ^^^^^^^^^^^^^^^^^^^^^^ + df=df, + ^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse + client.insert_arrow( + ~~~~~~~~~~~~~~~~~~~^ + table=table_name, + ^^^^^^^^^^^^^^^^^ + arrow_table=arrow_table, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow + return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings) + ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert + response = self._raw_request(insert_block, params, headers, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column project_id in table kelloggs.OQaD (baea0f0f-5216-49dd-b156-fa06fe3e62f1). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123) +2026-06-15 11:55:57 | INFO | ================================================================================ +2026-06-15 11:55:57 | INFO | Hello from data-move Python data pipeline! +2026-06-15 11:55:57 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 11:55:57 | INFO | Connecting to databases... +2026-06-15 11:55:58 | INFO | +2026-06-15 11:56:00 | INFO | +2026-06-15 11:56:00 | INFO | Database connections established +2026-06-15 11:56:00 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 11:56:00 | INFO | Found 571 MIDs +2026-06-15 11:56:01 | INFO | ================================================================================ +2026-06-15 11:56:01 | INFO | Processing Table: SOS_OneApp +2026-06-15 11:56:01 | INFO | Fetching data for 571 MIDs +2026-06-15 11:56:01 | INFO | Fetched 1,995 rows from SQL Server +2026-06-15 11:56:01 | INFO | Fetched 1995 rows +2026-06-15 11:56:01 | INFO | Creating table SOS_OneApp +2026-06-15 11:56:01 | INFO | Table ready: SOS_OneApp +2026-06-15 11:56:02 | INFO | SOS_OneApp: inserted 1,995 rows into ClickHouse +2026-06-15 11:56:02 | INFO | SOS_OneApp loaded successfully (1995 rows) +2026-06-15 11:56:02 | INFO | ================================================================================ +2026-06-15 11:56:02 | INFO | Processing Table: OQaD +2026-06-15 11:56:02 | INFO | Fetching OQaD data for run_date=2026-06-14 +2026-06-15 11:56:04 | INFO | Fetched 443 rows +2026-06-15 11:56:04 | INFO | Fetched 443 rows +2026-06-15 11:56:04 | INFO | Creating table OQaD +2026-06-15 11:56:04 | INFO | Table ready: OQaD +2026-06-15 11:56:04 | INFO | OQaD: inserted 443 rows into ClickHouse +2026-06-15 11:56:04 | INFO | OQaD loaded successfully (443 rows) +2026-06-15 11:56:04 | INFO | ================================================================================ +2026-06-15 11:56:04 | INFO | Processing Table: Survey +2026-06-15 11:56:04 | INFO | Fetching Survey data for 571 MIDs +2026-06-15 11:56:07 | INFO | Fetched 173 Survey rows +2026-06-15 11:56:07 | INFO | Fetched 173 rows +2026-06-15 11:56:07 | INFO | Creating table Survey +2026-06-15 11:56:07 | INFO | Table ready: Survey +2026-06-15 11:56:07 | INFO | Survey: inserted 173 rows into ClickHouse +2026-06-15 11:56:07 | INFO | Survey loaded successfully (173 rows) +2026-06-15 11:56:07 | INFO | ================================================================================ +2026-06-15 11:56:07 | INFO | Processing Table: additional_visibility +2026-06-15 11:56:07 | INFO | Fetching Additional Visibility data for 571 MIDs +2026-06-15 11:56:17 | INFO | Fetched 1,648 Additional Visibility rows +2026-06-15 11:56:17 | INFO | Fetched 1648 rows +2026-06-15 11:56:17 | INFO | Creating table additional_visibility +2026-06-15 11:56:18 | INFO | Table ready: additional_visibility +2026-06-15 11:56:18 | INFO | additional_visibility: inserted 1,648 rows into ClickHouse +2026-06-15 11:56:18 | INFO | additional_visibility loaded successfully (1648 rows) +2026-06-15 11:56:18 | INFO | ================================================================================ +2026-06-15 11:56:18 | INFO | Processing Table: Coverage +2026-06-15 11:56:18 | INFO | Fetching coverage data for 571 MIDs +2026-06-15 11:56:20 | INFO | Fetched 514 rows from SQL Server +2026-06-15 11:56:20 | INFO | Fetched 514 rows +2026-06-15 11:56:20 | INFO | Creating table Coverage +2026-06-15 11:56:20 | INFO | Table ready: Coverage +2026-06-15 11:56:20 | INFO | Coverage: inserted 514 rows into ClickHouse +2026-06-15 11:56:20 | INFO | Coverage loaded successfully (514 rows) +2026-06-15 11:56:20 | INFO | ================================================================================ +2026-06-15 11:56:20 | INFO | Processing Table: Login +2026-06-15 11:56:20 | INFO | Fetching Login data for yesterday +2026-06-15 11:56:22 | INFO | Fetched 454 Login rows +2026-06-15 11:56:22 | INFO | Fetched 454 rows +2026-06-15 11:56:22 | INFO | Creating table Login +2026-06-15 11:56:22 | INFO | Table ready: Login +2026-06-15 11:56:23 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-15 11:56:23 | INFO | Login loaded successfully (454 rows) +2026-06-15 11:56:23 | INFO | ================================================================================ +2026-06-15 11:56:23 | INFO | Processing Table: Stock_Details +2026-06-15 11:56:23 | INFO | Fetching Stock Details data for 571 MIDs +2026-06-15 11:56:30 | INFO | Fetched 28,493 Stock Details rows +2026-06-15 11:56:30 | INFO | Fetched 28493 rows +2026-06-15 11:56:31 | INFO | Creating table Stock_Details +2026-06-15 11:56:31 | INFO | Table ready: Stock_Details +2026-06-15 11:56:31 | INFO | Stock_Details: inserted 28,493 rows into ClickHouse +2026-06-15 11:56:31 | INFO | Stock_Details loaded successfully (28493 rows) +2026-06-15 11:56:31 | INFO | ================================================================================ +2026-06-15 11:56:31 | INFO | Processing Table: Attendance +2026-06-15 11:56:31 | INFO | Fetching Attendance data from 2026-05-30 to 2026-06-14 +2026-06-15 11:56:46 | INFO | Fetched 11,543 attendance rows for 594 employees +2026-06-15 11:56:46 | INFO | Fetched 11543 rows +2026-06-15 11:56:47 | INFO | Creating table Attendance +2026-06-15 11:56:47 | INFO | Table ready: Attendance +2026-06-15 11:56:47 | INFO | Attendance: inserted 11,543 rows into ClickHouse +2026-06-15 11:56:47 | INFO | Attendance loaded successfully (11543 rows) +2026-06-15 11:56:47 | INFO | ================================================================================ +2026-06-15 11:56:47 | INFO | Processing Table: Store_Master +2026-06-15 11:56:47 | INFO | Fetching Store Master data +2026-06-15 11:57:01 | INFO | Fetched 5,984 stores +2026-06-15 11:57:01 | INFO | Fetched 5984 rows +2026-06-15 11:57:01 | INFO | Creating table Store_Master +2026-06-15 11:57:01 | INFO | Table ready: Store_Master +2026-06-15 11:57:02 | INFO | Store_Master: inserted 5,984 rows into ClickHouse +2026-06-15 11:57:02 | INFO | Store_Master loaded successfully (5984 rows) +2026-06-15 11:57:02 | INFO | ================================================================================ +2026-06-15 11:57:02 | INFO | Processing Table: SKU_Master +2026-06-15 11:57:02 | INFO | Fetching SKU Master data +2026-06-15 11:57:02 | INFO | Fetched 160 SKU Master rows +2026-06-15 11:57:02 | INFO | Fetched 160 rows +2026-06-15 11:57:02 | INFO | Creating table SKU_Master +2026-06-15 11:57:02 | INFO | Table ready: SKU_Master +2026-06-15 11:57:02 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-15 11:57:02 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-15 11:57:02 | INFO | ================================================================================ +2026-06-15 11:57:02 | INFO | Processing Table: display_master +2026-06-15 11:57:02 | INFO | Fetching Display Master data +2026-06-15 11:57:03 | INFO | Fetched 135 Display Master records +2026-06-15 11:57:03 | INFO | Fetched 135 rows +2026-06-15 11:57:03 | INFO | Creating table display_master +2026-06-15 11:57:03 | INFO | Table ready: display_master +2026-06-15 11:57:03 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-15 11:57:03 | INFO | display_master loaded successfully (135 rows) +2026-06-15 11:57:03 | INFO | ================================================================================ +2026-06-15 11:57:03 | INFO | Processing Table: Employee_Master +2026-06-15 11:57:03 | INFO | Fetching Employee Master data +2026-06-15 11:57:04 | INFO | Fetched 2,273 Employee Master records +2026-06-15 11:57:04 | INFO | Fetched 2273 rows +2026-06-15 11:57:04 | INFO | Creating table Employee_Master +2026-06-15 11:57:04 | INFO | Table ready: Employee_Master +2026-06-15 11:57:04 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse +2026-06-15 11:57:04 | INFO | Employee_Master loaded successfully (2273 rows) +2026-06-15 11:57:04 | INFO | ================================================================================ +2026-06-15 11:57:04 | INFO | Processing Table: Journey_Plan +2026-06-15 11:57:04 | INFO | Fetching Journey Plan for 2026-06 +2026-06-15 11:57:10 | INFO | Fetched 21,133 Journey Plan records +2026-06-15 11:57:10 | INFO | Fetched 21133 rows +2026-06-15 11:57:10 | INFO | Creating table Journey_Plan +2026-06-15 11:57:10 | INFO | Table ready: Journey_Plan +2026-06-15 11:57:10 | INFO | Journey_Plan: inserted 21,133 rows into ClickHouse +2026-06-15 11:57:10 | INFO | Journey_Plan loaded successfully (21133 rows) +2026-06-15 11:57:10 | INFO | ================================================================================ +2026-06-15 11:57:10 | INFO | Processing Table: coverage_remarks +2026-06-15 11:57:10 | INFO | Fetching Coverage Remarks +2026-06-15 11:57:10 | INFO | Fetched 29 Coverage Remark records +2026-06-15 11:57:10 | INFO | Fetched 29 rows +2026-06-15 11:57:10 | INFO | Creating table coverage_remarks +2026-06-15 11:57:10 | INFO | Table ready: coverage_remarks +2026-06-15 11:57:11 | INFO | coverage_remarks: inserted 29 rows into ClickHouse +2026-06-15 11:57:11 | INFO | coverage_remarks loaded successfully (29 rows) +2026-06-15 11:57:11 | INFO | ================================================================================ +2026-06-15 11:57:11 | INFO | Processing Table: mapping_storevisibility +2026-06-15 11:57:11 | INFO | Fetching Mapping Store Visibility for 2026-06-14 +2026-06-15 11:57:11 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-15 11:57:11 | WARNING | mapping_storevisibility returned no rows +2026-06-15 11:57:11 | INFO | ================================================================================ +2026-06-15 11:57:11 | INFO | Processing Table: Master_VisibilityReason +2026-06-15 11:57:11 | INFO | Fetching Master Visibility Reason data +2026-06-15 11:57:11 | INFO | Fetched 17 Master Visibility Reason records +2026-06-15 11:57:11 | INFO | Fetched 17 rows +2026-06-15 11:57:11 | INFO | Creating table Master_VisibilityReason +2026-06-15 11:57:11 | INFO | Table ready: Master_VisibilityReason +2026-06-15 11:57:12 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-15 11:57:12 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-15 11:57:12 | INFO | ================================================================================ +2026-06-15 11:57:12 | INFO | Processing Table: Master_VisibilityDefinition +2026-06-15 11:57:12 | INFO | Fetching Master Visibility Definition data +2026-06-15 11:57:12 | INFO | Fetched 861 Master Visibility Definition records +2026-06-15 11:57:12 | INFO | Fetched 861 rows +2026-06-15 11:57:12 | INFO | Creating table Master_VisibilityDefinition +2026-06-15 11:57:12 | INFO | Table ready: Master_VisibilityDefinition +2026-06-15 11:57:12 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-15 11:57:12 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-15 11:57:12 | INFO | ================================================================================ +2026-06-15 11:57:12 | INFO | Processing Table: Web_Logins +2026-06-15 11:57:12 | INFO | Fetching Web Login data for 2026-06-14 +2026-06-15 11:57:13 | INFO | Fetched 25 Web Login records +2026-06-15 11:57:13 | INFO | Fetched 25 rows +2026-06-15 11:57:13 | INFO | Creating table Web_Logins +2026-06-15 11:57:13 | INFO | Table ready: Web_Logins +2026-06-15 11:57:13 | INFO | Web_Logins: inserted 25 rows into ClickHouse +2026-06-15 11:57:13 | INFO | Web_Logins loaded successfully (25 rows) +2026-06-15 11:57:13 | INFO | ================================================================================ +2026-06-15 11:57:13 | INFO | Processing Table: Promotion +2026-06-15 11:57:13 | INFO | Fetching Promotion data for 571 MIDs +2026-06-15 11:57:34 | INFO | Fetched 4,426 Promotion records +2026-06-15 11:57:34 | INFO | Fetched 4426 rows +2026-06-15 11:57:34 | INFO | Creating table Promotion +2026-06-15 11:57:34 | INFO | Table ready: Promotion +2026-06-15 11:57:34 | INFO | Promotion: inserted 4,426 rows into ClickHouse +2026-06-15 11:57:34 | INFO | Promotion loaded successfully (4426 rows) +2026-06-15 11:57:34 | INFO | ================================================================================ +2026-06-15 11:57:34 | INFO | Processing Table: PaidVisibility +2026-06-15 11:57:34 | INFO | Fetching Paid Visibility data for 571 MIDs +2026-06-15 11:57:37 | INFO | Fetched 1,296 Paid Visibility records +2026-06-15 11:57:37 | INFO | Fetched 1296 rows +2026-06-15 11:57:37 | INFO | Creating table PaidVisibility +2026-06-15 11:57:37 | INFO | Table ready: PaidVisibility +2026-06-15 11:57:37 | INFO | PaidVisibility: inserted 1,296 rows into ClickHouse +2026-06-15 11:57:37 | INFO | PaidVisibility loaded successfully (1296 rows) +2026-06-15 11:57:37 | INFO | ================================================================================ +2026-06-15 11:57:37 | INFO | Processing Table: Master_Salesterritorylayer +2026-06-15 11:57:37 | INFO | Fetching Master Sales Territory Layer data +2026-06-15 11:57:38 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-15 11:57:38 | INFO | Fetched 33 rows +2026-06-15 11:57:38 | INFO | Creating table Master_Salesterritorylayer +2026-06-15 11:57:38 | INFO | Table ready: Master_Salesterritorylayer +2026-06-15 11:57:38 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-15 11:57:38 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-15 11:57:38 | INFO | ================================================================================ +2026-06-15 11:57:38 | INFO | Pipeline Completed Successfully +2026-06-15 11:57:38 | INFO | ================================================================================ +2026-06-15 11:58:05 | INFO | ================================================================================ +2026-06-15 11:58:05 | INFO | Hello from data-move Python data pipeline! +2026-06-15 11:58:05 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 11:58:05 | INFO | Connecting to databases... +2026-06-15 11:58:06 | INFO | +2026-06-15 11:58:07 | INFO | +2026-06-15 11:58:08 | INFO | Database connections established +2026-06-15 11:58:08 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 11:58:08 | INFO | Found 571 MIDs +2026-06-15 11:58:09 | INFO | ================================================================================ +2026-06-15 11:58:09 | INFO | Processing Table: SOS_OneApp +2026-06-15 11:58:09 | INFO | Fetching data for 571 MIDs +2026-06-15 11:58:09 | INFO | Fetched 1,995 rows from SQL Server +2026-06-15 11:58:09 | INFO | Fetched 1995 rows +2026-06-15 11:58:09 | INFO | Deleting from SOS_OneApp +2026-06-15 11:58:09 | INFO | SOS_OneApp: inserted 1,995 rows into ClickHouse +2026-06-15 11:58:09 | INFO | SOS_OneApp loaded successfully (1995 rows) +2026-06-15 11:58:09 | INFO | ================================================================================ +2026-06-15 11:58:09 | INFO | Processing Table: OQaD +2026-06-15 11:58:09 | INFO | Fetching OQaD data for run_date=2026-06-14 +2026-06-15 11:58:12 | INFO | Fetched 443 rows +2026-06-15 11:58:12 | INFO | Fetched 443 rows +2026-06-15 11:58:12 | INFO | Deleting from OQaD +2026-06-15 11:58:12 | INFO | OQaD: inserted 443 rows into ClickHouse +2026-06-15 11:58:12 | INFO | OQaD loaded successfully (443 rows) +2026-06-15 11:58:12 | INFO | ================================================================================ +2026-06-15 11:58:12 | INFO | Processing Table: Survey +2026-06-15 11:58:12 | INFO | Fetching Survey data for 571 MIDs +2026-06-15 11:58:12 | INFO | Fetched 173 Survey rows +2026-06-15 11:58:12 | INFO | Fetched 173 rows +2026-06-15 11:58:13 | INFO | Deleting from Survey +2026-06-15 11:58:13 | INFO | Survey: inserted 173 rows into ClickHouse +2026-06-15 11:58:13 | INFO | Survey loaded successfully (173 rows) +2026-06-15 11:58:13 | INFO | ================================================================================ +2026-06-15 11:58:13 | INFO | Processing Table: additional_visibility +2026-06-15 11:58:13 | INFO | Fetching Additional Visibility data for 571 MIDs +2026-06-15 11:58:13 | INFO | Fetched 1,648 Additional Visibility rows +2026-06-15 11:58:13 | INFO | Fetched 1648 rows +2026-06-15 11:58:13 | INFO | Deleting from additional_visibility +2026-06-15 11:58:14 | INFO | additional_visibility: inserted 1,648 rows into ClickHouse +2026-06-15 11:58:14 | INFO | additional_visibility loaded successfully (1648 rows) +2026-06-15 11:58:14 | INFO | ================================================================================ +2026-06-15 11:58:14 | INFO | Processing Table: Coverage +2026-06-15 11:58:14 | INFO | Fetching coverage data for 571 MIDs +2026-06-15 11:58:14 | INFO | Fetched 514 rows from SQL Server +2026-06-15 11:58:14 | INFO | Fetched 514 rows +2026-06-15 11:58:14 | INFO | Deleting from Coverage +2026-06-15 11:58:14 | INFO | Coverage: inserted 514 rows into ClickHouse +2026-06-15 11:58:14 | INFO | Coverage loaded successfully (514 rows) +2026-06-15 11:58:14 | INFO | ================================================================================ +2026-06-15 11:58:14 | INFO | Processing Table: Login +2026-06-15 11:58:14 | INFO | Fetching Login data for yesterday +2026-06-15 11:58:15 | INFO | Fetched 454 Login rows +2026-06-15 11:58:15 | INFO | Fetched 454 rows +2026-06-15 11:58:15 | INFO | Deleting from Login +2026-06-15 11:58:15 | ERROR | Failed processing table Login +Traceback (most recent call last): + File "D:\data_move\test2.py", line 236, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 83, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + table_name, + ^^^^^^^^^^^ + f"MID IN ({mids_str})", + ^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'MID' while processing: 'isZeroOrNull(MID IN (1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650)), _database, _part_data_version, _part_granule_offset, _part_offset, _disk_name, _sample_factor, _block_offset, _table, login_date, _part_uuid, _part_starting_offset, first_store_in_time, _part_index, _part, unique_id, _block_number, login_time, _partition_id, last_store_out_time, _distance, employee_id', required columns: '_part_data_version' '_database' '_part_offset' '_part_granule_offset' 'MID' '_disk_name' '_sample_factor' 'login_date' '_block_offset' '_table' '_part_uuid' '_part' '_part_index' '_part_starting_offset' 'first_store_in_time' 'login_time' '_block_number' 'unique_id' '_partition_id' 'employee_id' '_distance' 'last_store_out_time', maybe you meant: 'login_date', 'first_store_in_time', 'login_time', 'unique_id', 'employee_id' or 'last_store_out_time'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) +2026-06-15 12:03:38 | INFO | ================================================================================ +2026-06-15 12:03:38 | INFO | Hello from data-move Python data pipeline! +2026-06-15 12:03:38 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 12:03:38 | INFO | Connecting to databases... +2026-06-15 12:03:39 | INFO | +2026-06-15 12:03:40 | INFO | +2026-06-15 12:03:41 | INFO | Database connections established +2026-06-15 12:03:41 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 12:03:41 | INFO | Found 571 MIDs +2026-06-15 12:03:41 | INFO | ================================================================================ +2026-06-15 12:03:41 | INFO | Processing Table: SOS_OneApp +2026-06-15 12:03:41 | INFO | Fetching data for 571 MIDs +2026-06-15 12:03:42 | INFO | Fetched 1,995 rows from SQL Server +2026-06-15 12:03:42 | INFO | Fetched 1995 rows +2026-06-15 12:03:42 | INFO | Deleting from SOS_OneApp +2026-06-15 12:03:42 | INFO | SOS_OneApp: inserted 1,995 rows into ClickHouse +2026-06-15 12:03:42 | INFO | SOS_OneApp loaded successfully (1995 rows) +2026-06-15 12:03:42 | INFO | ================================================================================ +2026-06-15 12:03:42 | INFO | Processing Table: OQaD +2026-06-15 12:03:42 | INFO | Fetching OQaD data for run_date=2026-06-14 +2026-06-15 12:03:45 | INFO | Fetched 443 rows +2026-06-15 12:03:45 | INFO | Fetched 443 rows +2026-06-15 12:03:45 | INFO | Deleting from OQaD +2026-06-15 12:03:45 | INFO | OQaD: inserted 443 rows into ClickHouse +2026-06-15 12:03:45 | INFO | OQaD loaded successfully (443 rows) +2026-06-15 12:03:45 | INFO | ================================================================================ +2026-06-15 12:03:45 | INFO | Processing Table: Survey +2026-06-15 12:03:45 | INFO | Fetching Survey data for 571 MIDs +2026-06-15 12:03:45 | INFO | Fetched 173 Survey rows +2026-06-15 12:03:45 | INFO | Fetched 173 rows +2026-06-15 12:03:45 | INFO | Deleting from Survey +2026-06-15 12:03:45 | INFO | Survey: inserted 173 rows into ClickHouse +2026-06-15 12:03:45 | INFO | Survey loaded successfully (173 rows) +2026-06-15 12:03:45 | INFO | ================================================================================ +2026-06-15 12:03:45 | INFO | Processing Table: additional_visibility +2026-06-15 12:03:45 | INFO | Fetching Additional Visibility data for 571 MIDs +2026-06-15 12:03:46 | INFO | Fetched 1,648 Additional Visibility rows +2026-06-15 12:03:46 | INFO | Fetched 1648 rows +2026-06-15 12:03:46 | INFO | Deleting from additional_visibility +2026-06-15 12:03:46 | INFO | additional_visibility: inserted 1,648 rows into ClickHouse +2026-06-15 12:03:46 | INFO | additional_visibility loaded successfully (1648 rows) +2026-06-15 12:03:46 | INFO | ================================================================================ +2026-06-15 12:03:46 | INFO | Processing Table: Coverage +2026-06-15 12:03:46 | INFO | Fetching coverage data for 571 MIDs +2026-06-15 12:03:46 | INFO | Fetched 514 rows from SQL Server +2026-06-15 12:03:46 | INFO | Fetched 514 rows +2026-06-15 12:03:46 | INFO | Deleting from Coverage +2026-06-15 12:03:46 | INFO | Coverage: inserted 514 rows into ClickHouse +2026-06-15 12:03:46 | INFO | Coverage loaded successfully (514 rows) +2026-06-15 12:03:46 | INFO | ================================================================================ +2026-06-15 12:03:46 | INFO | Processing Table: Login +2026-06-15 12:03:46 | INFO | Fetching Login data for yesterday +2026-06-15 12:03:47 | INFO | Fetched 454 Login rows +2026-06-15 12:03:47 | INFO | Fetched 454 rows +2026-06-15 12:03:47 | INFO | Deleting from Login +2026-06-15 12:03:47 | ERROR | Failed processing table Login +Traceback (most recent call last): + File "D:\data_move\test2.py", line 236, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 83, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + table_name, + ^^^^^^^^^^^ + f"MID IN ({mids_str})", + ^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'MID' while processing: 'isZeroOrNull(MID IN (1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650)), _database, _part_data_version, _part_granule_offset, _part_offset, _disk_name, _sample_factor, _block_offset, _table, login_date, _part_uuid, _part_starting_offset, first_store_in_time, _part_index, _part, unique_id, _block_number, login_time, _partition_id, last_store_out_time, _distance, employee_id', required columns: '_part_data_version' '_database' '_part_offset' '_part_granule_offset' 'MID' '_disk_name' '_sample_factor' 'login_date' '_block_offset' '_table' '_part_uuid' '_part' '_part_index' '_part_starting_offset' 'first_store_in_time' 'login_time' '_block_number' 'unique_id' '_partition_id' 'employee_id' '_distance' 'last_store_out_time', maybe you meant: 'login_date', 'first_store_in_time', 'login_time', 'unique_id', 'employee_id' or 'last_store_out_time'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) diff --git a/main.py b/main.py index bf402e7..e1e6aa1 100644 --- a/main.py +++ b/main.py @@ -71,7 +71,50 @@ def main(): log.info("Both databases connected successfully") - mids=collect_mids(sql_engine , run_date) + mids = MID_TABLE_COV(sql_engine, run_date) + + emp_visit_df = MID_TABLE_COV1( + sql_engine, + run_date + ) + + delete_existing_data( + client=client, + run_date=run_date, + mids=mids, + emp_visit_df=emp_visit_df, + ) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + mid_list = ",".join(map(str, mids)) conditions = { "mids": f"MID IN ({mid_list})", diff --git a/masters/dimensions.py b/masters/dimensions.py index 5095fc8..9f950d6 100644 --- a/masters/dimensions.py +++ b/masters/dimensions.py @@ -30,7 +30,7 @@ p=40148 def fetch_Store_Master(engine: Engine) -> pl.DataFrame: sql = """ SELECT - RegionId AS region_id, + RegionName AS region, StateId AS state_id, StateName AS state, diff --git a/mids.py b/mids.py index f7aa1c9..25760c2 100644 --- a/mids.py +++ b/mids.py @@ -19,7 +19,7 @@ from db_con.connection import * -def collect_mids(engine: Engine, target_date: date) -> list[int]: +def MID_TABLE_COV(engine: Engine, target_date: date) -> list[int]: sql = text(""" SELECT MID FROM OneApp_KelloggsMT.dbo.T_StoreCoverage @@ -30,9 +30,35 @@ def collect_mids(engine: Engine, target_date: date) -> list[int]: """) log.info(f"Collecting MIDs for: {target_date}") - + with engine.connect() as conn: result = conn.execute(sql, {"target_date": target_date}) mids = [row[0] for row in result.fetchall()] log.info(f"Found {len(mids):,} MIDs") - return mids \ No newline at end of file + return mids + +def MID_TABLE_COV1( + engine: Engine, + target_date: date, +) -> pl.DataFrame: + + query = f""" + SELECT + EmpId, + CAST(VisitDate AS DATE) AS VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CAST(CreateDate AS DATE) = '{target_date}' + + UNION + + SELECT + EmpId, + CAST(VisitDate AS DATE) AS VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CAST(UpdateDate AS DATE) = '{target_date}' + """ + + return pl.read_database( + query=query, + connection=engine, + ) \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 7c2e9b5..f894d7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "clickhouse-sqlalchemy>=0.3.2", "dotenv>=0.9.9", "polars>=1.41.2", + "pyarrow>=24.0.0", "pyodbc>=5.3.0", "pyyaml>=6.0.3", ] diff --git a/tables.yml b/tables.yml index a13e628..0453b80 100644 --- a/tables.yml +++ b/tables.yml @@ -8,7 +8,7 @@ tables: - name: OQaD type: FACT operation: INSERT - fetch_by: mids + fetch_by: run_date condition: mids - name: Survey diff --git a/test.py b/test.py index 7e0d556..e4fa967 100644 --- a/test.py +++ b/test.py @@ -1,41 +1,267 @@ +from __future__ import annotations + +import sys +from datetime import date, datetime, timedelta + +import polars as pl import yaml -from datetime import date, timedelta -mids_list=[1,3] +from log import log -run_date=date.today() -conditions = { - "mids": f"MID IN ({','.join(map(str, mids_list))})", +from clickhouse_task.create_table import create_clickhouse_table +from clickhouse_task.delete_task import delete_existing_data , truncate_table +from clickhouse_task.load_table import load_to_clickhouse - "j_plan": ( - f"MONTH(VisitDate) = {run_date.month} " - f"AND YEAR(VisitDate) = {run_date.year}" - ), +from db_con.connection import ( + build_sql_server_engine, + build_clickhouse_engine, + get_clickhouse_client, +) - "mapping": ( - f"CAST(Z.FromDate AS DATE) <= '{run_date}' " - f"AND CAST(Z.ToDate AS DATE) >= '{run_date}'" - ), +from mids import MID_TABLE_COV, MID_TABLE_COV1 - "web": ( - f"CAST(login_date AS DATE) = '{run_date}'" - ), - - "none": None, -} +from masters.dimensions import * +from masters.bridge import * +from kpi.facts import * -with open("tables.yml", "r") as file: - config = yaml.safe_load(file) +# ============================================================ +# Helpers +# ============================================================ -for table in config["tables"]: - - table_name=table["name"] - table_type=table["type"] - operation=table["operation"] - condition=table["condition"] - c = conditions[condition] - print(table_name) - print(table_type) - print(operation) - print(c) \ No newline at end of file +def get_dataframe( + fn_name: str, + fetch_by: str, + sql_engine, + mids, + run_date, +) -> pl.DataFrame: + + fn = globals()[fn_name] + + if fetch_by == "mids": + return fn(sql_engine, mids) + + if fetch_by == "run_date": + return fn(sql_engine, run_date) + + return fn(sql_engine) + + +def ensure_table_exists( + client, + clickhouse_engine, + table_name: str, + df: pl.DataFrame, +) -> bool: + + exists = client.command( + f"EXISTS TABLE {table_name}" + ) + + if not exists: + log.info( + "Creating ClickHouse table: %s", + table_name, + ) + + create_clickhouse_table( + df=df, + table_name=table_name, + clickhouse_engine=clickhouse_engine, + ) + + return bool(exists) + + +# ============================================================ +# Main +# ============================================================ + +def main(): + + log.info("=" * 80) + log.info("Hello from data-move Python data pipeline !") + + # -------------------------------------------------------- + # Run Date + # -------------------------------------------------------- + + if len(sys.argv) > 1: + run_date = datetime.strptime( + sys.argv[1], + "%Y-%m-%d", + ).date() + else: + run_date = date.today() - timedelta(days=1) + + log.info( + "Pipeline Run Date : %s", + run_date, + ) + + # -------------------------------------------------------- + # Connections + # -------------------------------------------------------- + + log.info( + "Connecting to SQL Server and ClickHouse" + ) + + sql_engine = build_sql_server_engine() + + clickhouse_engine = ( + build_clickhouse_engine() + ) + + client = get_clickhouse_client() + + log.info( + "Database connections established" + ) + + # -------------------------------------------------------- + # Collect Delete Keys + # -------------------------------------------------------- + + mids = MID_TABLE_COV( + sql_engine, + run_date, + ) + + emp_visit_df = MID_TABLE_COV1( + sql_engine, + run_date, + ) + + # -------------------------------------------------------- + # Delete Existing Data + # -------------------------------------------------------- + + delete_existing_data( + client=client, + run_date=run_date, + mids=mids, + emp_visit_df=emp_visit_df, + ) + + # -------------------------------------------------------- + # Load Config + # -------------------------------------------------------- + + with open( + "tables.yml", + "r", + ) as file: + + config = yaml.safe_load(file) + + # -------------------------------------------------------- + # Process Tables + # -------------------------------------------------------- + + for table in config["tables"]: + + table_name = table["name"] + table_type = table["type"] + operation = table["operation"] + fetch_by = table["fetch_by"] + + log.info("=" * 80) + + log.info( + "TABLE=%s | TYPE=%s | OPERATION=%s", + table_name, + table_type, + operation, + ) + + try: + + # -------------------------------------------- + # Fetch + # -------------------------------------------- + + fn_name = f"fetch_{table_name}" + + df = get_dataframe( + fn_name=fn_name, + fetch_by=fetch_by, + sql_engine=sql_engine, + mids=mids, + run_date=run_date, + ) + + if df.is_empty(): + + log.warning( + "%s returned no rows", + table_name, + ) + + continue + + log.info( + "Fetched %s rows", + len(df), + ) + + # -------------------------------------------- + # Create Table if Missing + # -------------------------------------------- + + exists = ensure_table_exists( + client=client, + clickhouse_engine=clickhouse_engine, + table_name=table_name, + df=df, + ) + + # -------------------------------------------- + # Full Refresh Tables + # -------------------------------------------- + + if exists and operation == "DELETE+INSERT": + + truncate_table( + client, + table_name, + ) + + log.info( + "Truncated %s", + table_name, + ) + + # -------------------------------------------- + # Load + # -------------------------------------------- + + load_to_clickhouse( + client=client, + table_name=table_name, + df=df, + ) + + log.info( + "%s loaded successfully (%s rows)", + table_name, + len(df), + ) + + except Exception as e: + + log.exception( + "Failed processing table %s", + table_name, + ) + + raise + + log.info("=" * 80) + log.info("Pipeline Completed Successfully") + log.info("=" * 80) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/test2.py b/test2.py new file mode 100644 index 0000000..720d2a5 --- /dev/null +++ b/test2.py @@ -0,0 +1,275 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "polars>=0.20.0", +# "pyarrow>=18.0.0", +# "sqlalchemy>=2.0.0", +# "pyodbc>=5.0.0", +# "clickhouse-connect>=0.7.0", +# "clickhouse-sqlalchemy>=0.3.2", +# "pyyaml>=6.0.3", +# "python-dotenv>=1.0.0", +# ] +# /// + + +from __future__ import annotations + +import sys +from datetime import date, datetime, timedelta + +import polars as pl +import yaml + + +from log import log + +from clickhouse_task.create_table import create_clickhouse_table +from clickhouse_task.delete_task import ( + delete_existing_data, + truncate_table, +) + +from clickhouse_task.load_table import load_to_clickhouse + +from db_con.connection import ( + build_sql_server_engine, + build_clickhouse_engine, + get_clickhouse_client, +) + +from mids import ( + MID_TABLE_COV, + MID_TABLE_COV1, +) + +from masters.dimensions import * +from masters.bridge import * +from kpi.facts import * + + +# ========================================================== +# Helpers +# ========================================================== + +def table_exists( + client, + table_name: str, +) -> bool: + + return bool( + client.command( + f"EXISTS TABLE {table_name}" + ) + ) + + +def get_dataframe( + fn_name: str, + fetch_by: str, + sql_engine, + mids, + run_date, +): + + fn = globals()[fn_name] + + if fetch_by == "mids": + return fn(sql_engine, mids) + + if fetch_by == "run_date": + return fn(sql_engine, run_date) + + return fn(sql_engine) + + +# ========================================================== +# Main +# ========================================================== + +def main(): + + log.info("=" * 80) + log.info("Hello from data-move Python data pipeline!") + + # ------------------------------------------------------ + # Run Date + # ------------------------------------------------------ + + if len(sys.argv) > 1: + run_date = datetime.strptime( + sys.argv[1], + "%Y-%m-%d", + ).date() + else: + run_date = date.today() - timedelta(days=1) + + log.info( + "Pipeline Run Date: %s", + run_date, + ) + + # ------------------------------------------------------ + # Connections + # ------------------------------------------------------ + + log.info( + "Connecting to databases..." + ) + + sql_engine = build_sql_server_engine() + clickhouse_engine = build_clickhouse_engine() + client = get_clickhouse_client() + + log.info( + "Database connections established" + ) + + # ------------------------------------------------------ + # Delete Keys + # ------------------------------------------------------ + + mids = MID_TABLE_COV( + sql_engine, + run_date, + ) + + emp_visit_df = MID_TABLE_COV1( + sql_engine, + run_date, + ) + + # ------------------------------------------------------ + # Config + # ------------------------------------------------------ + + with open( + "tables.yml", + "r", + ) as file: + + config = yaml.safe_load(file) + + # ------------------------------------------------------ + # Process Tables + # ------------------------------------------------------ + + for table in config["tables"]: + + table_name = table["name"] + operation = table["operation"] + fetch_by = table["fetch_by"] + + log.info("=" * 80) + log.info( + "Processing Table: %s", + table_name, + ) + + try: + + # ------------------------------------------ + # Fetch Data + # ------------------------------------------ + + fn_name = f"fetch_{table_name}" + + df = get_dataframe( + fn_name=fn_name, + fetch_by=fetch_by, + sql_engine=sql_engine, + mids=mids, + run_date=run_date, + ) + + if df.is_empty(): + + log.warning( + "%s returned no rows", + table_name, + ) + + continue + + log.info( + "Fetched %s rows", + len(df), + ) + + # ------------------------------------------ + # Create Table If Missing + # ------------------------------------------ + + exists = table_exists( + client, + table_name, + ) + + if not exists: + + log.info( + "Creating table %s", + table_name, + ) + + create_clickhouse_table( + df=df, + table_name=table_name, + clickhouse_engine=clickhouse_engine, + ) + + # ------------------------------------------ + # Existing Table Logic + # ------------------------------------------ + + else: + + if operation == "DELETE+INSERT": + + truncate_table( + client, + table_name, + ) + + else: + + delete_existing_data( + client=client, + table_name=table_name, + run_date=run_date, + mids=mids, + emp_visit_df=emp_visit_df, + ) + + # ------------------------------------------ + # Load Data + # ------------------------------------------ + + load_to_clickhouse( + client=client, + table_name=table_name, + df=df, + ) + + log.info( + "%s loaded successfully (%s rows)", + table_name, + len(df), + ) + + except Exception: + + log.exception( + "Failed processing table %s", + table_name, + ) + + raise + + log.info("=" * 80) + log.info("Pipeline Completed Successfully") + log.info("=" * 80) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/uv.lock b/uv.lock index 40eedd6..b3e1319 100644 --- a/uv.lock +++ b/uv.lock @@ -163,6 +163,7 @@ dependencies = [ { name = "clickhouse-sqlalchemy" }, { name = "dotenv" }, { name = "polars" }, + { name = "pyarrow" }, { name = "pyodbc" }, { name = "pyyaml" }, ] @@ -173,6 +174,7 @@ requires-dist = [ { name = "clickhouse-sqlalchemy", specifier = ">=0.3.2" }, { name = "dotenv", specifier = ">=0.9.9" }, { name = "polars", specifier = ">=1.41.2" }, + { name = "pyarrow", specifier = ">=24.0.0" }, { name = "pyodbc", specifier = ">=5.3.0" }, { name = "pyyaml", specifier = ">=6.0.3" }, ] @@ -289,6 +291,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cd/c6/79e9f3f270270d7ed5575d92b7bfef49f01abd9275447161275b23b553a8/polars_runtime_32-1.41.2-cp310-abi3-win_arm64.whl", hash = "sha256:843d96f69d18eca53429c1198e58891db7f18111f83b9c419bb45ad9d73eaed5", size = 46006901, upload-time = "2026-05-29T17:37:42.522Z" }, ] +[[package]] +name = "pyarrow" +version = "24.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" }, + { url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" }, + { url = "https://files.pythonhosted.org/packages/29/88/f4e9145da0417b3d2c12035a8492b35ff4a3dbc653e614fcfb51d9dedb38/pyarrow-24.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:38be1808cdd068605b787e6ca9119b27eb275a0234e50212c3492331680c3b1e", size = 28001155, upload-time = "2026-04-21T10:51:22.337Z" }, + { url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" }, + { url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" }, + { url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" }, + { url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" }, + { url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" }, + { url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" }, + { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" }, +] + [[package]] name = "pyodbc" version = "5.3.0"