From e77941c23d5b090de7d68ad33fc5baee8937cb33 Mon Sep 17 00:00:00 2001 From: Ankit Malik Date: Mon, 15 Jun 2026 13:52:24 +0530 Subject: [PATCH] 14-06-2026 2nd commit --- clickhouse_task/delete_task.py | 5 +- db_con/log.py | 24 + kpi/facts.py | 59 +- logs/etl_20260615.log | 2480 ++++++++++++++++++++++++++++++++ main.py | 366 ++--- masters/dimensions.py | 29 + t.yml | 105 ++ tables.yml | 6 +- test.py | 267 ---- test2.py | 275 ---- 10 files changed, 2844 insertions(+), 772 deletions(-) create mode 100644 db_con/log.py create mode 100644 t.yml delete mode 100644 test.py delete mode 100644 test2.py diff --git a/clickhouse_task/delete_task.py b/clickhouse_task/delete_task.py index a9f6a99..dde7e58 100644 --- a/clickhouse_task/delete_task.py +++ b/clickhouse_task/delete_task.py @@ -70,8 +70,6 @@ def delete_existing_data( "PaidVisibility", "SOS_OneApp", "Stock_Details", - "Login", - "coverage_remarks", } if table_name in mid_tables and mids: @@ -113,8 +111,7 @@ def delete_existing_data( client, table_name, f""" - project_id = 40148 - AND toDate(login_date) = toDate('{run_date}') + toDate(login_date) = toDate('{run_date}') """ ) diff --git a/db_con/log.py b/db_con/log.py new file mode 100644 index 0000000..11f11e0 --- /dev/null +++ b/db_con/log.py @@ -0,0 +1,24 @@ + +from pathlib import Path +from datetime import datetime +import logging + +# Create logs folder if it doesn't exist +Path("logs").mkdir(exist_ok=True) + +# Daily log file +log_file = Path("logs") / f"etl_{datetime.now():%Y%m%d}.log" + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)-8s | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + handlers=[ + logging.StreamHandler(), + logging.FileHandler(log_file, encoding="utf-8"), + ], +) + +# Export logger +log = logging.getLogger("etl") \ No newline at end of file diff --git a/kpi/facts.py b/kpi/facts.py index 58b6581..e12fe5f 100644 --- a/kpi/facts.py +++ b/kpi/facts.py @@ -807,35 +807,6 @@ def fetch_Journey_Plan( -def fetch_coverage_remarks(engine: Engine , mids: list[int]) -> pl.DataFrame: - """ - Source: - OneApp_KelloggsMT.dbo.Master_NonWorkingReason - - Target: - coverage_remarks - """ - - sql = """ - SELECT - 40148 AS project_id, - ReasonId AS reason_id, - Reason AS reason_remarks - FROM OneApp_KelloggsMT.dbo.Master_NonWorkingReason - """ - - log.info("Fetching Coverage Remarks") - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info( - f"Fetched {len(df):,} Coverage Remark records" - ) - - return df def fetch_Web_Logins( @@ -851,56 +822,40 @@ def fetch_Web_Logins( """ sql = f""" - SELECT DISTINCT + SELECT DISTINCT 40148 AS project_id, - EM1.Id AS supervisor_id, EM1.EmployeeName AS supervisor_name, - EM.Id AS emp_id, EM.EmployeeName AS employee_name, - DM.DesignationName AS designation, - - CAST(AL.Date AS DATE) AS login_date, - CONVERT(VARCHAR, AL.Date, 108) AS login_time, - + CAST(AL.Date AS DATE) AS date, + CONVERT(VARCHAR(8), AL.Date, 108) AS time, AL.Thread AS activity_name, AL.Level AS activity_type, - R.RightName AS right_name, - - GETDATE() AS create_date, - 'Pius' AS create_by - + GETDATE() AS CreateDate, + 'Pius' AS CreateBy FROM OneApp_KelloggsMT.dbo.T_User_Activity_Log AL - LEFT JOIN OneApp_KelloggsMT.dbo.AspNetUsers EM - ON AL.Logger = EM.UserName - + ON AL.Logger = EM.USERNAME INNER JOIN OneApp_KelloggsMT.dbo.AspNetUsers EM1 ON EM1.Id = EM.ManagerId - INNER JOIN OneApp_KelloggsMT.dbo.AspNetUsers EM2 ON EM2.Id = EM1.ManagerId - INNER JOIN OneApp_KelloggsMT.dbo.Master_Designation DM ON EM.DesignationId = DM.DesignationId - INNER JOIN OneApp_KelloggsMT.dbo.Master_City CM ON EM.CityId = CM.CityId - INNER JOIN OneApp_KelloggsMT.dbo.Master_State ST ON ST.StateId = CM.StateId - INNER JOIN OneApp_KelloggsMT.dbo.Master_Region RM ON RM.RegionId = ST.RegionId - INNER JOIN OneApp_KelloggsMT.dbo.Right_Master R ON EM.RightId = R.RightId - WHERE CAST(AL.Date AS DATE) = '{run_date}' """ + log.info(f"Fetching Web Login data for {run_date}") diff --git a/logs/etl_20260615.log b/logs/etl_20260615.log index 7b4d799..d3342e2 100644 --- a/logs/etl_20260615.log +++ b/logs/etl_20260615.log @@ -1101,3 +1101,2483 @@ Traceback (most recent call last): File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler raise err_type(err_str, code=code, name=name) from None clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'MID' while processing: 'isZeroOrNull(MID IN (1055918, 1057103, 1057202, 1057487, 1057606, 1058004, 1058337, 1058351, 1058352, 1058353, 1058423, 1058433, 1058456, 1058499, 1058500, 1058612, 1058627, 1058639, 1058640, 1058650, 1058726, 1058777, 1058782, 1058835, 1058845, 1058870, 1058967, 1059022, 1059036, 1059057, 1059058, 1059069, 1059097, 1059113, 1059114, 1059115, 1059116, 1059117, 1059118, 1059119, 1059120, 1059121, 1059122, 1059123, 1059124, 1059125, 1059126, 1059127, 1059128, 1059129, 1059130, 1059131, 1059132, 1059133, 1059134, 1059135, 1059136, 1059137, 1059138, 1059139, 1059140, 1059141, 1059142, 1059143, 1059144, 1059145, 1059146, 1059147, 1059148, 1059149, 1059150, 1059151, 1059152, 1059153, 1059154, 1059155, 1059156, 1059157, 1059158, 1059159, 1059160, 1059161, 1059162, 1059163, 1059164, 1059165, 1059166, 1059167, 1059168, 1059169, 1059170, 1059171, 1059172, 1059173, 1059174, 1059175, 1059176, 1059177, 1059178, 1059179, 1059180, 1059181, 1059182, 1059183, 1059184, 1059185, 1059186, 1059187, 1059188, 1059189, 1059190, 1059191, 1059192, 1059193, 1059194, 1059195, 1059196, 1059197, 1059198, 1059199, 1059200, 1059201, 1059202, 1059203, 1059204, 1059205, 1059206, 1059207, 1059208, 1059209, 1059210, 1059211, 1059212, 1059213, 1059214, 1059215, 1059216, 1059217, 1059218, 1059219, 1059220, 1059221, 1059222, 1059223, 1059224, 1059225, 1059226, 1059227, 1059228, 1059229, 1059230, 1059231, 1059232, 1059233, 1059234, 1059235, 1059236, 1059237, 1059238, 1059239, 1059240, 1059241, 1059242, 1059243, 1059244, 1059245, 1059246, 1059247, 1059248, 1059249, 1059250, 1059251, 1059252, 1059253, 1059254, 1059255, 1059256, 1059257, 1059258, 1059259, 1059260, 1059261, 1059262, 1059263, 1059264, 1059265, 1059266, 1059267, 1059268, 1059269, 1059270, 1059271, 1059272, 1059273, 1059274, 1059275, 1059276, 1059277, 1059278, 1059279, 1059280, 1059281, 1059282, 1059283, 1059284, 1059285, 1059286, 1059287, 1059288, 1059289, 1059290, 1059291, 1059292, 1059293, 1059294, 1059295, 1059296, 1059297, 1059298, 1059299, 1059300, 1059301, 1059302, 1059303, 1059304, 1059305, 1059306, 1059307, 1059308, 1059309, 1059310, 1059311, 1059312, 1059313, 1059314, 1059315, 1059316, 1059317, 1059318, 1059319, 1059320, 1059321, 1059322, 1059323, 1059324, 1059325, 1059326, 1059327, 1059328, 1059329, 1059330, 1059331, 1059332, 1059333, 1059334, 1059335, 1059336, 1059337, 1059338, 1059339, 1059340, 1059341, 1059342, 1059343, 1059344, 1059345, 1059346, 1059347, 1059348, 1059349, 1059350, 1059351, 1059352, 1059353, 1059354, 1059355, 1059356, 1059357, 1059358, 1059359, 1059360, 1059361, 1059362, 1059363, 1059364, 1059365, 1059366, 1059367, 1059368, 1059369, 1059370, 1059371, 1059372, 1059373, 1059374, 1059375, 1059376, 1059377, 1059378, 1059379, 1059380, 1059381, 1059382, 1059383, 1059384, 1059385, 1059386, 1059387, 1059388, 1059389, 1059390, 1059391, 1059392, 1059393, 1059394, 1059395, 1059396, 1059397, 1059398, 1059399, 1059400, 1059401, 1059402, 1059403, 1059404, 1059405, 1059406, 1059407, 1059408, 1059409, 1059410, 1059411, 1059412, 1059413, 1059414, 1059415, 1059416, 1059417, 1059418, 1059419, 1059420, 1059421, 1059422, 1059423, 1059424, 1059425, 1059426, 1059427, 1059428, 1059429, 1059430, 1059431, 1059432, 1059433, 1059434, 1059435, 1059436, 1059437, 1059438, 1059439, 1059440, 1059441, 1059442, 1059443, 1059444, 1059445, 1059446, 1059447, 1059448, 1059449, 1059450, 1059451, 1059452, 1059453, 1059454, 1059455, 1059456, 1059457, 1059458, 1059459, 1059460, 1059461, 1059462, 1059463, 1059464, 1059465, 1059466, 1059467, 1059468, 1059469, 1059470, 1059471, 1059472, 1059473, 1059474, 1059475, 1059476, 1059477, 1059478, 1059479, 1059480, 1059481, 1059482, 1059483, 1059484, 1059485, 1059486, 1059487, 1059488, 1059489, 1059490, 1059491, 1059492, 1059493, 1059494, 1059495, 1059496, 1059497, 1059498, 1059499, 1059500, 1059501, 1059502, 1059503, 1059504, 1059505, 1059506, 1059507, 1059508, 1059509, 1059510, 1059511, 1059512, 1059513, 1059514, 1059515, 1059516, 1059517, 1059518, 1059519, 1059520, 1059521, 1059522, 1059523, 1059524, 1059525, 1059526, 1059527, 1059528, 1059529, 1059530, 1059531, 1059532, 1059533, 1059534, 1059535, 1059536, 1059537, 1059538, 1059539, 1059540, 1059541, 1059542, 1059543, 1059544, 1059545, 1059546, 1059547, 1059548, 1059549, 1059550, 1059551, 1059552, 1059553, 1059554, 1059555, 1059556, 1059557, 1059558, 1059559, 1059560, 1059561, 1059562, 1059563, 1059564, 1059565, 1059566, 1059567, 1059568, 1059569, 1059570, 1059571, 1059572, 1059573, 1059574, 1059575, 1059576, 1059577, 1059578, 1059579, 1059580, 1059581, 1059582, 1059583, 1059584, 1059585, 1059586, 1059587, 1059588, 1059589, 1059590, 1059591, 1059592, 1059593, 1059594, 1059595, 1059596, 1059597, 1059598, 1059599, 1059600, 1059601, 1059602, 1059603, 1059604, 1059605, 1059606, 1059607, 1059608, 1059609, 1059610, 1059611, 1059612, 1059613, 1059614, 1059615, 1059616, 1059617, 1059618, 1059619, 1059620, 1059621, 1059622, 1059623, 1059624, 1059625, 1059626, 1059627, 1059628, 1059629, 1059630, 1059631, 1059632, 1059633, 1059634, 1059635, 1059636, 1059637, 1059638, 1059639, 1059640, 1059641, 1059642, 1059643, 1059644, 1059645, 1059646, 1059647, 1059648, 1059649, 1059650)), _database, _part_data_version, _part_granule_offset, _part_offset, _disk_name, _sample_factor, _block_offset, _table, login_date, _part_uuid, _part_starting_offset, first_store_in_time, _part_index, _part, unique_id, _block_number, login_time, _partition_id, last_store_out_time, _distance, employee_id', required columns: '_part_data_version' '_database' '_part_offset' '_part_granule_offset' 'MID' '_disk_name' '_sample_factor' 'login_date' '_block_offset' '_table' '_part_uuid' '_part' '_part_index' '_part_starting_offset' 'first_store_in_time' 'login_time' '_block_number' 'unique_id' '_partition_id' 'employee_id' '_distance' 'last_store_out_time', maybe you meant: 'login_date', 'first_store_in_time', 'login_time', 'unique_id', 'employee_id' or 'last_store_out_time'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) +2026-06-15 12:53:14 | INFO | ================================================================================ +2026-06-15 12:53:14 | INFO | Hello from data-move Python data pipeline! +2026-06-15 12:53:14 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 12:53:14 | INFO | Connecting to databases... +2026-06-15 12:53:17 | INFO | +2026-06-15 12:53:18 | INFO | +2026-06-15 12:53:19 | INFO | Database connections established +2026-06-15 12:53:19 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 12:53:21 | INFO | Found 571 MIDs +2026-06-15 12:53:21 | INFO | ================================================================================ +2026-06-15 12:53:21 | INFO | Processing Table: SOS_OneApp +2026-06-15 12:53:21 | INFO | Fetching data for 571 MIDs +2026-06-15 12:53:22 | INFO | Fetched 1,997 rows from SQL Server +2026-06-15 12:53:22 | INFO | Fetched 1997 rows +2026-06-15 12:53:22 | INFO | Deleting from SOS_OneApp +2026-06-15 12:53:23 | INFO | SOS_OneApp: inserted 1,997 rows into ClickHouse +2026-06-15 12:53:23 | INFO | SOS_OneApp loaded successfully (1997 rows) +2026-06-15 12:53:23 | INFO | ================================================================================ +2026-06-15 12:53:23 | INFO | Processing Table: OQaD +2026-06-15 12:53:23 | INFO | Fetching OQaD data for run_date=2026-06-14 +2026-06-15 12:53:26 | INFO | Fetched 443 rows +2026-06-15 12:53:26 | INFO | Fetched 443 rows +2026-06-15 12:53:26 | INFO | Deleting from OQaD +2026-06-15 12:53:26 | INFO | OQaD: inserted 443 rows into ClickHouse +2026-06-15 12:53:26 | INFO | OQaD loaded successfully (443 rows) +2026-06-15 12:53:26 | INFO | ================================================================================ +2026-06-15 12:53:26 | INFO | Processing Table: Survey +2026-06-15 12:53:26 | INFO | Fetching Survey data for 571 MIDs +2026-06-15 12:53:27 | INFO | Fetched 174 Survey rows +2026-06-15 12:53:27 | INFO | Fetched 174 rows +2026-06-15 12:53:27 | INFO | Deleting from Survey +2026-06-15 12:53:27 | INFO | Survey: inserted 174 rows into ClickHouse +2026-06-15 12:53:27 | INFO | Survey loaded successfully (174 rows) +2026-06-15 12:53:27 | INFO | ================================================================================ +2026-06-15 12:53:27 | INFO | Processing Table: additional_visibility +2026-06-15 12:53:27 | INFO | Fetching Additional Visibility data for 571 MIDs +2026-06-15 12:53:28 | INFO | Fetched 1,651 Additional Visibility rows +2026-06-15 12:53:28 | INFO | Fetched 1651 rows +2026-06-15 12:53:28 | INFO | Deleting from additional_visibility +2026-06-15 12:53:28 | INFO | additional_visibility: inserted 1,651 rows into ClickHouse +2026-06-15 12:53:28 | INFO | additional_visibility loaded successfully (1651 rows) +2026-06-15 12:53:28 | INFO | ================================================================================ +2026-06-15 12:53:28 | INFO | Processing Table: Coverage +2026-06-15 12:53:28 | INFO | Fetching coverage data for 571 MIDs +2026-06-15 12:53:28 | INFO | Fetched 514 rows from SQL Server +2026-06-15 12:53:28 | INFO | Fetched 514 rows +2026-06-15 12:53:29 | INFO | Deleting from Coverage +2026-06-15 12:53:29 | INFO | Coverage: inserted 514 rows into ClickHouse +2026-06-15 12:53:29 | INFO | Coverage loaded successfully (514 rows) +2026-06-15 12:53:29 | INFO | ================================================================================ +2026-06-15 12:53:29 | INFO | Processing Table: Login +2026-06-15 12:53:29 | INFO | Fetching Login data for yesterday +2026-06-15 12:53:31 | INFO | Fetched 454 Login rows +2026-06-15 12:53:31 | INFO | Fetched 454 rows +2026-06-15 12:53:31 | INFO | Deleting from Login +2026-06-15 12:53:31 | ERROR | Failed processing table Login +Traceback (most recent call last): + File "D:\data_move\test2.py", line 236, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 112, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + ...<4 lines>... + """ + ^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'project_id' while processing: 'isZeroOrNull((project_id = 40148) AND (toDate(login_date) = toDate('2026-06-14'))), _database, _part_data_version, _part_granule_offset, _part_offset, _disk_name, _sample_factor, _block_offset, _table, login_date, _part_uuid, _part_starting_offset, first_store_in_time, _part_index, _part, unique_id, _block_number, login_time, _partition_id, last_store_out_time, _distance, employee_id', required columns: '_part_offset' '_sample_factor' '_disk_name' '_part_data_version' 'project_id' '_database' '_part_granule_offset' '_part_uuid' '_block_offset' '_table' 'login_date' '_part' '_part_index' '_part_starting_offset' 'first_store_in_time' 'login_time' '_block_number' 'unique_id' '_partition_id' 'employee_id' '_distance' 'last_store_out_time', maybe you meant: 'login_date', 'first_store_in_time', 'login_time', 'unique_id', 'employee_id' or 'last_store_out_time'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) +2026-06-15 13:19:07 | INFO | ================================================================================ +2026-06-15 13:19:07 | INFO | Hello from data-move Python data pipeline! +2026-06-15 13:19:07 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 13:19:07 | INFO | Connecting to databases... +2026-06-15 13:19:07 | INFO | +2026-06-15 13:19:08 | INFO | +2026-06-15 13:19:09 | INFO | Database connections established +2026-06-15 13:19:09 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 13:19:09 | INFO | Found 571 MIDs +2026-06-15 13:19:09 | INFO | ================================================================================ +2026-06-15 13:19:09 | INFO | Processing Table: SOS_OneApp +2026-06-15 13:19:09 | INFO | Fetching data for 571 MIDs +2026-06-15 13:19:10 | INFO | Fetched 1,997 rows from SQL Server +2026-06-15 13:19:10 | INFO | Fetched 1997 rows +2026-06-15 13:19:10 | INFO | Deleting from SOS_OneApp +2026-06-15 13:19:10 | INFO | SOS_OneApp: inserted 1,997 rows into ClickHouse +2026-06-15 13:19:10 | INFO | SOS_OneApp loaded successfully (1997 rows) +2026-06-15 13:19:10 | INFO | ================================================================================ +2026-06-15 13:19:10 | INFO | Processing Table: OQaD +2026-06-15 13:19:10 | INFO | Fetching OQaD data for run_date=2026-06-14 +2026-06-15 13:19:13 | INFO | Fetched 443 rows +2026-06-15 13:19:13 | INFO | Fetched 443 rows +2026-06-15 13:19:13 | INFO | Deleting from OQaD +2026-06-15 13:19:14 | INFO | OQaD: inserted 443 rows into ClickHouse +2026-06-15 13:19:14 | INFO | OQaD loaded successfully (443 rows) +2026-06-15 13:19:14 | INFO | ================================================================================ +2026-06-15 13:19:14 | INFO | Processing Table: Survey +2026-06-15 13:19:14 | INFO | Fetching Survey data for 571 MIDs +2026-06-15 13:19:14 | INFO | Fetched 174 Survey rows +2026-06-15 13:19:14 | INFO | Fetched 174 rows +2026-06-15 13:19:14 | INFO | Deleting from Survey +2026-06-15 13:19:14 | INFO | Survey: inserted 174 rows into ClickHouse +2026-06-15 13:19:14 | INFO | Survey loaded successfully (174 rows) +2026-06-15 13:19:14 | INFO | ================================================================================ +2026-06-15 13:19:14 | INFO | Processing Table: additional_visibility +2026-06-15 13:19:14 | INFO | Fetching Additional Visibility data for 571 MIDs +2026-06-15 13:19:32 | INFO | Fetched 1,651 Additional Visibility rows +2026-06-15 13:19:32 | INFO | Fetched 1651 rows +2026-06-15 13:19:32 | INFO | Deleting from additional_visibility +2026-06-15 13:19:33 | INFO | additional_visibility: inserted 1,651 rows into ClickHouse +2026-06-15 13:19:33 | INFO | additional_visibility loaded successfully (1651 rows) +2026-06-15 13:19:33 | INFO | ================================================================================ +2026-06-15 13:19:33 | INFO | Processing Table: Coverage +2026-06-15 13:19:33 | INFO | Fetching coverage data for 571 MIDs +2026-06-15 13:19:34 | INFO | Fetched 514 rows from SQL Server +2026-06-15 13:19:34 | INFO | Fetched 514 rows +2026-06-15 13:19:34 | INFO | Deleting from Coverage +2026-06-15 13:19:34 | INFO | Coverage: inserted 514 rows into ClickHouse +2026-06-15 13:19:34 | INFO | Coverage loaded successfully (514 rows) +2026-06-15 13:19:34 | INFO | ================================================================================ +2026-06-15 13:19:34 | INFO | Processing Table: Login +2026-06-15 13:19:34 | INFO | Fetching Login data for yesterday +2026-06-15 13:19:36 | INFO | Fetched 454 Login rows +2026-06-15 13:19:36 | INFO | Fetched 454 rows +2026-06-15 13:19:36 | INFO | Deleting from Login +2026-06-15 13:19:36 | ERROR | Failed processing table Login +Traceback (most recent call last): + File "D:\data_move\test2.py", line 236, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 110, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + ...<3 lines>... + """ + ^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 62, server response: Code: 62. DB::Exception: Syntax error: failed at position 58 (toDate) (line 5, col 13): toDate(login_date) = toDate('2026-06-14') + + . Expected one of: token sequence, Dot, token, OR, AND, IS NOT DISTINCT FROM, IS DISTINCT FROM, IS NULL, IS NOT NULL, BETWEEN, NOT BETWEEN, LIKE, ILIKE, NOT LIKE, NOT ILIKE, REGEXP, IN, NOT IN, GLOBAL IN, GLOBAL NOT IN, MOD, DIV, alias, AS, Comma, INTO OUTFILE, FORMAT, SETTINGS, ParallelWithClause, PARALLEL WITH, end of query. (SYNTAX_ERROR) (for url http://172.188.12.194:8123) +2026-06-15 13:21:54 | INFO | ================================================================================ +2026-06-15 13:21:54 | INFO | Hello from data-move Python data pipeline! +2026-06-15 13:21:54 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 13:21:54 | INFO | Connecting to databases... +2026-06-15 13:21:55 | INFO | +2026-06-15 13:21:56 | INFO | +2026-06-15 13:21:56 | INFO | Database connections established +2026-06-15 13:21:56 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 13:21:57 | INFO | Found 571 MIDs +2026-06-15 13:21:57 | INFO | ================================================================================ +2026-06-15 13:21:57 | INFO | Processing Table: SOS_OneApp +2026-06-15 13:21:57 | INFO | Fetching data for 571 MIDs +2026-06-15 13:21:58 | INFO | Fetched 1,997 rows from SQL Server +2026-06-15 13:21:58 | INFO | Fetched 1997 rows +2026-06-15 13:21:58 | INFO | Deleting from SOS_OneApp +2026-06-15 13:21:58 | INFO | SOS_OneApp: inserted 1,997 rows into ClickHouse +2026-06-15 13:21:58 | INFO | SOS_OneApp loaded successfully (1997 rows) +2026-06-15 13:21:58 | INFO | ================================================================================ +2026-06-15 13:21:58 | INFO | Processing Table: OQaD +2026-06-15 13:21:58 | INFO | Fetching OQaD data for run_date=2026-06-14 +2026-06-15 13:22:00 | INFO | Fetched 443 rows +2026-06-15 13:22:00 | INFO | Fetched 443 rows +2026-06-15 13:22:00 | INFO | Deleting from OQaD +2026-06-15 13:22:00 | INFO | OQaD: inserted 443 rows into ClickHouse +2026-06-15 13:22:00 | INFO | OQaD loaded successfully (443 rows) +2026-06-15 13:22:01 | INFO | ================================================================================ +2026-06-15 13:22:01 | INFO | Processing Table: Survey +2026-06-15 13:22:01 | INFO | Fetching Survey data for 571 MIDs +2026-06-15 13:22:01 | INFO | Fetched 174 Survey rows +2026-06-15 13:22:01 | INFO | Fetched 174 rows +2026-06-15 13:22:01 | INFO | Deleting from Survey +2026-06-15 13:22:01 | INFO | Survey: inserted 174 rows into ClickHouse +2026-06-15 13:22:01 | INFO | Survey loaded successfully (174 rows) +2026-06-15 13:22:01 | INFO | ================================================================================ +2026-06-15 13:22:01 | INFO | Processing Table: additional_visibility +2026-06-15 13:22:01 | INFO | Fetching Additional Visibility data for 571 MIDs +2026-06-15 13:22:02 | INFO | Fetched 1,651 Additional Visibility rows +2026-06-15 13:22:02 | INFO | Fetched 1651 rows +2026-06-15 13:22:02 | INFO | Deleting from additional_visibility +2026-06-15 13:22:02 | INFO | additional_visibility: inserted 1,651 rows into ClickHouse +2026-06-15 13:22:02 | INFO | additional_visibility loaded successfully (1651 rows) +2026-06-15 13:22:02 | INFO | ================================================================================ +2026-06-15 13:22:02 | INFO | Processing Table: Coverage +2026-06-15 13:22:02 | INFO | Fetching coverage data for 571 MIDs +2026-06-15 13:22:02 | INFO | Fetched 514 rows from SQL Server +2026-06-15 13:22:02 | INFO | Fetched 514 rows +2026-06-15 13:22:02 | INFO | Deleting from Coverage +2026-06-15 13:22:02 | INFO | Coverage: inserted 514 rows into ClickHouse +2026-06-15 13:22:02 | INFO | Coverage loaded successfully (514 rows) +2026-06-15 13:22:02 | INFO | ================================================================================ +2026-06-15 13:22:02 | INFO | Processing Table: Login +2026-06-15 13:22:02 | INFO | Fetching Login data for yesterday +2026-06-15 13:22:03 | INFO | Fetched 454 Login rows +2026-06-15 13:22:03 | INFO | Fetched 454 rows +2026-06-15 13:22:03 | INFO | Deleting from Login +2026-06-15 13:22:03 | INFO | No delete logic required for Login +2026-06-15 13:22:03 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-15 13:22:03 | INFO | Login loaded successfully (454 rows) +2026-06-15 13:22:03 | INFO | ================================================================================ +2026-06-15 13:22:03 | INFO | Processing Table: Stock_Details +2026-06-15 13:22:03 | INFO | Fetching Stock Details data for 571 MIDs +2026-06-15 13:22:12 | INFO | Fetched 28,574 Stock Details rows +2026-06-15 13:22:12 | INFO | Fetched 28574 rows +2026-06-15 13:22:12 | INFO | Deleting from Stock_Details +2026-06-15 13:22:13 | INFO | Stock_Details: inserted 28,574 rows into ClickHouse +2026-06-15 13:22:13 | INFO | Stock_Details loaded successfully (28574 rows) +2026-06-15 13:22:13 | INFO | ================================================================================ +2026-06-15 13:22:13 | INFO | Processing Table: Attendance +2026-06-15 13:22:13 | INFO | Fetching Attendance data from 2026-05-30 to 2026-06-14 +2026-06-15 13:22:14 | INFO | Fetched 11,543 attendance rows for 594 employees +2026-06-15 13:22:14 | INFO | Fetched 11543 rows +2026-06-15 13:22:14 | INFO | Truncated table Attendance +2026-06-15 13:22:15 | INFO | Attendance: inserted 11,543 rows into ClickHouse +2026-06-15 13:22:15 | INFO | Attendance loaded successfully (11543 rows) +2026-06-15 13:22:15 | INFO | ================================================================================ +2026-06-15 13:22:15 | INFO | Processing Table: Store_Master +2026-06-15 13:22:15 | INFO | Fetching Store Master data +2026-06-15 13:22:17 | INFO | Fetched 5,984 stores +2026-06-15 13:22:17 | INFO | Fetched 5984 rows +2026-06-15 13:22:17 | INFO | Truncated table Store_Master +2026-06-15 13:22:18 | INFO | Store_Master: inserted 5,984 rows into ClickHouse +2026-06-15 13:22:18 | INFO | Store_Master loaded successfully (5984 rows) +2026-06-15 13:22:18 | INFO | ================================================================================ +2026-06-15 13:22:18 | INFO | Processing Table: SKU_Master +2026-06-15 13:22:18 | INFO | Fetching SKU Master data +2026-06-15 13:22:18 | INFO | Fetched 160 SKU Master rows +2026-06-15 13:22:18 | INFO | Fetched 160 rows +2026-06-15 13:22:18 | INFO | Truncated table SKU_Master +2026-06-15 13:22:18 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-15 13:22:18 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-15 13:22:18 | INFO | ================================================================================ +2026-06-15 13:22:18 | INFO | Processing Table: display_master +2026-06-15 13:22:18 | INFO | Fetching Display Master data +2026-06-15 13:22:18 | INFO | Fetched 135 Display Master records +2026-06-15 13:22:18 | INFO | Fetched 135 rows +2026-06-15 13:22:19 | INFO | Truncated table display_master +2026-06-15 13:22:19 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-15 13:22:19 | INFO | display_master loaded successfully (135 rows) +2026-06-15 13:22:19 | INFO | ================================================================================ +2026-06-15 13:22:19 | INFO | Processing Table: Employee_Master +2026-06-15 13:22:19 | INFO | Fetching Employee Master data +2026-06-15 13:22:19 | INFO | Fetched 2,273 Employee Master records +2026-06-15 13:22:19 | INFO | Fetched 2273 rows +2026-06-15 13:22:19 | INFO | Truncated table Employee_Master +2026-06-15 13:22:20 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse +2026-06-15 13:22:20 | INFO | Employee_Master loaded successfully (2273 rows) +2026-06-15 13:22:20 | INFO | ================================================================================ +2026-06-15 13:22:20 | INFO | Processing Table: Journey_Plan +2026-06-15 13:22:20 | INFO | Fetching Journey Plan for 2026-06 +2026-06-15 13:22:20 | INFO | Fetched 21,141 Journey Plan records +2026-06-15 13:22:20 | INFO | Fetched 21141 rows +2026-06-15 13:22:20 | INFO | Deleting from Journey_Plan +2026-06-15 13:22:21 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse +2026-06-15 13:22:21 | INFO | Journey_Plan loaded successfully (21141 rows) +2026-06-15 13:22:21 | INFO | ================================================================================ +2026-06-15 13:22:21 | INFO | Processing Table: coverage_remarks +2026-06-15 13:22:21 | INFO | Fetching Coverage Remarks +2026-06-15 13:22:21 | INFO | Fetched 29 Coverage Remark records +2026-06-15 13:22:21 | INFO | Fetched 29 rows +2026-06-15 13:22:21 | INFO | Truncated table coverage_remarks +2026-06-15 13:22:21 | INFO | coverage_remarks: inserted 29 rows into ClickHouse +2026-06-15 13:22:21 | INFO | coverage_remarks loaded successfully (29 rows) +2026-06-15 13:22:21 | INFO | ================================================================================ +2026-06-15 13:22:21 | INFO | Processing Table: mapping_storevisibility +2026-06-15 13:22:21 | INFO | Fetching Mapping Store Visibility for 2026-06-14 +2026-06-15 13:22:21 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-15 13:22:21 | WARNING | mapping_storevisibility returned no rows +2026-06-15 13:22:21 | INFO | ================================================================================ +2026-06-15 13:22:21 | INFO | Processing Table: Master_VisibilityReason +2026-06-15 13:22:21 | INFO | Fetching Master Visibility Reason data +2026-06-15 13:22:21 | INFO | Fetched 17 Master Visibility Reason records +2026-06-15 13:22:21 | INFO | Fetched 17 rows +2026-06-15 13:22:21 | INFO | Truncated table Master_VisibilityReason +2026-06-15 13:22:21 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-15 13:22:21 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-15 13:22:21 | INFO | ================================================================================ +2026-06-15 13:22:22 | INFO | Processing Table: Master_VisibilityDefinition +2026-06-15 13:22:22 | INFO | Fetching Master Visibility Definition data +2026-06-15 13:22:22 | INFO | Fetched 861 Master Visibility Definition records +2026-06-15 13:22:22 | INFO | Fetched 861 rows +2026-06-15 13:22:22 | INFO | Truncated table Master_VisibilityDefinition +2026-06-15 13:22:22 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-15 13:22:22 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-15 13:22:22 | INFO | ================================================================================ +2026-06-15 13:22:22 | INFO | Processing Table: Web_Logins +2026-06-15 13:22:22 | INFO | Fetching Web Login data for 2026-06-14 +2026-06-15 13:22:22 | INFO | Fetched 25 Web Login records +2026-06-15 13:22:22 | INFO | Fetched 25 rows +2026-06-15 13:22:22 | INFO | Deleting from Web_Logins +2026-06-15 13:22:22 | ERROR | Failed processing table Web_Logins +Traceback (most recent call last): + File "D:\data_move\test2.py", line 236, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 129, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + ...<4 lines>... + """, + ^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'date' while processing: 'isZeroOrNull(toDate(date) = toDate('2026-06-14')), _block_offset, _block_number, _table, _database, _distance, _part_starting_offset, _part_data_version, _part_granule_offset, _disk_name, _sample_factor, _partition_id, right_name, _part_index, _part, create_by, employee_name, activity_type, _part_uuid, login_time, create_date, project_id, activity_name, login_date, _part_offset, emp_id, designation, supervisor_name, supervisor_id', required columns: 'date' '_block_number' '_table' '_block_offset' '_part_uuid' 'activity_name' 'project_id' '_distance' '_database' '_part_starting_offset' '_part_data_version' '_part_granule_offset' '_disk_name' '_sample_factor' '_partition_id' 'create_date' 'login_time' 'designation' '_part_index' 'right_name' 'activity_type' 'employee_name' 'login_date' 'create_by' '_part_offset' 'emp_id' '_part' 'supervisor_name' 'supervisor_id', maybe you meant: 'activity_name', 'project_id', 'create_date', 'login_time', 'designation', 'right_name', 'activity_type', 'employee_name', 'login_date', 'create_by', 'emp_id', 'supervisor_name' or 'supervisor_id'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) +2026-06-15 13:26:05 | INFO | ================================================================================ +2026-06-15 13:26:05 | INFO | Hello from data-move Python data pipeline! +2026-06-15 13:26:05 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 13:26:05 | INFO | Connecting to databases... +2026-06-15 13:26:06 | INFO | +2026-06-15 13:26:08 | INFO | +2026-06-15 13:26:08 | INFO | Database connections established +2026-06-15 13:26:08 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 13:26:08 | INFO | Found 571 MIDs +2026-06-15 13:26:09 | INFO | ================================================================================ +2026-06-15 13:26:09 | INFO | Processing Table: SOS_OneApp +2026-06-15 13:26:09 | INFO | Fetching data for 571 MIDs +2026-06-15 13:26:09 | INFO | Fetched 1,997 rows from SQL Server +2026-06-15 13:26:09 | INFO | Fetched 1997 rows +2026-06-15 13:26:09 | INFO | Deleting from SOS_OneApp +2026-06-15 13:26:09 | INFO | SOS_OneApp: inserted 1,997 rows into ClickHouse +2026-06-15 13:26:09 | INFO | SOS_OneApp loaded successfully (1997 rows) +2026-06-15 13:26:09 | INFO | ================================================================================ +2026-06-15 13:26:09 | INFO | Processing Table: OQaD +2026-06-15 13:26:09 | INFO | Fetching OQaD data for run_date=2026-06-14 +2026-06-15 13:26:11 | INFO | Fetched 443 rows +2026-06-15 13:26:11 | INFO | Fetched 443 rows +2026-06-15 13:26:12 | INFO | Deleting from OQaD +2026-06-15 13:26:12 | INFO | OQaD: inserted 443 rows into ClickHouse +2026-06-15 13:26:12 | INFO | OQaD loaded successfully (443 rows) +2026-06-15 13:26:12 | INFO | ================================================================================ +2026-06-15 13:26:12 | INFO | Processing Table: Survey +2026-06-15 13:26:12 | INFO | Fetching Survey data for 571 MIDs +2026-06-15 13:26:12 | INFO | Fetched 174 Survey rows +2026-06-15 13:26:12 | INFO | Fetched 174 rows +2026-06-15 13:26:12 | INFO | Deleting from Survey +2026-06-15 13:26:12 | INFO | Survey: inserted 174 rows into ClickHouse +2026-06-15 13:26:12 | INFO | Survey loaded successfully (174 rows) +2026-06-15 13:26:12 | INFO | ================================================================================ +2026-06-15 13:26:12 | INFO | Processing Table: additional_visibility +2026-06-15 13:26:12 | INFO | Fetching Additional Visibility data for 571 MIDs +2026-06-15 13:26:13 | INFO | Fetched 1,651 Additional Visibility rows +2026-06-15 13:26:13 | INFO | Fetched 1651 rows +2026-06-15 13:26:13 | INFO | Deleting from additional_visibility +2026-06-15 13:26:13 | INFO | additional_visibility: inserted 1,651 rows into ClickHouse +2026-06-15 13:26:13 | INFO | additional_visibility loaded successfully (1651 rows) +2026-06-15 13:26:13 | INFO | ================================================================================ +2026-06-15 13:26:13 | INFO | Processing Table: Coverage +2026-06-15 13:26:13 | INFO | Fetching coverage data for 571 MIDs +2026-06-15 13:26:13 | INFO | Fetched 514 rows from SQL Server +2026-06-15 13:26:13 | INFO | Fetched 514 rows +2026-06-15 13:26:13 | INFO | Deleting from Coverage +2026-06-15 13:26:13 | INFO | Coverage: inserted 514 rows into ClickHouse +2026-06-15 13:26:13 | INFO | Coverage loaded successfully (514 rows) +2026-06-15 13:26:13 | INFO | ================================================================================ +2026-06-15 13:26:13 | INFO | Processing Table: Login +2026-06-15 13:26:13 | INFO | Fetching Login data for yesterday +2026-06-15 13:26:14 | INFO | Fetched 454 Login rows +2026-06-15 13:26:14 | INFO | Fetched 454 rows +2026-06-15 13:26:14 | INFO | Deleting from Login +2026-06-15 13:26:14 | INFO | No delete logic required for Login +2026-06-15 13:26:14 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-15 13:26:14 | INFO | Login loaded successfully (454 rows) +2026-06-15 13:26:14 | INFO | ================================================================================ +2026-06-15 13:26:14 | INFO | Processing Table: Stock_Details +2026-06-15 13:26:14 | INFO | Fetching Stock Details data for 571 MIDs +2026-06-15 13:26:17 | INFO | Fetched 28,574 Stock Details rows +2026-06-15 13:26:17 | INFO | Fetched 28574 rows +2026-06-15 13:26:17 | INFO | Deleting from Stock_Details +2026-06-15 13:26:18 | INFO | Stock_Details: inserted 28,574 rows into ClickHouse +2026-06-15 13:26:18 | INFO | Stock_Details loaded successfully (28574 rows) +2026-06-15 13:26:18 | INFO | ================================================================================ +2026-06-15 13:26:18 | INFO | Processing Table: Attendance +2026-06-15 13:26:18 | INFO | Fetching Attendance data from 2026-05-30 to 2026-06-14 +2026-06-15 13:26:18 | INFO | Fetched 11,543 attendance rows for 594 employees +2026-06-15 13:26:18 | INFO | Fetched 11543 rows +2026-06-15 13:26:19 | INFO | Truncated table Attendance +2026-06-15 13:26:19 | INFO | Attendance: inserted 11,543 rows into ClickHouse +2026-06-15 13:26:19 | INFO | Attendance loaded successfully (11543 rows) +2026-06-15 13:26:19 | INFO | ================================================================================ +2026-06-15 13:26:19 | INFO | Processing Table: Store_Master +2026-06-15 13:26:19 | INFO | Fetching Store Master data +2026-06-15 13:26:21 | INFO | Fetched 5,984 stores +2026-06-15 13:26:21 | INFO | Fetched 5984 rows +2026-06-15 13:26:22 | INFO | Truncated table Store_Master +2026-06-15 13:26:22 | INFO | Store_Master: inserted 5,984 rows into ClickHouse +2026-06-15 13:26:22 | INFO | Store_Master loaded successfully (5984 rows) +2026-06-15 13:26:22 | INFO | ================================================================================ +2026-06-15 13:26:22 | INFO | Processing Table: SKU_Master +2026-06-15 13:26:22 | INFO | Fetching SKU Master data +2026-06-15 13:26:22 | INFO | Fetched 160 SKU Master rows +2026-06-15 13:26:22 | INFO | Fetched 160 rows +2026-06-15 13:26:22 | INFO | Truncated table SKU_Master +2026-06-15 13:26:22 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-15 13:26:22 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-15 13:26:22 | INFO | ================================================================================ +2026-06-15 13:26:22 | INFO | Processing Table: display_master +2026-06-15 13:26:22 | INFO | Fetching Display Master data +2026-06-15 13:26:23 | INFO | Fetched 135 Display Master records +2026-06-15 13:26:23 | INFO | Fetched 135 rows +2026-06-15 13:26:23 | INFO | Truncated table display_master +2026-06-15 13:26:23 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-15 13:26:23 | INFO | display_master loaded successfully (135 rows) +2026-06-15 13:26:23 | INFO | ================================================================================ +2026-06-15 13:26:23 | INFO | Processing Table: Employee_Master +2026-06-15 13:26:23 | INFO | Fetching Employee Master data +2026-06-15 13:26:24 | INFO | Fetched 2,273 Employee Master records +2026-06-15 13:26:24 | INFO | Fetched 2273 rows +2026-06-15 13:26:24 | INFO | Truncated table Employee_Master +2026-06-15 13:26:24 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse +2026-06-15 13:26:24 | INFO | Employee_Master loaded successfully (2273 rows) +2026-06-15 13:26:24 | INFO | ================================================================================ +2026-06-15 13:26:24 | INFO | Processing Table: Journey_Plan +2026-06-15 13:26:24 | INFO | Fetching Journey Plan for 2026-06 +2026-06-15 13:26:24 | INFO | Fetched 21,141 Journey Plan records +2026-06-15 13:26:24 | INFO | Fetched 21141 rows +2026-06-15 13:26:24 | INFO | Deleting from Journey_Plan +2026-06-15 13:26:25 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse +2026-06-15 13:26:25 | INFO | Journey_Plan loaded successfully (21141 rows) +2026-06-15 13:26:25 | INFO | ================================================================================ +2026-06-15 13:26:25 | INFO | Processing Table: coverage_remarks +2026-06-15 13:26:25 | INFO | Fetching Coverage Remarks +2026-06-15 13:26:25 | INFO | Fetched 29 Coverage Remark records +2026-06-15 13:26:25 | INFO | Fetched 29 rows +2026-06-15 13:26:25 | INFO | Truncated table coverage_remarks +2026-06-15 13:26:25 | INFO | coverage_remarks: inserted 29 rows into ClickHouse +2026-06-15 13:26:25 | INFO | coverage_remarks loaded successfully (29 rows) +2026-06-15 13:26:25 | INFO | ================================================================================ +2026-06-15 13:26:25 | INFO | Processing Table: mapping_storevisibility +2026-06-15 13:26:25 | INFO | Fetching Mapping Store Visibility for 2026-06-14 +2026-06-15 13:26:25 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-15 13:26:25 | WARNING | mapping_storevisibility returned no rows +2026-06-15 13:26:25 | INFO | ================================================================================ +2026-06-15 13:26:25 | INFO | Processing Table: Master_VisibilityReason +2026-06-15 13:26:25 | INFO | Fetching Master Visibility Reason data +2026-06-15 13:26:25 | INFO | Fetched 17 Master Visibility Reason records +2026-06-15 13:26:25 | INFO | Fetched 17 rows +2026-06-15 13:26:25 | INFO | Truncated table Master_VisibilityReason +2026-06-15 13:26:25 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-15 13:26:25 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-15 13:26:25 | INFO | ================================================================================ +2026-06-15 13:26:25 | INFO | Processing Table: Master_VisibilityDefinition +2026-06-15 13:26:25 | INFO | Fetching Master Visibility Definition data +2026-06-15 13:26:25 | INFO | Fetched 861 Master Visibility Definition records +2026-06-15 13:26:25 | INFO | Fetched 861 rows +2026-06-15 13:26:26 | INFO | Truncated table Master_VisibilityDefinition +2026-06-15 13:26:26 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-15 13:26:26 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-15 13:26:26 | INFO | ================================================================================ +2026-06-15 13:26:26 | INFO | Processing Table: Web_Logins +2026-06-15 13:26:26 | INFO | Fetching Web Login data for 2026-06-14 +2026-06-15 13:26:26 | INFO | Fetched 25 Web Login records +2026-06-15 13:26:26 | INFO | Fetched 25 rows +2026-06-15 13:26:26 | INFO | Deleting from Web_Logins +2026-06-15 13:26:26 | ERROR | Failed processing table Web_Logins +Traceback (most recent call last): + File "D:\data_move\test2.py", line 236, in main + delete_existing_data( + ~~~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + ...<3 lines>... + emp_visit_df=emp_visit_df, + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 129, in delete_existing_data + delete_rows( + ~~~~~~~~~~~^ + client, + ^^^^^^^ + ...<4 lines>... + """, + ^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\delete_task.py", line 46, in delete_rows + client.command(query) + ~~~~~~~~~~~~~~^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 486, in command + response = self._raw_request(payload, params, headers, method, fields=fields, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 47, server response: Code: 47. DB::Exception: Missing columns: 'date' while processing: 'isZeroOrNull(toDate(date) = toDate('2026-06-14')), _block_offset, _block_number, _table, _database, _distance, _part_starting_offset, _part_data_version, _part_granule_offset, _disk_name, _sample_factor, _partition_id, right_name, _part_index, _part, create_by, employee_name, activity_type, _part_uuid, login_time, create_date, project_id, activity_name, login_date, _part_offset, emp_id, designation, supervisor_name, supervisor_id', required columns: 'date' '_block_number' '_table' '_block_offset' '_part_uuid' 'activity_name' 'project_id' '_distance' '_database' '_part_starting_offset' '_part_data_version' '_part_granule_offset' '_disk_name' '_sample_factor' '_partition_id' 'create_date' 'login_time' 'designation' '_part_index' 'right_name' 'activity_type' 'employee_name' 'login_date' 'create_by' '_part_offset' 'emp_id' '_part' 'supervisor_name' 'supervisor_id', maybe you meant: 'activity_name', 'project_id', 'create_date', 'login_time', 'designation', 'right_name', 'activity_type', 'employee_name', 'login_date', 'create_by', 'emp_id', 'supervisor_name' or 'supervisor_id'. (UNKNOWN_IDENTIFIER) (for url http://172.188.12.194:8123) +2026-06-15 13:27:51 | INFO | ================================================================================ +2026-06-15 13:27:51 | INFO | Hello from data-move Python data pipeline! +2026-06-15 13:27:51 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 13:27:51 | INFO | Connecting to databases... +2026-06-15 13:27:52 | INFO | +2026-06-15 13:27:53 | INFO | +2026-06-15 13:27:53 | INFO | Database connections established +2026-06-15 13:27:53 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 13:27:54 | INFO | Found 571 MIDs +2026-06-15 13:27:54 | INFO | ================================================================================ +2026-06-15 13:27:54 | INFO | Processing Table: SOS_OneApp +2026-06-15 13:27:54 | INFO | Fetching data for 571 MIDs +2026-06-15 13:27:54 | INFO | Fetched 1,997 rows from SQL Server +2026-06-15 13:27:54 | INFO | Fetched 1997 rows +2026-06-15 13:27:54 | INFO | Deleting from SOS_OneApp +2026-06-15 13:27:55 | INFO | SOS_OneApp: inserted 1,997 rows into ClickHouse +2026-06-15 13:27:55 | INFO | SOS_OneApp loaded successfully (1997 rows) +2026-06-15 13:27:55 | INFO | ================================================================================ +2026-06-15 13:27:55 | INFO | Processing Table: OQaD +2026-06-15 13:27:55 | INFO | Fetching OQaD data for run_date=2026-06-14 +2026-06-15 13:27:57 | INFO | Fetched 443 rows +2026-06-15 13:27:57 | INFO | Fetched 443 rows +2026-06-15 13:27:57 | INFO | Deleting from OQaD +2026-06-15 13:27:57 | INFO | OQaD: inserted 443 rows into ClickHouse +2026-06-15 13:27:57 | INFO | OQaD loaded successfully (443 rows) +2026-06-15 13:27:57 | INFO | ================================================================================ +2026-06-15 13:27:57 | INFO | Processing Table: Survey +2026-06-15 13:27:57 | INFO | Fetching Survey data for 571 MIDs +2026-06-15 13:27:57 | INFO | Fetched 174 Survey rows +2026-06-15 13:27:57 | INFO | Fetched 174 rows +2026-06-15 13:27:57 | INFO | Deleting from Survey +2026-06-15 13:27:58 | INFO | Survey: inserted 174 rows into ClickHouse +2026-06-15 13:27:58 | INFO | Survey loaded successfully (174 rows) +2026-06-15 13:27:58 | INFO | ================================================================================ +2026-06-15 13:27:58 | INFO | Processing Table: additional_visibility +2026-06-15 13:27:58 | INFO | Fetching Additional Visibility data for 571 MIDs +2026-06-15 13:27:58 | INFO | Fetched 1,651 Additional Visibility rows +2026-06-15 13:27:58 | INFO | Fetched 1651 rows +2026-06-15 13:27:58 | INFO | Deleting from additional_visibility +2026-06-15 13:27:58 | INFO | additional_visibility: inserted 1,651 rows into ClickHouse +2026-06-15 13:27:58 | INFO | additional_visibility loaded successfully (1651 rows) +2026-06-15 13:27:58 | INFO | ================================================================================ +2026-06-15 13:27:58 | INFO | Processing Table: Coverage +2026-06-15 13:27:58 | INFO | Fetching coverage data for 571 MIDs +2026-06-15 13:27:58 | INFO | Fetched 514 rows from SQL Server +2026-06-15 13:27:58 | INFO | Fetched 514 rows +2026-06-15 13:27:59 | INFO | Deleting from Coverage +2026-06-15 13:27:59 | INFO | Coverage: inserted 514 rows into ClickHouse +2026-06-15 13:27:59 | INFO | Coverage loaded successfully (514 rows) +2026-06-15 13:27:59 | INFO | ================================================================================ +2026-06-15 13:27:59 | INFO | Processing Table: Login +2026-06-15 13:27:59 | INFO | Fetching Login data for yesterday +2026-06-15 13:27:59 | INFO | Fetched 454 Login rows +2026-06-15 13:27:59 | INFO | Fetched 454 rows +2026-06-15 13:27:59 | INFO | Deleting from Login +2026-06-15 13:28:00 | INFO | No delete logic required for Login +2026-06-15 13:28:00 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-15 13:28:00 | INFO | Login loaded successfully (454 rows) +2026-06-15 13:28:00 | INFO | ================================================================================ +2026-06-15 13:28:00 | INFO | Processing Table: Stock_Details +2026-06-15 13:28:00 | INFO | Fetching Stock Details data for 571 MIDs +2026-06-15 13:28:03 | INFO | Fetched 28,574 Stock Details rows +2026-06-15 13:28:03 | INFO | Fetched 28574 rows +2026-06-15 13:28:03 | INFO | Deleting from Stock_Details +2026-06-15 13:28:03 | INFO | Stock_Details: inserted 28,574 rows into ClickHouse +2026-06-15 13:28:03 | INFO | Stock_Details loaded successfully (28574 rows) +2026-06-15 13:28:03 | INFO | ================================================================================ +2026-06-15 13:28:03 | INFO | Processing Table: Attendance +2026-06-15 13:28:03 | INFO | Fetching Attendance data from 2026-05-30 to 2026-06-14 +2026-06-15 13:28:04 | INFO | Fetched 11,543 attendance rows for 594 employees +2026-06-15 13:28:04 | INFO | Fetched 11543 rows +2026-06-15 13:28:04 | INFO | Truncated table Attendance +2026-06-15 13:28:05 | INFO | Attendance: inserted 11,543 rows into ClickHouse +2026-06-15 13:28:05 | INFO | Attendance loaded successfully (11543 rows) +2026-06-15 13:28:05 | INFO | ================================================================================ +2026-06-15 13:28:05 | INFO | Processing Table: Store_Master +2026-06-15 13:28:05 | INFO | Fetching Store Master data +2026-06-15 13:28:06 | INFO | Fetched 5,984 stores +2026-06-15 13:28:06 | INFO | Fetched 5984 rows +2026-06-15 13:28:06 | INFO | Truncated table Store_Master +2026-06-15 13:28:07 | INFO | Store_Master: inserted 5,984 rows into ClickHouse +2026-06-15 13:28:07 | INFO | Store_Master loaded successfully (5984 rows) +2026-06-15 13:28:07 | INFO | ================================================================================ +2026-06-15 13:28:07 | INFO | Processing Table: SKU_Master +2026-06-15 13:28:07 | INFO | Fetching SKU Master data +2026-06-15 13:28:07 | INFO | Fetched 160 SKU Master rows +2026-06-15 13:28:07 | INFO | Fetched 160 rows +2026-06-15 13:28:07 | INFO | Truncated table SKU_Master +2026-06-15 13:28:07 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-15 13:28:07 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-15 13:28:07 | INFO | ================================================================================ +2026-06-15 13:28:07 | INFO | Processing Table: display_master +2026-06-15 13:28:07 | INFO | Fetching Display Master data +2026-06-15 13:28:08 | INFO | Fetched 135 Display Master records +2026-06-15 13:28:08 | INFO | Fetched 135 rows +2026-06-15 13:28:08 | INFO | Truncated table display_master +2026-06-15 13:28:08 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-15 13:28:08 | INFO | display_master loaded successfully (135 rows) +2026-06-15 13:28:08 | INFO | ================================================================================ +2026-06-15 13:28:08 | INFO | Processing Table: Employee_Master +2026-06-15 13:28:08 | INFO | Fetching Employee Master data +2026-06-15 13:28:08 | INFO | Fetched 2,273 Employee Master records +2026-06-15 13:28:08 | INFO | Fetched 2273 rows +2026-06-15 13:28:08 | INFO | Truncated table Employee_Master +2026-06-15 13:28:09 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse +2026-06-15 13:28:09 | INFO | Employee_Master loaded successfully (2273 rows) +2026-06-15 13:28:09 | INFO | ================================================================================ +2026-06-15 13:28:09 | INFO | Processing Table: Journey_Plan +2026-06-15 13:28:09 | INFO | Fetching Journey Plan for 2026-06 +2026-06-15 13:28:09 | INFO | Fetched 21,141 Journey Plan records +2026-06-15 13:28:09 | INFO | Fetched 21141 rows +2026-06-15 13:28:09 | INFO | Deleting from Journey_Plan +2026-06-15 13:28:09 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse +2026-06-15 13:28:09 | INFO | Journey_Plan loaded successfully (21141 rows) +2026-06-15 13:28:09 | INFO | ================================================================================ +2026-06-15 13:28:09 | INFO | Processing Table: coverage_remarks +2026-06-15 13:28:09 | INFO | Fetching Coverage Remarks +2026-06-15 13:28:09 | INFO | Fetched 29 Coverage Remark records +2026-06-15 13:28:09 | INFO | Fetched 29 rows +2026-06-15 13:28:10 | INFO | Truncated table coverage_remarks +2026-06-15 13:28:10 | INFO | coverage_remarks: inserted 29 rows into ClickHouse +2026-06-15 13:28:10 | INFO | coverage_remarks loaded successfully (29 rows) +2026-06-15 13:28:10 | INFO | ================================================================================ +2026-06-15 13:28:10 | INFO | Processing Table: mapping_storevisibility +2026-06-15 13:28:10 | INFO | Fetching Mapping Store Visibility for 2026-06-14 +2026-06-15 13:28:10 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-15 13:28:10 | WARNING | mapping_storevisibility returned no rows +2026-06-15 13:28:10 | INFO | ================================================================================ +2026-06-15 13:28:10 | INFO | Processing Table: Master_VisibilityReason +2026-06-15 13:28:10 | INFO | Fetching Master Visibility Reason data +2026-06-15 13:28:10 | INFO | Fetched 17 Master Visibility Reason records +2026-06-15 13:28:10 | INFO | Fetched 17 rows +2026-06-15 13:28:10 | INFO | Truncated table Master_VisibilityReason +2026-06-15 13:28:10 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-15 13:28:10 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-15 13:28:10 | INFO | ================================================================================ +2026-06-15 13:28:10 | INFO | Processing Table: Master_VisibilityDefinition +2026-06-15 13:28:10 | INFO | Fetching Master Visibility Definition data +2026-06-15 13:28:10 | INFO | Fetched 861 Master Visibility Definition records +2026-06-15 13:28:10 | INFO | Fetched 861 rows +2026-06-15 13:28:10 | INFO | Truncated table Master_VisibilityDefinition +2026-06-15 13:28:11 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-15 13:28:11 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-15 13:28:11 | INFO | ================================================================================ +2026-06-15 13:28:11 | INFO | Processing Table: Web_Logins +2026-06-15 13:28:11 | INFO | Fetching Web Login data for 2026-06-14 +2026-06-15 13:28:11 | INFO | Fetched 25 Web Login records +2026-06-15 13:28:11 | INFO | Fetched 25 rows +2026-06-15 13:28:11 | INFO | Deleting from Web_Logins +2026-06-15 13:28:11 | ERROR | Failed processing table Web_Logins +Traceback (most recent call last): + File "D:\data_move\test2.py", line 248, in main + load_to_clickhouse( + ~~~~~~~~~~~~~~~~~~^ + client=client, + ^^^^^^^^^^^^^^ + table_name=table_name, + ^^^^^^^^^^^^^^^^^^^^^^ + df=df, + ^^^^^^ + ) + ^ + File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse + client.insert_arrow( + ~~~~~~~~~~~~~~~~~~~^ + table=table_name, + ^^^^^^^^^^^^^^^^^ + arrow_table=arrow_table, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow + return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings) + ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert + response = self._raw_request(insert_block, params, headers, server_wait=False) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\test2-86cf58bc5ffcfdee\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column date in table kelloggs_1.Web_Logins (c845112d-d41d-478b-ba61-6c113b115c9d). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123) +2026-06-15 13:34:41 | INFO | ================================================================================ +2026-06-15 13:34:41 | INFO | Hello from data-move Python data pipeline! +2026-06-15 13:34:41 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 13:34:41 | INFO | Connecting to databases... +2026-06-15 13:34:41 | INFO | +2026-06-15 13:34:42 | INFO | +2026-06-15 13:34:43 | INFO | Database connections established +2026-06-15 13:34:43 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 13:34:43 | INFO | Found 571 MIDs +2026-06-15 13:34:43 | INFO | ================================================================================ +2026-06-15 13:34:43 | INFO | Processing Table: SOS_OneApp +2026-06-15 13:34:43 | INFO | Fetching data for 571 MIDs +2026-06-15 13:34:44 | INFO | Fetched 1,997 rows from SQL Server +2026-06-15 13:34:44 | INFO | Fetched 1997 rows +2026-06-15 13:34:44 | INFO | Creating table SOS_OneApp +2026-06-15 13:34:44 | INFO | Table ready: SOS_OneApp +2026-06-15 13:34:44 | INFO | SOS_OneApp: inserted 1,997 rows into ClickHouse +2026-06-15 13:34:44 | INFO | SOS_OneApp loaded successfully (1997 rows) +2026-06-15 13:34:44 | INFO | ================================================================================ +2026-06-15 13:34:44 | INFO | Processing Table: OQaD +2026-06-15 13:34:44 | INFO | Fetching OQaD data for run_date=2026-06-14 +2026-06-15 13:34:47 | INFO | Fetched 443 rows +2026-06-15 13:34:47 | INFO | Fetched 443 rows +2026-06-15 13:34:47 | INFO | Creating table OQaD +2026-06-15 13:34:47 | INFO | Table ready: OQaD +2026-06-15 13:34:47 | INFO | OQaD: inserted 443 rows into ClickHouse +2026-06-15 13:34:47 | INFO | OQaD loaded successfully (443 rows) +2026-06-15 13:34:47 | INFO | ================================================================================ +2026-06-15 13:34:47 | INFO | Processing Table: Survey +2026-06-15 13:34:47 | INFO | Fetching Survey data for 571 MIDs +2026-06-15 13:34:47 | INFO | Fetched 174 Survey rows +2026-06-15 13:34:47 | INFO | Fetched 174 rows +2026-06-15 13:34:47 | INFO | Creating table Survey +2026-06-15 13:34:47 | INFO | Table ready: Survey +2026-06-15 13:34:48 | INFO | Survey: inserted 174 rows into ClickHouse +2026-06-15 13:34:48 | INFO | Survey loaded successfully (174 rows) +2026-06-15 13:34:48 | INFO | ================================================================================ +2026-06-15 13:34:48 | INFO | Processing Table: additional_visibility +2026-06-15 13:34:48 | INFO | Fetching Additional Visibility data for 571 MIDs +2026-06-15 13:34:48 | INFO | Fetched 1,651 Additional Visibility rows +2026-06-15 13:34:48 | INFO | Fetched 1651 rows +2026-06-15 13:34:48 | INFO | Creating table additional_visibility +2026-06-15 13:34:48 | INFO | Table ready: additional_visibility +2026-06-15 13:34:48 | INFO | additional_visibility: inserted 1,651 rows into ClickHouse +2026-06-15 13:34:48 | INFO | additional_visibility loaded successfully (1651 rows) +2026-06-15 13:34:48 | INFO | ================================================================================ +2026-06-15 13:34:48 | INFO | Processing Table: Coverage +2026-06-15 13:34:48 | INFO | Fetching coverage data for 571 MIDs +2026-06-15 13:34:48 | INFO | Fetched 514 rows from SQL Server +2026-06-15 13:34:48 | INFO | Fetched 514 rows +2026-06-15 13:34:48 | INFO | Creating table Coverage +2026-06-15 13:34:49 | INFO | Table ready: Coverage +2026-06-15 13:34:49 | INFO | Coverage: inserted 514 rows into ClickHouse +2026-06-15 13:34:49 | INFO | Coverage loaded successfully (514 rows) +2026-06-15 13:34:49 | INFO | ================================================================================ +2026-06-15 13:34:49 | INFO | Processing Table: Login +2026-06-15 13:34:49 | INFO | Fetching Login data for yesterday +2026-06-15 13:34:49 | INFO | Fetched 454 Login rows +2026-06-15 13:34:49 | INFO | Fetched 454 rows +2026-06-15 13:34:49 | INFO | Creating table Login +2026-06-15 13:34:49 | INFO | Table ready: Login +2026-06-15 13:34:50 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-15 13:34:50 | INFO | Login loaded successfully (454 rows) +2026-06-15 13:34:50 | INFO | ================================================================================ +2026-06-15 13:34:50 | INFO | Processing Table: Stock_Details +2026-06-15 13:34:50 | INFO | Fetching Stock Details data for 571 MIDs +2026-06-15 13:34:52 | INFO | Fetched 28,574 Stock Details rows +2026-06-15 13:34:52 | INFO | Fetched 28574 rows +2026-06-15 13:34:52 | INFO | Creating table Stock_Details +2026-06-15 13:34:52 | INFO | Table ready: Stock_Details +2026-06-15 13:34:52 | INFO | Stock_Details: inserted 28,574 rows into ClickHouse +2026-06-15 13:34:52 | INFO | Stock_Details loaded successfully (28574 rows) +2026-06-15 13:34:52 | INFO | ================================================================================ +2026-06-15 13:34:52 | INFO | Processing Table: Attendance +2026-06-15 13:34:52 | INFO | Fetching Attendance data from 2026-05-30 to 2026-06-14 +2026-06-15 13:34:53 | INFO | Fetched 11,543 attendance rows for 594 employees +2026-06-15 13:34:53 | INFO | Fetched 11543 rows +2026-06-15 13:34:53 | INFO | Creating table Attendance +2026-06-15 13:34:54 | INFO | Table ready: Attendance +2026-06-15 13:34:54 | INFO | Attendance: inserted 11,543 rows into ClickHouse +2026-06-15 13:34:54 | INFO | Attendance loaded successfully (11543 rows) +2026-06-15 13:34:54 | INFO | ================================================================================ +2026-06-15 13:34:54 | INFO | Processing Table: Store_Master +2026-06-15 13:34:54 | INFO | Fetching Store Master data +2026-06-15 13:34:56 | INFO | Fetched 5,984 stores +2026-06-15 13:34:56 | INFO | Fetched 5984 rows +2026-06-15 13:34:56 | INFO | Creating table Store_Master +2026-06-15 13:34:56 | INFO | Table ready: Store_Master +2026-06-15 13:34:57 | INFO | Store_Master: inserted 5,984 rows into ClickHouse +2026-06-15 13:34:57 | INFO | Store_Master loaded successfully (5984 rows) +2026-06-15 13:34:57 | INFO | ================================================================================ +2026-06-15 13:34:57 | INFO | Processing Table: SKU_Master +2026-06-15 13:34:57 | INFO | Fetching SKU Master data +2026-06-15 13:34:57 | INFO | Fetched 160 SKU Master rows +2026-06-15 13:34:57 | INFO | Fetched 160 rows +2026-06-15 13:34:57 | INFO | Creating table SKU_Master +2026-06-15 13:34:57 | INFO | Table ready: SKU_Master +2026-06-15 13:34:57 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-15 13:34:57 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-15 13:34:57 | INFO | ================================================================================ +2026-06-15 13:34:57 | INFO | Processing Table: display_master +2026-06-15 13:34:57 | INFO | Fetching Display Master data +2026-06-15 13:34:57 | INFO | Fetched 135 Display Master records +2026-06-15 13:34:57 | INFO | Fetched 135 rows +2026-06-15 13:34:57 | INFO | Creating table display_master +2026-06-15 13:34:58 | INFO | Table ready: display_master +2026-06-15 13:34:58 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-15 13:34:58 | INFO | display_master loaded successfully (135 rows) +2026-06-15 13:34:58 | INFO | ================================================================================ +2026-06-15 13:34:58 | INFO | Processing Table: Employee_Master +2026-06-15 13:34:58 | INFO | Fetching Employee Master data +2026-06-15 13:34:58 | INFO | Fetched 2,273 Employee Master records +2026-06-15 13:34:58 | INFO | Fetched 2273 rows +2026-06-15 13:34:58 | INFO | Creating table Employee_Master +2026-06-15 13:34:58 | INFO | Table ready: Employee_Master +2026-06-15 13:34:59 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse +2026-06-15 13:34:59 | INFO | Employee_Master loaded successfully (2273 rows) +2026-06-15 13:34:59 | INFO | ================================================================================ +2026-06-15 13:34:59 | INFO | Processing Table: Journey_Plan +2026-06-15 13:34:59 | INFO | Fetching Journey Plan for 2026-06 +2026-06-15 13:35:00 | INFO | Fetched 21,141 Journey Plan records +2026-06-15 13:35:00 | INFO | Fetched 21141 rows +2026-06-15 13:35:00 | INFO | Creating table Journey_Plan +2026-06-15 13:35:00 | INFO | Table ready: Journey_Plan +2026-06-15 13:35:00 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse +2026-06-15 13:35:00 | INFO | Journey_Plan loaded successfully (21141 rows) +2026-06-15 13:35:00 | INFO | ================================================================================ +2026-06-15 13:35:00 | INFO | Processing Table: coverage_remarks +2026-06-15 13:35:00 | INFO | Fetching Coverage Remarks +2026-06-15 13:35:00 | INFO | Fetched 29 Coverage Remark records +2026-06-15 13:35:00 | INFO | Fetched 29 rows +2026-06-15 13:35:00 | INFO | Creating table coverage_remarks +2026-06-15 13:35:00 | INFO | Table ready: coverage_remarks +2026-06-15 13:35:01 | INFO | coverage_remarks: inserted 29 rows into ClickHouse +2026-06-15 13:35:01 | INFO | coverage_remarks loaded successfully (29 rows) +2026-06-15 13:35:01 | INFO | ================================================================================ +2026-06-15 13:35:01 | INFO | Processing Table: mapping_storevisibility +2026-06-15 13:35:01 | INFO | Fetching Mapping Store Visibility for 2026-06-14 +2026-06-15 13:35:01 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-15 13:35:01 | WARNING | mapping_storevisibility returned no rows +2026-06-15 13:35:01 | INFO | ================================================================================ +2026-06-15 13:35:01 | INFO | Processing Table: Master_VisibilityReason +2026-06-15 13:35:01 | INFO | Fetching Master Visibility Reason data +2026-06-15 13:35:01 | INFO | Fetched 17 Master Visibility Reason records +2026-06-15 13:35:01 | INFO | Fetched 17 rows +2026-06-15 13:35:01 | INFO | Creating table Master_VisibilityReason +2026-06-15 13:35:01 | INFO | Table ready: Master_VisibilityReason +2026-06-15 13:35:01 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-15 13:35:01 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-15 13:35:01 | INFO | ================================================================================ +2026-06-15 13:35:01 | INFO | Processing Table: Master_VisibilityDefinition +2026-06-15 13:35:01 | INFO | Fetching Master Visibility Definition data +2026-06-15 13:35:01 | INFO | Fetched 861 Master Visibility Definition records +2026-06-15 13:35:01 | INFO | Fetched 861 rows +2026-06-15 13:35:01 | INFO | Creating table Master_VisibilityDefinition +2026-06-15 13:35:02 | INFO | Table ready: Master_VisibilityDefinition +2026-06-15 13:35:02 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-15 13:35:02 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-15 13:35:02 | INFO | ================================================================================ +2026-06-15 13:35:02 | INFO | Processing Table: Web_Logins +2026-06-15 13:35:02 | INFO | Fetching Web Login data for 2026-06-14 +2026-06-15 13:35:02 | INFO | Fetched 25 Web Login records +2026-06-15 13:35:02 | INFO | Fetched 25 rows +2026-06-15 13:35:02 | INFO | Creating table Web_Logins +2026-06-15 13:35:02 | INFO | Table ready: Web_Logins +2026-06-15 13:35:02 | INFO | Web_Logins: inserted 25 rows into ClickHouse +2026-06-15 13:35:02 | INFO | Web_Logins loaded successfully (25 rows) +2026-06-15 13:35:02 | INFO | ================================================================================ +2026-06-15 13:35:02 | INFO | Processing Table: Promotion +2026-06-15 13:35:02 | INFO | Fetching Promotion data for 571 MIDs +2026-06-15 13:35:05 | INFO | Fetched 4,426 Promotion records +2026-06-15 13:35:05 | INFO | Fetched 4426 rows +2026-06-15 13:35:05 | INFO | Creating table Promotion +2026-06-15 13:35:06 | INFO | Table ready: Promotion +2026-06-15 13:35:06 | INFO | Promotion: inserted 4,426 rows into ClickHouse +2026-06-15 13:35:06 | INFO | Promotion loaded successfully (4426 rows) +2026-06-15 13:35:06 | INFO | ================================================================================ +2026-06-15 13:35:06 | INFO | Processing Table: PaidVisibility +2026-06-15 13:35:06 | INFO | Fetching Paid Visibility data for 571 MIDs +2026-06-15 13:35:07 | INFO | Fetched 1,296 Paid Visibility records +2026-06-15 13:35:07 | INFO | Fetched 1296 rows +2026-06-15 13:35:07 | INFO | Creating table PaidVisibility +2026-06-15 13:35:07 | INFO | Table ready: PaidVisibility +2026-06-15 13:35:07 | INFO | PaidVisibility: inserted 1,296 rows into ClickHouse +2026-06-15 13:35:07 | INFO | PaidVisibility loaded successfully (1296 rows) +2026-06-15 13:35:07 | INFO | ================================================================================ +2026-06-15 13:35:07 | INFO | Processing Table: Master_Salesterritorylayer +2026-06-15 13:35:07 | INFO | Fetching Master Sales Territory Layer data +2026-06-15 13:35:08 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-15 13:35:08 | INFO | Fetched 33 rows +2026-06-15 13:35:08 | INFO | Creating table Master_Salesterritorylayer +2026-06-15 13:35:08 | INFO | Table ready: Master_Salesterritorylayer +2026-06-15 13:35:08 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-15 13:35:08 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-15 13:35:08 | INFO | ================================================================================ +2026-06-15 13:35:08 | INFO | Pipeline Completed Successfully +2026-06-15 13:35:08 | INFO | ================================================================================ +2026-06-15 13:35:25 | INFO | ================================================================================ +2026-06-15 13:35:25 | INFO | Hello from data-move Python data pipeline! +2026-06-15 13:35:25 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 13:35:25 | INFO | Connecting to databases... +2026-06-15 13:35:26 | INFO | +2026-06-15 13:35:27 | INFO | +2026-06-15 13:35:27 | INFO | Database connections established +2026-06-15 13:35:27 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 13:35:27 | INFO | Found 571 MIDs +2026-06-15 13:35:28 | INFO | ================================================================================ +2026-06-15 13:35:28 | INFO | Processing Table: SOS_OneApp +2026-06-15 13:35:28 | INFO | Fetching data for 571 MIDs +2026-06-15 13:35:28 | INFO | Fetched 1,997 rows from SQL Server +2026-06-15 13:35:28 | INFO | Fetched 1997 rows +2026-06-15 13:35:28 | INFO | Deleting from SOS_OneApp +2026-06-15 13:35:29 | INFO | SOS_OneApp: inserted 1,997 rows into ClickHouse +2026-06-15 13:35:29 | INFO | SOS_OneApp loaded successfully (1997 rows) +2026-06-15 13:35:29 | INFO | ================================================================================ +2026-06-15 13:35:29 | INFO | Processing Table: OQaD +2026-06-15 13:35:29 | INFO | Fetching OQaD data for run_date=2026-06-14 +2026-06-15 13:35:31 | INFO | Fetched 443 rows +2026-06-15 13:35:31 | INFO | Fetched 443 rows +2026-06-15 13:35:31 | INFO | Deleting from OQaD +2026-06-15 13:35:31 | INFO | OQaD: inserted 443 rows into ClickHouse +2026-06-15 13:35:31 | INFO | OQaD loaded successfully (443 rows) +2026-06-15 13:35:31 | INFO | ================================================================================ +2026-06-15 13:35:31 | INFO | Processing Table: Survey +2026-06-15 13:35:31 | INFO | Fetching Survey data for 571 MIDs +2026-06-15 13:35:31 | INFO | Fetched 174 Survey rows +2026-06-15 13:35:31 | INFO | Fetched 174 rows +2026-06-15 13:35:32 | INFO | Deleting from Survey +2026-06-15 13:35:32 | INFO | Survey: inserted 174 rows into ClickHouse +2026-06-15 13:35:32 | INFO | Survey loaded successfully (174 rows) +2026-06-15 13:35:32 | INFO | ================================================================================ +2026-06-15 13:35:32 | INFO | Processing Table: additional_visibility +2026-06-15 13:35:32 | INFO | Fetching Additional Visibility data for 571 MIDs +2026-06-15 13:35:32 | INFO | Fetched 1,651 Additional Visibility rows +2026-06-15 13:35:32 | INFO | Fetched 1651 rows +2026-06-15 13:35:32 | INFO | Deleting from additional_visibility +2026-06-15 13:35:32 | INFO | additional_visibility: inserted 1,651 rows into ClickHouse +2026-06-15 13:35:32 | INFO | additional_visibility loaded successfully (1651 rows) +2026-06-15 13:35:32 | INFO | ================================================================================ +2026-06-15 13:35:32 | INFO | Processing Table: Coverage +2026-06-15 13:35:32 | INFO | Fetching coverage data for 571 MIDs +2026-06-15 13:35:33 | INFO | Fetched 514 rows from SQL Server +2026-06-15 13:35:33 | INFO | Fetched 514 rows +2026-06-15 13:35:33 | INFO | Deleting from Coverage +2026-06-15 13:35:33 | INFO | Coverage: inserted 514 rows into ClickHouse +2026-06-15 13:35:33 | INFO | Coverage loaded successfully (514 rows) +2026-06-15 13:35:33 | INFO | ================================================================================ +2026-06-15 13:35:33 | INFO | Processing Table: Login +2026-06-15 13:35:33 | INFO | Fetching Login data for yesterday +2026-06-15 13:35:33 | INFO | Fetched 454 Login rows +2026-06-15 13:35:33 | INFO | Fetched 454 rows +2026-06-15 13:35:34 | INFO | Deleting from Login +2026-06-15 13:35:34 | INFO | No delete logic required for Login +2026-06-15 13:35:34 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-15 13:35:34 | INFO | Login loaded successfully (454 rows) +2026-06-15 13:35:34 | INFO | ================================================================================ +2026-06-15 13:35:34 | INFO | Processing Table: Stock_Details +2026-06-15 13:35:34 | INFO | Fetching Stock Details data for 571 MIDs +2026-06-15 13:35:36 | INFO | Fetched 28,574 Stock Details rows +2026-06-15 13:35:36 | INFO | Fetched 28574 rows +2026-06-15 13:35:36 | INFO | Deleting from Stock_Details +2026-06-15 13:35:36 | INFO | Stock_Details: inserted 28,574 rows into ClickHouse +2026-06-15 13:35:36 | INFO | Stock_Details loaded successfully (28574 rows) +2026-06-15 13:35:36 | INFO | ================================================================================ +2026-06-15 13:35:36 | INFO | Processing Table: Attendance +2026-06-15 13:35:36 | INFO | Fetching Attendance data from 2026-05-30 to 2026-06-14 +2026-06-15 13:35:37 | INFO | Fetched 11,543 attendance rows for 594 employees +2026-06-15 13:35:37 | INFO | Fetched 11543 rows +2026-06-15 13:35:37 | INFO | Truncated table Attendance +2026-06-15 13:35:37 | INFO | Attendance: inserted 11,543 rows into ClickHouse +2026-06-15 13:35:37 | INFO | Attendance loaded successfully (11543 rows) +2026-06-15 13:35:37 | INFO | ================================================================================ +2026-06-15 13:35:37 | INFO | Processing Table: Store_Master +2026-06-15 13:35:37 | INFO | Fetching Store Master data +2026-06-15 13:35:39 | INFO | Fetched 5,984 stores +2026-06-15 13:35:39 | INFO | Fetched 5984 rows +2026-06-15 13:35:39 | INFO | Truncated table Store_Master +2026-06-15 13:35:40 | INFO | Store_Master: inserted 5,984 rows into ClickHouse +2026-06-15 13:35:40 | INFO | Store_Master loaded successfully (5984 rows) +2026-06-15 13:35:40 | INFO | ================================================================================ +2026-06-15 13:35:40 | INFO | Processing Table: SKU_Master +2026-06-15 13:35:40 | INFO | Fetching SKU Master data +2026-06-15 13:35:40 | INFO | Fetched 160 SKU Master rows +2026-06-15 13:35:40 | INFO | Fetched 160 rows +2026-06-15 13:35:40 | INFO | Truncated table SKU_Master +2026-06-15 13:35:40 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-15 13:35:40 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-15 13:35:40 | INFO | ================================================================================ +2026-06-15 13:35:40 | INFO | Processing Table: display_master +2026-06-15 13:35:40 | INFO | Fetching Display Master data +2026-06-15 13:35:40 | INFO | Fetched 135 Display Master records +2026-06-15 13:35:40 | INFO | Fetched 135 rows +2026-06-15 13:35:41 | INFO | Truncated table display_master +2026-06-15 13:35:41 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-15 13:35:41 | INFO | display_master loaded successfully (135 rows) +2026-06-15 13:35:41 | INFO | ================================================================================ +2026-06-15 13:35:41 | INFO | Processing Table: Employee_Master +2026-06-15 13:35:41 | INFO | Fetching Employee Master data +2026-06-15 13:35:41 | INFO | Fetched 2,273 Employee Master records +2026-06-15 13:35:41 | INFO | Fetched 2273 rows +2026-06-15 13:35:41 | INFO | Truncated table Employee_Master +2026-06-15 13:35:42 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse +2026-06-15 13:35:42 | INFO | Employee_Master loaded successfully (2273 rows) +2026-06-15 13:35:42 | INFO | ================================================================================ +2026-06-15 13:35:42 | INFO | Processing Table: Journey_Plan +2026-06-15 13:35:42 | INFO | Fetching Journey Plan for 2026-06 +2026-06-15 13:35:42 | INFO | Fetched 21,141 Journey Plan records +2026-06-15 13:35:42 | INFO | Fetched 21141 rows +2026-06-15 13:35:42 | INFO | Deleting from Journey_Plan +2026-06-15 13:35:42 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse +2026-06-15 13:35:42 | INFO | Journey_Plan loaded successfully (21141 rows) +2026-06-15 13:35:42 | INFO | ================================================================================ +2026-06-15 13:35:42 | INFO | Processing Table: coverage_remarks +2026-06-15 13:35:42 | INFO | Fetching Coverage Remarks +2026-06-15 13:35:43 | INFO | Fetched 29 Coverage Remark records +2026-06-15 13:35:43 | INFO | Fetched 29 rows +2026-06-15 13:35:43 | INFO | Truncated table coverage_remarks +2026-06-15 13:35:43 | INFO | coverage_remarks: inserted 29 rows into ClickHouse +2026-06-15 13:35:43 | INFO | coverage_remarks loaded successfully (29 rows) +2026-06-15 13:35:43 | INFO | ================================================================================ +2026-06-15 13:35:43 | INFO | Processing Table: mapping_storevisibility +2026-06-15 13:35:43 | INFO | Fetching Mapping Store Visibility for 2026-06-14 +2026-06-15 13:35:43 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-15 13:35:43 | WARNING | mapping_storevisibility returned no rows +2026-06-15 13:35:43 | INFO | ================================================================================ +2026-06-15 13:35:43 | INFO | Processing Table: Master_VisibilityReason +2026-06-15 13:35:43 | INFO | Fetching Master Visibility Reason data +2026-06-15 13:35:43 | INFO | Fetched 17 Master Visibility Reason records +2026-06-15 13:35:43 | INFO | Fetched 17 rows +2026-06-15 13:35:43 | INFO | Truncated table Master_VisibilityReason +2026-06-15 13:35:43 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-15 13:35:43 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-15 13:35:43 | INFO | ================================================================================ +2026-06-15 13:35:43 | INFO | Processing Table: Master_VisibilityDefinition +2026-06-15 13:35:43 | INFO | Fetching Master Visibility Definition data +2026-06-15 13:35:44 | INFO | Fetched 861 Master Visibility Definition records +2026-06-15 13:35:44 | INFO | Fetched 861 rows +2026-06-15 13:35:44 | INFO | Truncated table Master_VisibilityDefinition +2026-06-15 13:35:44 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-15 13:35:44 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-15 13:35:44 | INFO | ================================================================================ +2026-06-15 13:35:44 | INFO | Processing Table: Web_Logins +2026-06-15 13:35:44 | INFO | Fetching Web Login data for 2026-06-14 +2026-06-15 13:35:44 | INFO | Fetched 25 Web Login records +2026-06-15 13:35:44 | INFO | Fetched 25 rows +2026-06-15 13:35:44 | INFO | Deleting from Web_Logins +2026-06-15 13:35:45 | INFO | Web_Logins: inserted 25 rows into ClickHouse +2026-06-15 13:35:45 | INFO | Web_Logins loaded successfully (25 rows) +2026-06-15 13:35:45 | INFO | ================================================================================ +2026-06-15 13:35:45 | INFO | Processing Table: Promotion +2026-06-15 13:35:45 | INFO | Fetching Promotion data for 571 MIDs +2026-06-15 13:35:46 | INFO | Fetched 4,426 Promotion records +2026-06-15 13:35:46 | INFO | Fetched 4426 rows +2026-06-15 13:35:46 | INFO | Deleting from Promotion +2026-06-15 13:35:47 | INFO | Promotion: inserted 4,426 rows into ClickHouse +2026-06-15 13:35:47 | INFO | Promotion loaded successfully (4426 rows) +2026-06-15 13:35:47 | INFO | ================================================================================ +2026-06-15 13:35:47 | INFO | Processing Table: PaidVisibility +2026-06-15 13:35:47 | INFO | Fetching Paid Visibility data for 571 MIDs +2026-06-15 13:35:48 | INFO | Fetched 1,296 Paid Visibility records +2026-06-15 13:35:48 | INFO | Fetched 1296 rows +2026-06-15 13:35:48 | INFO | Deleting from PaidVisibility +2026-06-15 13:35:48 | INFO | PaidVisibility: inserted 1,296 rows into ClickHouse +2026-06-15 13:35:48 | INFO | PaidVisibility loaded successfully (1296 rows) +2026-06-15 13:35:48 | INFO | ================================================================================ +2026-06-15 13:35:48 | INFO | Processing Table: Master_Salesterritorylayer +2026-06-15 13:35:48 | INFO | Fetching Master Sales Territory Layer data +2026-06-15 13:35:48 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-15 13:35:48 | INFO | Fetched 33 rows +2026-06-15 13:35:48 | INFO | Truncated table Master_Salesterritorylayer +2026-06-15 13:35:48 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-15 13:35:48 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-15 13:35:48 | INFO | ================================================================================ +2026-06-15 13:35:48 | INFO | Pipeline Completed Successfully +2026-06-15 13:35:48 | INFO | ================================================================================ +2026-06-15 13:35:54 | INFO | ================================================================================ +2026-06-15 13:35:54 | INFO | Hello from data-move Python data pipeline! +2026-06-15 13:35:54 | INFO | Pipeline Run Date: 2026-06-14 +2026-06-15 13:35:54 | INFO | Connecting to databases... +2026-06-15 13:35:55 | INFO | +2026-06-15 13:35:56 | INFO | +2026-06-15 13:35:56 | INFO | Database connections established +2026-06-15 13:35:56 | INFO | Collecting MIDs for: 2026-06-14 +2026-06-15 13:35:56 | INFO | Found 571 MIDs +2026-06-15 13:35:57 | INFO | ================================================================================ +2026-06-15 13:35:57 | INFO | Processing Table: SOS_OneApp +2026-06-15 13:35:57 | INFO | Fetching data for 571 MIDs +2026-06-15 13:35:57 | INFO | Fetched 1,997 rows from SQL Server +2026-06-15 13:35:57 | INFO | Fetched 1997 rows +2026-06-15 13:35:57 | INFO | Deleting from SOS_OneApp +2026-06-15 13:35:58 | INFO | SOS_OneApp: inserted 1,997 rows into ClickHouse +2026-06-15 13:35:58 | INFO | SOS_OneApp loaded successfully (1997 rows) +2026-06-15 13:35:58 | INFO | ================================================================================ +2026-06-15 13:35:58 | INFO | Processing Table: OQaD +2026-06-15 13:35:58 | INFO | Fetching OQaD data for run_date=2026-06-14 +2026-06-15 13:36:01 | INFO | Fetched 443 rows +2026-06-15 13:36:01 | INFO | Fetched 443 rows +2026-06-15 13:36:01 | INFO | Deleting from OQaD +2026-06-15 13:36:01 | INFO | OQaD: inserted 443 rows into ClickHouse +2026-06-15 13:36:01 | INFO | OQaD loaded successfully (443 rows) +2026-06-15 13:36:01 | INFO | ================================================================================ +2026-06-15 13:36:01 | INFO | Processing Table: Survey +2026-06-15 13:36:01 | INFO | Fetching Survey data for 571 MIDs +2026-06-15 13:36:01 | INFO | Fetched 174 Survey rows +2026-06-15 13:36:01 | INFO | Fetched 174 rows +2026-06-15 13:36:01 | INFO | Deleting from Survey +2026-06-15 13:36:01 | INFO | Survey: inserted 174 rows into ClickHouse +2026-06-15 13:36:01 | INFO | Survey loaded successfully (174 rows) +2026-06-15 13:36:01 | INFO | ================================================================================ +2026-06-15 13:36:01 | INFO | Processing Table: additional_visibility +2026-06-15 13:36:01 | INFO | Fetching Additional Visibility data for 571 MIDs +2026-06-15 13:36:02 | INFO | Fetched 1,651 Additional Visibility rows +2026-06-15 13:36:02 | INFO | Fetched 1651 rows +2026-06-15 13:36:02 | INFO | Deleting from additional_visibility +2026-06-15 13:36:02 | INFO | additional_visibility: inserted 1,651 rows into ClickHouse +2026-06-15 13:36:02 | INFO | additional_visibility loaded successfully (1651 rows) +2026-06-15 13:36:02 | INFO | ================================================================================ +2026-06-15 13:36:02 | INFO | Processing Table: Coverage +2026-06-15 13:36:02 | INFO | Fetching coverage data for 571 MIDs +2026-06-15 13:36:02 | INFO | Fetched 514 rows from SQL Server +2026-06-15 13:36:02 | INFO | Fetched 514 rows +2026-06-15 13:36:02 | INFO | Deleting from Coverage +2026-06-15 13:36:02 | INFO | Coverage: inserted 514 rows into ClickHouse +2026-06-15 13:36:02 | INFO | Coverage loaded successfully (514 rows) +2026-06-15 13:36:02 | INFO | ================================================================================ +2026-06-15 13:36:02 | INFO | Processing Table: Login +2026-06-15 13:36:02 | INFO | Fetching Login data for yesterday +2026-06-15 13:36:03 | INFO | Fetched 454 Login rows +2026-06-15 13:36:03 | INFO | Fetched 454 rows +2026-06-15 13:36:03 | INFO | Deleting from Login +2026-06-15 13:36:03 | INFO | No delete logic required for Login +2026-06-15 13:36:03 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-15 13:36:03 | INFO | Login loaded successfully (454 rows) +2026-06-15 13:36:03 | INFO | ================================================================================ +2026-06-15 13:36:03 | INFO | Processing Table: Stock_Details +2026-06-15 13:36:03 | INFO | Fetching Stock Details data for 571 MIDs +2026-06-15 13:36:09 | INFO | Fetched 28,574 Stock Details rows +2026-06-15 13:36:09 | INFO | Fetched 28574 rows +2026-06-15 13:36:09 | INFO | Deleting from Stock_Details +2026-06-15 13:36:09 | INFO | Stock_Details: inserted 28,574 rows into ClickHouse +2026-06-15 13:36:09 | INFO | Stock_Details loaded successfully (28574 rows) +2026-06-15 13:36:09 | INFO | ================================================================================ +2026-06-15 13:36:09 | INFO | Processing Table: Attendance +2026-06-15 13:36:09 | INFO | Fetching Attendance data from 2026-05-30 to 2026-06-14 +2026-06-15 13:36:10 | INFO | Fetched 11,543 attendance rows for 594 employees +2026-06-15 13:36:10 | INFO | Fetched 11543 rows +2026-06-15 13:36:10 | INFO | Truncated table Attendance +2026-06-15 13:36:10 | INFO | Attendance: inserted 11,543 rows into ClickHouse +2026-06-15 13:36:10 | INFO | Attendance loaded successfully (11543 rows) +2026-06-15 13:36:10 | INFO | ================================================================================ +2026-06-15 13:36:10 | INFO | Processing Table: Store_Master +2026-06-15 13:36:10 | INFO | Fetching Store Master data +2026-06-15 13:36:12 | INFO | Fetched 5,984 stores +2026-06-15 13:36:12 | INFO | Fetched 5984 rows +2026-06-15 13:36:12 | INFO | Truncated table Store_Master +2026-06-15 13:36:13 | INFO | Store_Master: inserted 5,984 rows into ClickHouse +2026-06-15 13:36:13 | INFO | Store_Master loaded successfully (5984 rows) +2026-06-15 13:36:13 | INFO | ================================================================================ +2026-06-15 13:36:13 | INFO | Processing Table: SKU_Master +2026-06-15 13:36:13 | INFO | Fetching SKU Master data +2026-06-15 13:36:13 | INFO | Fetched 160 SKU Master rows +2026-06-15 13:36:13 | INFO | Fetched 160 rows +2026-06-15 13:36:13 | INFO | Truncated table SKU_Master +2026-06-15 13:36:13 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-15 13:36:13 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-15 13:36:13 | INFO | ================================================================================ +2026-06-15 13:36:13 | INFO | Processing Table: display_master +2026-06-15 13:36:13 | INFO | Fetching Display Master data +2026-06-15 13:36:13 | INFO | Fetched 135 Display Master records +2026-06-15 13:36:13 | INFO | Fetched 135 rows +2026-06-15 13:36:13 | INFO | Truncated table display_master +2026-06-15 13:36:13 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-15 13:36:13 | INFO | display_master loaded successfully (135 rows) +2026-06-15 13:36:13 | INFO | ================================================================================ +2026-06-15 13:36:13 | INFO | Processing Table: Employee_Master +2026-06-15 13:36:13 | INFO | Fetching Employee Master data +2026-06-15 13:36:14 | INFO | Fetched 2,273 Employee Master records +2026-06-15 13:36:14 | INFO | Fetched 2273 rows +2026-06-15 13:36:14 | INFO | Truncated table Employee_Master +2026-06-15 13:36:14 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse +2026-06-15 13:36:14 | INFO | Employee_Master loaded successfully (2273 rows) +2026-06-15 13:36:14 | INFO | ================================================================================ +2026-06-15 13:36:14 | INFO | Processing Table: Journey_Plan +2026-06-15 13:36:14 | INFO | Fetching Journey Plan for 2026-06 +2026-06-15 13:36:15 | INFO | Fetched 21,141 Journey Plan records +2026-06-15 13:36:15 | INFO | Fetched 21141 rows +2026-06-15 13:36:15 | INFO | Deleting from Journey_Plan +2026-06-15 13:36:15 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse +2026-06-15 13:36:15 | INFO | Journey_Plan loaded successfully (21141 rows) +2026-06-15 13:36:15 | INFO | ================================================================================ +2026-06-15 13:36:15 | INFO | Processing Table: coverage_remarks +2026-06-15 13:36:15 | INFO | Fetching Coverage Remarks +2026-06-15 13:36:15 | INFO | Fetched 29 Coverage Remark records +2026-06-15 13:36:15 | INFO | Fetched 29 rows +2026-06-15 13:36:15 | INFO | Truncated table coverage_remarks +2026-06-15 13:36:15 | INFO | coverage_remarks: inserted 29 rows into ClickHouse +2026-06-15 13:36:15 | INFO | coverage_remarks loaded successfully (29 rows) +2026-06-15 13:36:15 | INFO | ================================================================================ +2026-06-15 13:36:15 | INFO | Processing Table: mapping_storevisibility +2026-06-15 13:36:15 | INFO | Fetching Mapping Store Visibility for 2026-06-14 +2026-06-15 13:36:15 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-15 13:36:15 | WARNING | mapping_storevisibility returned no rows +2026-06-15 13:36:15 | INFO | ================================================================================ +2026-06-15 13:36:15 | INFO | Processing Table: Master_VisibilityReason +2026-06-15 13:36:15 | INFO | Fetching Master Visibility Reason data +2026-06-15 13:36:15 | INFO | Fetched 17 Master Visibility Reason records +2026-06-15 13:36:15 | INFO | Fetched 17 rows +2026-06-15 13:36:16 | INFO | Truncated table Master_VisibilityReason +2026-06-15 13:36:16 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-15 13:36:16 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-15 13:36:16 | INFO | ================================================================================ +2026-06-15 13:36:16 | INFO | Processing Table: Master_VisibilityDefinition +2026-06-15 13:36:16 | INFO | Fetching Master Visibility Definition data +2026-06-15 13:36:16 | INFO | Fetched 861 Master Visibility Definition records +2026-06-15 13:36:16 | INFO | Fetched 861 rows +2026-06-15 13:36:16 | INFO | Truncated table Master_VisibilityDefinition +2026-06-15 13:36:16 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-15 13:36:16 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-15 13:36:16 | INFO | ================================================================================ +2026-06-15 13:36:16 | INFO | Processing Table: Web_Logins +2026-06-15 13:36:16 | INFO | Fetching Web Login data for 2026-06-14 +2026-06-15 13:36:16 | INFO | Fetched 25 Web Login records +2026-06-15 13:36:16 | INFO | Fetched 25 rows +2026-06-15 13:36:16 | INFO | Deleting from Web_Logins +2026-06-15 13:36:16 | INFO | Web_Logins: inserted 25 rows into ClickHouse +2026-06-15 13:36:16 | INFO | Web_Logins loaded successfully (25 rows) +2026-06-15 13:36:16 | INFO | ================================================================================ +2026-06-15 13:36:16 | INFO | Processing Table: Promotion +2026-06-15 13:36:16 | INFO | Fetching Promotion data for 571 MIDs +2026-06-15 13:36:18 | INFO | Fetched 4,426 Promotion records +2026-06-15 13:36:18 | INFO | Fetched 4426 rows +2026-06-15 13:36:18 | INFO | Deleting from Promotion +2026-06-15 13:36:18 | INFO | Promotion: inserted 4,426 rows into ClickHouse +2026-06-15 13:36:18 | INFO | Promotion loaded successfully (4426 rows) +2026-06-15 13:36:18 | INFO | ================================================================================ +2026-06-15 13:36:18 | INFO | Processing Table: PaidVisibility +2026-06-15 13:36:18 | INFO | Fetching Paid Visibility data for 571 MIDs +2026-06-15 13:36:19 | INFO | Fetched 1,296 Paid Visibility records +2026-06-15 13:36:19 | INFO | Fetched 1296 rows +2026-06-15 13:36:19 | INFO | Deleting from PaidVisibility +2026-06-15 13:36:20 | INFO | PaidVisibility: inserted 1,296 rows into ClickHouse +2026-06-15 13:36:20 | INFO | PaidVisibility loaded successfully (1296 rows) +2026-06-15 13:36:20 | INFO | ================================================================================ +2026-06-15 13:36:20 | INFO | Processing Table: Master_Salesterritorylayer +2026-06-15 13:36:20 | INFO | Fetching Master Sales Territory Layer data +2026-06-15 13:36:20 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-15 13:36:20 | INFO | Fetched 33 rows +2026-06-15 13:36:20 | INFO | Truncated table Master_Salesterritorylayer +2026-06-15 13:36:20 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-15 13:36:20 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-15 13:36:20 | INFO | ================================================================================ +2026-06-15 13:36:20 | INFO | Pipeline Completed Successfully +2026-06-15 13:36:20 | INFO | ================================================================================ +2026-06-15 13:37:03 | INFO | ================================================================================ +2026-06-15 13:37:03 | INFO | Hello from data-move Python data pipeline! +2026-06-15 13:37:03 | INFO | Pipeline Run Date: 2026-06-03 +2026-06-15 13:37:03 | INFO | Connecting to databases... +2026-06-15 13:37:04 | INFO | +2026-06-15 13:37:05 | INFO | +2026-06-15 13:37:06 | INFO | Database connections established +2026-06-15 13:37:06 | INFO | Collecting MIDs for: 2026-06-03 +2026-06-15 13:37:06 | INFO | Found 808 MIDs +2026-06-15 13:37:06 | INFO | ================================================================================ +2026-06-15 13:37:06 | INFO | Processing Table: SOS_OneApp +2026-06-15 13:37:06 | INFO | Fetching data for 808 MIDs +2026-06-15 13:37:07 | INFO | Fetched 3,565 rows from SQL Server +2026-06-15 13:37:07 | INFO | Fetched 3565 rows +2026-06-15 13:37:08 | INFO | Deleting from SOS_OneApp +2026-06-15 13:37:08 | INFO | SOS_OneApp: inserted 3,565 rows into ClickHouse +2026-06-15 13:37:08 | INFO | SOS_OneApp loaded successfully (3565 rows) +2026-06-15 13:37:08 | INFO | ================================================================================ +2026-06-15 13:37:08 | INFO | Processing Table: OQaD +2026-06-15 13:37:08 | INFO | Fetching OQaD data for run_date=2026-06-03 +2026-06-15 13:37:12 | INFO | Fetched 455 rows +2026-06-15 13:37:12 | INFO | Fetched 455 rows +2026-06-15 13:37:12 | INFO | Deleting from OQaD +2026-06-15 13:37:12 | INFO | OQaD: inserted 455 rows into ClickHouse +2026-06-15 13:37:12 | INFO | OQaD loaded successfully (455 rows) +2026-06-15 13:37:12 | INFO | ================================================================================ +2026-06-15 13:37:12 | INFO | Processing Table: Survey +2026-06-15 13:37:12 | INFO | Fetching Survey data for 808 MIDs +2026-06-15 13:37:13 | INFO | Fetched 139 Survey rows +2026-06-15 13:37:13 | INFO | Fetched 139 rows +2026-06-15 13:37:13 | INFO | Deleting from Survey +2026-06-15 13:37:13 | INFO | Survey: inserted 139 rows into ClickHouse +2026-06-15 13:37:13 | INFO | Survey loaded successfully (139 rows) +2026-06-15 13:37:13 | INFO | ================================================================================ +2026-06-15 13:37:13 | INFO | Processing Table: additional_visibility +2026-06-15 13:37:13 | INFO | Fetching Additional Visibility data for 808 MIDs +2026-06-15 13:37:14 | INFO | Fetched 1,884 Additional Visibility rows +2026-06-15 13:37:14 | INFO | Fetched 1884 rows +2026-06-15 13:37:14 | INFO | Deleting from additional_visibility +2026-06-15 13:37:14 | INFO | additional_visibility: inserted 1,884 rows into ClickHouse +2026-06-15 13:37:14 | INFO | additional_visibility loaded successfully (1884 rows) +2026-06-15 13:37:14 | INFO | ================================================================================ +2026-06-15 13:37:14 | INFO | Processing Table: Coverage +2026-06-15 13:37:14 | INFO | Fetching coverage data for 808 MIDs +2026-06-15 13:37:15 | INFO | Fetched 751 rows from SQL Server +2026-06-15 13:37:15 | INFO | Fetched 751 rows +2026-06-15 13:37:15 | INFO | Deleting from Coverage +2026-06-15 13:37:15 | INFO | Coverage: inserted 751 rows into ClickHouse +2026-06-15 13:37:15 | INFO | Coverage loaded successfully (751 rows) +2026-06-15 13:37:15 | INFO | ================================================================================ +2026-06-15 13:37:15 | INFO | Processing Table: Login +2026-06-15 13:37:15 | INFO | Fetching Login data for yesterday +2026-06-15 13:37:16 | INFO | Fetched 454 Login rows +2026-06-15 13:37:16 | INFO | Fetched 454 rows +2026-06-15 13:37:16 | INFO | Deleting from Login +2026-06-15 13:37:16 | INFO | No delete logic required for Login +2026-06-15 13:37:16 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-15 13:37:16 | INFO | Login loaded successfully (454 rows) +2026-06-15 13:37:16 | INFO | ================================================================================ +2026-06-15 13:37:16 | INFO | Processing Table: Stock_Details +2026-06-15 13:37:16 | INFO | Fetching Stock Details data for 808 MIDs +2026-06-15 13:37:19 | INFO | Fetched 40,840 Stock Details rows +2026-06-15 13:37:19 | INFO | Fetched 40840 rows +2026-06-15 13:37:19 | INFO | Deleting from Stock_Details +2026-06-15 13:37:20 | INFO | Stock_Details: inserted 40,840 rows into ClickHouse +2026-06-15 13:37:20 | INFO | Stock_Details loaded successfully (40840 rows) +2026-06-15 13:37:20 | INFO | ================================================================================ +2026-06-15 13:37:20 | INFO | Processing Table: Attendance +2026-06-15 13:37:20 | INFO | Fetching Attendance data from 2026-05-19 to 2026-06-03 +2026-06-15 13:37:21 | INFO | Fetched 12,432 attendance rows for 579 employees +2026-06-15 13:37:21 | INFO | Fetched 12432 rows +2026-06-15 13:37:21 | INFO | Truncated table Attendance +2026-06-15 13:37:21 | INFO | Attendance: inserted 12,432 rows into ClickHouse +2026-06-15 13:37:21 | INFO | Attendance loaded successfully (12432 rows) +2026-06-15 13:37:21 | INFO | ================================================================================ +2026-06-15 13:37:21 | INFO | Processing Table: Store_Master +2026-06-15 13:37:21 | INFO | Fetching Store Master data +2026-06-15 13:37:22 | INFO | Fetched 5,984 stores +2026-06-15 13:37:22 | INFO | Fetched 5984 rows +2026-06-15 13:37:22 | INFO | Truncated table Store_Master +2026-06-15 13:37:22 | INFO | Store_Master: inserted 5,984 rows into ClickHouse +2026-06-15 13:37:22 | INFO | Store_Master loaded successfully (5984 rows) +2026-06-15 13:37:22 | INFO | ================================================================================ +2026-06-15 13:37:22 | INFO | Processing Table: SKU_Master +2026-06-15 13:37:22 | INFO | Fetching SKU Master data +2026-06-15 13:37:22 | INFO | Fetched 160 SKU Master rows +2026-06-15 13:37:22 | INFO | Fetched 160 rows +2026-06-15 13:37:23 | INFO | Truncated table SKU_Master +2026-06-15 13:37:23 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-15 13:37:23 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-15 13:37:23 | INFO | ================================================================================ +2026-06-15 13:37:23 | INFO | Processing Table: display_master +2026-06-15 13:37:23 | INFO | Fetching Display Master data +2026-06-15 13:37:23 | INFO | Fetched 135 Display Master records +2026-06-15 13:37:23 | INFO | Fetched 135 rows +2026-06-15 13:37:23 | INFO | Truncated table display_master +2026-06-15 13:37:23 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-15 13:37:23 | INFO | display_master loaded successfully (135 rows) +2026-06-15 13:37:23 | INFO | ================================================================================ +2026-06-15 13:37:23 | INFO | Processing Table: Employee_Master +2026-06-15 13:37:23 | INFO | Fetching Employee Master data +2026-06-15 13:37:24 | INFO | Fetched 2,273 Employee Master records +2026-06-15 13:37:24 | INFO | Fetched 2273 rows +2026-06-15 13:37:24 | INFO | Truncated table Employee_Master +2026-06-15 13:37:25 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse +2026-06-15 13:37:25 | INFO | Employee_Master loaded successfully (2273 rows) +2026-06-15 13:37:25 | INFO | ================================================================================ +2026-06-15 13:37:25 | INFO | Processing Table: Journey_Plan +2026-06-15 13:37:25 | INFO | Fetching Journey Plan for 2026-06 +2026-06-15 13:37:25 | INFO | Fetched 21,141 Journey Plan records +2026-06-15 13:37:25 | INFO | Fetched 21141 rows +2026-06-15 13:37:25 | INFO | Deleting from Journey_Plan +2026-06-15 13:37:25 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse +2026-06-15 13:37:25 | INFO | Journey_Plan loaded successfully (21141 rows) +2026-06-15 13:37:25 | INFO | ================================================================================ +2026-06-15 13:37:25 | INFO | Processing Table: coverage_remarks +2026-06-15 13:37:25 | INFO | Fetching Coverage Remarks +2026-06-15 13:37:25 | INFO | Fetched 29 Coverage Remark records +2026-06-15 13:37:25 | INFO | Fetched 29 rows +2026-06-15 13:37:25 | INFO | Truncated table coverage_remarks +2026-06-15 13:37:26 | INFO | coverage_remarks: inserted 29 rows into ClickHouse +2026-06-15 13:37:26 | INFO | coverage_remarks loaded successfully (29 rows) +2026-06-15 13:37:26 | INFO | ================================================================================ +2026-06-15 13:37:26 | INFO | Processing Table: mapping_storevisibility +2026-06-15 13:37:26 | INFO | Fetching Mapping Store Visibility for 2026-06-03 +2026-06-15 13:37:26 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-15 13:37:26 | WARNING | mapping_storevisibility returned no rows +2026-06-15 13:37:26 | INFO | ================================================================================ +2026-06-15 13:37:26 | INFO | Processing Table: Master_VisibilityReason +2026-06-15 13:37:26 | INFO | Fetching Master Visibility Reason data +2026-06-15 13:37:26 | INFO | Fetched 17 Master Visibility Reason records +2026-06-15 13:37:26 | INFO | Fetched 17 rows +2026-06-15 13:37:26 | INFO | Truncated table Master_VisibilityReason +2026-06-15 13:37:26 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-15 13:37:26 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-15 13:37:26 | INFO | ================================================================================ +2026-06-15 13:37:26 | INFO | Processing Table: Master_VisibilityDefinition +2026-06-15 13:37:26 | INFO | Fetching Master Visibility Definition data +2026-06-15 13:37:26 | INFO | Fetched 861 Master Visibility Definition records +2026-06-15 13:37:26 | INFO | Fetched 861 rows +2026-06-15 13:37:26 | INFO | Truncated table Master_VisibilityDefinition +2026-06-15 13:37:26 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-15 13:37:26 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-15 13:37:26 | INFO | ================================================================================ +2026-06-15 13:37:26 | INFO | Processing Table: Web_Logins +2026-06-15 13:37:26 | INFO | Fetching Web Login data for 2026-06-03 +2026-06-15 13:37:27 | INFO | Fetched 200 Web Login records +2026-06-15 13:37:27 | INFO | Fetched 200 rows +2026-06-15 13:37:27 | INFO | Deleting from Web_Logins +2026-06-15 13:37:27 | INFO | Web_Logins: inserted 200 rows into ClickHouse +2026-06-15 13:37:27 | INFO | Web_Logins loaded successfully (200 rows) +2026-06-15 13:37:27 | INFO | ================================================================================ +2026-06-15 13:37:27 | INFO | Processing Table: Promotion +2026-06-15 13:37:27 | INFO | Fetching Promotion data for 808 MIDs +2026-06-15 13:37:32 | INFO | Fetched 8,162 Promotion records +2026-06-15 13:37:32 | INFO | Fetched 8162 rows +2026-06-15 13:37:32 | INFO | Deleting from Promotion +2026-06-15 13:37:32 | INFO | Promotion: inserted 8,162 rows into ClickHouse +2026-06-15 13:37:32 | INFO | Promotion loaded successfully (8162 rows) +2026-06-15 13:37:32 | INFO | ================================================================================ +2026-06-15 13:37:32 | INFO | Processing Table: PaidVisibility +2026-06-15 13:37:32 | INFO | Fetching Paid Visibility data for 808 MIDs +2026-06-15 13:37:34 | INFO | Fetched 1,064 Paid Visibility records +2026-06-15 13:37:34 | INFO | Fetched 1064 rows +2026-06-15 13:37:34 | INFO | Deleting from PaidVisibility +2026-06-15 13:37:34 | INFO | PaidVisibility: inserted 1,064 rows into ClickHouse +2026-06-15 13:37:34 | INFO | PaidVisibility loaded successfully (1064 rows) +2026-06-15 13:37:34 | INFO | ================================================================================ +2026-06-15 13:37:34 | INFO | Processing Table: Master_Salesterritorylayer +2026-06-15 13:37:34 | INFO | Fetching Master Sales Territory Layer data +2026-06-15 13:37:34 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-15 13:37:34 | INFO | Fetched 33 rows +2026-06-15 13:37:34 | INFO | Truncated table Master_Salesterritorylayer +2026-06-15 13:37:34 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-15 13:37:34 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-15 13:37:34 | INFO | ================================================================================ +2026-06-15 13:37:34 | INFO | Pipeline Completed Successfully +2026-06-15 13:37:34 | INFO | ================================================================================ +2026-06-15 13:37:43 | INFO | ================================================================================ +2026-06-15 13:37:43 | INFO | Hello from data-move Python data pipeline! +2026-06-15 13:37:43 | INFO | Pipeline Run Date: 2026-06-03 +2026-06-15 13:37:43 | INFO | Connecting to databases... +2026-06-15 13:37:44 | INFO | +2026-06-15 13:37:44 | INFO | +2026-06-15 13:37:45 | INFO | Database connections established +2026-06-15 13:37:45 | INFO | Collecting MIDs for: 2026-06-03 +2026-06-15 13:37:45 | INFO | Found 808 MIDs +2026-06-15 13:37:45 | INFO | ================================================================================ +2026-06-15 13:37:45 | INFO | Processing Table: SOS_OneApp +2026-06-15 13:37:45 | INFO | Fetching data for 808 MIDs +2026-06-15 13:37:46 | INFO | Fetched 3,565 rows from SQL Server +2026-06-15 13:37:46 | INFO | Fetched 3565 rows +2026-06-15 13:37:46 | INFO | Deleting from SOS_OneApp +2026-06-15 13:37:46 | INFO | SOS_OneApp: inserted 3,565 rows into ClickHouse +2026-06-15 13:37:46 | INFO | SOS_OneApp loaded successfully (3565 rows) +2026-06-15 13:37:46 | INFO | ================================================================================ +2026-06-15 13:37:46 | INFO | Processing Table: OQaD +2026-06-15 13:37:46 | INFO | Fetching OQaD data for run_date=2026-06-03 +2026-06-15 13:37:49 | INFO | Fetched 455 rows +2026-06-15 13:37:49 | INFO | Fetched 455 rows +2026-06-15 13:37:49 | INFO | Deleting from OQaD +2026-06-15 13:37:49 | INFO | OQaD: inserted 455 rows into ClickHouse +2026-06-15 13:37:49 | INFO | OQaD loaded successfully (455 rows) +2026-06-15 13:37:49 | INFO | ================================================================================ +2026-06-15 13:37:49 | INFO | Processing Table: Survey +2026-06-15 13:37:49 | INFO | Fetching Survey data for 808 MIDs +2026-06-15 13:37:49 | INFO | Fetched 139 Survey rows +2026-06-15 13:37:49 | INFO | Fetched 139 rows +2026-06-15 13:37:49 | INFO | Deleting from Survey +2026-06-15 13:37:49 | INFO | Survey: inserted 139 rows into ClickHouse +2026-06-15 13:37:49 | INFO | Survey loaded successfully (139 rows) +2026-06-15 13:37:49 | INFO | ================================================================================ +2026-06-15 13:37:49 | INFO | Processing Table: additional_visibility +2026-06-15 13:37:49 | INFO | Fetching Additional Visibility data for 808 MIDs +2026-06-15 13:37:50 | INFO | Fetched 1,884 Additional Visibility rows +2026-06-15 13:37:50 | INFO | Fetched 1884 rows +2026-06-15 13:37:50 | INFO | Deleting from additional_visibility +2026-06-15 13:37:50 | INFO | additional_visibility: inserted 1,884 rows into ClickHouse +2026-06-15 13:37:50 | INFO | additional_visibility loaded successfully (1884 rows) +2026-06-15 13:37:50 | INFO | ================================================================================ +2026-06-15 13:37:50 | INFO | Processing Table: Coverage +2026-06-15 13:37:50 | INFO | Fetching coverage data for 808 MIDs +2026-06-15 13:37:50 | INFO | Fetched 751 rows from SQL Server +2026-06-15 13:37:50 | INFO | Fetched 751 rows +2026-06-15 13:37:50 | INFO | Deleting from Coverage +2026-06-15 13:37:50 | INFO | Coverage: inserted 751 rows into ClickHouse +2026-06-15 13:37:50 | INFO | Coverage loaded successfully (751 rows) +2026-06-15 13:37:50 | INFO | ================================================================================ +2026-06-15 13:37:50 | INFO | Processing Table: Login +2026-06-15 13:37:50 | INFO | Fetching Login data for yesterday +2026-06-15 13:37:51 | INFO | Fetched 454 Login rows +2026-06-15 13:37:51 | INFO | Fetched 454 rows +2026-06-15 13:37:51 | INFO | Deleting from Login +2026-06-15 13:37:51 | INFO | No delete logic required for Login +2026-06-15 13:37:51 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-15 13:37:51 | INFO | Login loaded successfully (454 rows) +2026-06-15 13:37:51 | INFO | ================================================================================ +2026-06-15 13:37:51 | INFO | Processing Table: Stock_Details +2026-06-15 13:37:51 | INFO | Fetching Stock Details data for 808 MIDs +2026-06-15 13:37:55 | INFO | Fetched 40,840 Stock Details rows +2026-06-15 13:37:55 | INFO | Fetched 40840 rows +2026-06-15 13:37:55 | INFO | Deleting from Stock_Details +2026-06-15 13:37:55 | INFO | Stock_Details: inserted 40,840 rows into ClickHouse +2026-06-15 13:37:55 | INFO | Stock_Details loaded successfully (40840 rows) +2026-06-15 13:37:55 | INFO | ================================================================================ +2026-06-15 13:37:55 | INFO | Processing Table: Attendance +2026-06-15 13:37:55 | INFO | Fetching Attendance data from 2026-05-19 to 2026-06-03 +2026-06-15 13:37:56 | INFO | Fetched 12,432 attendance rows for 579 employees +2026-06-15 13:37:56 | INFO | Fetched 12432 rows +2026-06-15 13:37:57 | INFO | Truncated table Attendance +2026-06-15 13:37:57 | INFO | Attendance: inserted 12,432 rows into ClickHouse +2026-06-15 13:37:57 | INFO | Attendance loaded successfully (12432 rows) +2026-06-15 13:37:57 | INFO | ================================================================================ +2026-06-15 13:37:57 | INFO | Processing Table: Store_Master +2026-06-15 13:37:57 | INFO | Fetching Store Master data +2026-06-15 13:37:58 | INFO | Fetched 5,984 stores +2026-06-15 13:37:58 | INFO | Fetched 5984 rows +2026-06-15 13:37:58 | INFO | Truncated table Store_Master +2026-06-15 13:37:59 | INFO | Store_Master: inserted 5,984 rows into ClickHouse +2026-06-15 13:37:59 | INFO | Store_Master loaded successfully (5984 rows) +2026-06-15 13:37:59 | INFO | ================================================================================ +2026-06-15 13:37:59 | INFO | Processing Table: SKU_Master +2026-06-15 13:37:59 | INFO | Fetching SKU Master data +2026-06-15 13:37:59 | INFO | Fetched 160 SKU Master rows +2026-06-15 13:37:59 | INFO | Fetched 160 rows +2026-06-15 13:37:59 | INFO | Truncated table SKU_Master +2026-06-15 13:37:59 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-15 13:37:59 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-15 13:37:59 | INFO | ================================================================================ +2026-06-15 13:37:59 | INFO | Processing Table: display_master +2026-06-15 13:37:59 | INFO | Fetching Display Master data +2026-06-15 13:37:59 | INFO | Fetched 135 Display Master records +2026-06-15 13:37:59 | INFO | Fetched 135 rows +2026-06-15 13:37:59 | INFO | Truncated table display_master +2026-06-15 13:37:59 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-15 13:37:59 | INFO | display_master loaded successfully (135 rows) +2026-06-15 13:37:59 | INFO | ================================================================================ +2026-06-15 13:37:59 | INFO | Processing Table: Employee_Master +2026-06-15 13:37:59 | INFO | Fetching Employee Master data +2026-06-15 13:38:00 | INFO | Fetched 2,273 Employee Master records +2026-06-15 13:38:00 | INFO | Fetched 2273 rows +2026-06-15 13:38:00 | INFO | Truncated table Employee_Master +2026-06-15 13:38:00 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse +2026-06-15 13:38:00 | INFO | Employee_Master loaded successfully (2273 rows) +2026-06-15 13:38:00 | INFO | ================================================================================ +2026-06-15 13:38:00 | INFO | Processing Table: Journey_Plan +2026-06-15 13:38:00 | INFO | Fetching Journey Plan for 2026-06 +2026-06-15 13:38:00 | INFO | Fetched 21,141 Journey Plan records +2026-06-15 13:38:00 | INFO | Fetched 21141 rows +2026-06-15 13:38:00 | INFO | Deleting from Journey_Plan +2026-06-15 13:38:01 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse +2026-06-15 13:38:01 | INFO | Journey_Plan loaded successfully (21141 rows) +2026-06-15 13:38:01 | INFO | ================================================================================ +2026-06-15 13:38:01 | INFO | Processing Table: coverage_remarks +2026-06-15 13:38:01 | INFO | Fetching Coverage Remarks +2026-06-15 13:38:01 | INFO | Fetched 29 Coverage Remark records +2026-06-15 13:38:01 | INFO | Fetched 29 rows +2026-06-15 13:38:01 | INFO | Truncated table coverage_remarks +2026-06-15 13:38:01 | INFO | coverage_remarks: inserted 29 rows into ClickHouse +2026-06-15 13:38:01 | INFO | coverage_remarks loaded successfully (29 rows) +2026-06-15 13:38:01 | INFO | ================================================================================ +2026-06-15 13:38:01 | INFO | Processing Table: mapping_storevisibility +2026-06-15 13:38:01 | INFO | Fetching Mapping Store Visibility for 2026-06-03 +2026-06-15 13:38:01 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-15 13:38:01 | WARNING | mapping_storevisibility returned no rows +2026-06-15 13:38:01 | INFO | ================================================================================ +2026-06-15 13:38:01 | INFO | Processing Table: Master_VisibilityReason +2026-06-15 13:38:01 | INFO | Fetching Master Visibility Reason data +2026-06-15 13:38:01 | INFO | Fetched 17 Master Visibility Reason records +2026-06-15 13:38:01 | INFO | Fetched 17 rows +2026-06-15 13:38:01 | INFO | Truncated table Master_VisibilityReason +2026-06-15 13:38:01 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-15 13:38:01 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-15 13:38:01 | INFO | ================================================================================ +2026-06-15 13:38:01 | INFO | Processing Table: Master_VisibilityDefinition +2026-06-15 13:38:01 | INFO | Fetching Master Visibility Definition data +2026-06-15 13:38:02 | INFO | Fetched 861 Master Visibility Definition records +2026-06-15 13:38:02 | INFO | Fetched 861 rows +2026-06-15 13:38:02 | INFO | Truncated table Master_VisibilityDefinition +2026-06-15 13:38:02 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-15 13:38:02 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-15 13:38:02 | INFO | ================================================================================ +2026-06-15 13:38:02 | INFO | Processing Table: Web_Logins +2026-06-15 13:38:02 | INFO | Fetching Web Login data for 2026-06-03 +2026-06-15 13:38:02 | INFO | Fetched 200 Web Login records +2026-06-15 13:38:02 | INFO | Fetched 200 rows +2026-06-15 13:38:02 | INFO | Deleting from Web_Logins +2026-06-15 13:38:02 | INFO | Web_Logins: inserted 200 rows into ClickHouse +2026-06-15 13:38:02 | INFO | Web_Logins loaded successfully (200 rows) +2026-06-15 13:38:02 | INFO | ================================================================================ +2026-06-15 13:38:02 | INFO | Processing Table: Promotion +2026-06-15 13:38:02 | INFO | Fetching Promotion data for 808 MIDs +2026-06-15 13:38:04 | INFO | Fetched 8,162 Promotion records +2026-06-15 13:38:04 | INFO | Fetched 8162 rows +2026-06-15 13:38:04 | INFO | Deleting from Promotion +2026-06-15 13:38:04 | INFO | Promotion: inserted 8,162 rows into ClickHouse +2026-06-15 13:38:04 | INFO | Promotion loaded successfully (8162 rows) +2026-06-15 13:38:04 | INFO | ================================================================================ +2026-06-15 13:38:04 | INFO | Processing Table: PaidVisibility +2026-06-15 13:38:04 | INFO | Fetching Paid Visibility data for 808 MIDs +2026-06-15 13:38:05 | INFO | Fetched 1,064 Paid Visibility records +2026-06-15 13:38:05 | INFO | Fetched 1064 rows +2026-06-15 13:38:06 | INFO | Deleting from PaidVisibility +2026-06-15 13:38:06 | INFO | PaidVisibility: inserted 1,064 rows into ClickHouse +2026-06-15 13:38:06 | INFO | PaidVisibility loaded successfully (1064 rows) +2026-06-15 13:38:06 | INFO | ================================================================================ +2026-06-15 13:38:06 | INFO | Processing Table: Master_Salesterritorylayer +2026-06-15 13:38:06 | INFO | Fetching Master Sales Territory Layer data +2026-06-15 13:38:06 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-15 13:38:06 | INFO | Fetched 33 rows +2026-06-15 13:38:06 | INFO | Truncated table Master_Salesterritorylayer +2026-06-15 13:38:07 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-15 13:38:07 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-15 13:38:07 | INFO | ================================================================================ +2026-06-15 13:38:07 | INFO | Pipeline Completed Successfully +2026-06-15 13:38:07 | INFO | ================================================================================ +2026-06-15 13:40:53 | INFO | ================================================================================ +2026-06-15 13:40:53 | INFO | Hello from data-move Python data pipeline! +2026-06-15 13:40:53 | INFO | Pipeline Run Date: 2026-06-03 +2026-06-15 13:40:53 | INFO | Connecting to databases... +2026-06-15 13:40:53 | INFO | +2026-06-15 13:40:55 | INFO | +2026-06-15 13:40:56 | INFO | Database connections established +2026-06-15 13:40:56 | INFO | Collecting MIDs for: 2026-06-03 +2026-06-15 13:40:56 | INFO | Found 808 MIDs +2026-06-15 13:40:56 | INFO | ================================================================================ +2026-06-15 13:40:56 | INFO | Processing Table: SOS_OneApp +2026-06-15 13:40:56 | INFO | Fetching data for 808 MIDs +2026-06-15 13:40:57 | INFO | Fetched 3,565 rows from SQL Server +2026-06-15 13:40:57 | INFO | Fetched 3565 rows +2026-06-15 13:40:57 | INFO | Deleting from SOS_OneApp +2026-06-15 13:40:57 | INFO | SOS_OneApp: inserted 3,565 rows into ClickHouse +2026-06-15 13:40:57 | INFO | SOS_OneApp loaded successfully (3565 rows) +2026-06-15 13:40:57 | INFO | ================================================================================ +2026-06-15 13:40:57 | INFO | Processing Table: OQaD +2026-06-15 13:40:57 | INFO | Fetching OQaD data for run_date=2026-06-03 +2026-06-15 13:41:00 | INFO | Fetched 455 rows +2026-06-15 13:41:00 | INFO | Fetched 455 rows +2026-06-15 13:41:00 | INFO | Deleting from OQaD +2026-06-15 13:41:00 | INFO | OQaD: inserted 455 rows into ClickHouse +2026-06-15 13:41:00 | INFO | OQaD loaded successfully (455 rows) +2026-06-15 13:41:00 | INFO | ================================================================================ +2026-06-15 13:41:00 | INFO | Processing Table: Survey +2026-06-15 13:41:00 | INFO | Fetching Survey data for 808 MIDs +2026-06-15 13:41:00 | INFO | Fetched 139 Survey rows +2026-06-15 13:41:00 | INFO | Fetched 139 rows +2026-06-15 13:41:00 | INFO | Deleting from Survey +2026-06-15 13:41:00 | INFO | Survey: inserted 139 rows into ClickHouse +2026-06-15 13:41:00 | INFO | Survey loaded successfully (139 rows) +2026-06-15 13:41:00 | INFO | ================================================================================ +2026-06-15 13:41:00 | INFO | Processing Table: additional_visibility +2026-06-15 13:41:00 | INFO | Fetching Additional Visibility data for 808 MIDs +2026-06-15 13:41:01 | INFO | Fetched 1,884 Additional Visibility rows +2026-06-15 13:41:01 | INFO | Fetched 1884 rows +2026-06-15 13:41:01 | INFO | Deleting from additional_visibility +2026-06-15 13:41:01 | INFO | additional_visibility: inserted 1,884 rows into ClickHouse +2026-06-15 13:41:01 | INFO | additional_visibility loaded successfully (1884 rows) +2026-06-15 13:41:01 | INFO | ================================================================================ +2026-06-15 13:41:01 | INFO | Processing Table: Coverage +2026-06-15 13:41:01 | INFO | Fetching coverage data for 808 MIDs +2026-06-15 13:41:01 | INFO | Fetched 751 rows from SQL Server +2026-06-15 13:41:01 | INFO | Fetched 751 rows +2026-06-15 13:41:01 | INFO | Deleting from Coverage +2026-06-15 13:41:02 | INFO | Coverage: inserted 751 rows into ClickHouse +2026-06-15 13:41:02 | INFO | Coverage loaded successfully (751 rows) +2026-06-15 13:41:02 | INFO | ================================================================================ +2026-06-15 13:41:02 | INFO | Processing Table: Login +2026-06-15 13:41:02 | INFO | Fetching Login data for yesterday +2026-06-15 13:41:02 | INFO | Fetched 454 Login rows +2026-06-15 13:41:02 | INFO | Fetched 454 rows +2026-06-15 13:41:02 | INFO | Deleting from Login +2026-06-15 13:41:02 | INFO | No delete logic required for Login +2026-06-15 13:41:02 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-15 13:41:02 | INFO | Login loaded successfully (454 rows) +2026-06-15 13:41:02 | INFO | ================================================================================ +2026-06-15 13:41:02 | INFO | Processing Table: Stock_Details +2026-06-15 13:41:02 | INFO | Fetching Stock Details data for 808 MIDs +2026-06-15 13:41:07 | INFO | Fetched 40,840 Stock Details rows +2026-06-15 13:41:07 | INFO | Fetched 40840 rows +2026-06-15 13:41:08 | INFO | Deleting from Stock_Details +2026-06-15 13:41:08 | INFO | Stock_Details: inserted 40,840 rows into ClickHouse +2026-06-15 13:41:08 | INFO | Stock_Details loaded successfully (40840 rows) +2026-06-15 13:41:08 | INFO | ================================================================================ +2026-06-15 13:41:08 | INFO | Processing Table: Attendance +2026-06-15 13:41:08 | INFO | Fetching Attendance data from 2026-05-19 to 2026-06-03 +2026-06-15 13:41:10 | INFO | Fetched 12,432 attendance rows for 579 employees +2026-06-15 13:41:10 | INFO | Fetched 12432 rows +2026-06-15 13:41:10 | INFO | Truncated table Attendance +2026-06-15 13:41:10 | INFO | Attendance: inserted 12,432 rows into ClickHouse +2026-06-15 13:41:10 | INFO | Attendance loaded successfully (12432 rows) +2026-06-15 13:41:10 | INFO | ================================================================================ +2026-06-15 13:41:10 | INFO | Processing Table: Store_Master +2026-06-15 13:41:10 | INFO | Fetching Store Master data +2026-06-15 13:41:18 | INFO | Fetched 5,984 stores +2026-06-15 13:41:18 | INFO | Fetched 5984 rows +2026-06-15 13:41:18 | INFO | Truncated table Store_Master +2026-06-15 13:41:18 | INFO | Store_Master: inserted 5,984 rows into ClickHouse +2026-06-15 13:41:18 | INFO | Store_Master loaded successfully (5984 rows) +2026-06-15 13:41:18 | INFO | ================================================================================ +2026-06-15 13:41:18 | INFO | Processing Table: SKU_Master +2026-06-15 13:41:18 | INFO | Fetching SKU Master data +2026-06-15 13:41:18 | INFO | Fetched 160 SKU Master rows +2026-06-15 13:41:18 | INFO | Fetched 160 rows +2026-06-15 13:41:18 | INFO | Truncated table SKU_Master +2026-06-15 13:41:19 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-15 13:41:19 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-15 13:41:19 | INFO | ================================================================================ +2026-06-15 13:41:19 | INFO | Processing Table: display_master +2026-06-15 13:41:19 | INFO | Fetching Display Master data +2026-06-15 13:41:19 | INFO | Fetched 135 Display Master records +2026-06-15 13:41:19 | INFO | Fetched 135 rows +2026-06-15 13:41:19 | INFO | Truncated table display_master +2026-06-15 13:41:19 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-15 13:41:19 | INFO | display_master loaded successfully (135 rows) +2026-06-15 13:41:19 | INFO | ================================================================================ +2026-06-15 13:41:19 | INFO | Processing Table: Employee_Master +2026-06-15 13:41:19 | INFO | Fetching Employee Master data +2026-06-15 13:41:20 | INFO | Fetched 2,273 Employee Master records +2026-06-15 13:41:20 | INFO | Fetched 2273 rows +2026-06-15 13:41:20 | INFO | Truncated table Employee_Master +2026-06-15 13:41:20 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse +2026-06-15 13:41:20 | INFO | Employee_Master loaded successfully (2273 rows) +2026-06-15 13:41:20 | INFO | ================================================================================ +2026-06-15 13:41:20 | INFO | Processing Table: Journey_Plan +2026-06-15 13:41:20 | INFO | Fetching Journey Plan for 2026-06 +2026-06-15 13:41:21 | INFO | Fetched 21,141 Journey Plan records +2026-06-15 13:41:21 | INFO | Fetched 21141 rows +2026-06-15 13:41:21 | INFO | Deleting from Journey_Plan +2026-06-15 13:41:21 | INFO | Journey_Plan: inserted 21,141 rows into ClickHouse +2026-06-15 13:41:21 | INFO | Journey_Plan loaded successfully (21141 rows) +2026-06-15 13:41:21 | INFO | ================================================================================ +2026-06-15 13:41:21 | INFO | Processing Table: coverage_remarks +2026-06-15 13:41:21 | INFO | Fetching Coverage Remarks +2026-06-15 13:41:21 | INFO | Fetched 29 Coverage Remark records +2026-06-15 13:41:21 | INFO | Fetched 29 rows +2026-06-15 13:41:21 | INFO | Truncated table coverage_remarks +2026-06-15 13:41:22 | INFO | coverage_remarks: inserted 29 rows into ClickHouse +2026-06-15 13:41:22 | INFO | coverage_remarks loaded successfully (29 rows) +2026-06-15 13:41:22 | INFO | ================================================================================ +2026-06-15 13:41:22 | INFO | Processing Table: mapping_storevisibility +2026-06-15 13:41:22 | INFO | Fetching Mapping Store Visibility for 2026-06-03 +2026-06-15 13:41:22 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-15 13:41:22 | WARNING | mapping_storevisibility returned no rows +2026-06-15 13:41:22 | INFO | ================================================================================ +2026-06-15 13:41:22 | INFO | Processing Table: Master_VisibilityReason +2026-06-15 13:41:22 | INFO | Fetching Master Visibility Reason data +2026-06-15 13:41:22 | INFO | Fetched 17 Master Visibility Reason records +2026-06-15 13:41:22 | INFO | Fetched 17 rows +2026-06-15 13:41:22 | INFO | Truncated table Master_VisibilityReason +2026-06-15 13:41:22 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-15 13:41:22 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-15 13:41:22 | INFO | ================================================================================ +2026-06-15 13:41:22 | INFO | Processing Table: Master_VisibilityDefinition +2026-06-15 13:41:22 | INFO | Fetching Master Visibility Definition data +2026-06-15 13:41:22 | INFO | Fetched 861 Master Visibility Definition records +2026-06-15 13:41:22 | INFO | Fetched 861 rows +2026-06-15 13:41:22 | INFO | Truncated table Master_VisibilityDefinition +2026-06-15 13:41:22 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-15 13:41:22 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-15 13:41:22 | INFO | ================================================================================ +2026-06-15 13:41:22 | INFO | Processing Table: Web_Logins +2026-06-15 13:41:22 | INFO | Fetching Web Login data for 2026-06-03 +2026-06-15 13:41:23 | INFO | Fetched 200 Web Login records +2026-06-15 13:41:23 | INFO | Fetched 200 rows +2026-06-15 13:41:23 | INFO | Deleting from Web_Logins +2026-06-15 13:41:23 | INFO | Web_Logins: inserted 200 rows into ClickHouse +2026-06-15 13:41:23 | INFO | Web_Logins loaded successfully (200 rows) +2026-06-15 13:41:23 | INFO | ================================================================================ +2026-06-15 13:41:23 | INFO | Processing Table: Promotion +2026-06-15 13:41:23 | INFO | Fetching Promotion data for 808 MIDs +2026-06-15 13:41:28 | INFO | Fetched 8,162 Promotion records +2026-06-15 13:41:28 | INFO | Fetched 8162 rows +2026-06-15 13:41:28 | INFO | Deleting from Promotion +2026-06-15 13:41:28 | INFO | Promotion: inserted 8,162 rows into ClickHouse +2026-06-15 13:41:28 | INFO | Promotion loaded successfully (8162 rows) +2026-06-15 13:41:28 | INFO | ================================================================================ +2026-06-15 13:41:28 | INFO | Processing Table: PaidVisibility +2026-06-15 13:41:28 | INFO | Fetching Paid Visibility data for 808 MIDs +2026-06-15 13:41:29 | INFO | Fetched 1,064 Paid Visibility records +2026-06-15 13:41:29 | INFO | Fetched 1064 rows +2026-06-15 13:41:29 | INFO | Deleting from PaidVisibility +2026-06-15 13:41:30 | INFO | PaidVisibility: inserted 1,064 rows into ClickHouse +2026-06-15 13:41:30 | INFO | PaidVisibility loaded successfully (1064 rows) +2026-06-15 13:41:30 | INFO | ================================================================================ +2026-06-15 13:41:30 | INFO | Processing Table: Master_Salesterritorylayer +2026-06-15 13:41:30 | INFO | Fetching Master Sales Territory Layer data +2026-06-15 13:41:30 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-15 13:41:30 | INFO | Fetched 33 rows +2026-06-15 13:41:30 | INFO | Truncated table Master_Salesterritorylayer +2026-06-15 13:41:30 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-15 13:41:30 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-15 13:41:30 | INFO | ================================================================================ +2026-06-15 13:41:30 | INFO | Pipeline Completed Successfully +2026-06-15 13:41:30 | INFO | ================================================================================ +2026-06-15 13:46:09 | INFO | ================================================================================ +2026-06-15 13:46:09 | INFO | Hello from data-move Python data pipeline! +2026-06-15 13:46:09 | INFO | Pipeline Run Date: 2026-06-03 +2026-06-15 13:46:09 | INFO | Connecting to databases... +2026-06-15 13:46:10 | INFO | +2026-06-15 13:46:11 | INFO | +2026-06-15 13:46:11 | INFO | Database connections established +2026-06-15 13:46:11 | INFO | Collecting MIDs for: 2026-06-03 +2026-06-15 13:46:11 | INFO | Found 808 MIDs +2026-06-15 13:46:11 | INFO | ================================================================================ +2026-06-15 13:46:11 | INFO | Processing Table: SOS_OneApp +2026-06-15 13:46:11 | INFO | Fetching data for 808 MIDs +2026-06-15 13:46:12 | INFO | Fetched 3,565 rows from SQL Server +2026-06-15 13:46:12 | INFO | Fetched 3565 rows +2026-06-15 13:46:12 | INFO | Deleting from SOS_OneApp +2026-06-15 13:46:13 | INFO | SOS_OneApp: inserted 3,565 rows into ClickHouse +2026-06-15 13:46:13 | INFO | SOS_OneApp loaded successfully (3565 rows) +2026-06-15 13:46:13 | INFO | ================================================================================ +2026-06-15 13:46:13 | INFO | Processing Table: OQaD +2026-06-15 13:46:13 | INFO | Fetching OQaD data for run_date=2026-06-03 +2026-06-15 13:46:15 | INFO | Fetched 455 rows +2026-06-15 13:46:15 | INFO | Fetched 455 rows +2026-06-15 13:46:15 | INFO | Deleting from OQaD +2026-06-15 13:46:15 | INFO | OQaD: inserted 455 rows into ClickHouse +2026-06-15 13:46:15 | INFO | OQaD loaded successfully (455 rows) +2026-06-15 13:46:15 | INFO | ================================================================================ +2026-06-15 13:46:15 | INFO | Processing Table: Survey +2026-06-15 13:46:15 | INFO | Fetching Survey data for 808 MIDs +2026-06-15 13:46:15 | INFO | Fetched 139 Survey rows +2026-06-15 13:46:15 | INFO | Fetched 139 rows +2026-06-15 13:46:15 | INFO | Deleting from Survey +2026-06-15 13:46:16 | INFO | Survey: inserted 139 rows into ClickHouse +2026-06-15 13:46:16 | INFO | Survey loaded successfully (139 rows) +2026-06-15 13:46:16 | INFO | ================================================================================ +2026-06-15 13:46:16 | INFO | Processing Table: additional_visibility +2026-06-15 13:46:16 | INFO | Fetching Additional Visibility data for 808 MIDs +2026-06-15 13:46:16 | INFO | Fetched 1,884 Additional Visibility rows +2026-06-15 13:46:16 | INFO | Fetched 1884 rows +2026-06-15 13:46:16 | INFO | Deleting from additional_visibility +2026-06-15 13:46:16 | INFO | additional_visibility: inserted 1,884 rows into ClickHouse +2026-06-15 13:46:16 | INFO | additional_visibility loaded successfully (1884 rows) +2026-06-15 13:46:16 | INFO | ================================================================================ +2026-06-15 13:46:16 | INFO | Processing Table: Coverage +2026-06-15 13:46:16 | INFO | Fetching coverage data for 808 MIDs +2026-06-15 13:46:16 | INFO | Fetched 751 rows from SQL Server +2026-06-15 13:46:16 | INFO | Fetched 751 rows +2026-06-15 13:46:16 | INFO | Deleting from Coverage +2026-06-15 13:46:17 | INFO | Coverage: inserted 751 rows into ClickHouse +2026-06-15 13:46:17 | INFO | Coverage loaded successfully (751 rows) +2026-06-15 13:46:17 | INFO | ================================================================================ +2026-06-15 13:46:17 | INFO | Processing Table: Login +2026-06-15 13:46:17 | INFO | Fetching Login data for yesterday +2026-06-15 13:46:17 | INFO | Fetched 454 Login rows +2026-06-15 13:46:17 | INFO | Fetched 454 rows +2026-06-15 13:46:17 | INFO | Deleting from Login +2026-06-15 13:46:17 | INFO | No delete logic required for Login +2026-06-15 13:46:18 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-15 13:46:18 | INFO | Login loaded successfully (454 rows) +2026-06-15 13:46:18 | INFO | ================================================================================ +2026-06-15 13:46:18 | INFO | Processing Table: Stock_Details +2026-06-15 13:46:18 | INFO | Fetching Stock Details data for 808 MIDs +2026-06-15 13:46:22 | INFO | Fetched 40,840 Stock Details rows +2026-06-15 13:46:22 | INFO | Fetched 40840 rows +2026-06-15 13:46:22 | INFO | Deleting from Stock_Details +2026-06-15 13:46:23 | INFO | Stock_Details: inserted 40,840 rows into ClickHouse +2026-06-15 13:46:23 | INFO | Stock_Details loaded successfully (40840 rows) +2026-06-15 13:46:23 | INFO | ================================================================================ +2026-06-15 13:46:23 | INFO | Processing Table: Attendance +2026-06-15 13:46:23 | INFO | Fetching Attendance data from 2026-05-19 to 2026-06-03 +2026-06-15 13:46:25 | INFO | Fetched 12,432 attendance rows for 579 employees +2026-06-15 13:46:25 | INFO | Fetched 12432 rows +2026-06-15 13:46:25 | INFO | Truncated table Attendance +2026-06-15 13:46:25 | INFO | Attendance: inserted 12,432 rows into ClickHouse +2026-06-15 13:46:25 | INFO | Attendance loaded successfully (12432 rows) +2026-06-15 13:46:25 | INFO | ================================================================================ +2026-06-15 13:46:25 | INFO | Processing Table: Store_Master +2026-06-15 13:46:25 | INFO | Fetching Store Master data +2026-06-15 13:46:29 | INFO | Fetched 5,984 stores +2026-06-15 13:46:29 | INFO | Fetched 5984 rows +2026-06-15 13:46:29 | INFO | Truncated table Store_Master +2026-06-15 13:46:29 | INFO | Store_Master: inserted 5,984 rows into ClickHouse +2026-06-15 13:46:29 | INFO | Store_Master loaded successfully (5984 rows) +2026-06-15 13:46:29 | INFO | ================================================================================ +2026-06-15 13:46:29 | INFO | Processing Table: SKU_Master +2026-06-15 13:46:29 | INFO | Fetching SKU Master data +2026-06-15 13:46:29 | INFO | Fetched 160 SKU Master rows +2026-06-15 13:46:29 | INFO | Fetched 160 rows +2026-06-15 13:46:30 | INFO | Truncated table SKU_Master +2026-06-15 13:46:30 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-15 13:46:30 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-15 13:46:30 | INFO | ================================================================================ +2026-06-15 13:46:30 | INFO | Processing Table: display_master +2026-06-15 13:46:30 | INFO | Fetching Display Master data +2026-06-15 13:46:30 | INFO | Fetched 135 Display Master records +2026-06-15 13:46:30 | INFO | Fetched 135 rows +2026-06-15 13:46:30 | INFO | Truncated table display_master +2026-06-15 13:46:30 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-15 13:46:30 | INFO | display_master loaded successfully (135 rows) +2026-06-15 13:46:30 | INFO | ================================================================================ +2026-06-15 13:46:30 | INFO | Processing Table: Employee_Master +2026-06-15 13:46:30 | INFO | Fetching Employee Master data +2026-06-15 13:46:31 | INFO | Fetched 2,273 Employee Master records +2026-06-15 13:46:31 | INFO | Fetched 2273 rows +2026-06-15 13:46:31 | INFO | Truncated table Employee_Master +2026-06-15 13:46:31 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse +2026-06-15 13:46:31 | INFO | Employee_Master loaded successfully (2273 rows) +2026-06-15 13:46:31 | INFO | ================================================================================ +2026-06-15 13:46:31 | INFO | Processing Table: Journey_Plan +2026-06-15 13:46:31 | INFO | Fetching Journey Plan for 2026-06 +2026-06-15 13:46:31 | INFO | Fetched 21,142 Journey Plan records +2026-06-15 13:46:31 | INFO | Fetched 21142 rows +2026-06-15 13:46:32 | INFO | Deleting from Journey_Plan +2026-06-15 13:46:32 | INFO | Journey_Plan: inserted 21,142 rows into ClickHouse +2026-06-15 13:46:32 | INFO | Journey_Plan loaded successfully (21142 rows) +2026-06-15 13:46:32 | INFO | ================================================================================ +2026-06-15 13:46:32 | INFO | Processing Table: coverage_remarks +2026-06-15 13:46:32 | INFO | Fetching Coverage Remarks +2026-06-15 13:46:32 | INFO | Fetched 29 Coverage Remark records +2026-06-15 13:46:32 | INFO | Fetched 29 rows +2026-06-15 13:46:32 | INFO | Truncated table coverage_remarks +2026-06-15 13:46:33 | INFO | coverage_remarks: inserted 29 rows into ClickHouse +2026-06-15 13:46:33 | INFO | coverage_remarks loaded successfully (29 rows) +2026-06-15 13:46:33 | INFO | ================================================================================ +2026-06-15 13:46:33 | INFO | Processing Table: mapping_storevisibility +2026-06-15 13:46:33 | INFO | Fetching Mapping Store Visibility for 2026-06-03 +2026-06-15 13:46:33 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-15 13:46:33 | WARNING | mapping_storevisibility returned no rows +2026-06-15 13:46:33 | INFO | ================================================================================ +2026-06-15 13:46:33 | INFO | Processing Table: Master_VisibilityReason +2026-06-15 13:46:33 | INFO | Fetching Master Visibility Reason data +2026-06-15 13:46:33 | INFO | Fetched 17 Master Visibility Reason records +2026-06-15 13:46:33 | INFO | Fetched 17 rows +2026-06-15 13:46:33 | INFO | Truncated table Master_VisibilityReason +2026-06-15 13:46:33 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-15 13:46:33 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-15 13:46:33 | INFO | ================================================================================ +2026-06-15 13:46:33 | INFO | Processing Table: Master_VisibilityDefinition +2026-06-15 13:46:33 | INFO | Fetching Master Visibility Definition data +2026-06-15 13:46:33 | INFO | Fetched 861 Master Visibility Definition records +2026-06-15 13:46:33 | INFO | Fetched 861 rows +2026-06-15 13:46:34 | INFO | Truncated table Master_VisibilityDefinition +2026-06-15 13:46:34 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-15 13:46:34 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-15 13:46:34 | INFO | ================================================================================ +2026-06-15 13:46:34 | INFO | Processing Table: Web_Logins +2026-06-15 13:46:34 | INFO | Fetching Web Login data for 2026-06-03 +2026-06-15 13:46:34 | INFO | Fetched 200 Web Login records +2026-06-15 13:46:34 | INFO | Fetched 200 rows +2026-06-15 13:46:34 | INFO | Deleting from Web_Logins +2026-06-15 13:46:34 | INFO | Web_Logins: inserted 200 rows into ClickHouse +2026-06-15 13:46:34 | INFO | Web_Logins loaded successfully (200 rows) +2026-06-15 13:46:34 | INFO | ================================================================================ +2026-06-15 13:46:34 | INFO | Processing Table: Promotion +2026-06-15 13:46:34 | INFO | Fetching Promotion data for 808 MIDs +2026-06-15 13:46:37 | INFO | Fetched 8,162 Promotion records +2026-06-15 13:46:37 | INFO | Fetched 8162 rows +2026-06-15 13:46:37 | INFO | Deleting from Promotion +2026-06-15 13:46:37 | INFO | Promotion: inserted 8,162 rows into ClickHouse +2026-06-15 13:46:37 | INFO | Promotion loaded successfully (8162 rows) +2026-06-15 13:46:37 | INFO | ================================================================================ +2026-06-15 13:46:37 | INFO | Processing Table: PaidVisibility +2026-06-15 13:46:37 | INFO | Fetching Paid Visibility data for 808 MIDs +2026-06-15 13:46:38 | INFO | Fetched 1,064 Paid Visibility records +2026-06-15 13:46:38 | INFO | Fetched 1064 rows +2026-06-15 13:46:38 | INFO | Deleting from PaidVisibility +2026-06-15 13:46:38 | INFO | PaidVisibility: inserted 1,064 rows into ClickHouse +2026-06-15 13:46:38 | INFO | PaidVisibility loaded successfully (1064 rows) +2026-06-15 13:46:38 | INFO | ================================================================================ +2026-06-15 13:46:38 | INFO | Processing Table: Master_Salesterritorylayer +2026-06-15 13:46:38 | INFO | Fetching Master Sales Territory Layer data +2026-06-15 13:46:38 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-15 13:46:38 | INFO | Fetched 33 rows +2026-06-15 13:46:38 | INFO | Truncated table Master_Salesterritorylayer +2026-06-15 13:46:38 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-15 13:46:38 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-15 13:46:38 | INFO | ================================================================================ +2026-06-15 13:46:38 | INFO | Pipeline Completed Successfully +2026-06-15 13:46:38 | INFO | ================================================================================ +2026-06-15 13:46:43 | INFO | ================================================================================ +2026-06-15 13:46:43 | INFO | Hello from data-move Python data pipeline! +2026-06-15 13:46:43 | INFO | Pipeline Run Date: 2026-06-03 +2026-06-15 13:46:43 | INFO | Connecting to databases... +2026-06-15 13:46:43 | INFO | +2026-06-15 13:46:44 | INFO | +2026-06-15 13:46:45 | INFO | Database connections established +2026-06-15 13:46:45 | INFO | Collecting MIDs for: 2026-06-03 +2026-06-15 13:46:45 | INFO | Found 808 MIDs +2026-06-15 13:46:45 | INFO | ================================================================================ +2026-06-15 13:46:45 | INFO | Processing Table: SOS_OneApp +2026-06-15 13:46:45 | INFO | Fetching data for 808 MIDs +2026-06-15 13:46:45 | INFO | Fetched 3,565 rows from SQL Server +2026-06-15 13:46:45 | INFO | Fetched 3565 rows +2026-06-15 13:46:46 | INFO | Deleting from SOS_OneApp +2026-06-15 13:46:46 | INFO | SOS_OneApp: inserted 3,565 rows into ClickHouse +2026-06-15 13:46:46 | INFO | SOS_OneApp loaded successfully (3565 rows) +2026-06-15 13:46:46 | INFO | ================================================================================ +2026-06-15 13:46:46 | INFO | Processing Table: OQaD +2026-06-15 13:46:46 | INFO | Fetching OQaD data for run_date=2026-06-03 +2026-06-15 13:46:48 | INFO | Fetched 455 rows +2026-06-15 13:46:48 | INFO | Fetched 455 rows +2026-06-15 13:46:48 | INFO | Deleting from OQaD +2026-06-15 13:46:48 | INFO | OQaD: inserted 455 rows into ClickHouse +2026-06-15 13:46:48 | INFO | OQaD loaded successfully (455 rows) +2026-06-15 13:46:48 | INFO | ================================================================================ +2026-06-15 13:46:48 | INFO | Processing Table: Survey +2026-06-15 13:46:48 | INFO | Fetching Survey data for 808 MIDs +2026-06-15 13:46:48 | INFO | Fetched 139 Survey rows +2026-06-15 13:46:48 | INFO | Fetched 139 rows +2026-06-15 13:46:49 | INFO | Deleting from Survey +2026-06-15 13:46:49 | INFO | Survey: inserted 139 rows into ClickHouse +2026-06-15 13:46:49 | INFO | Survey loaded successfully (139 rows) +2026-06-15 13:46:49 | INFO | ================================================================================ +2026-06-15 13:46:49 | INFO | Processing Table: additional_visibility +2026-06-15 13:46:49 | INFO | Fetching Additional Visibility data for 808 MIDs +2026-06-15 13:46:50 | INFO | Fetched 1,884 Additional Visibility rows +2026-06-15 13:46:50 | INFO | Fetched 1884 rows +2026-06-15 13:46:50 | INFO | Deleting from additional_visibility +2026-06-15 13:46:50 | INFO | additional_visibility: inserted 1,884 rows into ClickHouse +2026-06-15 13:46:50 | INFO | additional_visibility loaded successfully (1884 rows) +2026-06-15 13:46:50 | INFO | ================================================================================ +2026-06-15 13:46:50 | INFO | Processing Table: Coverage +2026-06-15 13:46:50 | INFO | Fetching coverage data for 808 MIDs +2026-06-15 13:46:50 | INFO | Fetched 751 rows from SQL Server +2026-06-15 13:46:50 | INFO | Fetched 751 rows +2026-06-15 13:46:50 | INFO | Deleting from Coverage +2026-06-15 13:46:50 | INFO | Coverage: inserted 751 rows into ClickHouse +2026-06-15 13:46:50 | INFO | Coverage loaded successfully (751 rows) +2026-06-15 13:46:50 | INFO | ================================================================================ +2026-06-15 13:46:50 | INFO | Processing Table: Login +2026-06-15 13:46:50 | INFO | Fetching Login data for yesterday +2026-06-15 13:46:51 | INFO | Fetched 454 Login rows +2026-06-15 13:46:51 | INFO | Fetched 454 rows +2026-06-15 13:46:51 | INFO | Deleting from Login +2026-06-15 13:46:51 | INFO | No delete logic required for Login +2026-06-15 13:46:51 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-15 13:46:51 | INFO | Login loaded successfully (454 rows) +2026-06-15 13:46:51 | INFO | ================================================================================ +2026-06-15 13:46:51 | INFO | Processing Table: Stock_Details +2026-06-15 13:46:51 | INFO | Fetching Stock Details data for 808 MIDs +2026-06-15 13:46:53 | INFO | Fetched 40,840 Stock Details rows +2026-06-15 13:46:53 | INFO | Fetched 40840 rows +2026-06-15 13:46:53 | INFO | Deleting from Stock_Details +2026-06-15 13:46:54 | INFO | Stock_Details: inserted 40,840 rows into ClickHouse +2026-06-15 13:46:54 | INFO | Stock_Details loaded successfully (40840 rows) +2026-06-15 13:46:54 | INFO | ================================================================================ +2026-06-15 13:46:54 | INFO | Processing Table: Attendance +2026-06-15 13:46:54 | INFO | Fetching Attendance data from 2026-05-19 to 2026-06-03 +2026-06-15 13:46:54 | INFO | Fetched 12,432 attendance rows for 579 employees +2026-06-15 13:46:54 | INFO | Fetched 12432 rows +2026-06-15 13:46:55 | INFO | Truncated table Attendance +2026-06-15 13:46:55 | INFO | Attendance: inserted 12,432 rows into ClickHouse +2026-06-15 13:46:55 | INFO | Attendance loaded successfully (12432 rows) +2026-06-15 13:46:55 | INFO | ================================================================================ +2026-06-15 13:46:55 | INFO | Processing Table: Store_Master +2026-06-15 13:46:55 | INFO | Fetching Store Master data +2026-06-15 13:46:56 | INFO | Fetched 5,984 stores +2026-06-15 13:46:56 | INFO | Fetched 5984 rows +2026-06-15 13:46:56 | INFO | Truncated table Store_Master +2026-06-15 13:46:56 | INFO | Store_Master: inserted 5,984 rows into ClickHouse +2026-06-15 13:46:56 | INFO | Store_Master loaded successfully (5984 rows) +2026-06-15 13:46:56 | INFO | ================================================================================ +2026-06-15 13:46:56 | INFO | Processing Table: SKU_Master +2026-06-15 13:46:56 | INFO | Fetching SKU Master data +2026-06-15 13:46:57 | INFO | Fetched 160 SKU Master rows +2026-06-15 13:46:57 | INFO | Fetched 160 rows +2026-06-15 13:46:57 | INFO | Truncated table SKU_Master +2026-06-15 13:46:57 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-15 13:46:57 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-15 13:46:57 | INFO | ================================================================================ +2026-06-15 13:46:57 | INFO | Processing Table: display_master +2026-06-15 13:46:57 | INFO | Fetching Display Master data +2026-06-15 13:46:57 | INFO | Fetched 135 Display Master records +2026-06-15 13:46:57 | INFO | Fetched 135 rows +2026-06-15 13:46:57 | INFO | Truncated table display_master +2026-06-15 13:46:57 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-15 13:46:57 | INFO | display_master loaded successfully (135 rows) +2026-06-15 13:46:57 | INFO | ================================================================================ +2026-06-15 13:46:57 | INFO | Processing Table: Employee_Master +2026-06-15 13:46:57 | INFO | Fetching Employee Master data +2026-06-15 13:46:58 | INFO | Fetched 2,273 Employee Master records +2026-06-15 13:46:58 | INFO | Fetched 2273 rows +2026-06-15 13:46:58 | INFO | Truncated table Employee_Master +2026-06-15 13:46:58 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse +2026-06-15 13:46:58 | INFO | Employee_Master loaded successfully (2273 rows) +2026-06-15 13:46:58 | INFO | ================================================================================ +2026-06-15 13:46:58 | INFO | Processing Table: Journey_Plan +2026-06-15 13:46:58 | INFO | Fetching Journey Plan for 2026-06 +2026-06-15 13:46:58 | INFO | Fetched 21,142 Journey Plan records +2026-06-15 13:46:58 | INFO | Fetched 21142 rows +2026-06-15 13:46:58 | INFO | Deleting from Journey_Plan +2026-06-15 13:46:58 | INFO | Journey_Plan: inserted 21,142 rows into ClickHouse +2026-06-15 13:46:58 | INFO | Journey_Plan loaded successfully (21142 rows) +2026-06-15 13:46:58 | INFO | ================================================================================ +2026-06-15 13:46:58 | INFO | Processing Table: coverage_remarks +2026-06-15 13:46:58 | INFO | Fetching Coverage Remarks +2026-06-15 13:46:58 | INFO | Fetched 29 Coverage Remark records +2026-06-15 13:46:59 | INFO | Fetched 29 rows +2026-06-15 13:46:59 | INFO | Truncated table coverage_remarks +2026-06-15 13:46:59 | INFO | coverage_remarks: inserted 29 rows into ClickHouse +2026-06-15 13:46:59 | INFO | coverage_remarks loaded successfully (29 rows) +2026-06-15 13:46:59 | INFO | ================================================================================ +2026-06-15 13:46:59 | INFO | Processing Table: mapping_storevisibility +2026-06-15 13:46:59 | INFO | Fetching Mapping Store Visibility for 2026-06-03 +2026-06-15 13:46:59 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-15 13:46:59 | WARNING | mapping_storevisibility returned no rows +2026-06-15 13:46:59 | INFO | ================================================================================ +2026-06-15 13:46:59 | INFO | Processing Table: Master_VisibilityReason +2026-06-15 13:46:59 | INFO | Fetching Master Visibility Reason data +2026-06-15 13:46:59 | INFO | Fetched 17 Master Visibility Reason records +2026-06-15 13:46:59 | INFO | Fetched 17 rows +2026-06-15 13:46:59 | INFO | Truncated table Master_VisibilityReason +2026-06-15 13:46:59 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-15 13:46:59 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-15 13:46:59 | INFO | ================================================================================ +2026-06-15 13:46:59 | INFO | Processing Table: Master_VisibilityDefinition +2026-06-15 13:46:59 | INFO | Fetching Master Visibility Definition data +2026-06-15 13:46:59 | INFO | Fetched 861 Master Visibility Definition records +2026-06-15 13:46:59 | INFO | Fetched 861 rows +2026-06-15 13:46:59 | INFO | Truncated table Master_VisibilityDefinition +2026-06-15 13:47:00 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-15 13:47:00 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-15 13:47:00 | INFO | ================================================================================ +2026-06-15 13:47:00 | INFO | Processing Table: Web_Logins +2026-06-15 13:47:00 | INFO | Fetching Web Login data for 2026-06-03 +2026-06-15 13:47:00 | INFO | Fetched 200 Web Login records +2026-06-15 13:47:00 | INFO | Fetched 200 rows +2026-06-15 13:47:00 | INFO | Deleting from Web_Logins +2026-06-15 13:47:00 | INFO | Web_Logins: inserted 200 rows into ClickHouse +2026-06-15 13:47:00 | INFO | Web_Logins loaded successfully (200 rows) +2026-06-15 13:47:00 | INFO | ================================================================================ +2026-06-15 13:47:00 | INFO | Processing Table: Promotion +2026-06-15 13:47:00 | INFO | Fetching Promotion data for 808 MIDs +2026-06-15 13:47:02 | INFO | Fetched 8,162 Promotion records +2026-06-15 13:47:02 | INFO | Fetched 8162 rows +2026-06-15 13:47:02 | INFO | Deleting from Promotion +2026-06-15 13:47:02 | INFO | Promotion: inserted 8,162 rows into ClickHouse +2026-06-15 13:47:02 | INFO | Promotion loaded successfully (8162 rows) +2026-06-15 13:47:02 | INFO | ================================================================================ +2026-06-15 13:47:02 | INFO | Processing Table: PaidVisibility +2026-06-15 13:47:02 | INFO | Fetching Paid Visibility data for 808 MIDs +2026-06-15 13:47:03 | INFO | Fetched 1,064 Paid Visibility records +2026-06-15 13:47:03 | INFO | Fetched 1064 rows +2026-06-15 13:47:03 | INFO | Deleting from PaidVisibility +2026-06-15 13:47:03 | INFO | PaidVisibility: inserted 1,064 rows into ClickHouse +2026-06-15 13:47:03 | INFO | PaidVisibility loaded successfully (1064 rows) +2026-06-15 13:47:03 | INFO | ================================================================================ +2026-06-15 13:47:03 | INFO | Processing Table: Master_Salesterritorylayer +2026-06-15 13:47:03 | INFO | Fetching Master Sales Territory Layer data +2026-06-15 13:47:03 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-15 13:47:03 | INFO | Fetched 33 rows +2026-06-15 13:47:03 | INFO | Truncated table Master_Salesterritorylayer +2026-06-15 13:47:03 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-15 13:47:03 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-15 13:47:03 | INFO | ================================================================================ +2026-06-15 13:47:03 | INFO | Pipeline Completed Successfully +2026-06-15 13:47:03 | INFO | ================================================================================ +2026-06-15 13:50:19 | INFO | ================================================================================ +2026-06-15 13:50:19 | INFO | Hello from data-move Python data pipeline! +2026-06-15 13:50:19 | INFO | Pipeline Run Date: 2026-06-03 +2026-06-15 13:50:19 | INFO | Connecting to databases... +2026-06-15 13:50:20 | INFO | +2026-06-15 13:50:21 | INFO | +2026-06-15 13:50:21 | INFO | Database connections established +2026-06-15 13:50:21 | INFO | Collecting MIDs for: 2026-06-03 +2026-06-15 13:50:21 | INFO | Found 808 MIDs +2026-06-15 13:50:22 | INFO | ================================================================================ +2026-06-15 13:50:22 | INFO | Processing Table: SOS_OneApp +2026-06-15 13:50:22 | INFO | Fetching data for 808 MIDs +2026-06-15 13:50:22 | INFO | Fetched 3,565 rows from SQL Server +2026-06-15 13:50:22 | INFO | Fetched 3565 rows +2026-06-15 13:50:22 | INFO | Deleting from SOS_OneApp +2026-06-15 13:50:23 | INFO | SOS_OneApp: inserted 3,565 rows into ClickHouse +2026-06-15 13:50:23 | INFO | SOS_OneApp loaded successfully (3565 rows) +2026-06-15 13:50:23 | INFO | ================================================================================ +2026-06-15 13:50:23 | INFO | Processing Table: OQaD +2026-06-15 13:50:23 | INFO | Fetching OQaD data for run_date=2026-06-03 +2026-06-15 13:50:27 | INFO | Fetched 455 rows +2026-06-15 13:50:27 | INFO | Fetched 455 rows +2026-06-15 13:50:27 | INFO | Deleting from OQaD +2026-06-15 13:50:27 | INFO | OQaD: inserted 455 rows into ClickHouse +2026-06-15 13:50:27 | INFO | OQaD loaded successfully (455 rows) +2026-06-15 13:50:27 | INFO | ================================================================================ +2026-06-15 13:50:27 | INFO | Processing Table: Survey +2026-06-15 13:50:27 | INFO | Fetching Survey data for 808 MIDs +2026-06-15 13:50:27 | INFO | Fetched 139 Survey rows +2026-06-15 13:50:27 | INFO | Fetched 139 rows +2026-06-15 13:50:27 | INFO | Deleting from Survey +2026-06-15 13:50:27 | INFO | Survey: inserted 139 rows into ClickHouse +2026-06-15 13:50:27 | INFO | Survey loaded successfully (139 rows) +2026-06-15 13:50:27 | INFO | ================================================================================ +2026-06-15 13:50:27 | INFO | Processing Table: additional_visibility +2026-06-15 13:50:27 | INFO | Fetching Additional Visibility data for 808 MIDs +2026-06-15 13:50:28 | INFO | Fetched 1,884 Additional Visibility rows +2026-06-15 13:50:28 | INFO | Fetched 1884 rows +2026-06-15 13:50:28 | INFO | Deleting from additional_visibility +2026-06-15 13:50:28 | INFO | additional_visibility: inserted 1,884 rows into ClickHouse +2026-06-15 13:50:28 | INFO | additional_visibility loaded successfully (1884 rows) +2026-06-15 13:50:28 | INFO | ================================================================================ +2026-06-15 13:50:28 | INFO | Processing Table: Coverage +2026-06-15 13:50:28 | INFO | Fetching coverage data for 808 MIDs +2026-06-15 13:50:28 | INFO | Fetched 751 rows from SQL Server +2026-06-15 13:50:28 | INFO | Fetched 751 rows +2026-06-15 13:50:28 | INFO | Deleting from Coverage +2026-06-15 13:50:29 | INFO | Coverage: inserted 751 rows into ClickHouse +2026-06-15 13:50:29 | INFO | Coverage loaded successfully (751 rows) +2026-06-15 13:50:29 | INFO | ================================================================================ +2026-06-15 13:50:29 | INFO | Processing Table: Login +2026-06-15 13:50:29 | INFO | Fetching Login data for yesterday +2026-06-15 13:50:29 | INFO | Fetched 454 Login rows +2026-06-15 13:50:29 | INFO | Fetched 454 rows +2026-06-15 13:50:29 | INFO | Deleting from Login +2026-06-15 13:50:29 | INFO | No delete logic required for Login +2026-06-15 13:50:29 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-15 13:50:29 | INFO | Login loaded successfully (454 rows) +2026-06-15 13:50:29 | INFO | ================================================================================ +2026-06-15 13:50:29 | INFO | Processing Table: Stock_Details +2026-06-15 13:50:29 | INFO | Fetching Stock Details data for 808 MIDs +2026-06-15 13:50:33 | INFO | Fetched 40,840 Stock Details rows +2026-06-15 13:50:33 | INFO | Fetched 40840 rows +2026-06-15 13:50:33 | INFO | Deleting from Stock_Details +2026-06-15 13:50:34 | INFO | Stock_Details: inserted 40,840 rows into ClickHouse +2026-06-15 13:50:34 | INFO | Stock_Details loaded successfully (40840 rows) +2026-06-15 13:50:34 | INFO | ================================================================================ +2026-06-15 13:50:34 | INFO | Processing Table: Attendance +2026-06-15 13:50:34 | INFO | Fetching Attendance data from 2026-05-19 to 2026-06-03 +2026-06-15 13:50:35 | INFO | Fetched 12,432 attendance rows for 579 employees +2026-06-15 13:50:35 | INFO | Fetched 12432 rows +2026-06-15 13:50:35 | INFO | Truncated table Attendance +2026-06-15 13:50:35 | INFO | Attendance: inserted 12,432 rows into ClickHouse +2026-06-15 13:50:35 | INFO | Attendance loaded successfully (12432 rows) +2026-06-15 13:50:35 | INFO | ================================================================================ +2026-06-15 13:50:35 | INFO | Processing Table: Store_Master +2026-06-15 13:50:35 | INFO | Fetching Store Master data +2026-06-15 13:50:37 | INFO | Fetched 5,984 stores +2026-06-15 13:50:37 | INFO | Fetched 5984 rows +2026-06-15 13:50:37 | INFO | Truncated table Store_Master +2026-06-15 13:50:38 | INFO | Store_Master: inserted 5,984 rows into ClickHouse +2026-06-15 13:50:38 | INFO | Store_Master loaded successfully (5984 rows) +2026-06-15 13:50:38 | INFO | ================================================================================ +2026-06-15 13:50:38 | INFO | Processing Table: SKU_Master +2026-06-15 13:50:38 | INFO | Fetching SKU Master data +2026-06-15 13:50:38 | INFO | Fetched 160 SKU Master rows +2026-06-15 13:50:38 | INFO | Fetched 160 rows +2026-06-15 13:50:38 | INFO | Truncated table SKU_Master +2026-06-15 13:50:38 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-15 13:50:38 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-15 13:50:38 | INFO | ================================================================================ +2026-06-15 13:50:38 | INFO | Processing Table: display_master +2026-06-15 13:50:38 | INFO | Fetching Display Master data +2026-06-15 13:50:38 | INFO | Fetched 135 Display Master records +2026-06-15 13:50:38 | INFO | Fetched 135 rows +2026-06-15 13:50:38 | INFO | Truncated table display_master +2026-06-15 13:50:39 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-15 13:50:39 | INFO | display_master loaded successfully (135 rows) +2026-06-15 13:50:39 | INFO | ================================================================================ +2026-06-15 13:50:39 | INFO | Processing Table: Employee_Master +2026-06-15 13:50:39 | INFO | Fetching Employee Master data +2026-06-15 13:50:39 | INFO | Fetched 2,273 Employee Master records +2026-06-15 13:50:39 | INFO | Fetched 2273 rows +2026-06-15 13:50:39 | INFO | Truncated table Employee_Master +2026-06-15 13:50:39 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse +2026-06-15 13:50:39 | INFO | Employee_Master loaded successfully (2273 rows) +2026-06-15 13:50:39 | INFO | ================================================================================ +2026-06-15 13:50:39 | INFO | Processing Table: Journey_Plan +2026-06-15 13:50:39 | INFO | Fetching Journey Plan for 2026-06 +2026-06-15 13:50:40 | INFO | Fetched 21,142 Journey Plan records +2026-06-15 13:50:40 | INFO | Fetched 21142 rows +2026-06-15 13:50:40 | INFO | Deleting from Journey_Plan +2026-06-15 13:50:40 | INFO | Journey_Plan: inserted 21,142 rows into ClickHouse +2026-06-15 13:50:40 | INFO | Journey_Plan loaded successfully (21142 rows) +2026-06-15 13:50:40 | INFO | ================================================================================ +2026-06-15 13:50:40 | INFO | Processing Table: coverage_remarks +2026-06-15 13:50:40 | INFO | Fetching Coverage Remarks +2026-06-15 13:50:40 | INFO | Fetched 29 Coverage Remark records +2026-06-15 13:50:40 | INFO | Fetched 29 rows +2026-06-15 13:50:40 | INFO | Truncated table coverage_remarks +2026-06-15 13:50:40 | INFO | coverage_remarks: inserted 29 rows into ClickHouse +2026-06-15 13:50:40 | INFO | coverage_remarks loaded successfully (29 rows) +2026-06-15 13:50:40 | INFO | ================================================================================ +2026-06-15 13:50:40 | INFO | Processing Table: mapping_storevisibility +2026-06-15 13:50:40 | ERROR | Failed processing table mapping_storevisibility +Traceback (most recent call last): + File "D:\data_move\main.py", line 177, in main + df = get_dataframe( + fn_name=fn_name, + ...<3 lines>... + run_date=run_date, + ) + File "D:\data_move\main.py", line 83, in get_dataframe + return fn(sql_engine) +TypeError: fetch_mapping_storevisibility() missing 1 required positional argument: 'run_date' +2026-06-15 13:51:03 | INFO | ================================================================================ +2026-06-15 13:51:03 | INFO | Hello from data-move Python data pipeline! +2026-06-15 13:51:03 | INFO | Pipeline Run Date: 2026-06-03 +2026-06-15 13:51:03 | INFO | Connecting to databases... +2026-06-15 13:51:03 | INFO | +2026-06-15 13:51:04 | INFO | +2026-06-15 13:51:05 | INFO | Database connections established +2026-06-15 13:51:05 | INFO | Collecting MIDs for: 2026-06-03 +2026-06-15 13:51:05 | INFO | Found 808 MIDs +2026-06-15 13:51:05 | INFO | ================================================================================ +2026-06-15 13:51:05 | INFO | Processing Table: SOS_OneApp +2026-06-15 13:51:05 | INFO | Fetching data for 808 MIDs +2026-06-15 13:51:06 | INFO | Fetched 3,565 rows from SQL Server +2026-06-15 13:51:06 | INFO | Fetched 3565 rows +2026-06-15 13:51:06 | INFO | Deleting from SOS_OneApp +2026-06-15 13:51:06 | INFO | SOS_OneApp: inserted 3,565 rows into ClickHouse +2026-06-15 13:51:06 | INFO | SOS_OneApp loaded successfully (3565 rows) +2026-06-15 13:51:06 | INFO | ================================================================================ +2026-06-15 13:51:06 | INFO | Processing Table: OQaD +2026-06-15 13:51:06 | INFO | Fetching OQaD data for run_date=2026-06-03 +2026-06-15 13:51:09 | INFO | Fetched 455 rows +2026-06-15 13:51:09 | INFO | Fetched 455 rows +2026-06-15 13:51:09 | INFO | Deleting from OQaD +2026-06-15 13:51:09 | INFO | OQaD: inserted 455 rows into ClickHouse +2026-06-15 13:51:09 | INFO | OQaD loaded successfully (455 rows) +2026-06-15 13:51:09 | INFO | ================================================================================ +2026-06-15 13:51:09 | INFO | Processing Table: Survey +2026-06-15 13:51:09 | INFO | Fetching Survey data for 808 MIDs +2026-06-15 13:51:09 | INFO | Fetched 139 Survey rows +2026-06-15 13:51:09 | INFO | Fetched 139 rows +2026-06-15 13:51:09 | INFO | Deleting from Survey +2026-06-15 13:51:09 | INFO | Survey: inserted 139 rows into ClickHouse +2026-06-15 13:51:09 | INFO | Survey loaded successfully (139 rows) +2026-06-15 13:51:09 | INFO | ================================================================================ +2026-06-15 13:51:09 | INFO | Processing Table: additional_visibility +2026-06-15 13:51:09 | INFO | Fetching Additional Visibility data for 808 MIDs +2026-06-15 13:51:10 | INFO | Fetched 1,884 Additional Visibility rows +2026-06-15 13:51:10 | INFO | Fetched 1884 rows +2026-06-15 13:51:10 | INFO | Deleting from additional_visibility +2026-06-15 13:51:10 | INFO | additional_visibility: inserted 1,884 rows into ClickHouse +2026-06-15 13:51:10 | INFO | additional_visibility loaded successfully (1884 rows) +2026-06-15 13:51:10 | INFO | ================================================================================ +2026-06-15 13:51:10 | INFO | Processing Table: Coverage +2026-06-15 13:51:10 | INFO | Fetching coverage data for 808 MIDs +2026-06-15 13:51:10 | INFO | Fetched 751 rows from SQL Server +2026-06-15 13:51:10 | INFO | Fetched 751 rows +2026-06-15 13:51:10 | INFO | Deleting from Coverage +2026-06-15 13:51:11 | INFO | Coverage: inserted 751 rows into ClickHouse +2026-06-15 13:51:11 | INFO | Coverage loaded successfully (751 rows) +2026-06-15 13:51:11 | INFO | ================================================================================ +2026-06-15 13:51:11 | INFO | Processing Table: Login +2026-06-15 13:51:11 | INFO | Fetching Login data for yesterday +2026-06-15 13:51:11 | INFO | Fetched 454 Login rows +2026-06-15 13:51:11 | INFO | Fetched 454 rows +2026-06-15 13:51:11 | INFO | Deleting from Login +2026-06-15 13:51:11 | INFO | No delete logic required for Login +2026-06-15 13:51:11 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-15 13:51:11 | INFO | Login loaded successfully (454 rows) +2026-06-15 13:51:11 | INFO | ================================================================================ +2026-06-15 13:51:11 | INFO | Processing Table: Stock_Details +2026-06-15 13:51:11 | INFO | Fetching Stock Details data for 808 MIDs +2026-06-15 13:51:15 | INFO | Fetched 40,840 Stock Details rows +2026-06-15 13:51:15 | INFO | Fetched 40840 rows +2026-06-15 13:51:15 | INFO | Deleting from Stock_Details +2026-06-15 13:51:15 | INFO | Stock_Details: inserted 40,840 rows into ClickHouse +2026-06-15 13:51:15 | INFO | Stock_Details loaded successfully (40840 rows) +2026-06-15 13:51:15 | INFO | ================================================================================ +2026-06-15 13:51:15 | INFO | Processing Table: Attendance +2026-06-15 13:51:15 | INFO | Fetching Attendance data from 2026-05-19 to 2026-06-03 +2026-06-15 13:51:16 | INFO | Fetched 12,432 attendance rows for 579 employees +2026-06-15 13:51:16 | INFO | Fetched 12432 rows +2026-06-15 13:51:17 | INFO | Truncated table Attendance +2026-06-15 13:51:17 | INFO | Attendance: inserted 12,432 rows into ClickHouse +2026-06-15 13:51:17 | INFO | Attendance loaded successfully (12432 rows) +2026-06-15 13:51:17 | INFO | ================================================================================ +2026-06-15 13:51:17 | INFO | Processing Table: Store_Master +2026-06-15 13:51:17 | INFO | Fetching Store Master data +2026-06-15 13:51:20 | INFO | Fetched 5,984 stores +2026-06-15 13:51:20 | INFO | Fetched 5984 rows +2026-06-15 13:51:20 | INFO | Truncated table Store_Master +2026-06-15 13:51:20 | INFO | Store_Master: inserted 5,984 rows into ClickHouse +2026-06-15 13:51:20 | INFO | Store_Master loaded successfully (5984 rows) +2026-06-15 13:51:20 | INFO | ================================================================================ +2026-06-15 13:51:20 | INFO | Processing Table: SKU_Master +2026-06-15 13:51:20 | INFO | Fetching SKU Master data +2026-06-15 13:51:20 | INFO | Fetched 160 SKU Master rows +2026-06-15 13:51:20 | INFO | Fetched 160 rows +2026-06-15 13:51:21 | INFO | Truncated table SKU_Master +2026-06-15 13:51:21 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-15 13:51:21 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-15 13:51:21 | INFO | ================================================================================ +2026-06-15 13:51:21 | INFO | Processing Table: display_master +2026-06-15 13:51:21 | INFO | Fetching Display Master data +2026-06-15 13:51:21 | INFO | Fetched 135 Display Master records +2026-06-15 13:51:21 | INFO | Fetched 135 rows +2026-06-15 13:51:21 | INFO | Truncated table display_master +2026-06-15 13:51:21 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-15 13:51:21 | INFO | display_master loaded successfully (135 rows) +2026-06-15 13:51:21 | INFO | ================================================================================ +2026-06-15 13:51:21 | INFO | Processing Table: Employee_Master +2026-06-15 13:51:21 | INFO | Fetching Employee Master data +2026-06-15 13:51:21 | INFO | Fetched 2,273 Employee Master records +2026-06-15 13:51:21 | INFO | Fetched 2273 rows +2026-06-15 13:51:22 | INFO | Truncated table Employee_Master +2026-06-15 13:51:22 | INFO | Employee_Master: inserted 2,273 rows into ClickHouse +2026-06-15 13:51:22 | INFO | Employee_Master loaded successfully (2273 rows) +2026-06-15 13:51:22 | INFO | ================================================================================ +2026-06-15 13:51:22 | INFO | Processing Table: Journey_Plan +2026-06-15 13:51:22 | INFO | Fetching Journey Plan for 2026-06 +2026-06-15 13:51:22 | INFO | Fetched 21,142 Journey Plan records +2026-06-15 13:51:22 | INFO | Fetched 21142 rows +2026-06-15 13:51:22 | INFO | Deleting from Journey_Plan +2026-06-15 13:51:22 | INFO | Journey_Plan: inserted 21,142 rows into ClickHouse +2026-06-15 13:51:22 | INFO | Journey_Plan loaded successfully (21142 rows) +2026-06-15 13:51:22 | INFO | ================================================================================ +2026-06-15 13:51:22 | INFO | Processing Table: coverage_remarks +2026-06-15 13:51:22 | INFO | Fetching Coverage Remarks +2026-06-15 13:51:22 | INFO | Fetched 29 Coverage Remark records +2026-06-15 13:51:22 | INFO | Fetched 29 rows +2026-06-15 13:51:23 | INFO | Truncated table coverage_remarks +2026-06-15 13:51:23 | INFO | coverage_remarks: inserted 29 rows into ClickHouse +2026-06-15 13:51:23 | INFO | coverage_remarks loaded successfully (29 rows) +2026-06-15 13:51:23 | INFO | ================================================================================ +2026-06-15 13:51:23 | INFO | Processing Table: mapping_storevisibility +2026-06-15 13:51:23 | INFO | Fetching Mapping Store Visibility for 2026-06-03 +2026-06-15 13:51:23 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-15 13:51:23 | WARNING | mapping_storevisibility returned no rows +2026-06-15 13:51:23 | INFO | ================================================================================ +2026-06-15 13:51:23 | INFO | Processing Table: Master_VisibilityReason +2026-06-15 13:51:23 | INFO | Fetching Master Visibility Reason data +2026-06-15 13:51:23 | INFO | Fetched 17 Master Visibility Reason records +2026-06-15 13:51:23 | INFO | Fetched 17 rows +2026-06-15 13:51:23 | INFO | Truncated table Master_VisibilityReason +2026-06-15 13:51:23 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-15 13:51:23 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-15 13:51:23 | INFO | ================================================================================ +2026-06-15 13:51:23 | INFO | Processing Table: Master_VisibilityDefinition +2026-06-15 13:51:23 | INFO | Fetching Master Visibility Definition data +2026-06-15 13:51:23 | INFO | Fetched 861 Master Visibility Definition records +2026-06-15 13:51:23 | INFO | Fetched 861 rows +2026-06-15 13:51:23 | INFO | Truncated table Master_VisibilityDefinition +2026-06-15 13:51:24 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-15 13:51:24 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-15 13:51:24 | INFO | ================================================================================ +2026-06-15 13:51:24 | INFO | Processing Table: Web_Logins +2026-06-15 13:51:24 | INFO | Fetching Web Login data for 2026-06-03 +2026-06-15 13:51:24 | INFO | Fetched 200 Web Login records +2026-06-15 13:51:24 | INFO | Fetched 200 rows +2026-06-15 13:51:24 | INFO | Deleting from Web_Logins +2026-06-15 13:51:24 | INFO | Web_Logins: inserted 200 rows into ClickHouse +2026-06-15 13:51:24 | INFO | Web_Logins loaded successfully (200 rows) +2026-06-15 13:51:24 | INFO | ================================================================================ +2026-06-15 13:51:24 | INFO | Processing Table: Promotion +2026-06-15 13:51:24 | INFO | Fetching Promotion data for 808 MIDs +2026-06-15 13:51:28 | INFO | Fetched 8,162 Promotion records +2026-06-15 13:51:28 | INFO | Fetched 8162 rows +2026-06-15 13:51:28 | INFO | Deleting from Promotion +2026-06-15 13:51:29 | INFO | Promotion: inserted 8,162 rows into ClickHouse +2026-06-15 13:51:29 | INFO | Promotion loaded successfully (8162 rows) +2026-06-15 13:51:29 | INFO | ================================================================================ +2026-06-15 13:51:29 | INFO | Processing Table: PaidVisibility +2026-06-15 13:51:29 | INFO | Fetching Paid Visibility data for 808 MIDs +2026-06-15 13:51:30 | INFO | Fetched 1,064 Paid Visibility records +2026-06-15 13:51:30 | INFO | Fetched 1064 rows +2026-06-15 13:51:30 | INFO | Deleting from PaidVisibility +2026-06-15 13:51:30 | INFO | PaidVisibility: inserted 1,064 rows into ClickHouse +2026-06-15 13:51:30 | INFO | PaidVisibility loaded successfully (1064 rows) +2026-06-15 13:51:30 | INFO | ================================================================================ +2026-06-15 13:51:30 | INFO | Processing Table: Master_Salesterritorylayer +2026-06-15 13:51:30 | INFO | Fetching Master Sales Territory Layer data +2026-06-15 13:51:30 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-15 13:51:30 | INFO | Fetched 33 rows +2026-06-15 13:51:30 | INFO | Truncated table Master_Salesterritorylayer +2026-06-15 13:51:30 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-15 13:51:30 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-15 13:51:30 | INFO | ================================================================================ +2026-06-15 13:51:30 | INFO | Pipeline Completed Successfully +2026-06-15 13:51:30 | INFO | ================================================================================ diff --git a/main.py b/main.py index e1e6aa1..c59725a 100644 --- a/main.py +++ b/main.py @@ -13,238 +13,262 @@ # /// - from __future__ import annotations -import os -# import pyarrow import sys -from datetime import date, timedelta, datetime +from datetime import date, datetime, timedelta import polars as pl import yaml -from dotenv import load_dotenv -from sqlalchemy import create_engine, text -from sqlalchemy.engine import Engine, URL - -import clickhouse_connect from log import log +from clickhouse_task.create_table import create_clickhouse_table +from clickhouse_task.delete_task import ( + delete_existing_data, + truncate_table, +) +from clickhouse_task.load_table import load_to_clickhouse + +from db_con.connection import ( + build_sql_server_engine, + build_clickhouse_engine, + get_clickhouse_client, +) + +from mids import ( + MID_TABLE_COV, + MID_TABLE_COV1, +) -from clickhouse_task.create_table import * -from clickhouse_task.delete_task import * -from clickhouse_task.load_table import * -from db_con.connection import * -from mids import * from masters.dimensions import * from masters.bridge import * from kpi.facts import * +# ========================================================== +# Helpers +# ========================================================== + +def table_exists( + client, + table_name: str, +) -> bool: + + return bool( + client.command( + f"EXISTS TABLE {table_name}" + ) + ) +def get_dataframe( + fn_name: str, + fetch_by: str, + sql_engine, + mids, + run_date, +): + + fn = globals()[fn_name] + + if fetch_by == "mids": + return fn(sql_engine, mids) + + if fetch_by == "run_date": + return fn(sql_engine, run_date) + + return fn(sql_engine) +# ========================================================== +# Main +# ========================================================== def main(): log.info("=" * 80) - log.info("Hello from data-move Python data pipeline !") + log.info("Hello from data-move Python data pipeline!") + + # ------------------------------------------------------ + # Run Date + # ------------------------------------------------------ if len(sys.argv) > 1: - run_date = datetime.strptime(sys.argv[1], "%Y-%m-%d").date() + run_date = datetime.strptime( + sys.argv[1], + "%Y-%m-%d", + ).date() else: run_date = date.today() - timedelta(days=1) - log.info(f"Data-pipeline running Date is -:{run_date}") - # connecting with both db servers sql-server + log.info( + "Pipeline Run Date: %s", + run_date, + ) + + # ------------------------------------------------------ + # Connections + # ------------------------------------------------------ + + log.info( + "Connecting to databases..." + ) - - log.info("connecting with both db servers sql-serveras well as clickhouse DB") - sql_engine = build_sql_server_engine() clickhouse_engine = build_clickhouse_engine() - client=get_clickhouse_client() + client = get_clickhouse_client() - log.info("Both databases connected successfully") + log.info( + "Database connections established" + ) - mids = MID_TABLE_COV(sql_engine, run_date) + # ------------------------------------------------------ + # Delete Keys + # ------------------------------------------------------ + + mids = MID_TABLE_COV( + sql_engine, + run_date, + ) emp_visit_df = MID_TABLE_COV1( sql_engine, - run_date + run_date, ) - delete_existing_data( - client=client, - run_date=run_date, - mids=mids, - emp_visit_df=emp_visit_df, - ) + # ------------------------------------------------------ + # Config + # ------------------------------------------------------ + with open( + "t.yml", + "r", + ) as file: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - mid_list = ",".join(map(str, mids)) - conditions = { - "mids": f"MID IN ({mid_list})", - - "j_plan": ( - f"MONTH(VisitDate) = {run_date.month} " - f"AND YEAR(VisitDate) = {run_date.year}" - ), - - "mapping": ( - f"CAST(Z.FromDate AS DATE) <= '{run_date}' " - f"AND CAST(Z.ToDate AS DATE) >= '{run_date}'" - ), - - "web": ( - f"CAST(login_date AS DATE) = '{run_date}'" - ), - - "none": None, -} - - - - # fetching polar df from sql-server - - - - - - with open("tables.yml", "r") as file: config = yaml.safe_load(file) - for table in config["tables"]: - - table_name=table["name"] - table_type=table["type"] - operation=table["operation"] - condition=table["condition"] - - c = conditions.get(condition) - - log.info("=" * 80) - log.info("TABLE=%s | TYPE=%s | OPERATION=%s", - table_name, - table_type, - operation) - + # ------------------------------------------------------ + # Process Tables + # ------------------------------------------------------ - fn=f"fetch_{table_name}" + for table in config["tables"]: + + table_name = table["name"] + operation = table["operation"] fetch_by = table["fetch_by"] - if operation == "DELETE+INSERT" : - if fetch_by == "mids": - df = globals()[fn](sql_engine, mids) - check_query=f"EXISTS TABLE {table_name}" - exists = client.command(check_query) - if exists == 0 : - create_clickhouse_table(df, table_name, clickhouse_engine) + log.info("=" * 80) + log.info( + "Processing Table: %s", + table_name, + ) - else: - truncate_table(client , table_name ) - log.info(f"Truncate a ClickHouse table - {table_name}") + try: - load_to_clickhouse(client=client,table_name=table_name,df=df) + # ------------------------------------------ + # Fetch Data + # ------------------------------------------ - elif fetch_by == "run_date": - df = globals()[fn](sql_engine, run_date) - check_query=f"EXISTS TABLE {table_name}" - exists = client.command(check_query) - if exists == 0 : - create_clickhouse_table(df, table_name, clickhouse_engine) + fn_name = f"fetch_{table_name}" - else: - truncate_table(client , table_name ) - log.info(f"Truncate a ClickHouse table - {table_name}") + df = get_dataframe( + fn_name=fn_name, + fetch_by=fetch_by, + sql_engine=sql_engine, + mids=mids, + run_date=run_date, + ) + + if df.is_empty(): + + log.warning( + "%s returned no rows", + table_name, + ) + + continue + + log.info( + "Fetched %s rows", + len(df), + ) + + # ------------------------------------------ + # Create Table If Missing + # ------------------------------------------ + + exists = table_exists( + client, + table_name, + ) + + if not exists: + + log.info( + "Creating table %s", + table_name, + ) + + create_clickhouse_table( + df=df, + table_name=table_name, + clickhouse_engine=clickhouse_engine, + ) + + # ------------------------------------------ + # Existing Table Logic + # ------------------------------------------ + + else: + + if operation == "DELETE+INSERT": + + truncate_table( + client, + table_name, + ) - load_to_clickhouse(client=client,table_name=table_name,df=df) else: - df = globals()[fn](sql_engine) - check_query=f"EXISTS TABLE {table_name}" - exists = client.command(check_query) - if exists == 0 : - create_clickhouse_table(df, table_name, clickhouse_engine) - else: - truncate_table(client , table_name ) - log.info(f"Truncate a ClickHouse table - {table_name}") + delete_existing_data( + client=client, + table_name=table_name, + run_date=run_date, + mids=mids, + emp_visit_df=emp_visit_df, + ) - load_to_clickhouse(client=client,table_name=table_name,df=df) - else: - if fetch_by == "mids": - df = globals()[fn](sql_engine, mids) - check_query=f"EXISTS TABLE {table_name}" - exists = client.command(check_query) - if exists == 0 : - create_clickhouse_table(df, table_name, clickhouse_engine) + # ------------------------------------------ + # Load Data + # ------------------------------------------ - else: + load_to_clickhouse( + client=client, + table_name=table_name, + df=df, + ) - delete_rows(client, table_name, c ) + log.info( + "%s loaded successfully (%s rows)", + table_name, + len(df), + ) - load_to_clickhouse(client=client,table_name=table_name,df=df) - - elif fetch_by == "run_date": - df = globals()[fn](sql_engine, run_date) - check_query=f"EXISTS TABLE {table_name}" - exists = client.command(check_query) - if exists == 0 : - create_clickhouse_table(df, table_name, clickhouse_engine) - - else: - - delete_rows(client, table_name, c) - - load_to_clickhouse(client=client,table_name=table_name,df=df) - else: - df = globals()[fn](sql_engine) - check_query=f"EXISTS TABLE {table_name}" - exists = client.command(check_query) - if exists == 0 : - create_clickhouse_table(df, table_name, clickhouse_engine) - - else: - truncate_table(client , table_name ) - log.info(f"Truncate a ClickHouse table - {table_name}") - - load_to_clickhouse(client=client,table_name=table_name,df=df) + except Exception: + log.exception( + "Failed processing table %s", + table_name, + ) + raise + log.info("=" * 80) + log.info("Pipeline Completed Successfully") + log.info("=" * 80) if __name__ == "__main__": diff --git a/masters/dimensions.py b/masters/dimensions.py index 9f950d6..9f63bf8 100644 --- a/masters/dimensions.py +++ b/masters/dimensions.py @@ -484,3 +484,32 @@ def fetch_Master_Salesterritorylayer( return df +def fetch_coverage_remarks(engine: Engine ) -> pl.DataFrame: + """ + Source: + OneApp_KelloggsMT.dbo.Master_NonWorkingReason + + Target: + coverage_remarks + """ + + sql = """ + SELECT + 40148 AS project_id, + ReasonId AS reason_id, + Reason AS reason_remarks + FROM OneApp_KelloggsMT.dbo.Master_NonWorkingReason + """ + + log.info("Fetching Coverage Remarks") + + df = pl.read_database( + query=sql, + connection=engine + ) + + log.info( + f"Fetched {len(df):,} Coverage Remark records" + ) + + return df diff --git a/t.yml b/t.yml new file mode 100644 index 0000000..64c4b14 --- /dev/null +++ b/t.yml @@ -0,0 +1,105 @@ +tables: + - name: SOS_OneApp + type: FACT + operation: INSERT + fetch_by: mids + + - name: OQaD + type: FACT + operation: INSERT + fetch_by: run_date + + - name: Survey + type: FACT + operation: INSERT + fetch_by: mids + + - name: additional_visibility + type: FACT + operation: INSERT + fetch_by: mids + + - name: Coverage + type: FACT + operation: INSERT + fetch_by: mids + + - name: Login + type: FACT + operation: INSERT + fetch_by: run_date + + - name: Stock_Details + type: FACT + operation: INSERT + fetch_by: mids + + - name: Attendance + type: FACT + operation: DELETE+INSERT + fetch_by: run_date + + - name: Store_Master + type: DIMENSION + operation: DELETE+INSERT + fetch_by: none + + - name: SKU_Master + type: DIMENSION + operation: DELETE+INSERT + fetch_by: none + + - name: display_master + type: DIMENSION + operation: DELETE+INSERT + fetch_by: none + + - name: Employee_Master + type: DIMENSION + operation: DELETE+INSERT + fetch_by: none + + - name: Journey_Plan + type: FACT + operation: INSERT + fetch_by: run_date + + - name: coverage_remarks + type: DIMENSION + operation: DELETE+INSERT + fetch_by: none + + - name: mapping_storevisibility + type: BRIDGE + operation: DELETE+INSERT + fetch_by: run_date + + - name: Master_VisibilityReason + type: DIMENSION + operation: DELETE+INSERT + fetch_by: none + + - name: Master_VisibilityDefinition + type: DIMENSION + operation: DELETE+INSERT + fetch_by: none + + - name: Web_Logins + type: FACT + operation: INSERT + fetch_by: run_date + + - name: Promotion + type: FACT + operation: INSERT + fetch_by: mids + + - name: PaidVisibility + type: FACT + operation: INSERT + fetch_by: mids + + - name: Master_Salesterritorylayer + type: DIMENSION + operation: DELETE+INSERT + fetch_by: none \ No newline at end of file diff --git a/tables.yml b/tables.yml index 0453b80..9111820 100644 --- a/tables.yml +++ b/tables.yml @@ -9,7 +9,7 @@ tables: type: FACT operation: INSERT fetch_by: run_date - condition: mids + condition: none - name: Survey type: FACT @@ -32,8 +32,8 @@ tables: - name: Login type: FACT operation: INSERT - fetch_by: mids - condition: mids + fetch_by: run_date + condition: none - name: Stock_Details type: FACT diff --git a/test.py b/test.py deleted file mode 100644 index e4fa967..0000000 --- a/test.py +++ /dev/null @@ -1,267 +0,0 @@ -from __future__ import annotations - -import sys -from datetime import date, datetime, timedelta - -import polars as pl -import yaml - -from log import log - -from clickhouse_task.create_table import create_clickhouse_table -from clickhouse_task.delete_task import delete_existing_data , truncate_table -from clickhouse_task.load_table import load_to_clickhouse - -from db_con.connection import ( - build_sql_server_engine, - build_clickhouse_engine, - get_clickhouse_client, -) - -from mids import MID_TABLE_COV, MID_TABLE_COV1 - -from masters.dimensions import * -from masters.bridge import * -from kpi.facts import * - - -# ============================================================ -# Helpers -# ============================================================ - -def get_dataframe( - fn_name: str, - fetch_by: str, - sql_engine, - mids, - run_date, -) -> pl.DataFrame: - - fn = globals()[fn_name] - - if fetch_by == "mids": - return fn(sql_engine, mids) - - if fetch_by == "run_date": - return fn(sql_engine, run_date) - - return fn(sql_engine) - - -def ensure_table_exists( - client, - clickhouse_engine, - table_name: str, - df: pl.DataFrame, -) -> bool: - - exists = client.command( - f"EXISTS TABLE {table_name}" - ) - - if not exists: - log.info( - "Creating ClickHouse table: %s", - table_name, - ) - - create_clickhouse_table( - df=df, - table_name=table_name, - clickhouse_engine=clickhouse_engine, - ) - - return bool(exists) - - -# ============================================================ -# Main -# ============================================================ - -def main(): - - log.info("=" * 80) - log.info("Hello from data-move Python data pipeline !") - - # -------------------------------------------------------- - # Run Date - # -------------------------------------------------------- - - if len(sys.argv) > 1: - run_date = datetime.strptime( - sys.argv[1], - "%Y-%m-%d", - ).date() - else: - run_date = date.today() - timedelta(days=1) - - log.info( - "Pipeline Run Date : %s", - run_date, - ) - - # -------------------------------------------------------- - # Connections - # -------------------------------------------------------- - - log.info( - "Connecting to SQL Server and ClickHouse" - ) - - sql_engine = build_sql_server_engine() - - clickhouse_engine = ( - build_clickhouse_engine() - ) - - client = get_clickhouse_client() - - log.info( - "Database connections established" - ) - - # -------------------------------------------------------- - # Collect Delete Keys - # -------------------------------------------------------- - - mids = MID_TABLE_COV( - sql_engine, - run_date, - ) - - emp_visit_df = MID_TABLE_COV1( - sql_engine, - run_date, - ) - - # -------------------------------------------------------- - # Delete Existing Data - # -------------------------------------------------------- - - delete_existing_data( - client=client, - run_date=run_date, - mids=mids, - emp_visit_df=emp_visit_df, - ) - - # -------------------------------------------------------- - # Load Config - # -------------------------------------------------------- - - with open( - "tables.yml", - "r", - ) as file: - - config = yaml.safe_load(file) - - # -------------------------------------------------------- - # Process Tables - # -------------------------------------------------------- - - for table in config["tables"]: - - table_name = table["name"] - table_type = table["type"] - operation = table["operation"] - fetch_by = table["fetch_by"] - - log.info("=" * 80) - - log.info( - "TABLE=%s | TYPE=%s | OPERATION=%s", - table_name, - table_type, - operation, - ) - - try: - - # -------------------------------------------- - # Fetch - # -------------------------------------------- - - fn_name = f"fetch_{table_name}" - - df = get_dataframe( - fn_name=fn_name, - fetch_by=fetch_by, - sql_engine=sql_engine, - mids=mids, - run_date=run_date, - ) - - if df.is_empty(): - - log.warning( - "%s returned no rows", - table_name, - ) - - continue - - log.info( - "Fetched %s rows", - len(df), - ) - - # -------------------------------------------- - # Create Table if Missing - # -------------------------------------------- - - exists = ensure_table_exists( - client=client, - clickhouse_engine=clickhouse_engine, - table_name=table_name, - df=df, - ) - - # -------------------------------------------- - # Full Refresh Tables - # -------------------------------------------- - - if exists and operation == "DELETE+INSERT": - - truncate_table( - client, - table_name, - ) - - log.info( - "Truncated %s", - table_name, - ) - - # -------------------------------------------- - # Load - # -------------------------------------------- - - load_to_clickhouse( - client=client, - table_name=table_name, - df=df, - ) - - log.info( - "%s loaded successfully (%s rows)", - table_name, - len(df), - ) - - except Exception as e: - - log.exception( - "Failed processing table %s", - table_name, - ) - - raise - - log.info("=" * 80) - log.info("Pipeline Completed Successfully") - log.info("=" * 80) - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/test2.py b/test2.py deleted file mode 100644 index 720d2a5..0000000 --- a/test2.py +++ /dev/null @@ -1,275 +0,0 @@ -# /// script -# requires-python = ">=3.11" -# dependencies = [ -# "polars>=0.20.0", -# "pyarrow>=18.0.0", -# "sqlalchemy>=2.0.0", -# "pyodbc>=5.0.0", -# "clickhouse-connect>=0.7.0", -# "clickhouse-sqlalchemy>=0.3.2", -# "pyyaml>=6.0.3", -# "python-dotenv>=1.0.0", -# ] -# /// - - -from __future__ import annotations - -import sys -from datetime import date, datetime, timedelta - -import polars as pl -import yaml - - -from log import log - -from clickhouse_task.create_table import create_clickhouse_table -from clickhouse_task.delete_task import ( - delete_existing_data, - truncate_table, -) - -from clickhouse_task.load_table import load_to_clickhouse - -from db_con.connection import ( - build_sql_server_engine, - build_clickhouse_engine, - get_clickhouse_client, -) - -from mids import ( - MID_TABLE_COV, - MID_TABLE_COV1, -) - -from masters.dimensions import * -from masters.bridge import * -from kpi.facts import * - - -# ========================================================== -# Helpers -# ========================================================== - -def table_exists( - client, - table_name: str, -) -> bool: - - return bool( - client.command( - f"EXISTS TABLE {table_name}" - ) - ) - - -def get_dataframe( - fn_name: str, - fetch_by: str, - sql_engine, - mids, - run_date, -): - - fn = globals()[fn_name] - - if fetch_by == "mids": - return fn(sql_engine, mids) - - if fetch_by == "run_date": - return fn(sql_engine, run_date) - - return fn(sql_engine) - - -# ========================================================== -# Main -# ========================================================== - -def main(): - - log.info("=" * 80) - log.info("Hello from data-move Python data pipeline!") - - # ------------------------------------------------------ - # Run Date - # ------------------------------------------------------ - - if len(sys.argv) > 1: - run_date = datetime.strptime( - sys.argv[1], - "%Y-%m-%d", - ).date() - else: - run_date = date.today() - timedelta(days=1) - - log.info( - "Pipeline Run Date: %s", - run_date, - ) - - # ------------------------------------------------------ - # Connections - # ------------------------------------------------------ - - log.info( - "Connecting to databases..." - ) - - sql_engine = build_sql_server_engine() - clickhouse_engine = build_clickhouse_engine() - client = get_clickhouse_client() - - log.info( - "Database connections established" - ) - - # ------------------------------------------------------ - # Delete Keys - # ------------------------------------------------------ - - mids = MID_TABLE_COV( - sql_engine, - run_date, - ) - - emp_visit_df = MID_TABLE_COV1( - sql_engine, - run_date, - ) - - # ------------------------------------------------------ - # Config - # ------------------------------------------------------ - - with open( - "tables.yml", - "r", - ) as file: - - config = yaml.safe_load(file) - - # ------------------------------------------------------ - # Process Tables - # ------------------------------------------------------ - - for table in config["tables"]: - - table_name = table["name"] - operation = table["operation"] - fetch_by = table["fetch_by"] - - log.info("=" * 80) - log.info( - "Processing Table: %s", - table_name, - ) - - try: - - # ------------------------------------------ - # Fetch Data - # ------------------------------------------ - - fn_name = f"fetch_{table_name}" - - df = get_dataframe( - fn_name=fn_name, - fetch_by=fetch_by, - sql_engine=sql_engine, - mids=mids, - run_date=run_date, - ) - - if df.is_empty(): - - log.warning( - "%s returned no rows", - table_name, - ) - - continue - - log.info( - "Fetched %s rows", - len(df), - ) - - # ------------------------------------------ - # Create Table If Missing - # ------------------------------------------ - - exists = table_exists( - client, - table_name, - ) - - if not exists: - - log.info( - "Creating table %s", - table_name, - ) - - create_clickhouse_table( - df=df, - table_name=table_name, - clickhouse_engine=clickhouse_engine, - ) - - # ------------------------------------------ - # Existing Table Logic - # ------------------------------------------ - - else: - - if operation == "DELETE+INSERT": - - truncate_table( - client, - table_name, - ) - - else: - - delete_existing_data( - client=client, - table_name=table_name, - run_date=run_date, - mids=mids, - emp_visit_df=emp_visit_df, - ) - - # ------------------------------------------ - # Load Data - # ------------------------------------------ - - load_to_clickhouse( - client=client, - table_name=table_name, - df=df, - ) - - log.info( - "%s loaded successfully (%s rows)", - table_name, - len(df), - ) - - except Exception: - - log.exception( - "Failed processing table %s", - table_name, - ) - - raise - - log.info("=" * 80) - log.info("Pipeline Completed Successfully") - log.info("=" * 80) - - -if __name__ == "__main__": - main() \ No newline at end of file