From 8aaae1e27dcec47b3e9da926f65edab4b4d60677 Mon Sep 17 00:00:00 2001 From: Ankit Malik Date: Fri, 12 Jun 2026 15:39:39 +0530 Subject: [PATCH] 3rd commit --- clickhouse_task/load_table.py | 28 ++ db_con/connection.py | 3 +- kpi/facts.py | 183 ++++---- logs/etl_20260612.log | 839 ++++++++++++++++++++++++++++++++++ main.py | 65 ++- masters/bridge.py | 10 +- masters/dimensions.py | 77 +++- mids.py | 6 +- pyproject.toml | 4 + tables.yaml => tables.yml | 0 test.py | 47 ++ uv.lock | 138 ++++++ 12 files changed, 1280 insertions(+), 120 deletions(-) create mode 100644 logs/etl_20260612.log rename tables.yaml => tables.yml (100%) create mode 100644 test.py diff --git a/clickhouse_task/load_table.py b/clickhouse_task/load_table.py index e69de29..e678a8a 100644 --- a/clickhouse_task/load_table.py +++ b/clickhouse_task/load_table.py @@ -0,0 +1,28 @@ +import polars as pl +from clickhouse_connect.driver import Client +from log import log + + +def load_to_clickhouse( + client: Client, + table_name: str, + df: pl.DataFrame, +) -> None: + """ + Load a Polars DataFrame into ClickHouse using Arrow. + """ + + if df.is_empty(): + log.warning(f"{table_name}: DataFrame is empty. Skipping.") + return + + arrow_table = df.to_arrow() + + client.insert_arrow( + table=table_name, + arrow_table=arrow_table, + ) + + log.info( + f"{table_name}: inserted {len(df):,} rows into ClickHouse" + ) \ No newline at end of file diff --git a/db_con/connection.py b/db_con/connection.py index 02fb0f0..7da25d4 100644 --- a/db_con/connection.py +++ b/db_con/connection.py @@ -48,6 +48,7 @@ def build_sql_server_engine() -> Engine: ) with engine.connect() as conn: + log.info(conn.execute(text("SELECT 1"))) return engine @@ -81,6 +82,6 @@ def get_clickhouse_client(): host=CH_HOST, port=CH_PORT, username=CH_USER, - password="CH_PASS", + password=CH_PASS, database=CH_DB ) \ No newline at end of file diff --git a/kpi/facts.py b/kpi/facts.py index c92deeb..45bfc9b 100644 --- a/kpi/facts.py +++ b/kpi/facts.py @@ -1,5 +1,5 @@ import os -import pyarrow +# import pyarrow import sys import logging from datetime import date, timedelta @@ -10,7 +10,7 @@ import clickhouse_connect from dotenv import load_dotenv from log import log -from clickhouse_task.create_table import create_clickhouse_table , check +from clickhouse_task.create_table import * from db_con.connection import * from mids import * @@ -127,6 +127,7 @@ def fetch_SOS_OneApp(engine: Engine, mids: list[int]) -> pl.DataFrame: ) A """ + log.info(f"Fetching data for {len(mids):,} MIDs") df = pl.read_database( @@ -134,6 +135,7 @@ def fetch_SOS_OneApp(engine: Engine, mids: list[int]) -> pl.DataFrame: connection=engine ) + log.info(f"Fetched {len(df):,} rows from SQL Server") return df @@ -229,6 +231,7 @@ def fetch_OQaD(engine: Engine, mids: list[int]) -> pl.DataFrame: connection=engine ) + log.info(f"Fetched {len(df):,} rows") return df @@ -316,6 +319,7 @@ def fetch_Survey(engine: Engine, mids: list[int]) -> pl.DataFrame: ON Q.StoreId = SM.StoreId """ + log.info(f"Fetching Survey data for {len(mids):,} MIDs") df = pl.read_database( @@ -323,6 +327,7 @@ def fetch_Survey(engine: Engine, mids: list[int]) -> pl.DataFrame: connection=engine ) + log.info(f"Fetched {len(df):,} Survey rows") return df @@ -396,7 +401,7 @@ def fetch_additional_visibility( AND TS.Present = 1 AND SC.MID IN ({mid_list}) """ - + log.info( f"Fetching Additional Visibility data for {len(mids):,} MIDs" ) @@ -406,6 +411,7 @@ def fetch_additional_visibility( connection=engine ) + log.info( f"Fetched {len(df):,} Additional Visibility rows" ) @@ -485,6 +491,7 @@ def fetch_Coverage(engine: Engine, mids: list[int]) -> pl.DataFrame: AND Em.UserName NOT LIKE 'test%' """ + log.info(f"Fetching coverage data for {len(mids):,} MIDs") df = pl.read_database( @@ -492,14 +499,91 @@ def fetch_Coverage(engine: Engine, mids: list[int]) -> pl.DataFrame: connection=engine ) + log.info(f"Fetched {len(df):,} rows from SQL Server") return df +def fetch_Login(engine: Engine , mids: list[int]) -> pl.DataFrame: -def fetch_stock_details( + sql = """ + WITH login_data AS + ( + SELECT + UD.EmpId, + CAST(UD.LoginDate AS DATE) AS LoginDate, + CONVERT(VARCHAR(8), UD.InTime, 108) AS LoginTime, + + ROW_NUMBER() OVER + ( + PARTITION BY + UD.EmpId, + CAST(UD.LoginDate AS DATE) + ORDER BY UD.LoginDate + ) AS rn + + FROM OneApp_KelloggsMT.dbo.T_DeviceLogin UD + INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM + ON UD.EmpId = EM.EmpId + + WHERE CAST(UD.LoginDate AS DATE) = + CAST(DATEADD(DAY,-1,GETDATE()) AS DATE) + + AND EM.RightId = 6 + AND EM.EmpName NOT LIKE '%test%' + AND ( + EM.ResignDate IS NULL + OR CAST(EM.ResignDate AS DATE) >= + CAST(DATEADD(DAY,-1,GETDATE()) AS DATE) + ) + ) + + SELECT + LD.EmpId AS employee_id, + LD.LoginDate AS login_date, + LD.LoginTime AS login_time, + + ( + SELECT MIN(CONVERT(VARCHAR(8), SC.InTime, 108)) + FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC + WHERE SC.IsDel = 0 + AND SC.EmpId = LD.EmpId + AND CAST(SC.VisitDate AS DATE) = LD.LoginDate + ) AS first_store_in_time, + + ( + SELECT MAX(CONVERT(VARCHAR(8), SC.OutTime, 108)) + FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC + WHERE SC.IsDel = 0 + AND SC.EmpId = LD.EmpId + AND CAST(SC.VisitDate AS DATE) = LD.LoginDate + ) AS last_store_out_time, + + CONCAT('40148','_',CAST(LD.EmpId AS VARCHAR(50))) AS unique_id + + FROM login_data LD + WHERE LD.rn = 1 + """ + + + log.info("Fetching Login data for yesterday") + + df = pl.read_database( + query=sql, + connection=engine + ) + + + log.info(f"Fetched {len(df):,} Login rows") + + return df + + + + +def fetch_Stock_Details( engine: Engine, mids: list[int] ) -> pl.DataFrame: @@ -583,7 +667,7 @@ def fetch_stock_details( WHERE EM.EmpName NOT LIKE 'test%' AND SC.MID IN ({mid_list}) """ - + log.info( f"Fetching Stock Details data for {len(mids):,} MIDs" ) @@ -678,88 +762,10 @@ def fetch_Attendance( return df -def fetch_login(engine: Engine , mids: list[int]) -> pl.DataFrame: - - sql = """ - WITH login_data AS - ( - SELECT - UD.EmpId, - CAST(UD.LoginDate AS DATE) AS LoginDate, - CONVERT(VARCHAR(8), UD.InTime, 108) AS LoginTime, - - ROW_NUMBER() OVER - ( - PARTITION BY - UD.EmpId, - CAST(UD.LoginDate AS DATE) - ORDER BY UD.LoginDate - ) AS rn - - FROM OneApp_KelloggsMT.dbo.T_DeviceLogin UD - INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM - ON UD.EmpId = EM.EmpId - - WHERE CAST(UD.LoginDate AS DATE) = - CAST(DATEADD(DAY,-1,GETDATE()) AS DATE) - - AND EM.RightId = 6 - AND EM.EmpName NOT LIKE '%test%' - AND ( - EM.ResignDate IS NULL - OR CAST(EM.ResignDate AS DATE) >= - CAST(DATEADD(DAY,-1,GETDATE()) AS DATE) - ) - ) - - SELECT - LD.EmpId AS employee_id, - LD.LoginDate AS login_date, - LD.LoginTime AS login_time, - - ( - SELECT MIN(CONVERT(VARCHAR(8), SC.InTime, 108)) - FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC - WHERE SC.IsDel = 0 - AND SC.EmpId = LD.EmpId - AND CAST(SC.VisitDate AS DATE) = LD.LoginDate - ) AS first_store_in_time, - - ( - SELECT MAX(CONVERT(VARCHAR(8), SC.OutTime, 108)) - FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC - WHERE SC.IsDel = 0 - AND SC.EmpId = LD.EmpId - AND CAST(SC.VisitDate AS DATE) = LD.LoginDate - ) AS last_store_out_time, - - CONCAT('40148','_',CAST(LD.EmpId AS VARCHAR(50))) AS unique_id - - FROM login_data LD - WHERE LD.rn = 1 - """ - - - log.info("Fetching Login data for yesterday") - - df = pl.read_database( - query=sql, - connection=engine - ) - - log.info(f"Fetched {len(df):,} Login rows") - - return df - - - - - - -def fetch_journey_plan( +def fetch_Journey_Plan( engine: Engine, report_date: date ) -> pl.DataFrame: @@ -830,7 +836,7 @@ def fetch_coverage_remarks(engine: Engine , mids: list[int]) -> pl.DataFrame: return df -def fetch_web_logins( +def fetch_Web_Logins( engine: Engine, run_date: date ) -> pl.DataFrame: @@ -893,7 +899,7 @@ def fetch_web_logins( WHERE CAST(AL.Date AS DATE) = '{run_date}' """ - + log.info(f"Fetching Web Login data for {run_date}") df = pl.read_database( @@ -901,6 +907,7 @@ def fetch_web_logins( connection=engine ) + log.info( f"Fetched {len(df):,} Web Login records" ) @@ -910,7 +917,7 @@ def fetch_web_logins( -def fetch_promotion( +def fetch_Promotion( engine: Engine, mids: list[int] ) -> pl.DataFrame: @@ -1061,7 +1068,7 @@ def fetch_promotion( -def fetch_paid_visibility( +def fetch_PaidVisibility( engine: Engine, mids: list[int] ) -> pl.DataFrame: diff --git a/logs/etl_20260612.log b/logs/etl_20260612.log new file mode 100644 index 0000000..e1e7e7c --- /dev/null +++ b/logs/etl_20260612.log @@ -0,0 +1,839 @@ +2026-06-12 13:10:05 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 13:10:05 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 13:10:44 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 13:10:44 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 13:10:45 | INFO | +2026-06-12 13:10:50 | INFO | +2026-06-12 13:12:36 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 13:12:36 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 13:12:37 | INFO | +2026-06-12 13:12:39 | INFO | +2026-06-12 13:14:24 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 13:14:24 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 13:14:26 | INFO | +2026-06-12 13:14:27 | INFO | +2026-06-12 13:14:28 | INFO | Both databases connected successfully +2026-06-12 13:14:28 | INFO | Collecting MIDs for: 2026-06-11 +2026-06-12 13:14:28 | INFO | Found 818 MIDs +2026-06-12 13:15:39 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 13:15:39 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 13:15:40 | INFO | +2026-06-12 13:15:41 | INFO | +2026-06-12 13:15:42 | INFO | Both databases connected successfully +2026-06-12 13:15:42 | INFO | Collecting MIDs for: 2026-06-11 +2026-06-12 13:15:42 | INFO | Found 818 MIDs +2026-06-12 13:23:57 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 13:23:57 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 13:23:58 | INFO | +2026-06-12 13:24:01 | INFO | +2026-06-12 13:24:01 | INFO | Both databases connected successfully +2026-06-12 13:24:01 | INFO | Collecting MIDs for: 2026-06-11 +2026-06-12 13:24:02 | INFO | Found 818 MIDs +2026-06-12 13:24:02 | INFO | Fetching data for 818 MIDs +2026-06-12 13:24:06 | INFO | Fetched 3,677 rows from SQL Server +2026-06-12 13:24:06 | INFO | Fetching OQaD data for 818 MIDs +2026-06-12 13:24:09 | INFO | Fetched 464 rows +2026-06-12 13:24:09 | INFO | Fetching Survey data for 818 MIDs +2026-06-12 13:24:10 | INFO | Fetched 142 Survey rows +2026-06-12 13:24:10 | INFO | Fetching Additional Visibility data for 818 MIDs +2026-06-12 13:24:11 | INFO | Fetched 1,922 Additional Visibility rows +2026-06-12 13:24:11 | INFO | Fetching coverage data for 818 MIDs +2026-06-12 13:24:11 | INFO | Fetched 761 rows from SQL Server +2026-06-12 13:24:11 | INFO | Fetching Login data for yesterday +2026-06-12 13:24:12 | INFO | Fetched 479 Login rows +2026-06-12 13:24:12 | INFO | Fetching Stock Details data for 818 MIDs +2026-06-12 13:24:17 | INFO | Fetched 41,628 Stock Details rows +2026-06-12 13:24:17 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 +2026-06-12 13:24:18 | INFO | Fetched 11,918 attendance rows for 592 employees +2026-06-12 13:24:18 | INFO | Fetching Store Master data +2026-06-12 13:24:20 | INFO | Fetched 5,984 stores +2026-06-12 13:24:20 | INFO | Fetching SKU Master data +2026-06-12 13:24:20 | INFO | Fetched 160 SKU Master rows +2026-06-12 13:24:20 | INFO | Fetching Display Master data +2026-06-12 13:24:20 | INFO | Fetched 135 Display Master records +2026-06-12 13:24:20 | INFO | Fetching Employee Master data +2026-06-12 13:24:21 | INFO | Fetched 2,269 Employee Master records +2026-06-12 13:24:21 | INFO | Fetching Journey Plan for 2026-06 +2026-06-12 13:24:21 | INFO | Fetched 20,969 Journey Plan records +2026-06-12 13:24:21 | INFO | Fetching Coverage Remarks +2026-06-12 13:24:21 | INFO | Fetched 29 Coverage Remark records +2026-06-12 13:24:21 | INFO | Fetching Mapping Store Visibility for 2026-06-11 +2026-06-12 13:24:22 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-12 13:24:22 | INFO | Fetching Master Visibility Reason data +2026-06-12 13:24:22 | INFO | Fetched 17 Master Visibility Reason records +2026-06-12 13:24:22 | INFO | Fetching Master Visibility Definition data +2026-06-12 13:24:22 | INFO | Fetched 861 Master Visibility Definition records +2026-06-12 13:24:22 | INFO | Fetching Web Login data for 2026-06-11 +2026-06-12 13:24:22 | INFO | Fetched 223 Web Login records +2026-06-12 13:24:22 | INFO | Fetching Promotion data for 818 MIDs +2026-06-12 13:24:25 | INFO | Fetched 8,410 Promotion records +2026-06-12 13:26:34 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 13:26:34 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 13:26:34 | INFO | +2026-06-12 13:26:36 | INFO | +2026-06-12 13:26:36 | INFO | Both databases connected successfully +2026-06-12 13:26:36 | INFO | Collecting MIDs for: 2026-06-11 +2026-06-12 13:26:37 | INFO | Found 818 MIDs +2026-06-12 13:26:37 | INFO | Fetching data for 818 MIDs +2026-06-12 13:26:37 | INFO | Fetched 3,677 rows from SQL Server +2026-06-12 13:26:37 | INFO | Fetching OQaD data for 818 MIDs +2026-06-12 13:26:40 | INFO | Fetched 464 rows +2026-06-12 13:26:40 | INFO | Fetching Survey data for 818 MIDs +2026-06-12 13:26:40 | INFO | Fetched 142 Survey rows +2026-06-12 13:26:40 | INFO | Fetching Additional Visibility data for 818 MIDs +2026-06-12 13:26:41 | INFO | Fetched 1,922 Additional Visibility rows +2026-06-12 13:26:41 | INFO | Fetching coverage data for 818 MIDs +2026-06-12 13:26:41 | INFO | Fetched 761 rows from SQL Server +2026-06-12 13:26:41 | INFO | Fetching Login data for yesterday +2026-06-12 13:26:41 | INFO | Fetched 479 Login rows +2026-06-12 13:26:41 | INFO | Fetching Stock Details data for 818 MIDs +2026-06-12 13:26:45 | INFO | Fetched 41,628 Stock Details rows +2026-06-12 13:26:45 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 +2026-06-12 13:26:46 | INFO | Fetched 11,918 attendance rows for 592 employees +2026-06-12 13:26:46 | INFO | Fetching Store Master data +2026-06-12 13:26:48 | INFO | Fetched 5,984 stores +2026-06-12 13:26:48 | INFO | Fetching SKU Master data +2026-06-12 13:26:48 | INFO | Fetched 160 SKU Master rows +2026-06-12 13:26:48 | INFO | Fetching Display Master data +2026-06-12 13:26:48 | INFO | Fetched 135 Display Master records +2026-06-12 13:26:48 | INFO | Fetching Employee Master data +2026-06-12 13:26:49 | INFO | Fetched 2,269 Employee Master records +2026-06-12 13:26:49 | INFO | Fetching Journey Plan for 2026-06 +2026-06-12 13:26:50 | INFO | Fetched 20,969 Journey Plan records +2026-06-12 13:26:50 | INFO | Fetching Coverage Remarks +2026-06-12 13:26:50 | INFO | Fetched 29 Coverage Remark records +2026-06-12 13:26:50 | INFO | Fetching Mapping Store Visibility for 2026-06-11 +2026-06-12 13:26:50 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-12 13:26:50 | INFO | Fetching Master Visibility Reason data +2026-06-12 13:26:50 | INFO | Fetched 17 Master Visibility Reason records +2026-06-12 13:26:50 | INFO | Fetching Master Visibility Definition data +2026-06-12 13:26:50 | INFO | Fetched 861 Master Visibility Definition records +2026-06-12 13:26:50 | INFO | Fetching Web Login data for 2026-06-11 +2026-06-12 13:26:50 | INFO | Fetched 223 Web Login records +2026-06-12 13:26:50 | INFO | Fetching Promotion data for 818 MIDs +2026-06-12 13:26:53 | INFO | Fetched 8,410 Promotion records +2026-06-12 13:29:18 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 13:29:18 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 13:29:19 | INFO | +2026-06-12 13:29:21 | INFO | +2026-06-12 13:29:21 | INFO | Both databases connected successfully +2026-06-12 13:29:21 | INFO | Collecting MIDs for: 2026-06-11 +2026-06-12 13:29:21 | INFO | Found 818 MIDs +2026-06-12 13:29:21 | INFO | Fetching data for 818 MIDs +2026-06-12 13:29:22 | INFO | Fetched 3,677 rows from SQL Server +2026-06-12 13:29:22 | INFO | Fetching OQaD data for 818 MIDs +2026-06-12 13:29:24 | INFO | Fetched 464 rows +2026-06-12 13:29:24 | INFO | Fetching Survey data for 818 MIDs +2026-06-12 13:29:24 | INFO | Fetched 142 Survey rows +2026-06-12 13:29:24 | INFO | Fetching Additional Visibility data for 818 MIDs +2026-06-12 13:29:27 | INFO | Fetched 1,922 Additional Visibility rows +2026-06-12 13:29:27 | INFO | Fetching coverage data for 818 MIDs +2026-06-12 13:29:28 | INFO | Fetched 761 rows from SQL Server +2026-06-12 13:29:28 | INFO | Fetching Login data for yesterday +2026-06-12 13:29:28 | INFO | Fetched 479 Login rows +2026-06-12 13:29:28 | INFO | Fetching Stock Details data for 818 MIDs +2026-06-12 13:29:38 | INFO | Fetched 41,628 Stock Details rows +2026-06-12 13:29:38 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 +2026-06-12 13:29:41 | INFO | Fetched 11,918 attendance rows for 592 employees +2026-06-12 13:29:41 | INFO | Fetching Store Master data +2026-06-12 13:29:46 | INFO | Fetched 5,984 stores +2026-06-12 13:29:46 | INFO | Fetching SKU Master data +2026-06-12 13:29:46 | INFO | Fetched 160 SKU Master rows +2026-06-12 13:29:46 | INFO | Fetching Display Master data +2026-06-12 13:29:46 | INFO | Fetched 135 Display Master records +2026-06-12 13:29:46 | INFO | Fetching Employee Master data +2026-06-12 13:29:47 | INFO | Fetched 2,269 Employee Master records +2026-06-12 13:29:47 | INFO | Fetching Journey Plan for 2026-06 +2026-06-12 13:29:48 | INFO | Fetched 20,969 Journey Plan records +2026-06-12 13:29:48 | INFO | Fetching Coverage Remarks +2026-06-12 13:29:48 | INFO | Fetched 29 Coverage Remark records +2026-06-12 13:29:48 | INFO | Fetching Mapping Store Visibility for 2026-06-11 +2026-06-12 13:29:48 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-12 13:29:48 | INFO | Fetching Master Visibility Reason data +2026-06-12 13:29:48 | INFO | Fetched 17 Master Visibility Reason records +2026-06-12 13:29:48 | INFO | Fetching Master Visibility Definition data +2026-06-12 13:29:48 | INFO | Fetched 861 Master Visibility Definition records +2026-06-12 13:29:48 | INFO | Fetching Web Login data for 2026-06-11 +2026-06-12 13:29:49 | INFO | Fetched 223 Web Login records +2026-06-12 13:29:49 | INFO | Fetching Promotion data for 818 MIDs +2026-06-12 13:29:55 | INFO | Fetched 8,410 Promotion records +2026-06-12 13:29:55 | INFO | Fetching Paid Visibility data for 818 MIDs +2026-06-12 13:30:05 | INFO | Fetched 1,098 Paid Visibility records +2026-06-12 13:30:05 | INFO | Fetching Master Sales Territory Layer data +2026-06-12 13:30:05 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-12 13:31:55 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 13:31:55 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 13:31:56 | INFO | +2026-06-12 13:31:57 | INFO | +2026-06-12 13:31:58 | INFO | Both databases connected successfully +2026-06-12 13:31:58 | INFO | Collecting MIDs for: 2026-06-11 +2026-06-12 13:31:58 | INFO | Found 818 MIDs +2026-06-12 13:31:58 | INFO | Fetching data for 818 MIDs +2026-06-12 13:32:00 | INFO | Fetched 3,677 rows from SQL Server +2026-06-12 13:32:00 | INFO | Fetching OQaD data for 818 MIDs +2026-06-12 13:32:02 | INFO | Fetched 464 rows +2026-06-12 13:32:02 | INFO | Fetching Survey data for 818 MIDs +2026-06-12 13:32:02 | INFO | Fetched 142 Survey rows +2026-06-12 13:32:02 | INFO | Fetching Additional Visibility data for 818 MIDs +2026-06-12 13:32:04 | INFO | Fetched 1,922 Additional Visibility rows +2026-06-12 13:32:04 | INFO | Fetching coverage data for 818 MIDs +2026-06-12 13:32:04 | INFO | Fetched 761 rows from SQL Server +2026-06-12 13:32:04 | INFO | Fetching Login data for yesterday +2026-06-12 13:32:04 | INFO | Fetched 479 Login rows +2026-06-12 13:32:04 | INFO | Fetching Stock Details data for 818 MIDs +2026-06-12 13:32:08 | INFO | Fetched 41,628 Stock Details rows +2026-06-12 13:32:08 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 +2026-06-12 13:32:09 | INFO | Fetched 11,918 attendance rows for 592 employees +2026-06-12 13:32:09 | INFO | Fetching Store Master data +2026-06-12 13:32:12 | INFO | Fetched 5,984 stores +2026-06-12 13:32:12 | INFO | Fetching SKU Master data +2026-06-12 13:32:12 | INFO | Fetched 160 SKU Master rows +2026-06-12 13:32:12 | INFO | Fetching Display Master data +2026-06-12 13:32:12 | INFO | Fetched 135 Display Master records +2026-06-12 13:32:12 | INFO | Fetching Employee Master data +2026-06-12 13:32:13 | INFO | Fetched 2,269 Employee Master records +2026-06-12 13:32:13 | INFO | Fetching Journey Plan for 2026-06 +2026-06-12 13:32:13 | INFO | Fetched 20,969 Journey Plan records +2026-06-12 13:32:13 | INFO | Fetching Coverage Remarks +2026-06-12 13:32:13 | INFO | Fetched 29 Coverage Remark records +2026-06-12 13:32:13 | INFO | Fetching Mapping Store Visibility for 2026-06-11 +2026-06-12 13:32:14 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-12 13:32:14 | INFO | Fetching Master Visibility Reason data +2026-06-12 13:32:14 | INFO | Fetched 17 Master Visibility Reason records +2026-06-12 13:32:14 | INFO | Fetching Master Visibility Definition data +2026-06-12 13:32:14 | INFO | Fetched 861 Master Visibility Definition records +2026-06-12 13:32:14 | INFO | Fetching Web Login data for 2026-06-11 +2026-06-12 13:32:14 | INFO | Fetched 223 Web Login records +2026-06-12 13:32:14 | INFO | Fetching Promotion data for 818 MIDs +2026-06-12 13:32:21 | INFO | Fetched 8,410 Promotion records +2026-06-12 13:32:21 | INFO | Fetching Paid Visibility data for 818 MIDs +2026-06-12 13:32:28 | INFO | Fetched 1,098 Paid Visibility records +2026-06-12 13:32:28 | INFO | Fetching Master Sales Territory Layer data +2026-06-12 13:32:29 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-12 13:33:34 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 13:33:34 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 13:33:34 | INFO | +2026-06-12 13:33:36 | INFO | +2026-06-12 13:33:36 | INFO | Both databases connected successfully +2026-06-12 13:33:36 | INFO | Collecting MIDs for: 2026-06-11 +2026-06-12 13:33:36 | INFO | Found 818 MIDs +2026-06-12 13:33:37 | INFO | Fetching data for 818 MIDs +2026-06-12 13:33:37 | INFO | Fetched 3,677 rows from SQL Server +2026-06-12 13:33:37 | INFO | Fetching OQaD data for 818 MIDs +2026-06-12 13:33:39 | INFO | Fetched 464 rows +2026-06-12 13:33:39 | INFO | Fetching Survey data for 818 MIDs +2026-06-12 13:33:40 | INFO | Fetched 142 Survey rows +2026-06-12 13:33:40 | INFO | Fetching Additional Visibility data for 818 MIDs +2026-06-12 13:35:57 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 13:35:57 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 13:35:57 | INFO | +2026-06-12 13:35:59 | INFO | +2026-06-12 13:35:59 | INFO | Both databases connected successfully +2026-06-12 13:35:59 | INFO | Collecting MIDs for: 2026-06-11 +2026-06-12 13:36:00 | INFO | Found 818 MIDs +2026-06-12 13:36:00 | INFO | ================================================================================ +2026-06-12 13:36:00 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:36:00 | INFO | Fetching data for 818 MIDs +2026-06-12 13:36:00 | INFO | Fetched 3,677 rows from SQL Server +2026-06-12 13:36:00 | INFO | ================================================================================ +2026-06-12 13:36:00 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:36:00 | INFO | Fetching OQaD data for 818 MIDs +2026-06-12 13:36:03 | INFO | Fetched 464 rows +2026-06-12 13:36:03 | INFO | ================================================================================ +2026-06-12 13:36:03 | INFO | TABLE=Survey | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:36:03 | INFO | Fetching Survey data for 818 MIDs +2026-06-12 13:36:03 | INFO | Fetched 142 Survey rows +2026-06-12 13:36:03 | INFO | ================================================================================ +2026-06-12 13:36:03 | INFO | TABLE=additional_visibility | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:36:03 | INFO | Fetching Additional Visibility data for 818 MIDs +2026-06-12 13:36:04 | INFO | Fetched 1,922 Additional Visibility rows +2026-06-12 13:36:04 | INFO | ================================================================================ +2026-06-12 13:36:04 | INFO | TABLE=Coverage | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:36:04 | INFO | Fetching coverage data for 818 MIDs +2026-06-12 13:36:04 | INFO | Fetched 761 rows from SQL Server +2026-06-12 13:36:04 | INFO | ================================================================================ +2026-06-12 13:36:04 | INFO | TABLE=Login | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:36:04 | INFO | Fetching Login data for yesterday +2026-06-12 13:36:04 | INFO | Fetched 479 Login rows +2026-06-12 13:36:04 | INFO | ================================================================================ +2026-06-12 13:36:04 | INFO | TABLE=Stock_Details | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:36:04 | INFO | Fetching Stock Details data for 818 MIDs +2026-06-12 13:36:08 | INFO | Fetched 41,628 Stock Details rows +2026-06-12 13:36:08 | INFO | ================================================================================ +2026-06-12 13:36:08 | INFO | TABLE=Attendance | TYPE=FACT | OPERATION=DELETE+INSERT +2026-06-12 13:36:08 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 +2026-06-12 13:36:13 | INFO | Fetched 11,918 attendance rows for 592 employees +2026-06-12 13:36:13 | INFO | ================================================================================ +2026-06-12 13:36:13 | INFO | TABLE=Store_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 13:36:13 | INFO | Fetching Store Master data +2026-06-12 13:36:17 | INFO | Fetched 5,984 stores +2026-06-12 13:36:17 | INFO | ================================================================================ +2026-06-12 13:36:17 | INFO | TABLE=SKU_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 13:36:17 | INFO | Fetching SKU Master data +2026-06-12 13:36:17 | INFO | Fetched 160 SKU Master rows +2026-06-12 13:36:17 | INFO | ================================================================================ +2026-06-12 13:36:17 | INFO | TABLE=display_master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 13:36:17 | INFO | Fetching Display Master data +2026-06-12 13:36:17 | INFO | Fetched 135 Display Master records +2026-06-12 13:36:17 | INFO | ================================================================================ +2026-06-12 13:36:17 | INFO | TABLE=Employee_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 13:36:17 | INFO | Fetching Employee Master data +2026-06-12 13:36:18 | INFO | Fetched 2,269 Employee Master records +2026-06-12 13:36:18 | INFO | ================================================================================ +2026-06-12 13:36:18 | INFO | TABLE=Journey_Plan | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:36:18 | INFO | Fetching Journey Plan for 2026-06 +2026-06-12 13:36:18 | INFO | Fetched 20,970 Journey Plan records +2026-06-12 13:36:18 | INFO | ================================================================================ +2026-06-12 13:36:18 | INFO | TABLE=coverage_remarks | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:36:18 | INFO | Fetching Coverage Remarks +2026-06-12 13:36:18 | INFO | Fetched 29 Coverage Remark records +2026-06-12 13:36:18 | INFO | ================================================================================ +2026-06-12 13:36:18 | INFO | TABLE=mapping_storevisibility | TYPE=BRIDGE | OPERATION=INSERT +2026-06-12 13:36:18 | INFO | Fetching Mapping Store Visibility for 2026-06-11 +2026-06-12 13:36:18 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-12 13:36:18 | INFO | ================================================================================ +2026-06-12 13:36:18 | INFO | TABLE=Master_VisibilityReason | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 13:36:18 | INFO | Fetching Master Visibility Reason data +2026-06-12 13:36:19 | INFO | Fetched 17 Master Visibility Reason records +2026-06-12 13:36:19 | INFO | ================================================================================ +2026-06-12 13:36:19 | INFO | TABLE=Master_VisibilityDefinition | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 13:36:19 | INFO | Fetching Master Visibility Definition data +2026-06-12 13:36:19 | INFO | Fetched 861 Master Visibility Definition records +2026-06-12 13:36:19 | INFO | ================================================================================ +2026-06-12 13:36:19 | INFO | TABLE=Web_Logins | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:36:19 | INFO | Fetching Web Login data for 2026-06-11 +2026-06-12 13:36:19 | INFO | Fetched 223 Web Login records +2026-06-12 13:36:19 | INFO | ================================================================================ +2026-06-12 13:36:19 | INFO | TABLE=Promotion | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:36:19 | INFO | Fetching Promotion data for 818 MIDs +2026-06-12 13:36:22 | INFO | Fetched 8,410 Promotion records +2026-06-12 13:36:22 | INFO | ================================================================================ +2026-06-12 13:36:22 | INFO | TABLE=PaidVisibility | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:36:22 | INFO | Fetching Paid Visibility data for 818 MIDs +2026-06-12 13:36:30 | INFO | Fetched 1,098 Paid Visibility records +2026-06-12 13:36:30 | INFO | ================================================================================ +2026-06-12 13:36:30 | INFO | TABLE=Master_Salesterritorylayer | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 13:36:30 | INFO | Fetching Master Sales Territory Layer data +2026-06-12 13:36:31 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-12 13:40:32 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 13:40:32 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 13:40:32 | INFO | +2026-06-12 13:40:34 | INFO | +2026-06-12 13:40:35 | INFO | Both databases connected successfully +2026-06-12 13:40:35 | INFO | Collecting MIDs for: 2026-06-11 +2026-06-12 13:40:35 | INFO | Found 818 MIDs +2026-06-12 13:40:35 | INFO | ================================================================================ +2026-06-12 13:40:35 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:40:35 | INFO | Fetching data for 818 MIDs +2026-06-12 13:40:36 | INFO | Fetched 3,677 rows from SQL Server +2026-06-12 13:40:36 | INFO | ================================================================================ +2026-06-12 13:40:36 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:40:36 | INFO | Fetching OQaD data for 818 MIDs +2026-06-12 13:40:38 | INFO | Fetched 464 rows +2026-06-12 13:40:38 | INFO | ================================================================================ +2026-06-12 13:40:38 | INFO | TABLE=Survey | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:40:38 | INFO | Fetching Survey data for 818 MIDs +2026-06-12 13:40:38 | INFO | Fetched 142 Survey rows +2026-06-12 13:40:38 | INFO | ================================================================================ +2026-06-12 13:40:38 | INFO | TABLE=additional_visibility | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:40:38 | INFO | Fetching Additional Visibility data for 818 MIDs +2026-06-12 13:40:39 | INFO | Fetched 1,922 Additional Visibility rows +2026-06-12 13:40:39 | INFO | ================================================================================ +2026-06-12 13:40:39 | INFO | TABLE=Coverage | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:40:39 | INFO | Fetching coverage data for 818 MIDs +2026-06-12 13:40:39 | INFO | Fetched 761 rows from SQL Server +2026-06-12 13:40:39 | INFO | ================================================================================ +2026-06-12 13:40:39 | INFO | TABLE=Login | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:40:39 | INFO | Fetching Login data for yesterday +2026-06-12 13:40:39 | INFO | Fetched 479 Login rows +2026-06-12 13:40:39 | INFO | ================================================================================ +2026-06-12 13:40:39 | INFO | TABLE=Stock_Details | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:40:39 | INFO | Fetching Stock Details data for 818 MIDs +2026-06-12 13:40:41 | INFO | Fetched 41,628 Stock Details rows +2026-06-12 13:40:41 | INFO | ================================================================================ +2026-06-12 13:40:41 | INFO | TABLE=Attendance | TYPE=FACT | OPERATION=DELETE+INSERT +2026-06-12 13:40:41 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 +2026-06-12 13:40:42 | INFO | Fetched 11,918 attendance rows for 592 employees +2026-06-12 13:40:42 | INFO | ================================================================================ +2026-06-12 13:40:42 | INFO | TABLE=Store_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 13:40:42 | INFO | Fetching Store Master data +2026-06-12 13:40:44 | INFO | Fetched 5,984 stores +2026-06-12 13:40:44 | INFO | ================================================================================ +2026-06-12 13:40:44 | INFO | TABLE=SKU_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 13:40:44 | INFO | Fetching SKU Master data +2026-06-12 13:40:44 | INFO | Fetched 160 SKU Master rows +2026-06-12 13:40:44 | INFO | ================================================================================ +2026-06-12 13:40:44 | INFO | TABLE=display_master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 13:40:44 | INFO | Fetching Display Master data +2026-06-12 13:40:44 | INFO | Fetched 135 Display Master records +2026-06-12 13:40:44 | INFO | ================================================================================ +2026-06-12 13:40:44 | INFO | TABLE=Employee_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 13:40:44 | INFO | Fetching Employee Master data +2026-06-12 13:40:44 | INFO | Fetched 2,269 Employee Master records +2026-06-12 13:40:44 | INFO | ================================================================================ +2026-06-12 13:40:44 | INFO | TABLE=Journey_Plan | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:40:44 | INFO | Fetching Journey Plan for 2026-06 +2026-06-12 13:40:45 | INFO | Fetched 20,972 Journey Plan records +2026-06-12 13:40:45 | INFO | ================================================================================ +2026-06-12 13:40:45 | INFO | TABLE=coverage_remarks | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:40:45 | INFO | Fetching Coverage Remarks +2026-06-12 13:40:45 | INFO | Fetched 29 Coverage Remark records +2026-06-12 13:40:45 | INFO | ================================================================================ +2026-06-12 13:40:45 | INFO | TABLE=mapping_storevisibility | TYPE=BRIDGE | OPERATION=INSERT +2026-06-12 13:40:45 | INFO | Fetching Mapping Store Visibility for 2026-06-11 +2026-06-12 13:40:45 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-12 13:40:45 | INFO | ================================================================================ +2026-06-12 13:40:45 | INFO | TABLE=Master_VisibilityReason | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 13:40:45 | INFO | Fetching Master Visibility Reason data +2026-06-12 13:40:45 | INFO | Fetched 17 Master Visibility Reason records +2026-06-12 13:40:45 | INFO | ================================================================================ +2026-06-12 13:40:45 | INFO | TABLE=Master_VisibilityDefinition | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 13:40:45 | INFO | Fetching Master Visibility Definition data +2026-06-12 13:40:45 | INFO | Fetched 861 Master Visibility Definition records +2026-06-12 13:40:45 | INFO | ================================================================================ +2026-06-12 13:40:45 | INFO | TABLE=Web_Logins | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:40:45 | INFO | Fetching Web Login data for 2026-06-11 +2026-06-12 13:40:45 | INFO | Fetched 223 Web Login records +2026-06-12 13:40:45 | INFO | ================================================================================ +2026-06-12 13:40:45 | INFO | TABLE=Promotion | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:40:45 | INFO | Fetching Promotion data for 818 MIDs +2026-06-12 13:40:48 | INFO | Fetched 8,410 Promotion records +2026-06-12 13:40:48 | INFO | ================================================================================ +2026-06-12 13:40:48 | INFO | TABLE=PaidVisibility | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:40:48 | INFO | Fetching Paid Visibility data for 818 MIDs +2026-06-12 13:40:55 | INFO | Fetched 1,098 Paid Visibility records +2026-06-12 13:40:55 | INFO | ================================================================================ +2026-06-12 13:40:55 | INFO | TABLE=Master_Salesterritorylayer | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 13:40:55 | INFO | Fetching Master Sales Territory Layer data +2026-06-12 13:40:55 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-12 13:45:23 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 13:45:23 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 13:45:24 | INFO | +2026-06-12 13:45:26 | INFO | +2026-06-12 13:45:26 | INFO | Both databases connected successfully +2026-06-12 13:45:26 | INFO | Collecting MIDs for: 2026-06-11 +2026-06-12 13:45:26 | INFO | Found 818 MIDs +2026-06-12 13:45:26 | INFO | ================================================================================ +2026-06-12 13:45:26 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:45:26 | INFO | Fetching data for 818 MIDs +2026-06-12 13:45:27 | INFO | Fetched 3,677 rows from SQL Server +2026-06-12 13:45:27 | INFO | ================================================================================ +2026-06-12 13:45:27 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:45:27 | INFO | Fetching OQaD data for 818 MIDs +2026-06-12 13:45:29 | INFO | Fetched 464 rows +2026-06-12 13:45:29 | INFO | ================================================================================ +2026-06-12 13:45:29 | INFO | TABLE=Survey | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:45:29 | INFO | Fetching Survey data for 818 MIDs +2026-06-12 13:45:30 | INFO | Fetched 142 Survey rows +2026-06-12 13:45:30 | INFO | ================================================================================ +2026-06-12 13:45:30 | INFO | TABLE=additional_visibility | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:45:30 | INFO | Fetching Additional Visibility data for 818 MIDs +2026-06-12 13:45:31 | INFO | Fetched 1,922 Additional Visibility rows +2026-06-12 13:45:31 | INFO | ================================================================================ +2026-06-12 13:45:31 | INFO | TABLE=Coverage | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:45:31 | INFO | Fetching coverage data for 818 MIDs +2026-06-12 13:45:31 | INFO | Fetched 761 rows from SQL Server +2026-06-12 13:45:31 | INFO | ================================================================================ +2026-06-12 13:45:31 | INFO | TABLE=Login | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:45:31 | INFO | Fetching Login data for yesterday +2026-06-12 13:45:31 | INFO | Fetched 479 Login rows +2026-06-12 13:45:31 | INFO | ================================================================================ +2026-06-12 13:45:31 | INFO | TABLE=Stock_Details | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:45:31 | INFO | Fetching Stock Details data for 818 MIDs +2026-06-12 13:45:34 | INFO | Fetched 41,628 Stock Details rows +2026-06-12 13:45:34 | INFO | ================================================================================ +2026-06-12 13:45:34 | INFO | TABLE=Attendance | TYPE=FACT | OPERATION=DELETE+INSERT +2026-06-12 13:45:34 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 +2026-06-12 13:45:35 | INFO | Fetched 11,918 attendance rows for 592 employees +2026-06-12 13:45:35 | INFO | ================================================================================ +2026-06-12 13:45:35 | INFO | TABLE=Store_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 13:45:35 | INFO | ================================================================================ +2026-06-12 13:45:35 | INFO | Fetching Store Master data +2026-06-12 13:45:37 | INFO | ================================================================================ +2026-06-12 13:45:37 | INFO | Fetched 5,984 stores +2026-06-12 13:45:37 | INFO | ================================================================================ +2026-06-12 13:45:37 | INFO | TABLE=SKU_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 13:45:37 | INFO | ================================================================================ +2026-06-12 13:45:37 | INFO | Fetching SKU Master data +2026-06-12 13:45:37 | INFO | ================================================================================ +2026-06-12 13:45:37 | INFO | Fetched 160 SKU Master rows +2026-06-12 13:45:37 | INFO | ================================================================================ +2026-06-12 13:45:37 | INFO | TABLE=display_master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 13:45:37 | INFO | ================================================================================ +2026-06-12 13:45:37 | INFO | Fetching Display Master data +2026-06-12 13:45:37 | INFO | ================================================================================ +2026-06-12 13:45:37 | INFO | Fetched 135 Display Master records +2026-06-12 13:45:37 | INFO | ================================================================================ +2026-06-12 13:45:37 | INFO | TABLE=Employee_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 13:45:37 | INFO | ================================================================================ +2026-06-12 13:45:37 | INFO | Fetching Employee Master data +2026-06-12 13:45:38 | INFO | ================================================================================ +2026-06-12 13:45:38 | INFO | Fetched 2,269 Employee Master records +2026-06-12 13:45:38 | INFO | ================================================================================ +2026-06-12 13:45:38 | INFO | TABLE=Journey_Plan | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:45:38 | INFO | Fetching Journey Plan for 2026-06 +2026-06-12 13:45:38 | INFO | Fetched 20,972 Journey Plan records +2026-06-12 13:45:38 | INFO | ================================================================================ +2026-06-12 13:45:38 | INFO | TABLE=coverage_remarks | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:45:38 | INFO | Fetching Coverage Remarks +2026-06-12 13:45:38 | INFO | Fetched 29 Coverage Remark records +2026-06-12 13:45:38 | INFO | ================================================================================ +2026-06-12 13:45:38 | INFO | TABLE=mapping_storevisibility | TYPE=BRIDGE | OPERATION=INSERT +2026-06-12 13:45:38 | INFO | Fetching Mapping Store Visibility for 2026-06-11 +2026-06-12 13:45:38 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-12 13:45:38 | INFO | ================================================================================ +2026-06-12 13:45:38 | INFO | TABLE=Master_VisibilityReason | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 13:45:38 | INFO | ================================================================================ +2026-06-12 13:45:38 | INFO | Fetching Master Visibility Reason data +2026-06-12 13:45:38 | INFO | ================================================================================ +2026-06-12 13:45:38 | INFO | Fetched 17 Master Visibility Reason records +2026-06-12 13:45:38 | INFO | ================================================================================ +2026-06-12 13:45:38 | INFO | TABLE=Master_VisibilityDefinition | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 13:45:38 | INFO | ================================================================================ +2026-06-12 13:45:38 | INFO | Fetching Master Visibility Definition data +2026-06-12 13:45:38 | INFO | ================================================================================ +2026-06-12 13:45:38 | INFO | Fetched 861 Master Visibility Definition records +2026-06-12 13:45:38 | INFO | ================================================================================ +2026-06-12 13:45:38 | INFO | TABLE=Web_Logins | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:45:38 | INFO | Fetching Web Login data for 2026-06-11 +2026-06-12 13:45:39 | INFO | Fetched 223 Web Login records +2026-06-12 13:45:39 | INFO | ================================================================================ +2026-06-12 13:45:39 | INFO | TABLE=Promotion | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:45:39 | INFO | Fetching Promotion data for 818 MIDs +2026-06-12 13:45:42 | INFO | Fetched 8,410 Promotion records +2026-06-12 13:45:42 | INFO | ================================================================================ +2026-06-12 13:45:42 | INFO | TABLE=PaidVisibility | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:45:42 | INFO | Fetching Paid Visibility data for 818 MIDs +2026-06-12 13:45:49 | INFO | Fetched 1,098 Paid Visibility records +2026-06-12 13:45:49 | INFO | ================================================================================ +2026-06-12 13:45:50 | INFO | TABLE=Master_Salesterritorylayer | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 13:45:50 | INFO | ================================================================================ +2026-06-12 13:45:50 | INFO | Fetching Master Sales Territory Layer data +2026-06-12 13:45:50 | INFO | ================================================================================ +2026-06-12 13:45:50 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-12 13:48:24 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 13:48:24 | INFO | ================================================================================ +2026-06-12 13:48:24 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 13:48:24 | INFO | ================================================================================ +2026-06-12 13:48:24 | INFO | +2026-06-12 13:48:26 | INFO | ================================================================================ +2026-06-12 13:48:26 | INFO | +2026-06-12 13:48:26 | INFO | ================================================================================ +2026-06-12 13:48:26 | INFO | Both databases connected successfully +2026-06-12 13:48:26 | INFO | ================================================================================ +2026-06-12 13:48:26 | INFO | Collecting MIDs for: 2026-06-11 +2026-06-12 13:48:27 | INFO | ================================================================================ +2026-06-12 13:48:27 | INFO | Found 818 MIDs +2026-06-12 13:48:27 | INFO | ================================================================================ +2026-06-12 13:48:27 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:48:27 | INFO | ================================================================================ +2026-06-12 13:48:27 | INFO | Fetching data for 818 MIDs +2026-06-12 13:48:27 | INFO | ================================================================================ +2026-06-12 13:48:27 | INFO | Fetched 3,677 rows from SQL Server +2026-06-12 13:48:27 | INFO | ================================================================================ +2026-06-12 13:48:27 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:48:27 | INFO | ================================================================================ +2026-06-12 13:48:27 | INFO | Fetching OQaD data for 818 MIDs +2026-06-12 13:48:29 | INFO | ================================================================================ +2026-06-12 13:48:29 | INFO | Fetched 464 rows +2026-06-12 13:48:29 | INFO | ================================================================================ +2026-06-12 13:48:29 | INFO | TABLE=Survey | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:48:29 | INFO | ================================================================================ +2026-06-12 13:48:29 | INFO | Fetching Survey data for 818 MIDs +2026-06-12 13:48:30 | INFO | ================================================================================ +2026-06-12 13:48:30 | INFO | Fetched 142 Survey rows +2026-06-12 13:48:30 | INFO | ================================================================================ +2026-06-12 13:48:30 | INFO | TABLE=additional_visibility | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:48:30 | INFO | ================================================================================ +2026-06-12 13:48:30 | INFO | Fetching Additional Visibility data for 818 MIDs +2026-06-12 13:48:30 | INFO | ================================================================================ +2026-06-12 13:48:30 | INFO | Fetched 1,922 Additional Visibility rows +2026-06-12 13:48:30 | INFO | ================================================================================ +2026-06-12 13:48:30 | INFO | TABLE=Coverage | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:48:30 | INFO | ================================================================================ +2026-06-12 13:48:30 | INFO | Fetching coverage data for 818 MIDs +2026-06-12 13:48:30 | INFO | ================================================================================ +2026-06-12 13:48:30 | INFO | Fetched 761 rows from SQL Server +2026-06-12 13:48:30 | INFO | ================================================================================ +2026-06-12 13:48:30 | INFO | TABLE=Login | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:48:30 | INFO | ================================================================================ +2026-06-12 13:48:30 | INFO | Fetching Login data for yesterday +2026-06-12 13:48:31 | INFO | ================================================================================ +2026-06-12 13:48:31 | INFO | Fetched 479 Login rows +2026-06-12 13:48:31 | INFO | ================================================================================ +2026-06-12 13:48:31 | INFO | TABLE=Stock_Details | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:48:31 | INFO | ================================================================================ +2026-06-12 13:48:31 | INFO | Fetching Stock Details data for 818 MIDs +2026-06-12 13:48:34 | INFO | Fetched 41,628 Stock Details rows +2026-06-12 13:48:34 | INFO | ================================================================================ +2026-06-12 13:48:34 | INFO | TABLE=Attendance | TYPE=FACT | OPERATION=DELETE+INSERT +2026-06-12 13:48:34 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 +2026-06-12 13:48:35 | INFO | Fetched 11,918 attendance rows for 592 employees +2026-06-12 13:48:35 | INFO | ================================================================================ +2026-06-12 13:48:35 | INFO | TABLE=Store_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 13:48:35 | INFO | Fetching Store Master data +2026-06-12 13:48:37 | INFO | Fetched 5,984 stores +2026-06-12 13:48:37 | INFO | ================================================================================ +2026-06-12 13:48:37 | INFO | TABLE=SKU_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 13:48:37 | INFO | Fetching SKU Master data +2026-06-12 13:48:37 | INFO | Fetched 160 SKU Master rows +2026-06-12 13:48:37 | INFO | ================================================================================ +2026-06-12 13:48:37 | INFO | TABLE=display_master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 13:48:37 | INFO | Fetching Display Master data +2026-06-12 13:48:37 | INFO | Fetched 135 Display Master records +2026-06-12 13:48:37 | INFO | ================================================================================ +2026-06-12 13:48:37 | INFO | TABLE=Employee_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 13:48:37 | INFO | Fetching Employee Master data +2026-06-12 13:48:37 | INFO | Fetched 2,269 Employee Master records +2026-06-12 13:48:37 | INFO | ================================================================================ +2026-06-12 13:48:37 | INFO | TABLE=Journey_Plan | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:48:37 | INFO | Fetching Journey Plan for 2026-06 +2026-06-12 13:48:38 | INFO | Fetched 20,973 Journey Plan records +2026-06-12 13:48:38 | INFO | ================================================================================ +2026-06-12 13:48:38 | INFO | TABLE=coverage_remarks | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:48:38 | INFO | Fetching Coverage Remarks +2026-06-12 13:48:38 | INFO | Fetched 29 Coverage Remark records +2026-06-12 13:48:38 | INFO | ================================================================================ +2026-06-12 13:48:38 | INFO | TABLE=mapping_storevisibility | TYPE=BRIDGE | OPERATION=INSERT +2026-06-12 13:48:38 | INFO | ================================================================================ +2026-06-12 13:48:38 | INFO | Fetching Mapping Store Visibility for 2026-06-11 +2026-06-12 13:48:38 | INFO | ================================================================================ +2026-06-12 13:48:38 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-12 13:48:38 | INFO | ================================================================================ +2026-06-12 13:48:38 | INFO | TABLE=Master_VisibilityReason | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 13:48:38 | INFO | Fetching Master Visibility Reason data +2026-06-12 13:48:38 | INFO | Fetched 17 Master Visibility Reason records +2026-06-12 13:48:38 | INFO | ================================================================================ +2026-06-12 13:48:38 | INFO | TABLE=Master_VisibilityDefinition | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 13:48:38 | INFO | Fetching Master Visibility Definition data +2026-06-12 13:48:38 | INFO | Fetched 861 Master Visibility Definition records +2026-06-12 13:48:38 | INFO | ================================================================================ +2026-06-12 13:48:38 | INFO | TABLE=Web_Logins | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:48:38 | INFO | ================================================================================ +2026-06-12 13:48:38 | INFO | Fetching Web Login data for 2026-06-11 +2026-06-12 13:48:38 | INFO | ================================================================================ +2026-06-12 13:48:38 | INFO | Fetched 223 Web Login records +2026-06-12 13:48:38 | INFO | ================================================================================ +2026-06-12 13:48:38 | INFO | TABLE=Promotion | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:48:38 | INFO | ================================================================================ +2026-06-12 13:48:38 | INFO | Fetching Promotion data for 818 MIDs +2026-06-12 13:48:41 | INFO | ================================================================================ +2026-06-12 13:48:41 | INFO | Fetched 8,410 Promotion records +2026-06-12 13:48:41 | INFO | ================================================================================ +2026-06-12 13:48:41 | INFO | TABLE=PaidVisibility | TYPE=FACT | OPERATION=INSERT +2026-06-12 13:48:41 | INFO | ================================================================================ +2026-06-12 13:48:41 | INFO | Fetching Paid Visibility data for 818 MIDs +2026-06-12 13:48:48 | INFO | ================================================================================ +2026-06-12 13:48:48 | INFO | Fetched 1,098 Paid Visibility records +2026-06-12 13:48:48 | INFO | ================================================================================ +2026-06-12 13:48:48 | INFO | TABLE=Master_Salesterritorylayer | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 13:48:48 | INFO | Fetching Master Sales Territory Layer data +2026-06-12 13:48:48 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-12 14:57:00 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 14:57:00 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 14:57:01 | INFO | ================================================================================ +2026-06-12 14:57:01 | INFO | +2026-06-12 14:57:03 | INFO | +2026-06-12 14:57:04 | INFO | Both databases connected successfully +2026-06-12 14:57:04 | INFO | Collecting MIDs for: 2026-06-11 +2026-06-12 14:57:04 | INFO | Found 818 MIDs +2026-06-12 14:59:11 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 14:59:11 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 14:59:12 | INFO | ================================================================================ +2026-06-12 14:59:12 | INFO | +2026-06-12 14:59:14 | INFO | +2026-06-12 14:59:14 | INFO | Both databases connected successfully +2026-06-12 14:59:14 | INFO | Collecting MIDs for: 2026-06-11 +2026-06-12 14:59:14 | INFO | Found 818 MIDs +2026-06-12 14:59:14 | INFO | ================================================================================ +2026-06-12 14:59:14 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT +2026-06-12 14:59:14 | INFO | ================================================================================ +2026-06-12 14:59:14 | INFO | Fetching data for 818 MIDs +2026-06-12 14:59:15 | INFO | ================================================================================ +2026-06-12 14:59:15 | INFO | Fetched 3,677 rows from SQL Server +2026-06-12 14:59:15 | INFO | ================================================================================ +2026-06-12 14:59:15 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT +2026-06-12 14:59:15 | INFO | ================================================================================ +2026-06-12 14:59:15 | INFO | Fetching OQaD data for 818 MIDs +2026-06-12 14:59:18 | INFO | ================================================================================ +2026-06-12 14:59:18 | INFO | Fetched 464 rows +2026-06-12 14:59:18 | INFO | ================================================================================ +2026-06-12 14:59:18 | INFO | TABLE=Survey | TYPE=FACT | OPERATION=INSERT +2026-06-12 14:59:18 | INFO | ================================================================================ +2026-06-12 14:59:18 | INFO | Fetching Survey data for 818 MIDs +2026-06-12 14:59:19 | INFO | ================================================================================ +2026-06-12 14:59:19 | INFO | Fetched 142 Survey rows +2026-06-12 14:59:19 | INFO | ================================================================================ +2026-06-12 14:59:19 | INFO | TABLE=additional_visibility | TYPE=FACT | OPERATION=INSERT +2026-06-12 14:59:19 | INFO | ================================================================================ +2026-06-12 14:59:19 | INFO | Fetching Additional Visibility data for 818 MIDs +2026-06-12 14:59:19 | INFO | ================================================================================ +2026-06-12 14:59:19 | INFO | Fetched 1,922 Additional Visibility rows +2026-06-12 14:59:19 | INFO | ================================================================================ +2026-06-12 14:59:19 | INFO | TABLE=Coverage | TYPE=FACT | OPERATION=INSERT +2026-06-12 14:59:19 | INFO | ================================================================================ +2026-06-12 14:59:19 | INFO | Fetching coverage data for 818 MIDs +2026-06-12 14:59:20 | INFO | ================================================================================ +2026-06-12 14:59:20 | INFO | Fetched 761 rows from SQL Server +2026-06-12 14:59:20 | INFO | ================================================================================ +2026-06-12 14:59:20 | INFO | TABLE=Login | TYPE=FACT | OPERATION=INSERT +2026-06-12 14:59:20 | INFO | ================================================================================ +2026-06-12 14:59:20 | INFO | Fetching Login data for yesterday +2026-06-12 14:59:21 | INFO | ================================================================================ +2026-06-12 14:59:21 | INFO | Fetched 479 Login rows +2026-06-12 14:59:21 | INFO | ================================================================================ +2026-06-12 14:59:21 | INFO | TABLE=Stock_Details | TYPE=FACT | OPERATION=INSERT +2026-06-12 14:59:21 | INFO | ================================================================================ +2026-06-12 14:59:21 | INFO | Fetching Stock Details data for 818 MIDs +2026-06-12 14:59:26 | INFO | Fetched 41,628 Stock Details rows +2026-06-12 14:59:26 | INFO | ================================================================================ +2026-06-12 14:59:26 | INFO | TABLE=Attendance | TYPE=FACT | OPERATION=DELETE+INSERT +2026-06-12 14:59:26 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 +2026-06-12 14:59:27 | INFO | Fetched 11,918 attendance rows for 592 employees +2026-06-12 14:59:27 | INFO | ================================================================================ +2026-06-12 14:59:27 | INFO | TABLE=Store_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 14:59:27 | INFO | Fetching Store Master data +2026-06-12 14:59:28 | INFO | Fetched 5,984 stores +2026-06-12 14:59:29 | INFO | ================================================================================ +2026-06-12 14:59:29 | INFO | TABLE=SKU_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 14:59:29 | INFO | Fetching SKU Master data +2026-06-12 14:59:29 | INFO | Fetched 160 SKU Master rows +2026-06-12 14:59:29 | INFO | ================================================================================ +2026-06-12 14:59:29 | INFO | TABLE=display_master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 14:59:29 | INFO | Fetching Display Master data +2026-06-12 14:59:29 | INFO | Fetched 135 Display Master records +2026-06-12 14:59:29 | INFO | ================================================================================ +2026-06-12 14:59:29 | INFO | TABLE=Employee_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 14:59:29 | INFO | Fetching Employee Master data +2026-06-12 14:59:29 | INFO | Fetched 2,269 Employee Master records +2026-06-12 14:59:29 | INFO | ================================================================================ +2026-06-12 14:59:29 | INFO | TABLE=Journey_Plan | TYPE=FACT | OPERATION=INSERT +2026-06-12 14:59:29 | INFO | Fetching Journey Plan for 2026-06 +2026-06-12 14:59:30 | INFO | Fetched 20,976 Journey Plan records +2026-06-12 14:59:30 | INFO | ================================================================================ +2026-06-12 14:59:30 | INFO | TABLE=coverage_remarks | TYPE=FACT | OPERATION=INSERT +2026-06-12 14:59:30 | INFO | Fetching Coverage Remarks +2026-06-12 14:59:30 | INFO | Fetched 29 Coverage Remark records +2026-06-12 14:59:30 | INFO | ================================================================================ +2026-06-12 14:59:30 | INFO | TABLE=mapping_storevisibility | TYPE=BRIDGE | OPERATION=INSERT +2026-06-12 14:59:30 | INFO | ================================================================================ +2026-06-12 14:59:30 | INFO | Fetching Mapping Store Visibility for 2026-06-11 +2026-06-12 14:59:30 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-12 14:59:30 | INFO | ================================================================================ +2026-06-12 14:59:30 | INFO | TABLE=Master_VisibilityReason | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 14:59:30 | INFO | Fetching Master Visibility Reason data +2026-06-12 14:59:30 | INFO | Fetched 17 Master Visibility Reason records +2026-06-12 14:59:30 | INFO | ================================================================================ +2026-06-12 14:59:30 | INFO | TABLE=Master_VisibilityDefinition | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 14:59:30 | INFO | Fetching Master Visibility Definition data +2026-06-12 14:59:30 | INFO | Fetched 861 Master Visibility Definition records +2026-06-12 14:59:30 | INFO | ================================================================================ +2026-06-12 14:59:30 | INFO | TABLE=Web_Logins | TYPE=FACT | OPERATION=INSERT +2026-06-12 14:59:30 | INFO | ================================================================================ +2026-06-12 14:59:30 | INFO | Fetching Web Login data for 2026-06-11 +2026-06-12 14:59:31 | INFO | Fetched 223 Web Login records +2026-06-12 14:59:31 | INFO | ================================================================================ +2026-06-12 14:59:31 | INFO | TABLE=Promotion | TYPE=FACT | OPERATION=INSERT +2026-06-12 14:59:31 | INFO | Fetching Promotion data for 818 MIDs +2026-06-12 14:59:33 | INFO | Fetched 8,410 Promotion records +2026-06-12 14:59:33 | INFO | ================================================================================ +2026-06-12 14:59:33 | INFO | TABLE=PaidVisibility | TYPE=FACT | OPERATION=INSERT +2026-06-12 14:59:33 | INFO | Fetching Paid Visibility data for 818 MIDs +2026-06-12 14:59:41 | INFO | Fetched 1,098 Paid Visibility records +2026-06-12 14:59:41 | INFO | ================================================================================ +2026-06-12 14:59:41 | INFO | TABLE=Master_Salesterritorylayer | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 14:59:41 | INFO | Fetching Master Sales Territory Layer data +2026-06-12 14:59:41 | INFO | Fetched 33 Master Sales Territory Layer records +2026-06-12 15:20:11 | INFO | ================================================================================ +2026-06-12 15:20:11 | INFO | Hello from data-move Python data pipeline ! +2026-06-12 15:20:11 | INFO | Data-pipeline running Date is -:2026-06-11 +2026-06-12 15:20:11 | INFO | connecting with both db servers sql-serveras well as clickhouse DB +2026-06-12 15:20:12 | INFO | +2026-06-12 15:20:13 | INFO | +2026-06-12 15:20:14 | INFO | Both databases connected successfully +2026-06-12 15:20:14 | INFO | Collecting MIDs for: 2026-06-11 +2026-06-12 15:20:14 | INFO | Found 818 MIDs +2026-06-12 15:20:14 | INFO | ================================================================================ +2026-06-12 15:20:14 | INFO | TABLE=SOS_OneApp | TYPE=FACT | OPERATION=INSERT +2026-06-12 15:20:14 | INFO | Fetching data for 818 MIDs +2026-06-12 15:20:16 | INFO | Fetched 3,677 rows from SQL Server +2026-06-12 15:20:16 | INFO | ================================================================================ +2026-06-12 15:20:16 | INFO | TABLE=OQaD | TYPE=FACT | OPERATION=INSERT +2026-06-12 15:20:16 | INFO | Fetching OQaD data for 818 MIDs +2026-06-12 15:20:20 | INFO | Fetched 464 rows +2026-06-12 15:20:20 | INFO | ================================================================================ +2026-06-12 15:20:20 | INFO | TABLE=Survey | TYPE=FACT | OPERATION=INSERT +2026-06-12 15:20:20 | INFO | Fetching Survey data for 818 MIDs +2026-06-12 15:20:21 | INFO | Fetched 142 Survey rows +2026-06-12 15:20:21 | INFO | ================================================================================ +2026-06-12 15:20:21 | INFO | TABLE=additional_visibility | TYPE=FACT | OPERATION=INSERT +2026-06-12 15:20:21 | INFO | Fetching Additional Visibility data for 818 MIDs +2026-06-12 15:20:22 | INFO | Fetched 1,922 Additional Visibility rows +2026-06-12 15:20:22 | INFO | ================================================================================ +2026-06-12 15:20:22 | INFO | TABLE=Coverage | TYPE=FACT | OPERATION=INSERT +2026-06-12 15:20:22 | INFO | Fetching coverage data for 818 MIDs +2026-06-12 15:20:24 | INFO | Fetched 761 rows from SQL Server +2026-06-12 15:20:24 | INFO | ================================================================================ +2026-06-12 15:20:24 | INFO | TABLE=Login | TYPE=FACT | OPERATION=INSERT +2026-06-12 15:20:24 | INFO | Fetching Login data for yesterday +2026-06-12 15:20:25 | INFO | Fetched 479 Login rows +2026-06-12 15:20:25 | INFO | ================================================================================ +2026-06-12 15:20:25 | INFO | TABLE=Stock_Details | TYPE=FACT | OPERATION=INSERT +2026-06-12 15:20:25 | INFO | Fetching Stock Details data for 818 MIDs +2026-06-12 15:20:30 | INFO | Fetched 41,628 Stock Details rows +2026-06-12 15:20:30 | INFO | ================================================================================ +2026-06-12 15:20:30 | INFO | TABLE=Attendance | TYPE=FACT | OPERATION=DELETE+INSERT +2026-06-12 15:20:30 | INFO | Fetching Attendance data from 2026-05-27 to 2026-06-11 +2026-06-12 15:20:31 | INFO | Fetched 11,918 attendance rows for 592 employees +2026-06-12 15:20:31 | INFO | ================================================================================ +2026-06-12 15:20:31 | INFO | TABLE=Store_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 15:20:31 | INFO | Fetching Store Master data +2026-06-12 15:20:33 | INFO | Fetched 5,984 stores +2026-06-12 15:20:33 | INFO | ================================================================================ +2026-06-12 15:20:33 | INFO | TABLE=SKU_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 15:20:33 | INFO | Fetching SKU Master data +2026-06-12 15:20:33 | INFO | Fetched 160 SKU Master rows +2026-06-12 15:20:33 | INFO | ================================================================================ +2026-06-12 15:20:33 | INFO | TABLE=display_master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 15:20:33 | INFO | Fetching Display Master data +2026-06-12 15:20:33 | INFO | Fetched 135 Display Master records +2026-06-12 15:20:33 | INFO | ================================================================================ +2026-06-12 15:20:33 | INFO | TABLE=Employee_Master | TYPE=DIMENSION | OPERATION=DELETE+INSERT +2026-06-12 15:20:33 | INFO | Fetching Employee Master data +2026-06-12 15:20:34 | INFO | Fetched 2,269 Employee Master records +2026-06-12 15:20:34 | INFO | ================================================================================ +2026-06-12 15:20:34 | INFO | TABLE=Journey_Plan | TYPE=FACT | OPERATION=INSERT +2026-06-12 15:20:34 | INFO | Fetching Journey Plan for 2026-06 +2026-06-12 15:20:35 | INFO | Fetched 20,979 Journey Plan records +2026-06-12 15:20:35 | INFO | ================================================================================ +2026-06-12 15:20:35 | INFO | TABLE=coverage_remarks | TYPE=FACT | OPERATION=INSERT +2026-06-12 15:20:35 | INFO | Fetching Coverage Remarks +2026-06-12 15:20:35 | INFO | Fetched 29 Coverage Remark records +2026-06-12 15:20:35 | INFO | ================================================================================ +2026-06-12 15:20:35 | INFO | TABLE=mapping_storevisibility | TYPE=BRIDGE | OPERATION=INSERT +2026-06-12 15:20:35 | INFO | Fetching Mapping Store Visibility for 2026-06-11 +2026-06-12 15:20:35 | INFO | Fetched 0 Mapping Store Visibility records +2026-06-12 15:20:35 | INFO | ================================================================================ +2026-06-12 15:20:35 | INFO | TABLE=Master_VisibilityReason | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 15:20:35 | INFO | Fetching Master Visibility Reason data +2026-06-12 15:20:35 | INFO | Fetched 17 Master Visibility Reason records +2026-06-12 15:20:35 | INFO | ================================================================================ +2026-06-12 15:20:35 | INFO | TABLE=Master_VisibilityDefinition | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 15:20:35 | INFO | Fetching Master Visibility Definition data +2026-06-12 15:20:35 | INFO | Fetched 861 Master Visibility Definition records +2026-06-12 15:20:35 | INFO | ================================================================================ +2026-06-12 15:20:35 | INFO | TABLE=Web_Logins | TYPE=FACT | OPERATION=INSERT +2026-06-12 15:20:35 | INFO | Fetching Web Login data for 2026-06-11 +2026-06-12 15:20:36 | INFO | Fetched 223 Web Login records +2026-06-12 15:20:36 | INFO | ================================================================================ +2026-06-12 15:20:36 | INFO | TABLE=Promotion | TYPE=FACT | OPERATION=INSERT +2026-06-12 15:20:36 | INFO | Fetching Promotion data for 818 MIDs +2026-06-12 15:20:39 | INFO | Fetched 8,410 Promotion records +2026-06-12 15:20:39 | INFO | ================================================================================ +2026-06-12 15:20:39 | INFO | TABLE=PaidVisibility | TYPE=FACT | OPERATION=INSERT +2026-06-12 15:20:39 | INFO | Fetching Paid Visibility data for 818 MIDs +2026-06-12 15:20:47 | INFO | Fetched 1,098 Paid Visibility records +2026-06-12 15:20:47 | INFO | ================================================================================ +2026-06-12 15:20:47 | INFO | TABLE=Master_Salesterritorylayer | TYPE=DIMENSION | OPERATION=INSERT +2026-06-12 15:20:47 | INFO | Fetching Master Sales Territory Layer data +2026-06-12 15:20:47 | INFO | Fetched 33 Master Sales Territory Layer records diff --git a/main.py b/main.py index 42294e3..8709564 100644 --- a/main.py +++ b/main.py @@ -2,34 +2,46 @@ # requires-python = ">=3.11" # dependencies = [ # "polars>=0.20.0", +# "pyarrow>=18.0.0", # "sqlalchemy>=2.0.0", # "pyodbc>=5.0.0", -# "pyyaml>=6.0.3", # "clickhouse-connect>=0.7.0", # "clickhouse-sqlalchemy>=0.3.2", +# "pyyaml>=6.0.3", # "python-dotenv>=1.0.0", -# "pyarrow>=18.0.0" # ] # /// +from __future__ import annotations + import os -import yaml -import pyarrow +# import pyarrow import sys -import logging from datetime import date, timedelta + import polars as pl -from sqlalchemy import create_engine, text -from sqlalchemy.engine import Engine, URL -import clickhouse_connect +import yaml from dotenv import load_dotenv -from log import log -from clickhouse_task.create_table import create_clickhouse_table , check +from sqlalchemy import create_engine, text +from sqlalchemy.engine import Engine, URL + +import clickhouse_connect + +from log import log + + + +from clickhouse_task.create_table import * +from clickhouse_task.delete_task import * +from clickhouse_task.load_table import * from db_con.connection import * from mids import * +from masters.dimensions import * +from masters.bridge import * +from kpi.facts import * @@ -39,16 +51,15 @@ from mids import * def main(): + log.info("=" * 80) log.info("Hello from data-move Python data pipeline !") - check() if len(sys.argv) > 1: run_date = datetime.strptime(sys.argv[1], "%Y-%m-%d").date() else: run_date = date.today() - timedelta(days=1) - print(run_date) - print(type(run_date)) + log.info(f"Data-pipeline running Date is -:{run_date}") # connecting with both db servers sql-server @@ -74,11 +85,15 @@ def main(): for table in config["tables"]: - table_name=table["name"], - table_type=table["type"], + table_name=table["name"] + table_type=table["type"] operation=table["operation"] - log.info(table_name, operation) + log.info("=" * 80) + log.info("TABLE=%s | TYPE=%s | OPERATION=%s", + table_name, + table_type, + operation) fn=f"fetch_{table_name}" list=["Attendance", "Journey_Plan", "Web_Logins"] @@ -101,6 +116,24 @@ def main(): + # Step 2 + + if operation == "DELETE+INSERT" : + truncate_table(client , table_name ) + log.info(f"Truncate a ClickHouse table - {table_name}") + + load_to_clickhouse( + client=client, + table_name=table_name, + df=df, + ) + + else: + print("table is fact ") + + + + diff --git a/masters/bridge.py b/masters/bridge.py index a622bf4..3d6722b 100644 --- a/masters/bridge.py +++ b/masters/bridge.py @@ -1,5 +1,5 @@ import os -import pyarrow +# import pyarrow import sys import logging from datetime import date, timedelta @@ -10,7 +10,7 @@ import clickhouse_connect from dotenv import load_dotenv from log import log -from clickhouse_task.create_table import create_clickhouse_table , check +from clickhouse_task.create_table import * from db_con.connection import * from mids import * @@ -22,7 +22,7 @@ from mids import * -def fetch_mapping_store_visibility( +def fetch_mapping_storevisibility( engine: Engine, run_date: date ) -> pl.DataFrame: @@ -47,7 +47,7 @@ def fetch_mapping_store_visibility( WHERE MenuId = 22 ) """ - + log.info( f"Fetching Mapping Store Visibility for {run_date}" ) @@ -56,7 +56,7 @@ def fetch_mapping_store_visibility( query=sql, connection=engine ) - + log.info( f"Fetched {len(df):,} Mapping Store Visibility records" ) diff --git a/masters/dimensions.py b/masters/dimensions.py index b028a70..5095fc8 100644 --- a/masters/dimensions.py +++ b/masters/dimensions.py @@ -1,5 +1,5 @@ import os -import pyarrow +# import pyarrow import sys import logging from datetime import date, timedelta @@ -10,7 +10,7 @@ import clickhouse_connect from dotenv import load_dotenv from log import log -from clickhouse_task.create_table import create_clickhouse_table , check +from clickhouse_task.create_table import * from db_con.connection import * from mids import * @@ -27,7 +27,7 @@ p=40148 -def fetch_Store_master(engine: Engine) -> pl.DataFrame: +def fetch_Store_Master(engine: Engine) -> pl.DataFrame: sql = """ SELECT RegionId AS region_id, @@ -87,7 +87,7 @@ def fetch_Store_master(engine: Engine) -> pl.DataFrame: -def fetch_sku_master(engine: Engine) -> pl.DataFrame: +def fetch_SKU_Master(engine: Engine) -> pl.DataFrame: sql = """ SELECT @@ -193,7 +193,7 @@ def fetch_display_master(engine: Engine) -> pl.DataFrame: -def fetch_employee_master(engine: Engine) -> pl.DataFrame: +def fetch_Employee_Master(engine: Engine) -> pl.DataFrame: """ Fetch Employee Master data. Source: vw_Employee_Detail + Mapping_PositionUser + Master_Position @@ -276,7 +276,7 @@ def fetch_employee_master(engine: Engine) -> pl.DataFrame: -def fetch_employee_master(engine: Engine) -> pl.DataFrame: +def fetch_Employee_Master(engine: Engine) -> pl.DataFrame: """ Fetch Employee Master data. Source: vw_Employee_Detail + Mapping_PositionUser + Master_Position @@ -357,7 +357,7 @@ def fetch_employee_master(engine: Engine) -> pl.DataFrame: -def fetch_master_visibility_reason(engine: Engine) -> pl.DataFrame: +def fetch_Master_VisibilityReason(engine: Engine) -> pl.DataFrame: """ Source: Master_VisibilityReason @@ -390,7 +390,7 @@ def fetch_master_visibility_reason(engine: Engine) -> pl.DataFrame: -def fetch_master_visibility_definition(engine: Engine) -> pl.DataFrame: +def fetch_Master_VisibilityDefinition(engine: Engine) -> pl.DataFrame: """ Source: OneApp_KelloggsMT.dbo.Master_VisibilityDefinition @@ -423,3 +423,64 @@ def fetch_master_visibility_definition(engine: Engine) -> pl.DataFrame: return df + + +def fetch_Master_Salesterritorylayer( + engine: Engine +) -> pl.DataFrame: + """ + Source: + Master_SalesTerritoryLayerOne + Master_SalesTerritoryLayerTwo + Master_SalesTerritoryLayerThree + Master_SalesTerritoryLayerFour + + Target: + Master_Salesterritorylayer + """ + + sql = """ + SELECT DISTINCT + 40148 AS project_id, + + D.StLayerOneId AS st_layer_one_id, + D.StLayerOneName AS st_layer_one_name, + + C.StLayerTwoId AS st_layer_two_id, + C.StLayerTwoName AS st_layer_two_name, + + B.StLayerThreeId AS st_layer_three_id, + B.StLayerThreeName AS st_layer_three_name, + + A.StLayerFourId AS st_layer_four_id, + A.StLayerFourName AS st_layer_four_name, + + GETDATE() AS create_date, + 'SP-Pius' AS create_by + + FROM OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerFour A + + INNER JOIN OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerThree B + ON A.StLayerThreeId = B.StLayerThreeId + + INNER JOIN OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerTwo C + ON B.StLayerTwoId = C.StLayerTwoId + + INNER JOIN OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerOne D + ON C.StLayerOneId = D.StLayerOneId + """ + + log.info("Fetching Master Sales Territory Layer data") + + df = pl.read_database( + query=sql, + connection=engine + ) + + log.info( + f"Fetched {len(df):,} Master Sales Territory Layer records" + ) + + return df + + diff --git a/mids.py b/mids.py index 833f79c..f7aa1c9 100644 --- a/mids.py +++ b/mids.py @@ -1,5 +1,5 @@ import os -import pyarrow +# import pyarrow import sys import logging from datetime import date, timedelta @@ -10,7 +10,7 @@ import clickhouse_connect from dotenv import load_dotenv from log import log -from clickhouse_task.create_table import create_clickhouse_table , check +from clickhouse_task.create_table import * from db_con.connection import * @@ -28,7 +28,9 @@ def collect_mids(engine: Engine, target_date: date) -> list[int]: SELECT MID FROM OneApp_KelloggsMT.dbo.T_StoreCoverage WHERE CONVERT(date, UpdateDate) = :target_date """) + log.info(f"Collecting MIDs for: {target_date}") + with engine.connect() as conn: result = conn.execute(sql, {"target_date": target_date}) mids = [row[0] for row in result.fetchall()] diff --git a/pyproject.toml b/pyproject.toml index dfea37a..7c2e9b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,10 @@ description = "Add your description here" readme = "README.md" requires-python = ">=3.14" dependencies = [ + "clickhouse-connect>=1.3.0", "clickhouse-sqlalchemy>=0.3.2", + "dotenv>=0.9.9", + "polars>=1.41.2", + "pyodbc>=5.3.0", "pyyaml>=6.0.3", ] diff --git a/tables.yaml b/tables.yml similarity index 100% rename from tables.yaml rename to tables.yml diff --git a/test.py b/test.py new file mode 100644 index 0000000..8517bfb --- /dev/null +++ b/test.py @@ -0,0 +1,47 @@ + +from __future__ import annotations + +import os +# import pyarrow +import sys +from datetime import date, timedelta + +import polars as pl +import yaml +from dotenv import load_dotenv + +from sqlalchemy import create_engine, text +from sqlalchemy.engine import Engine, URL + +import clickhouse_connect + +from log import log + + + +from clickhouse_task.create_table import * +from clickhouse_task.delete_task import * +from clickhouse_task.load_table import * +from db_con.connection import * +from mids import * +from masters.dimensions import * +from masters.bridge import * +from kpi.facts import * + + + + + + + +with open("tables.yml", "r") as file: + config = yaml.safe_load(file) + + for table in config["tables"]: + + table_name=table["name"] + table_type=table["type"] + operation=table["operation"] + print(table_name) + print(table_type) + print(operation) \ No newline at end of file diff --git a/uv.lock b/uv.lock index bb3b816..40eedd6 100644 --- a/uv.lock +++ b/uv.lock @@ -85,6 +85,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/21/8e/515f9404faa39af8df5e2b899cafbca5dbe7cd2ffe5cc124ef393ffdaf1c/ciso8601-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:7657ba9730dc1340d73b9e61eca14f341c41dd308128c808b8b084d2b85bc03e", size = 17977, upload-time = "2025-08-20T16:31:03.429Z" }, ] +[[package]] +name = "clickhouse-connect" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "lz4" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, + { name = "urllib3" }, + { name = "zstandard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/b4/720c5f4cdfe4bf716001d3a8db24b9ba94e7ad709e626a067502ed457199/clickhouse_connect-1.3.0.tar.gz", hash = "sha256:32e780ff3de62dbff2ff21eaf0501582b5365fba6c42227e203664379312e33e", size = 164472, upload-time = "2026-06-11T14:50:18.159Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/cc/5d6d0b385f1ae33a67b702c04672ff2203ffc298bb1c845cc5884c9d4e3b/clickhouse_connect-1.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2d1645af60860edb2f61b474daf5aa0af6b48724eb4d2c748edfa4027fdbe58a", size = 342602, upload-time = "2026-06-11T14:49:40.664Z" }, + { url = "https://files.pythonhosted.org/packages/30/ec/97fdf0c4a949c7de7da40ca2960a134f2d4303bef2a52b71012a21f0aa16/clickhouse_connect-1.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:225de4ab00609e599f2529a8a5256da5f473ce9544a04ab9b18b8fdd5baf9005", size = 332000, upload-time = "2026-06-11T14:49:42.28Z" }, + { url = "https://files.pythonhosted.org/packages/7d/f5/c136e03a9513f8c6a8546d04064ea7c8f42e694260ebceaeba749058670a/clickhouse_connect-1.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c18c0773a85f26c7eaeb59e0fd0a142e464312fda1c54fc7feae6115eb1759d4", size = 1252990, upload-time = "2026-06-11T14:49:43.721Z" }, + { url = "https://files.pythonhosted.org/packages/76/69/601dff4fcb2a1b157dbf6fd608db884ae5fd408fab646a40f64e96a173b3/clickhouse_connect-1.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:61f2f32403ac23354a572b160fa0a51ad5e76ba88aea37ebe5371b8863659339", size = 1266406, upload-time = "2026-06-11T14:49:45.359Z" }, + { url = "https://files.pythonhosted.org/packages/6f/ec/611b7782d1acf5b5f6368ea0062f8cbcf5913f76f5814d067bed1db0f6ea/clickhouse_connect-1.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e655403a35d016ac2100ca49fece73d1f293c6623809a04e530aa05c0369d69c", size = 1232665, upload-time = "2026-06-11T14:49:46.879Z" }, + { url = "https://files.pythonhosted.org/packages/73/16/801c28dc369b67a5362ae77a70dd90f40cf7589c9edd52f4e1ed633f9bfc/clickhouse_connect-1.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:bf99dc475626d1fc1720c3cbc8bd6154d26f829cf272c3c2524e5b7ce132fca1", size = 1263038, upload-time = "2026-06-11T14:49:48.44Z" }, + { url = "https://files.pythonhosted.org/packages/68/6b/718106208297d7ba7b8c3708fa42ee9da9269ab6d3fe94b744b659bb2c5f/clickhouse_connect-1.3.0-cp314-cp314-win32.whl", hash = "sha256:3954d59ced274163e1243549994eeaa6ae7fb46c1635ebd661eefefd25ce7c3d", size = 312209, upload-time = "2026-06-11T14:49:50.019Z" }, + { url = "https://files.pythonhosted.org/packages/d6/b9/a4173e4f9eb44ee9b4094d42ac5fc3da0763ecc538174f9ff6511b812ca0/clickhouse_connect-1.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:6447aaa3cda9a01580bffd821519199901124e979673a76f8c00d353293b2bd2", size = 331541, upload-time = "2026-06-11T14:49:51.567Z" }, + { url = "https://files.pythonhosted.org/packages/fe/d4/a2733ecbcba3e1497f8213ab4eb123a4b6c335400ce5dda42b1ff6c73719/clickhouse_connect-1.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:059f0ef645734cd5425b6af3bdac87c57f93e9c0dd1c86ef01da68846e2dd949", size = 312207, upload-time = "2026-06-11T14:49:53.095Z" }, + { url = "https://files.pythonhosted.org/packages/cf/fd/6e4f73c18b8f9e8136f88528ff93f067d50b09c3517e4699625d3ad491a8/clickhouse_connect-1.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:331d2aaadd1ef7fe238209da1d6bdbcb9244da6bb94f581beed8193d6b04ac2d", size = 360564, upload-time = "2026-06-11T14:49:54.557Z" }, + { url = "https://files.pythonhosted.org/packages/90/8a/be24013502d401f53485e738eb41e3374817b528dd8fc134ddc7748dbfc5/clickhouse_connect-1.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ab4380888830b92878231415fabab0b1b90bb3982d599e8444f8f9aa22174249", size = 353025, upload-time = "2026-06-11T14:49:56.31Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9e/f2a22a5f88bb7a8e0135d0b1e44f5b2526a94151f31471f0495167e066d5/clickhouse_connect-1.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a63c5f646142679b9288cf9ab1cc6af028389c894a67efa372f9cf60f2d97264", size = 1339647, upload-time = "2026-06-11T14:49:58.18Z" }, + { url = "https://files.pythonhosted.org/packages/bf/a2/ef2de3755a233184af2a3057d49ce319a0c3eebb8bdb9eaea606ad8a1317/clickhouse_connect-1.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7139165a4beccf7604c66849cd5da1cc2be97de76e39ca03fb93c34dc9a4c570", size = 1322984, upload-time = "2026-06-11T14:50:00.19Z" }, + { url = "https://files.pythonhosted.org/packages/77/eb/316dc1bd136070edd9ca75dcf1bd529bf854951bc5ced116903a19df2ba7/clickhouse_connect-1.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:73b089407354a0a2903561a5c86c2cb359705d38e9701dbef5fe13bbf69d94fd", size = 1297190, upload-time = "2026-06-11T14:50:01.994Z" }, + { url = "https://files.pythonhosted.org/packages/0c/a5/b0c98b795db1b5b403aeb2f03daab9539ce31536d8752c0d71f5997dfbbd/clickhouse_connect-1.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a9bce3cd4af89d6a0271938f1fabfc10a8889e1a7c4dbc4a866d2a804c4ed506", size = 1313649, upload-time = "2026-06-11T14:50:03.679Z" }, + { url = "https://files.pythonhosted.org/packages/2b/46/ab707f11890d0f3d2a8ae270e196fbe05c797b6d9ae99a3635057e84c85c/clickhouse_connect-1.3.0-cp314-cp314t-win32.whl", hash = "sha256:6c26e0175dddc2e3eafb606781c0a10be4fee13831117375f645eb70db722319", size = 336750, upload-time = "2026-06-11T14:50:05.555Z" }, + { url = "https://files.pythonhosted.org/packages/1b/d6/bd82db4324b917aaa7cb868fb4e11c6bc44b3d647ead0489e06748dee93c/clickhouse_connect-1.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:016cdfe4c17d898044f618963d11cc9604e98b239e269b5e69dcabcf2625ace9", size = 363227, upload-time = "2026-06-11T14:50:07.049Z" }, + { url = "https://files.pythonhosted.org/packages/83/60/7d95de37486b00671454047967c850bac40a1f07ebe3f56f750e47437f18/clickhouse_connect-1.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:f9341e3072d353bb834300c2a27e88a6fcde7a6c259c27c99fb530b97fcbc788", size = 323303, upload-time = "2026-06-11T14:50:08.839Z" }, +] + [[package]] name = "clickhouse-driver" version = "0.2.10" @@ -126,16 +159,35 @@ name = "data-move" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "clickhouse-connect" }, { name = "clickhouse-sqlalchemy" }, + { name = "dotenv" }, + { name = "polars" }, + { name = "pyodbc" }, { name = "pyyaml" }, ] [package.metadata] requires-dist = [ + { name = "clickhouse-connect", specifier = ">=1.3.0" }, { name = "clickhouse-sqlalchemy", specifier = ">=0.3.2" }, + { name = "dotenv", specifier = ">=0.9.9" }, + { name = "polars", specifier = ">=1.41.2" }, + { name = "pyodbc", specifier = ">=5.3.0" }, { name = "pyyaml", specifier = ">=6.0.3" }, ] +[[package]] +name = "dotenv" +version = "0.9.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dotenv" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/b7/545d2c10c1fc15e48653c91efde329a790f2eecfbbf2bd16003b5db2bab0/dotenv-0.9.9-py2.py3-none-any.whl", hash = "sha256:29cf74a087b31dafdb5a446b6d7e11cbce8ed2741540e2339c69fbef92c94ce9", size = 1892, upload-time = "2025-02-19T22:15:01.647Z" }, +] + [[package]] name = "greenlet" version = "3.5.1" @@ -209,6 +261,69 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/28/2635a8141c9a4f4bc23f5135a92bbcf48d928d8ca094088c962df1879d64/lz4-4.4.5-cp314-cp314-win_arm64.whl", hash = "sha256:d994b87abaa7a88ceb7a37c90f547b8284ff9da694e6afcfaa8568d739faf3f7", size = 93812, upload-time = "2025-11-03T13:02:26.133Z" }, ] +[[package]] +name = "polars" +version = "1.41.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "polars-runtime-32" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/f9/aeda46259b0669247a160315d2d51269de9504b9dd2f70acadbcb22f46b7/polars-1.41.2.tar.gz", hash = "sha256:256d6731162371b77f3f29a55eacb8c0fc740ddb1a293a01d2ef5b5393c5c708", size = 737996, upload-time = "2026-05-29T17:39:15.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/22/28f62d24f7db56ac4343588f9362d49b7b4177e55ac47a466fe696b0099b/polars-1.41.2-py3-none-any.whl", hash = "sha256:23ce9a2910b6e3e8d4258770bf44aa17170958df7af6e85feedf4458a04d8d29", size = 833445, upload-time = "2026-05-29T17:37:05.576Z" }, +] + +[[package]] +name = "polars-runtime-32" +version = "1.41.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/56/54e3ea0e9b64f327179049e4742241cc6b1d3e8fa414b05a057dd26df367/polars_runtime_32-1.41.2.tar.gz", hash = "sha256:7af09ec1ab053da2c9669e8d15f809a4083a29be05db57111688b8051062af56", size = 2989474, upload-time = "2026-05-29T17:39:17.257Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/9b/fe72a3811c0357cdb06c67bdc7695fa1623ad47948fc523195f5ac31037f/polars_runtime_32-1.41.2-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:95a08346dac337357cdb825c8076df7d36da54c4caa59a5cb41d0a30691c5edd", size = 52265283, upload-time = "2026-05-29T17:37:09.407Z" }, + { url = "https://files.pythonhosted.org/packages/0a/93/fab9da803fd80d9e83ef88c20932f637a10bc611b20415fc322eec84bc44/polars_runtime_32-1.41.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:dedfaeec2c7f995298da7319dd9431d662e5dd1d0ec51b1459df4a0234ceff52", size = 46571222, upload-time = "2026-05-29T17:37:13.698Z" }, + { url = "https://files.pythonhosted.org/packages/c8/2a/8843f34a8ac57acd058a39b87b03b580dd352a490e9dae0415e02033bdd4/polars_runtime_32-1.41.2-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18eea22c5cc34e27f8a60950458ad81e6a9ea75e89363ca1367e14e7e7f781fc", size = 50409372, upload-time = "2026-05-29T17:37:17.875Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c6/92b352fe88cf51bd0a19fb99e1c0cbe46aa26c14dcf7995b89869cd932ae/polars_runtime_32-1.41.2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2630540dfdfb0f36f9b04a07c7c2e3f50bf2ad384113263c1c812007ee9141e0", size = 56405484, upload-time = "2026-05-29T17:37:22.684Z" }, + { url = "https://files.pythonhosted.org/packages/74/c4/bae3174c3b02f6b441d2e58594387abcd509f67a098f682a83b195f08966/polars_runtime_32-1.41.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:20e969e08f9b137e233c04cc04de73d9795f89eb77d34854e40a025965a43763", size = 50603512, upload-time = "2026-05-29T17:37:27.422Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ed/f2d26ae02d92c2689056838ed59e2a626326ad23c2831d58637d25f6c82a/polars_runtime_32-1.41.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e7016a3deb641b64a31447abbbee0f34bd020a6a9ae34ee6b743837def15e2a4", size = 54328561, upload-time = "2026-05-29T17:37:32.587Z" }, + { url = "https://files.pythonhosted.org/packages/9b/c4/9c3831cc885dc7769e59abf8f583821a5fb4403fd0e4eba0ccc6d47a3d4b/polars_runtime_32-1.41.2-cp310-abi3-win_amd64.whl", hash = "sha256:1e5e5377c315e0dcafdfb2a31adc546abbaeb3f9cb1864e6536523d2af473265", size = 51978643, upload-time = "2026-05-29T17:37:37.443Z" }, + { url = "https://files.pythonhosted.org/packages/cd/c6/79e9f3f270270d7ed5575d92b7bfef49f01abd9275447161275b23b553a8/polars_runtime_32-1.41.2-cp310-abi3-win_arm64.whl", hash = "sha256:843d96f69d18eca53429c1198e58891db7f18111f83b9c419bb45ad9d73eaed5", size = 46006901, upload-time = "2026-05-29T17:37:42.522Z" }, +] + +[[package]] +name = "pyodbc" +version = "5.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/85/44b10070a769a56bd910009bb185c0c0a82daff8d567cd1a116d7d730c7d/pyodbc-5.3.0.tar.gz", hash = "sha256:2fe0e063d8fb66efd0ac6dc39236c4de1a45f17c33eaded0d553d21c199f4d05", size = 121770, upload-time = "2025-10-17T18:04:09.43Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/f2/c26d82a7ce1e90b8bbb8731d3d53de73814e2f6606b9db9d978303aa8d5f/pyodbc-5.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3f1bdb3ce6480a17afaaef4b5242b356d4997a872f39e96f015cabef00613797", size = 73513, upload-time = "2025-10-17T18:03:40.536Z" }, + { url = "https://files.pythonhosted.org/packages/82/d5/1ab1b7c4708cbd701990a8f7183c5bb5e0712d5e8479b919934e46dadab4/pyodbc-5.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7713c740a10f33df3cb08f49a023b7e1e25de0c7c99650876bbe717bc95ee780", size = 72631, upload-time = "2025-10-17T18:03:41.713Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f1/7e3831eeac2b09b31a77e6b3495491ce162035ff2903d7261b49d35aa3c2/pyodbc-5.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf18797a12e70474e1b7f5027deeeccea816372497e3ff2d46b15bec2d18a0cc", size = 344580, upload-time = "2025-10-17T18:03:42.67Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a6/71d26d626a3c45951620b7ff356ec920e420f0e09b0a924123682aa5e4ab/pyodbc-5.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:08b2439500e212625471d32f8fde418075a5ddec556e095e5a4ba56d61df2dc6", size = 350224, upload-time = "2025-10-17T18:03:43.731Z" }, + { url = "https://files.pythonhosted.org/packages/93/14/f702c5e8c2d595776266934498505f11b7f1545baf21ffec1d32c258e9d3/pyodbc-5.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:729c535341bb09c476f219d6f7ab194bcb683c4a0a368010f1cb821a35136f05", size = 1301503, upload-time = "2025-10-17T18:03:45.013Z" }, + { url = "https://files.pythonhosted.org/packages/d9/b2/ad92ebdd1b5c7fec36b065e586d1d34b57881e17ba5beec5c705f1031058/pyodbc-5.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c67e7f2ce649155ea89beb54d3b42d83770488f025cf3b6f39ca82e9c598a02e", size = 1361050, upload-time = "2025-10-17T18:03:46.298Z" }, + { url = "https://files.pythonhosted.org/packages/19/40/dc84e232da07056cb5aaaf5f759ba4c874bc12f37569f7f1670fc71e7ae1/pyodbc-5.3.0-cp314-cp314-win32.whl", hash = "sha256:a48d731432abaee5256ed6a19a3e1528b8881f9cb25cb9cf72d8318146ea991b", size = 65670, upload-time = "2025-10-17T18:03:56.414Z" }, + { url = "https://files.pythonhosted.org/packages/b8/79/c48be07e8634f764662d7a279ac204f93d64172162dbf90f215e2398b0bd/pyodbc-5.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:58635a1cc859d5af3f878c85910e5d7228fe5c406d4571bffcdd281375a54b39", size = 72177, upload-time = "2025-10-17T18:03:57.296Z" }, + { url = "https://files.pythonhosted.org/packages/fc/79/e304574446b2263f428ce14df590ba52c2e0e0205e8d34b235b582b7d57e/pyodbc-5.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:754d052030d00c3ac38da09ceb9f3e240e8dd1c11da8906f482d5419c65b9ef5", size = 66668, upload-time = "2025-10-17T18:03:58.174Z" }, + { url = "https://files.pythonhosted.org/packages/43/17/f4eabf443b838a2728773554017d08eee3aca353102934a7e3ba96fb0e31/pyodbc-5.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:f927b440c38ade1668f0da64047ffd20ec34e32d817f9a60d07553301324b364", size = 75780, upload-time = "2025-10-17T18:03:47.273Z" }, + { url = "https://files.pythonhosted.org/packages/59/ea/e79e168c3d38c27d59d5d96273fd9e3c3ba55937cc944c4e60618f51de90/pyodbc-5.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:25c4cfb2c08e77bc6e82f666d7acd52f0e52a0401b1876e60f03c73c3b8aedc0", size = 75503, upload-time = "2025-10-17T18:03:48.171Z" }, + { url = "https://files.pythonhosted.org/packages/90/81/d1d7c125ec4a20e83fdc28e119b8321192b2bd694f432cf63e1199b2b929/pyodbc-5.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc834567c2990584b9726cba365834d039380c9dbbcef3030ddeb00c6541b943", size = 398356, upload-time = "2025-10-17T18:03:49.131Z" }, + { url = "https://files.pythonhosted.org/packages/5e/fc/f6be4b3cc3910f8c2aba37aa41671121fd6f37b402ae0fefe53a70ac7cd5/pyodbc-5.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8339d3094858893c1a68ee1af93efc4dff18b8b65de54d99104b99af6306320d", size = 397291, upload-time = "2025-10-17T18:03:50.18Z" }, + { url = "https://files.pythonhosted.org/packages/03/2e/0610b1ed05a5625528d52f6cece9610e84617d35f475c89c2a52f66d13f7/pyodbc-5.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74528fe148980d0c735c0ebb4a4dc74643ac4574337c43c1006ac4d09593f92d", size = 1353900, upload-time = "2025-10-17T18:03:51.339Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f1/43497e1d37f9f71b43b2b3172e7b1bdf50851e278390c3fb6b46a3630c53/pyodbc-5.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d89a7f2e24227150c13be8164774b7e1f9678321a4248f1356a465b9cc17d31e", size = 1406062, upload-time = "2025-10-17T18:03:52.546Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/88a1277c2f7d9ab1cec0a71e074ba24fd4a1710a43974682546da90a1343/pyodbc-5.3.0-cp314-cp314t-win32.whl", hash = "sha256:af4d8c9842fc4a6360c31c35508d6594d5a3b39922f61b282c2b4c9d9da99514", size = 70132, upload-time = "2025-10-17T18:03:53.715Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c7/ee98c62050de4aa8bafb6eb1e11b95e0b0c898bd5930137c6dc776e06a9b/pyodbc-5.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:bfeb3e34795d53b7d37e66dd54891d4f9c13a3889a8f5fe9640e56a82d770955", size = 79452, upload-time = "2025-10-17T18:03:54.664Z" }, + { url = "https://files.pythonhosted.org/packages/4b/8f/d8889efd96bbe8e5d43ff9701f6b1565a8e09c3e1f58c388d550724f777b/pyodbc-5.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:13656184faa3f2d5c6f19b701b8f247342ed581484f58bf39af7315c054e69db", size = 70142, upload-time = "2025-10-17T18:03:55.551Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + [[package]] name = "pytz" version = "2025.2" @@ -325,6 +440,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] +[[package]] +name = "zstandard" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/5c/f8923b595b55fe49e30612987ad8bf053aef555c14f05bb659dd5dbe3e8a/zstandard-0.25.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3", size = 795887, upload-time = "2025-09-14T22:17:54.198Z" }, + { url = "https://files.pythonhosted.org/packages/8d/09/d0a2a14fc3439c5f874042dca72a79c70a532090b7ba0003be73fee37ae2/zstandard-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f", size = 640658, upload-time = "2025-09-14T22:17:55.423Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7c/8b6b71b1ddd517f68ffb55e10834388d4f793c49c6b83effaaa05785b0b4/zstandard-0.25.0-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c", size = 5379849, upload-time = "2025-09-14T22:17:57.372Z" }, + { url = "https://files.pythonhosted.org/packages/a4/86/a48e56320d0a17189ab7a42645387334fba2200e904ee47fc5a26c1fd8ca/zstandard-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439", size = 5058095, upload-time = "2025-09-14T22:17:59.498Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ad/eb659984ee2c0a779f9d06dbfe45e2dc39d99ff40a319895df2d3d9a48e5/zstandard-0.25.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043", size = 5551751, upload-time = "2025-09-14T22:18:01.618Z" }, + { url = "https://files.pythonhosted.org/packages/61/b3/b637faea43677eb7bd42ab204dfb7053bd5c4582bfe6b1baefa80ac0c47b/zstandard-0.25.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859", size = 6364818, upload-time = "2025-09-14T22:18:03.769Z" }, + { url = "https://files.pythonhosted.org/packages/31/dc/cc50210e11e465c975462439a492516a73300ab8caa8f5e0902544fd748b/zstandard-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0", size = 5560402, upload-time = "2025-09-14T22:18:05.954Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ae/56523ae9c142f0c08efd5e868a6da613ae76614eca1305259c3bf6a0ed43/zstandard-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7", size = 4955108, upload-time = "2025-09-14T22:18:07.68Z" }, + { url = "https://files.pythonhosted.org/packages/98/cf/c899f2d6df0840d5e384cf4c4121458c72802e8bda19691f3b16619f51e9/zstandard-0.25.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2", size = 5269248, upload-time = "2025-09-14T22:18:09.753Z" }, + { url = "https://files.pythonhosted.org/packages/1b/c0/59e912a531d91e1c192d3085fc0f6fb2852753c301a812d856d857ea03c6/zstandard-0.25.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344", size = 5430330, upload-time = "2025-09-14T22:18:11.966Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/7e31db1240de2df22a58e2ea9a93fc6e38cc29353e660c0272b6735d6669/zstandard-0.25.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c", size = 5811123, upload-time = "2025-09-14T22:18:13.907Z" }, + { url = "https://files.pythonhosted.org/packages/f6/49/fac46df5ad353d50535e118d6983069df68ca5908d4d65b8c466150a4ff1/zstandard-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088", size = 5359591, upload-time = "2025-09-14T22:18:16.465Z" }, + { url = "https://files.pythonhosted.org/packages/c2/38/f249a2050ad1eea0bb364046153942e34abba95dd5520af199aed86fbb49/zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12", size = 444513, upload-time = "2025-09-14T22:18:20.61Z" }, + { url = "https://files.pythonhosted.org/packages/3a/43/241f9615bcf8ba8903b3f0432da069e857fc4fd1783bd26183db53c4804b/zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2", size = 516118, upload-time = "2025-09-14T22:18:17.849Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ef/da163ce2450ed4febf6467d77ccb4cd52c4c30ab45624bad26ca0a27260c/zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d", size = 476940, upload-time = "2025-09-14T22:18:19.088Z" }, +] + [[package]] name = "zstd" version = "1.5.7.3"