From e218aafc26583609cc8ef0a894ed95b26d5d8e94 Mon Sep 17 00:00:00 2001 From: Ankit Malik Date: Mon, 22 Jun 2026 16:03:00 +0530 Subject: [PATCH] first commit --- .gitignore | 10 + .python-version | 1 + Dockerfile | 27 + Pipeline_config.yml | 5 + README.md | 0 clickhouse_task/create_table.py | 54 + clickhouse_task/delete_task.py | 185 ++ clickhouse_task/load_table.py | 32 + config.yml | 4 + dags/sqlserver_clickhouse.py | 21 + db_con/.env | 12 + db_con/connection.py | 90 + db_con/log.py | 24 + log.py | 24 + logs/etl_20260622.log | 1989 +++++++++++++++++++ main2.py | 313 +++ mids.py | 74 + pyproject.toml | 15 + src/OQAD.py | 0 src/bridge.py | 110 + src/dim.py | 138 ++ src/fact.py | 723 +++++++ src/sql/bridge/mapping_storevisibility.sql | 11 + src/sql/dim/Employee_Master.sql | 17 + src/sql/dim/Master_Salesterritorylayer.sql | 14 + src/sql/dim/Master_VisibilityDefinition.sql | 5 + src/sql/dim/Master_VisibilityReason.sql | 7 + src/sql/dim/SKU_Master.sql | 18 + src/sql/dim/Store_Master.sql | 14 + src/sql/dim/coverage_remarks.sql | 8 + src/sql/dim/display_master.sql | 8 + src/sql/fact/Attendance.sql | 134 ++ src/sql/fact/Coverage.sql | 108 + src/sql/fact/Journey_Plan.sql | 10 + src/sql/fact/Login.sql | 81 + src/sql/fact/OQaD.sql | 81 + src/sql/fact/PaidVisibility.sql | 56 + src/sql/fact/Promotion.sql | 43 + src/sql/fact/SOS_OneApp.sql | 70 + src/sql/fact/Stock_Details.sql | 29 + src/sql/fact/Survey.sql | 68 + src/sql/fact/Web_Logins.sql | 23 + src/sql/fact/additional_visibility.sql | 31 + t.yml | 107 + uv.lock | 513 +++++ y.yml | 109 + 46 files changed, 5416 insertions(+) create mode 100644 .gitignore create mode 100644 .python-version create mode 100644 Dockerfile create mode 100644 Pipeline_config.yml create mode 100644 README.md create mode 100644 clickhouse_task/create_table.py create mode 100644 clickhouse_task/delete_task.py create mode 100644 clickhouse_task/load_table.py create mode 100644 config.yml create mode 100644 dags/sqlserver_clickhouse.py create mode 100644 db_con/.env create mode 100644 db_con/connection.py create mode 100644 db_con/log.py create mode 100644 log.py create mode 100644 logs/etl_20260622.log create mode 100644 main2.py create mode 100644 mids.py create mode 100644 pyproject.toml create mode 100644 src/OQAD.py create mode 100644 src/bridge.py create mode 100644 src/dim.py create mode 100644 src/fact.py create mode 100644 src/sql/bridge/mapping_storevisibility.sql create mode 100644 src/sql/dim/Employee_Master.sql create mode 100644 src/sql/dim/Master_Salesterritorylayer.sql create mode 100644 src/sql/dim/Master_VisibilityDefinition.sql create mode 100644 src/sql/dim/Master_VisibilityReason.sql create mode 100644 src/sql/dim/SKU_Master.sql create mode 100644 src/sql/dim/Store_Master.sql create mode 100644 src/sql/dim/coverage_remarks.sql create mode 100644 src/sql/dim/display_master.sql create mode 100644 src/sql/fact/Attendance.sql create mode 100644 src/sql/fact/Coverage.sql create mode 100644 src/sql/fact/Journey_Plan.sql create mode 100644 src/sql/fact/Login.sql create mode 100644 src/sql/fact/OQaD.sql create mode 100644 src/sql/fact/PaidVisibility.sql create mode 100644 src/sql/fact/Promotion.sql create mode 100644 src/sql/fact/SOS_OneApp.sql create mode 100644 src/sql/fact/Stock_Details.sql create mode 100644 src/sql/fact/Survey.sql create mode 100644 src/sql/fact/Web_Logins.sql create mode 100644 src/sql/fact/additional_visibility.sql create mode 100644 t.yml create mode 100644 uv.lock create mode 100644 y.yml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..505a3b1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +# Python-generated files +__pycache__/ +*.py[oc] +build/ +dist/ +wheels/ +*.egg-info + +# Virtual environments +.venv diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..6324d40 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.14 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3e30da5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM apache/airflow:3.2.1-python3.14 + +USER root + +RUN apt-get update && apt-get install -y \ + curl \ + gcc \ + g++ \ + unixodbc-dev \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -LsSf https://astral.sh/uv/install.sh | sh + +ENV PATH="/root/.local/bin:${PATH}" + +USER airflow + +WORKDIR /opt/airflow/project + +COPY pyproject.toml . +COPY uv.lock . + +RUN uv sync --frozen + +COPY . . + +ENV PATH="/opt/airflow/project/.venv/bin:${PATH}" \ No newline at end of file diff --git a/Pipeline_config.yml b/Pipeline_config.yml new file mode 100644 index 0000000..fc5e27a --- /dev/null +++ b/Pipeline_config.yml @@ -0,0 +1,5 @@ +pipeline: + error_message: null + run_date: null + status: null + last_successful_run_date: '2026-06-22' diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/clickhouse_task/create_table.py b/clickhouse_task/create_table.py new file mode 100644 index 0000000..8b1db76 --- /dev/null +++ b/clickhouse_task/create_table.py @@ -0,0 +1,54 @@ +import polars as pl +from sqlalchemy import text + +from log import * + + +def create_clickhouse_table( + df: pl.DataFrame, + table_name: str, + clickhouse_engine +): + type_mapping = { + pl.Int8: "Nullable(Int8)", + pl.Int16: "Nullable(Int16)", + pl.Int32: "Nullable(Int32)", + pl.Int64: "Nullable(Int64)", + pl.UInt8: "Nullable(UInt8)", + pl.UInt16: "Nullable(UInt16)", + pl.UInt32: "Nullable(UInt32)", + pl.UInt64: "Nullable(UInt64)", + pl.Float32: "Nullable(Float32)", + pl.Float64: "Nullable(Float64)", + pl.Boolean: "Nullable(Bool)", + pl.String: "Nullable(String)", + pl.Date: "Nullable(Date)", + pl.Datetime: "Nullable(DateTime)", + } + + columns = [] + + for col_name, dtype in df.schema.items(): + + clickhouse_type = type_mapping.get( + dtype, + "Nullable(String)" + ) + + columns.append( + f"`{col_name}` {clickhouse_type}" + ) + + create_sql = f""" + CREATE TABLE IF NOT EXISTS {table_name} + ( + {', '.join(columns)} + ) + ENGINE = MergeTree() + ORDER BY tuple() + """ + + with clickhouse_engine.begin() as conn: + conn.execute(text(create_sql)) + + log.info(f"Table ready: {table_name}") diff --git a/clickhouse_task/delete_task.py b/clickhouse_task/delete_task.py new file mode 100644 index 0000000..2f289df --- /dev/null +++ b/clickhouse_task/delete_task.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import polars as pl +from datetime import date, datetime, timedelta + +from log import log + + +def truncate_table( + client, + table_name: str, +) -> None: + """ + Full refresh tables. + """ + + client.command( + f"TRUNCATE TABLE {table_name}" + ) + + log.info( + "Truncated table %s", + table_name, + ) + + +def delete_rows( + client, + table_name: str, + where_clause: str, +) -> None: + """ + Generic ClickHouse delete. + """ + + query = f""" + ALTER TABLE {table_name} + DELETE + WHERE {where_clause} + """ + + log.info(f"_ _ _ _ Deleting Data from ClickHouse for {table_name} _ _ _ _") + + client.command(query) + + +def delete_existing_data( + client, + table_name: str, + run_date, + mids: list[int], + emp_visit_df: pl.DataFrame, +) -> None: + """ + Incremental delete logic. + Matches the old SQL procedure. + """ + + # -------------------------------------------------- + # MID based tables + # -------------------------------------------------- + + mid_tables = { + "additional_visibility", + "Coverage", + "Survey", + "Promotion", + "PaidVisibility", + "SOS_OneApp", + "Stock_Details", + } + + if table_name in mid_tables and mids : + + mids_str = ",".join(str(mid) for mid in mids) + + delete_rows( + client, + table_name, + f"Mid IN ({mids_str})", + ) + + return + + # -------------------------------------------------- + # Journey Plan + # -------------------------------------------------- + + if table_name == "Journey_Plan": + + delete_rows( + client, + table_name, + f""" + project_id = 40148 + AND toMonth(visit_date) = toMonth(toDate('2026-06-18')) + AND toYear(visit_date) = toYear(toDate('2026-06-18')) + """, + ) + + return + + # -------------------------------------------------- + # Logins + # -------------------------------------------------- + if table_name == "Login": + delete_rows( + client, + table_name, + f""" + toDate(login_date) = toDate('{run_date}') + """ + ) + + + # -------------------------------------------------- + # Web Logins + # -------------------------------------------------- + + + + + + if table_name == "Web_Logins": + + delete_rows( + client, + table_name, + f""" + toDate(date) + = toDate('{run_date}') + """, + ) + + return + + # -------------------------------------------------- + # Attendance + # -------------------------------------------------- + + if table_name == "Attendance": + + delete_rows( + client, + table_name, + f""" + toDate(visit_date) BETWEEN toDate('{run_date - timedelta(days=15) }') AND toDate('{run_date}') + = + """, + ) + + return + + # -------------------------------------------------- + # OQaD + # -------------------------------------------------- + + if ( + table_name == "OQaD" + and not emp_visit_df.is_empty() + ): + + conditions = [ + ( + f"(employee_id={row['EmpId']} " + f"AND toDate(visit_date)=" + f"toDate('{row['VisitDate']}'))" + ) + for row in emp_visit_df.iter_rows( + named=True + ) + ] + + delete_rows( + client, + table_name, + " OR ".join(conditions), + ) + + return + + log.info( + "No delete logic required for %s", + table_name, + ) \ No newline at end of file diff --git a/clickhouse_task/load_table.py b/clickhouse_task/load_table.py new file mode 100644 index 0000000..b7ace82 --- /dev/null +++ b/clickhouse_task/load_table.py @@ -0,0 +1,32 @@ +from sqlalchemy import create_engine, text +from sqlalchemy.engine import URL, Engine +import os +import clickhouse_connect +import polars as pd +import pyarrow +from log import log + + +def load_to_clickhouse( + client: Client, + table_name: str, + df: pl.DataFrame, +) -> None: + """ + Load a Polars DataFrame into ClickHouse using Arrow. + """ + + if df.is_empty(): + log.warning(f"{table_name}: DataFrame is empty. Skipping.") + return + + arrow_table = df.to_arrow() + + client.insert_arrow( + table=table_name, + arrow_table=arrow_table, + ) + + log.info( + f"{table_name}: inserted {len(df):,} rows into ClickHouse" + ) \ No newline at end of file diff --git a/config.yml b/config.yml new file mode 100644 index 0000000..64903c8 --- /dev/null +++ b/config.yml @@ -0,0 +1,4 @@ +pipeline: + error_message: null + run_date: null + status: null diff --git a/dags/sqlserver_clickhouse.py b/dags/sqlserver_clickhouse.py new file mode 100644 index 0000000..a4f1cd9 --- /dev/null +++ b/dags/sqlserver_clickhouse.py @@ -0,0 +1,21 @@ +from airflow import DAG +from airflow.operators.bash import BashOperator +from datetime import datetime, timedelta + +with DAG( + dag_id="sqlserver_clickhouse", + start_date=datetime(2023, 1, 1), + schedule="@daily", + catchup=True, # Important + max_active_runs=1, # Process one date at a time +): + + run_pipeline = BashOperator( + task_id="run_pipeline", + bash_command=""" + cd /opt/airflow/project && + uv run main.py {{ ds }} + """, + retries=3, + retry_delay=timedelta(minutes=5), + ) \ No newline at end of file diff --git a/db_con/.env b/db_con/.env new file mode 100644 index 0000000..f02885b --- /dev/null +++ b/db_con/.env @@ -0,0 +1,12 @@ +# SQL Server +MSSQL_SERVER=43.242.212.54,21443 +MSSQL_DB=OneApp_KelloggsMT +MSSQL_USER=ankit_malik +MSSQL_PASS=M4l!k#Ank1t001 + +# ClickHouse +CH_HOST=172.188.12.194 +CH_PORT=8123 +CH_USER=default +CH_PASS=dipanshu_k +CH_DB=kelloggs_1 \ No newline at end of file diff --git a/db_con/connection.py b/db_con/connection.py new file mode 100644 index 0000000..2c1a698 --- /dev/null +++ b/db_con/connection.py @@ -0,0 +1,90 @@ +from dotenv import load_dotenv +from sqlalchemy import create_engine, text +from sqlalchemy.engine import URL, Engine +import os +import clickhouse_connect + +from log import * + +load_dotenv() + +# SQL Server +MSSQL_SERVER = os.getenv("MSSQL_SERVER") +MSSQL_DB = os.getenv("MSSQL_DB") +MSSQL_USER = os.getenv("MSSQL_USER") +MSSQL_PASS = os.getenv("MSSQL_PASS") + +# ClickHouse +CH_HOST = os.getenv("CH_HOST") +CH_PORT = int(os.getenv("CH_PORT", "8123")) +CH_USER = os.getenv("CH_USER") +CH_PASS = os.getenv("CH_PASS") +CH_DB = os.getenv("CH_DB") + + +def build_sql_server_engine() -> Engine: + odbc_dsn = ( + f"DRIVER={{ODBC Driver 18 for SQL Server}};" + f"SERVER={MSSQL_SERVER};" + f"DATABASE={MSSQL_DB};" + f"UID={MSSQL_USER};" + f"PWD={MSSQL_PASS};" + "TrustServerCertificate=yes;" + "Connect Timeout=30;" + "Query Timeout=0;" + "Keep Alive=30;" + ) + + sql_url = URL.create( + drivername="mssql+pyodbc", + query={"odbc_connect": odbc_dsn}, + ) + + engine = create_engine( + sql_url, + pool_size=3, + max_overflow=2, + pool_pre_ping=True, + connect_args={ + "fast_executemany": True, + }, + ) + + with engine.connect() as conn: + + log.info(conn.execute(text("SELECT 1"))) + + return engine + + +def build_clickhouse_engine() -> Engine: + clickhouse_url = URL.create( + drivername="clickhouse+http", + username=CH_USER, + password=CH_PASS, + host=CH_HOST, + port=CH_PORT, + database=CH_DB, + ) + + engine = create_engine( + clickhouse_url, + pool_pre_ping=True, + ) + + with engine.connect() as conn: + log.info( conn.execute(text("SELECT 1"))) + + return engine + + + + +def get_clickhouse_client(): + return clickhouse_connect.get_client( + host=CH_HOST, + port=CH_PORT, + username=CH_USER, + password=CH_PASS, + database=CH_DB + ) \ No newline at end of file diff --git a/db_con/log.py b/db_con/log.py new file mode 100644 index 0000000..11f11e0 --- /dev/null +++ b/db_con/log.py @@ -0,0 +1,24 @@ + +from pathlib import Path +from datetime import datetime +import logging + +# Create logs folder if it doesn't exist +Path("logs").mkdir(exist_ok=True) + +# Daily log file +log_file = Path("logs") / f"etl_{datetime.now():%Y%m%d}.log" + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)-8s | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + handlers=[ + logging.StreamHandler(), + logging.FileHandler(log_file, encoding="utf-8"), + ], +) + +# Export logger +log = logging.getLogger("etl") \ No newline at end of file diff --git a/log.py b/log.py new file mode 100644 index 0000000..11f11e0 --- /dev/null +++ b/log.py @@ -0,0 +1,24 @@ + +from pathlib import Path +from datetime import datetime +import logging + +# Create logs folder if it doesn't exist +Path("logs").mkdir(exist_ok=True) + +# Daily log file +log_file = Path("logs") / f"etl_{datetime.now():%Y%m%d}.log" + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(levelname)-8s | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + handlers=[ + logging.StreamHandler(), + logging.FileHandler(log_file, encoding="utf-8"), + ], +) + +# Export logger +log = logging.getLogger("etl") \ No newline at end of file diff --git a/logs/etl_20260622.log b/logs/etl_20260622.log new file mode 100644 index 0000000..8907576 --- /dev/null +++ b/logs/etl_20260622.log @@ -0,0 +1,1989 @@ +2026-06-22 12:12:58 | INFO | ================================================================================ +2026-06-22 12:12:58 | INFO | Hello from data-move Python data pipeline! +2026-06-22 12:12:58 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 12:12:58 | INFO | Connecting to databases... +2026-06-22 12:12:59 | INFO | +2026-06-22 12:13:03 | INFO | +2026-06-22 12:13:03 | INFO | Database connections established +2026-06-22 12:13:03 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 12:13:04 | INFO | Found 569 MIDs +2026-06-22 12:13:05 | INFO | ================================================================================ +2026-06-22 12:13:05 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 12:13:05 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-22 12:13:05 | INFO | Fetching data from sql server for Master table...... +2026-06-22 12:13:05 | INFO | Fetching in progress .... +2026-06-22 12:13:05 | INFO | Fetched 17 rows from SQL Server +2026-06-22 12:13:05 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-22 12:13:05 | INFO | Fetched 17 rows +2026-06-22 12:13:05 | INFO | Truncated table Master_VisibilityReason +2026-06-22 12:13:05 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:13:05 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-22 12:13:05 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-22 12:13:05 | INFO | ================================================================================ +2026-06-22 12:13:05 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 12:13:05 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. +2026-06-22 12:13:05 | INFO | Fetching data from sql server for Master table...... +2026-06-22 12:13:05 | INFO | Fetching in progress .... +2026-06-22 12:13:05 | INFO | Fetched 861 rows from SQL Server +2026-06-22 12:13:05 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! +2026-06-22 12:13:05 | INFO | Fetched 861 rows +2026-06-22 12:13:05 | INFO | Truncated table Master_VisibilityDefinition +2026-06-22 12:13:05 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:13:06 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-22 12:13:06 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-22 12:13:06 | INFO | ================================================================================ +2026-06-22 12:13:06 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 12:13:06 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. +2026-06-22 12:13:06 | INFO | Fetching data from sql server for Master table...... +2026-06-22 12:13:06 | INFO | Fetching in progress .... +2026-06-22 12:13:06 | INFO | Fetched 33 rows from SQL Server +2026-06-22 12:13:06 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! +2026-06-22 12:13:06 | INFO | Fetched 33 rows +2026-06-22 12:13:06 | INFO | Truncated table Master_Salesterritorylayer +2026-06-22 12:13:06 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:13:06 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-22 12:13:06 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-22 12:13:06 | INFO | ================================================================================ +2026-06-22 12:13:06 | INFO | Pipeline Completed Successfully +2026-06-22 12:13:06 | INFO | ================================================================================ +2026-06-22 12:16:47 | INFO | ================================================================================ +2026-06-22 12:16:47 | INFO | Hello from data-move Python data pipeline! +2026-06-22 12:16:47 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 12:16:47 | INFO | Connecting to databases... +2026-06-22 12:16:48 | INFO | +2026-06-22 12:16:50 | INFO | +2026-06-22 12:16:52 | INFO | Database connections established +2026-06-22 12:16:52 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 12:16:52 | INFO | Found 569 MIDs +2026-06-22 12:16:53 | INFO | ================================================================================ +2026-06-22 12:16:53 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:16:53 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 12:16:53 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:16:53 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:16:53 | INFO | Fetching data for 569 MIDs +2026-06-22 12:16:57 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 12:16:57 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 12:16:57 | INFO | Fetched 1907 rows +2026-06-22 12:16:57 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 12:16:57 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:16:58 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 12:16:58 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 12:16:58 | INFO | ================================================================================ +2026-06-22 12:16:58 | INFO | Processing Table: OQaD | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:16:58 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 12:16:58 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 12:17:14 | INFO | Fetched 291,134 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 12:17:14 | WARNING | Table 'OQaD' does not exist. +2026-06-22 12:17:14 | INFO | Fetched 0 matched empids fetched for OQAD +2026-06-22 12:17:14 | INFO | Exists: True +2026-06-22 12:17:14 | INFO | Path: D:\z\data_move\src\sql\fact\OQaD.sql +2026-06-22 12:17:14 | INFO | Fetching data for 0 EMPIDs +2026-06-22 12:17:14 | INFO | Fetching OQaD data for run_date=2026-06-21 +2026-06-22 12:17:14 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ')'. (102) (SQLExecDirectW)") + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "D:\z\data_move\main2.py", line 159, in main + df=fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\z\data_move\src\fact.py", line 285, in fetch_OQaD + df=fetch_data( engine=sql_engine, + table_name=table_name, + ...<2 lines>... + run_date=run_date + ) + File "D:\z\data_move\src\fact.py", line 277, in fetch_data + df = pl.read_database( + query=sql, + connection=engine, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database + return cx.execute( + ~~~~~~~~~~^ + query=query, + ^^^^^^^^^^^^ + options=execute_options, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ).to_polars( + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute + result = cursor_execute(query, **options) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute + return meth( + self, + distilled_parameters, + execution_options or NO_OPTIONS, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection + return connection._execute_clauseelement( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + self, distilled_params, execution_options + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement + ret = self._execute_context( + dialect, + ...<8 lines>... + cache_hit=cache_hit, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context + return self._exec_single_context( + ~~~~~~~~~~~~~~~~~~~~~~~~~^ + dialect, context, statement, parameters + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context + self._handle_dbapi_exception( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + e, str_statement, effective_parameters, cursor, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-bb41b3d8f97c187a\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near ')'. (102) (SQLExecDirectW)") +[SQL: WITH MID_TABLE_COV1 AS +( + SELECT EmpId, VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CreateDate >= 2026-06-21 + AND CreateDate < DATEADD(DAY,1,'2026-06-21') + + UNION ALL + + SELECT EmpId, VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE UpdateDate >= 2026-06-21 + AND UpdateDate < DATEADD(DAY,1, '2026-06-21') +), + + +QUIZ AS +( + SELECT DISTINCT + E.EmpId, + E.EmpName, + E.SupervisorId, + E.SupervisorName, + E.DesignationName, + E.CityName, + E.StateName, + E.RegionName, + CAST(DQ.VisitDate AS DATE) AS VisitDate, + DQ.QuestionId, + DQ.AnswerId, + QC.QuestionCategoryId, + QC.QuestionCategory + FROM OneApp_KelloggsMT.dbo.T_OQAD DQ + INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E + ON DQ.EmpId = E.EmpId + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU + ON DQ.QuestionId = QU.QuestionId + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC + ON QU.QuestionCategoryId = QC.QuestionCategoryId + WHERE E.EmpName NOT LIKE '%TEST%' + AND E.RightId = 6 + AND ( + E.ResignDate IS NULL + OR CAST(E.ResignDate AS DATE) >= '2026-06-21' + ) + AND EXISTS + ( + SELECT 1 + FROM MID_TABLE_COV1 A + WHERE A.EmpId = DQ.EmpId + AND CAST(A.VisitDate AS DATE) = CAST(DQ.VisitDate AS DATE) + ) +) + +SELECT + 40148 AS project_id, + Q.EmpId AS employee_id, + 0 AS process_id, + Q.VisitDate AS visit_date, + Q.QuestionCategoryId AS question_category_id, + Q.QuestionCategory AS question_category, + QM.QuestionId AS question_id, + QM.Question AS question, + ISNULL(QA.AnswerId,0) AS answer_id, + ISNULL(QA.Answer,'') AS answer, + CASE + WHEN QA.AnswerId IS NULL THEN 'Not Answer' + WHEN QA.RightAnswer = 1 THEN 'Y' + WHEN QA.RightAnswer IS NULL THEN 'Not Answer' + ELSE 'N' + END AS correct_answer, + GETDATE() AS update_date, + 'SP-Pius' AS update_by +FROM QUIZ Q +INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM + ON Q.QuestionId = QM.QuestionId +LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA + ON Q.AnswerId = QA.AnswerId +where Q.EmpId not in () + + +] +(Background on this error at: https://sqlalche.me/e/20/f405) +2026-06-22 12:27:42 | INFO | ================================================================================ +2026-06-22 12:27:42 | INFO | Hello from data-move Python data pipeline! +2026-06-22 12:27:42 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 12:27:42 | INFO | Connecting to databases... +2026-06-22 12:27:43 | INFO | +2026-06-22 12:27:44 | INFO | +2026-06-22 12:27:44 | INFO | Database connections established +2026-06-22 12:27:44 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 12:27:45 | INFO | Found 569 MIDs +2026-06-22 12:27:45 | INFO | ================================================================================ +2026-06-22 12:27:45 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:27:45 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 12:27:45 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:27:45 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:27:45 | INFO | Fetching data for 569 MIDs +2026-06-22 12:27:50 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 12:27:50 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 12:27:50 | INFO | Fetched 1907 rows +2026-06-22 12:27:50 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 12:27:51 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:27:52 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 12:27:52 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 12:27:52 | INFO | ================================================================================ +2026-06-22 12:27:52 | INFO | Processing Table: OQaD | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:27:52 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 12:27:53 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 12:28:12 | INFO | Fetched 291,145 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 12:28:12 | WARNING | Table 'OQaD' does not exist. +2026-06-22 12:28:12 | WARNING | OQaD Matched df in OQaD returned no rows +2026-06-22 12:28:12 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "D:\z\data_move\main2.py", line 159, in main + df=fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\z\data_move\src\fact.py", line 257, in fetch_OQaD + log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ") + ^^^^^^ +UnboundLocalError: cannot access local variable 'empids' where it is not associated with a value +2026-06-22 12:28:18 | INFO | ================================================================================ +2026-06-22 12:28:18 | INFO | Hello from data-move Python data pipeline! +2026-06-22 12:28:18 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 12:28:18 | INFO | Connecting to databases... +2026-06-22 12:28:19 | INFO | +2026-06-22 12:28:19 | INFO | +2026-06-22 12:28:20 | INFO | Database connections established +2026-06-22 12:28:20 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 12:28:20 | INFO | Found 569 MIDs +2026-06-22 12:28:20 | INFO | ================================================================================ +2026-06-22 12:28:20 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:28:20 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 12:28:20 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:28:20 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:28:20 | INFO | Fetching data for 569 MIDs +2026-06-22 12:28:25 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 12:28:25 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 12:28:25 | INFO | Fetched 1907 rows +2026-06-22 12:28:25 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 12:28:25 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:28:25 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 12:28:25 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 12:28:25 | INFO | ================================================================================ +2026-06-22 12:28:25 | INFO | Processing Table: OQaD | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:28:25 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 12:28:25 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 12:28:47 | INFO | Fetched 291,145 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 12:28:47 | WARNING | Table 'OQaD' does not exist. +2026-06-22 12:28:47 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "D:\z\data_move\main2.py", line 159, in main + df=fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\z\data_move\src\fact.py", line 248, in fetch_OQaD + log.warning( + ~~~~~~~~~~~^ + "%s Matched df in OQaD returned no rows", + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + empids=[0] + ^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Roaming\uv\python\cpython-3.14-windows-x86_64-none\Lib\logging\__init__.py", line 1532, in warning + self._log(WARNING, msg, args, **kwargs) + ~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +TypeError: Logger._log() got an unexpected keyword argument 'empids' +2026-06-22 12:29:11 | INFO | ================================================================================ +2026-06-22 12:29:11 | INFO | Hello from data-move Python data pipeline! +2026-06-22 12:29:11 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 12:29:11 | INFO | Connecting to databases... +2026-06-22 12:29:12 | INFO | +2026-06-22 12:29:13 | INFO | +2026-06-22 12:29:13 | INFO | Database connections established +2026-06-22 12:29:13 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 12:29:14 | INFO | Found 569 MIDs +2026-06-22 12:29:14 | INFO | ================================================================================ +2026-06-22 12:29:14 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:29:14 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 12:29:14 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:29:14 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:29:14 | INFO | Fetching data for 569 MIDs +2026-06-22 12:29:15 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 12:29:15 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 12:29:15 | INFO | Fetched 1907 rows +2026-06-22 12:29:15 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 12:29:15 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:29:15 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 12:29:15 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 12:29:15 | INFO | ================================================================================ +2026-06-22 12:29:15 | INFO | Processing Table: OQaD | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:29:15 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 12:29:16 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 12:29:27 | INFO | Fetched 291,145 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 12:29:27 | WARNING | Table 'OQaD' does not exist. +2026-06-22 12:29:27 | ERROR | Failed processing table OQaD +Traceback (most recent call last): + File "D:\z\data_move\main2.py", line 159, in main + df=fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\z\data_move\src\fact.py", line 248, in fetch_OQaD + log.warning( + ~~~~~~~~~~~^ + "%s Matched df in OQaD returned no rows", + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + empids=[0] + ^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Roaming\uv\python\cpython-3.14-windows-x86_64-none\Lib\logging\__init__.py", line 1532, in warning + self._log(WARNING, msg, args, **kwargs) + ~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +TypeError: Logger._log() got an unexpected keyword argument 'empids' +2026-06-22 12:30:00 | INFO | ================================================================================ +2026-06-22 12:30:00 | INFO | Hello from data-move Python data pipeline! +2026-06-22 12:30:00 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 12:30:00 | INFO | Connecting to databases... +2026-06-22 12:30:02 | INFO | +2026-06-22 12:30:03 | INFO | +2026-06-22 12:30:03 | INFO | Database connections established +2026-06-22 12:30:03 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 12:30:04 | INFO | Found 569 MIDs +2026-06-22 12:30:04 | INFO | ================================================================================ +2026-06-22 12:30:04 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:30:04 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 12:30:04 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:30:04 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:30:04 | INFO | Fetching data for 569 MIDs +2026-06-22 12:30:06 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 12:30:06 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 12:30:06 | INFO | Fetched 1907 rows +2026-06-22 12:30:06 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 12:30:06 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:30:06 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 12:30:06 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 12:30:06 | INFO | ================================================================================ +2026-06-22 12:30:06 | INFO | Processing Table: OQaD | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:30:06 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 12:30:07 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 12:30:24 | INFO | Fetched 291,147 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 12:30:24 | WARNING | Table 'OQaD' does not exist. +2026-06-22 12:30:24 | WARNING | OQaD Matched df in OQaD returned no rows +2026-06-22 12:30:24 | INFO | Fetched 1 matched empids fetched for OQAD +2026-06-22 12:30:24 | INFO | Exists: True +2026-06-22 12:30:24 | INFO | Path: D:\z\data_move\src\sql\fact\OQaD.sql +2026-06-22 12:30:24 | INFO | Fetching data for 1 EMPIDs +2026-06-22 12:30:24 | INFO | Fetching OQaD data for run_date=2026-06-21 +2026-06-22 12:32:08 | INFO | fn name is fetch_OQad ------Fetched 290730 rows +2026-06-22 12:32:08 | INFO | Fetched 290,730 rows from SQL Server +2026-06-22 12:32:08 | INFO | Fetched total row -: 290730 from sql server for table-:OQaD ...........!!! +2026-06-22 12:32:08 | INFO | Fetched 290730 rows +2026-06-22 12:32:08 | INFO | Creating table OQaD +2026-06-22 12:32:08 | INFO | Table ready: OQaD +2026-06-22 12:32:08 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:32:12 | INFO | OQaD: inserted 290,730 rows into ClickHouse +2026-06-22 12:32:12 | INFO | OQaD loaded successfully (290730 rows) +2026-06-22 12:32:12 | INFO | ================================================================================ +2026-06-22 12:32:12 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:32:12 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-22 12:32:12 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:32:12 | INFO | Fetching data for 569 MIDs +2026-06-22 12:32:13 | INFO | Fetched 165 rows from SQL Server +2026-06-22 12:32:13 | INFO | Fetched total row -: 165 from sql server for table-:Survey ...........!!! +2026-06-22 12:32:13 | INFO | Fetched 165 rows +2026-06-22 12:32:13 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-22 12:32:13 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:32:13 | INFO | Survey: inserted 165 rows into ClickHouse +2026-06-22 12:32:13 | INFO | Survey loaded successfully (165 rows) +2026-06-22 12:32:13 | INFO | ================================================================================ +2026-06-22 12:32:13 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:32:13 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-22 12:32:13 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:32:13 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:32:13 | INFO | Fetching data for 569 MIDs +2026-06-22 12:32:14 | INFO | Fetched 1,579 rows from SQL Server +2026-06-22 12:32:14 | INFO | Fetched total row -: 1579 from sql server for table-:additional_visibility ...........!!! +2026-06-22 12:32:14 | INFO | Fetched 1579 rows +2026-06-22 12:32:14 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-22 12:32:14 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:32:14 | INFO | additional_visibility: inserted 1,579 rows into ClickHouse +2026-06-22 12:32:14 | INFO | additional_visibility loaded successfully (1579 rows) +2026-06-22 12:32:14 | INFO | ================================================================================ +2026-06-22 12:32:14 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:32:14 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-22 12:32:14 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:32:14 | INFO | Fetching data for 569 MIDs +2026-06-22 12:32:33 | INFO | Fetched 512 rows from SQL Server +2026-06-22 12:32:33 | INFO | Fetched total row -: 512 from sql server for table-:Coverage ...........!!! +2026-06-22 12:32:33 | INFO | Fetched 512 rows +2026-06-22 12:32:33 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-22 12:32:33 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:32:33 | INFO | Coverage: inserted 512 rows into ClickHouse +2026-06-22 12:32:33 | INFO | Coverage loaded successfully (512 rows) +2026-06-22 12:32:33 | INFO | ================================================================================ +2026-06-22 12:32:33 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:32:33 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-22 12:32:33 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:32:33 | ERROR | Failed processing table Login +Traceback (most recent call last): + File "D:\z\data_move\main2.py", line 159, in main + df=fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\z\data_move\src\fact.py", line 443, in fetch_Login + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) +KeyError: 'start_date' +2026-06-22 12:43:20 | INFO | ================================================================================ +2026-06-22 12:43:20 | INFO | Hello from data-move Python data pipeline! +2026-06-22 12:43:20 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 12:43:20 | INFO | Connecting to databases... +2026-06-22 12:43:21 | INFO | +2026-06-22 12:43:22 | INFO | +2026-06-22 12:43:23 | INFO | Database connections established +2026-06-22 12:43:23 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 12:43:23 | INFO | Found 569 MIDs +2026-06-22 12:43:23 | INFO | ================================================================================ +2026-06-22 12:43:23 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:43:23 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 12:43:23 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:43:23 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:43:23 | INFO | Fetching data for 569 MIDs +2026-06-22 12:43:25 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 12:43:25 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 12:43:25 | INFO | Fetched 1907 rows +2026-06-22 12:43:25 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 12:43:25 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:43:26 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 12:43:26 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 12:43:26 | INFO | ================================================================================ +2026-06-22 12:43:26 | INFO | Processing Table: OQaD | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:43:26 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 12:43:26 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 12:43:57 | INFO | Fetched 291,158 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 12:43:59 | INFO | Fetched 290,722 matched empids fetched for OQAD +2026-06-22 12:43:59 | INFO | Exists: True +2026-06-22 12:43:59 | INFO | Path: D:\z\data_move\src\sql\fact\OQaD.sql +2026-06-22 12:43:59 | INFO | Fetching data for 290,722 EMPIDs +2026-06-22 12:43:59 | INFO | Fetching OQaD data for run_date=2026-06-21 +2026-06-22 12:44:40 | INFO | fn name is fetch_OQad ------Fetched 0 rows +2026-06-22 12:44:40 | INFO | Fetched 0 rows from SQL Server +2026-06-22 12:44:40 | INFO | Fetched total row -: 0 from sql server for table-:OQaD ...........!!! +2026-06-22 12:44:40 | WARNING | OQaD returned no rows +2026-06-22 12:44:40 | INFO | ================================================================================ +2026-06-22 12:44:40 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:44:40 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-22 12:44:40 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:44:40 | INFO | Fetching data for 569 MIDs +2026-06-22 12:44:40 | INFO | Fetched 165 rows from SQL Server +2026-06-22 12:44:40 | INFO | Fetched total row -: 165 from sql server for table-:Survey ...........!!! +2026-06-22 12:44:40 | INFO | Fetched 165 rows +2026-06-22 12:44:40 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-22 12:44:40 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:44:41 | INFO | Survey: inserted 165 rows into ClickHouse +2026-06-22 12:44:41 | INFO | Survey loaded successfully (165 rows) +2026-06-22 12:44:41 | INFO | ================================================================================ +2026-06-22 12:44:41 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:44:41 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-22 12:44:41 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:44:41 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:44:41 | INFO | Fetching data for 569 MIDs +2026-06-22 12:44:41 | INFO | Fetched 1,579 rows from SQL Server +2026-06-22 12:44:41 | INFO | Fetched total row -: 1579 from sql server for table-:additional_visibility ...........!!! +2026-06-22 12:44:41 | INFO | Fetched 1579 rows +2026-06-22 12:44:41 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-22 12:44:41 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:44:42 | INFO | additional_visibility: inserted 1,579 rows into ClickHouse +2026-06-22 12:44:42 | INFO | additional_visibility loaded successfully (1579 rows) +2026-06-22 12:44:42 | INFO | ================================================================================ +2026-06-22 12:44:42 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:44:42 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-22 12:44:42 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:44:42 | INFO | Fetching data for 569 MIDs +2026-06-22 12:44:58 | INFO | Fetched 512 rows from SQL Server +2026-06-22 12:44:58 | INFO | Fetched total row -: 512 from sql server for table-:Coverage ...........!!! +2026-06-22 12:44:58 | INFO | Fetched 512 rows +2026-06-22 12:44:58 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-22 12:44:58 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:44:59 | INFO | Coverage: inserted 512 rows into ClickHouse +2026-06-22 12:44:59 | INFO | Coverage loaded successfully (512 rows) +2026-06-22 12:44:59 | INFO | ================================================================================ +2026-06-22 12:44:59 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:44:59 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-22 12:44:59 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:44:59 | INFO | Fetching data for 569 MIDs +2026-06-22 12:45:02 | INFO | Fetched 454 rows from SQL Server +2026-06-22 12:45:02 | INFO | Fetched total row -: 454 from sql server for table-:Login ...........!!! +2026-06-22 12:45:02 | INFO | Fetched 454 rows +2026-06-22 12:45:02 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-22 12:45:02 | INFO | No delete logic required for Login +2026-06-22 12:45:02 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:45:03 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-22 12:45:03 | INFO | Login loaded successfully (454 rows) +2026-06-22 12:45:03 | INFO | ================================================================================ +2026-06-22 12:45:03 | INFO | Processing Table: Stock_Details | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:45:03 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-22 12:45:03 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:45:03 | INFO | Fetching data for 569 MIDs +2026-06-22 12:45:19 | INFO | Fetched 27,359 rows from SQL Server +2026-06-22 12:45:19 | INFO | Fetched total row -: 27359 from sql server for table-:Stock_Details ...........!!! +2026-06-22 12:45:19 | INFO | Fetched 27359 rows +2026-06-22 12:45:19 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-22 12:45:19 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:45:19 | INFO | Stock_Details: inserted 27,359 rows into ClickHouse +2026-06-22 12:45:19 | INFO | Stock_Details loaded successfully (27359 rows) +2026-06-22 12:45:19 | INFO | ================================================================================ +2026-06-22 12:45:19 | INFO | Processing Table: Attendance | Table type is -: FACT | Based on -run_date and operation is used -DELETE+INSERT +2026-06-22 12:45:19 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-22 12:45:19 | INFO | Fetching Attendance data from 2026-06-06 to 2026-06-21 +2026-06-22 12:47:52 | INFO | Fetched 121,639 attendance rows for 544 employees +2026-06-22 12:47:52 | INFO | Fetched total row -: 121639 from sql server for table-:Attendance ...........!!! +2026-06-22 12:47:52 | INFO | Fetched 121639 rows +2026-06-22 12:47:52 | INFO | Truncated table Attendance +2026-06-22 12:47:52 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:47:54 | INFO | Attendance: inserted 121,639 rows into ClickHouse +2026-06-22 12:47:54 | INFO | Attendance loaded successfully (121639 rows) +2026-06-22 12:47:54 | INFO | ================================================================================ +2026-06-22 12:47:54 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:47:54 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-22 12:47:54 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:47:54 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:47:54 | INFO | Fetching data for 569 MIDs +2026-06-22 12:48:07 | INFO | Fetched 21,621 rows from SQL Server +2026-06-22 12:48:07 | INFO | Fetched total row -: 21621 from sql server for table-:Journey_Plan ...........!!! +2026-06-22 12:48:07 | INFO | Fetched 21621 rows +2026-06-22 12:48:07 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-22 12:48:07 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:48:08 | INFO | Journey_Plan: inserted 21,621 rows into ClickHouse +2026-06-22 12:48:08 | INFO | Journey_Plan loaded successfully (21621 rows) +2026-06-22 12:48:08 | INFO | ================================================================================ +2026-06-22 12:48:08 | INFO | Processing Table: Web_Logins | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 12:48:08 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-22 12:48:08 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 12:48:08 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 12:48:08 | INFO | Fetching data for 569 MIDs +2026-06-22 12:48:08 | INFO | Fetched 12 rows from SQL Server +2026-06-22 12:48:08 | INFO | Fetched total row -: 12 from sql server for table-:Web_Logins ...........!!! +2026-06-22 12:48:08 | INFO | Fetched 12 rows +2026-06-22 12:48:08 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ +2026-06-22 12:48:09 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:48:09 | INFO | Web_Logins: inserted 12 rows into ClickHouse +2026-06-22 12:48:09 | INFO | Web_Logins loaded successfully (12 rows) +2026-06-22 12:48:09 | INFO | ================================================================================ +2026-06-22 12:48:09 | INFO | Processing Table: Promotion | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 12:48:09 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-22 12:48:09 | ERROR | Failed processing table Promotion +Traceback (most recent call last): + File "D:\z\data_move\main2.py", line 158, in main + fn = globals()[fn_name] + ~~~~~~~~~^^^^^^^^^ +KeyError: 'fetch_Promotion' +2026-06-22 13:04:30 | INFO | ================================================================================ +2026-06-22 13:04:30 | INFO | Hello from data-move Python data pipeline! +2026-06-22 13:04:30 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 13:04:30 | INFO | Connecting to databases... +2026-06-22 13:04:30 | INFO | +2026-06-22 13:04:31 | INFO | +2026-06-22 13:04:32 | INFO | Database connections established +2026-06-22 13:04:32 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 13:04:32 | INFO | Found 569 MIDs +2026-06-22 13:04:32 | INFO | ================================================================================ +2026-06-22 13:04:32 | INFO | Processing Table: Promotion | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:04:32 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-22 13:04:32 | INFO | Fetching data for 569 MIDs +2026-06-22 13:04:36 | INFO | Fetched 4,223 rows from SQL Server +2026-06-22 13:04:36 | INFO | Fetched total row -: 4223 from sql server for table-:Promotion ...........!!! +2026-06-22 13:04:36 | INFO | Fetched 4223 rows +2026-06-22 13:04:36 | INFO | Creating table Promotion +2026-06-22 13:04:36 | INFO | Table ready: Promotion +2026-06-22 13:04:36 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:04:36 | INFO | Promotion: inserted 4,223 rows into ClickHouse +2026-06-22 13:04:36 | INFO | Promotion loaded successfully (4223 rows) +2026-06-22 13:04:36 | INFO | ================================================================================ +2026-06-22 13:04:36 | INFO | Pipeline Completed Successfully +2026-06-22 13:04:36 | INFO | ================================================================================ +2026-06-22 13:04:41 | INFO | ================================================================================ +2026-06-22 13:04:41 | INFO | Hello from data-move Python data pipeline! +2026-06-22 13:04:41 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 13:04:41 | INFO | Connecting to databases... +2026-06-22 13:04:42 | INFO | +2026-06-22 13:04:43 | INFO | +2026-06-22 13:04:44 | INFO | Database connections established +2026-06-22 13:04:44 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 13:04:44 | INFO | Found 569 MIDs +2026-06-22 13:04:44 | INFO | ================================================================================ +2026-06-22 13:04:44 | INFO | Processing Table: Promotion | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:04:44 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-22 13:04:44 | INFO | Fetching data for 569 MIDs +2026-06-22 13:04:54 | INFO | Fetched 4,223 rows from SQL Server +2026-06-22 13:04:54 | INFO | Fetched total row -: 4223 from sql server for table-:Promotion ...........!!! +2026-06-22 13:04:54 | INFO | Fetched 4223 rows +2026-06-22 13:04:54 | INFO | _ _ _ _ Deleting Data from ClickHouse for Promotion _ _ _ _ +2026-06-22 13:04:54 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:04:55 | INFO | Promotion: inserted 4,223 rows into ClickHouse +2026-06-22 13:04:55 | INFO | Promotion loaded successfully (4223 rows) +2026-06-22 13:04:55 | INFO | ================================================================================ +2026-06-22 13:04:55 | INFO | Pipeline Completed Successfully +2026-06-22 13:04:55 | INFO | ================================================================================ +2026-06-22 13:05:14 | INFO | ================================================================================ +2026-06-22 13:05:14 | INFO | Hello from data-move Python data pipeline! +2026-06-22 13:05:14 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 13:05:14 | INFO | Connecting to databases... +2026-06-22 13:05:15 | INFO | +2026-06-22 13:05:16 | INFO | +2026-06-22 13:05:17 | INFO | Database connections established +2026-06-22 13:05:17 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 13:05:17 | INFO | Found 569 MIDs +2026-06-22 13:05:17 | INFO | ================================================================================ +2026-06-22 13:05:17 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:05:17 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 13:05:17 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 13:05:17 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:05:17 | INFO | Fetching data for 569 MIDs +2026-06-22 13:05:19 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 13:05:19 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 13:05:19 | INFO | Fetched 1907 rows +2026-06-22 13:05:19 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 13:05:20 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:05:20 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 13:05:20 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 13:05:20 | INFO | ================================================================================ +2026-06-22 13:05:20 | INFO | Processing Table: OQaD | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 13:05:20 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 13:05:20 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 13:05:32 | INFO | Fetched 291,165 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 13:05:34 | INFO | Fetched 290,722 matched empids fetched for OQAD +2026-06-22 13:05:34 | INFO | Exists: True +2026-06-22 13:05:34 | INFO | Path: D:\z\data_move\src\sql\fact\OQaD.sql +2026-06-22 13:05:34 | INFO | Fetching data for 290,722 EMPIDs +2026-06-22 13:05:34 | INFO | Fetching OQaD data for run_date=2026-06-21 +2026-06-22 13:06:10 | INFO | fn name is fetch_OQad ------Fetched 0 rows +2026-06-22 13:06:10 | INFO | Fetched 0 rows from SQL Server +2026-06-22 13:06:10 | INFO | Fetched total row -: 0 from sql server for table-:OQaD ...........!!! +2026-06-22 13:06:10 | WARNING | OQaD returned no rows +2026-06-22 13:06:10 | INFO | ================================================================================ +2026-06-22 13:06:10 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:06:10 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-22 13:06:10 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:06:10 | INFO | Fetching data for 569 MIDs +2026-06-22 13:06:10 | INFO | Fetched 165 rows from SQL Server +2026-06-22 13:06:10 | INFO | Fetched total row -: 165 from sql server for table-:Survey ...........!!! +2026-06-22 13:06:10 | INFO | Fetched 165 rows +2026-06-22 13:06:11 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-22 13:06:11 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:06:11 | INFO | Survey: inserted 165 rows into ClickHouse +2026-06-22 13:06:11 | INFO | Survey loaded successfully (165 rows) +2026-06-22 13:06:11 | INFO | ================================================================================ +2026-06-22 13:06:11 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:06:11 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-22 13:06:11 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 13:06:11 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:06:11 | INFO | Fetching data for 569 MIDs +2026-06-22 13:06:11 | INFO | Fetched 1,579 rows from SQL Server +2026-06-22 13:06:11 | INFO | Fetched total row -: 1579 from sql server for table-:additional_visibility ...........!!! +2026-06-22 13:06:11 | INFO | Fetched 1579 rows +2026-06-22 13:06:11 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-22 13:06:12 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:06:12 | INFO | additional_visibility: inserted 1,579 rows into ClickHouse +2026-06-22 13:06:12 | INFO | additional_visibility loaded successfully (1579 rows) +2026-06-22 13:06:12 | INFO | ================================================================================ +2026-06-22 13:06:12 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:06:12 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-22 13:06:12 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:06:12 | INFO | Fetching data for 569 MIDs +2026-06-22 13:06:27 | INFO | Fetched 512 rows from SQL Server +2026-06-22 13:06:27 | INFO | Fetched total row -: 512 from sql server for table-:Coverage ...........!!! +2026-06-22 13:06:27 | INFO | Fetched 512 rows +2026-06-22 13:06:27 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-22 13:06:27 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:06:28 | INFO | Coverage: inserted 512 rows into ClickHouse +2026-06-22 13:06:28 | INFO | Coverage loaded successfully (512 rows) +2026-06-22 13:06:28 | INFO | ================================================================================ +2026-06-22 13:06:28 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 13:06:28 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-22 13:06:28 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:06:28 | INFO | Fetching data for 569 MIDs +2026-06-22 13:06:29 | INFO | Fetched 454 rows from SQL Server +2026-06-22 13:06:29 | INFO | Fetched total row -: 454 from sql server for table-:Login ...........!!! +2026-06-22 13:06:29 | INFO | Fetched 454 rows +2026-06-22 13:06:29 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-22 13:06:29 | INFO | No delete logic required for Login +2026-06-22 13:06:29 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:06:30 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-22 13:06:30 | INFO | Login loaded successfully (454 rows) +2026-06-22 13:06:30 | INFO | ================================================================================ +2026-06-22 13:06:30 | INFO | Processing Table: Stock_Details | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:06:30 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-22 13:06:30 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:06:30 | INFO | Fetching data for 569 MIDs +2026-06-22 13:06:35 | INFO | Fetched 27,359 rows from SQL Server +2026-06-22 13:06:35 | INFO | Fetched total row -: 27359 from sql server for table-:Stock_Details ...........!!! +2026-06-22 13:06:35 | INFO | Fetched 27359 rows +2026-06-22 13:06:35 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-22 13:06:35 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:06:35 | INFO | Stock_Details: inserted 27,359 rows into ClickHouse +2026-06-22 13:06:35 | INFO | Stock_Details loaded successfully (27359 rows) +2026-06-22 13:06:35 | INFO | ================================================================================ +2026-06-22 13:06:35 | INFO | Processing Table: Attendance | Table type is -: FACT | Based on -run_date and operation is used -DELETE+INSERT +2026-06-22 13:06:35 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-22 13:06:35 | INFO | Fetching Attendance data from 2026-06-06 to 2026-06-21 +2026-06-22 13:06:51 | INFO | Fetched 121,639 attendance rows for 544 employees +2026-06-22 13:06:51 | INFO | Fetched total row -: 121639 from sql server for table-:Attendance ...........!!! +2026-06-22 13:06:51 | INFO | Fetched 121639 rows +2026-06-22 13:06:51 | INFO | Truncated table Attendance +2026-06-22 13:06:51 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:06:56 | INFO | Attendance: inserted 121,639 rows into ClickHouse +2026-06-22 13:06:56 | INFO | Attendance loaded successfully (121639 rows) +2026-06-22 13:06:56 | INFO | ================================================================================ +2026-06-22 13:06:56 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 13:06:56 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-22 13:06:56 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 13:06:56 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:06:56 | INFO | Fetching data for 569 MIDs +2026-06-22 13:06:57 | INFO | Fetched 21,625 rows from SQL Server +2026-06-22 13:06:57 | INFO | Fetched total row -: 21625 from sql server for table-:Journey_Plan ...........!!! +2026-06-22 13:06:57 | INFO | Fetched 21625 rows +2026-06-22 13:06:57 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-22 13:06:57 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:06:57 | INFO | Journey_Plan: inserted 21,625 rows into ClickHouse +2026-06-22 13:06:57 | INFO | Journey_Plan loaded successfully (21625 rows) +2026-06-22 13:06:57 | INFO | ================================================================================ +2026-06-22 13:06:57 | INFO | Processing Table: Web_Logins | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 13:06:57 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-22 13:06:57 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 13:06:57 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:06:57 | INFO | Fetching data for 569 MIDs +2026-06-22 13:06:58 | INFO | Fetched 12 rows from SQL Server +2026-06-22 13:06:58 | INFO | Fetched total row -: 12 from sql server for table-:Web_Logins ...........!!! +2026-06-22 13:06:58 | INFO | Fetched 12 rows +2026-06-22 13:06:58 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ +2026-06-22 13:06:58 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:06:58 | INFO | Web_Logins: inserted 12 rows into ClickHouse +2026-06-22 13:06:58 | INFO | Web_Logins loaded successfully (12 rows) +2026-06-22 13:06:58 | INFO | ================================================================================ +2026-06-22 13:06:58 | INFO | Processing Table: Promotion | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:06:58 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-22 13:06:58 | INFO | Fetching data for 569 MIDs +2026-06-22 13:07:00 | INFO | Fetched 4,223 rows from SQL Server +2026-06-22 13:07:00 | INFO | Fetched total row -: 4223 from sql server for table-:Promotion ...........!!! +2026-06-22 13:07:00 | INFO | Fetched 4223 rows +2026-06-22 13:07:00 | INFO | _ _ _ _ Deleting Data from ClickHouse for Promotion _ _ _ _ +2026-06-22 13:07:01 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:01 | INFO | Promotion: inserted 4,223 rows into ClickHouse +2026-06-22 13:07:01 | INFO | Promotion loaded successfully (4223 rows) +2026-06-22 13:07:01 | INFO | ================================================================================ +2026-06-22 13:07:01 | INFO | Processing Table: PaidVisibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:07:01 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-22 13:07:01 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 13:07:01 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:07:01 | INFO | Fetching data for 569 MIDs +2026-06-22 13:07:02 | INFO | Fetched 1,284 rows from SQL Server +2026-06-22 13:07:02 | INFO | Fetched total row -: 1284 from sql server for table-:PaidVisibility ...........!!! +2026-06-22 13:07:02 | INFO | Fetched 1284 rows +2026-06-22 13:07:02 | INFO | _ _ _ _ Deleting Data from ClickHouse for PaidVisibility _ _ _ _ +2026-06-22 13:07:02 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:03 | INFO | PaidVisibility: inserted 1,284 rows into ClickHouse +2026-06-22 13:07:03 | INFO | PaidVisibility loaded successfully (1284 rows) +2026-06-22 13:07:03 | INFO | ================================================================================ +2026-06-22 13:07:03 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-22 13:07:03 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-22 13:07:03 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:03 | INFO | Fetching in progress .... +2026-06-22 13:07:05 | INFO | Fetched 5,998 rows from SQL Server +2026-06-22 13:07:05 | INFO | Fetched total row -: 5998 from sql server for table-:Store_Master ...........!!! +2026-06-22 13:07:05 | INFO | Fetched 5998 rows +2026-06-22 13:07:05 | INFO | Truncated table Store_Master +2026-06-22 13:07:05 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:07 | INFO | Store_Master: inserted 5,998 rows into ClickHouse +2026-06-22 13:07:07 | INFO | Store_Master loaded successfully (5998 rows) +2026-06-22 13:07:07 | INFO | ================================================================================ +2026-06-22 13:07:07 | INFO | Processing Table: SKU_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-22 13:07:07 | INFO | Fetching Data from sql server for table-: SKU_Master .............. +2026-06-22 13:07:07 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:07 | INFO | Fetching in progress .... +2026-06-22 13:07:07 | INFO | Fetched 160 rows from SQL Server +2026-06-22 13:07:07 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! +2026-06-22 13:07:07 | INFO | Fetched 160 rows +2026-06-22 13:07:07 | INFO | Truncated table SKU_Master +2026-06-22 13:07:07 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:07 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-22 13:07:07 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-22 13:07:07 | INFO | ================================================================================ +2026-06-22 13:07:07 | INFO | Processing Table: display_master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-22 13:07:07 | INFO | Fetching Data from sql server for table-: display_master .............. +2026-06-22 13:07:07 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:07 | INFO | Fetching in progress .... +2026-06-22 13:07:07 | INFO | Fetched 135 rows from SQL Server +2026-06-22 13:07:07 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! +2026-06-22 13:07:07 | INFO | Fetched 135 rows +2026-06-22 13:07:07 | INFO | Truncated table display_master +2026-06-22 13:07:07 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:07 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-22 13:07:07 | INFO | display_master loaded successfully (135 rows) +2026-06-22 13:07:07 | INFO | ================================================================================ +2026-06-22 13:07:07 | INFO | Processing Table: Employee_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-22 13:07:07 | INFO | Fetching Data from sql server for table-: Employee_Master .............. +2026-06-22 13:07:07 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:07 | INFO | Fetching in progress .... +2026-06-22 13:07:08 | INFO | Fetched 2,288 rows from SQL Server +2026-06-22 13:07:08 | INFO | Fetched total row -: 2288 from sql server for table-:Employee_Master ...........!!! +2026-06-22 13:07:08 | INFO | Fetched 2288 rows +2026-06-22 13:07:08 | INFO | Truncated table Employee_Master +2026-06-22 13:07:08 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:08 | INFO | Employee_Master: inserted 2,288 rows into ClickHouse +2026-06-22 13:07:08 | INFO | Employee_Master loaded successfully (2288 rows) +2026-06-22 13:07:08 | INFO | ================================================================================ +2026-06-22 13:07:08 | INFO | Processing Table: coverage_remarks | Table type is -: DIMENSION | Based on -reason_id and operation is used -DELETE+INSERT +2026-06-22 13:07:08 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. +2026-06-22 13:07:09 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:09 | INFO | Fetching in progress .... +2026-06-22 13:07:09 | INFO | Fetched 0 rows from SQL Server +2026-06-22 13:07:09 | INFO | Fetched 0 rows from SQL Server +2026-06-22 13:07:09 | INFO | Fetched total row -: 0 from sql server for table-:coverage_remarks ...........!!! +2026-06-22 13:07:09 | WARNING | coverage_remarks returned no rows +2026-06-22 13:07:09 | INFO | ================================================================================ +2026-06-22 13:07:09 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT +2026-06-22 13:07:09 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-22 13:07:10 | WARNING | Table 'mapping_storevisibility' does not exist. During collecting store_ids +2026-06-22 13:07:10 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:10 | INFO | Fetching in progress .... +2026-06-22 13:07:10 | INFO | Fetched 0 rows from SQL Server +2026-06-22 13:07:10 | INFO | Fetched 0 rows from SQL Server +2026-06-22 13:07:10 | INFO | Fetched total row -: 0 from sql server for table-:mapping_storevisibility ...........!!! +2026-06-22 13:07:10 | WARNING | mapping_storevisibility returned no rows +2026-06-22 13:07:10 | INFO | ================================================================================ +2026-06-22 13:07:10 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 13:07:10 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-22 13:07:10 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:10 | INFO | Fetching in progress .... +2026-06-22 13:07:10 | INFO | Fetched 17 rows from SQL Server +2026-06-22 13:07:10 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-22 13:07:10 | INFO | Fetched 17 rows +2026-06-22 13:07:11 | INFO | Truncated table Master_VisibilityReason +2026-06-22 13:07:11 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:11 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-22 13:07:11 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-22 13:07:11 | INFO | ================================================================================ +2026-06-22 13:07:11 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 13:07:11 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. +2026-06-22 13:07:11 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:11 | INFO | Fetching in progress .... +2026-06-22 13:07:11 | INFO | Fetched 861 rows from SQL Server +2026-06-22 13:07:11 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! +2026-06-22 13:07:11 | INFO | Fetched 861 rows +2026-06-22 13:07:11 | INFO | Truncated table Master_VisibilityDefinition +2026-06-22 13:07:11 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:11 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-22 13:07:11 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-22 13:07:11 | INFO | ================================================================================ +2026-06-22 13:07:11 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 13:07:11 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. +2026-06-22 13:07:11 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:07:11 | INFO | Fetching in progress .... +2026-06-22 13:07:11 | INFO | Fetched 33 rows from SQL Server +2026-06-22 13:07:11 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! +2026-06-22 13:07:11 | INFO | Fetched 33 rows +2026-06-22 13:07:11 | INFO | Truncated table Master_Salesterritorylayer +2026-06-22 13:07:11 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:07:12 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-22 13:07:12 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-22 13:07:12 | INFO | ================================================================================ +2026-06-22 13:07:12 | INFO | Pipeline Completed Successfully +2026-06-22 13:07:12 | INFO | ================================================================================ +2026-06-22 13:15:45 | INFO | ================================================================================ +2026-06-22 13:15:45 | INFO | Hello from data-move Python data pipeline! +2026-06-22 13:15:45 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 13:15:45 | INFO | Connecting to databases... +2026-06-22 13:15:46 | INFO | +2026-06-22 13:15:47 | INFO | +2026-06-22 13:15:48 | INFO | Database connections established +2026-06-22 13:15:48 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 13:15:49 | INFO | Found 569 MIDs +2026-06-22 13:15:49 | INFO | ================================================================================ +2026-06-22 13:15:49 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:15:49 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 13:15:49 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 13:15:49 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:15:49 | INFO | Fetching data for 569 MIDs +2026-06-22 13:15:51 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 13:15:51 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 13:15:51 | INFO | Fetched 1907 rows +2026-06-22 13:15:51 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 13:15:51 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:15:52 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 13:15:52 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 13:15:52 | INFO | ================================================================================ +2026-06-22 13:15:52 | INFO | Processing Table: OQaD | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 13:15:52 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 13:15:52 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 13:16:09 | INFO | Fetched 291,166 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 13:16:10 | INFO | Fetched 290,722 matched empids fetched for OQAD +2026-06-22 13:16:10 | INFO | Exists: True +2026-06-22 13:16:10 | INFO | Path: D:\z\data_move\src\sql\fact\OQaD.sql +2026-06-22 13:16:10 | INFO | Fetching data for 290,722 EMPIDs +2026-06-22 13:16:10 | INFO | Fetching OQaD data for run_date=2026-06-21 +2026-06-22 13:16:46 | INFO | fn name is fetch_OQad ------Fetched 0 rows +2026-06-22 13:16:46 | INFO | Fetched 0 rows from SQL Server +2026-06-22 13:16:46 | INFO | Fetched total row -: 0 from sql server for table-:OQaD ...........!!! +2026-06-22 13:16:46 | WARNING | OQaD returned no rows +2026-06-22 13:16:46 | INFO | ================================================================================ +2026-06-22 13:16:46 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:16:46 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-22 13:16:46 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:16:46 | INFO | Fetching data for 569 MIDs +2026-06-22 13:16:47 | INFO | Fetched 165 rows from SQL Server +2026-06-22 13:16:47 | INFO | Fetched total row -: 165 from sql server for table-:Survey ...........!!! +2026-06-22 13:16:47 | INFO | Fetched 165 rows +2026-06-22 13:16:47 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-22 13:16:47 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:16:47 | INFO | Survey: inserted 165 rows into ClickHouse +2026-06-22 13:16:47 | INFO | Survey loaded successfully (165 rows) +2026-06-22 13:16:47 | INFO | ================================================================================ +2026-06-22 13:16:47 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:16:47 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-22 13:16:47 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 13:16:47 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:16:47 | INFO | Fetching data for 569 MIDs +2026-06-22 13:16:47 | INFO | Fetched 1,579 rows from SQL Server +2026-06-22 13:16:47 | INFO | Fetched total row -: 1579 from sql server for table-:additional_visibility ...........!!! +2026-06-22 13:16:47 | INFO | Fetched 1579 rows +2026-06-22 13:16:48 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-22 13:16:48 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:16:48 | INFO | additional_visibility: inserted 1,579 rows into ClickHouse +2026-06-22 13:16:48 | INFO | additional_visibility loaded successfully (1579 rows) +2026-06-22 13:16:48 | INFO | ================================================================================ +2026-06-22 13:16:48 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:16:48 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-22 13:16:48 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:16:48 | INFO | Fetching data for 569 MIDs +2026-06-22 13:17:03 | INFO | Fetched 512 rows from SQL Server +2026-06-22 13:17:03 | INFO | Fetched total row -: 512 from sql server for table-:Coverage ...........!!! +2026-06-22 13:17:03 | INFO | Fetched 512 rows +2026-06-22 13:17:03 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-22 13:17:03 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:17:03 | INFO | Coverage: inserted 512 rows into ClickHouse +2026-06-22 13:17:03 | INFO | Coverage loaded successfully (512 rows) +2026-06-22 13:17:03 | INFO | ================================================================================ +2026-06-22 13:17:03 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 13:17:03 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-22 13:17:03 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:17:03 | INFO | Fetching data for 569 MIDs +2026-06-22 13:17:05 | INFO | Fetched 454 rows from SQL Server +2026-06-22 13:17:05 | INFO | Fetched total row -: 454 from sql server for table-:Login ...........!!! +2026-06-22 13:17:05 | INFO | Fetched 454 rows +2026-06-22 13:17:05 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-22 13:17:05 | INFO | No delete logic required for Login +2026-06-22 13:17:05 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:17:05 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-22 13:17:05 | INFO | Login loaded successfully (454 rows) +2026-06-22 13:17:05 | INFO | ================================================================================ +2026-06-22 13:17:05 | INFO | Processing Table: Stock_Details | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:17:05 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-22 13:17:05 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:17:05 | INFO | Fetching data for 569 MIDs +2026-06-22 13:17:10 | INFO | Fetched 27,359 rows from SQL Server +2026-06-22 13:17:10 | INFO | Fetched total row -: 27359 from sql server for table-:Stock_Details ...........!!! +2026-06-22 13:17:10 | INFO | Fetched 27359 rows +2026-06-22 13:17:10 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-22 13:17:10 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:17:11 | INFO | Stock_Details: inserted 27,359 rows into ClickHouse +2026-06-22 13:17:11 | INFO | Stock_Details loaded successfully (27359 rows) +2026-06-22 13:17:11 | INFO | ================================================================================ +2026-06-22 13:17:11 | INFO | Processing Table: Attendance | Table type is -: FACT | Based on -run_date and operation is used -DELETE+INSERT +2026-06-22 13:17:11 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-22 13:17:11 | INFO | Fetching Attendance data from 2026-06-06 to 2026-06-21 +2026-06-22 13:17:27 | INFO | Fetched 121,639 attendance rows for 544 employees +2026-06-22 13:17:27 | INFO | Fetched total row -: 121639 from sql server for table-:Attendance ...........!!! +2026-06-22 13:17:27 | INFO | Fetched 121639 rows +2026-06-22 13:17:27 | INFO | Truncated table Attendance +2026-06-22 13:17:27 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:17:28 | INFO | Attendance: inserted 121,639 rows into ClickHouse +2026-06-22 13:17:28 | INFO | Attendance loaded successfully (121639 rows) +2026-06-22 13:17:28 | INFO | ================================================================================ +2026-06-22 13:17:28 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 13:17:28 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-22 13:17:28 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 13:17:28 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:17:28 | INFO | Fetching data for 569 MIDs +2026-06-22 13:17:29 | INFO | Fetched 21,625 rows from SQL Server +2026-06-22 13:17:29 | INFO | Fetched total row -: 21625 from sql server for table-:Journey_Plan ...........!!! +2026-06-22 13:17:29 | INFO | Fetched 21625 rows +2026-06-22 13:17:30 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-22 13:17:30 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:17:30 | INFO | Journey_Plan: inserted 21,625 rows into ClickHouse +2026-06-22 13:17:30 | INFO | Journey_Plan loaded successfully (21625 rows) +2026-06-22 13:17:30 | INFO | ================================================================================ +2026-06-22 13:17:30 | INFO | Processing Table: Web_Logins | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 13:17:30 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-22 13:17:30 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 13:17:30 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:17:30 | INFO | Fetching data for 569 MIDs +2026-06-22 13:17:31 | INFO | Fetched 12 rows from SQL Server +2026-06-22 13:17:31 | INFO | Fetched total row -: 12 from sql server for table-:Web_Logins ...........!!! +2026-06-22 13:17:31 | INFO | Fetched 12 rows +2026-06-22 13:17:31 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ +2026-06-22 13:17:31 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:17:31 | INFO | Web_Logins: inserted 12 rows into ClickHouse +2026-06-22 13:17:31 | INFO | Web_Logins loaded successfully (12 rows) +2026-06-22 13:17:31 | INFO | ================================================================================ +2026-06-22 13:17:31 | INFO | Processing Table: Promotion | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:17:31 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-22 13:17:31 | INFO | Fetching data for 569 MIDs +2026-06-22 13:17:32 | INFO | Fetched 4,223 rows from SQL Server +2026-06-22 13:17:32 | INFO | Fetched total row -: 4223 from sql server for table-:Promotion ...........!!! +2026-06-22 13:17:32 | INFO | Fetched 4223 rows +2026-06-22 13:17:32 | INFO | _ _ _ _ Deleting Data from ClickHouse for Promotion _ _ _ _ +2026-06-22 13:17:32 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:17:33 | INFO | Promotion: inserted 4,223 rows into ClickHouse +2026-06-22 13:17:33 | INFO | Promotion loaded successfully (4223 rows) +2026-06-22 13:17:33 | INFO | ================================================================================ +2026-06-22 13:17:33 | INFO | Processing Table: PaidVisibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 13:17:33 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-22 13:17:33 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 13:17:33 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 13:17:33 | INFO | Fetching data for 569 MIDs +2026-06-22 13:17:34 | INFO | Fetched 1,284 rows from SQL Server +2026-06-22 13:17:34 | INFO | Fetched total row -: 1284 from sql server for table-:PaidVisibility ...........!!! +2026-06-22 13:17:34 | INFO | Fetched 1284 rows +2026-06-22 13:17:34 | INFO | _ _ _ _ Deleting Data from ClickHouse for PaidVisibility _ _ _ _ +2026-06-22 13:17:34 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:17:34 | INFO | PaidVisibility: inserted 1,284 rows into ClickHouse +2026-06-22 13:17:34 | INFO | PaidVisibility loaded successfully (1284 rows) +2026-06-22 13:17:34 | INFO | ================================================================================ +2026-06-22 13:17:34 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-22 13:17:34 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-22 13:17:34 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:17:34 | INFO | Fetching in progress .... +2026-06-22 13:17:38 | INFO | Fetched 5,998 rows from SQL Server +2026-06-22 13:17:38 | INFO | Fetched total row -: 5998 from sql server for table-:Store_Master ...........!!! +2026-06-22 13:17:38 | INFO | Fetched 5998 rows +2026-06-22 13:17:38 | INFO | Truncated table Store_Master +2026-06-22 13:17:38 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:17:39 | INFO | Store_Master: inserted 5,998 rows into ClickHouse +2026-06-22 13:17:39 | INFO | Store_Master loaded successfully (5998 rows) +2026-06-22 13:17:39 | INFO | ================================================================================ +2026-06-22 13:17:39 | INFO | Processing Table: SKU_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-22 13:17:39 | INFO | Fetching Data from sql server for table-: SKU_Master .............. +2026-06-22 13:17:39 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:17:39 | INFO | Fetching in progress .... +2026-06-22 13:17:39 | INFO | Fetched 160 rows from SQL Server +2026-06-22 13:17:39 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! +2026-06-22 13:17:39 | INFO | Fetched 160 rows +2026-06-22 13:17:39 | INFO | Truncated table SKU_Master +2026-06-22 13:17:39 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:17:40 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-22 13:17:40 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-22 13:17:40 | INFO | ================================================================================ +2026-06-22 13:17:40 | INFO | Processing Table: display_master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-22 13:17:40 | INFO | Fetching Data from sql server for table-: display_master .............. +2026-06-22 13:17:40 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:17:40 | INFO | Fetching in progress .... +2026-06-22 13:17:40 | INFO | Fetched 135 rows from SQL Server +2026-06-22 13:17:40 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! +2026-06-22 13:17:40 | INFO | Fetched 135 rows +2026-06-22 13:17:40 | INFO | Truncated table display_master +2026-06-22 13:17:40 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:17:40 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-22 13:17:40 | INFO | display_master loaded successfully (135 rows) +2026-06-22 13:17:40 | INFO | ================================================================================ +2026-06-22 13:17:40 | INFO | Processing Table: Employee_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-22 13:17:40 | INFO | Fetching Data from sql server for table-: Employee_Master .............. +2026-06-22 13:17:40 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:17:40 | INFO | Fetching in progress .... +2026-06-22 13:17:41 | INFO | Fetched 2,288 rows from SQL Server +2026-06-22 13:17:41 | INFO | Fetched total row -: 2288 from sql server for table-:Employee_Master ...........!!! +2026-06-22 13:17:41 | INFO | Fetched 2288 rows +2026-06-22 13:17:41 | INFO | Truncated table Employee_Master +2026-06-22 13:17:41 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:17:41 | INFO | Employee_Master: inserted 2,288 rows into ClickHouse +2026-06-22 13:17:41 | INFO | Employee_Master loaded successfully (2288 rows) +2026-06-22 13:17:41 | INFO | ================================================================================ +2026-06-22 13:17:41 | INFO | Processing Table: coverage_remarks | Table type is -: DIMENSION | Based on -reason_id and operation is used -DELETE+INSERT +2026-06-22 13:17:41 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. +2026-06-22 13:17:42 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:17:42 | INFO | Fetching in progress .... +2026-06-22 13:17:42 | INFO | Fetched 0 rows from SQL Server +2026-06-22 13:17:42 | INFO | Fetched 0 rows from SQL Server +2026-06-22 13:17:42 | INFO | Fetched total row -: 0 from sql server for table-:coverage_remarks ...........!!! +2026-06-22 13:17:42 | WARNING | coverage_remarks returned no rows +2026-06-22 13:17:42 | INFO | ================================================================================ +2026-06-22 13:17:42 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT +2026-06-22 13:17:42 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-22 13:17:42 | WARNING | Table 'mapping_storevisibility' does not exist. During collecting store_ids +2026-06-22 13:17:42 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:17:42 | INFO | Fetching in progress .... +2026-06-22 13:17:42 | INFO | Fetched 0 rows from SQL Server +2026-06-22 13:17:42 | INFO | Fetched 0 rows from SQL Server +2026-06-22 13:17:42 | INFO | Fetched total row -: 0 from sql server for table-:mapping_storevisibility ...........!!! +2026-06-22 13:17:42 | WARNING | mapping_storevisibility returned no rows +2026-06-22 13:17:42 | INFO | ================================================================================ +2026-06-22 13:17:42 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 13:17:42 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-22 13:17:42 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:17:42 | INFO | Fetching in progress .... +2026-06-22 13:17:42 | INFO | Fetched 17 rows from SQL Server +2026-06-22 13:17:42 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-22 13:17:42 | INFO | Fetched 17 rows +2026-06-22 13:17:43 | INFO | Truncated table Master_VisibilityReason +2026-06-22 13:17:43 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:17:43 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-22 13:17:43 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-22 13:17:43 | INFO | ================================================================================ +2026-06-22 13:17:43 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 13:17:43 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. +2026-06-22 13:17:43 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:17:43 | INFO | Fetching in progress .... +2026-06-22 13:17:43 | INFO | Fetched 861 rows from SQL Server +2026-06-22 13:17:43 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! +2026-06-22 13:17:43 | INFO | Fetched 861 rows +2026-06-22 13:17:43 | INFO | Truncated table Master_VisibilityDefinition +2026-06-22 13:17:43 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:17:43 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-22 13:17:43 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-22 13:17:43 | INFO | ================================================================================ +2026-06-22 13:17:43 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 13:17:43 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. +2026-06-22 13:17:43 | INFO | Fetching data from sql server for Master table...... +2026-06-22 13:17:43 | INFO | Fetching in progress .... +2026-06-22 13:17:43 | INFO | Fetched 33 rows from SQL Server +2026-06-22 13:17:43 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! +2026-06-22 13:17:43 | INFO | Fetched 33 rows +2026-06-22 13:17:43 | INFO | Truncated table Master_Salesterritorylayer +2026-06-22 13:17:43 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 13:17:44 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-22 13:17:44 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-22 13:17:44 | INFO | ================================================================================ +2026-06-22 13:17:44 | INFO | Pipeline Completed Successfully +2026-06-22 13:17:44 | INFO | ================================================================================ +2026-06-22 15:38:19 | INFO | ================================================================================ +2026-06-22 15:38:19 | INFO | Hello from data-move Python data pipeline! +2026-06-22 15:38:19 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 15:38:19 | INFO | Connecting to databases... +2026-06-22 15:38:21 | INFO | +2026-06-22 15:38:25 | INFO | +2026-06-22 15:38:26 | INFO | Database connections established +2026-06-22 15:38:26 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 15:38:26 | INFO | Found 569 MIDs +2026-06-22 15:38:26 | INFO | ================================================================================ +2026-06-22 15:38:26 | INFO | Processing Table: SOS_OneApp | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 15:38:26 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 15:38:26 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 15:38:26 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:38:26 | INFO | Fetching data for 569 MIDs +2026-06-22 15:38:36 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 15:38:36 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 15:38:36 | INFO | Fetched 1907 rows +2026-06-22 15:38:36 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 15:38:36 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:38:37 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 15:38:37 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 15:38:37 | INFO | ================================================================================ +2026-06-22 15:38:37 | INFO | Processing Table: OQaD | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 15:38:37 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 15:38:37 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 15:39:02 | INFO | Fetched 291,198 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 15:39:06 | INFO | Fetched 290,722 matched empids fetched for OQAD +2026-06-22 15:39:06 | INFO | Exists: True +2026-06-22 15:39:06 | INFO | Path: D:\z\airflow-project\src\sql\fact\OQaD.sql +2026-06-22 15:39:06 | INFO | Fetching data for 290,722 EMPIDs +2026-06-22 15:39:06 | INFO | Fetching OQaD data for run_date=2026-06-21 +2026-06-22 15:40:18 | INFO | fn name is fetch_OQad ------Fetched 0 rows +2026-06-22 15:40:18 | INFO | Fetched 0 rows from SQL Server +2026-06-22 15:40:18 | INFO | Fetched total row -: 0 from sql server for table-:OQaD ...........!!! +2026-06-22 15:40:18 | WARNING | OQaD returned no rows +2026-06-22 15:40:18 | INFO | ================================================================================ +2026-06-22 15:40:18 | INFO | Processing Table: Survey | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 15:40:18 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-22 15:40:18 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:40:18 | INFO | Fetching data for 569 MIDs +2026-06-22 15:40:19 | INFO | Fetched 165 rows from SQL Server +2026-06-22 15:40:19 | INFO | Fetched total row -: 165 from sql server for table-:Survey ...........!!! +2026-06-22 15:40:19 | INFO | Fetched 165 rows +2026-06-22 15:40:19 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-22 15:40:19 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:40:19 | INFO | Survey: inserted 165 rows into ClickHouse +2026-06-22 15:40:19 | INFO | Survey loaded successfully (165 rows) +2026-06-22 15:40:19 | INFO | ================================================================================ +2026-06-22 15:40:19 | INFO | Processing Table: additional_visibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 15:40:19 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-22 15:40:19 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 15:40:19 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:40:19 | INFO | Fetching data for 569 MIDs +2026-06-22 15:40:20 | INFO | Fetched 1,579 rows from SQL Server +2026-06-22 15:40:20 | INFO | Fetched total row -: 1579 from sql server for table-:additional_visibility ...........!!! +2026-06-22 15:40:20 | INFO | Fetched 1579 rows +2026-06-22 15:40:20 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-22 15:40:20 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:40:20 | INFO | additional_visibility: inserted 1,579 rows into ClickHouse +2026-06-22 15:40:20 | INFO | additional_visibility loaded successfully (1579 rows) +2026-06-22 15:40:20 | INFO | ================================================================================ +2026-06-22 15:40:20 | INFO | Processing Table: Coverage | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 15:40:20 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-22 15:40:20 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:40:20 | INFO | Fetching data for 569 MIDs +2026-06-22 15:40:42 | INFO | Fetched 512 rows from SQL Server +2026-06-22 15:40:42 | INFO | Fetched total row -: 512 from sql server for table-:Coverage ...........!!! +2026-06-22 15:40:42 | INFO | Fetched 512 rows +2026-06-22 15:40:43 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-22 15:40:43 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:40:43 | INFO | Coverage: inserted 512 rows into ClickHouse +2026-06-22 15:40:43 | INFO | Coverage loaded successfully (512 rows) +2026-06-22 15:40:43 | INFO | ================================================================================ +2026-06-22 15:40:43 | INFO | Processing Table: Login | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 15:40:43 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-22 15:40:43 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:40:43 | INFO | Fetching data for 569 MIDs +2026-06-22 15:40:46 | INFO | Fetched 454 rows from SQL Server +2026-06-22 15:40:46 | INFO | Fetched total row -: 454 from sql server for table-:Login ...........!!! +2026-06-22 15:40:46 | INFO | Fetched 454 rows +2026-06-22 15:40:46 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-22 15:40:46 | INFO | No delete logic required for Login +2026-06-22 15:40:46 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:40:46 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-22 15:40:46 | INFO | Login loaded successfully (454 rows) +2026-06-22 15:40:46 | INFO | ================================================================================ +2026-06-22 15:40:46 | INFO | Processing Table: Stock_Details | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 15:40:46 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-22 15:40:46 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:40:46 | INFO | Fetching data for 569 MIDs +2026-06-22 15:40:54 | INFO | Fetched 27,359 rows from SQL Server +2026-06-22 15:40:54 | INFO | Fetched total row -: 27359 from sql server for table-:Stock_Details ...........!!! +2026-06-22 15:40:54 | INFO | Fetched 27359 rows +2026-06-22 15:40:54 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-22 15:40:54 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:40:55 | INFO | Stock_Details: inserted 27,359 rows into ClickHouse +2026-06-22 15:40:55 | INFO | Stock_Details loaded successfully (27359 rows) +2026-06-22 15:40:55 | INFO | ================================================================================ +2026-06-22 15:40:55 | INFO | Processing Table: Attendance | Table type is -: FACT | Based on -run_date and operation is used -DELETE+INSERT +2026-06-22 15:40:55 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-22 15:40:55 | INFO | Fetching Attendance data from 2026-06-06 to 2026-06-21 +2026-06-22 15:41:17 | INFO | Fetched 121,642 attendance rows for 545 employees +2026-06-22 15:41:17 | INFO | Fetched total row -: 121642 from sql server for table-:Attendance ...........!!! +2026-06-22 15:41:17 | INFO | Fetched 121642 rows +2026-06-22 15:41:17 | INFO | Truncated table Attendance +2026-06-22 15:41:17 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:41:18 | INFO | Attendance: inserted 121,642 rows into ClickHouse +2026-06-22 15:41:18 | INFO | Attendance loaded successfully (121642 rows) +2026-06-22 15:41:18 | INFO | ================================================================================ +2026-06-22 15:41:18 | INFO | Processing Table: Journey_Plan | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 15:41:18 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-22 15:41:18 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 15:41:18 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:41:18 | INFO | Fetching data for 569 MIDs +2026-06-22 15:41:20 | INFO | Fetched 21,643 rows from SQL Server +2026-06-22 15:41:20 | INFO | Fetched total row -: 21643 from sql server for table-:Journey_Plan ...........!!! +2026-06-22 15:41:20 | INFO | Fetched 21643 rows +2026-06-22 15:41:20 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-22 15:41:20 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:41:21 | INFO | Journey_Plan: inserted 21,643 rows into ClickHouse +2026-06-22 15:41:21 | INFO | Journey_Plan loaded successfully (21643 rows) +2026-06-22 15:41:21 | INFO | ================================================================================ +2026-06-22 15:41:21 | INFO | Processing Table: Web_Logins | Table type is -: FACT | Based on -run_date and operation is used -INSERT +2026-06-22 15:41:21 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-22 15:41:21 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 15:41:21 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:41:21 | INFO | Fetching data for 569 MIDs +2026-06-22 15:41:21 | INFO | Fetched 12 rows from SQL Server +2026-06-22 15:41:21 | INFO | Fetched total row -: 12 from sql server for table-:Web_Logins ...........!!! +2026-06-22 15:41:21 | INFO | Fetched 12 rows +2026-06-22 15:41:21 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ +2026-06-22 15:41:21 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:41:22 | INFO | Web_Logins: inserted 12 rows into ClickHouse +2026-06-22 15:41:22 | INFO | Web_Logins loaded successfully (12 rows) +2026-06-22 15:41:22 | INFO | ================================================================================ +2026-06-22 15:41:22 | INFO | Processing Table: Promotion | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 15:41:22 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-22 15:41:22 | INFO | Fetching data for 569 MIDs +2026-06-22 15:41:25 | INFO | Fetched 4,223 rows from SQL Server +2026-06-22 15:41:25 | INFO | Fetched total row -: 4223 from sql server for table-:Promotion ...........!!! +2026-06-22 15:41:25 | INFO | Fetched 4223 rows +2026-06-22 15:41:25 | INFO | _ _ _ _ Deleting Data from ClickHouse for Promotion _ _ _ _ +2026-06-22 15:41:25 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:41:25 | INFO | Promotion: inserted 4,223 rows into ClickHouse +2026-06-22 15:41:25 | INFO | Promotion loaded successfully (4223 rows) +2026-06-22 15:41:25 | INFO | ================================================================================ +2026-06-22 15:41:25 | INFO | Processing Table: PaidVisibility | Table type is -: FACT | Based on -mids and operation is used -INSERT +2026-06-22 15:41:25 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-22 15:41:25 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 15:41:25 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:41:25 | INFO | Fetching data for 569 MIDs +2026-06-22 15:41:27 | INFO | Fetched 1,284 rows from SQL Server +2026-06-22 15:41:27 | INFO | Fetched total row -: 1284 from sql server for table-:PaidVisibility ...........!!! +2026-06-22 15:41:27 | INFO | Fetched 1284 rows +2026-06-22 15:41:27 | INFO | _ _ _ _ Deleting Data from ClickHouse for PaidVisibility _ _ _ _ +2026-06-22 15:41:27 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:41:28 | INFO | PaidVisibility: inserted 1,284 rows into ClickHouse +2026-06-22 15:41:28 | INFO | PaidVisibility loaded successfully (1284 rows) +2026-06-22 15:41:28 | INFO | ================================================================================ +2026-06-22 15:41:28 | INFO | Processing Table: Store_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-22 15:41:28 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-22 15:41:28 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:41:28 | INFO | Fetching in progress .... +2026-06-22 15:41:34 | INFO | Fetched 5,998 rows from SQL Server +2026-06-22 15:41:34 | INFO | Fetched total row -: 5998 from sql server for table-:Store_Master ...........!!! +2026-06-22 15:41:34 | INFO | Fetched 5998 rows +2026-06-22 15:41:35 | INFO | Truncated table Store_Master +2026-06-22 15:41:35 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:41:35 | INFO | Store_Master: inserted 5,998 rows into ClickHouse +2026-06-22 15:41:35 | INFO | Store_Master loaded successfully (5998 rows) +2026-06-22 15:41:35 | INFO | ================================================================================ +2026-06-22 15:41:35 | INFO | Processing Table: SKU_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-22 15:41:35 | INFO | Fetching Data from sql server for table-: SKU_Master .............. +2026-06-22 15:41:35 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:41:35 | INFO | Fetching in progress .... +2026-06-22 15:41:36 | INFO | Fetched 160 rows from SQL Server +2026-06-22 15:41:36 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! +2026-06-22 15:41:36 | INFO | Fetched 160 rows +2026-06-22 15:41:36 | INFO | Truncated table SKU_Master +2026-06-22 15:41:36 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:41:36 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-22 15:41:36 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-22 15:41:36 | INFO | ================================================================================ +2026-06-22 15:41:36 | INFO | Processing Table: display_master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-22 15:41:36 | INFO | Fetching Data from sql server for table-: display_master .............. +2026-06-22 15:41:36 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:41:36 | INFO | Fetching in progress .... +2026-06-22 15:41:36 | INFO | Fetched 135 rows from SQL Server +2026-06-22 15:41:36 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! +2026-06-22 15:41:36 | INFO | Fetched 135 rows +2026-06-22 15:41:36 | INFO | Truncated table display_master +2026-06-22 15:41:36 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:41:36 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-22 15:41:36 | INFO | display_master loaded successfully (135 rows) +2026-06-22 15:41:36 | INFO | ================================================================================ +2026-06-22 15:41:36 | INFO | Processing Table: Employee_Master | Table type is -: DIMENSION | Based on -master and operation is used -DELETE+INSERT +2026-06-22 15:41:36 | INFO | Fetching Data from sql server for table-: Employee_Master .............. +2026-06-22 15:41:36 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:41:36 | INFO | Fetching in progress .... +2026-06-22 15:41:37 | INFO | Fetched 2,289 rows from SQL Server +2026-06-22 15:41:37 | INFO | Fetched total row -: 2289 from sql server for table-:Employee_Master ...........!!! +2026-06-22 15:41:37 | INFO | Fetched 2289 rows +2026-06-22 15:41:37 | INFO | Truncated table Employee_Master +2026-06-22 15:41:37 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:41:37 | INFO | Employee_Master: inserted 2,289 rows into ClickHouse +2026-06-22 15:41:37 | INFO | Employee_Master loaded successfully (2289 rows) +2026-06-22 15:41:37 | INFO | ================================================================================ +2026-06-22 15:41:37 | INFO | Processing Table: coverage_remarks | Table type is -: DIMENSION | Based on -reason_id and operation is used -DELETE+INSERT +2026-06-22 15:41:37 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. +2026-06-22 15:41:38 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:41:38 | INFO | Fetching in progress .... +2026-06-22 15:41:38 | INFO | Fetched 0 rows from SQL Server +2026-06-22 15:41:38 | INFO | Fetched 0 rows from SQL Server +2026-06-22 15:41:38 | INFO | Fetched total row -: 0 from sql server for table-:coverage_remarks ...........!!! +2026-06-22 15:41:38 | WARNING | coverage_remarks returned no rows +2026-06-22 15:41:38 | INFO | ================================================================================ +2026-06-22 15:41:38 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT +2026-06-22 15:41:38 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-22 15:41:38 | WARNING | Table 'mapping_storevisibility' does not exist. During collecting store_ids +2026-06-22 15:41:38 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:41:38 | INFO | Fetching in progress .... +2026-06-22 15:41:39 | INFO | Fetched 0 rows from SQL Server +2026-06-22 15:41:39 | INFO | Fetched 0 rows from SQL Server +2026-06-22 15:41:39 | INFO | Fetched total row -: 0 from sql server for table-:mapping_storevisibility ...........!!! +2026-06-22 15:41:39 | WARNING | mapping_storevisibility returned no rows +2026-06-22 15:41:39 | INFO | ================================================================================ +2026-06-22 15:41:39 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 15:41:39 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-22 15:41:39 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:41:39 | INFO | Fetching in progress .... +2026-06-22 15:41:39 | INFO | Fetched 17 rows from SQL Server +2026-06-22 15:41:39 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-22 15:41:39 | INFO | Fetched 17 rows +2026-06-22 15:41:39 | INFO | Truncated table Master_VisibilityReason +2026-06-22 15:41:39 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:41:39 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-22 15:41:39 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-22 15:41:39 | INFO | ================================================================================ +2026-06-22 15:41:39 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 15:41:39 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. +2026-06-22 15:41:39 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:41:39 | INFO | Fetching in progress .... +2026-06-22 15:41:39 | INFO | Fetched 861 rows from SQL Server +2026-06-22 15:41:39 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! +2026-06-22 15:41:39 | INFO | Fetched 861 rows +2026-06-22 15:41:39 | INFO | Truncated table Master_VisibilityDefinition +2026-06-22 15:41:39 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:41:39 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-22 15:41:39 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-22 15:41:39 | INFO | ================================================================================ +2026-06-22 15:41:39 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 15:41:39 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. +2026-06-22 15:41:39 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:41:39 | INFO | Fetching in progress .... +2026-06-22 15:41:40 | INFO | Fetched 33 rows from SQL Server +2026-06-22 15:41:40 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! +2026-06-22 15:41:40 | INFO | Fetched 33 rows +2026-06-22 15:41:40 | INFO | Truncated table Master_Salesterritorylayer +2026-06-22 15:41:40 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:41:40 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-22 15:41:40 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-22 15:41:40 | INFO | ================================================================================ +2026-06-22 15:41:40 | INFO | Pipeline Completed Successfully +2026-06-22 15:41:40 | INFO | ================================================================================ +2026-06-22 15:47:12 | INFO | ================================================================================ +2026-06-22 15:47:12 | INFO | Hello from data-move Python data pipeline! +2026-06-22 15:47:12 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 15:47:12 | INFO | Connecting to databases... +2026-06-22 15:47:13 | INFO | +2026-06-22 15:47:14 | INFO | +2026-06-22 15:47:15 | INFO | Database connections established +2026-06-22 15:47:15 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 15:47:15 | INFO | Found 569 MIDs +2026-06-22 15:47:16 | INFO | ================================================================================ +2026-06-22 15:47:16 | INFO | Processing Table-:SOS_OneApp | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-22 15:47:16 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 15:47:16 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 15:47:16 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:47:16 | INFO | Fetching data for 569 MIDs +2026-06-22 15:47:19 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 15:47:19 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 15:47:19 | INFO | Fetched 1907 rows +2026-06-22 15:47:19 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 15:47:19 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:47:20 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 15:47:20 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 15:47:20 | INFO | ================================================================================ +2026-06-22 15:47:20 | INFO | Processing Table-:OQaD | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-22 15:47:20 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 15:47:20 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 15:47:35 | INFO | Fetched 291,198 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 15:47:37 | INFO | Fetched 290,722 matched empids fetched for OQAD +2026-06-22 15:47:37 | INFO | Exists: True +2026-06-22 15:47:37 | INFO | Path: D:\z\airflow-project\src\sql\fact\OQaD.sql +2026-06-22 15:47:37 | INFO | Fetching data for 290,722 EMPIDs +2026-06-22 15:47:37 | INFO | Fetching OQaD data for run_date=2026-06-21 +2026-06-22 15:48:22 | INFO | fn name is fetch_OQad ------Fetched 0 rows +2026-06-22 15:48:22 | INFO | Fetched 0 rows from SQL Server +2026-06-22 15:48:22 | INFO | Fetched total row -: 0 from sql server for table-:OQaD ...........!!! +2026-06-22 15:48:22 | WARNING | OQaD returned no rows +2026-06-22 15:48:22 | INFO | ================================================================================ +2026-06-22 15:48:22 | INFO | Processing Table-:Survey | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-22 15:48:22 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-22 15:48:22 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:48:22 | INFO | Fetching data for 569 MIDs +2026-06-22 15:48:22 | INFO | Fetched 165 rows from SQL Server +2026-06-22 15:48:22 | INFO | Fetched total row -: 165 from sql server for table-:Survey ...........!!! +2026-06-22 15:48:22 | INFO | Fetched 165 rows +2026-06-22 15:48:22 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-22 15:48:22 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:48:23 | INFO | Survey: inserted 165 rows into ClickHouse +2026-06-22 15:48:23 | INFO | Survey loaded successfully (165 rows) +2026-06-22 15:48:23 | INFO | ================================================================================ +2026-06-22 15:48:23 | INFO | Processing Table-:additional_visibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-22 15:48:23 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-22 15:48:23 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 15:48:23 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:48:23 | INFO | Fetching data for 569 MIDs +2026-06-22 15:48:23 | INFO | Fetched 1,579 rows from SQL Server +2026-06-22 15:48:23 | INFO | Fetched total row -: 1579 from sql server for table-:additional_visibility ...........!!! +2026-06-22 15:48:23 | INFO | Fetched 1579 rows +2026-06-22 15:48:23 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-22 15:48:23 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:48:24 | INFO | additional_visibility: inserted 1,579 rows into ClickHouse +2026-06-22 15:48:24 | INFO | additional_visibility loaded successfully (1579 rows) +2026-06-22 15:48:24 | INFO | ================================================================================ +2026-06-22 15:48:24 | INFO | Processing Table-:Coverage | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-22 15:48:24 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-22 15:48:24 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:48:24 | INFO | Fetching data for 569 MIDs +2026-06-22 15:48:38 | INFO | Fetched 512 rows from SQL Server +2026-06-22 15:48:38 | INFO | Fetched total row -: 512 from sql server for table-:Coverage ...........!!! +2026-06-22 15:48:38 | INFO | Fetched 512 rows +2026-06-22 15:48:38 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-22 15:48:38 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:48:38 | INFO | Coverage: inserted 512 rows into ClickHouse +2026-06-22 15:48:38 | INFO | Coverage loaded successfully (512 rows) +2026-06-22 15:48:38 | INFO | ================================================================================ +2026-06-22 15:48:38 | INFO | Processing Table-:Login | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-22 15:48:38 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-22 15:48:38 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:48:38 | INFO | Fetching data for 569 MIDs +2026-06-22 15:48:40 | INFO | Fetched 454 rows from SQL Server +2026-06-22 15:48:40 | INFO | Fetched total row -: 454 from sql server for table-:Login ...........!!! +2026-06-22 15:48:40 | INFO | Fetched 454 rows +2026-06-22 15:48:40 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-22 15:48:40 | INFO | No delete logic required for Login +2026-06-22 15:48:40 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:48:40 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-22 15:48:40 | INFO | Login loaded successfully (454 rows) +2026-06-22 15:48:40 | INFO | ================================================================================ +2026-06-22 15:48:40 | INFO | Processing Table-:Stock_Details | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-22 15:48:40 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-22 15:48:40 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:48:40 | INFO | Fetching data for 569 MIDs +2026-06-22 15:48:45 | INFO | Fetched 27,359 rows from SQL Server +2026-06-22 15:48:45 | INFO | Fetched total row -: 27359 from sql server for table-:Stock_Details ...........!!! +2026-06-22 15:48:45 | INFO | Fetched 27359 rows +2026-06-22 15:48:45 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-22 15:48:45 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:48:45 | INFO | Stock_Details: inserted 27,359 rows into ClickHouse +2026-06-22 15:48:45 | INFO | Stock_Details loaded successfully (27359 rows) +2026-06-22 15:48:45 | INFO | ================================================================================ +2026-06-22 15:48:45 | INFO | Processing Table-:Attendance | Table type -:FACT | fetcht by-:run_date | operation-:DELETE+INSERT +2026-06-22 15:48:45 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-22 15:48:45 | INFO | Fetching Attendance data from 2026-06-06 to 2026-06-21 +2026-06-22 15:49:08 | INFO | Fetched 121,642 attendance rows for 545 employees +2026-06-22 15:49:08 | INFO | Fetched total row -: 121642 from sql server for table-:Attendance ...........!!! +2026-06-22 15:49:08 | INFO | Fetched 121642 rows +2026-06-22 15:49:08 | INFO | Truncated table Attendance +2026-06-22 15:49:08 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:49:09 | INFO | Attendance: inserted 121,642 rows into ClickHouse +2026-06-22 15:49:09 | INFO | Attendance loaded successfully (121642 rows) +2026-06-22 15:49:09 | INFO | ================================================================================ +2026-06-22 15:49:09 | INFO | Processing Table-:Journey_Plan | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-22 15:49:09 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-22 15:49:09 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 15:49:09 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:49:09 | INFO | Fetching data for 569 MIDs +2026-06-22 15:49:10 | INFO | Fetched 21,643 rows from SQL Server +2026-06-22 15:49:10 | INFO | Fetched total row -: 21643 from sql server for table-:Journey_Plan ...........!!! +2026-06-22 15:49:10 | INFO | Fetched 21643 rows +2026-06-22 15:49:10 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-22 15:49:11 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:49:11 | INFO | Journey_Plan: inserted 21,643 rows into ClickHouse +2026-06-22 15:49:11 | INFO | Journey_Plan loaded successfully (21643 rows) +2026-06-22 15:49:11 | INFO | ================================================================================ +2026-06-22 15:49:11 | INFO | Processing Table-:Web_Logins | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-22 15:49:11 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-22 15:49:11 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 15:49:11 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:49:11 | INFO | Fetching data for 569 MIDs +2026-06-22 15:49:11 | INFO | Fetched 12 rows from SQL Server +2026-06-22 15:49:11 | INFO | Fetched total row -: 12 from sql server for table-:Web_Logins ...........!!! +2026-06-22 15:49:11 | INFO | Fetched 12 rows +2026-06-22 15:49:11 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ +2026-06-22 15:49:11 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:49:11 | INFO | Web_Logins: inserted 12 rows into ClickHouse +2026-06-22 15:49:11 | INFO | Web_Logins loaded successfully (12 rows) +2026-06-22 15:49:11 | INFO | ================================================================================ +2026-06-22 15:49:11 | INFO | Processing Table-:Promotion | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-22 15:49:11 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-22 15:49:11 | INFO | Fetching data for 569 MIDs +2026-06-22 15:49:14 | INFO | Fetched 4,223 rows from SQL Server +2026-06-22 15:49:14 | INFO | Fetched total row -: 4223 from sql server for table-:Promotion ...........!!! +2026-06-22 15:49:14 | INFO | Fetched 4223 rows +2026-06-22 15:49:14 | INFO | _ _ _ _ Deleting Data from ClickHouse for Promotion _ _ _ _ +2026-06-22 15:49:14 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:49:14 | INFO | Promotion: inserted 4,223 rows into ClickHouse +2026-06-22 15:49:14 | INFO | Promotion loaded successfully (4223 rows) +2026-06-22 15:49:14 | INFO | ================================================================================ +2026-06-22 15:49:14 | INFO | Processing Table-:PaidVisibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-22 15:49:14 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-22 15:49:14 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 15:49:14 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:49:14 | INFO | Fetching data for 569 MIDs +2026-06-22 15:49:16 | INFO | Fetched 1,284 rows from SQL Server +2026-06-22 15:49:16 | INFO | Fetched total row -: 1284 from sql server for table-:PaidVisibility ...........!!! +2026-06-22 15:49:16 | INFO | Fetched 1284 rows +2026-06-22 15:49:16 | INFO | _ _ _ _ Deleting Data from ClickHouse for PaidVisibility _ _ _ _ +2026-06-22 15:49:16 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:49:16 | INFO | PaidVisibility: inserted 1,284 rows into ClickHouse +2026-06-22 15:49:16 | INFO | PaidVisibility loaded successfully (1284 rows) +2026-06-22 15:49:16 | INFO | ================================================================================ +2026-06-22 15:49:16 | INFO | Processing Table-:Store_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-22 15:49:16 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-22 15:49:16 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:49:16 | INFO | Fetching in progress .... +2026-06-22 15:49:20 | INFO | Fetched 5,998 rows from SQL Server +2026-06-22 15:49:20 | INFO | Fetched total row -: 5998 from sql server for table-:Store_Master ...........!!! +2026-06-22 15:49:20 | INFO | Fetched 5998 rows +2026-06-22 15:49:20 | INFO | Truncated table Store_Master +2026-06-22 15:49:20 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:49:21 | INFO | Store_Master: inserted 5,998 rows into ClickHouse +2026-06-22 15:49:21 | INFO | Store_Master loaded successfully (5998 rows) +2026-06-22 15:49:21 | INFO | ================================================================================ +2026-06-22 15:49:21 | INFO | Processing Table-:SKU_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-22 15:49:21 | INFO | Fetching Data from sql server for table-: SKU_Master .............. +2026-06-22 15:49:21 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:49:21 | INFO | Fetching in progress .... +2026-06-22 15:49:21 | INFO | Fetched 160 rows from SQL Server +2026-06-22 15:49:21 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! +2026-06-22 15:49:21 | INFO | Fetched 160 rows +2026-06-22 15:49:21 | INFO | Truncated table SKU_Master +2026-06-22 15:49:21 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:49:21 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-22 15:49:21 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-22 15:49:21 | INFO | ================================================================================ +2026-06-22 15:49:21 | INFO | Processing Table-:display_master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-22 15:49:21 | INFO | Fetching Data from sql server for table-: display_master .............. +2026-06-22 15:49:21 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:49:21 | INFO | Fetching in progress .... +2026-06-22 15:49:21 | INFO | Fetched 135 rows from SQL Server +2026-06-22 15:49:21 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! +2026-06-22 15:49:21 | INFO | Fetched 135 rows +2026-06-22 15:49:22 | INFO | Truncated table display_master +2026-06-22 15:49:22 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:49:22 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-22 15:49:22 | INFO | display_master loaded successfully (135 rows) +2026-06-22 15:49:22 | INFO | ================================================================================ +2026-06-22 15:49:22 | INFO | Processing Table-:Employee_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-22 15:49:22 | INFO | Fetching Data from sql server for table-: Employee_Master .............. +2026-06-22 15:49:22 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:49:22 | INFO | Fetching in progress .... +2026-06-22 15:49:22 | INFO | Fetched 2,289 rows from SQL Server +2026-06-22 15:49:22 | INFO | Fetched total row -: 2289 from sql server for table-:Employee_Master ...........!!! +2026-06-22 15:49:22 | INFO | Fetched 2289 rows +2026-06-22 15:49:22 | INFO | Truncated table Employee_Master +2026-06-22 15:49:22 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:49:23 | INFO | Employee_Master: inserted 2,289 rows into ClickHouse +2026-06-22 15:49:23 | INFO | Employee_Master loaded successfully (2289 rows) +2026-06-22 15:49:23 | INFO | ================================================================================ +2026-06-22 15:49:23 | INFO | Processing Table-:coverage_remarks | Table type -:DIMENSION | fetcht by-:reason_id | operation-:DELETE+INSERT +2026-06-22 15:49:23 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. +2026-06-22 15:49:23 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:49:23 | INFO | Fetching in progress .... +2026-06-22 15:49:23 | INFO | Fetched 0 rows from SQL Server +2026-06-22 15:49:23 | INFO | Fetched 0 rows from SQL Server +2026-06-22 15:49:23 | INFO | Fetched total row -: 0 from sql server for table-:coverage_remarks ...........!!! +2026-06-22 15:49:23 | WARNING | coverage_remarks returned no rows +2026-06-22 15:49:23 | INFO | ================================================================================ +2026-06-22 15:49:23 | INFO | Processing Table-:mapping_storevisibility | Table type -:BRIDGE | fetcht by-:run_date | operation-:ONLY_INSERT +2026-06-22 15:49:23 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-22 15:49:24 | WARNING | Table 'mapping_storevisibility' does not exist. During collecting store_ids +2026-06-22 15:49:24 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:49:24 | INFO | Fetching in progress .... +2026-06-22 15:49:24 | INFO | Fetched 0 rows from SQL Server +2026-06-22 15:49:24 | INFO | Fetched 0 rows from SQL Server +2026-06-22 15:49:24 | INFO | Fetched total row -: 0 from sql server for table-:mapping_storevisibility ...........!!! +2026-06-22 15:49:24 | WARNING | mapping_storevisibility returned no rows +2026-06-22 15:49:24 | INFO | ================================================================================ +2026-06-22 15:49:24 | INFO | Processing Table-:Master_VisibilityReason | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-22 15:49:24 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-22 15:49:24 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:49:24 | INFO | Fetching in progress .... +2026-06-22 15:49:24 | INFO | Fetched 17 rows from SQL Server +2026-06-22 15:49:24 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-22 15:49:24 | INFO | Fetched 17 rows +2026-06-22 15:49:24 | INFO | Truncated table Master_VisibilityReason +2026-06-22 15:49:24 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:49:24 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-22 15:49:24 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-22 15:49:24 | INFO | ================================================================================ +2026-06-22 15:49:24 | INFO | Processing Table-:Master_VisibilityDefinition | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-22 15:49:24 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. +2026-06-22 15:49:24 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:49:24 | INFO | Fetching in progress .... +2026-06-22 15:49:25 | INFO | Fetched 861 rows from SQL Server +2026-06-22 15:49:25 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! +2026-06-22 15:49:25 | INFO | Fetched 861 rows +2026-06-22 15:49:25 | INFO | Truncated table Master_VisibilityDefinition +2026-06-22 15:49:25 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:49:25 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-22 15:49:25 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-22 15:49:25 | INFO | ================================================================================ +2026-06-22 15:49:25 | INFO | Processing Table-:Master_Salesterritorylayer | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-22 15:49:25 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. +2026-06-22 15:49:25 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:49:25 | INFO | Fetching in progress .... +2026-06-22 15:49:25 | INFO | Fetched 33 rows from SQL Server +2026-06-22 15:49:25 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! +2026-06-22 15:49:25 | INFO | Fetched 33 rows +2026-06-22 15:49:25 | INFO | Truncated table Master_Salesterritorylayer +2026-06-22 15:49:25 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:49:25 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-22 15:49:25 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-22 15:49:25 | INFO | ================================================================================ +2026-06-22 15:49:25 | INFO | Pipeline Completed Successfully +2026-06-22 15:49:25 | INFO | ================================================================================ +2026-06-22 15:49:34 | INFO | ================================================================================ +2026-06-22 15:49:34 | INFO | Hello from data-move Python data pipeline! +2026-06-22 15:49:34 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 15:49:34 | INFO | Connecting to databases... +2026-06-22 15:49:35 | INFO | +2026-06-22 15:49:37 | INFO | +2026-06-22 15:49:37 | INFO | Database connections established +2026-06-22 15:49:37 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 15:49:38 | INFO | Found 569 MIDs +2026-06-22 15:49:38 | INFO | ================================================================================ +2026-06-22 15:49:38 | INFO | Processing Table-:SOS_OneApp | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-22 15:49:38 | INFO | Fetching Data from sql server for table-: SOS_OneApp .............. +2026-06-22 15:49:38 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 15:49:38 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:49:38 | INFO | Fetching data for 569 MIDs +2026-06-22 15:49:39 | INFO | Fetched 1,907 rows from SQL Server +2026-06-22 15:49:39 | INFO | Fetched total row -: 1907 from sql server for table-:SOS_OneApp ...........!!! +2026-06-22 15:49:39 | INFO | Fetched 1907 rows +2026-06-22 15:49:39 | INFO | _ _ _ _ Deleting Data from ClickHouse for SOS_OneApp _ _ _ _ +2026-06-22 15:49:40 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:49:40 | INFO | SOS_OneApp: inserted 1,907 rows into ClickHouse +2026-06-22 15:49:40 | INFO | SOS_OneApp loaded successfully (1907 rows) +2026-06-22 15:49:40 | INFO | ================================================================================ +2026-06-22 15:49:40 | INFO | Processing Table-:OQaD | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-22 15:49:40 | INFO | Fetching Data from sql server for table-: OQaD .............. +2026-06-22 15:49:41 | INFO | Fetching quiz_empids data for EMPID and Visitid +2026-06-22 15:49:54 | INFO | Fetched 291,198 total empid and visitdate fetched for OQAD from SQL Server +2026-06-22 15:49:58 | INFO | Fetched 290,722 matched empids fetched for OQAD +2026-06-22 15:49:58 | INFO | Exists: True +2026-06-22 15:49:58 | INFO | Path: D:\z\airflow-project\src\sql\fact\OQaD.sql +2026-06-22 15:49:58 | INFO | Fetching data for 290,722 EMPIDs +2026-06-22 15:49:58 | INFO | Fetching OQaD data for run_date=2026-06-21 +2026-06-22 15:49:59 | INFO | fn name is fetch_OQad ------Fetched 0 rows +2026-06-22 15:49:59 | INFO | Fetched 0 rows from SQL Server +2026-06-22 15:49:59 | INFO | Fetched total row -: 0 from sql server for table-:OQaD ...........!!! +2026-06-22 15:49:59 | WARNING | OQaD returned no rows +2026-06-22 15:49:59 | INFO | ================================================================================ +2026-06-22 15:49:59 | INFO | Processing Table-:Survey | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-22 15:49:59 | INFO | Fetching Data from sql server for table-: Survey .............. +2026-06-22 15:49:59 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:49:59 | INFO | Fetching data for 569 MIDs +2026-06-22 15:50:00 | INFO | Fetched 165 rows from SQL Server +2026-06-22 15:50:00 | INFO | Fetched total row -: 165 from sql server for table-:Survey ...........!!! +2026-06-22 15:50:00 | INFO | Fetched 165 rows +2026-06-22 15:50:00 | INFO | _ _ _ _ Deleting Data from ClickHouse for Survey _ _ _ _ +2026-06-22 15:50:00 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:50:00 | INFO | Survey: inserted 165 rows into ClickHouse +2026-06-22 15:50:00 | INFO | Survey loaded successfully (165 rows) +2026-06-22 15:50:00 | INFO | ================================================================================ +2026-06-22 15:50:00 | INFO | Processing Table-:additional_visibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-22 15:50:00 | INFO | Fetching Data from sql server for table-: additional_visibility .............. +2026-06-22 15:50:00 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 15:50:00 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:50:00 | INFO | Fetching data for 569 MIDs +2026-06-22 15:50:01 | INFO | Fetched 1,579 rows from SQL Server +2026-06-22 15:50:01 | INFO | Fetched total row -: 1579 from sql server for table-:additional_visibility ...........!!! +2026-06-22 15:50:01 | INFO | Fetched 1579 rows +2026-06-22 15:50:01 | INFO | _ _ _ _ Deleting Data from ClickHouse for additional_visibility _ _ _ _ +2026-06-22 15:50:01 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:50:01 | INFO | additional_visibility: inserted 1,579 rows into ClickHouse +2026-06-22 15:50:01 | INFO | additional_visibility loaded successfully (1579 rows) +2026-06-22 15:50:01 | INFO | ================================================================================ +2026-06-22 15:50:01 | INFO | Processing Table-:Coverage | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-22 15:50:01 | INFO | Fetching Data from sql server for table-: Coverage .............. +2026-06-22 15:50:01 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:50:01 | INFO | Fetching data for 569 MIDs +2026-06-22 15:50:31 | INFO | Fetched 512 rows from SQL Server +2026-06-22 15:50:31 | INFO | Fetched total row -: 512 from sql server for table-:Coverage ...........!!! +2026-06-22 15:50:31 | INFO | Fetched 512 rows +2026-06-22 15:50:33 | INFO | _ _ _ _ Deleting Data from ClickHouse for Coverage _ _ _ _ +2026-06-22 15:50:33 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:50:34 | INFO | Coverage: inserted 512 rows into ClickHouse +2026-06-22 15:50:34 | INFO | Coverage loaded successfully (512 rows) +2026-06-22 15:50:34 | INFO | ================================================================================ +2026-06-22 15:50:34 | INFO | Processing Table-:Login | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-22 15:50:34 | INFO | Fetching Data from sql server for table-: Login .............. +2026-06-22 15:50:34 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:50:34 | INFO | Fetching data for 569 MIDs +2026-06-22 15:50:37 | INFO | Fetched 454 rows from SQL Server +2026-06-22 15:50:37 | INFO | Fetched total row -: 454 from sql server for table-:Login ...........!!! +2026-06-22 15:50:37 | INFO | Fetched 454 rows +2026-06-22 15:50:38 | INFO | _ _ _ _ Deleting Data from ClickHouse for Login _ _ _ _ +2026-06-22 15:50:38 | INFO | No delete logic required for Login +2026-06-22 15:50:38 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:50:38 | INFO | Login: inserted 454 rows into ClickHouse +2026-06-22 15:50:38 | INFO | Login loaded successfully (454 rows) +2026-06-22 15:50:38 | INFO | ================================================================================ +2026-06-22 15:50:38 | INFO | Processing Table-:Stock_Details | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-22 15:50:38 | INFO | Fetching Data from sql server for table-: Stock_Details .............. +2026-06-22 15:50:38 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:50:38 | INFO | Fetching data for 569 MIDs +2026-06-22 15:50:46 | INFO | Fetched 27,359 rows from SQL Server +2026-06-22 15:50:46 | INFO | Fetched total row -: 27359 from sql server for table-:Stock_Details ...........!!! +2026-06-22 15:50:46 | INFO | Fetched 27359 rows +2026-06-22 15:50:46 | INFO | _ _ _ _ Deleting Data from ClickHouse for Stock_Details _ _ _ _ +2026-06-22 15:50:46 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:50:47 | INFO | Stock_Details: inserted 27,359 rows into ClickHouse +2026-06-22 15:50:47 | INFO | Stock_Details loaded successfully (27359 rows) +2026-06-22 15:50:47 | INFO | ================================================================================ +2026-06-22 15:50:47 | INFO | Processing Table-:Attendance | Table type -:FACT | fetcht by-:run_date | operation-:DELETE+INSERT +2026-06-22 15:50:47 | INFO | Fetching Data from sql server for table-: Attendance .............. +2026-06-22 15:50:47 | INFO | Fetching Attendance data from 2026-06-06 to 2026-06-21 +2026-06-22 15:51:08 | INFO | Fetched 121,642 attendance rows for 545 employees +2026-06-22 15:51:08 | INFO | Fetched total row -: 121642 from sql server for table-:Attendance ...........!!! +2026-06-22 15:51:08 | INFO | Fetched 121642 rows +2026-06-22 15:51:08 | INFO | Truncated table Attendance +2026-06-22 15:51:08 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:51:09 | INFO | Attendance: inserted 121,642 rows into ClickHouse +2026-06-22 15:51:09 | INFO | Attendance loaded successfully (121642 rows) +2026-06-22 15:51:09 | INFO | ================================================================================ +2026-06-22 15:51:09 | INFO | Processing Table-:Journey_Plan | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-22 15:51:09 | INFO | Fetching Data from sql server for table-: Journey_Plan .............. +2026-06-22 15:51:09 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 15:51:09 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:51:09 | INFO | Fetching data for 569 MIDs +2026-06-22 15:51:10 | INFO | Fetched 21,643 rows from SQL Server +2026-06-22 15:51:10 | INFO | Fetched total row -: 21643 from sql server for table-:Journey_Plan ...........!!! +2026-06-22 15:51:10 | INFO | Fetched 21643 rows +2026-06-22 15:51:11 | INFO | _ _ _ _ Deleting Data from ClickHouse for Journey_Plan _ _ _ _ +2026-06-22 15:51:11 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:51:11 | INFO | Journey_Plan: inserted 21,643 rows into ClickHouse +2026-06-22 15:51:11 | INFO | Journey_Plan loaded successfully (21643 rows) +2026-06-22 15:51:11 | INFO | ================================================================================ +2026-06-22 15:51:11 | INFO | Processing Table-:Web_Logins | Table type -:FACT | fetcht by-:run_date | operation-:INSERT +2026-06-22 15:51:11 | INFO | Fetching Data from sql server for table-: Web_Logins .............. +2026-06-22 15:51:11 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 15:51:11 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:51:11 | INFO | Fetching data for 569 MIDs +2026-06-22 15:51:11 | INFO | Fetched 12 rows from SQL Server +2026-06-22 15:51:11 | INFO | Fetched total row -: 12 from sql server for table-:Web_Logins ...........!!! +2026-06-22 15:51:11 | INFO | Fetched 12 rows +2026-06-22 15:51:11 | INFO | _ _ _ _ Deleting Data from ClickHouse for Web_Logins _ _ _ _ +2026-06-22 15:51:11 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:51:11 | INFO | Web_Logins: inserted 12 rows into ClickHouse +2026-06-22 15:51:11 | INFO | Web_Logins loaded successfully (12 rows) +2026-06-22 15:51:11 | INFO | ================================================================================ +2026-06-22 15:51:11 | INFO | Processing Table-:Promotion | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-22 15:51:11 | INFO | Fetching Data from sql server for table-: Promotion .............. +2026-06-22 15:51:11 | INFO | Fetching data for 569 MIDs +2026-06-22 15:51:14 | INFO | Fetched 4,223 rows from SQL Server +2026-06-22 15:51:14 | INFO | Fetched total row -: 4223 from sql server for table-:Promotion ...........!!! +2026-06-22 15:51:14 | INFO | Fetched 4223 rows +2026-06-22 15:51:14 | INFO | _ _ _ _ Deleting Data from ClickHouse for Promotion _ _ _ _ +2026-06-22 15:51:14 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:51:15 | INFO | Promotion: inserted 4,223 rows into ClickHouse +2026-06-22 15:51:15 | INFO | Promotion loaded successfully (4223 rows) +2026-06-22 15:51:15 | INFO | ================================================================================ +2026-06-22 15:51:15 | INFO | Processing Table-:PaidVisibility | Table type -:FACT | fetcht by-:mids | operation-:INSERT +2026-06-22 15:51:15 | INFO | Fetching Data from sql server for table-: PaidVisibility .............. +2026-06-22 15:51:15 | INFO | Start Fetching data for these 569 MIDs +2026-06-22 15:51:15 | INFO | Start Fetching data for these 569 MIDs or based on this date 2026-06-21 +2026-06-22 15:51:15 | INFO | Fetching data for 569 MIDs +2026-06-22 15:51:16 | INFO | Fetched 1,284 rows from SQL Server +2026-06-22 15:51:16 | INFO | Fetched total row -: 1284 from sql server for table-:PaidVisibility ...........!!! +2026-06-22 15:51:16 | INFO | Fetched 1284 rows +2026-06-22 15:51:16 | INFO | _ _ _ _ Deleting Data from ClickHouse for PaidVisibility _ _ _ _ +2026-06-22 15:51:16 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:51:16 | INFO | PaidVisibility: inserted 1,284 rows into ClickHouse +2026-06-22 15:51:16 | INFO | PaidVisibility loaded successfully (1284 rows) +2026-06-22 15:51:16 | INFO | ================================================================================ +2026-06-22 15:51:16 | INFO | Processing Table-:Store_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-22 15:51:16 | INFO | Fetching Data from sql server for table-: Store_Master .............. +2026-06-22 15:51:16 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:51:16 | INFO | Fetching in progress .... +2026-06-22 15:51:18 | INFO | Fetched 5,998 rows from SQL Server +2026-06-22 15:51:18 | INFO | Fetched total row -: 5998 from sql server for table-:Store_Master ...........!!! +2026-06-22 15:51:18 | INFO | Fetched 5998 rows +2026-06-22 15:51:18 | INFO | Truncated table Store_Master +2026-06-22 15:51:18 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:51:18 | INFO | Store_Master: inserted 5,998 rows into ClickHouse +2026-06-22 15:51:18 | INFO | Store_Master loaded successfully (5998 rows) +2026-06-22 15:51:18 | INFO | ================================================================================ +2026-06-22 15:51:18 | INFO | Processing Table-:SKU_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-22 15:51:18 | INFO | Fetching Data from sql server for table-: SKU_Master .............. +2026-06-22 15:51:18 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:51:18 | INFO | Fetching in progress .... +2026-06-22 15:51:19 | INFO | Fetched 160 rows from SQL Server +2026-06-22 15:51:19 | INFO | Fetched total row -: 160 from sql server for table-:SKU_Master ...........!!! +2026-06-22 15:51:19 | INFO | Fetched 160 rows +2026-06-22 15:51:19 | INFO | Truncated table SKU_Master +2026-06-22 15:51:19 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:51:19 | INFO | SKU_Master: inserted 160 rows into ClickHouse +2026-06-22 15:51:19 | INFO | SKU_Master loaded successfully (160 rows) +2026-06-22 15:51:19 | INFO | ================================================================================ +2026-06-22 15:51:19 | INFO | Processing Table-:display_master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-22 15:51:19 | INFO | Fetching Data from sql server for table-: display_master .............. +2026-06-22 15:51:19 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:51:19 | INFO | Fetching in progress .... +2026-06-22 15:51:19 | INFO | Fetched 135 rows from SQL Server +2026-06-22 15:51:19 | INFO | Fetched total row -: 135 from sql server for table-:display_master ...........!!! +2026-06-22 15:51:19 | INFO | Fetched 135 rows +2026-06-22 15:51:19 | INFO | Truncated table display_master +2026-06-22 15:51:19 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:51:19 | INFO | display_master: inserted 135 rows into ClickHouse +2026-06-22 15:51:19 | INFO | display_master loaded successfully (135 rows) +2026-06-22 15:51:19 | INFO | ================================================================================ +2026-06-22 15:51:19 | INFO | Processing Table-:Employee_Master | Table type -:DIMENSION | fetcht by-:master | operation-:DELETE+INSERT +2026-06-22 15:51:19 | INFO | Fetching Data from sql server for table-: Employee_Master .............. +2026-06-22 15:51:19 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:51:19 | INFO | Fetching in progress .... +2026-06-22 15:51:20 | INFO | Fetched 2,289 rows from SQL Server +2026-06-22 15:51:20 | INFO | Fetched total row -: 2289 from sql server for table-:Employee_Master ...........!!! +2026-06-22 15:51:20 | INFO | Fetched 2289 rows +2026-06-22 15:51:20 | INFO | Truncated table Employee_Master +2026-06-22 15:51:20 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:51:21 | INFO | Employee_Master: inserted 2,289 rows into ClickHouse +2026-06-22 15:51:21 | INFO | Employee_Master loaded successfully (2289 rows) +2026-06-22 15:51:21 | INFO | ================================================================================ +2026-06-22 15:51:21 | INFO | Processing Table-:coverage_remarks | Table type -:DIMENSION | fetcht by-:reason_id | operation-:DELETE+INSERT +2026-06-22 15:51:21 | INFO | Fetching Data from sql server for table-: coverage_remarks .............. +2026-06-22 15:51:21 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:51:21 | INFO | Fetching in progress .... +2026-06-22 15:51:21 | INFO | Fetched 0 rows from SQL Server +2026-06-22 15:51:21 | INFO | Fetched 0 rows from SQL Server +2026-06-22 15:51:21 | INFO | Fetched total row -: 0 from sql server for table-:coverage_remarks ...........!!! +2026-06-22 15:51:21 | WARNING | coverage_remarks returned no rows +2026-06-22 15:51:21 | INFO | ================================================================================ +2026-06-22 15:51:21 | INFO | Processing Table-:mapping_storevisibility | Table type -:BRIDGE | fetcht by-:run_date | operation-:ONLY_INSERT +2026-06-22 15:51:21 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-22 15:51:22 | WARNING | Table 'mapping_storevisibility' does not exist. During collecting store_ids +2026-06-22 15:51:22 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:51:22 | INFO | Fetching in progress .... +2026-06-22 15:51:22 | INFO | Fetched 0 rows from SQL Server +2026-06-22 15:51:22 | INFO | Fetched 0 rows from SQL Server +2026-06-22 15:51:22 | INFO | Fetched total row -: 0 from sql server for table-:mapping_storevisibility ...........!!! +2026-06-22 15:51:22 | WARNING | mapping_storevisibility returned no rows +2026-06-22 15:51:22 | INFO | ================================================================================ +2026-06-22 15:51:22 | INFO | Processing Table-:Master_VisibilityReason | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-22 15:51:22 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-22 15:51:22 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:51:22 | INFO | Fetching in progress .... +2026-06-22 15:51:22 | INFO | Fetched 17 rows from SQL Server +2026-06-22 15:51:22 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-22 15:51:22 | INFO | Fetched 17 rows +2026-06-22 15:51:22 | INFO | Truncated table Master_VisibilityReason +2026-06-22 15:51:22 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:51:22 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-22 15:51:22 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-22 15:51:22 | INFO | ================================================================================ +2026-06-22 15:51:22 | INFO | Processing Table-:Master_VisibilityDefinition | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-22 15:51:22 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. +2026-06-22 15:51:22 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:51:22 | INFO | Fetching in progress .... +2026-06-22 15:51:23 | INFO | Fetched 861 rows from SQL Server +2026-06-22 15:51:23 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! +2026-06-22 15:51:23 | INFO | Fetched 861 rows +2026-06-22 15:51:23 | INFO | Truncated table Master_VisibilityDefinition +2026-06-22 15:51:23 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:51:23 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-22 15:51:23 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-22 15:51:23 | INFO | ================================================================================ +2026-06-22 15:51:23 | INFO | Processing Table-:Master_Salesterritorylayer | Table type -:DIMENSION | fetcht by-:none | operation-:DELETE+INSERT +2026-06-22 15:51:23 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. +2026-06-22 15:51:23 | INFO | Fetching data from sql server for Master table...... +2026-06-22 15:51:23 | INFO | Fetching in progress .... +2026-06-22 15:51:23 | INFO | Fetched 33 rows from SQL Server +2026-06-22 15:51:23 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! +2026-06-22 15:51:23 | INFO | Fetched 33 rows +2026-06-22 15:51:23 | INFO | Truncated table Master_Salesterritorylayer +2026-06-22 15:51:23 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 15:51:23 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-22 15:51:23 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-22 15:51:23 | INFO | ================================================================================ +2026-06-22 15:51:23 | INFO | Pipeline Completed Successfully +2026-06-22 15:51:23 | INFO | ================================================================================ +2026-06-22 15:51:23 | INFO | Pipeline completed successfully. last_successful_run_date=2026-06-22 diff --git a/main2.py b/main2.py new file mode 100644 index 0000000..87439c4 --- /dev/null +++ b/main2.py @@ -0,0 +1,313 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "polars>=0.20.0", +# "pyarrow>=18.0.0", +# "sqlalchemy>=2.0.0", +# "pyodbc>=5.0.0", +# "clickhouse-connect>=0.7.0", +# "clickhouse-sqlalchemy>=0.3.2", +# "pyyaml>=6.0.3", +# "python-dotenv>=1.0.0", +# ] +# /// + + +from __future__ import annotations +from time import sleep +import sys +from datetime import date, datetime, timedelta + +import polars as pl +import yaml + + +from log import log + +from clickhouse_task.create_table import create_clickhouse_table +from clickhouse_task.delete_task import ( + delete_existing_data, + truncate_table, +) + +from clickhouse_task.load_table import load_to_clickhouse + +from db_con.connection import ( + build_sql_server_engine, + build_clickhouse_engine, + get_clickhouse_client, +) + +from mids import ( + MID_TABLE_COV, + MID_TABLE_COV1, +) + +from src.bridge import * +from src.fact import * +from src.dim import * + + +# ========================================================== +# Helpers +# ========================================================== + +def table_exists( + client, + table_name: str, +) -> bool: + + return bool( + client.command( + f"EXISTS TABLE {table_name}" + ) + ) + + +# ========================================================== +# Main +# ========================================================== + +def main(): + + log.info("=" * 80) + log.info("Hello from data-move Python data pipeline!") + + # ------------------------------------------------------ + # Run Date + # ------------------------------------------------------ + + if len(sys.argv) > 1: + run_date = datetime.strptime( + sys.argv[1], + "%Y-%m-%d", + ).date() + else: + run_date = date.today() - timedelta(days=1) + + log.info( + "Pipeline Run Date: %s", + run_date, + ) + + # ------------------------------------------------------ + # Connections + # ------------------------------------------------------ + + log.info( + "Connecting to databases..." + ) + + sql_engine = build_sql_server_engine() + clickhouse_engine = build_clickhouse_engine() + client = get_clickhouse_client() + + log.info( + "Database connections established" + ) + + # ------------------------------------------------------ + # mids Keys + # ------------------------------------------------------ + + mids = MID_TABLE_COV( + sql_engine, + run_date, + ) + + emp_visit_df = MID_TABLE_COV1( + sql_engine, + run_date, + ) + + + # ------------------------------------------------------ + # Config + # ------------------------------------------------------ + + with open( + "t.yml", + "r", + ) as file: + + config = yaml.safe_load(file) + + # ------------------------------------------------------ + # Process Tables + # ------------------------------------------------------ + for table in config["tables"]: + table_name = table["name"] + operation = table["operation"] + fetch_by = table["fetch_by"] + table_type=table["type"] + + log.info("=" * 80) + log.info(f"Processing Table-:{table_name} | Table type -:{table_type} | fetcht by-:{fetch_by} | operation-:{operation}" ) + + try: + + # ------------------------------------------ + # Fetch Data + # ------------------------------------------ + + + log.info(f"Fetching Data from sql server for table-: {table_name} ..............") + fetch_list=["mids" ,"run_date", "reason_id"] + if fetch_by in fetch_list : + fn_name = f"fetch_{table_name}" + fn = globals()[fn_name] + df=fn(sql_engine, table_name , table_type, mids, run_date) + else: + df = fetch_data(sql_engine ,table_name,table_type) + + log.info(f"Fetched total row -: {len(df)} from sql server for table-:{table_name} ...........!!!") + + if df.is_empty(): + + log.warning( + "%s returned no rows", + table_name, + ) + + continue + + log.info( + "Fetched %s rows", + len(df), + ) + + # ------------------------------------------ + # Create Table If Missing + # ------------------------------------------ + + exists = table_exists( + client, + table_name, + ) + + if not exists: + + log.info( + "Creating table %s", + table_name, + ) + + create_clickhouse_table( + df=df, + table_name=table_name, + clickhouse_engine=clickhouse_engine, + ) + + # ------------------------------------------ + # Existing Table Logic + # ------------------------------------------ + + else: + + if operation == "DELETE+INSERT": + + truncate_table( + client, + table_name, + ) + + elif operation =="ONLY_INSERT" : + continue + else: + + delete_existing_data( + client=client, + table_name=table_name, + run_date=run_date, + mids=mids, + emp_visit_df=emp_visit_df, + ) + + # ------------------------------------------ + # Load Data + # ------------------------------------------ + log.info("_ _ _ _Inserting data into clickhouse db from sql server_ _ _ _") + load_to_clickhouse( + client=client, + table_name=table_name, + df=df, + ) + + log.info( + "%s loaded successfully (%s rows)", + table_name, + len(df), + ) + + except Exception: + + log.exception( + "Failed processing table %s", + table_name, + ) + + raise + + log.info("=" * 80) + log.info("Pipeline Completed Successfully") + log.info("=" * 80) + + +if __name__ == "__main__": + + config_file = Path("Pipeline_config.yml") + + if not config_file.exists(): + default_config = { + "pipeline": { + "run_date": None, + "status": None, + "error_message": None, + } + } + + with open(config_file, "w") as f: + yaml.safe_dump(default_config, f) + + with open(config_file, "r") as f: + config = yaml.safe_load(f) + + + + for attempt in range(3): + try: + main() + + with open("Pipeline_config.yml", "r") as f: + config = yaml.safe_load(f) + + config["pipeline"]["last_successful_run_date"] = str(date.today()) + + with open("Pipeline_config.yml", "w") as f: + yaml.safe_dump(config, f, sort_keys=False) + + log.info( + f"Pipeline completed successfully. " + f"last_successful_run_date={date.today()}" + ) + + break + + except Exception as e: + + with open("Pipeline_config.yml", "r") as f: + config = yaml.safe_load(f) + + config["pipeline"]["run_date"] = str(date.today()) + + with open("Pipeline_config.yml", "w") as f: + yaml.safe_dump(config, f, sort_keys=False) + + if attempt == 2: + raise + + log.warning( + f"Pipeline failed. Retry {attempt + 1}/3. Error: {e}" + ) + + sleep(5) \ No newline at end of file diff --git a/mids.py b/mids.py new file mode 100644 index 0000000..417b658 --- /dev/null +++ b/mids.py @@ -0,0 +1,74 @@ +import os +# import pyarrow +import sys +import logging +from datetime import date, timedelta +import polars as pl +from sqlalchemy import create_engine, text +from sqlalchemy.engine import Engine, URL +import clickhouse_connect +from dotenv import load_dotenv + +from log import log +from clickhouse_task.create_table import * +from db_con.connection import * + + + + + + + +def MID_TABLE_COV(engine: Engine, target_date: date) -> list[int]: + + sql = text(""" + SELECT MID FROM OneApp_KelloggsMT.dbo.T_StoreCoverage + WHERE CONVERT(date, CreateDate) = :target_date + UNION + SELECT MID FROM OneApp_KelloggsMT.dbo.T_StoreCoverage + WHERE CONVERT(date, UpdateDate) = :target_date + """) + + log.info(f"Collecting MIDs for: {target_date}") + + with engine.connect() as conn: + result = conn.execute(sql, {"target_date": target_date}) + mids = [row[0] for row in result.fetchall()] + log.info(f"Found {len(mids):,} MIDs") + return mids + +def MID_TABLE_COV1( + engine: Engine, + target_date: date, +) -> pl.DataFrame: + + query = f""" + SELECT + EmpId, + CAST(VisitDate AS DATE) AS VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CAST(CreateDate AS DATE) = '{target_date}' + + UNION + + SELECT + EmpId, + CAST(VisitDate AS DATE) AS VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CAST(UpdateDate AS DATE) = '{target_date}' + """ + + return pl.read_database( + query=query, + connection=engine, + ) + + + + + + + + + + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f894d7a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "data-move" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.14" +dependencies = [ + "clickhouse-connect>=1.3.0", + "clickhouse-sqlalchemy>=0.3.2", + "dotenv>=0.9.9", + "polars>=1.41.2", + "pyarrow>=24.0.0", + "pyodbc>=5.3.0", + "pyyaml>=6.0.3", +] diff --git a/src/OQAD.py b/src/OQAD.py new file mode 100644 index 0000000..e69de29 diff --git a/src/bridge.py b/src/bridge.py new file mode 100644 index 0000000..8edb255 --- /dev/null +++ b/src/bridge.py @@ -0,0 +1,110 @@ +from pathlib import Path +import polars as pl +from sqlalchemy import Engine +from datetime import date , timedelta +from log import log + + + + +from db_con.connection import ( + build_sql_server_engine, + build_clickhouse_engine, + get_clickhouse_client, +) + + + + + +def fetch_mapping_storevisibility( + sql_engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + run_date = run_date + timedelta(days=1) + client= get_clickhouse_client() + def table_exists( + client, + table_name: str, + ) -> bool: + + return bool( + client.command( + f"EXISTS TABLE {table_name}" + ) + ) + def get_reason_ids_mapping_storevisibility( + client, + run_date: date, + table_name: str = "mapping_storevisibility", + ) -> list[int] : + + if not table_exists(client, table_name): + log.warning(f"Table '{table_name}' does not exist. During collecting store_ids") + return [0] + + + query = f""" + SELECT DISTINCT StoreId + FROM mapping_storevisibility + WHERE toDate(Fromdate) <= toDate('{run_date + timedelta(days= 1)}') + AND toDate(Todate) >= toDate('{run_date + timedelta(days= 1)}') + AND project_Id = '40148' + + """ + + # ClickHouse -> PyArrow -> Polars + arrow_table = client.query_arrow(query) + + df= pl.from_arrow(arrow_table) + list=df["reason_id"].to_list() + return list + + def fetch_data( + engine: Engine, + table_name: str, + table_type: str, + run_date: date, + store_id: list[int] +) -> pl.DataFrame: + log.info(f"Fetching data from sql server for Master table......") + + store_id_list = ",".join(str(sid) for sid in store_id) + + sql_file = Path("src") / "sql" / f"bridge" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + store_id_list=store_id_list, + run_date=run_date.strftime("%Y-%m-%d") + + ) + + log.info(f"Fetching in progress .... ") + + df = pl.read_database( + query=sql, + connection=engine + ) + + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + store_id=get_reason_ids_mapping_storevisibility(client, run_date, "mapping_storevisibility") + df=fetch_data(engine=sql_engine, + table_name=table_name, + table_type=table_type, + run_date=run_date, + store_id=store_id, + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + diff --git a/src/dim.py b/src/dim.py new file mode 100644 index 0000000..0a1085a --- /dev/null +++ b/src/dim.py @@ -0,0 +1,138 @@ +from pathlib import Path +import polars as pl +from sqlalchemy import Engine +from datetime import date , timedelta +from log import log + + + + +from db_con.connection import ( + build_sql_server_engine, + build_clickhouse_engine, + get_clickhouse_client, +) + + + + + + + + + + +def fetch_data( + engine: Engine, + table_name: str, + table_type: str + +) -> pl.DataFrame: + + + + log.info(f"Fetching data from sql server for Master table......") + + sql_file = Path("src") / "sql" / f"dim" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( ) + + log.info(f"Fetching in progress .... ") + + df = pl.read_database( + query=sql, + connection=engine + ) + + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + +def fetch_coverage_remarks( + sql_engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date + +) -> pl.DataFrame: + + + client= get_clickhouse_client() + def table_exists( + client, + table_name: str, + ) -> bool: + + return bool( + client.command( + f"EXISTS TABLE {table_name}" + ) + ) + def get_reason_ids_coverage_remarks( + client, + table_name: str = "coverage_remarks", + ) -> list[int] : + + if not table_exists(client, table_name): + log.warning(f"Table '{table_name}' does not exist. During collecting reason_ids") + return [0] + + + query = f""" + SELECT DISTINCT + reason_id + FROM {table_name} + """ + + # ClickHouse -> PyArrow -> Polars + arrow_table = client.query_arrow(query) + + df= pl.from_arrow(arrow_table) + list=df["reason_id"].to_list() + return list + + def fetch_data( + engine: Engine, + table_name: str, + table_type: str, + reason_ids: list[int] +) -> pl.DataFrame: + log.info(f"Fetching data from sql server for Master table......") + + resaon_id_list = ",".join(str(rid) for rid in reason_ids) + + sql_file = Path("src") / "sql" / f"dim" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + resaon_id_list=resaon_id_list + + ) + + log.info(f"Fetching in progress .... ") + + df = pl.read_database( + query=sql, + connection=engine + ) + + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + reason_ids=get_reason_ids_coverage_remarks(client, "coverage_remarks") + df=fetch_data(engine=sql_engine, + table_name=table_name, + table_type=table_type, + reason_ids=reason_ids, + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df diff --git a/src/fact.py b/src/fact.py new file mode 100644 index 0000000..dcf2f21 --- /dev/null +++ b/src/fact.py @@ -0,0 +1,723 @@ +from pathlib import Path +import polars as pl +from sqlalchemy import Engine +from datetime import date , timedelta +from log import log + + + + +from db_con.connection import ( + build_sql_server_engine, + build_clickhouse_engine, + get_clickhouse_client, +) + + + + + + + + + + + + + + +# def fetch_data( +# engine: Engine, +# table_name: str, +# table_type: str, +# mids: list[int], +# run_date: date +# ) -> pl.DataFrame: + +# if not mids: +# log.warning("No MIDs — nothing to fetch.") +# return pl.DataFrame() + +# mid_list = ",".join(str(mid) for mid in mids) + +# sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + +# with open(sql_file, "r", encoding="utf-8") as f: +# sql_template = f.read() + +# sql = sql_template.format( +# mid_list=mid_list, +# run_date=run_date.strftime("%Y-%m-%d") +# ) + +# log.info(f"Fetching data for {len(mids):,} MIDs") + +# df = pl.read_database( +# query=sql, +# connection=engine +# ) + +# log.info(f"Fetched {len(df):,} rows from SQL Server") + +# return df + + + +def fetch_SOS_OneApp( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + log.info(f" Start Fetching data for these {len(mids):} MIDs ") + mid_list = ",".join(str(mid) for mid in mids) + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + +def fetch_additional_visibility( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + + log.info(f" Start Fetching data for these {len(mids):} MIDs ") + mid_list = ",".join(str(mid) for mid in mids) + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + +def fetch_OQaD( + sql_engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + client= get_clickhouse_client() + def table_exists( + client, + table_name: str, + ) -> bool: + + return bool( + client.command( + f"EXISTS TABLE {table_name}" + ) + ) + + + + def fetch_quiz_empids(engine: Engine, run_date : date) -> pl.DataFrame: + + sql_template = f""" + WITH MID_TABLE_COV1 AS + ( + SELECT EmpId, VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CreateDate >= {run_date} + AND CreateDate < DATEADD(DAY,1,'{run_date}') + + UNION ALL + + SELECT EmpId, VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE UpdateDate >= {run_date} + AND UpdateDate < DATEADD(DAY,1, '{run_date}') + ), + + QUIZ AS + ( + SELECT Distinct E.EmpId as empid + , CONVERT(date,DQ.VisitDate) AS visitdate + FROM OneApp_KelloggsMT.dbo.T_OQAD DQ INNER JOIN + OneApp_KelloggsMT.dbo.vw_Employee_Detail E ON DQ.EmpId = E.EmpId inner join + OneApp_KelloggsMT.dbo.Master_OQAD_Question QU on DQ.QuestionId= qu.QuestionId inner join + OneApp_KelloggsMT.dbo.Master_OQAD_Category qc on qu.QuestionCategoryId= qc.QuestionCategoryId + where e.EmpName not like 'test%' and e.RightId in (6) + and (E.ResignDate is null or E.ResignDate>=''+CONVERT(VARCHAR,'{run_date}')+'') AND E.EmpName NOT LIKE '%TEST%' + AND DQ.EmpId IN (SELECT EmpId FROM MID_TABLE_COV1 A WHERE + DQ.EmpId=A.EmpId AND CONVERT(date,VisitDate)=CONVERT(date,A.VisitDate) ) + ) select * from quiz + """ + sql = sql_template.format( + run_date=run_date.strftime("%Y-%m-%d") + ) + + log.info(f"Fetching quiz_empids data for EMPID and Visitid") + + df = pl.read_database( + query=sql, + connection=engine + ) + + + log.info(f"Fetched {len(df):,} total empid and visitdate fetched for OQAD from SQL Server") + + return df + + + def get_empids_clickhouse_OQAD( + client, + table_name: str = "OQaD", + ) -> pl.DataFrame: + + if not table_exists(client, table_name): + log.warning(f"Table '{table_name}' does not exist.") + return pl.DataFrame( + schema={ + "empid": pl.Int64, + "visitdate": pl.Date, + } + ) + + query = f""" + SELECT DISTINCT + employee_id AS empid, + visit_date AS visitdate + FROM {table_name} + """ + + # ClickHouse -> PyArrow -> Polars + arrow_table = client.query_arrow(query) + + return pl.from_arrow(arrow_table) + + + + qf=fetch_quiz_empids(sql_engine,run_date) + db_df = get_empids_clickhouse_OQAD(client) + + matched = qf.join( + db_df, + on=["empid", "visitdate"], + how="inner", + ) + + if matched.is_empty(): + + empids=[0] + log.warning( + "%s Matched df in OQaD returned no rows", + table_name, + ) + + else: + empids=matched["empid"].to_list() + + + log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ") + + def fetch_data( + engine: Engine, + table_name: str, + table_type: str, + empids: list[int], + run_date: date + ) -> pl.DataFrame: + + empid_list = ",".join(str(empid) for empid in empids) + + + sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql" + + log.info(f"Exists: {sql_file.exists()}") + log.info(f"Path: {sql_file.resolve()}") + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + empid_list=empid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + + log.info(f"Fetching data for {len(empids):,} EMPIDs") + + log.info("Fetching OQaD data for run_date=%s", run_date) + + df = pl.read_database( + query=sql, + connection=engine, + ) + + log.info("fn name is fetch_OQad ------Fetched %s rows", len(df)) + + return df + df=fetch_data( engine=sql_engine, + table_name=table_name, + table_type=table_type, + empids=empids, + run_date=run_date + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + +# def fetch_OQaD( +# engine: Engine, +# table_name: str, +# table_type: str, +# empids: list[int], +# run_date: date +# ) -> pl.DataFrame: + +# empid_list = ",".join(str(empid) for empid in empids) + + +# sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql" + +# log.info(f"Exists: {sql_file.exists()}") +# log.info(f"Path: {sql_file.resolve()}") + +# with open(sql_file, "r", encoding="utf-8") as f: +# sql_template = f.read() + +# sql = sql_template.format( +# empid_list=empid_list, +# run_date=run_date.strftime("%Y-%m-%d") +# ) + +# log.info(f"Fetching data for {len(empids):,} EMPIDs") + +# log.info("Fetching OQaD data for run_date=%s", run_date) + +# df = pl.read_database( +# query=sql, +# connection=engine, +# ) + +# log.info("fn name is fetch_OQad ------Fetched %s rows", len(df)) + +# return df + + + + + + +def fetch_Coverage( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + mid_list = ",".join(str(mid) for mid in mids) + + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + + +def fetch_Survey( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + + mid_list = ",".join(str(mid) for mid in mids) + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + + + + +def fetch_Login( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + # if not mids: + # log.warning("No MIDs — nothing to fetch.") + # return pl.DataFrame() + + mid_list = ",".join(str(mid) for mid in mids) + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + # mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + + + + + +def fetch_Stock_Details( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + + mid_list = ",".join(str(mid) for mid in mids) + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + + + +# def fetch_Coverage( engine: Engine, +# table_name: str, +# table_type: str, +# mids: list[int], +# run_date: date +# ) -> pl.DataFrame: + + + +# if not mids: +# log.warning("No MIDs — nothing to fetch.") +# return pl.DataFrame() + +# mid_list = ",".join(str(mid) for mid in mids) +# log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") +# sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + +# with open(sql_file, "r", encoding="utf-8") as f: +# sql_template = f.read() + +# sql = sql_template.format( +# mid_list=mid_list, +# run_date=run_date.strftime("%Y-%m-%d") +# ) +# log.info(f"Fetching data for {len(mids):,} MIDs") + +# df = pl.read_database( +# query=sql, +# connection=engine +# ) +# log.info(f"Fetched {len(df):,} rows from SQL Server") + +# return df + + + +def fetch_Attendance( + engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date | None = None, + days_back: int = 15 +) -> pl.DataFrame: + """ + Fetch attendance source data. + + Default: + end_date = yesterday + start_date = yesterday - 15 days + """ + + if run_date is None: + run_date = date.today() - timedelta(days=1) + + start_date = run_date - timedelta(days=days_back) + + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + start_date=start_date.strftime("%Y-%m-%d"), + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info( + f"Fetching Attendance data from {start_date} to {run_date}" + ) + + df = pl.read_database( + query=sql, + connection=engine + ) + + log.info( + f"Fetched {len(df):,} attendance rows " + f"for {df['employee_id'].n_unique():,} employees" + ) + + return df + + + +def fetch_Journey_Plan( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + + log.info(f" Start Fetching data for these {len(mids):} MIDs ") + mid_list = ",".join(str(mid) for mid in mids) + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + +def fetch_PaidVisibility( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + + log.info(f" Start Fetching data for these {len(mids):} MIDs ") + mid_list = ",".join(str(mid) for mid in mids) + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + +def fetch_Web_Logins( engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + + log.info(f" Start Fetching data for these {len(mids):} MIDs ") + mid_list = ",".join(str(mid) for mid in mids) + log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}") + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df + + +def fetch_Promotion( + engine: Engine, + table_name: str, + table_type: str, + mids: list[int], + run_date: date +) -> pl.DataFrame: + + if not mids: + log.warning("No MIDs — nothing to fetch.") + return pl.DataFrame() + + mid_list = ",".join(str(mid) for mid in mids) + + sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" + + with open(sql_file, "r", encoding="utf-8") as f: + sql_template = f.read() + + sql = sql_template.format( + mid_list=mid_list, + run_date=run_date.strftime("%Y-%m-%d") + ) + + log.info(f"Fetching data for {len(mids):,} MIDs") + + df = pl.read_database( + query=sql, + connection=engine + ) + + log.info(f"Fetched {len(df):,} rows from SQL Server") + + return df \ No newline at end of file diff --git a/src/sql/bridge/mapping_storevisibility.sql b/src/sql/bridge/mapping_storevisibility.sql new file mode 100644 index 0000000..af35f60 --- /dev/null +++ b/src/sql/bridge/mapping_storevisibility.sql @@ -0,0 +1,11 @@ +with mapping_storevisibility +(Project_Id,StoreId,VisibilityDefinitionid,Fromdate,Todate,CreateDate,CreateBy) +AS ( +select DISTINCT '40148' as Project_Id,StoreId,VisibilityDefinitionid,Fromdate,Todate,getdate(),'SP-Pius' +FROM OneApp_KelloggsMT.dbo.mapping_storevisibility z WHERE +convert(date,FROMDATE,101)<=convert(Date,getdate(),101) AND CONVERT(DATE,ToDate,101)>=convert(Date,getdate(),101) +AND z.VisibilityDefinitionid IN +(SELECT DISTINCT VisibilityDefinitionid FROM OneApp_KelloggsMT.dbo.MASTER_VISIBILITYDEFINITION WHERE MENUID=22 ) +AND z.StoreId NOT IN ({store_id_list}) +) +select * from mapping_storevisibility \ No newline at end of file diff --git a/src/sql/dim/Employee_Master.sql b/src/sql/dim/Employee_Master.sql new file mode 100644 index 0000000..7cd00b9 --- /dev/null +++ b/src/sql/dim/Employee_Master.sql @@ -0,0 +1,17 @@ + with Employee_Master (project_id,region_id,region,state_id,state,city_id, + city,employee_id,employee_name,gender,designation_id,designation,manager_id, + manager_name,employee_joining_date,employee_resign_date,position_code,employee_legacy_code,employee_role,EMPLOYEE_TYPE) + as ( +select '40148' as ProjectId, RegionId, + RegionName,StateId,StateName, CityId,CityName, + a.EmpId,EmpName,Gender,a.DesignationId,DesignationName,SupervisorId,SupervisorName,JoinDate,ResignDate,c.PositionCode ,EmpCode ,RIGHTNAME, + + case when RIGHTNAME in ('Client','Client HO') Then 'NON CPM' else 'CPM' END AS EMPLOYEE_TYPE +from OneApp_KelloggsMT.dbo.vw_Employee_Detail a Left join (Select distinct PositionId, EmpId from OneApp_KelloggsMT.dbo.Mapping_PositionUser b + Where DATEdiff(d,FromDate, GETDATE())>= 0 and DATEDIFF(d,ToDate,getdate())<=0) b on a.EmpId=b.EmpId + LEFT join (select PositionId,PositionCode from OneApp_KelloggsMT.dbo.Master_Position) c on b.PositionId=c.PositionId + where 1=1 + -- and EmpId not in ( + --select employee_id from [dbo].[Employee_Master] where project_id='40148') + ) +select * from Employee_Master \ No newline at end of file diff --git a/src/sql/dim/Master_Salesterritorylayer.sql b/src/sql/dim/Master_Salesterritorylayer.sql new file mode 100644 index 0000000..853c238 --- /dev/null +++ b/src/sql/dim/Master_Salesterritorylayer.sql @@ -0,0 +1,14 @@ +with Master_Salesterritorylayer(ProjectId,StLayerOneId,StLayerOneName,StLayerTwoId,StLayerTwoName,StLayerThreeId,StLayerThreeName,StLayerFourId, +StLayerFourName,CreateDate,CreateBy) +as ( +select DISTINCT '40148' as Project_Id,d.StLayerOneId,d.StLayerOneName,c.StLayerTwoId,c.StLayerTwoName , +b.StLayerThreeId,b.StLayerThreeName,a.StLayerFourId,a.StLayerFourName , +getdate(),'SP-Pius' +FROM OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerFour] A +INNER JOIN OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerThree]B ON + A.StLayerThreeId=B.StLayerThreeId + INNER JOIN OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerTWo]C ON + B.StLayerTwoId=C.StLayerTwoId + INNER JOIN OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerOne]D ON + C.StLayerOneId=D.StLayerOneId) + select * from Master_Salesterritorylayer \ No newline at end of file diff --git a/src/sql/dim/Master_VisibilityDefinition.sql b/src/sql/dim/Master_VisibilityDefinition.sql new file mode 100644 index 0000000..78d7731 --- /dev/null +++ b/src/sql/dim/Master_VisibilityDefinition.sql @@ -0,0 +1,5 @@ +with Master_VisibilityDefinition(Project_Id,VisibilityDefinitionId,VisibilityDefinitionName,CreateDate,CreateBy) +as ( +select DISTINCT '40148' as Project_Id,VisibilityDefinitionid,VisibilityDefinitionname,getdate(),'SP-Pius' +FROM OneApp_KelloggsMT.dbo.Master_VisibilityDefinition) +select * from Master_VisibilityDefinition \ No newline at end of file diff --git a/src/sql/dim/Master_VisibilityReason.sql b/src/sql/dim/Master_VisibilityReason.sql new file mode 100644 index 0000000..02f6453 --- /dev/null +++ b/src/sql/dim/Master_VisibilityReason.sql @@ -0,0 +1,7 @@ +with Master_VisibilityReason (ProjectId,MenuId,ReasonId,Reason,CreateDate,Createby) + as ( + +select DISTINCT '40148' as ProjectId,MenuId,VisibilityReasonId,VisibilityReason,getdate(),'SP-Pius' +FROM OneApp_KelloggsMT.dbo.Master_VisibilityReason + ) +select * from Master_VisibilityReason \ No newline at end of file diff --git a/src/sql/dim/SKU_Master.sql b/src/sql/dim/SKU_Master.sql new file mode 100644 index 0000000..18aa5be --- /dev/null +++ b/src/sql/dim/SKU_Master.sql @@ -0,0 +1,18 @@ + +with SKU_Master (project_id,category_id,category_code,Category_name,sub_category_id,sub_category_code,sub_category_name,brand_id, +brand_code,brand_name,sub_brand_id,sub_brand_code,sub_brand_name,product_id,product_name,product_code,mrp,flavour_id, +flavour,grammage,product_sequence,case_size,company_name,is_competitor,ptr,update_date,update_by) + as ( + + Select '40148' as ProjectId, cm.CategoryId, +cm.CategoryCode,cm.CategoryName,sca.SubCategoryId,sca.SubCategoryCode,sca.SubCategoryName,br.BrandId,br.BrandCode,br.BrandName +,sb.SubBrandId,sb.SubBrandCode,sb.SubBrandName,p.ProductId,p.ProductName,p.ProductCode,p.Mrp,fl.FlavourId, +fl.Flavour,p.Grammage,p.ProductSequence,p.CaseSize,mc.Company,mc.IsCompetitor,p.ptr,GETDATE(),'MB' +from OneApp_KelloggsMT.dbo.Master_Product p Right join +OneApp_KelloggsMT.dbo.Master_Flavour fl on p.FlavourId= Fl.FlavourId Right join +OneApp_KelloggsMT.dbo.Master_subbrand sb on p.SubBrandId= sb.SubBrandId Right join +OneApp_KelloggsMT.dbo.Master_Brand br on sb.BrandId = br.BrandId Right join +OneApp_KelloggsMT.dbo.Master_SubCategory sca on br.SubCategoryId= sca.SubCategoryId Right join +OneApp_KelloggsMT.dbo.Master_Category cm on sca.CategoryId= cm.CategoryId Right join +OneApp_KelloggsMT.dbo.Master_Company mc on mc.companyId=br.companyId ) +select * from SKU_Master diff --git a/src/sql/dim/Store_Master.sql b/src/sql/dim/Store_Master.sql new file mode 100644 index 0000000..a768d40 --- /dev/null +++ b/src/sql/dim/Store_Master.sql @@ -0,0 +1,14 @@ +with Store_Master (project_id, +region_id,region,state_id,state,city_id,city,cpm_city_id,channel_id,channel,distributor_id,distributor_name, keyaccount_id, keyaccount, +insight_store_id,client_store_code,latitude,longitude,store_category_id,store_category,store_type_id,store_type,store_classification_id,store_classification, StLayerFourId, +store_id,store_name,address) + AS ( +select '40148',RegionId,RegionName,StateId,StateName,CityId, +CityName,CityCode,ChannelId,ChannelName,DistributorId,Distributor,ChainId,ChainName,StoreUniqueCode,StoreCode,Latitude, +Longitude, StoreCategoryId,StoreCategory,StoreTypeId, +StoreType,StoreClassId,StoreClass,StLayerFourId,StoreId, +StoreName,Address from OneApp_KelloggsMT.dbo.vw_storedetail where 1=1 +--and storeid not in ( +--select store_id from Store_Master where project_id='40148') +) +select * from Store_Master \ No newline at end of file diff --git a/src/sql/dim/coverage_remarks.sql b/src/sql/dim/coverage_remarks.sql new file mode 100644 index 0000000..3eef46e --- /dev/null +++ b/src/sql/dim/coverage_remarks.sql @@ -0,0 +1,8 @@ + with coverage_remarks + (project_id,reason_id,reason_remarks) + as( + select + '40148' ,reasonid,reason from OneApp_KelloggsMT.dbo.master_nonworkingreason + where reasonid not in ( {resaon_id_list}) + ) + select * from coverage_remarks \ No newline at end of file diff --git a/src/sql/dim/display_master.sql b/src/sql/dim/display_master.sql new file mode 100644 index 0000000..cb0ea82 --- /dev/null +++ b/src/sql/dim/display_master.sql @@ -0,0 +1,8 @@ + with display_master (project_id,display_id,display_code,display_name,display_ref_url,created_date, + created_by) as ( + select '40148',displayid,displaycode,displayname,displayrefImage,getdate(),'Pius' + from OneApp_KelloggsMT.dbo.Master_display +-- and ProductId not in ( +--select product_id from [dbo].[SKU Master] where project_id='40148') + ) +select * from display_master \ No newline at end of file diff --git a/src/sql/fact/Attendance.sql b/src/sql/fact/Attendance.sql new file mode 100644 index 0000000..ecf3b4f --- /dev/null +++ b/src/sql/fact/Attendance.sql @@ -0,0 +1,134 @@ +with Emp_cte as + (Select T1.*, ISNULL(ps.PositionCode ,'') PositionCode,FromDate,ToDate from( + select a.*,b.date from + (Select rm.RegionId, rm.RegionName as Region, st.StateId, st.StateName , cm.CityId, cm.CityName + , em2.Id as [Area Manager Id], em2.EmployeeName AS [Area Manager] + , em1.Id as [Supervisor Id], isnull(em1.EmployeeName,'') as [Supervisor Name] + , em.Id as [Employee Id], isnull(em.EmployeeName,'') as [Employee Name] + , dm.DesignationId, dm.DesignationName as Designation + , CAST(EM.ResignDate AS DATE) as DOR, CAST(EM.JoinDate AS DATE) as DOJ, em.LegacyCode + --,ISNULL(ps.PositionCode ,'') PositionCode--, FromDate, ToDate + from + OneApp_KelloggsMT.dbo.AspNetUsers em inner join + OneApp_KelloggsMT.dbo.AspNetUsers em1 on em1.Id = em.ManagerId inner join + OneApp_KelloggsMT.dbo.AspNetUsers em2 on em2.Id = em1.ManagerId inner join + OneApp_KelloggsMT.dbo.Master_City cm on em.CityId= cm.CityId inner join + OneApp_KelloggsMT.dbo.Master_State st on cm.StateId= st.StateId inner join + OneApp_KelloggsMT.dbo.Master_Region rm on st.RegionId= rm.RegionId inner join + OneApp_KelloggsMT.dbo.Master_Designation dm on Em.DesignationId= dm.DesignationId + + where em.RightId in (6) and em.EmployeeName not like '%test%' and + (Em.ResignDate is null or Em.ResignDate>=CAST('{start_date}' AS DATE)) and CONVERT(date,em.JoinDate,101)<=CONVERT(date,CAST('{run_date}' AS DATE),101) + --and em.Id=2290 + + )a + cross join (select distinct DATE from DBO.GET_ALL_DAYS(CAST('{start_date}' AS DATE),CAST('{run_date}' AS DATE))) b + ) AS T1 LEFT JOIN + (Select distinct EmpId, PositionId, Date,FromDate,ToDate from + (Select distinct EmpId,PositionId,FromDate,ToDate from OneApp_KelloggsMT.dbo.Mapping_PositionUser + Where convert(Date,FromDate)<= CONVERT(date,CAST('{run_date}' AS DATE),101) and convert(Date,ToDate)>=CONVERT(date,CAST('{start_date}' AS DATE),101)) as m Inner Join + (Select Date from OneApp_KelloggsMT.dbo.Master_Calender Where 1=1 and Date between CONVERT(date,CAST('{start_date}' AS DATE),101) and CONVERT(date,CAST('{run_date}' AS DATE),101)) + cal on cal.Date >= m.FromDate AND cal.Date <= m.ToDate) pu + on T1.[Employee Id]=pu.EmpId and T1.date= pu.date Left Join + OneApp_KelloggsMT.dbo.Master_Position ps WITH (NOLOCK) ON pu.PositionId=ps.PositionId + ) , + Attendance + (project_id, + employee_id,position_code,legacy_code,supervisor_id,date_of_join,date_of_resign,visit_date,parinaam_attendance, + update_date,update_by,Unique_Id) + AS ( + + select '40148', emp.[Employee Id], Case When emp.DATE <= ToDate and emp.DATE>= FromDate + then emp.PositionCode Else '' End as PositionCode ,emp.LegacyCode, emp.[Supervisor Id] ,emp.DOJ,emp.DOR, + emp.DATE, case when emp.DATE>=emp.DOR then 'R' WHEN emp.DATE< emp.DOJ THEN '-' + WHEN b.COL IS NULL AND (emp.DATE<=emp.DOR OR emp.DOR IS NULL )AND (emp.DATE>=emp.DOJ OR emp.DOJ IS NULL) THEN 'W' ELSE b.COL + END AS Attendance,GETDATE(),'SP-Pius',CAST('40148' AS VARCHAR) + '_' + CAST(emp.[Employee Id] AS VARCHAR) + from Emp_cte as emp + + left join + + + (select a.* + + + FROM + ( + SELECT A.DATE,TT.[MERCHANDISER CD],TT.VisitDate,TT.DOJ1,TT.DOR1,TT.COL from DBO.GET_ALL_DAYS(CAST('{start_date}' AS DATE),CAST('{run_date}' AS DATE)) a + left join + + + ( + + SELECT distinct [EMPLOYEE ID] AS [MERCHANDISER CD], VISITDATE + , CASE WHEN COL=1 THEN 'H' WHEN COL=2 THEN 'L' WHEN COL=4 THEN 'M' WHEN COL=5 THEN 'CO' WHEN COL=6 THEN 'WO' + WHEN COL=7 THEN 'P' WHEN COL=10 THEN 'EH' WHEN COL=8 THEN 'HD' WHEN COL IS NULL THEN 'A' END AS COL + , DOJ AS DOJ1, DOR AS DOR1 + FROM + ( + SELECT DOJ, DOR, Region, [AREA MANAGER], SUPERVISOR_CD, [SUPERVISOR], City, [EMPLOYEE ID], MERCHANDISER, VISITDATE + , REMARKS, COL, ROW_NUMBER()OVER (PARTITION BY [EMPLOYEE ID], VISITDATE ORDER BY COL DESC,Intime, VISITDATE, ROW ) AS COL1 + FROM + ( + SELECT DOJ, DOR, Region, [AREA MANAGER], SUPERVISOR_CD, [SUPERVISOR], City, [EMPLOYEE ID], MERCHANDISER, InTime, VISITDATE + , REMARKS, COL, ROW_NUMBER()OVER (PARTITION BY [EMPLOYEE ID], VISITDATE ORDER BY COL) AS ROW + FROM + ( + SELECT RM.RegionName as Region, em2.EmployeeName AS [Area Manager] + , em1.Id as Supervisor_cd, em1.EmployeeName AS [Supervisor] + , CM.CityName as City, EM.Id as [Employee Id], em.EmployeeName AS Merchandiser + ,JP.VisitDate as VisitDate, sc.InTime + ,ISNULL((SELECT top 1 case + when d.ReasonId IS NULL OR d.ReasonId =0 then '' else REASON+' - ' + + case when replace(replace(D.REMARK,CHAR(13),''),CHAR(10),' ')<>'' then replace(replace(D.REMARK,CHAR(13),''),CHAR(10),' ') + else replace(replace(D.REMARK,CHAR(13),''),CHAR(10),' ') end end as REASON FROM OneApp_KelloggsMT.dbo.T_StoreCoverage D + INNER JOIN OneApp_KelloggsMT.dbo.Master_NonWorkingReason N ON N.ReasonId = D.ReasonId + WHERE D.ISDEL = 0 + AND D.EmpId = JP.EmpId AND D.StoreId = JP.StoreId AND d.VisitDate = JP.VisitDate),'') AS Remarks + , CASE WHEN (Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId = SC.EmpId AND JP.StoreId = SC.StoreId AND JP.VisitDate = SC.VisitDate + AND SC.ReasonId=11) >=1 THEN 1 + ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate + AND SC.ReasonId=2) >=1 THEN 2 + ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate + AND SC.ReasonId=6) >=1 THEN 4 + ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate + AND SC.ReasonId=12) >=1 THEN 5 + ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate + AND SC.ReasonId=13) >=1 THEN 6 + ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate + AND SC.ReasonId=7) >=1 THEN 8 + ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate + AND SC.ReasonId=26) >=1 THEN 10 + ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate + AND SC.ReasonId IN(0,1,3,4,5,8,9,10,14,15,17,18,19,20,21,22,24)) >=1 THEN 7 + END END END END END END END END AS COL, em.ResignDate as DOR, em.JoinDate as DOJ + + FROM OneApp_KelloggsMT.dbo.Mapping_JourneyPlan JP LEFT OUTER JOIN + OneApp_KelloggsMT.dbo.T_StoreCoverage SC ON JP.StoreId= SC.StoreId AND JP.EmpId= SC.EmpId AND JP.VisitDate= SC.VisitDate INNER JOIN + OneApp_KelloggsMT.dbo.Master_Store SM ON JP.StoreId= SM.StoreId INNER JOIN + OneApp_KelloggsMT.dbo.AspNetUsers em ON JP.EmpId= em.Id inner join + OneApp_KelloggsMT.dbo.AspNetUsers em1 on em1.Id = em.ManagerId inner join + OneApp_KelloggsMT.dbo.AspNetUsers em2 on em2.Id = em1.ManagerId inner join + OneApp_KelloggsMT.dbo.Master_City cm on em.CityId= cm.CityId inner join + OneApp_KelloggsMT.dbo.Master_State st on cm.StateId= st.StateId inner join + OneApp_KelloggsMT.dbo.Master_Region rm on st.RegionId= rm.RegionId --Left Join + + + + + WHERE EM.EMPLOYEENAME not like 'teststore%' AND JP.VisitDate BETWEEN CAST('{start_date}' AS DATE) AND CAST('{run_date}' AS DATE) + + ) AS T1 + ) AS T2 + ) AS T + WHERE COL1=1 + ) AS TT + on a.date=tt.visitdate)A)b + on emp.[Employee Id]=b.[MERCHANDISER CD] and ( + emp.DOR IS NULL + OR emp.DATE >= emp.DOR + ) ) + select * from Attendance where + parinaam_attendance !='-' and + project_id=40148 + AND visit_date > date_of_join + order by employee_id, visit_date \ No newline at end of file diff --git a/src/sql/fact/Coverage.sql b/src/sql/fact/Coverage.sql new file mode 100644 index 0000000..bd3f337 --- /dev/null +++ b/src/sql/fact/Coverage.sql @@ -0,0 +1,108 @@ + +with Executed as +(select JP.mid, CASE WHEN isnull(B.MID,'')<>'' then 'Y' +ELSE 'N' END AS COVERED, +CASE WHEN isnull(C.MID,'')<>'' then 'Y' +ELSE 'N' END AS EXECUTED,isnull(NR.REASON,'') AS REASON + +from OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) left join +(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) +inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B + ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) + AND JP.ReasonId in (0,1,3,9,10,19,20))b +on JP.MID=b.MID + +left join +(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock) +inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B + ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101) + AND JP.ReasonId in (0,19))c +on JP.MID=c.MID +left join +OneApp_KelloggsMT.dbo.Master_NonWorkingReason nr on JP.ReasonId=nr.ReasonId +Where 1=1 AND JP.MID in ({mid_list}) + +) , + Coverage + (project_id,Mid, +store_id,employee_id,visit_date,in_time,out_time,duration_in_minutes, +is_covered,is_executed,reason_remarks,storetype_id, +supervisor_id,coverage_type,distance_in_meters,reasonId,camera_allow,update_date, +update_by,Unique_Id) + +AS ( + +Select '40148' as ProjectId,JP.MID, sm.StoreId,JP.EmpId as EmpId, + Convert(VARCHAR,Jp.VisitDate) AS VisitDate +, Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, + Convert(Datetime,InTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, + Convert(Datetime,InTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage Where EmpId = Jp.EmpId And StoreId = Jp.StoreId + And VisitDate = Jp.VisitDate),'') As [In Time] + , Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time, + Convert(Datetime,OutTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time, + Convert(Datetime,OutTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage + Where EmpId = Jp.EmpId And StoreId = Jp.StoreId + And Convert(Date,VisitDate) = Jp.VisitDate),'') As [Out Time] + , + Isnull((Select Top 1 Case When convert(varchar,outtime) like '%00:00:00.00%' THEN 0 + Else Case When outtime0 Then 'Y' +Else 'N' End As Covered +, isnull(Exe.Executed,'') as Executed + +,Isnull((Select Top 1 Case +When D.ReasonId Is Null Or D.ReasonId =0 Then '' Else Reason+' - ' + +Case When Replace(Replace(D.Remark,Char(13),''''),Char(10),' ')<>'' Then Replace(Replace(D.Remark,Char(13),''),Char(10),' ') +Else Replace(Replace(D.Remark,Char(13),''),Char(10),' ') End End As Reason +From OneApp_KelloggsMT.dbo.T_StoreCoverage D +Inner Join OneApp_KelloggsMT.dbo.Master_NonWorkingReason N On N.ReasonId = D.ReasonId +Where D.Isdel = 0 +And D.EmpId = Jp.EmpId And D.StoreId = Jp.StoreId And D.VisitDate = Jp.VisitDate),'''') As [Detailed Remarks], +sm.StoreTypeId,Em.SupervisorId, +Case When jp.Deviation=0 Then 'Planned' When jp.Deviation=1 Then 'Adhoc' +When jp.Deviation=2 Then 'Beat Plan' When jp.Deviation=3 Then 'Non Merchandised' +When jp.Deviation=4 Then 'Add New Store' When jp.Deviation=5 Then 'Non Program' else '' End as [PJP Status] +,Isnull((Select Top 1 Case When (Sc.Latitude=0.00000000 Or Sc.Longitude Is Null) Then 0 Else Case When (sm.Latitude=0.00000000 Or sm.Latitude Is Null) Then 0 + Else Case When (sm.Longitude=0.00000000 Or sm.Longitude Is Null) Then 0 +Else SQRT(POWER(69.1 * ( Sc.Latitude - Sm.Latitude),2) + POWER(69.1 * ( Sm.Longitude - Sc.Longitude ) * COS(Sc.Latitude / 57.3), 2))*1000 + End End End As [Distance In Mtr] From OneApp_KelloggsMT.dbo.T_StoreCoverage Sc Where Sc.Isdel=0 And Sc.EmpId= Jp.EmpId +And Sc.StoreId= Jp.StoreId And Sc.VisitDate= Jp.VisitDate),'') As [Distance In Mtr] +,ISNULL(CAST (JP.ReasonId AS VARCHAR),'0'),sm.CameraAllow, + +GETDATE(),'SP-Pius' , + + CAST('40148' AS VARCHAR) + '_' + CAST(SM.storeid AS VARCHAR) + + '_' + CAST(EM.EMPID AS VARCHAR) + + + +FROM OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) Inner Join +Executed Exe on Jp.MID= Exe.MID Inner Join +OneApp_KelloggsMT.dbo.vw_StoreDetail sm on Jp.StoreId= sm.StoreId Inner Join +OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on JP.EmpId= Em.EmpId +left join +OneApp_KelloggsMT.dbo.T_StoreCoveragePositionPivot pv with (nolock) +on jp.mid=pv.MID +Where 1=1 and em.UserName not like 'test%' AND JP.MID in ({mid_list} ) + +) +select * from Coverage \ No newline at end of file diff --git a/src/sql/fact/Journey_Plan.sql b/src/sql/fact/Journey_Plan.sql new file mode 100644 index 0000000..f777999 --- /dev/null +++ b/src/sql/fact/Journey_Plan.sql @@ -0,0 +1,10 @@ +with Journey_Plan + (project_id,store_id,employee_id,visit_date,process_id,CreateDate,CreateBy,UpdateDate, + UpdateBy) + AS ( + select + '40148' ,storeid,EmpId,cast(visitdate as DATE),Deviation,GETDATE(),'Pius',GETDATE(),'Pius' from OneApp_KelloggsMT.dbo.mapping_journeyplan + where MONTH(VisitDate) = MONTH(CAST('{run_date}' AS DATE)) AND Year(VisitDate)=Year(CAST('{run_date}' AS DATE)) + AND EmpId NOT IN ( SELECT ID FROM OneApp_KelloggsMT.dbo.ASPNETUSERS + WHERE USERNAME LIKE 'TEST%')) +select * from Journey_Plan; \ No newline at end of file diff --git a/src/sql/fact/Login.sql b/src/sql/fact/Login.sql new file mode 100644 index 0000000..7fd83dd --- /dev/null +++ b/src/sql/fact/Login.sql @@ -0,0 +1,81 @@ +with employee_detail AS +( + SELECT rm.RegionId, rm.RegionName, st.StateId, st.StateName, + cm.CityId, cm.CityName, + em1.Id AS [Supervisor Code], ISNULL(em1.EmployeeName, '') AS Supervisor, + em.Id AS [Employee code], ISNULL(em.EmployeeName, '') AS [Employee Name], + dm.DesignationId, dm.DesignationName AS Designation + FROM OneApp_KelloggsMT.dbo.AspNetUsers em + INNER JOIN OneApp_KelloggsMT.dbo.AspNetUsers em1 ON em1.Id = em.ManagerId + INNER JOIN OneApp_KelloggsMT.dbo.Master_City cm ON em.CityId = cm.CityId + INNER JOIN OneApp_KelloggsMT.dbo.Master_State st ON cm.StateId = st.StateId + INNER JOIN OneApp_KelloggsMT.dbo.Master_Region rm ON st.RegionId = rm.RegionId + INNER JOIN OneApp_KelloggsMT.dbo.Master_Designation dm ON Em.DesignationId = dm.DesignationId + WHERE em.RightId IN (6) + AND (Em.ResignDate IS NULL OR CONVERT(DATE, Em.ResignDate, 101) >= CAST('{run_date}' AS DATE)) + AND em.EmployeeName NOT LIKE '%test%' +), +Login ( + project_id, + employee_id, login_date, login_time, process_id, + first_store_in_time, last_store_out_time, update_date, update_by, + Unique_Id +) AS ( + SELECT '40148' AS ProjectId, + Em.[Employee Code], + [Login Date], + [Login Time], + '0', + [1st Store In Time], + [Last Store Out Time], + GETDATE(), + 'SP-Pius', + CAST(40148 AS VARCHAR) + '_' + CAST(Em.[Employee Code] AS VARCHAR) + FROM employee_detail Em + LEFT OUTER JOIN + ( + SELECT SupervisorId AS [Supervisor Code], + SupervisorName, + EmpId AS [Employee Code], + EmpName, + DesignationName, + [Login Date], + [Login Time], + [1st Store In Time], + [Last Store Out Time], + AppVersion AS [App Version] + FROM ( + SELECT Em.SupervisorId, + Em.SupervisorName, + Em.EmpId, + Em.EmpName, + Em.DesignationName, + CONVERT(VARCHAR, ud.LoginDate, 101) AS [Login Date], + CONVERT(VARCHAR, ud.InTime, 108) AS [Login Time], + ud.AppVersion, + ROW_NUMBER() OVER ( + PARTITION BY ud.Empid, CONVERT(VARCHAR, ud.LoginDate, 101) + ORDER BY ud.LoginDate ASC + ) AS col, + (SELECT MIN(CONVERT(NVARCHAR, sc.InTime, 108)) + FROM OneApp_KelloggsMT.dbo.T_StoreCoverage sc + WHERE sc.Isdel = 0 + AND sc.EmpId = ud.EmpId + AND CONVERT(VARCHAR, sc.VisitDate, 101) = CONVERT(VARCHAR, ud.LoginDate, 101) + ) AS [1st Store In Time], + (SELECT MAX(CONVERT(NVARCHAR, OutTime, 108)) + FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC + WHERE SC.Isdel = 0 + AND SC.EmpId = UD.EmpId + AND CONVERT(VARCHAR, SC.VisitDate, 101) = CONVERT(VARCHAR, UD.LoginDate, 101) + ) AS [Last Store Out Time] + FROM OneApp_KelloggsMT.dbo.T_DeviceLogin ud + INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em ON ud.EmpId = Em.EmpId + WHERE CAST(ud.LoginDate AS DATE) = CAST('{run_date}' AS DATE) -- fixed: direct date comparison + ) AS T1 + WHERE T1.col = 1 + ) AS T ON Em.[Employee code] = T.[Employee Code] + WHERE T.[Login Date] IS NOT NULL +) + +SELECT * FROM Login; \ No newline at end of file diff --git a/src/sql/fact/OQaD.sql b/src/sql/fact/OQaD.sql new file mode 100644 index 0000000..c2cbeca --- /dev/null +++ b/src/sql/fact/OQaD.sql @@ -0,0 +1,81 @@ +WITH MID_TABLE_COV1 AS +( + SELECT EmpId, VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE CreateDate >= {run_date} + AND CreateDate < DATEADD(DAY,1,'{run_date}') + + UNION ALL + + SELECT EmpId, VisitDate + FROM OneApp_KelloggsMT.dbo.T_OQAD + WHERE UpdateDate >= {run_date} + AND UpdateDate < DATEADD(DAY,1, '{run_date}') +), + + +QUIZ AS +( + SELECT DISTINCT + E.EmpId, + E.EmpName, + E.SupervisorId, + E.SupervisorName, + E.DesignationName, + E.CityName, + E.StateName, + E.RegionName, + CAST(DQ.VisitDate AS DATE) AS VisitDate, + DQ.QuestionId, + DQ.AnswerId, + QC.QuestionCategoryId, + QC.QuestionCategory + FROM OneApp_KelloggsMT.dbo.T_OQAD DQ + INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E + ON DQ.EmpId = E.EmpId + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU + ON DQ.QuestionId = QU.QuestionId + INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC + ON QU.QuestionCategoryId = QC.QuestionCategoryId + WHERE E.EmpName NOT LIKE '%TEST%' + AND E.RightId = 6 + AND ( + E.ResignDate IS NULL + OR CAST(E.ResignDate AS DATE) >= '{run_date}' + ) + AND EXISTS + ( + SELECT 1 + FROM MID_TABLE_COV1 A + WHERE A.EmpId = DQ.EmpId + AND CAST(A.VisitDate AS DATE) = CAST(DQ.VisitDate AS DATE) + ) +) + +SELECT + 40148 AS project_id, + Q.EmpId AS employee_id, + 0 AS process_id, + Q.VisitDate AS visit_date, + Q.QuestionCategoryId AS question_category_id, + Q.QuestionCategory AS question_category, + QM.QuestionId AS question_id, + QM.Question AS question, + ISNULL(QA.AnswerId,0) AS answer_id, + ISNULL(QA.Answer,'') AS answer, + CASE + WHEN QA.AnswerId IS NULL THEN 'Not Answer' + WHEN QA.RightAnswer = 1 THEN 'Y' + WHEN QA.RightAnswer IS NULL THEN 'Not Answer' + ELSE 'N' + END AS correct_answer, + GETDATE() AS update_date, + 'SP-Pius' AS update_by +FROM QUIZ Q +INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM + ON Q.QuestionId = QM.QuestionId +LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA + ON Q.AnswerId = QA.AnswerId +where Q.EmpId not in ({empid_list}) + + diff --git a/src/sql/fact/PaidVisibility.sql b/src/sql/fact/PaidVisibility.sql new file mode 100644 index 0000000..84324a7 --- /dev/null +++ b/src/sql/fact/PaidVisibility.sql @@ -0,0 +1,56 @@ + + with PaidVisibility(Mid, + project_id,store_id,employee_id,visit_date,supervisor_id,channel_id,chain_id,storetype_id,Menuid,MenuName, + Visibility_Id, + Visibility_Name, + Visibility_definition_id,Visibility_definition_name, + Visibility_deatils,Visibility_deatils_id,Visibility_value_name, + present,REASONID,reason,VisibilityQuestion,VisibilityAnswer,image1,image2,update_date,update_by) + AS ( + + Select sc.MID, '40148' Projectid,sc.StoreId,Em.EmpId,sc.VisitDate,Em.SupervisorId,sm.ChannelId,sm.ChainId,sm.StoreTypeId, 0 as menuid ,'' as menuname, + ts.visibilityid,mv.Visibilityname,msd.VisibilityDefinitionId, + MSD.VisibilityDefinitionName, case when isnull(TS.VisibilityTable,'')='Master_Category' then 'Category' + when isnull(TS.VisibilityTable,'')='Master_SubCategory' then 'SubCategory' + when isnull(TS.VisibilityTable,'')='Master_Brand' then 'Brand' + when isnull(TS.VisibilityTable,'')='Master_SubBrand' then 'SubBrand' + end as PaidVisibilityDeatils ,ts.visibilityvalue[Paid_visiValueID], + CASE when isnull(TS.VisibilityTable,'')='Master_Category' THEN (SELECT a.CategoryName from OneApp_KelloggsMT.dbo.Master_Category a where ts.VisibilityValue=a.CategoryId ) + when isnull(TS.VisibilityTable,'')='Master_SubCategory' THEN (SELECT a.SubCategoryName from OneApp_KelloggsMT.dbo.Master_SubCategory a where ts.VisibilityValue=a.SubCategoryId ) + when isnull(TS.VisibilityTable,'')='Master_Brand' THEN (SELECT a.BrandName from OneApp_KelloggsMT.dbo.Master_Brand a where ts.VisibilityValue=a.BrandId ) + when isnull(TS.VisibilityTable,'')='Master_SubBrand' THEN (SELECT a.SubBrandName from OneApp_KelloggsMT.dbo.Master_SubBrand a where ts.VisibilityValue=a.SubBrandId ) end as + [Paid_VisiValueName],case when ts.Present=0 then 'N' else 'Y' end Present, + CASE WHEN TS.PRESENT=0 THEN TS.VISIBILITYREASONID ELSE '' END AS VISIBILITYREASONID , + case when ts.Present=0 then isnull(mnp.VisibilityReason,'') else '' end Reason, + ISNULL( MVQ.VisibilityQuestionName,'') AS Question, ISNULL(TVQ.VisibilityAnswerName,'') AS Answer , + case when ts.present=0 then '' else + case when isnull(SHI.VisibilityImage1,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PaidVisibilityImages/'+SHI.VisibilityImage1 + end end as PaidVisibilityImg1 ,case when isnull(SHI.VisibilityImage2,'')='' then '' else 'https://kimt.parinaam.in/Upload/PaidVisibilityImages/'+SHI.VisibilityImage2 + end as PaidVisibilityImg2 ,GETDATE(),'SP-Pius' + + + from + OneApp_KelloggsMT.dbo.T_Visibility ts WITH (NOLOCK) Inner join + OneApp_KelloggsMT.dbo.T_StoreCoverage sc WITH (NOLOCK) on ts.mid= sc.mid Inner Join + OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join + OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId + inner join + OneApp_KelloggsMT.dbo.Master_VisibilityDefinition msd WITH (NOLOCK) on + msd.VisibilityDefinitionId=ts.VisibilityDefinitionId --and msd.menuid=ts.menuid + --inner join OneApp_KelloggsMT.dbo.Master_menu MM WITH (NOLOCK) ON ts.menuid=mm.menuid and msd.menuid=mm.menuid + left join OneApp_KelloggsMT.DBO. master_visibility mv WITH (NOLOCK) on ts.visibilityid=mv.visibilityid + + + LEFT join + OneApp_KelloggsMT.dbo.T_VisibilityImages SHI WITH (NOLOCK) ON ts.VId=SHI.VId + left join + OneApp_KelloggsMT.dbo.Master_VisibilityReason mnp WITH (NOLOCK) on ts.VisibilityReasonId=mnp.VisibilityReasonId + LEFT JOIN OneApp_KelloggsMT.dbo.T_VisibilityStock TVS WITH (NOLOCK) ON TS.VID=TVS.VID + LEFT JOIN OneApp_KelloggsMT.dbo.T_VisibilityQuestion TVQ WITH (NOLOCK) ON TS.Vid=TVQ.Vid + LEFT JOIN OneApp_KelloggsMT.dbo.Master_VisibilityQuestion MVQ WITH (NOLOCK) on TVQ.VisibilityQuestionId=MVQ.VisibilityQuestionId + + Where 1=1 and Em.EmpName not like 'test%' + --and sc.visitdate between '10/01/2023' and '10/31/2023' + AND sc.MID in ({mid_list}) + ) +select * from PaidVisibility \ No newline at end of file diff --git a/src/sql/fact/Promotion.sql b/src/sql/fact/Promotion.sql new file mode 100644 index 0000000..4775b9f --- /dev/null +++ b/src/sql/fact/Promotion.sql @@ -0,0 +1,43 @@ +with Promotion (Mid, + project_id,store_id,employee_id,visit_date,supervisor_id,channel_id,chain_id,storetype_id,promo_definition_id,promo_definition_name, + promotion_deatils,promotion_deatils_id,promotion_value_name, + present,reason,PromoQuestion,PromoAnswer,image1,image2,update_date,update_by) + as ( + +Select +sc.MID, '40148' Projectid,sc.StoreId,Em.EmpId,sc.VisitDate,Em.SupervisorId,sm.ChannelId,sm.ChainId,sm.StoreTypeId,msd.PromoDefinitionId, + MSD.PromoDefinitionName, case when isnull(TS.PromoTable,'')='Master_Category' then 'Category' + when isnull(TS.PromoTable,'')='Master_SubCategory' then 'SubCategory' + when isnull(TS.PromoTable,'')='Master_Brand' then 'Brand' + when isnull(TS.PromoTable,'')='Master_SubBrand' then 'SubBrand' + end as PromotionDeatils , + TS.PromoValue[PromotionDeatilsId], + CASE when isnull(TS.PromoTable,'')='Master_Category' THEN (SELECT a.CategoryName from OneApp_KelloggsMT.dbo.Master_Category a where ts.PromoValue=a.CategoryId ) + when isnull(TS.PromoTable,'')='Master_SubCategory' THEN (SELECT a.SubCategoryName from OneApp_KelloggsMT.dbo.Master_SubCategory a where ts.PromoValue=a.SubCategoryId ) + when isnull(TS.PromoTable,'')='Master_Brand' THEN (SELECT a.BrandName from OneApp_KelloggsMT.dbo.Master_Brand a where ts.PromoValue=a.BrandId ) + when isnull(TS.PromoTable,'')='Master_SubBrand' THEN (SELECT a.SubBrandName from OneApp_KelloggsMT.dbo.Master_SubBrand a where ts.PromoValue=a.SubBrandId ) end as +[PromoValueName],case when ts.Present=0 then 'N' else 'Y' end Present, + case when ts.Present=1 then '' else isnull(mnp.PromoReason,'') end as Reason, ISNULL(mpq.PromoQuestionName,'') AS Question, + ISNULL(tpq.PromoAnswerName,'') AS Answer , + case when isnull(SHI.PromoImage1,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PromotionImages/'+SHI.PromoImage1 + end as Image1 ,case when isnull(SHI.PromoImage2,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PromotionImages/'+SHI.PromoImage2 + end as Image2 ,GETDATE(),'SP-Pius' + +from + OneApp_KelloggsMT.dbo.T_Promotion ts Inner join + OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join + OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join + OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId + inner join + OneApp_KelloggsMT.dbo.Master_PromotionDefinition msd on + msd.PromoDefinitionId=ts.PromoDefinitionId + LEFT join + OneApp_KelloggsMT.dbo.T_PromotionImages SHI ON ts.PId=SHI.PId + left join + OneApp_KelloggsMT.dbo.Master_PromotionReason mnp on ts.PromoReasonId=mnp.PromoReasonId + left join OneApp_KelloggsMT.dbo.t_promotionquestion tpq on ts.PId=tpq.PId + left join OneApp_KelloggsMT.dbo.Master_PromotionQuestion mpq on tpq.PromoQuestionId=mpq.PromoQuestionId + + Where 1=1 and Em.EmpName not like 'test%' +AND sc.MID in ({mid_list}) ) +select * from Promotion \ No newline at end of file diff --git a/src/sql/fact/SOS_OneApp.sql b/src/sql/fact/SOS_OneApp.sql new file mode 100644 index 0000000..df248e2 --- /dev/null +++ b/src/sql/fact/SOS_OneApp.sql @@ -0,0 +1,70 @@ +with SOS_OneApp ( project_id,Mid, +employee_id,store_id,visit_date,storetype_id,channel_id,SOSDefinitionName,SOSHeaderDeatils,SOSHeaderName,SOSHeaderID,SOSChildDeatils,SOSChildName,SOSChildID, +SOSHeaderFacing,ChildTotalFacing,ChildSelfFacing,SOSTarget,update_date,update_by) + as ( +Select '40148' AS ProjectId,MID, +EmpId +,StoreId, +VisitDate,StoreTypeid,ChannelId , +--CameraAllow, +SOSDefinitionName,SOSHeaderDeatils,SOSHeaderName,[SOSHeaderID] ,SOSChildDeatils , SOSChildName,SOSChildID,SOSHeaderFacing,ChildTotalFacing, +ChildSelfFacing ,SOSTarget,getdate(),'SP-Pius' from + ( +Select sm.CountryName,SC.MID, sm.RegionName, sm.StateName, sm.CityName, Em.SupervisorName, em.EmpId as EmpId +, Em.EmpName as EmployeeName, Em.DesignationName as Designation, sm.StoreId,convert(varchar,sc.VisitDate,101)VisitDate, sm.StoreCode, sm.StoreName, +sm.Address, sm.StoreTypeid ,sm.ChannelId,sm.ChainName, +MSD.SOSDefinitionName, case when isnull(TS.SOSHeaderTable,'')='Master_Category' then 'Category' +when isnull(TS.SOSHeaderTable,'')='Master_SubCategory' then 'SubCategory' +when isnull(TS.SOSHeaderTable,'')='Master_Brand' then 'Brand' +when isnull(TS.SOSHeaderTable,'')='Master_SubBrand' then 'SubBrand' +end as SOSHeaderDeatils , +TS.SOSHeaderName,TS.SOSHeaderValue[SOSHeaderID], 'Header'+'_'+'Image' as HDR1, +case when isnull(tsc.SOSChildTable,'')='Master_Category' then 'Category' +when isnull(tsc.SOSChildTable,'')='Master_SubCategory' then 'SubCategory' +when isnull(tsc.SOSChildTable,'')='Master_Brand' then 'Brand' +when isnull(tsc.SOSChildTable,'')='Master_SubBrand' then 'SubBrand' +end as SOSChildDeatils , +TSC.SOSChildName,TSC.SOSChildValue[SOSChildID],TSC.ChildTotalFacing,ts.SOSHeaderFacing,TSC.ChildSelfFacing +, +(select top 1 SOSTarget from OneApp_KelloggsMT.dbo.Mapping_StoreShareOfShelfTarget a where a.SOSDefinitionId=msd.SOSDefinitionId and +a.Storeid=sm.StoreId and a.fromdate<=sc.visitdate and a.todate>=sc.visitdate ) as SOSTarget, +case when isnull(SHI.SOSHeaderImage,'')='' then '' else 'https://kimt1.parinaam.in/Upload/SOSImages/'+SHI.SOSHeaderImage +end as SOSHeaderImg +--,case when isnull(SCI.SOSChildImage,'')='' then '' else 'https://di1.parinaam.in/Upload/SOSImages/'+SCI.SOSChildImage +--end as SOSChildImage + + from +OneApp_KelloggsMT.dbo.T_ShareOfShelfHeader ts Inner join +OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join +OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join +OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId +inner join +OneApp_KelloggsMT.dbo.T_ShareOfShelfChild tsc on ts.sosid=tsc.SOSId +inner join +OneApp_KelloggsMT.dbo.Master_ShareOfShelfDefinition msd on +msd.SOSDefinitionId=tsc.SOSDefinitionId + +LEFT join +OneApp_KelloggsMT.dbo.T_ShareOfShelfHeaderImages SHI ON ts.SOSId=SHI.SOSId AND TS.SOSHeaderValue=SHI.SOSHeaderValue +LEFT JOIN +OneApp_KelloggsMT.dbo.T_ShareOfShelfChildImages SCI ON TSC.SOSId=SCI.SOSId AND TSC.SOSChildValue=SCI.SOSChildValue +Where 1=1 AND EM.EMPNAME NOT LIKE 'test%' AND sc.mid IN ({mid_list}) +-- AND sc.VisitDate BETWEEN '09/18/2023' AND '09/21/2023' +--and sc.StoreId not in ( +--select store_id from SOS_OneApp a where a.project_id=40148 and a.store_id=sc.StoreId +--and a.employee_id=sc.EmpId and convert(date,sc.VisitDate,101)=convert(date,a.visit_date,101) ) + +) as A + +Pivot +( +min(SOSHeaderImg) +for HDR1 in([Header_Image]) +) as pvt1 + +GROUP BY CountryName,MID, RegionName, StateName, CityName, SupervisorName, EmpId +, EmployeeName, Designation, StoreId,VisitDate,StoreCode, StoreName, +Address, StoreTypeid ,ChannelId,ChainName, +SOSDefinitionName,SOSHeaderDeatils,SOSHeaderName,[SOSHeaderID] ,SOSChildDeatils , +SOSChildName,SOSChildID,ChildTotalFacing,SOSHeaderFacing,ChildSelfFacing,SOSTarget ) + select * from SOS_OneApp \ No newline at end of file diff --git a/src/sql/fact/Stock_Details.sql b/src/sql/fact/Stock_Details.sql new file mode 100644 index 0000000..0cb5b0a --- /dev/null +++ b/src/sql/fact/Stock_Details.sql @@ -0,0 +1,29 @@ +with Stock_Details (project_id, +Mid,supervisor_id,employee_id,store_id, +visitdate,storetype_id,store_category_id,product_id,MSL, MBQ,Stock_Qty, +damagedstock,loststock,expirystock,skuavailability,update_date,update_by,StockType) + +AS ( + +Select '40148' AS ProjectId,sc.MID,Em.SupervisorId,Em.EmpId,sm.StoreId,sc.VisitDate,sc.StoreTypeId,sm.StoreCategoryId, +vp.ProductId,ts.MSL,mp.MBQ, ISNULL(ts.OpeningStock,0)+ISNULL(TS.middaystock,0),0,0,0, +Case WHEN ISNULL(ts.OpeningStock,0)+ISNULL(TS.middaystock,0)>=1THEN 'Y' ELSE 'N' END AS +skuavailability, GETDATE(),'SP-Pius','Parinaam' + from +OneApp_KelloggsMT.dbo.T_Stock ts Inner join +OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join +OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join +OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId +inner join OneApp_KelloggsMT.dbo.vw_product vp on ts.ProductId=vp.ProductId inner JOIN +(SELECT MBQ,ProductId,StateId,ChainId,StoreTypeId,StoreCategoryId,StoreClassId FROM OneApp_KelloggsMT.dbo.Mapping_ProductAssortment where FromDate<='{run_date}' and ToDate>='{run_date}') mp on mp.StateId=sm.StateId and +mp.ChainId=sm.ChainId +and mp.StoreTypeId=sm.StoreTypeId +and mp.StoreCategoryId=sm.StoreCategoryId +and mp.StoreClassId=sm.StoreClassId +and mp.ProductId=ts.ProductId + + +Where Em.EmpName not like 'test%' AND sc.MID in ({mid_list}) + +) +select * from Stock_Details \ No newline at end of file diff --git a/src/sql/fact/Survey.sql b/src/sql/fact/Survey.sql new file mode 100644 index 0000000..674422d --- /dev/null +++ b/src/sql/fact/Survey.sql @@ -0,0 +1,68 @@ +WITH QUIZ AS +( + SELECT + SC.MID, + SC.StoreId, + EM.EmpId, + EM.EmpName, + EM.SupervisorId, + EM.SupervisorName, + EM.DesignationName, + CAST(SC.VisitDate AS DATE) AS VisitDate, + DQ.QuestionId, + QU.Question, + DQ.Answer, + QC.CategoryId, + QC.Category, + MSS.SubCategoryId, + MSS.SubCategory, + DQ.ImageUrl + + FROM OneApp_KelloggsMT.dbo.T_SURVEY DQ + + INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC + ON DQ.MID = SC.MID + + INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveyQuestion QU + ON DQ.QuestionId = QU.QuestionId + + INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveySubCategory MSS + ON QU.SubCategoryId = MSS.SubCategoryId + + INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveyCategory QC + ON MSS.CategoryId = QC.CategoryId + + INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM + ON SC.EmpId = EM.EmpId + + INNER JOIN OneApp_KelloggsMT.dbo.Master_Survey MS + ON DQ.SurveyId = MS.SurveyId + AND QU.SurveyId = MS.SurveyId + + WHERE EM.EmpName NOT LIKE 'test%' + AND SC.MID IN ({mid_list}) +) + +SELECT + 40148 AS Project_Id, + Q.MID AS Mid, + Q.SupervisorId AS SuperVisorId, + Q.EmpId AS EmpID, + SM.StoreId, + Q.VisitDate, + SM.StoreTypeId, + SM.ChainId, + Q.CategoryId, + Q.Category, + Q.SubCategoryId, + Q.SubCategory, + Q.QuestionId, + Q.Question, + Q.Answer, + GETDATE() AS CreateDate, + 'SP-Pius' AS CreateBy + +FROM QUIZ Q + +INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM + ON Q.StoreId = SM.StoreId \ No newline at end of file diff --git a/src/sql/fact/Web_Logins.sql b/src/sql/fact/Web_Logins.sql new file mode 100644 index 0000000..8e83d87 --- /dev/null +++ b/src/sql/fact/Web_Logins.sql @@ -0,0 +1,23 @@ + + with Web_Logins (project_id, + supervisor_id,supervisor_name,emp_id,employee_name,designation,date,time,activity_name,activity_type,right_name,CreateDate,CreateBy) + as ( + Select Distinct '40148' as ProjectId + , EM1.Id AS SupervisorId, EM1.EmployeeName AS Supervisor + , EM.Id AS EmployeeId , EM.EmployeeName as Employee + , DM.DesignationName as Designation + , cast( CONVERT(date,AL.Date) as Date) AS Date,CONVERT(NVARCHAR,AL.Date ,108) AS Time + , AL.Thread as ActivityName, AL.Level as ActivityType, R.RightName,GETDATE(),'Pius' + FROM OneApp_KelloggsMT.dbo.T_User_Activity_Log AL LEFT JOIN + OneApp_KelloggsMT.dbo.AspNetUsers EM ON AL.Logger=EM.USERNAME INNER JOIN + OneApp_KelloggsMT.dbo.AspNetUsers EM1 ON EM1.Id = EM.ManagerId INNER JOIN + OneApp_KelloggsMT.dbo.AspNetUsers EM2 ON EM2.Id = EM1.ManagerId INNER JOIN + OneApp_KelloggsMT.dbo.Master_Designation DM ON EM.DesignationId = DM.DesignationId INNER JOIN + OneApp_KelloggsMT.dbo.Master_City CM ON EM.CityId = CM.CityId INNER JOIN + OneApp_KelloggsMT.dbo.Master_State ST ON ST.StateId = cm.StateId INNER JOIN + OneApp_KelloggsMT.dbo.Master_Region RM ON RM.RegionId = ST.RegionId INNER JOIN + OneApp_KelloggsMT.dbo.Right_Master R ON EM.RightId=R.RightId + Where 1=1 and CONVERT(date,AL.Date,101) = CONVERT(DATE,CAST('{run_date}' AS DATE),101) + ) + select * from Web_Logins + ORDER BY employee_name \ No newline at end of file diff --git a/src/sql/fact/additional_visibility.sql b/src/sql/fact/additional_visibility.sql new file mode 100644 index 0000000..ecb0250 --- /dev/null +++ b/src/sql/fact/additional_visibility.sql @@ -0,0 +1,31 @@ + + +WITH + additional_visibility (project_id,Mid, + emp_id,store_id,storetype_id,channel_id,chain_id,camera_allowed,visit_date,is_present,brand_id, + display_id,Remarks,image_url,created_date,created_by) +as ( + + SELECT '40148' AS ProjectId,ts.mid, em.EmpId as EmpId + , sm.StoreId,sm.storetypeid,sm.channelid,sm.chainid,case when sm.cameraallow=1 then 'Y' ELSE 'N' END AS + cameraallow , convert(varchar,sc.VisitDate,101)VisitDate, + case when ts.Present=0 then 'N' else 'Y' end Present, + isnull(MB.BrandId,'')[BrandId], + isnull(MD.DisplayId,'')[DisplayId],ISNULL(TS.REMARK,'')[Remark], + case when isnull(ts.ImageUrl,'')='' then '' else 'https://kimt1.parinaam.in/Upload/VisibilityImages/'+ts.ImageUrl + end as AdditionalImageUrl ,GETDATE(),'Pius' + + from + OneApp_KelloggsMT.dbo.T_AdditionalVisibility ts Inner join + OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join + OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join + OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId + left join + OneApp_KelloggsMT.dbo.Master_Brand MB ON TS.BrandId=MB.BrandId + LEFT JOIN + OneApp_KelloggsMT.dbo.Master_Display MD ON TS.DisplayId=MD.DisplayId + Where Em.EmpName not like 'test%' and ts.Present=1 and sc.mid in ({mid_list}) + +) + +select * from additional_visibility; \ No newline at end of file diff --git a/t.yml b/t.yml new file mode 100644 index 0000000..285375d --- /dev/null +++ b/t.yml @@ -0,0 +1,107 @@ +tables: + - name: SOS_OneApp + type: FACT + operation: INSERT + fetch_by: mids + + - name: OQaD + type: FACT + operation: INSERT + fetch_by: run_date + + - name: Survey + type: FACT + operation: INSERT + fetch_by: mids + + - name: additional_visibility + type: FACT + operation: INSERT + fetch_by: mids + + - name: Coverage + type: FACT + operation: INSERT + fetch_by: mids + + - name: Login + type: FACT + operation: INSERT + fetch_by: run_date + + - name: Stock_Details + type: FACT + operation: INSERT + fetch_by: mids + + - name: Attendance + type: FACT + operation: DELETE+INSERT + fetch_by: run_date + + - name: Journey_Plan + type: FACT + operation: INSERT + fetch_by: run_date + + - name: Web_Logins + type: FACT + operation: INSERT + fetch_by: run_date + + - name: Promotion + type: FACT + operation: INSERT + fetch_by: mids + + - name: PaidVisibility + type: FACT + operation: INSERT + fetch_by: mids + + - name: Store_Master + type: DIMENSION + operation: DELETE+INSERT + fetch_by: master + + - name: SKU_Master + type: DIMENSION + operation: DELETE+INSERT + fetch_by: master + + - name: display_master + type: DIMENSION + operation: DELETE+INSERT + fetch_by: master + + - name: Employee_Master + type: DIMENSION + operation: DELETE+INSERT + fetch_by: master + + + - name: coverage_remarks + type: DIMENSION + operation: DELETE+INSERT + fetch_by: reason_id + + - name: mapping_storevisibility + type: BRIDGE + operation: ONLY_INSERT + fetch_by: run_date + + - name: Master_VisibilityReason + type: DIMENSION + operation: DELETE+INSERT + fetch_by: none + + - name: Master_VisibilityDefinition + type: DIMENSION + operation: DELETE+INSERT + fetch_by: none + + + - name: Master_Salesterritorylayer + type: DIMENSION + operation: DELETE+INSERT + fetch_by: none \ No newline at end of file diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..b3e1319 --- /dev/null +++ b/uv.lock @@ -0,0 +1,513 @@ +version = 1 +revision = 3 +requires-python = ">=3.14" + +[[package]] +name = "asynch" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ciso8601" }, + { name = "leb128" }, + { name = "lz4" }, + { name = "pytz" }, + { name = "tzlocal" }, + { name = "zstd" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/ce/f11ee2cafa6010d295db3ddee089f14cfc5fa9b8a19d0a21356c18a49267/asynch-0.3.1.tar.gz", hash = "sha256:d8bb3c1793a74a2e9d44cc7e8406ead3cf3818c11ecd6d8405b702b23148c584", size = 59825, upload-time = "2025-11-11T15:20:22.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/b0/9547aeadb577cb0c590162661b31110f7c3c02c1b23653a1b33222c7ff70/asynch-0.3.1-py3-none-any.whl", hash = "sha256:32b30a4409b70514077f7bf8ef713a248158fe5e63504c2769a4c7f772828412", size = 76502, upload-time = "2025-11-11T15:20:21.46Z" }, +] + +[[package]] +name = "certifi" +version = "2026.5.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", size = 309234, upload-time = "2026-04-02T09:27:07.194Z" }, + { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", size = 208042, upload-time = "2026-04-02T09:27:08.749Z" }, + { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", size = 228706, upload-time = "2026-04-02T09:27:09.951Z" }, + { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", size = 224727, upload-time = "2026-04-02T09:27:11.175Z" }, + { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", size = 215882, upload-time = "2026-04-02T09:27:12.446Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", size = 200860, upload-time = "2026-04-02T09:27:13.721Z" }, + { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", size = 211564, upload-time = "2026-04-02T09:27:15.272Z" }, + { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", size = 211276, upload-time = "2026-04-02T09:27:16.834Z" }, + { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", size = 201238, upload-time = "2026-04-02T09:27:18.229Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", size = 230189, upload-time = "2026-04-02T09:27:19.445Z" }, + { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", size = 211352, upload-time = "2026-04-02T09:27:20.79Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", size = 227024, upload-time = "2026-04-02T09:27:22.063Z" }, + { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", size = 217869, upload-time = "2026-04-02T09:27:23.486Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", size = 148541, upload-time = "2026-04-02T09:27:25.146Z" }, + { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", size = 159634, upload-time = "2026-04-02T09:27:26.642Z" }, + { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", size = 148384, upload-time = "2026-04-02T09:27:28.271Z" }, + { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", size = 330133, upload-time = "2026-04-02T09:27:29.474Z" }, + { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", size = 216257, upload-time = "2026-04-02T09:27:30.793Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", size = 234851, upload-time = "2026-04-02T09:27:32.44Z" }, + { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", size = 233393, upload-time = "2026-04-02T09:27:34.03Z" }, + { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", size = 223251, upload-time = "2026-04-02T09:27:35.369Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", size = 206609, upload-time = "2026-04-02T09:27:36.661Z" }, + { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", size = 220014, upload-time = "2026-04-02T09:27:38.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", size = 218979, upload-time = "2026-04-02T09:27:39.37Z" }, + { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", size = 209238, upload-time = "2026-04-02T09:27:40.722Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", size = 236110, upload-time = "2026-04-02T09:27:42.33Z" }, + { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", size = 219824, upload-time = "2026-04-02T09:27:43.924Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", size = 233103, upload-time = "2026-04-02T09:27:45.348Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", size = 225194, upload-time = "2026-04-02T09:27:46.706Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", size = 159827, upload-time = "2026-04-02T09:27:48.053Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", size = 174168, upload-time = "2026-04-02T09:27:49.795Z" }, + { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", size = 153018, upload-time = "2026-04-02T09:27:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, +] + +[[package]] +name = "ciso8601" +version = "2.3.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c1/8a/075724aea06c98626109bfd670c27c248c87b9ba33e637f069bf46e8c4c3/ciso8601-2.3.3.tar.gz", hash = "sha256:db5d78d9fb0de8686fbad1c1c2d168ed52efb6e8bf8774ae26226e5034a46dae", size = 31909, upload-time = "2025-08-20T16:31:33.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/3a/54ad0ae2257870076b4990545a8f16221470fecea0aa7a4e1f39506db8c5/ciso8601-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:82db4047d74d8b1d129e7a8da578518729912c3bd19cb71541b147e41f426381", size = 16115, upload-time = "2025-08-20T16:30:54.971Z" }, + { url = "https://files.pythonhosted.org/packages/23/fb/9fe767d44520691e2b706769466852fbdeb44a82dc294c2766bce1049d22/ciso8601-2.3.3-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:a553f3fc03a2ed5ca6f5716de0b314fa166461df01b45d8b36043ccac3a5e79f", size = 24214, upload-time = "2025-08-20T16:30:56.359Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ac/984fd3948f372c46c436a2b48da43f4fb7bc6f156a6f4bc858adaab79d42/ciso8601-2.3.3-cp314-cp314-macosx_11_0_x86_64.whl", hash = "sha256:ff59c26083b7bef6df4f0d96e4b649b484806d3d7bcc2de14ad43147c3aafb04", size = 15929, upload-time = "2025-08-20T16:30:58.352Z" }, + { url = "https://files.pythonhosted.org/packages/de/3a/5572917d4e0bec2c1ef0eda8652f9dc8d1850d29d3eef9e5e82ffe5d6791/ciso8601-2.3.3-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:99a1fa5a730790431d0bfcd1f3a6387f60cddc6853d8dcc5c2e140cd4d67a928", size = 41578, upload-time = "2025-08-20T16:30:59.351Z" }, + { url = "https://files.pythonhosted.org/packages/5e/cf/07321ce5cf099b98de0c02cd4bab4818610da69743003e94c8fb6e8a59cb/ciso8601-2.3.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c35265c1b0bd2ac30ed29b49818dd38b0d1dfda43086af605d8b91722727dec0", size = 42085, upload-time = "2025-08-20T16:31:00.338Z" }, + { url = "https://files.pythonhosted.org/packages/d3/c7/3c521d6779ee433d9596eb3fcded79549bbe371843f25e62006c04f74dc9/ciso8601-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:aa9df2f84ab25454f14df92b2dd4f9aae03dbfa581565a716b3e89b8e2110c03", size = 41313, upload-time = "2025-08-20T16:31:01.313Z" }, + { url = "https://files.pythonhosted.org/packages/f9/93/efd40db0d6b512be1cbe4e7e750882c2e88f580e17f35b3e9cc9c23004b5/ciso8601-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:32e06a35eb251cfc4bbe01a858c598da0a160e4ad7f42ff52477157ceaf48061", size = 41443, upload-time = "2025-08-20T16:31:02.357Z" }, + { url = "https://files.pythonhosted.org/packages/21/8e/515f9404faa39af8df5e2b899cafbca5dbe7cd2ffe5cc124ef393ffdaf1c/ciso8601-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:7657ba9730dc1340d73b9e61eca14f341c41dd308128c808b8b084d2b85bc03e", size = 17977, upload-time = "2025-08-20T16:31:03.429Z" }, +] + +[[package]] +name = "clickhouse-connect" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "lz4" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, + { name = "urllib3" }, + { name = "zstandard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/b4/720c5f4cdfe4bf716001d3a8db24b9ba94e7ad709e626a067502ed457199/clickhouse_connect-1.3.0.tar.gz", hash = "sha256:32e780ff3de62dbff2ff21eaf0501582b5365fba6c42227e203664379312e33e", size = 164472, upload-time = "2026-06-11T14:50:18.159Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/cc/5d6d0b385f1ae33a67b702c04672ff2203ffc298bb1c845cc5884c9d4e3b/clickhouse_connect-1.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2d1645af60860edb2f61b474daf5aa0af6b48724eb4d2c748edfa4027fdbe58a", size = 342602, upload-time = "2026-06-11T14:49:40.664Z" }, + { url = "https://files.pythonhosted.org/packages/30/ec/97fdf0c4a949c7de7da40ca2960a134f2d4303bef2a52b71012a21f0aa16/clickhouse_connect-1.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:225de4ab00609e599f2529a8a5256da5f473ce9544a04ab9b18b8fdd5baf9005", size = 332000, upload-time = "2026-06-11T14:49:42.28Z" }, + { url = "https://files.pythonhosted.org/packages/7d/f5/c136e03a9513f8c6a8546d04064ea7c8f42e694260ebceaeba749058670a/clickhouse_connect-1.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c18c0773a85f26c7eaeb59e0fd0a142e464312fda1c54fc7feae6115eb1759d4", size = 1252990, upload-time = "2026-06-11T14:49:43.721Z" }, + { url = "https://files.pythonhosted.org/packages/76/69/601dff4fcb2a1b157dbf6fd608db884ae5fd408fab646a40f64e96a173b3/clickhouse_connect-1.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:61f2f32403ac23354a572b160fa0a51ad5e76ba88aea37ebe5371b8863659339", size = 1266406, upload-time = "2026-06-11T14:49:45.359Z" }, + { url = "https://files.pythonhosted.org/packages/6f/ec/611b7782d1acf5b5f6368ea0062f8cbcf5913f76f5814d067bed1db0f6ea/clickhouse_connect-1.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e655403a35d016ac2100ca49fece73d1f293c6623809a04e530aa05c0369d69c", size = 1232665, upload-time = "2026-06-11T14:49:46.879Z" }, + { url = "https://files.pythonhosted.org/packages/73/16/801c28dc369b67a5362ae77a70dd90f40cf7589c9edd52f4e1ed633f9bfc/clickhouse_connect-1.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:bf99dc475626d1fc1720c3cbc8bd6154d26f829cf272c3c2524e5b7ce132fca1", size = 1263038, upload-time = "2026-06-11T14:49:48.44Z" }, + { url = "https://files.pythonhosted.org/packages/68/6b/718106208297d7ba7b8c3708fa42ee9da9269ab6d3fe94b744b659bb2c5f/clickhouse_connect-1.3.0-cp314-cp314-win32.whl", hash = "sha256:3954d59ced274163e1243549994eeaa6ae7fb46c1635ebd661eefefd25ce7c3d", size = 312209, upload-time = "2026-06-11T14:49:50.019Z" }, + { url = "https://files.pythonhosted.org/packages/d6/b9/a4173e4f9eb44ee9b4094d42ac5fc3da0763ecc538174f9ff6511b812ca0/clickhouse_connect-1.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:6447aaa3cda9a01580bffd821519199901124e979673a76f8c00d353293b2bd2", size = 331541, upload-time = "2026-06-11T14:49:51.567Z" }, + { url = "https://files.pythonhosted.org/packages/fe/d4/a2733ecbcba3e1497f8213ab4eb123a4b6c335400ce5dda42b1ff6c73719/clickhouse_connect-1.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:059f0ef645734cd5425b6af3bdac87c57f93e9c0dd1c86ef01da68846e2dd949", size = 312207, upload-time = "2026-06-11T14:49:53.095Z" }, + { url = "https://files.pythonhosted.org/packages/cf/fd/6e4f73c18b8f9e8136f88528ff93f067d50b09c3517e4699625d3ad491a8/clickhouse_connect-1.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:331d2aaadd1ef7fe238209da1d6bdbcb9244da6bb94f581beed8193d6b04ac2d", size = 360564, upload-time = "2026-06-11T14:49:54.557Z" }, + { url = "https://files.pythonhosted.org/packages/90/8a/be24013502d401f53485e738eb41e3374817b528dd8fc134ddc7748dbfc5/clickhouse_connect-1.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ab4380888830b92878231415fabab0b1b90bb3982d599e8444f8f9aa22174249", size = 353025, upload-time = "2026-06-11T14:49:56.31Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9e/f2a22a5f88bb7a8e0135d0b1e44f5b2526a94151f31471f0495167e066d5/clickhouse_connect-1.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a63c5f646142679b9288cf9ab1cc6af028389c894a67efa372f9cf60f2d97264", size = 1339647, upload-time = "2026-06-11T14:49:58.18Z" }, + { url = "https://files.pythonhosted.org/packages/bf/a2/ef2de3755a233184af2a3057d49ce319a0c3eebb8bdb9eaea606ad8a1317/clickhouse_connect-1.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7139165a4beccf7604c66849cd5da1cc2be97de76e39ca03fb93c34dc9a4c570", size = 1322984, upload-time = "2026-06-11T14:50:00.19Z" }, + { url = "https://files.pythonhosted.org/packages/77/eb/316dc1bd136070edd9ca75dcf1bd529bf854951bc5ced116903a19df2ba7/clickhouse_connect-1.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:73b089407354a0a2903561a5c86c2cb359705d38e9701dbef5fe13bbf69d94fd", size = 1297190, upload-time = "2026-06-11T14:50:01.994Z" }, + { url = "https://files.pythonhosted.org/packages/0c/a5/b0c98b795db1b5b403aeb2f03daab9539ce31536d8752c0d71f5997dfbbd/clickhouse_connect-1.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a9bce3cd4af89d6a0271938f1fabfc10a8889e1a7c4dbc4a866d2a804c4ed506", size = 1313649, upload-time = "2026-06-11T14:50:03.679Z" }, + { url = "https://files.pythonhosted.org/packages/2b/46/ab707f11890d0f3d2a8ae270e196fbe05c797b6d9ae99a3635057e84c85c/clickhouse_connect-1.3.0-cp314-cp314t-win32.whl", hash = "sha256:6c26e0175dddc2e3eafb606781c0a10be4fee13831117375f645eb70db722319", size = 336750, upload-time = "2026-06-11T14:50:05.555Z" }, + { url = "https://files.pythonhosted.org/packages/1b/d6/bd82db4324b917aaa7cb868fb4e11c6bc44b3d647ead0489e06748dee93c/clickhouse_connect-1.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:016cdfe4c17d898044f618963d11cc9604e98b239e269b5e69dcabcf2625ace9", size = 363227, upload-time = "2026-06-11T14:50:07.049Z" }, + { url = "https://files.pythonhosted.org/packages/83/60/7d95de37486b00671454047967c850bac40a1f07ebe3f56f750e47437f18/clickhouse_connect-1.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:f9341e3072d353bb834300c2a27e88a6fcde7a6c259c27c99fb530b97fcbc788", size = 323303, upload-time = "2026-06-11T14:50:08.839Z" }, +] + +[[package]] +name = "clickhouse-driver" +version = "0.2.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytz" }, + { name = "tzlocal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/46/9e/d8e40b29b6269a84552441a553fc64dff28f2d7e2d92e81c6be84fe12b4c/clickhouse_driver-0.2.10.tar.gz", hash = "sha256:925fc6ecda1e5314e3f03bcb493955c068b070cdba221fb8ce27329ee8a7f71b", size = 409448, upload-time = "2025-11-10T22:49:58.764Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/9f/513e661a80b0db9b6be5ce525d6cc86339c04dfc8d75f577b775e5ac5cf1/clickhouse_driver-0.2.10-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:19dd1de015d98dd1559d14ef05de2b6a0560c98b77ef160b3b99632a87fc8212", size = 211779, upload-time = "2025-11-10T22:48:36.873Z" }, + { url = "https://files.pythonhosted.org/packages/1c/6e/842610028ca39c826865402d46b7d7f499a9a249de219edfce889c0e025d/clickhouse_driver-0.2.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b6f35a9ee7f7a8ce3483764c3bc2d23c8be1c5ce3aef537b1a3cebe59fdb0c4d", size = 205709, upload-time = "2025-11-10T22:48:38.404Z" }, + { url = "https://files.pythonhosted.org/packages/c6/94/ddbcda962f7ca36cbcf7ded105f68f99b261dc29b9c28ca2eaec70382f66/clickhouse_driver-0.2.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:888ab59b2cccda24680cfbaeb723fd8922c6148b9a43ad4cf067fef55959f19f", size = 1032732, upload-time = "2025-11-10T22:48:40.047Z" }, + { url = "https://files.pythonhosted.org/packages/f6/1d/9d378af127ca1d35b734c965a9ec26318253f07d2a7eaa4a5643210fe327/clickhouse_driver-0.2.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fbf5d692ee4df241e82d200a195dc2ddbe7fbf490ebd185fbd9da32478245399", size = 1085199, upload-time = "2025-11-10T22:48:41.784Z" }, + { url = "https://files.pythonhosted.org/packages/3c/38/d65aedbe40e08a085fe3c2fcdcfcc0cec084d2e838e653c1ed4408771d9a/clickhouse_driver-0.2.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2f1d42ffcdc5b77346403e7ac49639a48c2fb58767a3e55657bd40e36290e9f0", size = 1089153, upload-time = "2025-11-10T22:48:43.283Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e3/e60fbcd5ec1458da9fe313984527e9ce5f95636c178bd2d08d6257c42c30/clickhouse_driver-0.2.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74b36dfb79311bcf1ba6edde926908a46778a5e9db6302f126a799550f1fb807", size = 1023508, upload-time = "2025-11-10T22:48:44.83Z" }, + { url = "https://files.pythonhosted.org/packages/9b/80/345be27d8011bcbd2fd3437d15b0e387369e2a8a8b73a7704af25bb6e90b/clickhouse_driver-0.2.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8c49d4b6619ce9237817185a41db6645f1557f7a77f2a822029ded1041179d55", size = 995994, upload-time = "2025-11-10T22:48:46.35Z" }, + { url = "https://files.pythonhosted.org/packages/0c/76/de011c4460a2b80e517c9a5f1b80a9fe3937e2987989e9776a79dc0a1e99/clickhouse_driver-0.2.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:503bbcbfedd65840489dadf3d0a2ac391888828b8f82579e6ec08efb1f362659", size = 1043865, upload-time = "2025-11-10T22:48:47.924Z" }, + { url = "https://files.pythonhosted.org/packages/5d/f4/264b6bb95488eebd79565372c8fbe397df2aac1478224629ebdc229eab00/clickhouse_driver-0.2.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:64aa403194bf4d3bdbecc2f8412ca85f4b9a35ed8dc228f23061a9889ce18dc4", size = 1038468, upload-time = "2025-11-10T22:48:50.706Z" }, + { url = "https://files.pythonhosted.org/packages/ec/95/01b3cc32ba060132ce9183f7af9059995fed2ded8d41865e86193f17ce30/clickhouse_driver-0.2.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20b16b066760cfcc6a8a5342e91585a84ba22152f83421030edd3582b6e420e3", size = 998721, upload-time = "2025-11-10T22:48:52.924Z" }, + { url = "https://files.pythonhosted.org/packages/85/d4/21e3203f8d22ad32224b0623ad2772103bb3a05ce367d6b27e78e118ea2f/clickhouse_driver-0.2.10-cp314-cp314-win32.whl", hash = "sha256:5ac6eed651dd6320a50338ff331fea93d20ed13dec27c7c7bfab71f8db3c76bb", size = 192639, upload-time = "2025-11-10T22:48:54.729Z" }, + { url = "https://files.pythonhosted.org/packages/ac/6b/1e87eb25614c0477fdadd5cf0b49675bc91cfcae95a10cbb83e3a688b01b/clickhouse_driver-0.2.10-cp314-cp314-win_amd64.whl", hash = "sha256:9f17746188833162e06e6af595e6a41c2ed09c7714b96860217c696fdb7689cf", size = 206586, upload-time = "2025-11-10T22:48:56.002Z" }, +] + +[[package]] +name = "clickhouse-sqlalchemy" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asynch" }, + { name = "clickhouse-driver" }, + { name = "requests" }, + { name = "sqlalchemy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2d/b9/801073bb20edd4efc5e7240ea318b755867fb0d62acb374daafc35419b39/clickhouse-sqlalchemy-0.3.2.tar.gz", hash = "sha256:267f3a9a1d0d186eb99a41895a684922d31125cea21702cd7dc73af1ccdd10e7", size = 45206, upload-time = "2024-06-12T12:08:58.044Z" } + +[[package]] +name = "data-move" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "clickhouse-connect" }, + { name = "clickhouse-sqlalchemy" }, + { name = "dotenv" }, + { name = "polars" }, + { name = "pyarrow" }, + { name = "pyodbc" }, + { name = "pyyaml" }, +] + +[package.metadata] +requires-dist = [ + { name = "clickhouse-connect", specifier = ">=1.3.0" }, + { name = "clickhouse-sqlalchemy", specifier = ">=0.3.2" }, + { name = "dotenv", specifier = ">=0.9.9" }, + { name = "polars", specifier = ">=1.41.2" }, + { name = "pyarrow", specifier = ">=24.0.0" }, + { name = "pyodbc", specifier = ">=5.3.0" }, + { name = "pyyaml", specifier = ">=6.0.3" }, +] + +[[package]] +name = "dotenv" +version = "0.9.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dotenv" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/b7/545d2c10c1fc15e48653c91efde329a790f2eecfbbf2bd16003b5db2bab0/dotenv-0.9.9-py2.py3-none-any.whl", hash = "sha256:29cf74a087b31dafdb5a446b6d7e11cbce8ed2741540e2339c69fbef92c94ce9", size = 1892, upload-time = "2025-02-19T22:15:01.647Z" }, +] + +[[package]] +name = "greenlet" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/6e/802acd792aebb2256fbbee8cacf2727faaeb6f240ac11008f09eae4414bc/greenlet-3.5.1.tar.gz", hash = "sha256:5a56aeb7d5d9cc4b3a735efb5095bd4b4f6f0e4f93e5ca876d0e2315137b7829", size = 197356, upload-time = "2026-05-20T15:05:03.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/cb/c62454606daf5640369c94d8a9dd540599b1bfc090e2d2180cb77f4038d2/greenlet-3.5.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d8ab31c9de8651a2facdd5c5bb0011f2380dd1a7af78ce2adf4b56095294fc07", size = 285579, upload-time = "2026-05-20T13:08:56.396Z" }, + { url = "https://files.pythonhosted.org/packages/ec/71/c4270398c2eba968a6071af1dfbdcaeee6ec1c24bc8b435b8cc452700da6/greenlet-3.5.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e300185139abc337ade480c327183adf42a875ac7181bfe66d7d4efea31fbea", size = 651106, upload-time = "2026-05-20T14:00:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ab/71e34b78a44ec271fb5f550c17bc46d301ddc5953890d935f270b0dcdb5a/greenlet-3.5.1-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7ffdb990dcaa0234cf9845aead5df2e3c3a8b6507d409274dd87e0d5ab05ffc2", size = 663478, upload-time = "2026-05-20T14:05:45.88Z" }, + { url = "https://files.pythonhosted.org/packages/77/96/4efd6fa5c62c85426a0c19077a586258ebc3a2a146ff2493e4312a697a22/greenlet-3.5.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2f82b3597e9d83b63408affed0b48fd0f54935edac4302237b9a837be0dae33c", size = 660800, upload-time = "2026-05-20T13:14:29.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/e0/6c71401a25cac7000261304e866a2f2cc04dc74810d40e2f118aa4799495/greenlet-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c0141e37414c10164e702b8fb1473304221ad98f71600850c6ef7ff4880feba0", size = 1617518, upload-time = "2026-05-20T14:02:28.662Z" }, + { url = "https://files.pythonhosted.org/packages/41/26/c5c06643e8c0af9e7bf18e16cb51d0ab7625155f0392e1c9015d66d556cd/greenlet-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:50ae25a67bea74ea41fb14b960bc532df73eb713417b2d61892dced82fe8d3bc", size = 1681593, upload-time = "2026-05-20T13:14:39.417Z" }, + { url = "https://files.pythonhosted.org/packages/8a/bd/e11a108317485075e68af9d23039619b86b28130c3b50d227d42edece64b/greenlet-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:8a17c42330e261299766b75ac1ea32caa437a9453c8f65d16a13140db378ecd3", size = 239800, upload-time = "2026-05-20T13:09:30.128Z" }, + { url = "https://files.pythonhosted.org/packages/47/f8/8e8e8417b7bf28639a5a56356ef934d0375e1d0c70a57e04d7701e870ffe/greenlet-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:7b5f5fae05b8ac6d176a61b60c394a8cbdc2b5b91b81793066e68745cf165e54", size = 236862, upload-time = "2026-05-20T13:09:10.498Z" }, + { url = "https://files.pythonhosted.org/packages/90/12/41bf27fde4d3605d3773ae57751eda182b8be2f5398011c041173b1d9534/greenlet-3.5.1-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:ea8da1e900d758d078810d4255d8c6aa572181896a31ec79d779eb79c3adc9ad", size = 293637, upload-time = "2026-05-20T13:12:35.529Z" }, + { url = "https://files.pythonhosted.org/packages/44/44/ba14b23e9757707050c2f397d305bbcae62e5d7cad122f8b6baec5ae4a1f/greenlet-3.5.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a19570c52a21420dcbc94e661994bc325c0b5b11304540fed514586da5dc8f2e", size = 650840, upload-time = "2026-05-20T14:00:11.079Z" }, + { url = "https://files.pythonhosted.org/packages/a8/37/5ddc2b686a6844f91abecef43411842426da2e1573f60b49ecf2547f4ae1/greenlet-3.5.1-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3d955c89b75eeca4723d7cc14135f393cd47c32e2a6cb4a8e4c6e760a26b0986", size = 656416, upload-time = "2026-05-20T14:05:47.118Z" }, + { url = "https://files.pythonhosted.org/packages/e1/f0/d17510297c35a2992712f0bf84de3779749999f7d3d63aa1f09db7c62dbe/greenlet-3.5.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2daaaebd1a5aa88c49045b6baf9310b3263796bd88db713edf37cf53e7bb4e", size = 654397, upload-time = "2026-05-20T13:14:30.696Z" }, + { url = "https://files.pythonhosted.org/packages/37/eb/147387705bb89092645b012586e7273cb5ed3c90ef7eaf3a69173eaf0209/greenlet-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3bfbd69cc349e43bf3a8ae1c85548ff0718efc887615c2db16c3833d7b0b072d", size = 1614469, upload-time = "2026-05-20T14:02:30.192Z" }, + { url = "https://files.pythonhosted.org/packages/a6/4e/37ee0da7732b7aa9896f17e15579a9df34b9fcb9dd494f0adfa749af6623/greenlet-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4378720dd888136c27215a0214d32a4d37c3852765d45bc37aad0623423cfd78", size = 1675115, upload-time = "2026-05-20T13:14:40.972Z" }, + { url = "https://files.pythonhosted.org/packages/57/f3/97dfcf4a6eb5077f8a672234216fb5923eb89f2cab7081cb10b2cf75b605/greenlet-3.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:45718441607f9325d948db98cbc691276059316d0358c188c246da4e1d4d23d2", size = 245246, upload-time = "2026-05-20T13:12:22.646Z" }, + { url = "https://files.pythonhosted.org/packages/5d/73/d7f72e34b582f694f4a9b248162db7b09cc458a259ba8f0c0bfa1a34ea7d/greenlet-3.5.1-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:2baee5ca02031757ffe8cc3d69f0cc0aec7065ce362622da74f32d3bcab1c541", size = 285575, upload-time = "2026-05-20T13:12:07.043Z" }, + { url = "https://files.pythonhosted.org/packages/df/59/fa9c6e87dc8ad27a95dabe2f29f372b733d05a8a67470f6c901ed9975655/greenlet-3.5.1-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b1ec3274918a81d3ea778b9e75b56b72b33f300edb6cf7f3a7fe1dae56683de", size = 656428, upload-time = "2026-05-20T14:00:12.556Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f9/e753408871eaa61dfe35e619cfc67512b036fde99893685d50eea9e07146/greenlet-3.5.1-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:111e2390ffffc47d5840b01711dd7fac07d4c09283d0283e7f3264b14e284c64", size = 667064, upload-time = "2026-05-20T14:05:48.662Z" }, + { url = "https://files.pythonhosted.org/packages/96/27/5565b5b40389f1c7753003a07e21892fda8660926787036d5bc0308b8113/greenlet-3.5.1-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e630136e905fe5ff43e86945ae41220b6d1470956a39220e708110ac48d01ea5", size = 665697, upload-time = "2026-05-20T13:14:32.943Z" }, + { url = "https://files.pythonhosted.org/packages/cf/82/e7de4178c0c2d1c9a5a3be3cc0b33e46a85b3ee4a77c071bf7ad8600e079/greenlet-3.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:975eac34b44a7077ca4d421348455b94f0f518246a7f14bc6d2fdcfe5b584368", size = 1621256, upload-time = "2026-05-20T14:02:31.91Z" }, + { url = "https://files.pythonhosted.org/packages/00/10/f2dddcf7dacac17dfc68691809589adad06135eb28930429cf58a6467a2f/greenlet-3.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:9ab3c3a0b2ae6198e67c898dad5215a49f9ae0d0081b3c3ec59f333e39eeca26", size = 1685956, upload-time = "2026-05-20T13:14:42.55Z" }, + { url = "https://files.pythonhosted.org/packages/22/17/4a232b32133230ada52f70e9d7f5b65b0caef8772f01849bd8d149e7e4ca/greenlet-3.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:cbfc69be86e10dcfef5b1e6269d1d6926552aa89ee39e1de3353360c1b6989ab", size = 239802, upload-time = "2026-05-20T13:13:15.481Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ae/4e623a7e6d4d2a5f4cb8e4c82de4169fc637942caae68d6e676b8a128ac5/greenlet-3.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:92fd6d44ac5e5a887c8a5dc4a8ba0ba908527c31c12f78c6bc7dcfe8aab279f6", size = 236853, upload-time = "2026-05-20T13:15:37.301Z" }, + { url = "https://files.pythonhosted.org/packages/7a/57/816d9cff29119da3505b3d6a5e14a8af89006ac36f47f891ff293ee05af1/greenlet-3.5.1-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:a6fdf2433a5441ef9a95464f7c3e674775da1c8c1177fff311cee1acad4626ed", size = 293877, upload-time = "2026-05-20T13:10:19.078Z" }, + { url = "https://files.pythonhosted.org/packages/23/a1/59b0a7c7d140ff1a75626680b9a9899b79a9176cab298b394968fb023295/greenlet-3.5.1-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7546556f0d649f99f6a361098a55f761181bb2ea12ff150bb16d26092ad88244", size = 655333, upload-time = "2026-05-20T14:00:14.758Z" }, + { url = "https://files.pythonhosted.org/packages/72/1b/5efe127597625042218939d01855109f352779050768b670b52edcc16a6c/greenlet-3.5.1-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d5ee3ea898009fa898f85f9982255d35278c477bebe185beca249cab42d4526c", size = 659443, upload-time = "2026-05-20T14:05:50.159Z" }, + { url = "https://files.pythonhosted.org/packages/6c/6d/c404246ea4d22d097a7426d0efb5b781bd7eb67715f09e79001bd552ab18/greenlet-3.5.1-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5c81f74d204d3edd136ebfd50dce53acbb776995d721a0fe801626cfc93b8cd", size = 658356, upload-time = "2026-05-20T13:14:35.091Z" }, + { url = "https://files.pythonhosted.org/packages/51/02/f8ee37fb6d2219329f350af241c27fcf12df57e723d11f6fc6d3bacdadaa/greenlet-3.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:2c18ef16bf6d4dd410e4dd52996888ea1497be26892fe5bbc73580aba4287b8e", size = 1619216, upload-time = "2026-05-20T14:02:33.403Z" }, + { url = "https://files.pythonhosted.org/packages/93/c5/3dc9475ace2c7a3680da12372cddd7f1ac874eb410a1ac48d3e9dab83782/greenlet-3.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:17d86354f0ae6b61bf9be5148d0dd34e06c3cb7c602c671f79f29ac3b150e659", size = 1678427, upload-time = "2026-05-20T13:14:43.71Z" }, + { url = "https://files.pythonhosted.org/packages/df/4e/750c15c317a41ffb36f0bf40b933e3d744a7dede61889f74443ea69690cf/greenlet-3.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:e7516cf6ae6b8a582c2770a0caed47b8a48373ed732c33d69a72913ae6ac923e", size = 245225, upload-time = "2026-05-20T13:13:59.366Z" }, + { url = "https://files.pythonhosted.org/packages/4f/fd/d3baea2eeb7b617efd47e87ca06e2ec2c6118d303aa9e918e0ce16eadc10/greenlet-3.5.1-cp315-cp315t-win_arm64.whl", hash = "sha256:5028648bf2253ec4745add746129d3904121fa7fe871a76bed23c5720573ce0a", size = 239590, upload-time = "2026-05-20T13:13:37.382Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "leb128" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/3c/ef6fe994b4b45d84187fea994f124173d587f4f8ab0641693ef269e80f56/leb128-1.0.9.tar.gz", hash = "sha256:8f8b0e2216ba8a318e2897360d9a34d88e2c968656fcb7c7bbb1aef31010f1c6", size = 26710, upload-time = "2026-01-09T08:29:39.261Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/f6/62cd379fe8527c6d685013ebed11c85fd4fced125bde9b3c80ebd5759850/leb128-1.0.9-py2.py3-none-any.whl", hash = "sha256:fef16ef20aca33dfdd2f4841d8004ec4acb7ed8545b63a7bc1183292c9a3e594", size = 3700, upload-time = "2026-01-09T08:29:37.446Z" }, +] + +[[package]] +name = "lz4" +version = "4.4.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/51/f1b86d93029f418033dddf9b9f79c8d2641e7454080478ee2aab5123173e/lz4-4.4.5.tar.gz", hash = "sha256:5f0b9e53c1e82e88c10d7c180069363980136b9d7a8306c4dca4f760d60c39f0", size = 172886, upload-time = "2025-11-03T13:02:36.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/9c/70bdbdb9f54053a308b200b4678afd13efd0eafb6ddcbb7f00077213c2e5/lz4-4.4.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c216b6d5275fc060c6280936bb3bb0e0be6126afb08abccde27eed23dead135f", size = 207586, upload-time = "2025-11-03T13:02:18.263Z" }, + { url = "https://files.pythonhosted.org/packages/b6/cb/bfead8f437741ce51e14b3c7d404e3a1f6b409c440bad9b8f3945d4c40a7/lz4-4.4.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c8e71b14938082ebaf78144f3b3917ac715f72d14c076f384a4c062df96f9df6", size = 207161, upload-time = "2025-11-03T13:02:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/e7/18/b192b2ce465dfbeabc4fc957ece7a1d34aded0d95a588862f1c8a86ac448/lz4-4.4.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9b5e6abca8df9f9bdc5c3085f33ff32cdc86ed04c65e0355506d46a5ac19b6e9", size = 1292415, upload-time = "2025-11-03T13:02:20.829Z" }, + { url = "https://files.pythonhosted.org/packages/67/79/a4e91872ab60f5e89bfad3e996ea7dc74a30f27253faf95865771225ccba/lz4-4.4.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3b84a42da86e8ad8537aabef062e7f661f4a877d1c74d65606c49d835d36d668", size = 1279920, upload-time = "2025-11-03T13:02:22.013Z" }, + { url = "https://files.pythonhosted.org/packages/f1/01/d52c7b11eaa286d49dae619c0eec4aabc0bf3cda7a7467eb77c62c4471f3/lz4-4.4.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bba042ec5a61fa77c7e380351a61cb768277801240249841defd2ff0a10742f", size = 1368661, upload-time = "2025-11-03T13:02:23.208Z" }, + { url = "https://files.pythonhosted.org/packages/f7/da/137ddeea14c2cb86864838277b2607d09f8253f152156a07f84e11768a28/lz4-4.4.5-cp314-cp314-win32.whl", hash = "sha256:bd85d118316b53ed73956435bee1997bd06cc66dd2fa74073e3b1322bd520a67", size = 90139, upload-time = "2025-11-03T13:02:24.301Z" }, + { url = "https://files.pythonhosted.org/packages/18/2c/8332080fd293f8337779a440b3a143f85e374311705d243439a3349b81ad/lz4-4.4.5-cp314-cp314-win_amd64.whl", hash = "sha256:92159782a4502858a21e0079d77cdcaade23e8a5d252ddf46b0652604300d7be", size = 101497, upload-time = "2025-11-03T13:02:25.187Z" }, + { url = "https://files.pythonhosted.org/packages/ca/28/2635a8141c9a4f4bc23f5135a92bbcf48d928d8ca094088c962df1879d64/lz4-4.4.5-cp314-cp314-win_arm64.whl", hash = "sha256:d994b87abaa7a88ceb7a37c90f547b8284ff9da694e6afcfaa8568d739faf3f7", size = 93812, upload-time = "2025-11-03T13:02:26.133Z" }, +] + +[[package]] +name = "polars" +version = "1.41.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "polars-runtime-32" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/f9/aeda46259b0669247a160315d2d51269de9504b9dd2f70acadbcb22f46b7/polars-1.41.2.tar.gz", hash = "sha256:256d6731162371b77f3f29a55eacb8c0fc740ddb1a293a01d2ef5b5393c5c708", size = 737996, upload-time = "2026-05-29T17:39:15.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/22/28f62d24f7db56ac4343588f9362d49b7b4177e55ac47a466fe696b0099b/polars-1.41.2-py3-none-any.whl", hash = "sha256:23ce9a2910b6e3e8d4258770bf44aa17170958df7af6e85feedf4458a04d8d29", size = 833445, upload-time = "2026-05-29T17:37:05.576Z" }, +] + +[[package]] +name = "polars-runtime-32" +version = "1.41.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/56/54e3ea0e9b64f327179049e4742241cc6b1d3e8fa414b05a057dd26df367/polars_runtime_32-1.41.2.tar.gz", hash = "sha256:7af09ec1ab053da2c9669e8d15f809a4083a29be05db57111688b8051062af56", size = 2989474, upload-time = "2026-05-29T17:39:17.257Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/9b/fe72a3811c0357cdb06c67bdc7695fa1623ad47948fc523195f5ac31037f/polars_runtime_32-1.41.2-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:95a08346dac337357cdb825c8076df7d36da54c4caa59a5cb41d0a30691c5edd", size = 52265283, upload-time = "2026-05-29T17:37:09.407Z" }, + { url = "https://files.pythonhosted.org/packages/0a/93/fab9da803fd80d9e83ef88c20932f637a10bc611b20415fc322eec84bc44/polars_runtime_32-1.41.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:dedfaeec2c7f995298da7319dd9431d662e5dd1d0ec51b1459df4a0234ceff52", size = 46571222, upload-time = "2026-05-29T17:37:13.698Z" }, + { url = "https://files.pythonhosted.org/packages/c8/2a/8843f34a8ac57acd058a39b87b03b580dd352a490e9dae0415e02033bdd4/polars_runtime_32-1.41.2-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18eea22c5cc34e27f8a60950458ad81e6a9ea75e89363ca1367e14e7e7f781fc", size = 50409372, upload-time = "2026-05-29T17:37:17.875Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c6/92b352fe88cf51bd0a19fb99e1c0cbe46aa26c14dcf7995b89869cd932ae/polars_runtime_32-1.41.2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2630540dfdfb0f36f9b04a07c7c2e3f50bf2ad384113263c1c812007ee9141e0", size = 56405484, upload-time = "2026-05-29T17:37:22.684Z" }, + { url = "https://files.pythonhosted.org/packages/74/c4/bae3174c3b02f6b441d2e58594387abcd509f67a098f682a83b195f08966/polars_runtime_32-1.41.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:20e969e08f9b137e233c04cc04de73d9795f89eb77d34854e40a025965a43763", size = 50603512, upload-time = "2026-05-29T17:37:27.422Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ed/f2d26ae02d92c2689056838ed59e2a626326ad23c2831d58637d25f6c82a/polars_runtime_32-1.41.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e7016a3deb641b64a31447abbbee0f34bd020a6a9ae34ee6b743837def15e2a4", size = 54328561, upload-time = "2026-05-29T17:37:32.587Z" }, + { url = "https://files.pythonhosted.org/packages/9b/c4/9c3831cc885dc7769e59abf8f583821a5fb4403fd0e4eba0ccc6d47a3d4b/polars_runtime_32-1.41.2-cp310-abi3-win_amd64.whl", hash = "sha256:1e5e5377c315e0dcafdfb2a31adc546abbaeb3f9cb1864e6536523d2af473265", size = 51978643, upload-time = "2026-05-29T17:37:37.443Z" }, + { url = "https://files.pythonhosted.org/packages/cd/c6/79e9f3f270270d7ed5575d92b7bfef49f01abd9275447161275b23b553a8/polars_runtime_32-1.41.2-cp310-abi3-win_arm64.whl", hash = "sha256:843d96f69d18eca53429c1198e58891db7f18111f83b9c419bb45ad9d73eaed5", size = 46006901, upload-time = "2026-05-29T17:37:42.522Z" }, +] + +[[package]] +name = "pyarrow" +version = "24.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" }, + { url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" }, + { url = "https://files.pythonhosted.org/packages/29/88/f4e9145da0417b3d2c12035a8492b35ff4a3dbc653e614fcfb51d9dedb38/pyarrow-24.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:38be1808cdd068605b787e6ca9119b27eb275a0234e50212c3492331680c3b1e", size = 28001155, upload-time = "2026-04-21T10:51:22.337Z" }, + { url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" }, + { url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" }, + { url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" }, + { url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" }, + { url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" }, + { url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" }, + { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" }, +] + +[[package]] +name = "pyodbc" +version = "5.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/85/44b10070a769a56bd910009bb185c0c0a82daff8d567cd1a116d7d730c7d/pyodbc-5.3.0.tar.gz", hash = "sha256:2fe0e063d8fb66efd0ac6dc39236c4de1a45f17c33eaded0d553d21c199f4d05", size = 121770, upload-time = "2025-10-17T18:04:09.43Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/f2/c26d82a7ce1e90b8bbb8731d3d53de73814e2f6606b9db9d978303aa8d5f/pyodbc-5.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3f1bdb3ce6480a17afaaef4b5242b356d4997a872f39e96f015cabef00613797", size = 73513, upload-time = "2025-10-17T18:03:40.536Z" }, + { url = "https://files.pythonhosted.org/packages/82/d5/1ab1b7c4708cbd701990a8f7183c5bb5e0712d5e8479b919934e46dadab4/pyodbc-5.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7713c740a10f33df3cb08f49a023b7e1e25de0c7c99650876bbe717bc95ee780", size = 72631, upload-time = "2025-10-17T18:03:41.713Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f1/7e3831eeac2b09b31a77e6b3495491ce162035ff2903d7261b49d35aa3c2/pyodbc-5.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf18797a12e70474e1b7f5027deeeccea816372497e3ff2d46b15bec2d18a0cc", size = 344580, upload-time = "2025-10-17T18:03:42.67Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a6/71d26d626a3c45951620b7ff356ec920e420f0e09b0a924123682aa5e4ab/pyodbc-5.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:08b2439500e212625471d32f8fde418075a5ddec556e095e5a4ba56d61df2dc6", size = 350224, upload-time = "2025-10-17T18:03:43.731Z" }, + { url = "https://files.pythonhosted.org/packages/93/14/f702c5e8c2d595776266934498505f11b7f1545baf21ffec1d32c258e9d3/pyodbc-5.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:729c535341bb09c476f219d6f7ab194bcb683c4a0a368010f1cb821a35136f05", size = 1301503, upload-time = "2025-10-17T18:03:45.013Z" }, + { url = "https://files.pythonhosted.org/packages/d9/b2/ad92ebdd1b5c7fec36b065e586d1d34b57881e17ba5beec5c705f1031058/pyodbc-5.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c67e7f2ce649155ea89beb54d3b42d83770488f025cf3b6f39ca82e9c598a02e", size = 1361050, upload-time = "2025-10-17T18:03:46.298Z" }, + { url = "https://files.pythonhosted.org/packages/19/40/dc84e232da07056cb5aaaf5f759ba4c874bc12f37569f7f1670fc71e7ae1/pyodbc-5.3.0-cp314-cp314-win32.whl", hash = "sha256:a48d731432abaee5256ed6a19a3e1528b8881f9cb25cb9cf72d8318146ea991b", size = 65670, upload-time = "2025-10-17T18:03:56.414Z" }, + { url = "https://files.pythonhosted.org/packages/b8/79/c48be07e8634f764662d7a279ac204f93d64172162dbf90f215e2398b0bd/pyodbc-5.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:58635a1cc859d5af3f878c85910e5d7228fe5c406d4571bffcdd281375a54b39", size = 72177, upload-time = "2025-10-17T18:03:57.296Z" }, + { url = "https://files.pythonhosted.org/packages/fc/79/e304574446b2263f428ce14df590ba52c2e0e0205e8d34b235b582b7d57e/pyodbc-5.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:754d052030d00c3ac38da09ceb9f3e240e8dd1c11da8906f482d5419c65b9ef5", size = 66668, upload-time = "2025-10-17T18:03:58.174Z" }, + { url = "https://files.pythonhosted.org/packages/43/17/f4eabf443b838a2728773554017d08eee3aca353102934a7e3ba96fb0e31/pyodbc-5.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:f927b440c38ade1668f0da64047ffd20ec34e32d817f9a60d07553301324b364", size = 75780, upload-time = "2025-10-17T18:03:47.273Z" }, + { url = "https://files.pythonhosted.org/packages/59/ea/e79e168c3d38c27d59d5d96273fd9e3c3ba55937cc944c4e60618f51de90/pyodbc-5.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:25c4cfb2c08e77bc6e82f666d7acd52f0e52a0401b1876e60f03c73c3b8aedc0", size = 75503, upload-time = "2025-10-17T18:03:48.171Z" }, + { url = "https://files.pythonhosted.org/packages/90/81/d1d7c125ec4a20e83fdc28e119b8321192b2bd694f432cf63e1199b2b929/pyodbc-5.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc834567c2990584b9726cba365834d039380c9dbbcef3030ddeb00c6541b943", size = 398356, upload-time = "2025-10-17T18:03:49.131Z" }, + { url = "https://files.pythonhosted.org/packages/5e/fc/f6be4b3cc3910f8c2aba37aa41671121fd6f37b402ae0fefe53a70ac7cd5/pyodbc-5.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8339d3094858893c1a68ee1af93efc4dff18b8b65de54d99104b99af6306320d", size = 397291, upload-time = "2025-10-17T18:03:50.18Z" }, + { url = "https://files.pythonhosted.org/packages/03/2e/0610b1ed05a5625528d52f6cece9610e84617d35f475c89c2a52f66d13f7/pyodbc-5.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74528fe148980d0c735c0ebb4a4dc74643ac4574337c43c1006ac4d09593f92d", size = 1353900, upload-time = "2025-10-17T18:03:51.339Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f1/43497e1d37f9f71b43b2b3172e7b1bdf50851e278390c3fb6b46a3630c53/pyodbc-5.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d89a7f2e24227150c13be8164774b7e1f9678321a4248f1356a465b9cc17d31e", size = 1406062, upload-time = "2025-10-17T18:03:52.546Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/88a1277c2f7d9ab1cec0a71e074ba24fd4a1710a43974682546da90a1343/pyodbc-5.3.0-cp314-cp314t-win32.whl", hash = "sha256:af4d8c9842fc4a6360c31c35508d6594d5a3b39922f61b282c2b4c9d9da99514", size = 70132, upload-time = "2025-10-17T18:03:53.715Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c7/ee98c62050de4aa8bafb6eb1e11b95e0b0c898bd5930137c6dc776e06a9b/pyodbc-5.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:bfeb3e34795d53b7d37e66dd54891d4f9c13a3889a8f5fe9640e56a82d770955", size = 79452, upload-time = "2025-10-17T18:03:54.664Z" }, + { url = "https://files.pythonhosted.org/packages/4b/8f/d8889efd96bbe8e5d43ff9701f6b1565a8e09c3e1f58c388d550724f777b/pyodbc-5.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:13656184faa3f2d5c6f19b701b8f247342ed581484f58bf39af7315c054e69db", size = 70142, upload-time = "2025-10-17T18:03:55.551Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + +[[package]] +name = "pytz" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "sqlalchemy" +version = "2.0.50" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/da/6fbf010c8ebb347679d0d100b22fe9ba5e13fd04046c5df7280d2f0bf706/sqlalchemy-2.0.50.tar.gz", hash = "sha256:af5607d11ef90fd6a5c0549fe0045dce1663d427426bcfb506dcb5346a85a3b9", size = 9907424, upload-time = "2026-05-24T19:20:04.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/32/10ac51b4be7cdecd7e93d069251c86dfbf70b7adbd7c67b48ccea6c49e1c/sqlalchemy-2.0.50-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c966932507a4d7d0a37314927dbfcd89720e3f37d2a1e3352e7ae7939fa8e8a0", size = 2158519, upload-time = "2026-05-24T19:27:56.472Z" }, + { url = "https://files.pythonhosted.org/packages/5a/76/e703d2f7681d7d66c4c891af3f07c7ccf4c76ad7f18351de035b5eda007a/sqlalchemy-2.0.50-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:faffef4bcc20a1892e65e155293d99d60855bbbc79250ab712819cfd56a8e6bb", size = 3282063, upload-time = "2026-05-24T20:09:38.57Z" }, + { url = "https://files.pythonhosted.org/packages/31/26/ef168b184a25701f9995e8fb7e503fafd7a99c1c77cda1bc1a26ea2ed486/sqlalchemy-2.0.50-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c206aec519a2e7bd08abbfb33436e325fd22c632d9c21a9047e376ce241646e", size = 3287069, upload-time = "2026-05-24T20:17:21.942Z" }, + { url = "https://files.pythonhosted.org/packages/c2/15/765acc2bc693bccc43ca4a95d5b69750da8aaf6db1b5c616536e087f8920/sqlalchemy-2.0.50-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bef4ac756363227ef6402a75fee025a4bc690f92328e825868939b3b3a446a6d", size = 3230453, upload-time = "2026-05-24T20:09:40.398Z" }, + { url = "https://files.pythonhosted.org/packages/63/61/08e03c3adbf5db0087a0b6816746fec8f3032fb2f7fc899a9bb9b2a48ce4/sqlalchemy-2.0.50-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:96fbee6b19c19cd1556c8bf9419447cf2ec149ffcab7ab64348c23e54ef8547f", size = 3252413, upload-time = "2026-05-24T20:17:24.067Z" }, + { url = "https://files.pythonhosted.org/packages/03/0c/370a1f2db38436c615e10134c8a37de3688e74084792380695f3f5083860/sqlalchemy-2.0.50-cp314-cp314-win32.whl", hash = "sha256:8f00e3eb43ba30eb1b238ee03a8a62309486d1321eda3328bb611e0340033ad8", size = 2120063, upload-time = "2026-05-24T19:50:11.08Z" }, + { url = "https://files.pythonhosted.org/packages/7f/a0/fe92bb9817863bc13ba093bda931979a26cc2ca69f8e8f26d07add3d7c6f/sqlalchemy-2.0.50-cp314-cp314-win_amd64.whl", hash = "sha256:15708c613cd5005b7dffe1f66ee6a63ee8f5e46799f71c70ebad74178c676a39", size = 2145830, upload-time = "2026-05-24T19:50:12.452Z" }, + { url = "https://files.pythonhosted.org/packages/cc/ff/e5640a98a0b2f491eb8fde10fb6c773621a2e44340de231fafcc9370f4a9/sqlalchemy-2.0.50-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3699dac4be410e97049a1658e9480da9cde956594aa0f3aebc60b88f21c5ba70", size = 2178435, upload-time = "2026-05-24T19:42:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/b7/85/337116e186f1236375b5fb70c21cfac98e8e8ab0d3a47be838dc47a59e08/sqlalchemy-2.0.50-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f96233858e3df43932ac11589e22520da6e8aeb624b03fedfeebb0e8ea213086", size = 3566059, upload-time = "2026-05-24T20:01:20.848Z" }, + { url = "https://files.pythonhosted.org/packages/96/34/bb0e190e161c3c2c24314a65add57218be14a4a9486886b7f5047c1ff7c8/sqlalchemy-2.0.50-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c4e70c46fad30c3bcc6a4708bc0130a3173e11a5b25f0ea4a9d8911b450f1f52", size = 3535366, upload-time = "2026-05-24T20:03:56.768Z" }, + { url = "https://files.pythonhosted.org/packages/df/5a/a7f759f97e4fd499c5d4e4488c760d5a7fbecf3028b465a04274fcd52384/sqlalchemy-2.0.50-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1918a3cf564d16d95bca7301005f41ab2ad50b07cd3b9da50d3ed986db148d6a", size = 3474879, upload-time = "2026-05-24T20:01:23.058Z" }, + { url = "https://files.pythonhosted.org/packages/9d/d9/2907ea38eb60687d297bf9c39e5ee58053c87b57fe8a9cae97090cecbf10/sqlalchemy-2.0.50-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b00098cdbdbd38c7be3d568b0c9c3122b8c0ec62b911b57cd5e6e0254d60a76d", size = 3486117, upload-time = "2026-05-24T20:03:59.052Z" }, + { url = "https://files.pythonhosted.org/packages/f2/e3/5aa06f167559f8c0bdae487e297d23ba548150ab016a3418265d617a4985/sqlalchemy-2.0.50-cp314-cp314t-win32.whl", hash = "sha256:1fbd55a969d7ac44a98e3dec75016074f809fa08f871585ace58dde110d1bf3e", size = 2150823, upload-time = "2026-05-24T20:08:58.644Z" }, + { url = "https://files.pythonhosted.org/packages/65/9b/112fb8f977582d7489d036e409e3723948bcf5320b3ac465f3c481bbe8f9/sqlalchemy-2.0.50-cp314-cp314t-win_amd64.whl", hash = "sha256:c5c3cdb753a9004183e1ccb634b41611654c989e61bc68617ce878e46d6f1e51", size = 2185794, upload-time = "2026-05-24T20:09:00.319Z" }, + { url = "https://files.pythonhosted.org/packages/d0/10/f7220e9b784d295d241c86ed99aeb537f92afcd469a64861f2717e9bb077/sqlalchemy-2.0.50-py3-none-any.whl", hash = "sha256:92064363517a3ff8212b5a93b8c62876579d8dfd1ca5b561335f30152d884fa9", size = 1943861, upload-time = "2026-05-24T19:59:01.119Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10", size = 198254, upload-time = "2026-04-24T15:22:08.651Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, +] + +[[package]] +name = "tzlocal" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[[package]] +name = "zstandard" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/5c/f8923b595b55fe49e30612987ad8bf053aef555c14f05bb659dd5dbe3e8a/zstandard-0.25.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3", size = 795887, upload-time = "2025-09-14T22:17:54.198Z" }, + { url = "https://files.pythonhosted.org/packages/8d/09/d0a2a14fc3439c5f874042dca72a79c70a532090b7ba0003be73fee37ae2/zstandard-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f", size = 640658, upload-time = "2025-09-14T22:17:55.423Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7c/8b6b71b1ddd517f68ffb55e10834388d4f793c49c6b83effaaa05785b0b4/zstandard-0.25.0-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c", size = 5379849, upload-time = "2025-09-14T22:17:57.372Z" }, + { url = "https://files.pythonhosted.org/packages/a4/86/a48e56320d0a17189ab7a42645387334fba2200e904ee47fc5a26c1fd8ca/zstandard-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439", size = 5058095, upload-time = "2025-09-14T22:17:59.498Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ad/eb659984ee2c0a779f9d06dbfe45e2dc39d99ff40a319895df2d3d9a48e5/zstandard-0.25.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043", size = 5551751, upload-time = "2025-09-14T22:18:01.618Z" }, + { url = "https://files.pythonhosted.org/packages/61/b3/b637faea43677eb7bd42ab204dfb7053bd5c4582bfe6b1baefa80ac0c47b/zstandard-0.25.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859", size = 6364818, upload-time = "2025-09-14T22:18:03.769Z" }, + { url = "https://files.pythonhosted.org/packages/31/dc/cc50210e11e465c975462439a492516a73300ab8caa8f5e0902544fd748b/zstandard-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0", size = 5560402, upload-time = "2025-09-14T22:18:05.954Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ae/56523ae9c142f0c08efd5e868a6da613ae76614eca1305259c3bf6a0ed43/zstandard-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7", size = 4955108, upload-time = "2025-09-14T22:18:07.68Z" }, + { url = "https://files.pythonhosted.org/packages/98/cf/c899f2d6df0840d5e384cf4c4121458c72802e8bda19691f3b16619f51e9/zstandard-0.25.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2", size = 5269248, upload-time = "2025-09-14T22:18:09.753Z" }, + { url = "https://files.pythonhosted.org/packages/1b/c0/59e912a531d91e1c192d3085fc0f6fb2852753c301a812d856d857ea03c6/zstandard-0.25.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344", size = 5430330, upload-time = "2025-09-14T22:18:11.966Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/7e31db1240de2df22a58e2ea9a93fc6e38cc29353e660c0272b6735d6669/zstandard-0.25.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c", size = 5811123, upload-time = "2025-09-14T22:18:13.907Z" }, + { url = "https://files.pythonhosted.org/packages/f6/49/fac46df5ad353d50535e118d6983069df68ca5908d4d65b8c466150a4ff1/zstandard-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088", size = 5359591, upload-time = "2025-09-14T22:18:16.465Z" }, + { url = "https://files.pythonhosted.org/packages/c2/38/f249a2050ad1eea0bb364046153942e34abba95dd5520af199aed86fbb49/zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12", size = 444513, upload-time = "2025-09-14T22:18:20.61Z" }, + { url = "https://files.pythonhosted.org/packages/3a/43/241f9615bcf8ba8903b3f0432da069e857fc4fd1783bd26183db53c4804b/zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2", size = 516118, upload-time = "2025-09-14T22:18:17.849Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ef/da163ce2450ed4febf6467d77ccb4cd52c4c30ab45624bad26ca0a27260c/zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d", size = 476940, upload-time = "2025-09-14T22:18:19.088Z" }, +] + +[[package]] +name = "zstd" +version = "1.5.7.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/62/b9c075ad664e7c4cbb3d8d2be7c246506abe1bc7f778eb58d260ef9538c8/zstd-1.5.7.3.tar.gz", hash = "sha256:403e5205f4ac04b92e6b0cda654be2f51de268228a0db0067bc087faacf2f495", size = 672559, upload-time = "2026-01-08T16:24:43.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d9/854b2643b677d22c914e352691cc46fea875867863fb453814fabaa9cc38/zstd-1.5.7.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5412c86c34cbaf6906433ef3f2c96c407f208782f06cd3e5f01f066788adb3b8", size = 268245, upload-time = "2026-01-08T16:43:39.347Z" }, + { url = "https://files.pythonhosted.org/packages/7d/b7/97f8c206efd27df3dc7ebe2bf2d69193a67d5de024726b15c2b79ddace30/zstd-1.5.7.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f94246befb1e473211a298c96e5768f3c63eaad814ac14d160d79ae9858e1d03", size = 230992, upload-time = "2026-01-08T16:43:37.79Z" }, + { url = "https://files.pythonhosted.org/packages/8d/d9/e2771b6144f3b2a26daaf6bb40c233ef887fce03627e7825e11a674ced65/zstd-1.5.7.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:31050e17a1a546fb82c90eee8ee3c30d22b9d0594b5937e69d38b7a5084af2a2", size = 2162418, upload-time = "2026-01-08T18:09:00.35Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a5/871ac984f6b8bad512104747e55eb3b65bd89342715e030522e9cbe3b1c7/zstd-1.5.7.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ba8ec5dfd48c86d19f880713246f85d09ee06e8cd17141956258650878000d6", size = 2202154, upload-time = "2026-01-08T18:08:57.748Z" }, + { url = "https://files.pythonhosted.org/packages/a7/5d/828157d6fe7f72d5c5473231a575e98e4e2c861d65a8acdd0e7200588ff6/zstd-1.5.7.3-cp314-cp314-manylinux_2_14_i686.whl", hash = "sha256:3005540ba406157f3e205c998709ab5f8e68b390c658c7c238eb8986092089d5", size = 300205, upload-time = "2026-01-08T16:29:58.595Z" }, + { url = "https://files.pythonhosted.org/packages/12/71/09617dcd1d3be40fc43f210f6d99c2011861db9c8e1a444a571e1022716e/zstd-1.5.7.3-cp314-cp314-manylinux_2_14_x86_64.whl", hash = "sha256:3934b54a3b7df039fcd4cf7b0f0a38c86ce44d26321255ffc3fac73d6cdcc59d", size = 304148, upload-time = "2026-01-08T16:41:47.079Z" }, + { url = "https://files.pythonhosted.org/packages/8d/c2/6abdab0a1b0b1326bd53fa187568630d78a01bd3734e035360869d80a52b/zstd-1.5.7.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e9230cd3e9153e2bed16f332558f8f3f7d869f4d15e8fa3f9c360bfa163a8b4a", size = 2096739, upload-time = "2026-01-08T18:09:03.943Z" }, + { url = "https://files.pythonhosted.org/packages/66/7b/3867d0279bab7d0a69fdc06403e4d23f7e6a8f54530ef6626bcf09a19d62/zstd-1.5.7.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5bffba70af539f14f9df5367b1add9119f14d5e35b658aef7b765417ea461e0e", size = 2125551, upload-time = "2026-01-08T18:09:02.133Z" }, + { url = "https://files.pythonhosted.org/packages/87/f3/75566c4d29ab144ac4e1c915e4f58ce239cbe5ae4f8376fb56edf4114866/zstd-1.5.7.3-cp314-cp314-win32.whl", hash = "sha256:a006e70c88ab67bb56989e11d820adc7601a6a7ad5558b3c6c690b19a1dadc5b", size = 153519, upload-time = "2026-01-08T17:20:15.507Z" }, + { url = "https://files.pythonhosted.org/packages/ff/22/6c42f636f92e74cda4acfeab0e64dd4a3db3a07d5f5723ae885b8155c85a/zstd-1.5.7.3-cp314-cp314-win_amd64.whl", hash = "sha256:cb4957c330c7b94b0546c7b9529723b49e865608683b9503a251fe793da9d4db", size = 171454, upload-time = "2026-01-08T17:20:17.124Z" }, + { url = "https://files.pythonhosted.org/packages/79/71/ff79a965e040d90de1cabb842ef22ef4475ef37cae81be3646cbb9c7a9b4/zstd-1.5.7.3-cp314-cp314-win_arm64.whl", hash = "sha256:a785426081ab7cafe4522876ac771d701766deea9a6d8352e87744da00e6637f", size = 163084, upload-time = "2026-01-08T16:37:52.448Z" }, + { url = "https://files.pythonhosted.org/packages/5f/46/56453790f9b7859e5593b767361adb2662318c55c3e4af0fffa1d389a2a5/zstd-1.5.7.3-cp314-cp314t-manylinux_2_14_i686.whl", hash = "sha256:b52ef154793be0399befd742328ec6f5dff95154248d6d18dd65851cf22a1a5f", size = 300364, upload-time = "2026-01-10T11:12:15.545Z" }, + { url = "https://files.pythonhosted.org/packages/4e/9e/501836085c9c6672effc0bc11d0383b439d414edd033958209bb93c68ac3/zstd-1.5.7.3-cp314-cp314t-manylinux_2_14_x86_64.whl", hash = "sha256:8024a8ba9156b1b2e64e69d147df5ddedeaed107f9da02a3428fd7baf3e5b920", size = 304338, upload-time = "2026-01-10T11:12:03.378Z" }, + { url = "https://files.pythonhosted.org/packages/fb/ee/b71a41ae810b429b909b4d68c9bd6decc4bca1407cac8d157b9470a654e2/zstd-1.5.7.3-cp315-cp315-manylinux_2_14_i686.whl", hash = "sha256:31ac7fbacca4759aad4b6abc13bbc05e68788e9e85a968255f7624b3b8db31df", size = 300146, upload-time = "2026-01-08T16:56:10.452Z" }, + { url = "https://files.pythonhosted.org/packages/93/24/3925528355ededa372951f10e85bec9870e3f4d325b04cfb6d9707ba7dec/zstd-1.5.7.3-cp315-cp315-manylinux_2_14_x86_64.whl", hash = "sha256:d03b2927c5843ded4d1319836a33a9c21675d2f86f916a2f234a060d4c67d87c", size = 304202, upload-time = "2026-01-08T16:35:34.94Z" }, + { url = "https://files.pythonhosted.org/packages/8d/37/2ed92c824e2f8ecff1a5f699677277a4d46ce9d527800aadf5f867455ccd/zstd-1.5.7.3-cp315-cp315t-manylinux_2_14_i686.whl", hash = "sha256:5dfbf2564eb574fc1f45613ecf28036a82533c3dd70e7bb1c9854168c638da7a", size = 300404, upload-time = "2026-01-08T16:24:41.593Z" }, + { url = "https://files.pythonhosted.org/packages/92/77/a152dde083129c3eee60c3bc83963b8c747112afe375b3d6d4a96fa15427/zstd-1.5.7.3-cp315-cp315t-manylinux_2_14_x86_64.whl", hash = "sha256:7f2f5776b902f41daf7b63e75a9384b0d7c855f824f14dabefc67814b8fa5611", size = 304333, upload-time = "2026-01-08T16:25:49.196Z" }, +] diff --git a/y.yml b/y.yml new file mode 100644 index 0000000..737c148 --- /dev/null +++ b/y.yml @@ -0,0 +1,109 @@ +tables: + # - name: SOS_OneApp + # type: FACT + # operation: INSERT + # fetch_by: mids + + # # - name: OQaD + # # type: FACT + # # operation: INSERT + # # fetch_by: run_date + + # - name: additional_visibility + # type: FACT + # operation: INSERT + # fetch_by: mids + + # - name: Coverage + # type: FACT + # operation: INSERT + # fetch_by: mids + + # - name: Survey + # type: FACT + # operation: INSERT + # fetch_by: mids + + # - name: Login + # type: FACT + # operation: INSERT + # fetch_by: run_date + + # - name: Stock_Details + # type: FACT + # operation: INSERT + # fetch_by: mids + + # - name: Attendance + # type: FACT + # operation: DELETE+INSERT + # fetch_by: run_date + + # - name: Journey_Plan + # type: FACT + # operation: INSERT + # fetch_by: run_date + + # - name: PaidVisibility + # type: FACT + # operation: INSERT + # fetch_by: mids + + # - name: Web_Logins + # type: FACT + # operation: INSERT + # fetch_by: run_date + + + # - name: Store_Master + # type: DIMENSION + # operation: DELETE+INSERT + # fetch_by: master + + # - name: coverage_remarks + # type: DIMENSION + # operation: DELETE+INSERT + # fetch_by: reason_id + + # - name: SKU_Master + # type: DIMENSION + # operation: DELETE+INSERT + # fetch_by: master + + # - name: display_master + # type: DIMENSION + # operation: DELETE+INSERT + # fetch_by: master + + # - name: Employee_Master + # type: DIMENSION + # operation: DELETE+INSERT + # fetch_by: master + + # - name: mapping_storevisibility + # type: BRIDGE + # operation: ONLY_INSERT + # fetch_by: run_date + + # - name: Master_VisibilityReason + # type: DIMENSION + # operation: DELETE+INSERT + # fetch_by: none + + + # - name: Master_VisibilityDefinition + # type: DIMENSION + # operation: DELETE+INSERT + # fetch_by: none + + + # - name: Master_Salesterritorylayer + # type: DIMENSION + # operation: DELETE+INSERT + # fetch_by: none + + - name: Promotion + type: FACT + operation: INSERT + fetch_by: mids +