2nd commit

masters/bridge.py

@@ -0,0 +1,64 @@
+import os
+import pyarrow
+import sys
+import logging
+from datetime import date, timedelta
+import polars as pl
+from sqlalchemy import create_engine, text
+from sqlalchemy.engine import Engine, URL
+import clickhouse_connect
+from dotenv import load_dotenv
+
+from log import log 
+from clickhouse_task.create_table import create_clickhouse_table , check
+from db_con.connection import *
+from mids import *
+
+
+
+
+
+
+
+
+
+def fetch_mapping_store_visibility(
+    engine: Engine,
+    run_date: date
+) -> pl.DataFrame:
+
+    sql = f"""
+    SELECT DISTINCT
+        40148 AS project_id,
+        Z.StoreId AS store_id,
+        Z.VisibilityDefinitionId AS visibility_definition_id,
+        Z.FromDate AS from_date,
+        Z.ToDate AS to_date
+
+    FROM OneApp_KelloggsMT.dbo.Mapping_StoreVisibility Z
+
+    WHERE CAST(Z.FromDate AS DATE) <= '{run_date}'
+      AND CAST(Z.ToDate AS DATE) >= '{run_date}'
+
+      AND Z.VisibilityDefinitionId IN
+      (
+          SELECT DISTINCT VisibilityDefinitionId
+          FROM OneApp_KelloggsMT.dbo.Master_VisibilityDefinition
+          WHERE MenuId = 22
+      )
+    """
+
+    log.info(
+        f"Fetching Mapping Store Visibility for {run_date}"
+    )
+
+    df = pl.read_database(
+        query=sql,
+        connection=engine
+    )
+
+    log.info(
+        f"Fetched {len(df):,} Mapping Store Visibility records"
+    )
+
+    return df

masters/dimensions.py

@@ -192,10 +192,6 @@ def fetch_display_master(engine: Engine) -> pl.DataFrame:
     return df
 
 
-import polars as pl
-from sqlalchemy.engine import Engine
-from loguru import logger as log
-
 
 def fetch_employee_master(engine: Engine) -> pl.DataFrame:
     """
@@ -360,3 +356,70 @@ def fetch_employee_master(engine: Engine) -> pl.DataFrame:
     return df
 
 
+
+def fetch_master_visibility_reason(engine: Engine) -> pl.DataFrame:
+    """
+    Source:
+        Master_VisibilityReason
+
+    Target:
+        Master_VisibilityReason
+    """
+
+    sql = """
+    SELECT DISTINCT
+        40148 AS project_id,
+        MenuId AS menu_id,
+        VisibilityReasonId AS reason_id,
+        VisibilityReason AS reason
+    FROM OneApp_KelloggsMT.dbo.Master_VisibilityReason
+    """
+
+    log.info("Fetching Master Visibility Reason data")
+
+    df = pl.read_database(
+        query=sql,
+        connection=engine
+    )
+
+    log.info(
+        f"Fetched {len(df):,} Master Visibility Reason records"
+    )
+
+    return df
+
+
+
+def fetch_master_visibility_definition(engine: Engine) -> pl.DataFrame:
+    """
+    Source:
+        OneApp_KelloggsMT.dbo.Master_VisibilityDefinition
+
+    Target:
+        Master_VisibilityDefinition
+    """
+
+    sql = """
+    SELECT DISTINCT
+        40148 AS project_id,
+        VisibilityDefinitionId AS visibility_definition_id,
+        VisibilityDefinitionName AS visibility_definition_name,
+        GETDATE() AS create_date,
+        'SP-Pius' AS create_by
+    FROM OneApp_KelloggsMT.dbo.Master_VisibilityDefinition
+    """
+
+    log.info("Fetching Master Visibility Definition data")
+
+    df = pl.read_database(
+        query=sql,
+        connection=engine
+    )
+
+    log.info(
+        f"Fetched {len(df):,} Master Visibility Definition records"
+    )
+
+    return df
+
+