3rd commit

masters/bridge.py

@@ -1,5 +1,5 @@
 import os
-import pyarrow
+# import pyarrow
 import sys
 import logging
 from datetime import date, timedelta
@@ -10,7 +10,7 @@ import clickhouse_connect
 from dotenv import load_dotenv
 
 from log import log 
-from clickhouse_task.create_table import create_clickhouse_table , check
+from clickhouse_task.create_table import *
 from db_con.connection import *
 from mids import *
 
@@ -22,7 +22,7 @@ from mids import *
 
 
 
-def fetch_mapping_store_visibility(
+def fetch_mapping_storevisibility(
     engine: Engine,
     run_date: date
 ) -> pl.DataFrame:
@@ -47,7 +47,7 @@ def fetch_mapping_store_visibility(
           WHERE MenuId = 22
       )
     """
-
+    
     log.info(
         f"Fetching Mapping Store Visibility for {run_date}"
     )
@@ -56,7 +56,7 @@ def fetch_mapping_store_visibility(
         query=sql,
         connection=engine
     )
-
+    
     log.info(
         f"Fetched {len(df):,} Mapping Store Visibility records"
     )

masters/dimensions.py

@@ -1,5 +1,5 @@
 import os
-import pyarrow
+# import pyarrow
 import sys
 import logging
 from datetime import date, timedelta
@@ -10,7 +10,7 @@ import clickhouse_connect
 from dotenv import load_dotenv
 
 from log import log 
-from clickhouse_task.create_table import create_clickhouse_table , check
+from clickhouse_task.create_table import *
 from db_con.connection import *
 from mids import *
 
@@ -27,7 +27,7 @@ p=40148
 
 
 
-def fetch_Store_master(engine: Engine) -> pl.DataFrame:
+def fetch_Store_Master(engine: Engine) -> pl.DataFrame:
     sql = """
     SELECT
         RegionId            AS region_id,
@@ -87,7 +87,7 @@ def fetch_Store_master(engine: Engine) -> pl.DataFrame:
 
 
 
-def fetch_sku_master(engine: Engine) -> pl.DataFrame:
+def fetch_SKU_Master(engine: Engine) -> pl.DataFrame:
 
     sql = """
     SELECT
@@ -193,7 +193,7 @@ def fetch_display_master(engine: Engine) -> pl.DataFrame:
 
 
 
-def fetch_employee_master(engine: Engine) -> pl.DataFrame:
+def fetch_Employee_Master(engine: Engine) -> pl.DataFrame:
     """
     Fetch Employee Master data.
     Source: vw_Employee_Detail + Mapping_PositionUser + Master_Position
@@ -276,7 +276,7 @@ def fetch_employee_master(engine: Engine) -> pl.DataFrame:
 
 
 
-def fetch_employee_master(engine: Engine) -> pl.DataFrame:
+def fetch_Employee_Master(engine: Engine) -> pl.DataFrame:
     """
     Fetch Employee Master data.
     Source: vw_Employee_Detail + Mapping_PositionUser + Master_Position
@@ -357,7 +357,7 @@ def fetch_employee_master(engine: Engine) -> pl.DataFrame:
 
 
 
-def fetch_master_visibility_reason(engine: Engine) -> pl.DataFrame:
+def fetch_Master_VisibilityReason(engine: Engine) -> pl.DataFrame:
     """
     Source:
         Master_VisibilityReason
@@ -390,7 +390,7 @@ def fetch_master_visibility_reason(engine: Engine) -> pl.DataFrame:
 
 
 
-def fetch_master_visibility_definition(engine: Engine) -> pl.DataFrame:
+def fetch_Master_VisibilityDefinition(engine: Engine) -> pl.DataFrame:
     """
     Source:
         OneApp_KelloggsMT.dbo.Master_VisibilityDefinition
@@ -423,3 +423,64 @@ def fetch_master_visibility_definition(engine: Engine) -> pl.DataFrame:
     return df
 
 
+
+
+def fetch_Master_Salesterritorylayer(
+    engine: Engine
+) -> pl.DataFrame:
+    """
+    Source:
+        Master_SalesTerritoryLayerOne
+        Master_SalesTerritoryLayerTwo
+        Master_SalesTerritoryLayerThree
+        Master_SalesTerritoryLayerFour
+
+    Target:
+        Master_Salesterritorylayer
+    """
+
+    sql = """
+    SELECT DISTINCT
+        40148 AS project_id,
+
+        D.StLayerOneId AS st_layer_one_id,
+        D.StLayerOneName AS st_layer_one_name,
+
+        C.StLayerTwoId AS st_layer_two_id,
+        C.StLayerTwoName AS st_layer_two_name,
+
+        B.StLayerThreeId AS st_layer_three_id,
+        B.StLayerThreeName AS st_layer_three_name,
+
+        A.StLayerFourId AS st_layer_four_id,
+        A.StLayerFourName AS st_layer_four_name,
+
+        GETDATE() AS create_date,
+        'SP-Pius' AS create_by
+
+    FROM OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerFour A
+
+    INNER JOIN OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerThree B
+        ON A.StLayerThreeId = B.StLayerThreeId
+
+    INNER JOIN OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerTwo C
+        ON B.StLayerTwoId = C.StLayerTwoId
+
+    INNER JOIN OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerOne D
+        ON C.StLayerOneId = D.StLayerOneId
+    """
+
+    log.info("Fetching Master Sales Territory Layer data")
+
+    df = pl.read_database(
+        query=sql,
+        connection=engine
+    )
+
+    log.info(
+        f"Fetched {len(df):,} Master Sales Territory Layer records"
+    )
+
+    return df
+
+