ankitm/data_pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

19-06-2026 2nd commit

Browse Source
This commit is contained in:
Ankit Malik
2026-06-19 18:15:27 +05:30
parent 97b22ea0f8
commit 1ba1545ea8
2 changed files with 19 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/bridge.py
+13 -46
View File
@@ -25,7 +25,7 @@ def fetch_mapping_storevisibility(
run_date: date
) -> pl.DataFrame:
run_date = run_date + timedelta(days=1)
client= get_clickhouse_client()
def table_exists(
client,
@@ -39,20 +39,21 @@ def fetch_mapping_storevisibility(
)
def get_reason_ids_mapping_storevisibility(
client,
table_name: str = "coverage_remarks",
run_date: date,
table_name: str = "mapping_storevisibility",
) -> list[int] :
if not table_exists(client, table_name):
log.warning(f"Table '{table_name}' does not exist. During collecting reason_ids")
log.warning(f"Table '{table_name}' does not exist. During collecting store_ids")
return [0]
query = f"""
SELECT DISTINCT StoreId
FROM mapping_storevisibility
WHERE toDate(Fromdate) <= {run_date}
AND toDate(Todate) >= {run_date}
AND a.Project_Id = '40148'
WHERE toDate(Fromdate) <= toDate('{run_date + timedelta(days= 1)}')
AND toDate(Todate) >= toDate('{run_date + timedelta(days= 1)}')
AND project_Id = '40148'
"""
@@ -67,11 +68,12 @@ def fetch_mapping_storevisibility(
engine: Engine,
table_name: str,
table_type: str,
reason_ids: list[int]
run_date: date,
store_id: list[int]
) -> pl.DataFrame:
log.info(f"Fetching data from sql server for Master table......")
resaon_id_list = ",".join(str(rid) for rid in reason_ids)
store_id_list = ",".join(str(sid) for sid in store_id)
sql_file = Path("src") / "sql" / f"dim" / f"{table_name}.sql"
@@ -79,7 +81,8 @@ def fetch_mapping_storevisibility(
sql_template = f.read()
sql = sql_template.format(
resaon_id_list=resaon_id_list
store_id_list=store_id_list,
run_date=run_date.strftime("%Y-%m-%d")
)
@@ -94,7 +97,7 @@ def fetch_mapping_storevisibility(
return df
store_id=get_reason_ids_mapping_storevisibility(client, "coverage_remarks")
store_id=get_reason_ids_mapping_storevisibility(client, run_date, "coverage_remarks")
df=fetch_data(engine=sql_engine,
table_name=table_name,
table_type=table_type,
@@ -104,39 +107,3 @@ def fetch_mapping_storevisibility(
return df
sql = f"""
SELECT DISTINCT
40148 AS project_id,
Z.StoreId AS store_id,
Z.VisibilityDefinitionId AS visibility_definition_id,
Z.FromDate AS from_date,
Z.ToDate AS to_date
FROM OneApp_KelloggsMT.dbo.Mapping_StoreVisibility Z
WHERE CAST(Z.FromDate AS DATE) <= '{run_date}'
AND CAST(Z.ToDate AS DATE) >= '{run_date}'
AND Z.VisibilityDefinitionId IN
(
SELECT DISTINCT VisibilityDefinitionId
FROM OneApp_KelloggsMT.dbo.Master_VisibilityDefinition
WHERE MenuId = 22
)
"""
log.info(
f"Fetching Mapping Store Visibility for {run_date}"
)
df = pl.read_database(
query=sql,
connection=engine
)
log.info(
f"Fetched {len(df):,} Mapping Store Visibility records"
)
return df
src/sql/bridge/mapping_storevisibility.sql
+6 -7
View File
@@ -1,12 +1,11 @@
INSERT INTO mapping_storevisibility(Project_Id,StoreId,VisibilityDefinitionid,Fromdate,Todate,CreateDate,CreateBy)
with mapping_storevisibility
(Project_Id,StoreId,VisibilityDefinitionid,Fromdate,Todate,CreateDate,CreateBy)
AS (
select DISTINCT '40148' as Project_Id,StoreId,VisibilityDefinitionid,Fromdate,Todate,getdate(),'SP-Pius'
FROM OneApp_KelloggsMT.dbo.mapping_storevisibility z WHERE
convert(date,FROMDATE,101)<=convert(Date,getdate(),101) AND CONVERT(DATE,ToDate,101)>=convert(Date,getdate(),101)
AND z.VisibilityDefinitionid IN
(SELECT DISTINCT VisibilityDefinitionid FROM OneApp_KelloggsMT.dbo.MASTER_VISIBILITYDEFINITION WHERE MENUID=22 )
AND z.StoreId NOT IN (
SELECT distinct a.STOREID FROM Mapping_StoreVisibility a WHERE convert(date,a.FROMDATE,101)<=convert(Date,getdate(),101)
AND CONVERT(DATE,a.ToDate,101)>=convert(Date,getdate(),101) AND a.Project_Id='40148'
and a.StoreId=z.StoreId and a.VisibilityDefinitionid=z.VisibilityDefinitionId
AND z.StoreId NOT IN ({store_id_list})
)
select * from mapping_storevisibility