ankitm/data_pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

19-06-2026 1st commit

Browse Source
This commit is contained in:
Ankit Malik
2026-06-19 16:51:03 +05:30
parent d60740ae48
commit 97b22ea0f8
23 changed files with 2026 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
clickhouse_task/delete_task.py
+6 -4
View File
@@ -1,6 +1,7 @@
from __future__ import annotations from __future__ import annotations
import polars as pl import polars as pl
from datetime import date, datetime, timedelta
from log import log from log import log
@@ -91,9 +92,10 @@ def delete_existing_data(
client, client,
table_name, table_name,
f""" f"""
toMonth(visit_date) = {run_date.month} project_id = 40148
AND toYear(visit_date) = {run_date.year} AND toMonth(visit_date) = toMonth(toDate('2026-06-18'))
""", AND toYear(visit_date) = toYear(toDate('2026-06-18'))
""",
) )
return return
@@ -142,7 +144,7 @@ def delete_existing_data(
client, client,
table_name, table_name,
f""" f"""
toDate(visit_date) BETWEEN toDate('{run_date - }') AND toDate('{run_date}') toDate(visit_date) BETWEEN toDate('{run_date - timedelta(days=15) }') AND toDate('{run_date}')
= =
""", """,
) )
logs/etl_20260619.log
+1295
View File
File diff suppressed because it is too large Load Diff
main2.py
+9 -36
View File
@@ -43,9 +43,9 @@ from mids import (
MID_TABLE_COV1, MID_TABLE_COV1,
) )
from masters.dimensions import *
from masters.bridge import * from masters.bridge import *
from src.fact import * from src.fact import *
from src.dim import *
# ========================================================== # ==========================================================
@@ -64,29 +64,6 @@ def table_exists(
) )
def get_dataframe(
sql_engine,
fn_name: str,
fetch_by: str,
table_name: str,
table_type: str ,
mids,
run_date,
):
fn = globals()[fn_name]
if fetch_by == "mids" or "run_date":
return fn(sql_engine, table_name , table_type, mids, run_date)
return fn(sql_engine ,table_name,table_type)
# ========================================================== # ==========================================================
# Main # Main
# ========================================================== # ==========================================================
@@ -158,9 +135,7 @@ def main():
# ------------------------------------------------------ # ------------------------------------------------------
# Process Tables # Process Tables
# ------------------------------------------------------ # ------------------------------------------------------
for table in config["tables"]: for table in config["tables"]:
table_name = table["name"] table_name = table["name"]
operation = table["operation"] operation = table["operation"]
fetch_by = table["fetch_by"] fetch_by = table["fetch_by"]
@@ -177,16 +152,14 @@ def main():
log.info(f"Fetching Data from sql server for table-: {table_name} ..............") log.info(f"Fetching Data from sql server for table-: {table_name} ..............")
fn_name = f"fetch_{table_name}" fetch_list=["mids" ,"run_date", "reason_id"]
df = get_dataframe( if fetch_by in fetch_list :
sql_engine, fn_name = f"fetch_{table_name}"
fn_name=fn_name, fn = globals()[fn_name]
fetch_by=fetch_by, df=fn(sql_engine, table_name , table_type, mids, run_date)
table_name=table_name, else:
table_type=table_type, df = fetch_data(sql_engine ,table_name,table_type)
mids=mids,
run_date=run_date,
)
log.info(f"Fetched total row -: {len(df)} from sql server for table-:{table_name} ...........!!!") log.info(f"Fetched total row -: {len(df)} from sql server for table-:{table_name} ...........!!!")
if df.is_empty(): if df.is_empty():
src/bridge.py
+142
View File
@@ -0,0 +1,142 @@
from pathlib import Path
import polars as pl
from sqlalchemy import Engine
from datetime import date , timedelta
from log import log
from db_con.connection import (
build_sql_server_engine,
build_clickhouse_engine,
get_clickhouse_client,
)
def fetch_mapping_storevisibility(
sql_engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
client= get_clickhouse_client()
def table_exists(
client,
table_name: str,
) -> bool:
return bool(
client.command(
f"EXISTS TABLE {table_name}"
)
)
def get_reason_ids_mapping_storevisibility(
client,
table_name: str = "coverage_remarks",
) -> list[int] :
if not table_exists(client, table_name):
log.warning(f"Table '{table_name}' does not exist. During collecting reason_ids")
return [0]
query = f"""
SELECT DISTINCT StoreId
FROM mapping_storevisibility
WHERE toDate(Fromdate) <= {run_date}
AND toDate(Todate) >= {run_date}
AND a.Project_Id = '40148'
"""
# ClickHouse -> PyArrow -> Polars
arrow_table = client.query_arrow(query)
df= pl.from_arrow(arrow_table)
list=df["reason_id"].to_list()
return list
def fetch_data(
engine: Engine,
table_name: str,
table_type: str,
reason_ids: list[int]
) -> pl.DataFrame:
log.info(f"Fetching data from sql server for Master table......")
resaon_id_list = ",".join(str(rid) for rid in reason_ids)
sql_file = Path("src") / "sql" / f"dim" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
resaon_id_list=resaon_id_list
)
log.info(f"Fetching in progress .... ")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
store_id=get_reason_ids_mapping_storevisibility(client, "coverage_remarks")
df=fetch_data(engine=sql_engine,
table_name=table_name,
table_type=table_type,
store_id=store_id,
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
sql = f"""
SELECT DISTINCT
40148 AS project_id,
Z.StoreId AS store_id,
Z.VisibilityDefinitionId AS visibility_definition_id,
Z.FromDate AS from_date,
Z.ToDate AS to_date
FROM OneApp_KelloggsMT.dbo.Mapping_StoreVisibility Z
WHERE CAST(Z.FromDate AS DATE) <= '{run_date}'
AND CAST(Z.ToDate AS DATE) >= '{run_date}'
AND Z.VisibilityDefinitionId IN
(
SELECT DISTINCT VisibilityDefinitionId
FROM OneApp_KelloggsMT.dbo.Master_VisibilityDefinition
WHERE MenuId = 22
)
"""
log.info(
f"Fetching Mapping Store Visibility for {run_date}"
)
df = pl.read_database(
query=sql,
connection=engine
)
log.info(
f"Fetched {len(df):,} Mapping Store Visibility records"
)
return df
src/dim.py
+138
View File
@@ -0,0 +1,138 @@
from pathlib import Path
import polars as pl
from sqlalchemy import Engine
from datetime import date , timedelta
from log import log
from db_con.connection import (
build_sql_server_engine,
build_clickhouse_engine,
get_clickhouse_client,
)
def fetch_data(
engine: Engine,
table_name: str,
table_type: str
) -> pl.DataFrame:
log.info(f"Fetching data from sql server for Master table......")
sql_file = Path("src") / "sql" / f"dim" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format( )
log.info(f"Fetching in progress .... ")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
def fetch_coverage_remarks(
sql_engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
client= get_clickhouse_client()
def table_exists(
client,
table_name: str,
) -> bool:
return bool(
client.command(
f"EXISTS TABLE {table_name}"
)
)
def get_reason_ids_coverage_remarks(
client,
table_name: str = "coverage_remarks",
) -> list[int] :
if not table_exists(client, table_name):
log.warning(f"Table '{table_name}' does not exist. During collecting reason_ids")
return [0]
query = f"""
SELECT DISTINCT
reason_id
FROM {table_name}
"""
# ClickHouse -> PyArrow -> Polars
arrow_table = client.query_arrow(query)
df= pl.from_arrow(arrow_table)
list=df["reason_id"].to_list()
return list
def fetch_data(
engine: Engine,
table_name: str,
table_type: str,
reason_ids: list[int]
) -> pl.DataFrame:
log.info(f"Fetching data from sql server for Master table......")
resaon_id_list = ",".join(str(rid) for rid in reason_ids)
sql_file = Path("src") / "sql" / f"dim" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
resaon_id_list=resaon_id_list
)
log.info(f"Fetching in progress .... ")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
reason_ids=get_reason_ids_coverage_remarks(client, "coverage_remarks")
df=fetch_data(engine=sql_engine,
table_name=table_name,
table_type=table_type,
reason_ids=reason_ids,
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
src/fact.py
+134 -26
View File
@@ -26,40 +26,40 @@ from db_con.connection import (
def fetch_data( # def fetch_data(
engine: Engine, # engine: Engine,
table_name: str, # table_name: str,
table_type: str, # table_type: str,
mids: list[int], # mids: list[int],
run_date: date # run_date: date
) -> pl.DataFrame: # ) -> pl.DataFrame:
if not mids: # if not mids:
log.warning("No MIDs — nothing to fetch.") # log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame() # return pl.DataFrame()
mid_list = ",".join(str(mid) for mid in mids) # mid_list = ",".join(str(mid) for mid in mids)
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql" # sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f: # with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read() # sql_template = f.read()
sql = sql_template.format( # sql = sql_template.format(
mid_list=mid_list, # mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d") # run_date=run_date.strftime("%Y-%m-%d")
) # )
log.info(f"Fetching data for {len(mids):,} MIDs") # log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database( # df = pl.read_database(
query=sql, # query=sql,
connection=engine # connection=engine
) # )
log.info(f"Fetched {len(df):,} rows from SQL Server") # log.info(f"Fetched {len(df):,} rows from SQL Server")
return df # return df
@@ -548,7 +548,7 @@ def fetch_Attendance(
sql_template = f.read() sql_template = f.read()
sql = sql_template.format( sql = sql_template.format(
star_date=start_date.strftime("%Y-%m-%d"), start_date=start_date.strftime("%Y-%m-%d"),
run_date=run_date.strftime("%Y-%m-%d") run_date=run_date.strftime("%Y-%m-%d")
) )
log.info( log.info(
@@ -566,3 +566,111 @@ def fetch_Attendance(
) )
return df return df
def fetch_Journey_Plan( engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
log.info(f" Start Fetching data for these {len(mids):} MIDs ")
mid_list = ",".join(str(mid) for mid in mids)
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
def fetch_PaidVisibility( engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
log.info(f" Start Fetching data for these {len(mids):} MIDs ")
mid_list = ",".join(str(mid) for mid in mids)
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
def fetch_Web_Logins( engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
log.info(f" Start Fetching data for these {len(mids):} MIDs ")
mid_list = ",".join(str(mid) for mid in mids)
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
src/sql/bridge/mapping_storevisibility.sql
+12
View File
@@ -0,0 +1,12 @@
INSERT INTO mapping_storevisibility(Project_Id,StoreId,VisibilityDefinitionid,Fromdate,Todate,CreateDate,CreateBy)
select DISTINCT '40148' as Project_Id,StoreId,VisibilityDefinitionid,Fromdate,Todate,getdate(),'SP-Pius'
FROM OneApp_KelloggsMT.dbo.mapping_storevisibility z WHERE
convert(date,FROMDATE,101)<=convert(Date,getdate(),101) AND CONVERT(DATE,ToDate,101)>=convert(Date,getdate(),101)
AND z.VisibilityDefinitionid IN
(SELECT DISTINCT VisibilityDefinitionid FROM OneApp_KelloggsMT.dbo.MASTER_VISIBILITYDEFINITION WHERE MENUID=22 )
AND z.StoreId NOT IN (
SELECT distinct a.STOREID FROM Mapping_StoreVisibility a WHERE convert(date,a.FROMDATE,101)<=convert(Date,getdate(),101)
AND CONVERT(DATE,a.ToDate,101)>=convert(Date,getdate(),101) AND a.Project_Id='40148'
and a.StoreId=z.StoreId and a.VisibilityDefinitionid=z.VisibilityDefinitionId
src/sql/dim/Employee_Master.sql
+17
View File
@@ -0,0 +1,17 @@
with Employee_Master (project_id,region_id,region,state_id,state,city_id,
city,employee_id,employee_name,gender,designation_id,designation,manager_id,
manager_name,employee_joining_date,employee_resign_date,position_code,employee_legacy_code,employee_role,EMPLOYEE_TYPE)
as (
select '40148' as ProjectId, RegionId,
RegionName,StateId,StateName, CityId,CityName,
a.EmpId,EmpName,Gender,a.DesignationId,DesignationName,SupervisorId,SupervisorName,JoinDate,ResignDate,c.PositionCode ,EmpCode ,RIGHTNAME,
case when RIGHTNAME in ('Client','Client HO') Then 'NON CPM' else 'CPM' END AS EMPLOYEE_TYPE
from OneApp_KelloggsMT.dbo.vw_Employee_Detail a Left join (Select distinct PositionId, EmpId from OneApp_KelloggsMT.dbo.Mapping_PositionUser b
Where DATEdiff(d,FromDate, GETDATE())>= 0 and DATEDIFF(d,ToDate,getdate())<=0) b on a.EmpId=b.EmpId
LEFT join (select PositionId,PositionCode from OneApp_KelloggsMT.dbo.Master_Position) c on b.PositionId=c.PositionId
where 1=1
-- and EmpId not in (
--select employee_id from [dbo].[Employee_Master] where project_id='40148')
)
select * from Employee_Master
src/sql/dim/Master_Salesterritorylayer.sql
View File
src/sql/dim/Master_VisibilityDefinition.sql
View File
src/sql/dim/Master_VisibilityReason.sql
View File
src/sql/dim/SKU_Master.sql
+18
View File
@@ -0,0 +1,18 @@
with SKU_Master (project_id,category_id,category_code,Category_name,sub_category_id,sub_category_code,sub_category_name,brand_id,
brand_code,brand_name,sub_brand_id,sub_brand_code,sub_brand_name,product_id,product_name,product_code,mrp,flavour_id,
flavour,grammage,product_sequence,case_size,company_name,is_competitor,ptr,update_date,update_by)
as (
Select '40148' as ProjectId, cm.CategoryId,
cm.CategoryCode,cm.CategoryName,sca.SubCategoryId,sca.SubCategoryCode,sca.SubCategoryName,br.BrandId,br.BrandCode,br.BrandName
,sb.SubBrandId,sb.SubBrandCode,sb.SubBrandName,p.ProductId,p.ProductName,p.ProductCode,p.Mrp,fl.FlavourId,
fl.Flavour,p.Grammage,p.ProductSequence,p.CaseSize,mc.Company,mc.IsCompetitor,p.ptr,GETDATE(),'MB'
from OneApp_KelloggsMT.dbo.Master_Product p Right join
OneApp_KelloggsMT.dbo.Master_Flavour fl on p.FlavourId= Fl.FlavourId Right join
OneApp_KelloggsMT.dbo.Master_subbrand sb on p.SubBrandId= sb.SubBrandId Right join
OneApp_KelloggsMT.dbo.Master_Brand br on sb.BrandId = br.BrandId Right join
OneApp_KelloggsMT.dbo.Master_SubCategory sca on br.SubCategoryId= sca.SubCategoryId Right join
OneApp_KelloggsMT.dbo.Master_Category cm on sca.CategoryId= cm.CategoryId Right join
OneApp_KelloggsMT.dbo.Master_Company mc on mc.companyId=br.companyId )
select * from SKU_Master
src/sql/dim/Store_Master.sql
+14
View File
@@ -0,0 +1,14 @@
with Store_Master (project_id,
region_id,region,state_id,state,city_id,city,cpm_city_id,channel_id,channel,distributor_id,distributor_name, keyaccount_id, keyaccount,
insight_store_id,client_store_code,latitude,longitude,store_category_id,store_category,store_type_id,store_type,store_classification_id,store_classification, StLayerFourId,
store_id,store_name,address)
AS (
select '40148',RegionId,RegionName,StateId,StateName,CityId,
CityName,CityCode,ChannelId,ChannelName,DistributorId,Distributor,ChainId,ChainName,StoreUniqueCode,StoreCode,Latitude,
Longitude, StoreCategoryId,StoreCategory,StoreTypeId,
StoreType,StoreClassId,StoreClass,StLayerFourId,StoreId,
StoreName,Address from OneApp_KelloggsMT.dbo.vw_storedetail where 1=1
--and storeid not in (
--select store_id from Store_Master where project_id='40148')
)
select * from Store_Master
src/sql/dim/coverage_remarks.sql
+8
View File
@@ -0,0 +1,8 @@
with coverage_remarks
(project_id,reason_id,reason_remarks)
as(
select
'40148' ,reasonid,reason from OneApp_KelloggsMT.dbo.master_nonworkingreason
where reasonid not in ( {resaon_id_list})
)
select * from coverage_remarks
src/sql/dim/display_master.sql
+8
View File
@@ -0,0 +1,8 @@
with display_master (project_id,display_id,display_code,display_name,display_ref_url,created_date,
created_by) as (
select '40148',displayid,displaycode,displayname,displayrefImage,getdate(),'Pius'
from OneApp_KelloggsMT.dbo.Master_display
-- and ProductId not in (
--select product_id from [dbo].[SKU Master] where project_id='40148')
)
select * from display_master
src/sql/fact/Attendance.sql
+13 -10
View File
@@ -6,7 +6,7 @@ with Emp_cte as
, em1.Id as [Supervisor Id], isnull(em1.EmployeeName,'') as [Supervisor Name] , em1.Id as [Supervisor Id], isnull(em1.EmployeeName,'') as [Supervisor Name]
, em.Id as [Employee Id], isnull(em.EmployeeName,'') as [Employee Name] , em.Id as [Employee Id], isnull(em.EmployeeName,'') as [Employee Name]
, dm.DesignationId, dm.DesignationName as Designation , dm.DesignationId, dm.DesignationName as Designation
, convert(varchar,EM.ResignDate,101) as DOR, convert(varchar,EM.JoinDate,101) as DOJ, em.LegacyCode , CAST(EM.ResignDate AS DATE) as DOR, CAST(EM.JoinDate AS DATE) as DOJ, em.LegacyCode
--,ISNULL(ps.PositionCode ,'') PositionCode--, FromDate, ToDate --,ISNULL(ps.PositionCode ,'') PositionCode--, FromDate, ToDate
from from
OneApp_KelloggsMT.dbo.AspNetUsers em inner join OneApp_KelloggsMT.dbo.AspNetUsers em inner join
@@ -18,16 +18,16 @@ with Emp_cte as
OneApp_KelloggsMT.dbo.Master_Designation dm on Em.DesignationId= dm.DesignationId OneApp_KelloggsMT.dbo.Master_Designation dm on Em.DesignationId= dm.DesignationId
where em.RightId in (6) and em.EmployeeName not like '%test%' and where em.RightId in (6) and em.EmployeeName not like '%test%' and
(Em.ResignDate is null or Em.ResignDate>='{start_date}') and CONVERT(date,em.JoinDate,101)<=CONVERT(date,'{run_date}',101) (Em.ResignDate is null or Em.ResignDate>=CAST('{start_date}' AS DATE)) and CONVERT(date,em.JoinDate,101)<=CONVERT(date,CAST('{run_date}' AS DATE),101)
--and em.Id=2290 --and em.Id=2290
)a )a
cross join (select distinct DATE from DBO.GET_ALL_DAYS('{start_date}','{run_date}')) b cross join (select distinct DATE from DBO.GET_ALL_DAYS(CAST('{start_date}' AS DATE),CAST('{run_date}' AS DATE))) b
) AS T1 LEFT JOIN ) AS T1 LEFT JOIN
(Select distinct EmpId, PositionId, Date,FromDate,ToDate from (Select distinct EmpId, PositionId, Date,FromDate,ToDate from
(Select distinct EmpId,PositionId,FromDate,ToDate from OneApp_KelloggsMT.dbo.Mapping_PositionUser (Select distinct EmpId,PositionId,FromDate,ToDate from OneApp_KelloggsMT.dbo.Mapping_PositionUser
Where convert(Date,FromDate)<= CONVERT(date,'{run_date}',101) and convert(Date,ToDate)>=CONVERT(date,'{start_date}',101)) as m Inner Join Where convert(Date,FromDate)<= CONVERT(date,CAST('{run_date}' AS DATE),101) and convert(Date,ToDate)>=CONVERT(date,CAST('{start_date}' AS DATE),101)) as m Inner Join
(Select Date from OneApp_KelloggsMT.dbo.Master_Calender Where 1=1 and Date between CONVERT(date,'{start_date}',101) and CONVERT(date,'{run_date}',101)) (Select Date from OneApp_KelloggsMT.dbo.Master_Calender Where 1=1 and Date between CONVERT(date,CAST('{start_date}' AS DATE),101) and CONVERT(date,CAST('{run_date}' AS DATE),101))
cal on cal.Date >= m.FromDate AND cal.Date <= m.ToDate) pu cal on cal.Date >= m.FromDate AND cal.Date <= m.ToDate) pu
on T1.[Employee Id]=pu.EmpId and T1.date= pu.date Left Join on T1.[Employee Id]=pu.EmpId and T1.date= pu.date Left Join
OneApp_KelloggsMT.dbo.Master_Position ps WITH (NOLOCK) ON pu.PositionId=ps.PositionId OneApp_KelloggsMT.dbo.Master_Position ps WITH (NOLOCK) ON pu.PositionId=ps.PositionId
@@ -53,7 +53,7 @@ with Emp_cte as
FROM FROM
( (
SELECT A.DATE,TT.[MERCHANDISER CD],TT.VisitDate,TT.DOJ1,TT.DOR1,TT.COL from DBO.GET_ALL_DAYS('{start_date}','{run_date}') a SELECT A.DATE,TT.[MERCHANDISER CD],TT.VisitDate,TT.DOJ1,TT.DOR1,TT.COL from DBO.GET_ALL_DAYS(CAST('{start_date}' AS DATE),CAST('{run_date}' AS DATE)) a
left join left join
@@ -115,7 +115,7 @@ with Emp_cte as
WHERE EM.EMPLOYEENAME not like 'teststore%' AND JP.VisitDate BETWEEN '{start_date}' AND '{run_date}' WHERE EM.EMPLOYEENAME not like 'teststore%' AND JP.VisitDate BETWEEN CAST('{start_date}' AS DATE) AND CAST('{run_date}' AS DATE)
) AS T1 ) AS T1
) AS T2 ) AS T2
@@ -123,9 +123,12 @@ with Emp_cte as
WHERE COL1=1 WHERE COL1=1
) AS TT ) AS TT
on a.date=tt.visitdate)A)b on a.date=tt.visitdate)A)b
on emp.[Employee Id]=b.[MERCHANDISER CD] and emp.DATE= b.DATE ) on emp.[Employee Id]=b.[MERCHANDISER CD] and (
emp.DOR IS NULL
OR emp.DATE >= emp.DOR
) )
select * from Attendance where select * from Attendance where
parinaam_attendance !='-' and parinaam_attendance !='-' and
project_id=40148 and project_id=40148
CONVERT(date,visit_date,101) > CONVERT(date,date_of_join,101) AND visit_date > date_of_join
order by employee_id, visit_date order by employee_id, visit_date
src/sql/fact/Journey_Plan.sql
+10
View File
@@ -0,0 +1,10 @@
with Journey_Plan
(project_id,store_id,employee_id,visit_date,process_id,CreateDate,CreateBy,UpdateDate,
UpdateBy)
AS (
select
'40148' ,storeid,EmpId,cast(visitdate as DATE),Deviation,GETDATE(),'Pius',GETDATE(),'Pius' from OneApp_KelloggsMT.dbo.mapping_journeyplan
where MONTH(VisitDate) = MONTH(CAST('{run_date}' AS DATE)) AND Year(VisitDate)=Year(CAST('{run_date}' AS DATE))
AND EmpId NOT IN ( SELECT ID FROM OneApp_KelloggsMT.dbo.ASPNETUSERS
WHERE USERNAME LIKE 'TEST%'))
select * from Journey_Plan;
src/sql/fact/Login.sql
+2 -2
View File
@@ -12,7 +12,7 @@ with employee_detail AS
INNER JOIN OneApp_KelloggsMT.dbo.Master_Region rm ON st.RegionId = rm.RegionId INNER JOIN OneApp_KelloggsMT.dbo.Master_Region rm ON st.RegionId = rm.RegionId
INNER JOIN OneApp_KelloggsMT.dbo.Master_Designation dm ON Em.DesignationId = dm.DesignationId INNER JOIN OneApp_KelloggsMT.dbo.Master_Designation dm ON Em.DesignationId = dm.DesignationId
WHERE em.RightId IN (6) WHERE em.RightId IN (6)
AND (Em.ResignDate IS NULL OR CONVERT(DATE, Em.ResignDate, 101) >= '{run_date}') AND (Em.ResignDate IS NULL OR CONVERT(DATE, Em.ResignDate, 101) >= CAST('{start_date}' AS DATE))
AND em.EmployeeName NOT LIKE '%test%' AND em.EmployeeName NOT LIKE '%test%'
), ),
Login ( Login (
@@ -71,7 +71,7 @@ Login (
) AS [Last Store Out Time] ) AS [Last Store Out Time]
FROM OneApp_KelloggsMT.dbo.T_DeviceLogin ud FROM OneApp_KelloggsMT.dbo.T_DeviceLogin ud
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em ON ud.EmpId = Em.EmpId INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em ON ud.EmpId = Em.EmpId
WHERE CAST(ud.LoginDate AS DATE) = '{run_date}' -- fixed: direct date comparison WHERE CAST(ud.LoginDate AS DATE) = CAST('{start_date}' AS DATE) -- fixed: direct date comparison
) AS T1 ) AS T1
WHERE T1.col = 1 WHERE T1.col = 1
) AS T ON Em.[Employee code] = T.[Employee Code] ) AS T ON Em.[Employee code] = T.[Employee Code]
src/sql/fact/PaidVisibility.sql
+56
View File
@@ -0,0 +1,56 @@
with PaidVisibility(Mid,
project_id,store_id,employee_id,visit_date,supervisor_id,channel_id,chain_id,storetype_id,Menuid,MenuName,
Visibility_Id,
Visibility_Name,
Visibility_definition_id,Visibility_definition_name,
Visibility_deatils,Visibility_deatils_id,Visibility_value_name,
present,REASONID,reason,VisibilityQuestion,VisibilityAnswer,image1,image2,update_date,update_by)
AS (
Select sc.MID, '40148' Projectid,sc.StoreId,Em.EmpId,sc.VisitDate,Em.SupervisorId,sm.ChannelId,sm.ChainId,sm.StoreTypeId, 0 as menuid ,'' as menuname,
ts.visibilityid,mv.Visibilityname,msd.VisibilityDefinitionId,
MSD.VisibilityDefinitionName, case when isnull(TS.VisibilityTable,'')='Master_Category' then 'Category'
when isnull(TS.VisibilityTable,'')='Master_SubCategory' then 'SubCategory'
when isnull(TS.VisibilityTable,'')='Master_Brand' then 'Brand'
when isnull(TS.VisibilityTable,'')='Master_SubBrand' then 'SubBrand'
end as PaidVisibilityDeatils ,ts.visibilityvalue[Paid_visiValueID],
CASE when isnull(TS.VisibilityTable,'')='Master_Category' THEN (SELECT a.CategoryName from OneApp_KelloggsMT.dbo.Master_Category a where ts.VisibilityValue=a.CategoryId )
when isnull(TS.VisibilityTable,'')='Master_SubCategory' THEN (SELECT a.SubCategoryName from OneApp_KelloggsMT.dbo.Master_SubCategory a where ts.VisibilityValue=a.SubCategoryId )
when isnull(TS.VisibilityTable,'')='Master_Brand' THEN (SELECT a.BrandName from OneApp_KelloggsMT.dbo.Master_Brand a where ts.VisibilityValue=a.BrandId )
when isnull(TS.VisibilityTable,'')='Master_SubBrand' THEN (SELECT a.SubBrandName from OneApp_KelloggsMT.dbo.Master_SubBrand a where ts.VisibilityValue=a.SubBrandId ) end as
[Paid_VisiValueName],case when ts.Present=0 then 'N' else 'Y' end Present,
CASE WHEN TS.PRESENT=0 THEN TS.VISIBILITYREASONID ELSE '' END AS VISIBILITYREASONID ,
case when ts.Present=0 then isnull(mnp.VisibilityReason,'') else '' end Reason,
ISNULL( MVQ.VisibilityQuestionName,'') AS Question, ISNULL(TVQ.VisibilityAnswerName,'') AS Answer ,
case when ts.present=0 then '' else
case when isnull(SHI.VisibilityImage1,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PaidVisibilityImages/'+SHI.VisibilityImage1
end end as PaidVisibilityImg1 ,case when isnull(SHI.VisibilityImage2,'')='' then '' else 'https://kimt.parinaam.in/Upload/PaidVisibilityImages/'+SHI.VisibilityImage2
end as PaidVisibilityImg2 ,GETDATE(),'SP-Pius'
from
OneApp_KelloggsMT.dbo.T_Visibility ts WITH (NOLOCK) Inner join
OneApp_KelloggsMT.dbo.T_StoreCoverage sc WITH (NOLOCK) on ts.mid= sc.mid Inner Join
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId
inner join
OneApp_KelloggsMT.dbo.Master_VisibilityDefinition msd WITH (NOLOCK) on
msd.VisibilityDefinitionId=ts.VisibilityDefinitionId --and msd.menuid=ts.menuid
--inner join OneApp_KelloggsMT.dbo.Master_menu MM WITH (NOLOCK) ON ts.menuid=mm.menuid and msd.menuid=mm.menuid
left join OneApp_KelloggsMT.DBO. master_visibility mv WITH (NOLOCK) on ts.visibilityid=mv.visibilityid
LEFT join
OneApp_KelloggsMT.dbo.T_VisibilityImages SHI WITH (NOLOCK) ON ts.VId=SHI.VId
left join
OneApp_KelloggsMT.dbo.Master_VisibilityReason mnp WITH (NOLOCK) on ts.VisibilityReasonId=mnp.VisibilityReasonId
LEFT JOIN OneApp_KelloggsMT.dbo.T_VisibilityStock TVS WITH (NOLOCK) ON TS.VID=TVS.VID
LEFT JOIN OneApp_KelloggsMT.dbo.T_VisibilityQuestion TVQ WITH (NOLOCK) ON TS.Vid=TVQ.Vid
LEFT JOIN OneApp_KelloggsMT.dbo.Master_VisibilityQuestion MVQ WITH (NOLOCK) on TVQ.VisibilityQuestionId=MVQ.VisibilityQuestionId
Where 1=1 and Em.EmpName not like 'test%'
--and sc.visitdate between '10/01/2023' and '10/31/2023'
AND sc.MID in ({mid_list})
)
select * from PaidVisibility
src/sql/fact/Promotion.sql
+43
View File
@@ -0,0 +1,43 @@
with Promotion(Mid,
project_id,store_id,employee_id,visit_date,supervisor_id,channel_id,chain_id,storetype_id,promo_definition_id,promo_definition_name,
promotion_deatils,promotion_deatils_id,promotion_value_name,
present,reason,PromoQuestion,PromoAnswer,image1,image2,update_date,update_by)
as (
Select
sc.MID, '40148' Projectid,sc.StoreId,Em.EmpId,sc.VisitDate,Em.SupervisorId,sm.ChannelId,sm.ChainId,sm.StoreTypeId,msd.PromoDefinitionId,
MSD.PromoDefinitionName, case when isnull(TS.PromoTable,'')='Master_Category' then 'Category'
when isnull(TS.PromoTable,'')='Master_SubCategory' then 'SubCategory'
when isnull(TS.PromoTable,'')='Master_Brand' then 'Brand'
when isnull(TS.PromoTable,'')='Master_SubBrand' then 'SubBrand'
end as PromotionDeatils ,
TS.PromoValue[PromotionDeatilsId],
CASE when isnull(TS.PromoTable,'')='Master_Category' THEN (SELECT a.CategoryName from OneApp_KelloggsMT.dbo.Master_Category a where ts.PromoValue=a.CategoryId )
when isnull(TS.PromoTable,'')='Master_SubCategory' THEN (SELECT a.SubCategoryName from OneApp_KelloggsMT.dbo.Master_SubCategory a where ts.PromoValue=a.SubCategoryId )
when isnull(TS.PromoTable,'')='Master_Brand' THEN (SELECT a.BrandName from OneApp_KelloggsMT.dbo.Master_Brand a where ts.PromoValue=a.BrandId )
when isnull(TS.PromoTable,'')='Master_SubBrand' THEN (SELECT a.SubBrandName from OneApp_KelloggsMT.dbo.Master_SubBrand a where ts.PromoValue=a.SubBrandId ) end as
[PromoValueName],case when ts.Present=0 then 'N' else 'Y' end Present,
case when ts.Present=1 then '' else isnull(mnp.PromoReason,'') end as Reason, ISNULL(mpq.PromoQuestionName,'') AS Question,
ISNULL(tpq.PromoAnswerName,'') AS Answer ,
case when isnull(SHI.PromoImage1,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PromotionImages/'+SHI.PromoImage1
end as Image1 ,case when isnull(SHI.PromoImage2,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PromotionImages/'+SHI.PromoImage2
end as Image2 ,GETDATE(),'SP-Pius'
from
OneApp_KelloggsMT.dbo.T_Promotion ts Inner join
OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId
inner join
OneApp_KelloggsMT.dbo.Master_PromotionDefinition msd on
msd.PromoDefinitionId=ts.PromoDefinitionId
LEFT join
OneApp_KelloggsMT.dbo.T_PromotionImages SHI ON ts.PId=SHI.PId
left join
OneApp_KelloggsMT.dbo.Master_PromotionReason mnp on ts.PromoReasonId=mnp.PromoReasonId
left join OneApp_KelloggsMT.dbo.t_promotionquestion tpq on ts.PId=tpq.PId
left join OneApp_KelloggsMT.dbo.Master_PromotionQuestion mpq on tpq.PromoQuestionId=mpq.PromoQuestionId
Where 1=1 and Em.EmpName not like 'test%'
AND sc.MID in ({mid_list})
)
select * from Promotion
src/sql/fact/Web_Logins.sql
+23
View File
@@ -0,0 +1,23 @@
with Web_Logins (project_id,
supervisor_id,supervisor_name,emp_id,employee_name,designation,date,time,activity_name,activity_type,right_name,CreateDate,CreateBy)
as (
Select Distinct '40148' as ProjectId
, EM1.Id AS SupervisorId, EM1.EmployeeName AS Supervisor
, EM.Id AS EmployeeId , EM.EmployeeName as Employee
, DM.DesignationName as Designation
, cast( CONVERT(date,AL.Date) as Date) AS Date,CONVERT(NVARCHAR,AL.Date ,108) AS Time
, AL.Thread as ActivityName, AL.Level as ActivityType, R.RightName,GETDATE(),'Pius'
FROM OneApp_KelloggsMT.dbo.T_User_Activity_Log AL LEFT JOIN
OneApp_KelloggsMT.dbo.AspNetUsers EM ON AL.Logger=EM.USERNAME INNER JOIN
OneApp_KelloggsMT.dbo.AspNetUsers EM1 ON EM1.Id = EM.ManagerId INNER JOIN
OneApp_KelloggsMT.dbo.AspNetUsers EM2 ON EM2.Id = EM1.ManagerId INNER JOIN
OneApp_KelloggsMT.dbo.Master_Designation DM ON EM.DesignationId = DM.DesignationId INNER JOIN
OneApp_KelloggsMT.dbo.Master_City CM ON EM.CityId = CM.CityId INNER JOIN
OneApp_KelloggsMT.dbo.Master_State ST ON ST.StateId = cm.StateId INNER JOIN
OneApp_KelloggsMT.dbo.Master_Region RM ON RM.RegionId = ST.RegionId INNER JOIN
OneApp_KelloggsMT.dbo.Right_Master R ON EM.RightId=R.RightId
Where 1=1 and CONVERT(date,AL.Date,101) = CONVERT(DATE,CAST('{run_date}' AS DATE),101)
)
select * from Web_Logins
ORDER BY employee_name
t.yml
+6 -6
View File
@@ -62,32 +62,32 @@ tables:
- name: Store_Master - name: Store_Master
type: DIMENSION type: DIMENSION
operation: DELETE+INSERT operation: DELETE+INSERT
fetch_by: none fetch_by: master
- name: SKU_Master - name: SKU_Master
type: DIMENSION type: DIMENSION
operation: DELETE+INSERT operation: DELETE+INSERT
fetch_by: none fetch_by: master
- name: display_master - name: display_master
type: DIMENSION type: DIMENSION
operation: DELETE+INSERT operation: DELETE+INSERT
fetch_by: none fetch_by: master
- name: Employee_Master - name: Employee_Master
type: DIMENSION type: DIMENSION
operation: DELETE+INSERT operation: DELETE+INSERT
fetch_by: none fetch_by: master
- name: coverage_remarks - name: coverage_remarks
type: DIMENSION type: DIMENSION
operation: DELETE+INSERT operation: DELETE+INSERT
fetch_by: none fetch_by: reason_id
- name: mapping_storevisibility - name: mapping_storevisibility
type: BRIDGE type: BRIDGE
operation: DELETE+INSERT operation: ONLY_INSERT
fetch_by: run_date fetch_by: run_date
- name: Master_VisibilityReason - name: Master_VisibilityReason
y.yml
+72 -25
View File
@@ -1,35 +1,82 @@
tables: tables:
- name: SOS_OneApp # - name: SOS_OneApp
type: FACT # type: FACT
operation: INSERT # operation: INSERT
fetch_by: mids # fetch_by: mids
# - name: OQaD # # - name: OQaD
# # type: FACT
# # operation: INSERT
# # fetch_by: run_date
# - name: additional_visibility
# type: FACT
# operation: INSERT
# fetch_by: mids
# - name: Coverage
# type: FACT
# operation: INSERT
# fetch_by: mids
# - name: Survey
# type: FACT
# operation: INSERT
# fetch_by: mids
# - name: Login
# type: FACT
# operation: INSERT
# fetch_by: run_date
# - name: Stock_Details
# type: FACT
# operation: INSERT
# fetch_by: mids
# - name: Attendance
# type: FACT
# operation: DELETE+INSERT
# fetch_by: run_date
# - name: Journey_Plan
# type: FACT # type: FACT
# operation: INSERT # operation: INSERT
# fetch_by: run_date # fetch_by: run_date
- name: additional_visibility # - name: PaidVisibility
type: FACT # type: FACT
operation: INSERT # operation: INSERT
fetch_by: mids # fetch_by: mids
- name: Coverage # - name: Web_Logins
type: FACT # type: FACT
operation: INSERT # operation: INSERT
fetch_by: mids # fetch_by: run_date
# - name: Store_Master
# type: DIMENSION
# operation: DELETE+INSERT
# fetch_by: master
- name: Survey # - name: coverage_remarks
type: FACT # type: DIMENSION
operation: INSERT # operation: DELETE+INSERT
fetch_by: mids # fetch_by: reason_id
- name: Login # - name: SKU_Master
type: FACT # type: DIMENSION
operation: INSERT # operation: DELETE+INSERT
fetch_by: run_date # fetch_by: master
# - name: display_master
# type: DIMENSION
# operation: DELETE+INSERT
# fetch_by: master
- name: Employee_Master
type: DIMENSION
operation: DELETE+INSERT
fetch_by: master
- name: Stock_Details
type: FACT
operation: INSERT
fetch_by: mids