ankitm/data_pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

18-06-2026 1st commit

Browse Source
This commit is contained in:
Ankit Malik
2026-06-18 18:16:50 +05:30
parent 9cd4135d7b
commit d60740ae48
11 changed files with 3104 additions and 311 deletions
Download Patch File Download Diff File Expand all files Collapse all files
clickhouse_task/delete_task.py
+6 -11
View File
@@ -38,10 +38,7 @@ def delete_rows(
WHERE {where_clause} WHERE {where_clause}
""" """
log.info( log.info(f"_ _ _ _ Deleting Data from ClickHouse for {table_name} _ _ _ _")
"Deleting from %s",
table_name,
)
client.command(query) client.command(query)
@@ -72,16 +69,14 @@ def delete_existing_data(
"Stock_Details", "Stock_Details",
} }
if table_name in mid_tables and mids: if table_name in mid_tables and mids :
mids_str = ",".join( mids_str = ",".join(str(mid) for mid in mids)
map(str, mids)
)
delete_rows( delete_rows(
client, client,
table_name, table_name,
f"MID IN ({mids_str})", f"Mid IN ({mids_str})",
) )
return return
@@ -147,8 +142,8 @@ def delete_existing_data(
client, client,
table_name, table_name,
f""" f"""
toDate(attendance_date) toDate(visit_date) BETWEEN toDate('{run_date - }') AND toDate('{run_date}')
= toDate('{run_date}') =
""", """,
) )
logs/etl_20260618.log
+2134
View File
File diff suppressed because one or more lines are too long
main2.py
+21 -65
View File
@@ -65,51 +65,27 @@ def table_exists(
def get_dataframe( def get_dataframe(
sql_engine,
fn_name: str, fn_name: str,
fetch_by: str, fetch_by: str,
sql_engine, table_name: str,
table_type: str ,
mids, mids,
run_date, run_date,
): ):
fn = globals()[fn_name] fn = globals()[fn_name]
if fetch_by == "mids": if fetch_by == "mids" or "run_date":
return fn(sql_engine, mids) return fn(sql_engine, table_name , table_type, mids, run_date)
if fetch_by == "run_date":
return fn(sql_engine, run_date)
return fn(sql_engine)
return fn(sql_engine ,table_name,table_type)
def get_empids_clickhouse_OQAD(
client,
table_name: str = "OQaD",
) -> pl.DataFrame:
if not table_exists(client, table_name):
log.warning(f"Table '{table_name}' does not exist.")
return pl.DataFrame(
schema={
"empid": pl.Int64,
"visitdate": pl.Date,
}
)
query = f"""
SELECT DISTINCT
employee_id AS empid,
visit_date AS visitdate
FROM {table_name}
"""
# ClickHouse -> PyArrow -> Polars
arrow_table = client.query_arrow(query)
return pl.from_arrow(arrow_table)
# ========================================================== # ==========================================================
# Main # Main
@@ -166,17 +142,7 @@ def main():
sql_engine, sql_engine,
run_date, run_date,
) )
qf=fetch_quiz_empids(sql_engine,run_date)
db_df = get_empids_clickhouse_OQAD(client)
matched = qf.join(
db_df,
on=["empid", "visitdate"],
how="inner",
)
empids=matched["empid"].to_list()
log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ")
# ------------------------------------------------------ # ------------------------------------------------------
# Config # Config
@@ -201,37 +167,27 @@ def main():
table_type=table["type"] table_type=table["type"]
log.info("=" * 80) log.info("=" * 80)
log.info( log.info(f"Processing Table: {table_name} | Table type is -: {table_type} | Based on -{fetch_by} and operation is used -{operation} " )
"Processing Table: %s",
table_name,
)
try: try:
# ------------------------------------------ # ------------------------------------------
# Fetch Data # Fetch Data
# ------------------------------------------ # ------------------------------------------
if table_name =="OQaD":
df=fetch_OQaD( engine=sql_engine, log.info(f"Fetching Data from sql server for table-: {table_name} ..............")
table_name=table_name, fn_name = f"fetch_{table_name}"
table_type=table_type, df = get_dataframe(
empids=empids, sql_engine,
run_date=run_date fn_name=fn_name,
) fetch_by=fetch_by,
table_name=table_name,
table_type=table_type,
else: mids=mids,
run_date=run_date,
)
df = fetch_data( log.info(f"Fetched total row -: {len(df)} from sql server for table-:{table_name} ...........!!!")
engine=sql_engine,
table_name=table_name,
table_type=table_type,
mids=mids,
run_date=run_date,
)
if df.is_empty(): if df.is_empty():
@@ -295,7 +251,7 @@ def main():
# ------------------------------------------ # ------------------------------------------
# Load Data # Load Data
# ------------------------------------------ # ------------------------------------------
log.info("_ _ _ _Inserting data into clickhouse db from sql server_ _ _ _")
load_to_clickhouse( load_to_clickhouse(
client=client, client=client,
table_name=table_name, table_name=table_name,
mids.py
-48
View File
@@ -72,51 +72,3 @@ def MID_TABLE_COV1(
def fetch_quiz_empids(engine: Engine, run_date : date) -> pl.DataFrame:
sql_template = f"""
WITH MID_TABLE_COV1 AS
(
SELECT EmpId, VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE CreateDate >= {run_date}
AND CreateDate < DATEADD(DAY,1,'{run_date}')
UNION ALL
SELECT EmpId, VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE UpdateDate >= {run_date}
AND UpdateDate < DATEADD(DAY,1, '{run_date}')
),
QUIZ AS
(
SELECT Distinct E.EmpId as empid
, CONVERT(date,DQ.VisitDate) AS visitdate
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ INNER JOIN
OneApp_KelloggsMT.dbo.vw_Employee_Detail E ON DQ.EmpId = E.EmpId inner join
OneApp_KelloggsMT.dbo.Master_OQAD_Question QU on DQ.QuestionId= qu.QuestionId inner join
OneApp_KelloggsMT.dbo.Master_OQAD_Category qc on qu.QuestionCategoryId= qc.QuestionCategoryId
where e.EmpName not like 'test%' and e.RightId in (6)
and (E.ResignDate is null or E.ResignDate>=''+CONVERT(VARCHAR,'{run_date}')+'') AND E.EmpName NOT LIKE '%TEST%'
AND DQ.EmpId IN (SELECT EmpId FROM MID_TABLE_COV1 A WHERE
DQ.EmpId=A.EmpId AND CONVERT(date,VisitDate)=CONVERT(date,A.VisitDate) )
) select * from quiz
"""
sql = sql_template.format(
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching quiz_empids data for EMPID and Visitid")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} total empid and visitdate fetched for OQAD from SQL Server")
return df
src/fact.py
+505 -18
View File
@@ -1,10 +1,31 @@
from pathlib import Path from pathlib import Path
import polars as pl import polars as pl
from sqlalchemy import Engine from sqlalchemy import Engine
from datetime import date from datetime import date , timedelta
from log import log from log import log
from db_con.connection import (
build_sql_server_engine,
build_clickhouse_engine,
get_clickhouse_client,
)
def fetch_data( def fetch_data(
engine: Engine, engine: Engine,
table_name: str, table_name: str,
@@ -42,40 +63,506 @@ def fetch_data(
def fetch_SOS_OneApp( engine: Engine,
def fetch_OQaD(
engine: Engine,
table_name: str, table_name: str,
table_type: str, table_type: str,
empids: list[int], mids: list[int],
run_date: date run_date: date
) -> pl.DataFrame: ) -> pl.DataFrame:
empid_list = ",".join(str(empid) for empid in empids)
sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql" if not mids:
log.warning("No MIDs — nothing to fetch.")
log.info(f"Exists: {sql_file.exists()}") return pl.DataFrame()
log.info(f"Path: {sql_file.resolve()}") log.info(f" Start Fetching data for these {len(mids):} MIDs ")
mid_list = ",".join(str(mid) for mid in mids)
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f: with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read() sql_template = f.read()
sql = sql_template.format( sql = sql_template.format(
empid_list=empid_list, mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d") run_date=run_date.strftime("%Y-%m-%d")
) )
log.info(f"Fetching data for {len(mids):,} MIDs")
log.info(f"Fetching data for {len(empids):,} EMPIDs")
log.info("Fetching OQaD data for run_date=%s", run_date)
df = pl.read_database( df = pl.read_database(
query=sql, query=sql,
connection=engine, connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
def fetch_additional_visibility( engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
log.info(f" Start Fetching data for these {len(mids):} MIDs ")
mid_list = ",".join(str(mid) for mid in mids)
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
def fetch_OQaD(
sql_engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
client= get_clickhouse_client()
def table_exists(
client,
table_name: str,
) -> bool:
return bool(
client.command(
f"EXISTS TABLE {table_name}"
)
)
def fetch_quiz_empids(engine: Engine, run_date : date) -> pl.DataFrame:
sql_template = f"""
WITH MID_TABLE_COV1 AS
(
SELECT EmpId, VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE CreateDate >= {run_date}
AND CreateDate < DATEADD(DAY,1,'{run_date}')
UNION ALL
SELECT EmpId, VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE UpdateDate >= {run_date}
AND UpdateDate < DATEADD(DAY,1, '{run_date}')
),
QUIZ AS
(
SELECT Distinct E.EmpId as empid
, CONVERT(date,DQ.VisitDate) AS visitdate
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ INNER JOIN
OneApp_KelloggsMT.dbo.vw_Employee_Detail E ON DQ.EmpId = E.EmpId inner join
OneApp_KelloggsMT.dbo.Master_OQAD_Question QU on DQ.QuestionId= qu.QuestionId inner join
OneApp_KelloggsMT.dbo.Master_OQAD_Category qc on qu.QuestionCategoryId= qc.QuestionCategoryId
where e.EmpName not like 'test%' and e.RightId in (6)
and (E.ResignDate is null or E.ResignDate>=''+CONVERT(VARCHAR,'{run_date}')+'') AND E.EmpName NOT LIKE '%TEST%'
AND DQ.EmpId IN (SELECT EmpId FROM MID_TABLE_COV1 A WHERE
DQ.EmpId=A.EmpId AND CONVERT(date,VisitDate)=CONVERT(date,A.VisitDate) )
) select * from quiz
"""
sql = sql_template.format(
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching quiz_empids data for EMPID and Visitid")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} total empid and visitdate fetched for OQAD from SQL Server")
return df
def get_empids_clickhouse_OQAD(
client,
table_name: str = "OQaD",
) -> pl.DataFrame:
if not table_exists(client, table_name):
log.warning(f"Table '{table_name}' does not exist.")
return pl.DataFrame(
schema={
"empid": pl.Int64,
"visitdate": pl.Date,
}
)
query = f"""
SELECT DISTINCT
employee_id AS empid,
visit_date AS visitdate
FROM {table_name}
"""
# ClickHouse -> PyArrow -> Polars
arrow_table = client.query_arrow(query)
return pl.from_arrow(arrow_table)
qf=fetch_quiz_empids(sql_engine,run_date)
db_df = get_empids_clickhouse_OQAD(client)
matched = qf.join(
db_df,
on=["empid", "visitdate"],
how="inner",
)
empids=matched["empid"].to_list()
log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ")
def fetch_data(
engine: Engine,
table_name: str,
table_type: str,
empids: list[int],
run_date: date
) -> pl.DataFrame:
empid_list = ",".join(str(empid) for empid in empids)
sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql"
log.info(f"Exists: {sql_file.exists()}")
log.info(f"Path: {sql_file.resolve()}")
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
empid_list=empid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(empids):,} EMPIDs")
log.info("Fetching OQaD data for run_date=%s", run_date)
df = pl.read_database(
query=sql,
connection=engine,
)
log.info("fn name is fetch_OQad ------Fetched %s rows", len(df))
return df
df=fetch_data( engine=sql_engine,
table_name=table_name,
table_type=table_type,
empids=empids,
run_date=run_date
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
# def fetch_OQaD(
# engine: Engine,
# table_name: str,
# table_type: str,
# empids: list[int],
# run_date: date
# ) -> pl.DataFrame:
# empid_list = ",".join(str(empid) for empid in empids)
# sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql"
# log.info(f"Exists: {sql_file.exists()}")
# log.info(f"Path: {sql_file.resolve()}")
# with open(sql_file, "r", encoding="utf-8") as f:
# sql_template = f.read()
# sql = sql_template.format(
# empid_list=empid_list,
# run_date=run_date.strftime("%Y-%m-%d")
# )
# log.info(f"Fetching data for {len(empids):,} EMPIDs")
# log.info("Fetching OQaD data for run_date=%s", run_date)
# df = pl.read_database(
# query=sql,
# connection=engine,
# )
# log.info("fn name is fetch_OQad ------Fetched %s rows", len(df))
# return df
def fetch_Coverage( engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
mid_list = ",".join(str(mid) for mid in mids)
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
def fetch_Survey( engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
mid_list = ",".join(str(mid) for mid in mids)
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
def fetch_Login( engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
mid_list = ",".join(str(mid) for mid in mids)
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
def fetch_Stock_Details( engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
mid_list = ",".join(str(mid) for mid in mids)
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
# def fetch_Coverage( engine: Engine,
# table_name: str,
# table_type: str,
# mids: list[int],
# run_date: date
# ) -> pl.DataFrame:
# if not mids:
# log.warning("No MIDs — nothing to fetch.")
# return pl.DataFrame()
# mid_list = ",".join(str(mid) for mid in mids)
# log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
# sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
# with open(sql_file, "r", encoding="utf-8") as f:
# sql_template = f.read()
# sql = sql_template.format(
# mid_list=mid_list,
# run_date=run_date.strftime("%Y-%m-%d")
# )
# log.info(f"Fetching data for {len(mids):,} MIDs")
# df = pl.read_database(
# query=sql,
# connection=engine
# )
# log.info(f"Fetched {len(df):,} rows from SQL Server")
# return df
def fetch_Attendance(
engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date | None = None,
days_back: int = 15
) -> pl.DataFrame:
"""
Fetch attendance source data.
Default:
end_date = yesterday
start_date = yesterday - 15 days
"""
if run_date is None:
run_date = date.today() - timedelta(days=1)
start_date = run_date - timedelta(days=days_back)
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
star_date=start_date.strftime("%Y-%m-%d"),
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(
f"Fetching Attendance data from {start_date} to {run_date}"
)
df = pl.read_database(
query=sql,
connection=engine
)
log.info(
f"Fetched {len(df):,} attendance rows "
f"for {df['employee_id'].n_unique():,} employees"
) )
log.info("fn name is fetch_OQad ------Fetched %s rows", len(df))
return df return df
src/sql/fact/Attendance.sql
+131
View File
@@ -0,0 +1,131 @@
with Emp_cte as
(Select T1.*, ISNULL(ps.PositionCode ,'') PositionCode,FromDate,ToDate from(
select a.*,b.date from
(Select rm.RegionId, rm.RegionName as Region, st.StateId, st.StateName , cm.CityId, cm.CityName
, em2.Id as [Area Manager Id], em2.EmployeeName AS [Area Manager]
, em1.Id as [Supervisor Id], isnull(em1.EmployeeName,'') as [Supervisor Name]
, em.Id as [Employee Id], isnull(em.EmployeeName,'') as [Employee Name]
, dm.DesignationId, dm.DesignationName as Designation
, convert(varchar,EM.ResignDate,101) as DOR, convert(varchar,EM.JoinDate,101) as DOJ, em.LegacyCode
--,ISNULL(ps.PositionCode ,'') PositionCode--, FromDate, ToDate
from
OneApp_KelloggsMT.dbo.AspNetUsers em inner join
OneApp_KelloggsMT.dbo.AspNetUsers em1 on em1.Id = em.ManagerId inner join
OneApp_KelloggsMT.dbo.AspNetUsers em2 on em2.Id = em1.ManagerId inner join
OneApp_KelloggsMT.dbo.Master_City cm on em.CityId= cm.CityId inner join
OneApp_KelloggsMT.dbo.Master_State st on cm.StateId= st.StateId inner join
OneApp_KelloggsMT.dbo.Master_Region rm on st.RegionId= rm.RegionId inner join
OneApp_KelloggsMT.dbo.Master_Designation dm on Em.DesignationId= dm.DesignationId
where em.RightId in (6) and em.EmployeeName not like '%test%' and
(Em.ResignDate is null or Em.ResignDate>='{start_date}') and CONVERT(date,em.JoinDate,101)<=CONVERT(date,'{run_date}',101)
--and em.Id=2290
)a
cross join (select distinct DATE from DBO.GET_ALL_DAYS('{start_date}','{run_date}')) b
) AS T1 LEFT JOIN
(Select distinct EmpId, PositionId, Date,FromDate,ToDate from
(Select distinct EmpId,PositionId,FromDate,ToDate from OneApp_KelloggsMT.dbo.Mapping_PositionUser
Where convert(Date,FromDate)<= CONVERT(date,'{run_date}',101) and convert(Date,ToDate)>=CONVERT(date,'{start_date}',101)) as m Inner Join
(Select Date from OneApp_KelloggsMT.dbo.Master_Calender Where 1=1 and Date between CONVERT(date,'{start_date}',101) and CONVERT(date,'{run_date}',101))
cal on cal.Date >= m.FromDate AND cal.Date <= m.ToDate) pu
on T1.[Employee Id]=pu.EmpId and T1.date= pu.date Left Join
OneApp_KelloggsMT.dbo.Master_Position ps WITH (NOLOCK) ON pu.PositionId=ps.PositionId
) ,
Attendance
(project_id,
employee_id,position_code,legacy_code,supervisor_id,date_of_join,date_of_resign,visit_date,parinaam_attendance,
update_date,update_by,Unique_Id)
AS (
select '40148', emp.[Employee Id], Case When emp.DATE <= ToDate and emp.DATE>= FromDate
then emp.PositionCode Else '' End as PositionCode ,emp.LegacyCode, emp.[Supervisor Id] ,emp.DOJ,emp.DOR,
emp.DATE, case when emp.DATE>=emp.DOR then 'R' WHEN emp.DATE< emp.DOJ THEN '-'
WHEN b.COL IS NULL AND (emp.DATE<=emp.DOR OR emp.DOR IS NULL )AND (emp.DATE>=emp.DOJ OR emp.DOJ IS NULL) THEN 'W' ELSE b.COL
END AS Attendance,GETDATE(),'SP-Pius',CAST('40148' AS VARCHAR) + '_' + CAST(emp.[Employee Id] AS VARCHAR)
from Emp_cte as emp
left join
(select a.*
FROM
(
SELECT A.DATE,TT.[MERCHANDISER CD],TT.VisitDate,TT.DOJ1,TT.DOR1,TT.COL from DBO.GET_ALL_DAYS('{start_date}','{run_date}') a
left join
(
SELECT distinct [EMPLOYEE ID] AS [MERCHANDISER CD], VISITDATE
, CASE WHEN COL=1 THEN 'H' WHEN COL=2 THEN 'L' WHEN COL=4 THEN 'M' WHEN COL=5 THEN 'CO' WHEN COL=6 THEN 'WO'
WHEN COL=7 THEN 'P' WHEN COL=10 THEN 'EH' WHEN COL=8 THEN 'HD' WHEN COL IS NULL THEN 'A' END AS COL
, DOJ AS DOJ1, DOR AS DOR1
FROM
(
SELECT DOJ, DOR, Region, [AREA MANAGER], SUPERVISOR_CD, [SUPERVISOR], City, [EMPLOYEE ID], MERCHANDISER, VISITDATE
, REMARKS, COL, ROW_NUMBER()OVER (PARTITION BY [EMPLOYEE ID], VISITDATE ORDER BY COL DESC,Intime, VISITDATE, ROW ) AS COL1
FROM
(
SELECT DOJ, DOR, Region, [AREA MANAGER], SUPERVISOR_CD, [SUPERVISOR], City, [EMPLOYEE ID], MERCHANDISER, InTime, VISITDATE
, REMARKS, COL, ROW_NUMBER()OVER (PARTITION BY [EMPLOYEE ID], VISITDATE ORDER BY COL) AS ROW
FROM
(
SELECT RM.RegionName as Region, em2.EmployeeName AS [Area Manager]
, em1.Id as Supervisor_cd, em1.EmployeeName AS [Supervisor]
, CM.CityName as City, EM.Id as [Employee Id], em.EmployeeName AS Merchandiser
,JP.VisitDate as VisitDate, sc.InTime
,ISNULL((SELECT top 1 case
when d.ReasonId IS NULL OR d.ReasonId =0 then '' else REASON+' - ' +
case when replace(replace(D.REMARK,CHAR(13),''),CHAR(10),' ')<>'' then replace(replace(D.REMARK,CHAR(13),''),CHAR(10),' ')
else replace(replace(D.REMARK,CHAR(13),''),CHAR(10),' ') end end as REASON FROM OneApp_KelloggsMT.dbo.T_StoreCoverage D
INNER JOIN OneApp_KelloggsMT.dbo.Master_NonWorkingReason N ON N.ReasonId = D.ReasonId
WHERE D.ISDEL = 0
AND D.EmpId = JP.EmpId AND D.StoreId = JP.StoreId AND d.VisitDate = JP.VisitDate),'') AS Remarks
, CASE WHEN (Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId = SC.EmpId AND JP.StoreId = SC.StoreId AND JP.VisitDate = SC.VisitDate
AND SC.ReasonId=11) >=1 THEN 1
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
AND SC.ReasonId=2) >=1 THEN 2
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
AND SC.ReasonId=6) >=1 THEN 4
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
AND SC.ReasonId=12) >=1 THEN 5
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
AND SC.ReasonId=13) >=1 THEN 6
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
AND SC.ReasonId=7) >=1 THEN 8
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
AND SC.ReasonId=26) >=1 THEN 10
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
AND SC.ReasonId IN(0,1,3,4,5,8,9,10,14,15,17,18,19,20,21,22,24)) >=1 THEN 7
END END END END END END END END AS COL, em.ResignDate as DOR, em.JoinDate as DOJ
FROM OneApp_KelloggsMT.dbo.Mapping_JourneyPlan JP LEFT OUTER JOIN
OneApp_KelloggsMT.dbo.T_StoreCoverage SC ON JP.StoreId= SC.StoreId AND JP.EmpId= SC.EmpId AND JP.VisitDate= SC.VisitDate INNER JOIN
OneApp_KelloggsMT.dbo.Master_Store SM ON JP.StoreId= SM.StoreId INNER JOIN
OneApp_KelloggsMT.dbo.AspNetUsers em ON JP.EmpId= em.Id inner join
OneApp_KelloggsMT.dbo.AspNetUsers em1 on em1.Id = em.ManagerId inner join
OneApp_KelloggsMT.dbo.AspNetUsers em2 on em2.Id = em1.ManagerId inner join
OneApp_KelloggsMT.dbo.Master_City cm on em.CityId= cm.CityId inner join
OneApp_KelloggsMT.dbo.Master_State st on cm.StateId= st.StateId inner join
OneApp_KelloggsMT.dbo.Master_Region rm on st.RegionId= rm.RegionId --Left Join
WHERE EM.EMPLOYEENAME not like 'teststore%' AND JP.VisitDate BETWEEN '{start_date}' AND '{run_date}'
) AS T1
) AS T2
) AS T
WHERE COL1=1
) AS TT
on a.date=tt.visitdate)A)b
on emp.[Employee Id]=b.[MERCHANDISER CD] and emp.DATE= b.DATE )
select * from Attendance where
parinaam_attendance !='-' and
project_id=40148 and
CONVERT(date,visit_date,101) > CONVERT(date,date_of_join,101)
order by employee_id, visit_date
src/sql/fact/Coverage.sql
+90 -146
View File
@@ -1,164 +1,108 @@
WITH SOS_BASE AS
(
SELECT
sm.CountryName,
sc.MID,
sm.RegionName,
sm.StateName,
sm.CityName,
Em.SupervisorName,
em.EmpId,
Em.EmpName AS EmployeeName,
Em.DesignationName AS Designation,
sm.StoreId,
CONVERT(varchar, sc.VisitDate, 101) AS VisitDate,
sm.StoreCode,
sm.StoreName,
sm.Address,
sm.StoreTypeid,
sm.ChannelId,
sm.ChainName,
MSD.SOSDefinitionName, with Executed as
(select JP.mid, CASE WHEN isnull(B.MID,'')<>'' then 'Y'
ELSE 'N' END AS COVERED,
CASE WHEN isnull(C.MID,'')<>'' then 'Y'
ELSE 'N' END AS EXECUTED,isnull(NR.REASON,'') AS REASON
CASE from OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) left join
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Category' THEN 'Category' (SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock)
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubCategory' THEN 'SubCategory' inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Brand' THEN 'Brand' ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101)
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubBrand' THEN 'SubBrand' AND JP.ReasonId in (0,1,3,9,10,19,20))b
END AS SOSHeaderDeatils, on JP.MID=b.MID
TS.SOSHeaderName, left join
TS.SOSHeaderValue AS SOSHeaderID, (SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock)
inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B
ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101)
AND JP.ReasonId in (0,19))c
on JP.MID=c.MID
left join
OneApp_KelloggsMT.dbo.Master_NonWorkingReason nr on JP.ReasonId=nr.ReasonId
Where 1=1 AND JP.MID in ({mid_list})
'Header_Image' AS HDR1, ) ,
Coverage
(project_id,Mid,
store_id,employee_id,visit_date,in_time,out_time,duration_in_minutes,
is_covered,is_executed,reason_remarks,storetype_id,
supervisor_id,coverage_type,distance_in_meters,reasonId,camera_allow,update_date,
update_by,Unique_Id)
CASE AS (
WHEN ISNULL(TSC.SOSChildTable,'')='Master_Category' THEN 'Category'
WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubCategory' THEN 'SubCategory'
WHEN ISNULL(TSC.SOSChildTable,'')='Master_Brand' THEN 'Brand'
WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubBrand' THEN 'SubBrand'
END AS SOSChildDeatils,
TSC.SOSChildName, Select '40148' as ProjectId,JP.MID, sm.StoreId,JP.EmpId as EmpId,
TSC.SOSChildValue AS SOSChildID, Convert(VARCHAR,Jp.VisitDate) AS VisitDate
TSC.ChildTotalFacing, , Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time,
TS.SOSHeaderFacing, Convert(Datetime,InTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time,
TSC.ChildSelfFacing, Convert(Datetime,InTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage Where EmpId = Jp.EmpId And StoreId = Jp.StoreId
And VisitDate = Jp.VisitDate),'') As [In Time]
, Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time,
Convert(Datetime,OutTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time,
Convert(Datetime,OutTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage
Where EmpId = Jp.EmpId And StoreId = Jp.StoreId
And Convert(Date,VisitDate) = Jp.VisitDate),'') As [Out Time]
,
Isnull((Select Top 1 Case When convert(varchar,outtime) like '%00:00:00.00%' THEN 0
Else Case When outtime<InTime THEN 0 ELSE DATEDIFF(SS,CONVERT(DATETIME, InTime)
, CONVERT(DATETIME, outtime))/60 END END From
OneApp_KelloggsMT.dbo.T_StoreCoverage sc
Where sc.EmpId = Jp.EmpId And sc.StoreId = Jp.StoreId
And Convert(Date,sc.visitdate,101) = convert(date,Jp.VisitDate,101)), '') As duration_in_minute,
-- Isnull((Select Top 1 case when min(convert(varchar,InTime,108))='00:00:00' OR max(convert(varchar,CheckOutTime,108))='00:00:00'
-- then '00:00:00' else Cast(Datediff(Ss,Min(Convert(Varchar,Convert(Datetime,InTime),108)),
-- Max(Convert(Varchar,Convert(Datetime, CheckOutTime),108)) ) /3600 As Varchar(2))+':'+ Cast((Datediff(Ss,Min(Convert(Varchar,Convert(Datetime,InTime),108))
(
SELECT TOP 1 SOSTarget
FROM OneApp_KelloggsMT.dbo.Mapping_StoreShareOfShelfTarget a
WHERE a.SOSDefinitionId = MSD.SOSDefinitionId
AND a.StoreId = sm.StoreId
AND a.FromDate <= sc.VisitDate
AND a.ToDate >= sc.VisitDate
) AS SOSTarget,
CASE -- ,Max(Convert(Varchar,Convert(Datetime, CheckOutTime),108)) ) /60 )%60 As Varchar(2)) +':'+ Cast(Datediff(Ss,Min(Convert(Varchar,Convert(Datetime,InTime),108)),
WHEN ISNULL(SHI.SOSHeaderImage,'') = ''
THEN ''
ELSE
'https://kimt1.parinaam.in/Upload/SOSImages/'
+ SHI.SOSHeaderImage
END AS SOSHeaderImg
FROM OneApp_KelloggsMT.dbo.T_ShareOfShelfHeader ts
INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage sc -- Max(Convert(Varchar,Convert(Datetime, CheckOutTime),108)) ) %60 As Varchar(2)) end as time_taken From OneApp_KelloggsMT.dbo.T_StoreCoverage sc left join
ON ts.MID = sc.MID --OneApp_KelloggsMT.dbo.T_StoreCheckOut UD ON UD.EmpId = sc.EmpId AND CONVERT(VARCHAR,UD.CheckOutDate,101) = sc.VisitDate AND sc.StoreId = UD.StoreId
-- Where sc.EmpId = Jp.EmpId And sc.StoreId = Jp.StoreId
-- And VisitDate = Jp.VisitDate),'') As [Time Taken]
Case When (Select Top 1 EmpId From OneApp_KelloggsMT.dbo.T_StoreCoverage Sc LEFT join
OneApp_KelloggsMT.dbo.Master_NonWorkingReason nw on Sc.ReasonId= nw.ReasonId
Where EmpId = Jp.EmpId And StoreId = Jp.StoreId And VisitDate = Jp.VisitDate
And (sc.ReasonId =0 Or nw.ForCoverage=1) ) >0 Then 'Y'
Else 'N' End As Covered
, isnull(Exe.Executed,'') as Executed
INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail sm ,Isnull((Select Top 1 Case
ON sc.StoreId = sm.StoreId When D.ReasonId Is Null Or D.ReasonId =0 Then '' Else Reason+' - ' +
Case When Replace(Replace(D.Remark,Char(13),''''),Char(10),' ')<>'' Then Replace(Replace(D.Remark,Char(13),''),Char(10),' ')
Else Replace(Replace(D.Remark,Char(13),''),Char(10),' ') End End As Reason
From OneApp_KelloggsMT.dbo.T_StoreCoverage D
Inner Join OneApp_KelloggsMT.dbo.Master_NonWorkingReason N On N.ReasonId = D.ReasonId
Where D.Isdel = 0
And D.EmpId = Jp.EmpId And D.StoreId = Jp.StoreId And D.VisitDate = Jp.VisitDate),'''') As [Detailed Remarks],
sm.StoreTypeId,Em.SupervisorId,
Case When jp.Deviation=0 Then 'Planned' When jp.Deviation=1 Then 'Adhoc'
When jp.Deviation=2 Then 'Beat Plan' When jp.Deviation=3 Then 'Non Merchandised'
When jp.Deviation=4 Then 'Add New Store' When jp.Deviation=5 Then 'Non Program' else '' End as [PJP Status]
,Isnull((Select Top 1 Case When (Sc.Latitude=0.00000000 Or Sc.Longitude Is Null) Then 0 Else Case When (sm.Latitude=0.00000000 Or sm.Latitude Is Null) Then 0
Else Case When (sm.Longitude=0.00000000 Or sm.Longitude Is Null) Then 0
Else SQRT(POWER(69.1 * ( Sc.Latitude - Sm.Latitude),2) + POWER(69.1 * ( Sm.Longitude - Sc.Longitude ) * COS(Sc.Latitude / 57.3), 2))*1000
End End End As [Distance In Mtr] From OneApp_KelloggsMT.dbo.T_StoreCoverage Sc Where Sc.Isdel=0 And Sc.EmpId= Jp.EmpId
And Sc.StoreId= Jp.StoreId And Sc.VisitDate= Jp.VisitDate),'') As [Distance In Mtr]
,ISNULL(CAST (JP.ReasonId AS VARCHAR),'0'),sm.CameraAllow,
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em GETDATE(),'SP-Pius' ,
ON sc.EmpId = Em.EmpId
INNER JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChild tsc CAST('40148' AS VARCHAR) + '_' + CAST(SM.storeid AS VARCHAR)
ON ts.SOSId = tsc.SOSId + '_' + CAST(EM.EMPID AS VARCHAR)
INNER JOIN OneApp_KelloggsMT.dbo.Master_ShareOfShelfDefinition msd
ON msd.SOSDefinitionId = tsc.SOSDefinitionId
LEFT JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfHeaderImages SHI
ON ts.SOSId = SHI.SOSId
AND ts.SOSHeaderValue = SHI.SOSHeaderValue
LEFT JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChildImages SCI FROM OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) Inner Join
ON tsc.SOSId = SCI.SOSId Executed Exe on Jp.MID= Exe.MID Inner Join
AND tsc.SOSChildValue = SCI.SOSChildValue OneApp_KelloggsMT.dbo.vw_StoreDetail sm on Jp.StoreId= sm.StoreId Inner Join
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on JP.EmpId= Em.EmpId
left join
OneApp_KelloggsMT.dbo.T_StoreCoveragePositionPivot pv with (nolock)
on jp.mid=pv.MID
Where 1=1 and em.UserName not like 'test%' AND JP.MID in ({mid_list} )
WHERE Em.EmpName NOT LIKE 'test%'
AND sc.MID IN ({mid_list})
),
SOS_PIVOT AS
(
SELECT *
FROM SOS_BASE
PIVOT
(
MIN(SOSHeaderImg)
FOR HDR1 IN ([Header_Image])
) pvt
) )
select * from Coverage
SELECT
'40148' AS ProjectId,
MID,
EmpId AS employee_id,
StoreId AS store_id,
VisitDate AS visit_date,
StoreTypeid AS storetype_id,
ChannelId AS channel_id,
SOSDefinitionName,
SOSHeaderDeatils,
SOSHeaderName,
SOSHeaderID,
SOSChildDeatils,
SOSChildName,
SOSChildID,
SOSHeaderFacing,
ChildTotalFacing,
ChildSelfFacing,
SOSTarget,
GETDATE() AS update_date,
'SP-Pius' AS update_by
FROM SOS_PIVOT
GROUP BY
CountryName,
MID,
RegionName,
StateName,
CityName,
SupervisorName,
EmpId,
EmployeeName,
Designation,
StoreId,
VisitDate,
StoreCode,
StoreName,
Address,
StoreTypeid,
ChannelId,
ChainName,
SOSDefinitionName,
SOSHeaderDeatils,
SOSHeaderName,
SOSHeaderID,
SOSChildDeatils,
SOSChildName,
SOSChildID,
ChildTotalFacing,
SOSHeaderFacing,
ChildSelfFacing,
SOSTarget
ORDER BY
RegionName,
StateName,
CityName,
VisitDate;
src/sql/fact/Login.sql
+81
View File
@@ -0,0 +1,81 @@
with employee_detail AS
(
SELECT rm.RegionId, rm.RegionName, st.StateId, st.StateName,
cm.CityId, cm.CityName,
em1.Id AS [Supervisor Code], ISNULL(em1.EmployeeName, '') AS Supervisor,
em.Id AS [Employee code], ISNULL(em.EmployeeName, '') AS [Employee Name],
dm.DesignationId, dm.DesignationName AS Designation
FROM OneApp_KelloggsMT.dbo.AspNetUsers em
INNER JOIN OneApp_KelloggsMT.dbo.AspNetUsers em1 ON em1.Id = em.ManagerId
INNER JOIN OneApp_KelloggsMT.dbo.Master_City cm ON em.CityId = cm.CityId
INNER JOIN OneApp_KelloggsMT.dbo.Master_State st ON cm.StateId = st.StateId
INNER JOIN OneApp_KelloggsMT.dbo.Master_Region rm ON st.RegionId = rm.RegionId
INNER JOIN OneApp_KelloggsMT.dbo.Master_Designation dm ON Em.DesignationId = dm.DesignationId
WHERE em.RightId IN (6)
AND (Em.ResignDate IS NULL OR CONVERT(DATE, Em.ResignDate, 101) >= '{run_date}')
AND em.EmployeeName NOT LIKE '%test%'
),
Login (
project_id,
employee_id, login_date, login_time, process_id,
first_store_in_time, last_store_out_time, update_date, update_by,
Unique_Id
) AS (
SELECT '40148' AS ProjectId,
Em.[Employee Code],
[Login Date],
[Login Time],
'0',
[1st Store In Time],
[Last Store Out Time],
GETDATE(),
'SP-Pius',
CAST(40148 AS VARCHAR) + '_' + CAST(Em.[Employee Code] AS VARCHAR)
FROM employee_detail Em
LEFT OUTER JOIN
(
SELECT SupervisorId AS [Supervisor Code],
SupervisorName,
EmpId AS [Employee Code],
EmpName,
DesignationName,
[Login Date],
[Login Time],
[1st Store In Time],
[Last Store Out Time],
AppVersion AS [App Version]
FROM (
SELECT Em.SupervisorId,
Em.SupervisorName,
Em.EmpId,
Em.EmpName,
Em.DesignationName,
CONVERT(VARCHAR, ud.LoginDate, 101) AS [Login Date],
CONVERT(VARCHAR, ud.InTime, 108) AS [Login Time],
ud.AppVersion,
ROW_NUMBER() OVER (
PARTITION BY ud.Empid, CONVERT(VARCHAR, ud.LoginDate, 101)
ORDER BY ud.LoginDate ASC
) AS col,
(SELECT MIN(CONVERT(NVARCHAR, sc.InTime, 108))
FROM OneApp_KelloggsMT.dbo.T_StoreCoverage sc
WHERE sc.Isdel = 0
AND sc.EmpId = ud.EmpId
AND CONVERT(VARCHAR, sc.VisitDate, 101) = CONVERT(VARCHAR, ud.LoginDate, 101)
) AS [1st Store In Time],
(SELECT MAX(CONVERT(NVARCHAR, OutTime, 108))
FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC
WHERE SC.Isdel = 0
AND SC.EmpId = UD.EmpId
AND CONVERT(VARCHAR, SC.VisitDate, 101) = CONVERT(VARCHAR, UD.LoginDate, 101)
) AS [Last Store Out Time]
FROM OneApp_KelloggsMT.dbo.T_DeviceLogin ud
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em ON ud.EmpId = Em.EmpId
WHERE CAST(ud.LoginDate AS DATE) = '{run_date}' -- fixed: direct date comparison
) AS T1
WHERE T1.col = 1
) AS T ON Em.[Employee code] = T.[Employee Code]
WHERE T.[Login Date] IS NOT NULL
)
SELECT * FROM Login;
src/sql/fact/SOS_OneApp.sql
+70
View File
@@ -0,0 +1,70 @@
with SOS_OneApp ( project_id,Mid,
employee_id,store_id,visit_date,storetype_id,channel_id,SOSDefinitionName,SOSHeaderDeatils,SOSHeaderName,SOSHeaderID,SOSChildDeatils,SOSChildName,SOSChildID,
SOSHeaderFacing,ChildTotalFacing,ChildSelfFacing,SOSTarget,update_date,update_by)
as (
Select '40148' AS ProjectId,MID,
EmpId
,StoreId,
VisitDate,StoreTypeid,ChannelId ,
--CameraAllow,
SOSDefinitionName,SOSHeaderDeatils,SOSHeaderName,[SOSHeaderID] ,SOSChildDeatils , SOSChildName,SOSChildID,SOSHeaderFacing,ChildTotalFacing,
ChildSelfFacing ,SOSTarget,getdate(),'SP-Pius' from
(
Select sm.CountryName,SC.MID, sm.RegionName, sm.StateName, sm.CityName, Em.SupervisorName, em.EmpId as EmpId
, Em.EmpName as EmployeeName, Em.DesignationName as Designation, sm.StoreId,convert(varchar,sc.VisitDate,101)VisitDate, sm.StoreCode, sm.StoreName,
sm.Address, sm.StoreTypeid ,sm.ChannelId,sm.ChainName,
MSD.SOSDefinitionName, case when isnull(TS.SOSHeaderTable,'')='Master_Category' then 'Category'
when isnull(TS.SOSHeaderTable,'')='Master_SubCategory' then 'SubCategory'
when isnull(TS.SOSHeaderTable,'')='Master_Brand' then 'Brand'
when isnull(TS.SOSHeaderTable,'')='Master_SubBrand' then 'SubBrand'
end as SOSHeaderDeatils ,
TS.SOSHeaderName,TS.SOSHeaderValue[SOSHeaderID], 'Header'+'_'+'Image' as HDR1,
case when isnull(tsc.SOSChildTable,'')='Master_Category' then 'Category'
when isnull(tsc.SOSChildTable,'')='Master_SubCategory' then 'SubCategory'
when isnull(tsc.SOSChildTable,'')='Master_Brand' then 'Brand'
when isnull(tsc.SOSChildTable,'')='Master_SubBrand' then 'SubBrand'
end as SOSChildDeatils ,
TSC.SOSChildName,TSC.SOSChildValue[SOSChildID],TSC.ChildTotalFacing,ts.SOSHeaderFacing,TSC.ChildSelfFacing
,
(select top 1 SOSTarget from OneApp_KelloggsMT.dbo.Mapping_StoreShareOfShelfTarget a where a.SOSDefinitionId=msd.SOSDefinitionId and
a.Storeid=sm.StoreId and a.fromdate<=sc.visitdate and a.todate>=sc.visitdate ) as SOSTarget,
case when isnull(SHI.SOSHeaderImage,'')='' then '' else 'https://kimt1.parinaam.in/Upload/SOSImages/'+SHI.SOSHeaderImage
end as SOSHeaderImg
--,case when isnull(SCI.SOSChildImage,'')='' then '' else 'https://di1.parinaam.in/Upload/SOSImages/'+SCI.SOSChildImage
--end as SOSChildImage
from
OneApp_KelloggsMT.dbo.T_ShareOfShelfHeader ts Inner join
OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId
inner join
OneApp_KelloggsMT.dbo.T_ShareOfShelfChild tsc on ts.sosid=tsc.SOSId
inner join
OneApp_KelloggsMT.dbo.Master_ShareOfShelfDefinition msd on
msd.SOSDefinitionId=tsc.SOSDefinitionId
LEFT join
OneApp_KelloggsMT.dbo.T_ShareOfShelfHeaderImages SHI ON ts.SOSId=SHI.SOSId AND TS.SOSHeaderValue=SHI.SOSHeaderValue
LEFT JOIN
OneApp_KelloggsMT.dbo.T_ShareOfShelfChildImages SCI ON TSC.SOSId=SCI.SOSId AND TSC.SOSChildValue=SCI.SOSChildValue
Where 1=1 AND EM.EMPNAME NOT LIKE 'test%' AND sc.mid IN ({mid_list})
-- AND sc.VisitDate BETWEEN '09/18/2023' AND '09/21/2023'
--and sc.StoreId not in (
--select store_id from SOS_OneApp a where a.project_id=40148 and a.store_id=sc.StoreId
--and a.employee_id=sc.EmpId and convert(date,sc.VisitDate,101)=convert(date,a.visit_date,101) )
) as A
Pivot
(
min(SOSHeaderImg)
for HDR1 in([Header_Image])
) as pvt1
GROUP BY CountryName,MID, RegionName, StateName, CityName, SupervisorName, EmpId
, EmployeeName, Designation, StoreId,VisitDate,StoreCode, StoreName,
Address, StoreTypeid ,ChannelId,ChainName,
SOSDefinitionName,SOSHeaderDeatils,SOSHeaderName,[SOSHeaderID] ,SOSChildDeatils ,
SOSChildName,SOSChildID,ChildTotalFacing,SOSHeaderFacing,ChildSelfFacing,SOSTarget )
select * from SOS_OneApp
src/sql/fact/Stock_Details.sql
+29
View File
@@ -0,0 +1,29 @@
with Stock_Details (project_id,
Mid,supervisor_id,employee_id,store_id,
visitdate,storetype_id,store_category_id,product_id,MSL, MBQ,Stock_Qty,
damagedstock,loststock,expirystock,skuavailability,update_date,update_by,StockType)
AS (
Select '40148' AS ProjectId,sc.MID,Em.SupervisorId,Em.EmpId,sm.StoreId,sc.VisitDate,sc.StoreTypeId,sm.StoreCategoryId,
vp.ProductId,ts.MSL,mp.MBQ, ISNULL(ts.OpeningStock,0)+ISNULL(TS.middaystock,0),0,0,0,
Case WHEN ISNULL(ts.OpeningStock,0)+ISNULL(TS.middaystock,0)>=1THEN 'Y' ELSE 'N' END AS
skuavailability, GETDATE(),'SP-Pius','Parinaam'
from
OneApp_KelloggsMT.dbo.T_Stock ts Inner join
OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId
inner join OneApp_KelloggsMT.dbo.vw_product vp on ts.ProductId=vp.ProductId inner JOIN
(SELECT MBQ,ProductId,StateId,ChainId,StoreTypeId,StoreCategoryId,StoreClassId FROM OneApp_KelloggsMT.dbo.Mapping_ProductAssortment where FromDate<='{run_date}' and ToDate>='{run_date}') mp on mp.StateId=sm.StateId and
mp.ChainId=sm.ChainId
and mp.StoreTypeId=sm.StoreTypeId
and mp.StoreCategoryId=sm.StoreCategoryId
and mp.StoreClassId=sm.StoreClassId
and mp.ProductId=ts.ProductId
Where Em.EmpName not like 'test%' AND sc.MID in ({mid_list})
)
select * from Stock_Details
y.yml
+22 -8
View File
@@ -1,8 +1,8 @@
tables: tables:
# - name: SOS_OneApp - name: SOS_OneApp
# type: FACT type: FACT
# operation: INSERT operation: INSERT
# fetch_by: mids fetch_by: mids
# - name: OQaD # - name: OQaD
# type: FACT # type: FACT
@@ -14,8 +14,22 @@ tables:
operation: INSERT operation: INSERT
fetch_by: mids fetch_by: mids
- name: Coverage
type: FACT
operation: INSERT
fetch_by: mids
# - name: Survey - name: Survey
# type: FACT type: FACT
# operation: INSERT operation: INSERT
# fetch_by: mids fetch_by: mids
- name: Login
type: FACT
operation: INSERT
fetch_by: run_date
- name: Stock_Details
type: FACT
operation: INSERT
fetch_by: mids