Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Ankit Malik
|
2ead99553f | ||
|
Ankit Malik
|
1ba1545ea8 | ||
|
Ankit Malik
|
97b22ea0f8 | ||
|
Ankit Malik
|
d60740ae48 | ||
|
Ankit Malik
|
9cd4135d7b |
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import polars as pl
|
||||
from datetime import date, datetime, timedelta
|
||||
|
||||
from log import log
|
||||
|
||||
@@ -38,10 +39,7 @@ def delete_rows(
|
||||
WHERE {where_clause}
|
||||
"""
|
||||
|
||||
log.info(
|
||||
"Deleting from %s",
|
||||
table_name,
|
||||
)
|
||||
log.info(f"_ _ _ _ Deleting Data from ClickHouse for {table_name} _ _ _ _")
|
||||
|
||||
client.command(query)
|
||||
|
||||
@@ -72,16 +70,14 @@ def delete_existing_data(
|
||||
"Stock_Details",
|
||||
}
|
||||
|
||||
if table_name in mid_tables and mids:
|
||||
if table_name in mid_tables and mids :
|
||||
|
||||
mids_str = ",".join(
|
||||
map(str, mids)
|
||||
)
|
||||
mids_str = ",".join(str(mid) for mid in mids)
|
||||
|
||||
delete_rows(
|
||||
client,
|
||||
table_name,
|
||||
f"MID IN ({mids_str})",
|
||||
f"Mid IN ({mids_str})",
|
||||
)
|
||||
|
||||
return
|
||||
@@ -96,9 +92,10 @@ def delete_existing_data(
|
||||
client,
|
||||
table_name,
|
||||
f"""
|
||||
toMonth(visit_date) = {run_date.month}
|
||||
AND toYear(visit_date) = {run_date.year}
|
||||
""",
|
||||
project_id = 40148
|
||||
AND toMonth(visit_date) = toMonth(toDate('2026-06-18'))
|
||||
AND toYear(visit_date) = toYear(toDate('2026-06-18'))
|
||||
""",
|
||||
)
|
||||
|
||||
return
|
||||
@@ -147,8 +144,8 @@ def delete_existing_data(
|
||||
client,
|
||||
table_name,
|
||||
f"""
|
||||
toDate(attendance_date)
|
||||
= toDate('{run_date}')
|
||||
toDate(visit_date) BETWEEN toDate('{run_date - timedelta(days=15) }') AND toDate('{run_date}')
|
||||
=
|
||||
""",
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
2026-06-16 17:56:17 | INFO | ================================================================================
|
||||
2026-06-16 17:56:17 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-16 17:56:17 | INFO | Pipeline Run Date: 2026-06-15
|
||||
2026-06-16 17:56:17 | INFO | Connecting to databases...
|
||||
2026-06-16 17:56:19 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000017B644A81B0>
|
||||
2026-06-16 17:56:23 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000017B6548EFD0>
|
||||
2026-06-16 17:56:24 | INFO | Database connections established
|
||||
2026-06-16 17:56:24 | INFO | Collecting MIDs for: 2026-06-15
|
||||
2026-06-16 17:56:24 | INFO | Found 849 MIDs
|
||||
2026-06-16 17:57:33 | INFO | ================================================================================
|
||||
2026-06-16 17:57:33 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-16 17:57:33 | INFO | Pipeline Run Date: 2026-06-15
|
||||
2026-06-16 17:57:33 | INFO | Connecting to databases...
|
||||
2026-06-16 17:57:34 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DCA21AB750>
|
||||
2026-06-16 17:57:36 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DCA3371590>
|
||||
2026-06-16 17:57:36 | INFO | Database connections established
|
||||
2026-06-16 17:57:36 | INFO | Collecting MIDs for: 2026-06-15
|
||||
2026-06-16 17:57:37 | INFO | Found 849 MIDs
|
||||
2026-06-16 17:58:52 | INFO | ================================================================================
|
||||
2026-06-16 17:58:52 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-16 17:58:52 | INFO | Pipeline Run Date: 2026-06-15
|
||||
2026-06-16 17:58:52 | INFO | Connecting to databases...
|
||||
2026-06-16 17:58:53 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E5AB2CF750>
|
||||
2026-06-16 17:58:54 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E5AC458910>
|
||||
2026-06-16 17:58:55 | INFO | Database connections established
|
||||
2026-06-16 17:58:55 | INFO | Collecting MIDs for: 2026-06-15
|
||||
2026-06-16 17:58:55 | INFO | Found 849 MIDs
|
||||
2026-06-16 18:00:31 | INFO | ================================================================================
|
||||
2026-06-16 18:00:31 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-16 18:00:31 | INFO | Pipeline Run Date: 2026-06-15
|
||||
2026-06-16 18:00:31 | INFO | Connecting to databases...
|
||||
2026-06-16 18:00:38 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E8C854F750>
|
||||
2026-06-16 18:00:40 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E8C9688910>
|
||||
2026-06-16 18:00:40 | INFO | Database connections established
|
||||
2026-06-16 18:00:40 | INFO | Collecting MIDs for: 2026-06-15
|
||||
2026-06-16 18:00:42 | INFO | Found 849 MIDs
|
||||
2026-06-16 18:00:44 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-16 18:02:58 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-16 18:07:47 | INFO | ================================================================================
|
||||
2026-06-16 18:07:47 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-16 18:07:47 | INFO | Pipeline Run Date: 2026-06-15
|
||||
2026-06-16 18:07:47 | INFO | Connecting to databases...
|
||||
2026-06-16 18:07:48 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DF4BB2F750>
|
||||
2026-06-16 18:07:50 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DF4CCE8A50>
|
||||
2026-06-16 18:07:50 | INFO | Database connections established
|
||||
2026-06-16 18:07:50 | INFO | Collecting MIDs for: 2026-06-15
|
||||
2026-06-16 18:07:50 | INFO | Found 849 MIDs
|
||||
2026-06-16 18:07:51 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-16 18:08:09 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-16 18:08:58 | INFO | ================================================================================
|
||||
2026-06-16 18:08:58 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-16 18:08:58 | INFO | Pipeline Run Date: 2026-06-15
|
||||
2026-06-16 18:08:58 | INFO | Connecting to databases...
|
||||
2026-06-16 18:08:59 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001639269F750>
|
||||
2026-06-16 18:09:01 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000016393858910>
|
||||
2026-06-16 18:09:01 | INFO | Database connections established
|
||||
2026-06-16 18:09:01 | INFO | Collecting MIDs for: 2026-06-15
|
||||
2026-06-16 18:09:02 | INFO | Found 849 MIDs
|
||||
2026-06-16 18:09:02 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-16 18:09:19 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-16 18:11:21 | INFO | ================================================================================
|
||||
2026-06-16 18:11:21 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-16 18:11:21 | INFO | Pipeline Run Date: 2026-06-15
|
||||
2026-06-16 18:11:21 | INFO | Connecting to databases...
|
||||
2026-06-16 18:11:23 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002240B7AF750>
|
||||
2026-06-16 18:11:24 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002240C978910>
|
||||
2026-06-16 18:11:24 | INFO | Database connections established
|
||||
2026-06-16 18:11:24 | INFO | Collecting MIDs for: 2026-06-15
|
||||
2026-06-16 18:11:25 | INFO | Found 849 MIDs
|
||||
2026-06-16 18:11:25 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-16 18:13:05 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
|
||||
@@ -0,0 +1,688 @@
|
||||
2026-06-17 10:17:16 | INFO | ================================================================================
|
||||
2026-06-17 10:17:16 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 10:17:16 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 10:17:16 | INFO | Connecting to databases...
|
||||
2026-06-17 10:17:18 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000019A9A2CF750>
|
||||
2026-06-17 10:17:21 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000019A9B778910>
|
||||
2026-06-17 10:17:22 | INFO | Database connections established
|
||||
2026-06-17 10:17:22 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 10:17:22 | INFO | Found 863 MIDs
|
||||
2026-06-17 10:17:23 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 10:17:44 | INFO | Fetched 290,984 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 10:43:05 | INFO | ================================================================================
|
||||
2026-06-17 10:43:05 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 10:43:05 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 10:43:05 | INFO | Connecting to databases...
|
||||
2026-06-17 10:43:07 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002CE210FF750>
|
||||
2026-06-17 10:43:08 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002CE2253CA50>
|
||||
2026-06-17 10:43:09 | INFO | Database connections established
|
||||
2026-06-17 10:43:09 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 10:43:10 | INFO | Found 863 MIDs
|
||||
2026-06-17 10:43:11 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 10:43:55 | INFO | Fetched 291,001 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 10:45:51 | INFO | ================================================================================
|
||||
2026-06-17 10:45:51 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 10:45:51 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 10:45:51 | INFO | Connecting to databases...
|
||||
2026-06-17 10:45:52 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000214EC45F750>
|
||||
2026-06-17 10:45:53 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000214ED908910>
|
||||
2026-06-17 10:45:54 | INFO | Database connections established
|
||||
2026-06-17 10:45:54 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 10:45:55 | INFO | Found 863 MIDs
|
||||
2026-06-17 10:45:55 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 10:46:11 | INFO | Fetched 291,005 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 10:46:11 | INFO | ================================================================================
|
||||
2026-06-17 10:46:11 | INFO | Processing Table: OQaD
|
||||
2026-06-17 10:46:11 | ERROR | Failed processing table OQaD
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 212, in main
|
||||
df=fetch_OQaD( engine=sql_engine,
|
||||
table_name=table_name,
|
||||
...<2 lines>...
|
||||
run_date=run_date
|
||||
)
|
||||
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
|
||||
2026-06-17 10:48:44 | INFO | ================================================================================
|
||||
2026-06-17 10:48:44 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 10:48:44 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 10:48:44 | INFO | Connecting to databases...
|
||||
2026-06-17 10:48:45 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000021A270378B0>
|
||||
2026-06-17 10:48:46 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000021A28538910>
|
||||
2026-06-17 10:48:47 | INFO | Database connections established
|
||||
2026-06-17 10:48:47 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 10:48:48 | INFO | Found 863 MIDs
|
||||
2026-06-17 10:48:48 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 10:49:00 | INFO | Fetched 291,005 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 10:49:01 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 10:49:01 | INFO | ================================================================================
|
||||
2026-06-17 10:49:01 | INFO | Processing Table: OQaD
|
||||
2026-06-17 10:49:01 | ERROR | Failed processing table OQaD
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 215, in main
|
||||
df=fetch_OQaD( engine=sql_engine,
|
||||
table_name=table_name,
|
||||
...<2 lines>...
|
||||
run_date=run_date
|
||||
)
|
||||
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
|
||||
2026-06-17 10:53:50 | INFO | ================================================================================
|
||||
2026-06-17 10:53:50 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 10:53:50 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 10:53:50 | INFO | Connecting to databases...
|
||||
2026-06-17 10:53:51 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001D1D6DB78B0>
|
||||
2026-06-17 10:53:52 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001D1D81F8910>
|
||||
2026-06-17 10:53:53 | INFO | Database connections established
|
||||
2026-06-17 10:53:53 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 10:53:54 | INFO | Found 863 MIDs
|
||||
2026-06-17 10:53:54 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 10:54:08 | INFO | Fetched 291,006 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 10:54:09 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 10:54:09 | INFO | ================================================================================
|
||||
2026-06-17 10:54:09 | INFO | Processing Table: OQaD
|
||||
2026-06-17 10:54:09 | ERROR | Failed processing table OQaD
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 217, in main
|
||||
df=fetch_OQaD( engine=sql_engine,
|
||||
table_name=table_name,
|
||||
...<2 lines>...
|
||||
run_date=run_date
|
||||
)
|
||||
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
|
||||
2026-06-17 10:57:28 | INFO | ================================================================================
|
||||
2026-06-17 10:57:28 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 10:57:28 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 10:57:28 | INFO | Connecting to databases...
|
||||
2026-06-17 10:57:30 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000197AB2778B0>
|
||||
2026-06-17 10:57:31 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000197AC768910>
|
||||
2026-06-17 10:57:32 | INFO | Database connections established
|
||||
2026-06-17 10:57:32 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 10:57:32 | INFO | Found 863 MIDs
|
||||
2026-06-17 10:57:33 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 10:57:45 | INFO | Fetched 291,010 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 10:57:46 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 10:57:46 | INFO | ================================================================================
|
||||
2026-06-17 10:57:46 | INFO | Processing Table: OQaD
|
||||
2026-06-17 10:57:46 | ERROR | Failed processing table OQaD
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 217, in main
|
||||
df=fetch_OQaD( engine=sql_engine,
|
||||
table_name=table_name,
|
||||
...<2 lines>...
|
||||
run_date=run_date
|
||||
)
|
||||
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
|
||||
2026-06-17 10:59:57 | INFO | ================================================================================
|
||||
2026-06-17 10:59:57 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 10:59:57 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 10:59:57 | INFO | Connecting to databases...
|
||||
2026-06-17 10:59:58 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001A32C2078B0>
|
||||
2026-06-17 11:00:00 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001A32D648910>
|
||||
2026-06-17 11:00:00 | INFO | Database connections established
|
||||
2026-06-17 11:00:00 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 11:00:01 | INFO | Found 863 MIDs
|
||||
2026-06-17 11:00:02 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 11:00:24 | INFO | Fetched 291,011 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 11:00:25 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 11:00:25 | INFO | ================================================================================
|
||||
2026-06-17 11:00:25 | INFO | Processing Table: OQaD
|
||||
2026-06-17 11:00:25 | ERROR | Failed processing table OQaD
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 217, in main
|
||||
df=fetch_OQaD( engine=sql_engine,
|
||||
table_name=table_name,
|
||||
...<2 lines>...
|
||||
run_date=run_date
|
||||
)
|
||||
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\fact\\OQaD.sql'
|
||||
2026-06-17 12:09:34 | INFO | ================================================================================
|
||||
2026-06-17 12:09:34 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 12:09:34 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 12:09:34 | INFO | Connecting to databases...
|
||||
2026-06-17 12:09:37 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000274B86078B0>
|
||||
2026-06-17 12:09:39 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000274B9A18910>
|
||||
2026-06-17 12:09:39 | INFO | Database connections established
|
||||
2026-06-17 12:09:39 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 12:09:41 | INFO | Found 863 MIDs
|
||||
2026-06-17 12:09:43 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 12:10:48 | INFO | Fetched 291,093 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 12:10:48 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 12:10:48 | INFO | ================================================================================
|
||||
2026-06-17 12:10:48 | INFO | Processing Table: OQaD
|
||||
2026-06-17 12:10:48 | INFO | Exists: False
|
||||
2026-06-17 12:10:48 | INFO | Path: D:\data_move\sql\fact\OQaD.sql
|
||||
2026-06-17 12:10:48 | ERROR | Failed processing table OQaD
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 217, in main
|
||||
df=fetch_OQaD( engine=sql_engine,
|
||||
table_name=table_name,
|
||||
...<2 lines>...
|
||||
run_date=run_date
|
||||
)
|
||||
File "D:\data_move\src\fact.py", line 62, in fetch_OQaD
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\fact\\OQaD.sql'
|
||||
2026-06-17 12:11:47 | INFO | ================================================================================
|
||||
2026-06-17 12:11:47 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 12:11:47 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 12:11:47 | INFO | Connecting to databases...
|
||||
2026-06-17 12:11:48 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000241C8DF78B0>
|
||||
2026-06-17 12:11:50 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000241CA238910>
|
||||
2026-06-17 12:11:51 | INFO | Database connections established
|
||||
2026-06-17 12:11:51 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 12:11:52 | INFO | Found 863 MIDs
|
||||
2026-06-17 12:11:52 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 12:12:14 | INFO | Fetched 291,097 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 12:12:14 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 12:12:14 | INFO | ================================================================================
|
||||
2026-06-17 12:12:14 | INFO | Processing Table: OQaD
|
||||
2026-06-17 12:12:14 | INFO | Exists: True
|
||||
2026-06-17 12:12:14 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
||||
2026-06-17 12:12:14 | INFO | Fetching data for 893 EMPIDs
|
||||
2026-06-17 12:12:14 | INFO | Fetching OQaD data for run_date=2026-06-16
|
||||
2026-06-17 12:12:14 | ERROR | Failed processing table OQaD
|
||||
Traceback (most recent call last):
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context
|
||||
self.dialect.do_execute(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
cursor, str_statement, effective_parameters, context
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute
|
||||
cursor.execute(statement, parameters)
|
||||
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near 'QUIZ'. (102) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'where'. (156)")
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 217, in main
|
||||
df=fetch_OQaD( engine=sql_engine,
|
||||
table_name=table_name,
|
||||
...<2 lines>...
|
||||
run_date=run_date
|
||||
)
|
||||
File "D:\data_move\src\fact.py", line 74, in fetch_OQaD
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine,
|
||||
)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database
|
||||
return cx.execute(
|
||||
~~~~~~~~~~^
|
||||
query=query,
|
||||
^^^^^^^^^^^^
|
||||
options=execute_options,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
).to_polars(
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute
|
||||
result = cursor_execute(query, **options)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute
|
||||
return meth(
|
||||
self,
|
||||
distilled_parameters,
|
||||
execution_options or NO_OPTIONS,
|
||||
)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection
|
||||
return connection._execute_clauseelement(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
self, distilled_params, execution_options
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement
|
||||
ret = self._execute_context(
|
||||
dialect,
|
||||
...<8 lines>...
|
||||
cache_hit=cache_hit,
|
||||
)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context
|
||||
return self._exec_single_context(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
dialect, context, statement, parameters
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context
|
||||
self._handle_dbapi_exception(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
e, str_statement, effective_parameters, cursor, context
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception
|
||||
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context
|
||||
self.dialect.do_execute(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
cursor, str_statement, effective_parameters, context
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute
|
||||
cursor.execute(statement, parameters)
|
||||
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near 'QUIZ'. (102) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'where'. (156)")
|
||||
[SQL: WITH MID_TABLE_COV1 AS
|
||||
(
|
||||
SELECT EmpId, VisitDate
|
||||
FROM OneApp_KelloggsMT.dbo.T_OQAD
|
||||
WHERE CreateDate >= 2026-06-16
|
||||
AND CreateDate < DATEADD(DAY,1,'2026-06-16')
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT EmpId, VisitDate
|
||||
FROM OneApp_KelloggsMT.dbo.T_OQAD
|
||||
WHERE UpdateDate >= 2026-06-16
|
||||
AND UpdateDate < DATEADD(DAY,1, '2026-06-16')
|
||||
),
|
||||
|
||||
OQaD
|
||||
|
||||
|
||||
QUIZ AS
|
||||
(
|
||||
SELECT DISTINCT
|
||||
E.EmpId,
|
||||
E.EmpName,
|
||||
E.SupervisorId,
|
||||
E.SupervisorName,
|
||||
E.DesignationName,
|
||||
E.CityName,
|
||||
E.StateName,
|
||||
E.RegionName,
|
||||
CAST(DQ.VisitDate AS DATE) AS VisitDate,
|
||||
DQ.QuestionId,
|
||||
DQ.AnswerId,
|
||||
QC.QuestionCategoryId,
|
||||
QC.QuestionCategory
|
||||
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E
|
||||
ON DQ.EmpId = E.EmpId
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU
|
||||
ON DQ.QuestionId = QU.QuestionId
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC
|
||||
ON QU.QuestionCategoryId = QC.QuestionCategoryId
|
||||
WHERE E.EmpName NOT LIKE '%TEST%'
|
||||
AND E.RightId = 6
|
||||
AND (
|
||||
E.ResignDate IS NULL
|
||||
OR CAST(E.ResignDate AS DATE) >= '2026-06-16'
|
||||
)
|
||||
AND EXISTS
|
||||
(
|
||||
SELECT 1
|
||||
FROM MID_TABLE_COV1 A
|
||||
WHERE A.EmpId = DQ.EmpId
|
||||
AND CAST(A.VisitDate AS DATE) = CAST(DQ.VisitDate AS DATE)
|
||||
)
|
||||
)
|
||||
|
||||
SELECT
|
||||
40148 AS project_id,
|
||||
Q.EmpId AS employee_id,
|
||||
0 AS process_id,
|
||||
Q.VisitDate AS visit_date,
|
||||
Q.QuestionCategoryId AS question_category_id,
|
||||
Q.QuestionCategory AS question_category,
|
||||
QM.QuestionId AS question_id,
|
||||
QM.Question AS question,
|
||||
ISNULL(QA.AnswerId,0) AS answer_id,
|
||||
ISNULL(QA.Answer,'') AS answer,
|
||||
CASE
|
||||
WHEN QA.AnswerId IS NULL THEN 'Not Answer'
|
||||
WHEN QA.RightAnswer = 1 THEN 'Y'
|
||||
WHEN QA.RightAnswer IS NULL THEN 'Not Answer'
|
||||
ELSE 'N'
|
||||
END AS correct_answer,
|
||||
GETDATE() AS update_date,
|
||||
'SP-Pius' AS update_by
|
||||
FROM QUIZ Q
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM
|
||||
ON Q.QuestionId = QM.QuestionId
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA
|
||||
ON Q.AnswerId = QA.AnswerId;
|
||||
where Q.EmpId not in (3167,4724,4019,4902,4375,3352,3081,5244,3181,4810,5166,4895,4817,5173,5237,3608,4646,4126,3243,3948,3950,3599,3777,5235,5346,3606,3079,5175,5242,5064,4366,5071,5339,4812,4722,3169,5270,3326,3689,4062,3131,5054,5178,3371,4829,5185,4698,4836,3760,4665,5218,4451,4280,3269,3098,5327,4978,5261,3335,5090,5320,4214,4919,4570,3150,4221,3328,3112,3290,4950,4430,5104,3383,5213,4864,5199,5206,4439,3207,5033,4684,3979,5211,4774,3117,5102,5280,3309,4931,4753,5123,3155,5085,4914,5092,4387,5325,5154,4100,4805,3402,5187,3580,3409,3060,3962,4696,4831,3715,4710,4242,5282,5121,4933,5289,5282,3444,5104,3095,4933,4142,3437,3530,3088,3715,4840,5002,4831,5187,3361,3962,5113,3086,5106,5256,5085,4914,5092,3470,4387,5277,3309,5239,4712,4945,5246,5123,4774,3952,3603,4999,3727,3378,3207,5341,4992,5199,5334,4672,5206,3342,3520,3454,3171,4916,3105,4396,3112,3290,5265,3119,5227,3143,3150,5056,4529,3328,5313,5220,4693,5320,4615,3755,4444,3406,4978,4451,3746,4280,3399,5322,4617,4795,5218,3330,3159,4698,5144,5054,3473,3131,4726,4688,4743,3689,3812,3107,3169,5332,3347,5023,5022,4812,3380,3067,3950,4954,5132,3081,5296,5244,3140,4019,3492,4375,5210,3126,5289,3482,4847,5296,4954,4436,3380,5127,5305,3306,5291,5023,5120,4531,4346,5229,3107,3812,4353,3145,3152,3330,5322,4439,5144,4795,3363,4054,4232,3399,4239,3755,5279,4930,5213,3406,4864,5286,5220,4693,5101,5227,3985,3297,3636,4529,3420,5265,4916,3591,3335,4460,5343,4638,3342,5327,3520,3171,4467,3700,5336,4987,3515,4992,3760,5267,4562,3247,5082,4555,4733,5061,5239,3090,4712,5246,4904,5270,3059,4866,5277,4750,5106,3361,4968,5175,5310,3496,3608,4840,5248,4543,3615,4142,3437,3088,3444,3078,4902,3095,3567,4543,4450,3140,5319,4614,3496,4970,4443,5132,3916,3149,5310,3327,5312,3156,4968,4894,3394,5065,4538,4688,4866,3251,3090,3268,4671,4849,4904,4726,5082,4555,5191,4733,4562,3515,5300,4951,5115,3173,5293,4944,4460,4987,4638,4467,3591,3064,4526,3821,3232,3420,5056,5046,5008,3985,3636,5350,3109,5272,5101,5279,3358,4396,5172,5110,4054,4232,5281,3363,4999,3638,3603,3104,5027,5205,4856,4151,3083,4346,3764,3819,3415,4353,3470,3299,5167,3346,5113,3306,5331,5291,5160,5120,5127,4913,5091,4564,5262,5196,4847,5077,4666,4139,3612,3978,3078,5340,4139,3612,4991,5248,3170,5155,5333,4628,5077,4217,5084,4032,4913,4564,5257,5324,4381,5331,5250,5160,4134,4312,4141,3059,3764,3415,4305,5027,3220,3104,5321,4972,5307,5267,5136,5314,4787,3315,3322,4540,3835,3700,3486,4547,5172,5001,5008,3427,3379,5046,3372,3821,4526,3650,5124,5302,5286,3130,5115,4937,3132,5293,5117,3118,4239,5300,4951,4343,3897,5219,5191,4671,4849,5212,5229,5003,5181,4531,5065,5345,3538,4538,4189,4229,3531,5276,5305,4578,3106,3149,5312,5269,3113,4436,3099,4443,4970,3916,3567,3975,4153,4844,5210,5200,4844,3533,4666,5022,3184,5200,4402,4153,3704,3975,4922,3099,5262,4751,5091,3106,5269,5093,5276,4578,3346,5202,4675,4229,3531,3182,5195,5167,3203,5003,5205,5150,5328,3220,4979,4856,5157,4452,5335,3514,5219,3638,4343,4383,3289,4390,3118,3296,5281,5110,3438,3267,5074,5117,5252,3780,3431,5005,3372,3379,5176,3232,5316,4611,4967,3068,5309,3139,3495,3673,4540,3835,4891,3486,3488,3315,3322,5307,5136,3070,4877,4787,5314,5055,3939,3234,4072,5181,4305,3251,3374,5119,4134,3080,5297,4065,4894,3127,5250,4763,4941,5257,3298,4381,3327,3156,4687,4338,5221,4450,5155,3400,5333,5207,4628,5162,3742,4098,3742,4943,5121,3129,5207,5114,5292,4687,3122,4338,4696,3464,3113,4333,3298,3127,5031,4941,4763,5209,5119,3388,3566,3538,4065,3744,4072,3374,4753,3234,3939,5102,5088,5266,3070,5055,4877,3096,3979,3132,3488,3139,4841,5019,5302,3495,5124,5309,5012,4611,3369,3509,5176,5145,5323,5005,3153,4656,4371,3431,5254,5252,4547,3091,5074,4198,3782,3433,4383,4651,4829,4302,3953,5349,5321,3514,4972,5178,5150,5328,4979,5342,4452,5157,5304,3203,5195,3490,4312,3141,5126,4141,4675,4366,5057,5235,5064,3243,5093,4217,4922,3407,4751,4810,3170,4283,3756,3704,4974,3533,4991,3184,4974,3492,3143,4098,4283,3400,3756,3407,4710,5221,3067,3580,5057,3409,5196,3060,4805,4100,3402,5325,5297,3490,3141,5154,5126,3155,5304,4931,5109,5287,4117,5280,3473,4651,4302,3953,5033,5211,3782,4684,3433,3084,4891,3746,3383,4371,4022,5254,3673,5313,4207,3153,5316,3509,5145,5323,4430,4919,4570,3369,4221,3105,5090,4563,5012,3269,4841,3098,5019,3454,5261,3438,4672,3267,4836,3096,4665,5259,5088,3727,5266,3378,5185,4390,4397,3371,4062,4945,3528,3326,4722,3388,5256,3566,3744,5202,5242,3086,5071,5209,3777,4333,3606,3079,5002,3948,4126,3599,3464,5344,4817,5173,5351,4646,3122,3352,3530,5337,3181,3359,5166,5292,5114,4943)
|
||||
-- select d.employee_id from OQaD d where project_id=40148 and d.employee_id=Q.EmpId and CONVERT(date,d.visit_date,101)=
|
||||
-- CONVERT(date,q.VisitDate,101))
|
||||
|
||||
]
|
||||
(Background on this error at: https://sqlalche.me/e/20/f405)
|
||||
2026-06-17 12:19:35 | INFO | ================================================================================
|
||||
2026-06-17 12:19:35 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 12:19:35 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 12:19:35 | INFO | Connecting to databases...
|
||||
2026-06-17 12:19:37 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000210484378B0>
|
||||
2026-06-17 12:19:38 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000021049928910>
|
||||
2026-06-17 12:19:39 | INFO | Database connections established
|
||||
2026-06-17 12:19:39 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 12:19:40 | INFO | Found 863 MIDs
|
||||
2026-06-17 12:19:40 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 12:20:14 | INFO | Fetched 291,100 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 12:20:14 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 12:20:14 | INFO | ================================================================================
|
||||
2026-06-17 12:20:14 | INFO | Processing Table: OQaD
|
||||
2026-06-17 12:20:14 | INFO | Exists: True
|
||||
2026-06-17 12:20:14 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
||||
2026-06-17 12:20:14 | INFO | Fetching data for 893 EMPIDs
|
||||
2026-06-17 12:20:14 | INFO | Fetching OQaD data for run_date=2026-06-16
|
||||
2026-06-17 12:20:30 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
|
||||
2026-06-17 12:20:30 | ERROR | Failed processing table OQaD
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 226, in main
|
||||
df = get_dataframe(
|
||||
fn_name=fn_name,
|
||||
...<3 lines>...
|
||||
run_date=run_date,
|
||||
)
|
||||
File "D:\data_move\main2.py", line 81, in get_dataframe
|
||||
return fn(sql_engine, run_date)
|
||||
TypeError: fetch_OQaD() missing 3 required positional arguments: 'table_type', 'empids', and 'run_date'
|
||||
2026-06-17 12:38:43 | INFO | ================================================================================
|
||||
2026-06-17 12:38:43 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 12:38:43 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 12:38:43 | INFO | Connecting to databases...
|
||||
2026-06-17 12:38:45 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001C35AC2F750>
|
||||
2026-06-17 12:38:46 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001C35BDF87D0>
|
||||
2026-06-17 12:38:46 | INFO | Database connections established
|
||||
2026-06-17 12:38:46 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 12:38:47 | INFO | Found 863 MIDs
|
||||
2026-06-17 12:38:47 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 12:38:58 | INFO | Fetched 291,112 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 12:38:58 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 12:38:58 | INFO | ================================================================================
|
||||
2026-06-17 12:38:58 | INFO | Processing Table: OQaD
|
||||
2026-06-17 12:38:58 | INFO | Exists: True
|
||||
2026-06-17 12:38:58 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
||||
2026-06-17 12:38:58 | INFO | Fetching data for 893 EMPIDs
|
||||
2026-06-17 12:38:58 | INFO | Fetching OQaD data for run_date=2026-06-16
|
||||
2026-06-17 12:39:05 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
|
||||
2026-06-17 12:39:05 | INFO | ================================================================================
|
||||
2026-06-17 12:39:05 | INFO | Pipeline Completed Successfully
|
||||
2026-06-17 12:39:05 | INFO | ================================================================================
|
||||
2026-06-17 12:39:30 | INFO | ================================================================================
|
||||
2026-06-17 12:39:30 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 12:39:30 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 12:39:30 | INFO | Connecting to databases...
|
||||
2026-06-17 12:39:31 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000025D1633F750>
|
||||
2026-06-17 12:39:32 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000025D175887D0>
|
||||
2026-06-17 12:39:33 | INFO | Database connections established
|
||||
2026-06-17 12:39:33 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 12:39:33 | INFO | Found 863 MIDs
|
||||
2026-06-17 12:39:33 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 12:39:43 | INFO | Fetched 291,112 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 12:39:43 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 12:39:43 | INFO | ================================================================================
|
||||
2026-06-17 12:39:43 | INFO | Processing Table: OQaD
|
||||
2026-06-17 12:39:43 | INFO | Exists: True
|
||||
2026-06-17 12:39:43 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
||||
2026-06-17 12:39:43 | INFO | Fetching data for 893 EMPIDs
|
||||
2026-06-17 12:39:43 | INFO | Fetching OQaD data for run_date=2026-06-16
|
||||
2026-06-17 12:39:46 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
|
||||
2026-06-17 12:39:46 | INFO | ================================================================================
|
||||
2026-06-17 12:39:46 | INFO | Pipeline Completed Successfully
|
||||
2026-06-17 12:39:46 | INFO | ================================================================================
|
||||
2026-06-17 12:44:50 | INFO | ================================================================================
|
||||
2026-06-17 12:44:50 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 12:44:50 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 12:44:50 | INFO | Connecting to databases...
|
||||
2026-06-17 12:44:51 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E3647178B0>
|
||||
2026-06-17 12:44:52 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E365BB8910>
|
||||
2026-06-17 12:44:53 | INFO | Database connections established
|
||||
2026-06-17 12:44:53 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 12:44:53 | INFO | Found 863 MIDs
|
||||
2026-06-17 12:44:53 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 12:45:07 | INFO | Fetched 291,115 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 12:45:08 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 12:45:08 | INFO | ================================================================================
|
||||
2026-06-17 12:45:08 | INFO | Processing Table: OQaD
|
||||
2026-06-17 12:45:08 | INFO | Exists: True
|
||||
2026-06-17 12:45:08 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
||||
2026-06-17 12:45:08 | INFO | Fetching data for 893 EMPIDs
|
||||
2026-06-17 12:45:08 | INFO | Fetching OQaD data for run_date=2026-06-16
|
||||
2026-06-17 12:45:10 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
|
||||
2026-06-17 12:45:10 | INFO | Fetched 4998 rows
|
||||
2026-06-17 12:45:10 | INFO | Deleting from OQaD
|
||||
2026-06-17 12:45:11 | INFO | OQaD: inserted 4,998 rows into ClickHouse
|
||||
2026-06-17 12:45:11 | INFO | OQaD loaded successfully (4998 rows)
|
||||
2026-06-17 12:45:11 | INFO | ================================================================================
|
||||
2026-06-17 12:45:11 | INFO | Pipeline Completed Successfully
|
||||
2026-06-17 12:45:11 | INFO | ================================================================================
|
||||
2026-06-17 13:23:53 | INFO | ================================================================================
|
||||
2026-06-17 13:23:53 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 13:23:53 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 13:23:53 | INFO | Connecting to databases...
|
||||
2026-06-17 13:23:54 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000281306878B0>
|
||||
2026-06-17 13:23:56 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000028131AD8910>
|
||||
2026-06-17 13:23:56 | INFO | Database connections established
|
||||
2026-06-17 13:23:56 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 13:23:57 | INFO | Found 863 MIDs
|
||||
2026-06-17 13:23:58 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 13:24:15 | INFO | Fetched 291,128 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 13:24:15 | INFO | Fetched 4,819 matched empids fetched for OQAD
|
||||
2026-06-17 13:24:15 | INFO | ================================================================================
|
||||
2026-06-17 13:24:15 | INFO | Processing Table: OQaD
|
||||
2026-06-17 13:24:15 | INFO | Exists: True
|
||||
2026-06-17 13:24:15 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
||||
2026-06-17 13:24:15 | INFO | Fetching data for 4,819 EMPIDs
|
||||
2026-06-17 13:24:15 | INFO | Fetching OQaD data for run_date=2026-06-16
|
||||
2026-06-17 13:24:22 | INFO | fn name is fetch_OQad ------Fetched 1072 rows
|
||||
2026-06-17 13:24:22 | INFO | Fetched 1072 rows
|
||||
2026-06-17 13:24:22 | INFO | Deleting from OQaD
|
||||
2026-06-17 13:24:23 | INFO | OQaD: inserted 1,072 rows into ClickHouse
|
||||
2026-06-17 13:24:23 | INFO | OQaD loaded successfully (1072 rows)
|
||||
2026-06-17 13:24:23 | INFO | ================================================================================
|
||||
2026-06-17 13:24:23 | INFO | Processing Table: Survey
|
||||
2026-06-17 13:24:23 | INFO | Fetching data for 863 MIDs
|
||||
2026-06-17 13:24:23 | INFO | Fetched 142 rows from SQL Server
|
||||
2026-06-17 13:24:23 | INFO | Fetched 142 rows
|
||||
2026-06-17 13:24:23 | INFO | Deleting from Survey
|
||||
2026-06-17 13:24:24 | ERROR | Failed processing table Survey
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 299, in main
|
||||
load_to_clickhouse(
|
||||
~~~~~~~~~~~~~~~~~~^
|
||||
client=client,
|
||||
^^^^^^^^^^^^^^
|
||||
table_name=table_name,
|
||||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
df=df,
|
||||
^^^^^^
|
||||
)
|
||||
^
|
||||
File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse
|
||||
client.insert_arrow(
|
||||
~~~~~~~~~~~~~~~~~~~^
|
||||
table=table_name,
|
||||
^^^^^^^^^^^^^^^^^
|
||||
arrow_table=arrow_table,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow
|
||||
return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings)
|
||||
~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert
|
||||
response = self._raw_request(insert_block, params, headers, server_wait=False)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request
|
||||
self._error_handler(response)
|
||||
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler
|
||||
raise err_type(err_str, code=code, name=name) from None
|
||||
clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column Project_Id in table kelloggs_1.Survey (d8c44908-2ed7-4d29-bd10-8e70fa84dfba). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123)
|
||||
2026-06-17 13:27:22 | INFO | ================================================================================
|
||||
2026-06-17 13:27:22 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 13:27:22 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 13:27:22 | INFO | Connecting to databases...
|
||||
2026-06-17 13:27:23 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000026D13D878B0>
|
||||
2026-06-17 13:27:25 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000026D151D8910>
|
||||
2026-06-17 13:27:25 | INFO | Database connections established
|
||||
2026-06-17 13:27:25 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 13:27:26 | INFO | Found 863 MIDs
|
||||
2026-06-17 13:27:26 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 13:27:37 | INFO | Fetched 291,128 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 13:27:38 | INFO | Fetched 4,814 matched empids fetched for OQAD
|
||||
2026-06-17 13:27:38 | INFO | ================================================================================
|
||||
2026-06-17 13:27:38 | INFO | Processing Table: OQaD
|
||||
2026-06-17 13:27:38 | INFO | Exists: True
|
||||
2026-06-17 13:27:38 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
||||
2026-06-17 13:27:38 | INFO | Fetching data for 4,814 EMPIDs
|
||||
2026-06-17 13:27:38 | INFO | Fetching OQaD data for run_date=2026-06-16
|
||||
2026-06-17 13:27:44 | INFO | fn name is fetch_OQad ------Fetched 1072 rows
|
||||
2026-06-17 13:27:44 | INFO | Fetched 1072 rows
|
||||
2026-06-17 13:27:44 | INFO | Deleting from OQaD
|
||||
2026-06-17 13:27:45 | INFO | OQaD: inserted 1,072 rows into ClickHouse
|
||||
2026-06-17 13:27:45 | INFO | OQaD loaded successfully (1072 rows)
|
||||
2026-06-17 13:27:45 | INFO | ================================================================================
|
||||
2026-06-17 13:27:45 | INFO | Processing Table: Survey
|
||||
2026-06-17 13:27:45 | INFO | Fetching data for 863 MIDs
|
||||
2026-06-17 13:27:45 | INFO | Fetched 142 rows from SQL Server
|
||||
2026-06-17 13:27:45 | INFO | Fetched 142 rows
|
||||
2026-06-17 13:27:45 | INFO | Creating table Survey
|
||||
2026-06-17 13:27:45 | INFO | Table ready: Survey
|
||||
2026-06-17 13:27:45 | INFO | Survey: inserted 142 rows into ClickHouse
|
||||
2026-06-17 13:27:45 | INFO | Survey loaded successfully (142 rows)
|
||||
2026-06-17 13:27:45 | INFO | ================================================================================
|
||||
2026-06-17 13:27:45 | INFO | Pipeline Completed Successfully
|
||||
2026-06-17 13:27:45 | INFO | ================================================================================
|
||||
2026-06-17 15:05:42 | INFO | ================================================================================
|
||||
2026-06-17 15:05:42 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 15:05:42 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 15:05:42 | INFO | Connecting to databases...
|
||||
2026-06-17 15:05:43 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002009FB778B0>
|
||||
2026-06-17 15:05:45 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000200A0FB8910>
|
||||
2026-06-17 15:05:45 | INFO | Database connections established
|
||||
2026-06-17 15:05:45 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 15:05:45 | INFO | Found 863 MIDs
|
||||
2026-06-17 15:05:46 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 15:05:57 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 15:05:57 | INFO | Fetched 4,814 matched empids fetched for OQAD
|
||||
2026-06-17 15:05:57 | INFO | ================================================================================
|
||||
2026-06-17 15:05:57 | INFO | Processing Table: additional_visibility
|
||||
2026-06-17 15:05:57 | INFO | Fetching data for 863 MIDs
|
||||
2026-06-17 15:05:58 | ERROR | Failed processing table additional_visibility
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 228, in main
|
||||
df = fetch_data(
|
||||
engine=sql_engine,
|
||||
...<3 lines>...
|
||||
run_date=run_date,
|
||||
)
|
||||
File "D:\data_move\src\fact.py", line 34, in fetch_data
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 287, in read_database
|
||||
).to_polars(
|
||||
~~~~~~~~~^
|
||||
batch_size=batch_size,
|
||||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
...<2 lines>...
|
||||
infer_schema_length=infer_schema_length,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 588, in to_polars
|
||||
frame = frame_init(
|
||||
batch_size=batch_size,
|
||||
...<2 lines>...
|
||||
infer_schema_length=infer_schema_length,
|
||||
)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 285, in _from_rows
|
||||
(d[0], d[1:]) for d in self.result.cursor.description
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
AttributeError: 'NoneType' object has no attribute 'description'
|
||||
2026-06-17 15:07:15 | INFO | ================================================================================
|
||||
2026-06-17 15:07:15 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 15:07:15 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 15:07:15 | INFO | Connecting to databases...
|
||||
2026-06-17 15:07:16 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001BD510978B0>
|
||||
2026-06-17 15:07:17 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001BD52518910>
|
||||
2026-06-17 15:07:18 | INFO | Database connections established
|
||||
2026-06-17 15:07:18 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 15:07:18 | INFO | Found 863 MIDs
|
||||
2026-06-17 15:07:18 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 15:07:31 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 15:07:31 | INFO | Fetched 4,814 matched empids fetched for OQAD
|
||||
2026-06-17 15:07:31 | INFO | ================================================================================
|
||||
2026-06-17 15:07:31 | INFO | Processing Table: additional_visibility
|
||||
2026-06-17 15:07:31 | INFO | Fetching data for 863 MIDs
|
||||
2026-06-17 15:07:32 | INFO | Fetched 1,773 rows from SQL Server
|
||||
2026-06-17 15:07:32 | INFO | Fetched 1773 rows
|
||||
2026-06-17 15:07:32 | INFO | Deleting from additional_visibility
|
||||
2026-06-17 15:07:32 | ERROR | Failed processing table additional_visibility
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 299, in main
|
||||
load_to_clickhouse(
|
||||
~~~~~~~~~~~~~~~~~~^
|
||||
client=client,
|
||||
^^^^^^^^^^^^^^
|
||||
table_name=table_name,
|
||||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
df=df,
|
||||
^^^^^^
|
||||
)
|
||||
^
|
||||
File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse
|
||||
client.insert_arrow(
|
||||
~~~~~~~~~~~~~~~~~~~^
|
||||
table=table_name,
|
||||
^^^^^^^^^^^^^^^^^
|
||||
arrow_table=arrow_table,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow
|
||||
return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings)
|
||||
~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert
|
||||
response = self._raw_request(insert_block, params, headers, server_wait=False)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request
|
||||
self._error_handler(response)
|
||||
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler
|
||||
raise err_type(err_str, code=code, name=name) from None
|
||||
clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column project_id in table kelloggs_1.additional_visibility (9e03e625-028a-47db-8f60-e8ab93e907ea). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123)
|
||||
2026-06-17 15:08:55 | INFO | ================================================================================
|
||||
2026-06-17 15:08:55 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 15:08:55 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 15:08:55 | INFO | Connecting to databases...
|
||||
2026-06-17 15:08:56 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001ED5A4C78B0>
|
||||
2026-06-17 15:08:58 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001ED5B948910>
|
||||
2026-06-17 15:08:58 | INFO | Database connections established
|
||||
2026-06-17 15:08:58 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 15:08:59 | INFO | Found 863 MIDs
|
||||
2026-06-17 15:08:59 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 15:09:12 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 15:09:12 | INFO | Fetched 4,814 matched empids fetched for OQAD
|
||||
2026-06-17 15:09:12 | INFO | ================================================================================
|
||||
2026-06-17 15:09:12 | INFO | Processing Table: additional_visibility
|
||||
2026-06-17 15:09:12 | INFO | Fetching data for 863 MIDs
|
||||
2026-06-17 15:09:14 | INFO | Fetched 1,773 rows from SQL Server
|
||||
2026-06-17 15:09:14 | INFO | Fetched 1773 rows
|
||||
2026-06-17 15:09:14 | INFO | Creating table additional_visibility
|
||||
2026-06-17 15:09:14 | INFO | Table ready: additional_visibility
|
||||
2026-06-17 15:09:15 | INFO | additional_visibility: inserted 1,773 rows into ClickHouse
|
||||
2026-06-17 15:09:15 | INFO | additional_visibility loaded successfully (1773 rows)
|
||||
2026-06-17 15:09:15 | INFO | ================================================================================
|
||||
2026-06-17 15:09:15 | INFO | Pipeline Completed Successfully
|
||||
2026-06-17 15:09:15 | INFO | ================================================================================
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,307 @@
|
||||
2026-06-22 10:05:56 | INFO | ================================================================================
|
||||
2026-06-22 10:05:56 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-22 10:05:56 | INFO | Pipeline Run Date: 2026-06-21
|
||||
2026-06-22 10:05:56 | INFO | Connecting to databases...
|
||||
2026-06-22 10:05:57 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001F93F950730>
|
||||
2026-06-22 10:06:00 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001F9409FC7D0>
|
||||
2026-06-22 10:06:03 | INFO | Database connections established
|
||||
2026-06-22 10:06:03 | INFO | Collecting MIDs for: 2026-06-21
|
||||
2026-06-22 10:06:05 | INFO | Found 569 MIDs
|
||||
2026-06-22 10:06:07 | INFO | ================================================================================
|
||||
2026-06-22 10:06:07 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||
2026-06-22 10:06:07 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
|
||||
2026-06-22 10:06:07 | INFO | Fetching data from sql server for Master table......
|
||||
2026-06-22 10:06:07 | INFO | Fetching in progress ....
|
||||
2026-06-22 10:06:31 | ERROR | Failed processing table Master_VisibilityReason
|
||||
Traceback (most recent call last):
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context
|
||||
self.dialect.do_execute(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
cursor, str_statement, effective_parameters, context
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute
|
||||
cursor.execute(statement, parameters)
|
||||
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
pyodbc.OperationalError: ('08S01', '[08S01] [Microsoft][ODBC Driver 18 for SQL Server]TCP Provider: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond.\r\n (10060) (SQLExecDirectW); [08S01] [Microsoft][ODBC Driver 18 for SQL Server]Communication link failure (10060)')
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 161, in main
|
||||
df = fetch_data(sql_engine ,table_name,table_type)
|
||||
File "D:\data_move\src\dim.py", line 45, in fetch_data
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database
|
||||
return cx.execute(
|
||||
~~~~~~~~~~^
|
||||
query=query,
|
||||
^^^^^^^^^^^^
|
||||
options=execute_options,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
).to_polars(
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute
|
||||
result = cursor_execute(query, **options)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute
|
||||
return meth(
|
||||
self,
|
||||
distilled_parameters,
|
||||
execution_options or NO_OPTIONS,
|
||||
)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection
|
||||
return connection._execute_clauseelement(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
self, distilled_params, execution_options
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement
|
||||
ret = self._execute_context(
|
||||
dialect,
|
||||
...<8 lines>...
|
||||
cache_hit=cache_hit,
|
||||
)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context
|
||||
return self._exec_single_context(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
dialect, context, statement, parameters
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context
|
||||
self._handle_dbapi_exception(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
e, str_statement, effective_parameters, cursor, context
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception
|
||||
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context
|
||||
self.dialect.do_execute(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
cursor, str_statement, effective_parameters, context
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute
|
||||
cursor.execute(statement, parameters)
|
||||
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
sqlalchemy.exc.OperationalError: (pyodbc.OperationalError) ('08S01', '[08S01] [Microsoft][ODBC Driver 18 for SQL Server]TCP Provider: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond.\r\n (10060) (SQLExecDirectW); [08S01] [Microsoft][ODBC Driver 18 for SQL Server]Communication link failure (10060)')
|
||||
[SQL: with Master_VisibilityReason (ProjectId,MenuId,ReasonId,Reason,CreateDate,Createby)
|
||||
as (
|
||||
|
||||
select DISTINCT '40148' as ProjectId,MenuId,VisibilityReasonId,VisibilityReason,getdate(),'SP-Pius'
|
||||
FROM OneApp_KelloggsMT.dbo.Master_VisibilityReason
|
||||
)
|
||||
select * from Master_VisibilityReason]
|
||||
(Background on this error at: https://sqlalche.me/e/20/e3q8)
|
||||
2026-06-22 10:10:52 | INFO | ================================================================================
|
||||
2026-06-22 10:10:52 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-22 10:10:52 | INFO | Pipeline Run Date: 2026-06-21
|
||||
2026-06-22 10:10:53 | INFO | Connecting to databases...
|
||||
2026-06-22 10:10:53 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001500A41C730>
|
||||
2026-06-22 10:10:55 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001500B56C7D0>
|
||||
2026-06-22 10:10:56 | INFO | Database connections established
|
||||
2026-06-22 10:10:56 | INFO | Collecting MIDs for: 2026-06-21
|
||||
2026-06-22 10:10:56 | INFO | Found 569 MIDs
|
||||
2026-06-22 10:10:57 | INFO | ================================================================================
|
||||
2026-06-22 10:10:57 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||
2026-06-22 10:10:57 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
|
||||
2026-06-22 10:10:57 | INFO | Fetching data from sql server for Master table......
|
||||
2026-06-22 10:10:57 | INFO | Fetching in progress ....
|
||||
2026-06-22 10:10:57 | INFO | Fetched 17 rows from SQL Server
|
||||
2026-06-22 10:10:57 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!!
|
||||
2026-06-22 10:10:57 | INFO | Fetched 17 rows
|
||||
2026-06-22 10:10:57 | INFO | Creating table Master_VisibilityReason
|
||||
2026-06-22 10:10:57 | INFO | Table ready: Master_VisibilityReason
|
||||
2026-06-22 10:10:57 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||
2026-06-22 10:10:58 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse
|
||||
2026-06-22 10:10:58 | INFO | Master_VisibilityReason loaded successfully (17 rows)
|
||||
2026-06-22 10:10:58 | INFO | ================================================================================
|
||||
2026-06-22 10:10:58 | INFO | Pipeline Completed Successfully
|
||||
2026-06-22 10:10:58 | INFO | ================================================================================
|
||||
2026-06-22 10:11:04 | INFO | ================================================================================
|
||||
2026-06-22 10:11:04 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-22 10:11:04 | INFO | Pipeline Run Date: 2026-06-21
|
||||
2026-06-22 10:11:04 | INFO | Connecting to databases...
|
||||
2026-06-22 10:11:05 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002DFCC410730>
|
||||
2026-06-22 10:11:06 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002DFCD4CC7D0>
|
||||
2026-06-22 10:11:07 | INFO | Database connections established
|
||||
2026-06-22 10:11:07 | INFO | Collecting MIDs for: 2026-06-21
|
||||
2026-06-22 10:11:07 | INFO | Found 569 MIDs
|
||||
2026-06-22 10:11:08 | INFO | ================================================================================
|
||||
2026-06-22 10:11:08 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||
2026-06-22 10:11:08 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
|
||||
2026-06-22 10:11:08 | INFO | Fetching data from sql server for Master table......
|
||||
2026-06-22 10:11:08 | INFO | Fetching in progress ....
|
||||
2026-06-22 10:11:08 | INFO | Fetched 17 rows from SQL Server
|
||||
2026-06-22 10:11:08 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!!
|
||||
2026-06-22 10:11:08 | INFO | Fetched 17 rows
|
||||
2026-06-22 10:11:08 | INFO | Truncated table Master_VisibilityReason
|
||||
2026-06-22 10:11:08 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||
2026-06-22 10:11:08 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse
|
||||
2026-06-22 10:11:08 | INFO | Master_VisibilityReason loaded successfully (17 rows)
|
||||
2026-06-22 10:11:08 | INFO | ================================================================================
|
||||
2026-06-22 10:11:08 | INFO | Pipeline Completed Successfully
|
||||
2026-06-22 10:11:08 | INFO | ================================================================================
|
||||
2026-06-22 11:07:54 | INFO | ================================================================================
|
||||
2026-06-22 11:07:54 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-22 11:07:54 | INFO | Pipeline Run Date: 2026-06-21
|
||||
2026-06-22 11:07:54 | INFO | Connecting to databases...
|
||||
2026-06-22 11:07:56 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001689BE04730>
|
||||
2026-06-22 11:08:00 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001689CEBC7D0>
|
||||
2026-06-22 11:08:01 | INFO | Database connections established
|
||||
2026-06-22 11:08:01 | INFO | Collecting MIDs for: 2026-06-21
|
||||
2026-06-22 11:08:02 | INFO | Found 569 MIDs
|
||||
2026-06-22 11:08:42 | INFO | ================================================================================
|
||||
2026-06-22 11:08:42 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-22 11:08:42 | INFO | Pipeline Run Date: 2026-06-21
|
||||
2026-06-22 11:08:42 | INFO | Connecting to databases...
|
||||
2026-06-22 11:08:44 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000015B77AE4730>
|
||||
2026-06-22 11:08:46 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000015B78B8C7D0>
|
||||
2026-06-22 11:08:47 | INFO | Database connections established
|
||||
2026-06-22 11:08:47 | INFO | Collecting MIDs for: 2026-06-21
|
||||
2026-06-22 11:08:48 | INFO | Found 569 MIDs
|
||||
2026-06-22 11:08:48 | INFO | ================================================================================
|
||||
2026-06-22 11:08:48 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||
2026-06-22 11:08:48 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
|
||||
2026-06-22 11:08:48 | INFO | Fetching data from sql server for Master table......
|
||||
2026-06-22 11:08:48 | INFO | Fetching in progress ....
|
||||
2026-06-22 11:08:48 | INFO | Fetched 17 rows from SQL Server
|
||||
2026-06-22 11:08:48 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!!
|
||||
2026-06-22 11:08:48 | INFO | Fetched 17 rows
|
||||
2026-06-22 11:08:48 | INFO | Truncated table Master_VisibilityReason
|
||||
2026-06-22 11:08:48 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||
2026-06-22 11:08:49 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse
|
||||
2026-06-22 11:08:49 | INFO | Master_VisibilityReason loaded successfully (17 rows)
|
||||
2026-06-22 11:08:49 | INFO | ================================================================================
|
||||
2026-06-22 11:08:49 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||
2026-06-22 11:08:49 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition ..............
|
||||
2026-06-22 11:08:49 | INFO | Fetching data from sql server for Master table......
|
||||
2026-06-22 11:08:49 | INFO | Fetching in progress ....
|
||||
2026-06-22 11:08:49 | INFO | Fetched 861 rows from SQL Server
|
||||
2026-06-22 11:08:49 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!!
|
||||
2026-06-22 11:08:49 | INFO | Fetched 861 rows
|
||||
2026-06-22 11:08:49 | INFO | Creating table Master_VisibilityDefinition
|
||||
2026-06-22 11:08:49 | INFO | Table ready: Master_VisibilityDefinition
|
||||
2026-06-22 11:08:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||
2026-06-22 11:08:49 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse
|
||||
2026-06-22 11:08:49 | INFO | Master_VisibilityDefinition loaded successfully (861 rows)
|
||||
2026-06-22 11:08:49 | INFO | ================================================================================
|
||||
2026-06-22 11:08:49 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||
2026-06-22 11:08:49 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer ..............
|
||||
2026-06-22 11:08:49 | INFO | Fetching data from sql server for Master table......
|
||||
2026-06-22 11:08:49 | INFO | Fetching in progress ....
|
||||
2026-06-22 11:08:49 | INFO | Fetched 33 rows from SQL Server
|
||||
2026-06-22 11:08:49 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!!
|
||||
2026-06-22 11:08:49 | INFO | Fetched 33 rows
|
||||
2026-06-22 11:08:50 | INFO | Creating table Master_Salesterritorylayer
|
||||
2026-06-22 11:08:50 | INFO | Table ready: Master_Salesterritorylayer
|
||||
2026-06-22 11:08:50 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||
2026-06-22 11:08:50 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse
|
||||
2026-06-22 11:08:50 | INFO | Master_Salesterritorylayer loaded successfully (33 rows)
|
||||
2026-06-22 11:08:50 | INFO | ================================================================================
|
||||
2026-06-22 11:08:50 | INFO | Pipeline Completed Successfully
|
||||
2026-06-22 11:08:50 | INFO | ================================================================================
|
||||
2026-06-22 11:08:56 | INFO | ================================================================================
|
||||
2026-06-22 11:08:56 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-22 11:08:56 | INFO | Pipeline Run Date: 2026-06-21
|
||||
2026-06-22 11:08:56 | INFO | Connecting to databases...
|
||||
2026-06-22 11:08:57 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000214E3374730>
|
||||
2026-06-22 11:08:59 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000214E44CC7D0>
|
||||
2026-06-22 11:09:00 | INFO | Database connections established
|
||||
2026-06-22 11:09:00 | INFO | Collecting MIDs for: 2026-06-21
|
||||
2026-06-22 11:09:00 | INFO | Found 569 MIDs
|
||||
2026-06-22 11:09:00 | INFO | ================================================================================
|
||||
2026-06-22 11:09:00 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||
2026-06-22 11:09:00 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
|
||||
2026-06-22 11:09:00 | INFO | Fetching data from sql server for Master table......
|
||||
2026-06-22 11:09:00 | INFO | Fetching in progress ....
|
||||
2026-06-22 11:09:01 | INFO | Fetched 17 rows from SQL Server
|
||||
2026-06-22 11:09:01 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!!
|
||||
2026-06-22 11:09:01 | INFO | Fetched 17 rows
|
||||
2026-06-22 11:09:01 | INFO | Truncated table Master_VisibilityReason
|
||||
2026-06-22 11:09:01 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||
2026-06-22 11:09:01 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse
|
||||
2026-06-22 11:09:01 | INFO | Master_VisibilityReason loaded successfully (17 rows)
|
||||
2026-06-22 11:09:01 | INFO | ================================================================================
|
||||
2026-06-22 11:09:01 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||
2026-06-22 11:09:01 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition ..............
|
||||
2026-06-22 11:09:01 | INFO | Fetching data from sql server for Master table......
|
||||
2026-06-22 11:09:01 | INFO | Fetching in progress ....
|
||||
2026-06-22 11:09:02 | INFO | Fetched 861 rows from SQL Server
|
||||
2026-06-22 11:09:02 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!!
|
||||
2026-06-22 11:09:02 | INFO | Fetched 861 rows
|
||||
2026-06-22 11:09:02 | INFO | Truncated table Master_VisibilityDefinition
|
||||
2026-06-22 11:09:02 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||
2026-06-22 11:09:02 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse
|
||||
2026-06-22 11:09:02 | INFO | Master_VisibilityDefinition loaded successfully (861 rows)
|
||||
2026-06-22 11:09:02 | INFO | ================================================================================
|
||||
2026-06-22 11:09:02 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||
2026-06-22 11:09:02 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer ..............
|
||||
2026-06-22 11:09:02 | INFO | Fetching data from sql server for Master table......
|
||||
2026-06-22 11:09:02 | INFO | Fetching in progress ....
|
||||
2026-06-22 11:09:02 | INFO | Fetched 33 rows from SQL Server
|
||||
2026-06-22 11:09:02 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!!
|
||||
2026-06-22 11:09:02 | INFO | Fetched 33 rows
|
||||
2026-06-22 11:09:02 | INFO | Truncated table Master_Salesterritorylayer
|
||||
2026-06-22 11:09:02 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||
2026-06-22 11:09:03 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse
|
||||
2026-06-22 11:09:03 | INFO | Master_Salesterritorylayer loaded successfully (33 rows)
|
||||
2026-06-22 11:09:03 | INFO | ================================================================================
|
||||
2026-06-22 11:09:03 | INFO | Pipeline Completed Successfully
|
||||
2026-06-22 11:09:03 | INFO | ================================================================================
|
||||
2026-06-22 12:04:40 | INFO | ================================================================================
|
||||
2026-06-22 12:04:40 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-22 12:04:40 | INFO | Pipeline Run Date: 2026-06-21
|
||||
2026-06-22 12:04:40 | INFO | Connecting to databases...
|
||||
2026-06-22 12:04:43 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000023A01CA4730>
|
||||
2026-06-22 12:04:44 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000023A02D1C7D0>
|
||||
2026-06-22 12:04:45 | INFO | Database connections established
|
||||
2026-06-22 12:04:45 | INFO | Collecting MIDs for: 2026-06-21
|
||||
2026-06-22 12:04:46 | INFO | Found 569 MIDs
|
||||
2026-06-22 12:04:48 | INFO | ================================================================================
|
||||
2026-06-22 12:04:48 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||
2026-06-22 12:04:48 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
|
||||
2026-06-22 12:04:48 | INFO | Fetching data from sql server for Master table......
|
||||
2026-06-22 12:04:48 | INFO | Fetching in progress ....
|
||||
2026-06-22 12:04:48 | INFO | Fetched 17 rows from SQL Server
|
||||
2026-06-22 12:04:48 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!!
|
||||
2026-06-22 12:04:48 | INFO | Fetched 17 rows
|
||||
2026-06-22 12:04:48 | INFO | Truncated table Master_VisibilityReason
|
||||
2026-06-22 12:04:48 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||
2026-06-22 12:04:48 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse
|
||||
2026-06-22 12:04:48 | INFO | Master_VisibilityReason loaded successfully (17 rows)
|
||||
2026-06-22 12:04:48 | INFO | ================================================================================
|
||||
2026-06-22 12:04:48 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||
2026-06-22 12:04:48 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition ..............
|
||||
2026-06-22 12:04:48 | INFO | Fetching data from sql server for Master table......
|
||||
2026-06-22 12:04:48 | INFO | Fetching in progress ....
|
||||
2026-06-22 12:04:49 | INFO | Fetched 861 rows from SQL Server
|
||||
2026-06-22 12:04:49 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!!
|
||||
2026-06-22 12:04:49 | INFO | Fetched 861 rows
|
||||
2026-06-22 12:04:49 | INFO | Truncated table Master_VisibilityDefinition
|
||||
2026-06-22 12:04:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||
2026-06-22 12:04:49 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse
|
||||
2026-06-22 12:04:49 | INFO | Master_VisibilityDefinition loaded successfully (861 rows)
|
||||
2026-06-22 12:04:49 | INFO | ================================================================================
|
||||
2026-06-22 12:04:49 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||
2026-06-22 12:04:49 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer ..............
|
||||
2026-06-22 12:04:49 | INFO | Fetching data from sql server for Master table......
|
||||
2026-06-22 12:04:49 | INFO | Fetching in progress ....
|
||||
2026-06-22 12:04:49 | INFO | Fetched 33 rows from SQL Server
|
||||
2026-06-22 12:04:49 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!!
|
||||
2026-06-22 12:04:49 | INFO | Fetched 33 rows
|
||||
2026-06-22 12:04:49 | INFO | Truncated table Master_Salesterritorylayer
|
||||
2026-06-22 12:04:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||
2026-06-22 12:04:50 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse
|
||||
2026-06-22 12:04:50 | INFO | Master_Salesterritorylayer loaded successfully (33 rows)
|
||||
2026-06-22 12:04:50 | INFO | ================================================================================
|
||||
2026-06-22 12:04:50 | INFO | Pipeline Completed Successfully
|
||||
2026-06-22 12:04:50 | INFO | ================================================================================
|
||||
@@ -43,9 +43,9 @@ from mids import (
|
||||
MID_TABLE_COV1,
|
||||
)
|
||||
|
||||
from masters.dimensions import *
|
||||
from masters.bridge import *
|
||||
from kpi.facts import *
|
||||
from src.bridge import *
|
||||
from src.fact import *
|
||||
from src.dim import *
|
||||
|
||||
|
||||
# ==========================================================
|
||||
@@ -64,53 +64,6 @@ def table_exists(
|
||||
)
|
||||
|
||||
|
||||
def get_dataframe(
|
||||
fn_name: str,
|
||||
fetch_by: str,
|
||||
sql_engine,
|
||||
mids,
|
||||
run_date,
|
||||
):
|
||||
|
||||
fn = globals()[fn_name]
|
||||
|
||||
if fetch_by == "mids":
|
||||
return fn(sql_engine, mids)
|
||||
|
||||
if fetch_by == "run_date":
|
||||
return fn(sql_engine, run_date)
|
||||
|
||||
return fn(sql_engine)
|
||||
|
||||
|
||||
|
||||
|
||||
def get_empids_clickhouse_OQAD(
|
||||
client,
|
||||
table_name: str = "OQaD",
|
||||
) -> pl.DataFrame:
|
||||
|
||||
if not table_exists(client, table_name):
|
||||
log.warning(f"Table '{table_name}' does not exist.")
|
||||
return pl.DataFrame(
|
||||
schema={
|
||||
"EmpId": pl.Int64,
|
||||
"VisitDate": pl.Date,
|
||||
}
|
||||
)
|
||||
|
||||
query = f"""
|
||||
SELECT DISTINCT
|
||||
employee_id AS EmpId,
|
||||
toDate(visit_date) AS VisitDate
|
||||
FROM {table_name}
|
||||
"""
|
||||
|
||||
# ClickHouse -> PyArrow -> Polars
|
||||
arrow_table = client.query_arrow(query)
|
||||
|
||||
return pl.from_arrow(arrow_table)
|
||||
|
||||
# ==========================================================
|
||||
# Main
|
||||
# ==========================================================
|
||||
@@ -166,20 +119,14 @@ def main():
|
||||
sql_engine,
|
||||
run_date,
|
||||
)
|
||||
qf=fetch_quiz_empids()
|
||||
db_df = get_empids_clickhouse_OQAD(client)
|
||||
|
||||
matched = qf.join(
|
||||
db_df,
|
||||
on=["EmpId", "VisitDate"],
|
||||
how="inner",
|
||||
)
|
||||
|
||||
# ------------------------------------------------------
|
||||
# Config
|
||||
# ------------------------------------------------------
|
||||
|
||||
with open(
|
||||
"t.yml",
|
||||
"y.yml",
|
||||
"r",
|
||||
) as file:
|
||||
|
||||
@@ -188,19 +135,14 @@ def main():
|
||||
# ------------------------------------------------------
|
||||
# Process Tables
|
||||
# ------------------------------------------------------
|
||||
|
||||
for table in config["tables"]:
|
||||
|
||||
table_name = table["name"]
|
||||
operation = table["operation"]
|
||||
fetch_by = table["fetch_by"]
|
||||
table_type=table["type"]
|
||||
|
||||
log.info("=" * 80)
|
||||
log.info(
|
||||
"Processing Table: %s",
|
||||
table_name,
|
||||
)
|
||||
log.info(f"Processing Table: {table_name} | Table type is -: {table_type} | Based on -{fetch_by} and operation is used -{operation} " )
|
||||
|
||||
try:
|
||||
|
||||
@@ -208,15 +150,17 @@ def main():
|
||||
# Fetch Data
|
||||
# ------------------------------------------
|
||||
|
||||
fn_name = f"fetch_{table_name}"
|
||||
|
||||
log.info(f"Fetching Data from sql server for table-: {table_name} ..............")
|
||||
fetch_list=["mids" ,"run_date", "reason_id"]
|
||||
if fetch_by in fetch_list :
|
||||
fn_name = f"fetch_{table_name}"
|
||||
fn = globals()[fn_name]
|
||||
df=fn(sql_engine, table_name , table_type, mids, run_date)
|
||||
else:
|
||||
df = fetch_data(sql_engine ,table_name,table_type)
|
||||
|
||||
df = get_dataframe(
|
||||
fn_name=fn_name,
|
||||
fetch_by=fetch_by,
|
||||
sql_engine=sql_engine,
|
||||
mids=mids,
|
||||
run_date=run_date,
|
||||
)
|
||||
log.info(f"Fetched total row -: {len(df)} from sql server for table-:{table_name} ...........!!!")
|
||||
|
||||
if df.is_empty():
|
||||
|
||||
@@ -267,6 +211,8 @@ def main():
|
||||
table_name,
|
||||
)
|
||||
|
||||
elif operation =="ONLY_INSERT" :
|
||||
continue
|
||||
else:
|
||||
|
||||
delete_existing_data(
|
||||
@@ -280,7 +226,7 @@ def main():
|
||||
# ------------------------------------------
|
||||
# Load Data
|
||||
# ------------------------------------------
|
||||
|
||||
log.info("_ _ _ _Inserting data into clickhouse db from sql server_ _ _ _")
|
||||
load_to_clickhouse(
|
||||
client=client,
|
||||
table_name=table_name,
|
||||
|
||||
@@ -72,51 +72,3 @@ def MID_TABLE_COV1(
|
||||
|
||||
|
||||
|
||||
|
||||
def fetch_quiz_empids(engine: Engine, run_date : date) -> pl.DataFrame:
|
||||
|
||||
sql_template = f"""
|
||||
WITH MID_TABLE_COV1 AS
|
||||
(
|
||||
SELECT EmpId, VisitDate
|
||||
FROM OneApp_KelloggsMT.dbo.T_OQAD
|
||||
WHERE CreateDate >= {run_date}
|
||||
AND CreateDate < DATEADD(DAY,1,'{run_date}')
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT EmpId, VisitDate
|
||||
FROM OneApp_KelloggsMT.dbo.T_OQAD
|
||||
WHERE UpdateDate >= {run_date}
|
||||
AND UpdateDate < DATEADD(DAY,1, '{run_date}')
|
||||
),
|
||||
|
||||
QUIZ AS
|
||||
(
|
||||
SELECT Distinct E.EmpId as empid
|
||||
, CONVERT(VARCHAR,DQ.VisitDate) AS VisitDate
|
||||
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ INNER JOIN
|
||||
OneApp_KelloggsMT.dbo.vw_Employee_Detail E ON DQ.EmpId = E.EmpId inner join
|
||||
OneApp_KelloggsMT.dbo.Master_OQAD_Question QU on DQ.QuestionId= qu.QuestionId inner join
|
||||
OneApp_KelloggsMT.dbo.Master_OQAD_Category qc on qu.QuestionCategoryId= qc.QuestionCategoryId
|
||||
where e.EmpName not like 'test%' and e.RightId in (6)
|
||||
and (E.ResignDate is null or E.ResignDate>=''+CONVERT(VARCHAR,'{run_date}')+'') AND E.EmpName NOT LIKE '%TEST%'
|
||||
AND DQ.EmpId IN (SELECT EmpId FROM MID_TABLE_COV1 A WHERE
|
||||
DQ.EmpId=A.EmpId AND CONVERT(date,VisitDate)=CONVERT(date,A.VisitDate) )
|
||||
) select * from quiz
|
||||
"""
|
||||
sql = sql_template.format(
|
||||
run_date=run_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
|
||||
log.info(f"Fetching quiz_empids data for EMPID and Visitid")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
|
||||
log.info(f"Fetched {len(df):,} total empid and visitdate fetched for OQAD from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
from pathlib import Path
|
||||
import polars as pl
|
||||
from sqlalchemy import Engine
|
||||
from datetime import date , timedelta
|
||||
from log import log
|
||||
|
||||
|
||||
|
||||
|
||||
from db_con.connection import (
|
||||
build_sql_server_engine,
|
||||
build_clickhouse_engine,
|
||||
get_clickhouse_client,
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def fetch_mapping_storevisibility(
|
||||
sql_engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
mids: list[int],
|
||||
run_date: date
|
||||
) -> pl.DataFrame:
|
||||
|
||||
run_date = run_date + timedelta(days=1)
|
||||
client= get_clickhouse_client()
|
||||
def table_exists(
|
||||
client,
|
||||
table_name: str,
|
||||
) -> bool:
|
||||
|
||||
return bool(
|
||||
client.command(
|
||||
f"EXISTS TABLE {table_name}"
|
||||
)
|
||||
)
|
||||
def get_reason_ids_mapping_storevisibility(
|
||||
client,
|
||||
run_date: date,
|
||||
table_name: str = "mapping_storevisibility",
|
||||
) -> list[int] :
|
||||
|
||||
if not table_exists(client, table_name):
|
||||
log.warning(f"Table '{table_name}' does not exist. During collecting store_ids")
|
||||
return [0]
|
||||
|
||||
|
||||
query = f"""
|
||||
SELECT DISTINCT StoreId
|
||||
FROM mapping_storevisibility
|
||||
WHERE toDate(Fromdate) <= toDate('{run_date + timedelta(days= 1)}')
|
||||
AND toDate(Todate) >= toDate('{run_date + timedelta(days= 1)}')
|
||||
AND project_Id = '40148'
|
||||
|
||||
"""
|
||||
|
||||
# ClickHouse -> PyArrow -> Polars
|
||||
arrow_table = client.query_arrow(query)
|
||||
|
||||
df= pl.from_arrow(arrow_table)
|
||||
list=df["reason_id"].to_list()
|
||||
return list
|
||||
|
||||
def fetch_data(
|
||||
engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
run_date: date,
|
||||
store_id: list[int]
|
||||
) -> pl.DataFrame:
|
||||
log.info(f"Fetching data from sql server for Master table......")
|
||||
|
||||
store_id_list = ",".join(str(sid) for sid in store_id)
|
||||
|
||||
sql_file = Path("src") / "sql" / f"bridge" / f"{table_name}.sql"
|
||||
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
sql_template = f.read()
|
||||
|
||||
sql = sql_template.format(
|
||||
store_id_list=store_id_list,
|
||||
run_date=run_date.strftime("%Y-%m-%d")
|
||||
|
||||
)
|
||||
|
||||
log.info(f"Fetching in progress .... ")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
store_id=get_reason_ids_mapping_storevisibility(client, run_date, "mapping_storevisibility")
|
||||
df=fetch_data(engine=sql_engine,
|
||||
table_name=table_name,
|
||||
table_type=table_type,
|
||||
run_date=run_date,
|
||||
store_id=store_id,
|
||||
)
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
from pathlib import Path
|
||||
import polars as pl
|
||||
from sqlalchemy import Engine
|
||||
from datetime import date , timedelta
|
||||
from log import log
|
||||
|
||||
|
||||
|
||||
|
||||
from db_con.connection import (
|
||||
build_sql_server_engine,
|
||||
build_clickhouse_engine,
|
||||
get_clickhouse_client,
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def fetch_data(
|
||||
engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str
|
||||
|
||||
) -> pl.DataFrame:
|
||||
|
||||
|
||||
|
||||
log.info(f"Fetching data from sql server for Master table......")
|
||||
|
||||
sql_file = Path("src") / "sql" / f"dim" / f"{table_name}.sql"
|
||||
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
sql_template = f.read()
|
||||
|
||||
sql = sql_template.format( )
|
||||
|
||||
log.info(f"Fetching in progress .... ")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def fetch_coverage_remarks(
|
||||
sql_engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
mids: list[int],
|
||||
run_date: date
|
||||
|
||||
) -> pl.DataFrame:
|
||||
|
||||
|
||||
client= get_clickhouse_client()
|
||||
def table_exists(
|
||||
client,
|
||||
table_name: str,
|
||||
) -> bool:
|
||||
|
||||
return bool(
|
||||
client.command(
|
||||
f"EXISTS TABLE {table_name}"
|
||||
)
|
||||
)
|
||||
def get_reason_ids_coverage_remarks(
|
||||
client,
|
||||
table_name: str = "coverage_remarks",
|
||||
) -> list[int] :
|
||||
|
||||
if not table_exists(client, table_name):
|
||||
log.warning(f"Table '{table_name}' does not exist. During collecting reason_ids")
|
||||
return [0]
|
||||
|
||||
|
||||
query = f"""
|
||||
SELECT DISTINCT
|
||||
reason_id
|
||||
FROM {table_name}
|
||||
"""
|
||||
|
||||
# ClickHouse -> PyArrow -> Polars
|
||||
arrow_table = client.query_arrow(query)
|
||||
|
||||
df= pl.from_arrow(arrow_table)
|
||||
list=df["reason_id"].to_list()
|
||||
return list
|
||||
|
||||
def fetch_data(
|
||||
engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
reason_ids: list[int]
|
||||
) -> pl.DataFrame:
|
||||
log.info(f"Fetching data from sql server for Master table......")
|
||||
|
||||
resaon_id_list = ",".join(str(rid) for rid in reason_ids)
|
||||
|
||||
sql_file = Path("src") / "sql" / f"dim" / f"{table_name}.sql"
|
||||
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
sql_template = f.read()
|
||||
|
||||
sql = sql_template.format(
|
||||
resaon_id_list=resaon_id_list
|
||||
|
||||
)
|
||||
|
||||
log.info(f"Fetching in progress .... ")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
reason_ids=get_reason_ids_coverage_remarks(client, "coverage_remarks")
|
||||
df=fetch_data(engine=sql_engine,
|
||||
table_name=table_name,
|
||||
table_type=table_type,
|
||||
reason_ids=reason_ids,
|
||||
)
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
+621
-22
@@ -1,25 +1,84 @@
|
||||
from pathlib import Path
|
||||
import polars as pl
|
||||
from sqlalchemy import Engine
|
||||
from datetime import date
|
||||
from datetime import date , timedelta
|
||||
from log import log
|
||||
|
||||
|
||||
def fetch_data(
|
||||
engine: Engine,
|
||||
|
||||
|
||||
from db_con.connection import (
|
||||
build_sql_server_engine,
|
||||
build_clickhouse_engine,
|
||||
get_clickhouse_client,
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# def fetch_data(
|
||||
# engine: Engine,
|
||||
# table_name: str,
|
||||
# table_type: str,
|
||||
# mids: list[int],
|
||||
# run_date: date
|
||||
# ) -> pl.DataFrame:
|
||||
|
||||
# if not mids:
|
||||
# log.warning("No MIDs — nothing to fetch.")
|
||||
# return pl.DataFrame()
|
||||
|
||||
# mid_list = ",".join(str(mid) for mid in mids)
|
||||
|
||||
# sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
|
||||
|
||||
# with open(sql_file, "r", encoding="utf-8") as f:
|
||||
# sql_template = f.read()
|
||||
|
||||
# sql = sql_template.format(
|
||||
# mid_list=mid_list,
|
||||
# run_date=run_date.strftime("%Y-%m-%d")
|
||||
# )
|
||||
|
||||
# log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||
|
||||
# df = pl.read_database(
|
||||
# query=sql,
|
||||
# connection=engine
|
||||
# )
|
||||
|
||||
# log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
# return df
|
||||
|
||||
|
||||
|
||||
def fetch_SOS_OneApp( engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
mids: list[int],
|
||||
run_date: date
|
||||
) -> pl.DataFrame:
|
||||
|
||||
|
||||
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
|
||||
log.info(f" Start Fetching data for these {len(mids):} MIDs ")
|
||||
mid_list = ",".join(str(mid) for mid in mids)
|
||||
|
||||
sql_file = Path("sql") / f"{table_type}" / f"{table_name}.sql"
|
||||
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
|
||||
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
|
||||
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
sql_template = f.read()
|
||||
@@ -28,14 +87,320 @@ def fetch_data(
|
||||
mid_list=mid_list,
|
||||
run_date=run_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
|
||||
log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||
log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def fetch_additional_visibility( engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
mids: list[int],
|
||||
run_date: date
|
||||
) -> pl.DataFrame:
|
||||
|
||||
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
|
||||
log.info(f" Start Fetching data for these {len(mids):} MIDs ")
|
||||
mid_list = ",".join(str(mid) for mid in mids)
|
||||
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
|
||||
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
|
||||
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
sql_template = f.read()
|
||||
|
||||
sql = sql_template.format(
|
||||
mid_list=mid_list,
|
||||
run_date=run_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def fetch_OQaD(
|
||||
sql_engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
mids: list[int],
|
||||
run_date: date
|
||||
) -> pl.DataFrame:
|
||||
|
||||
|
||||
client= get_clickhouse_client()
|
||||
def table_exists(
|
||||
client,
|
||||
table_name: str,
|
||||
) -> bool:
|
||||
|
||||
return bool(
|
||||
client.command(
|
||||
f"EXISTS TABLE {table_name}"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
|
||||
def fetch_quiz_empids(engine: Engine, run_date : date) -> pl.DataFrame:
|
||||
|
||||
sql_template = f"""
|
||||
WITH MID_TABLE_COV1 AS
|
||||
(
|
||||
SELECT EmpId, VisitDate
|
||||
FROM OneApp_KelloggsMT.dbo.T_OQAD
|
||||
WHERE CreateDate >= {run_date}
|
||||
AND CreateDate < DATEADD(DAY,1,'{run_date}')
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT EmpId, VisitDate
|
||||
FROM OneApp_KelloggsMT.dbo.T_OQAD
|
||||
WHERE UpdateDate >= {run_date}
|
||||
AND UpdateDate < DATEADD(DAY,1, '{run_date}')
|
||||
),
|
||||
|
||||
QUIZ AS
|
||||
(
|
||||
SELECT Distinct E.EmpId as empid
|
||||
, CONVERT(date,DQ.VisitDate) AS visitdate
|
||||
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ INNER JOIN
|
||||
OneApp_KelloggsMT.dbo.vw_Employee_Detail E ON DQ.EmpId = E.EmpId inner join
|
||||
OneApp_KelloggsMT.dbo.Master_OQAD_Question QU on DQ.QuestionId= qu.QuestionId inner join
|
||||
OneApp_KelloggsMT.dbo.Master_OQAD_Category qc on qu.QuestionCategoryId= qc.QuestionCategoryId
|
||||
where e.EmpName not like 'test%' and e.RightId in (6)
|
||||
and (E.ResignDate is null or E.ResignDate>=''+CONVERT(VARCHAR,'{run_date}')+'') AND E.EmpName NOT LIKE '%TEST%'
|
||||
AND DQ.EmpId IN (SELECT EmpId FROM MID_TABLE_COV1 A WHERE
|
||||
DQ.EmpId=A.EmpId AND CONVERT(date,VisitDate)=CONVERT(date,A.VisitDate) )
|
||||
) select * from quiz
|
||||
"""
|
||||
sql = sql_template.format(
|
||||
run_date=run_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
|
||||
log.info(f"Fetching quiz_empids data for EMPID and Visitid")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
|
||||
log.info(f"Fetched {len(df):,} total empid and visitdate fetched for OQAD from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def get_empids_clickhouse_OQAD(
|
||||
client,
|
||||
table_name: str = "OQaD",
|
||||
) -> pl.DataFrame:
|
||||
|
||||
if not table_exists(client, table_name):
|
||||
log.warning(f"Table '{table_name}' does not exist.")
|
||||
return pl.DataFrame(
|
||||
schema={
|
||||
"empid": pl.Int64,
|
||||
"visitdate": pl.Date,
|
||||
}
|
||||
)
|
||||
|
||||
query = f"""
|
||||
SELECT DISTINCT
|
||||
employee_id AS empid,
|
||||
visit_date AS visitdate
|
||||
FROM {table_name}
|
||||
"""
|
||||
|
||||
# ClickHouse -> PyArrow -> Polars
|
||||
arrow_table = client.query_arrow(query)
|
||||
|
||||
return pl.from_arrow(arrow_table)
|
||||
|
||||
|
||||
|
||||
qf=fetch_quiz_empids(sql_engine,run_date)
|
||||
db_df = get_empids_clickhouse_OQAD(client)
|
||||
|
||||
matched = qf.join(
|
||||
db_df,
|
||||
on=["empid", "visitdate"],
|
||||
how="inner",
|
||||
)
|
||||
empids=matched["empid"].to_list()
|
||||
|
||||
log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ")
|
||||
|
||||
def fetch_data(
|
||||
engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
empids: list[int],
|
||||
run_date: date
|
||||
) -> pl.DataFrame:
|
||||
|
||||
empid_list = ",".join(str(empid) for empid in empids)
|
||||
|
||||
|
||||
sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql"
|
||||
|
||||
log.info(f"Exists: {sql_file.exists()}")
|
||||
log.info(f"Path: {sql_file.resolve()}")
|
||||
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
sql_template = f.read()
|
||||
|
||||
sql = sql_template.format(
|
||||
empid_list=empid_list,
|
||||
run_date=run_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
|
||||
log.info(f"Fetching data for {len(empids):,} EMPIDs")
|
||||
|
||||
log.info("Fetching OQaD data for run_date=%s", run_date)
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine,
|
||||
)
|
||||
|
||||
log.info("fn name is fetch_OQad ------Fetched %s rows", len(df))
|
||||
|
||||
return df
|
||||
df=fetch_data( engine=sql_engine,
|
||||
table_name=table_name,
|
||||
table_type=table_type,
|
||||
empids=empids,
|
||||
run_date=run_date
|
||||
)
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
# def fetch_OQaD(
|
||||
# engine: Engine,
|
||||
# table_name: str,
|
||||
# table_type: str,
|
||||
# empids: list[int],
|
||||
# run_date: date
|
||||
# ) -> pl.DataFrame:
|
||||
|
||||
# empid_list = ",".join(str(empid) for empid in empids)
|
||||
|
||||
|
||||
# sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql"
|
||||
|
||||
# log.info(f"Exists: {sql_file.exists()}")
|
||||
# log.info(f"Path: {sql_file.resolve()}")
|
||||
|
||||
# with open(sql_file, "r", encoding="utf-8") as f:
|
||||
# sql_template = f.read()
|
||||
|
||||
# sql = sql_template.format(
|
||||
# empid_list=empid_list,
|
||||
# run_date=run_date.strftime("%Y-%m-%d")
|
||||
# )
|
||||
|
||||
# log.info(f"Fetching data for {len(empids):,} EMPIDs")
|
||||
|
||||
# log.info("Fetching OQaD data for run_date=%s", run_date)
|
||||
|
||||
# df = pl.read_database(
|
||||
# query=sql,
|
||||
# connection=engine,
|
||||
# )
|
||||
|
||||
# log.info("fn name is fetch_OQad ------Fetched %s rows", len(df))
|
||||
|
||||
# return df
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def fetch_Coverage( engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
mids: list[int],
|
||||
run_date: date
|
||||
) -> pl.DataFrame:
|
||||
|
||||
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
|
||||
mid_list = ",".join(str(mid) for mid in mids)
|
||||
|
||||
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
|
||||
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
sql_template = f.read()
|
||||
|
||||
sql = sql_template.format(
|
||||
mid_list=mid_list,
|
||||
run_date=run_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
def fetch_Survey( engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
mids: list[int],
|
||||
run_date: date
|
||||
) -> pl.DataFrame:
|
||||
|
||||
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
|
||||
mid_list = ",".join(str(mid) for mid in mids)
|
||||
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
|
||||
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
|
||||
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
sql_template = f.read()
|
||||
|
||||
sql = sql_template.format(
|
||||
mid_list=mid_list,
|
||||
run_date=run_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
@@ -43,35 +408,269 @@ def fetch_data(
|
||||
|
||||
|
||||
|
||||
def fetch_OQaD(
|
||||
engine: Engine,
|
||||
|
||||
def fetch_Login( engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
empids: list[int],
|
||||
mids: list[int],
|
||||
run_date: date
|
||||
) -> pl.DataFrame:
|
||||
|
||||
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
|
||||
empid_list = ",".join(str(empid) for empid in empids)
|
||||
|
||||
sql_file = Path("sql") / table_type / f"{table_name}.sql"
|
||||
mid_list = ",".join(str(mid) for mid in mids)
|
||||
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
|
||||
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
|
||||
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
sql_template = f.read()
|
||||
|
||||
sql = sql_template.format(
|
||||
empid_list=empid_list,
|
||||
mid_list=mid_list,
|
||||
run_date=run_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
|
||||
log.info(f"Fetching data for {len(empids):,} EMPIDs")
|
||||
|
||||
log.info("Fetching OQaD data for run_date=%s", run_date)
|
||||
log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine,
|
||||
connection=engine
|
||||
)
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def fetch_Stock_Details( engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
mids: list[int],
|
||||
run_date: date
|
||||
) -> pl.DataFrame:
|
||||
|
||||
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
|
||||
mid_list = ",".join(str(mid) for mid in mids)
|
||||
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
|
||||
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
|
||||
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
sql_template = f.read()
|
||||
|
||||
sql = sql_template.format(
|
||||
mid_list=mid_list,
|
||||
run_date=run_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
|
||||
# def fetch_Coverage( engine: Engine,
|
||||
# table_name: str,
|
||||
# table_type: str,
|
||||
# mids: list[int],
|
||||
# run_date: date
|
||||
# ) -> pl.DataFrame:
|
||||
|
||||
|
||||
|
||||
# if not mids:
|
||||
# log.warning("No MIDs — nothing to fetch.")
|
||||
# return pl.DataFrame()
|
||||
|
||||
# mid_list = ",".join(str(mid) for mid in mids)
|
||||
# log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
|
||||
# sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
|
||||
|
||||
# with open(sql_file, "r", encoding="utf-8") as f:
|
||||
# sql_template = f.read()
|
||||
|
||||
# sql = sql_template.format(
|
||||
# mid_list=mid_list,
|
||||
# run_date=run_date.strftime("%Y-%m-%d")
|
||||
# )
|
||||
# log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||
|
||||
# df = pl.read_database(
|
||||
# query=sql,
|
||||
# connection=engine
|
||||
# )
|
||||
# log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
# return df
|
||||
|
||||
|
||||
|
||||
def fetch_Attendance(
|
||||
engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
mids: list[int],
|
||||
run_date: date | None = None,
|
||||
days_back: int = 15
|
||||
) -> pl.DataFrame:
|
||||
"""
|
||||
Fetch attendance source data.
|
||||
|
||||
Default:
|
||||
end_date = yesterday
|
||||
start_date = yesterday - 15 days
|
||||
"""
|
||||
|
||||
if run_date is None:
|
||||
run_date = date.today() - timedelta(days=1)
|
||||
|
||||
start_date = run_date - timedelta(days=days_back)
|
||||
|
||||
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
|
||||
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
sql_template = f.read()
|
||||
|
||||
sql = sql_template.format(
|
||||
start_date=start_date.strftime("%Y-%m-%d"),
|
||||
run_date=run_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
log.info(
|
||||
f"Fetching Attendance data from {start_date} to {run_date}"
|
||||
)
|
||||
|
||||
log.info("Fetched %s rows", len(df))
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(
|
||||
f"Fetched {len(df):,} attendance rows "
|
||||
f"for {df['employee_id'].n_unique():,} employees"
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
def fetch_Journey_Plan( engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
mids: list[int],
|
||||
run_date: date
|
||||
) -> pl.DataFrame:
|
||||
|
||||
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
|
||||
log.info(f" Start Fetching data for these {len(mids):} MIDs ")
|
||||
mid_list = ",".join(str(mid) for mid in mids)
|
||||
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
|
||||
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
|
||||
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
sql_template = f.read()
|
||||
|
||||
sql = sql_template.format(
|
||||
mid_list=mid_list,
|
||||
run_date=run_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
def fetch_PaidVisibility( engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
mids: list[int],
|
||||
run_date: date
|
||||
) -> pl.DataFrame:
|
||||
|
||||
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
|
||||
log.info(f" Start Fetching data for these {len(mids):} MIDs ")
|
||||
mid_list = ",".join(str(mid) for mid in mids)
|
||||
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
|
||||
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
|
||||
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
sql_template = f.read()
|
||||
|
||||
sql = sql_template.format(
|
||||
mid_list=mid_list,
|
||||
run_date=run_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def fetch_Web_Logins( engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
mids: list[int],
|
||||
run_date: date
|
||||
) -> pl.DataFrame:
|
||||
|
||||
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
|
||||
log.info(f" Start Fetching data for these {len(mids):} MIDs ")
|
||||
mid_list = ",".join(str(mid) for mid in mids)
|
||||
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
|
||||
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
|
||||
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
sql_template = f.read()
|
||||
|
||||
sql = sql_template.format(
|
||||
mid_list=mid_list,
|
||||
run_date=run_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
@@ -0,0 +1,11 @@
|
||||
with mapping_storevisibility
|
||||
(Project_Id,StoreId,VisibilityDefinitionid,Fromdate,Todate,CreateDate,CreateBy)
|
||||
AS (
|
||||
select DISTINCT '40148' as Project_Id,StoreId,VisibilityDefinitionid,Fromdate,Todate,getdate(),'SP-Pius'
|
||||
FROM OneApp_KelloggsMT.dbo.mapping_storevisibility z WHERE
|
||||
convert(date,FROMDATE,101)<=convert(Date,getdate(),101) AND CONVERT(DATE,ToDate,101)>=convert(Date,getdate(),101)
|
||||
AND z.VisibilityDefinitionid IN
|
||||
(SELECT DISTINCT VisibilityDefinitionid FROM OneApp_KelloggsMT.dbo.MASTER_VISIBILITYDEFINITION WHERE MENUID=22 )
|
||||
AND z.StoreId NOT IN ({store_id_list})
|
||||
)
|
||||
select * from mapping_storevisibility
|
||||
@@ -0,0 +1,17 @@
|
||||
with Employee_Master (project_id,region_id,region,state_id,state,city_id,
|
||||
city,employee_id,employee_name,gender,designation_id,designation,manager_id,
|
||||
manager_name,employee_joining_date,employee_resign_date,position_code,employee_legacy_code,employee_role,EMPLOYEE_TYPE)
|
||||
as (
|
||||
select '40148' as ProjectId, RegionId,
|
||||
RegionName,StateId,StateName, CityId,CityName,
|
||||
a.EmpId,EmpName,Gender,a.DesignationId,DesignationName,SupervisorId,SupervisorName,JoinDate,ResignDate,c.PositionCode ,EmpCode ,RIGHTNAME,
|
||||
|
||||
case when RIGHTNAME in ('Client','Client HO') Then 'NON CPM' else 'CPM' END AS EMPLOYEE_TYPE
|
||||
from OneApp_KelloggsMT.dbo.vw_Employee_Detail a Left join (Select distinct PositionId, EmpId from OneApp_KelloggsMT.dbo.Mapping_PositionUser b
|
||||
Where DATEdiff(d,FromDate, GETDATE())>= 0 and DATEDIFF(d,ToDate,getdate())<=0) b on a.EmpId=b.EmpId
|
||||
LEFT join (select PositionId,PositionCode from OneApp_KelloggsMT.dbo.Master_Position) c on b.PositionId=c.PositionId
|
||||
where 1=1
|
||||
-- and EmpId not in (
|
||||
--select employee_id from [dbo].[Employee_Master] where project_id='40148')
|
||||
)
|
||||
select * from Employee_Master
|
||||
@@ -0,0 +1,14 @@
|
||||
with Master_Salesterritorylayer(ProjectId,StLayerOneId,StLayerOneName,StLayerTwoId,StLayerTwoName,StLayerThreeId,StLayerThreeName,StLayerFourId,
|
||||
StLayerFourName,CreateDate,CreateBy)
|
||||
as (
|
||||
select DISTINCT '40148' as Project_Id,d.StLayerOneId,d.StLayerOneName,c.StLayerTwoId,c.StLayerTwoName ,
|
||||
b.StLayerThreeId,b.StLayerThreeName,a.StLayerFourId,a.StLayerFourName ,
|
||||
getdate(),'SP-Pius'
|
||||
FROM OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerFour] A
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerThree]B ON
|
||||
A.StLayerThreeId=B.StLayerThreeId
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerTWo]C ON
|
||||
B.StLayerTwoId=C.StLayerTwoId
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerOne]D ON
|
||||
C.StLayerOneId=D.StLayerOneId)
|
||||
select * from Master_Salesterritorylayer
|
||||
@@ -0,0 +1,5 @@
|
||||
with Master_VisibilityDefinition(Project_Id,VisibilityDefinitionId,VisibilityDefinitionName,CreateDate,CreateBy)
|
||||
as (
|
||||
select DISTINCT '40148' as Project_Id,VisibilityDefinitionid,VisibilityDefinitionname,getdate(),'SP-Pius'
|
||||
FROM OneApp_KelloggsMT.dbo.Master_VisibilityDefinition)
|
||||
select * from Master_VisibilityDefinition
|
||||
@@ -0,0 +1,7 @@
|
||||
with Master_VisibilityReason (ProjectId,MenuId,ReasonId,Reason,CreateDate,Createby)
|
||||
as (
|
||||
|
||||
select DISTINCT '40148' as ProjectId,MenuId,VisibilityReasonId,VisibilityReason,getdate(),'SP-Pius'
|
||||
FROM OneApp_KelloggsMT.dbo.Master_VisibilityReason
|
||||
)
|
||||
select * from Master_VisibilityReason
|
||||
@@ -0,0 +1,18 @@
|
||||
|
||||
with SKU_Master (project_id,category_id,category_code,Category_name,sub_category_id,sub_category_code,sub_category_name,brand_id,
|
||||
brand_code,brand_name,sub_brand_id,sub_brand_code,sub_brand_name,product_id,product_name,product_code,mrp,flavour_id,
|
||||
flavour,grammage,product_sequence,case_size,company_name,is_competitor,ptr,update_date,update_by)
|
||||
as (
|
||||
|
||||
Select '40148' as ProjectId, cm.CategoryId,
|
||||
cm.CategoryCode,cm.CategoryName,sca.SubCategoryId,sca.SubCategoryCode,sca.SubCategoryName,br.BrandId,br.BrandCode,br.BrandName
|
||||
,sb.SubBrandId,sb.SubBrandCode,sb.SubBrandName,p.ProductId,p.ProductName,p.ProductCode,p.Mrp,fl.FlavourId,
|
||||
fl.Flavour,p.Grammage,p.ProductSequence,p.CaseSize,mc.Company,mc.IsCompetitor,p.ptr,GETDATE(),'MB'
|
||||
from OneApp_KelloggsMT.dbo.Master_Product p Right join
|
||||
OneApp_KelloggsMT.dbo.Master_Flavour fl on p.FlavourId= Fl.FlavourId Right join
|
||||
OneApp_KelloggsMT.dbo.Master_subbrand sb on p.SubBrandId= sb.SubBrandId Right join
|
||||
OneApp_KelloggsMT.dbo.Master_Brand br on sb.BrandId = br.BrandId Right join
|
||||
OneApp_KelloggsMT.dbo.Master_SubCategory sca on br.SubCategoryId= sca.SubCategoryId Right join
|
||||
OneApp_KelloggsMT.dbo.Master_Category cm on sca.CategoryId= cm.CategoryId Right join
|
||||
OneApp_KelloggsMT.dbo.Master_Company mc on mc.companyId=br.companyId )
|
||||
select * from SKU_Master
|
||||
@@ -0,0 +1,14 @@
|
||||
with Store_Master (project_id,
|
||||
region_id,region,state_id,state,city_id,city,cpm_city_id,channel_id,channel,distributor_id,distributor_name, keyaccount_id, keyaccount,
|
||||
insight_store_id,client_store_code,latitude,longitude,store_category_id,store_category,store_type_id,store_type,store_classification_id,store_classification, StLayerFourId,
|
||||
store_id,store_name,address)
|
||||
AS (
|
||||
select '40148',RegionId,RegionName,StateId,StateName,CityId,
|
||||
CityName,CityCode,ChannelId,ChannelName,DistributorId,Distributor,ChainId,ChainName,StoreUniqueCode,StoreCode,Latitude,
|
||||
Longitude, StoreCategoryId,StoreCategory,StoreTypeId,
|
||||
StoreType,StoreClassId,StoreClass,StLayerFourId,StoreId,
|
||||
StoreName,Address from OneApp_KelloggsMT.dbo.vw_storedetail where 1=1
|
||||
--and storeid not in (
|
||||
--select store_id from Store_Master where project_id='40148')
|
||||
)
|
||||
select * from Store_Master
|
||||
@@ -0,0 +1,8 @@
|
||||
with coverage_remarks
|
||||
(project_id,reason_id,reason_remarks)
|
||||
as(
|
||||
select
|
||||
'40148' ,reasonid,reason from OneApp_KelloggsMT.dbo.master_nonworkingreason
|
||||
where reasonid not in ( {resaon_id_list})
|
||||
)
|
||||
select * from coverage_remarks
|
||||
@@ -0,0 +1,8 @@
|
||||
with display_master (project_id,display_id,display_code,display_name,display_ref_url,created_date,
|
||||
created_by) as (
|
||||
select '40148',displayid,displaycode,displayname,displayrefImage,getdate(),'Pius'
|
||||
from OneApp_KelloggsMT.dbo.Master_display
|
||||
-- and ProductId not in (
|
||||
--select product_id from [dbo].[SKU Master] where project_id='40148')
|
||||
)
|
||||
select * from display_master
|
||||
@@ -0,0 +1,134 @@
|
||||
with Emp_cte as
|
||||
(Select T1.*, ISNULL(ps.PositionCode ,'') PositionCode,FromDate,ToDate from(
|
||||
select a.*,b.date from
|
||||
(Select rm.RegionId, rm.RegionName as Region, st.StateId, st.StateName , cm.CityId, cm.CityName
|
||||
, em2.Id as [Area Manager Id], em2.EmployeeName AS [Area Manager]
|
||||
, em1.Id as [Supervisor Id], isnull(em1.EmployeeName,'') as [Supervisor Name]
|
||||
, em.Id as [Employee Id], isnull(em.EmployeeName,'') as [Employee Name]
|
||||
, dm.DesignationId, dm.DesignationName as Designation
|
||||
, CAST(EM.ResignDate AS DATE) as DOR, CAST(EM.JoinDate AS DATE) as DOJ, em.LegacyCode
|
||||
--,ISNULL(ps.PositionCode ,'') PositionCode--, FromDate, ToDate
|
||||
from
|
||||
OneApp_KelloggsMT.dbo.AspNetUsers em inner join
|
||||
OneApp_KelloggsMT.dbo.AspNetUsers em1 on em1.Id = em.ManagerId inner join
|
||||
OneApp_KelloggsMT.dbo.AspNetUsers em2 on em2.Id = em1.ManagerId inner join
|
||||
OneApp_KelloggsMT.dbo.Master_City cm on em.CityId= cm.CityId inner join
|
||||
OneApp_KelloggsMT.dbo.Master_State st on cm.StateId= st.StateId inner join
|
||||
OneApp_KelloggsMT.dbo.Master_Region rm on st.RegionId= rm.RegionId inner join
|
||||
OneApp_KelloggsMT.dbo.Master_Designation dm on Em.DesignationId= dm.DesignationId
|
||||
|
||||
where em.RightId in (6) and em.EmployeeName not like '%test%' and
|
||||
(Em.ResignDate is null or Em.ResignDate>=CAST('{start_date}' AS DATE)) and CONVERT(date,em.JoinDate,101)<=CONVERT(date,CAST('{run_date}' AS DATE),101)
|
||||
--and em.Id=2290
|
||||
|
||||
)a
|
||||
cross join (select distinct DATE from DBO.GET_ALL_DAYS(CAST('{start_date}' AS DATE),CAST('{run_date}' AS DATE))) b
|
||||
) AS T1 LEFT JOIN
|
||||
(Select distinct EmpId, PositionId, Date,FromDate,ToDate from
|
||||
(Select distinct EmpId,PositionId,FromDate,ToDate from OneApp_KelloggsMT.dbo.Mapping_PositionUser
|
||||
Where convert(Date,FromDate)<= CONVERT(date,CAST('{run_date}' AS DATE),101) and convert(Date,ToDate)>=CONVERT(date,CAST('{start_date}' AS DATE),101)) as m Inner Join
|
||||
(Select Date from OneApp_KelloggsMT.dbo.Master_Calender Where 1=1 and Date between CONVERT(date,CAST('{start_date}' AS DATE),101) and CONVERT(date,CAST('{run_date}' AS DATE),101))
|
||||
cal on cal.Date >= m.FromDate AND cal.Date <= m.ToDate) pu
|
||||
on T1.[Employee Id]=pu.EmpId and T1.date= pu.date Left Join
|
||||
OneApp_KelloggsMT.dbo.Master_Position ps WITH (NOLOCK) ON pu.PositionId=ps.PositionId
|
||||
) ,
|
||||
Attendance
|
||||
(project_id,
|
||||
employee_id,position_code,legacy_code,supervisor_id,date_of_join,date_of_resign,visit_date,parinaam_attendance,
|
||||
update_date,update_by,Unique_Id)
|
||||
AS (
|
||||
|
||||
select '40148', emp.[Employee Id], Case When emp.DATE <= ToDate and emp.DATE>= FromDate
|
||||
then emp.PositionCode Else '' End as PositionCode ,emp.LegacyCode, emp.[Supervisor Id] ,emp.DOJ,emp.DOR,
|
||||
emp.DATE, case when emp.DATE>=emp.DOR then 'R' WHEN emp.DATE< emp.DOJ THEN '-'
|
||||
WHEN b.COL IS NULL AND (emp.DATE<=emp.DOR OR emp.DOR IS NULL )AND (emp.DATE>=emp.DOJ OR emp.DOJ IS NULL) THEN 'W' ELSE b.COL
|
||||
END AS Attendance,GETDATE(),'SP-Pius',CAST('40148' AS VARCHAR) + '_' + CAST(emp.[Employee Id] AS VARCHAR)
|
||||
from Emp_cte as emp
|
||||
|
||||
left join
|
||||
|
||||
|
||||
(select a.*
|
||||
|
||||
|
||||
FROM
|
||||
(
|
||||
SELECT A.DATE,TT.[MERCHANDISER CD],TT.VisitDate,TT.DOJ1,TT.DOR1,TT.COL from DBO.GET_ALL_DAYS(CAST('{start_date}' AS DATE),CAST('{run_date}' AS DATE)) a
|
||||
left join
|
||||
|
||||
|
||||
(
|
||||
|
||||
SELECT distinct [EMPLOYEE ID] AS [MERCHANDISER CD], VISITDATE
|
||||
, CASE WHEN COL=1 THEN 'H' WHEN COL=2 THEN 'L' WHEN COL=4 THEN 'M' WHEN COL=5 THEN 'CO' WHEN COL=6 THEN 'WO'
|
||||
WHEN COL=7 THEN 'P' WHEN COL=10 THEN 'EH' WHEN COL=8 THEN 'HD' WHEN COL IS NULL THEN 'A' END AS COL
|
||||
, DOJ AS DOJ1, DOR AS DOR1
|
||||
FROM
|
||||
(
|
||||
SELECT DOJ, DOR, Region, [AREA MANAGER], SUPERVISOR_CD, [SUPERVISOR], City, [EMPLOYEE ID], MERCHANDISER, VISITDATE
|
||||
, REMARKS, COL, ROW_NUMBER()OVER (PARTITION BY [EMPLOYEE ID], VISITDATE ORDER BY COL DESC,Intime, VISITDATE, ROW ) AS COL1
|
||||
FROM
|
||||
(
|
||||
SELECT DOJ, DOR, Region, [AREA MANAGER], SUPERVISOR_CD, [SUPERVISOR], City, [EMPLOYEE ID], MERCHANDISER, InTime, VISITDATE
|
||||
, REMARKS, COL, ROW_NUMBER()OVER (PARTITION BY [EMPLOYEE ID], VISITDATE ORDER BY COL) AS ROW
|
||||
FROM
|
||||
(
|
||||
SELECT RM.RegionName as Region, em2.EmployeeName AS [Area Manager]
|
||||
, em1.Id as Supervisor_cd, em1.EmployeeName AS [Supervisor]
|
||||
, CM.CityName as City, EM.Id as [Employee Id], em.EmployeeName AS Merchandiser
|
||||
,JP.VisitDate as VisitDate, sc.InTime
|
||||
,ISNULL((SELECT top 1 case
|
||||
when d.ReasonId IS NULL OR d.ReasonId =0 then '' else REASON+' - ' +
|
||||
case when replace(replace(D.REMARK,CHAR(13),''),CHAR(10),' ')<>'' then replace(replace(D.REMARK,CHAR(13),''),CHAR(10),' ')
|
||||
else replace(replace(D.REMARK,CHAR(13),''),CHAR(10),' ') end end as REASON FROM OneApp_KelloggsMT.dbo.T_StoreCoverage D
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_NonWorkingReason N ON N.ReasonId = D.ReasonId
|
||||
WHERE D.ISDEL = 0
|
||||
AND D.EmpId = JP.EmpId AND D.StoreId = JP.StoreId AND d.VisitDate = JP.VisitDate),'') AS Remarks
|
||||
, CASE WHEN (Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId = SC.EmpId AND JP.StoreId = SC.StoreId AND JP.VisitDate = SC.VisitDate
|
||||
AND SC.ReasonId=11) >=1 THEN 1
|
||||
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
|
||||
AND SC.ReasonId=2) >=1 THEN 2
|
||||
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
|
||||
AND SC.ReasonId=6) >=1 THEN 4
|
||||
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
|
||||
AND SC.ReasonId=12) >=1 THEN 5
|
||||
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
|
||||
AND SC.ReasonId=13) >=1 THEN 6
|
||||
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
|
||||
AND SC.ReasonId=7) >=1 THEN 8
|
||||
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
|
||||
AND SC.ReasonId=26) >=1 THEN 10
|
||||
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
|
||||
AND SC.ReasonId IN(0,1,3,4,5,8,9,10,14,15,17,18,19,20,21,22,24)) >=1 THEN 7
|
||||
END END END END END END END END AS COL, em.ResignDate as DOR, em.JoinDate as DOJ
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.Mapping_JourneyPlan JP LEFT OUTER JOIN
|
||||
OneApp_KelloggsMT.dbo.T_StoreCoverage SC ON JP.StoreId= SC.StoreId AND JP.EmpId= SC.EmpId AND JP.VisitDate= SC.VisitDate INNER JOIN
|
||||
OneApp_KelloggsMT.dbo.Master_Store SM ON JP.StoreId= SM.StoreId INNER JOIN
|
||||
OneApp_KelloggsMT.dbo.AspNetUsers em ON JP.EmpId= em.Id inner join
|
||||
OneApp_KelloggsMT.dbo.AspNetUsers em1 on em1.Id = em.ManagerId inner join
|
||||
OneApp_KelloggsMT.dbo.AspNetUsers em2 on em2.Id = em1.ManagerId inner join
|
||||
OneApp_KelloggsMT.dbo.Master_City cm on em.CityId= cm.CityId inner join
|
||||
OneApp_KelloggsMT.dbo.Master_State st on cm.StateId= st.StateId inner join
|
||||
OneApp_KelloggsMT.dbo.Master_Region rm on st.RegionId= rm.RegionId --Left Join
|
||||
|
||||
|
||||
|
||||
|
||||
WHERE EM.EMPLOYEENAME not like 'teststore%' AND JP.VisitDate BETWEEN CAST('{start_date}' AS DATE) AND CAST('{run_date}' AS DATE)
|
||||
|
||||
) AS T1
|
||||
) AS T2
|
||||
) AS T
|
||||
WHERE COL1=1
|
||||
) AS TT
|
||||
on a.date=tt.visitdate)A)b
|
||||
on emp.[Employee Id]=b.[MERCHANDISER CD] and (
|
||||
emp.DOR IS NULL
|
||||
OR emp.DATE >= emp.DOR
|
||||
) )
|
||||
select * from Attendance where
|
||||
parinaam_attendance !='-' and
|
||||
project_id=40148
|
||||
AND visit_date > date_of_join
|
||||
order by employee_id, visit_date
|
||||
+103
-159
@@ -1,164 +1,108 @@
|
||||
WITH SOS_BASE AS
|
||||
(
|
||||
SELECT
|
||||
sm.CountryName,
|
||||
sc.MID,
|
||||
sm.RegionName,
|
||||
sm.StateName,
|
||||
sm.CityName,
|
||||
Em.SupervisorName,
|
||||
em.EmpId,
|
||||
Em.EmpName AS EmployeeName,
|
||||
Em.DesignationName AS Designation,
|
||||
sm.StoreId,
|
||||
CONVERT(varchar, sc.VisitDate, 101) AS VisitDate,
|
||||
sm.StoreCode,
|
||||
sm.StoreName,
|
||||
sm.Address,
|
||||
sm.StoreTypeid,
|
||||
sm.ChannelId,
|
||||
sm.ChainName,
|
||||
|
||||
MSD.SOSDefinitionName,
|
||||
with Executed as
|
||||
(select JP.mid, CASE WHEN isnull(B.MID,'')<>'' then 'Y'
|
||||
ELSE 'N' END AS COVERED,
|
||||
CASE WHEN isnull(C.MID,'')<>'' then 'Y'
|
||||
ELSE 'N' END AS EXECUTED,isnull(NR.REASON,'') AS REASON
|
||||
|
||||
from OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) left join
|
||||
(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock)
|
||||
inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B
|
||||
ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101)
|
||||
AND JP.ReasonId in (0,1,3,9,10,19,20))b
|
||||
on JP.MID=b.MID
|
||||
|
||||
left join
|
||||
(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock)
|
||||
inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B
|
||||
ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101)
|
||||
AND JP.ReasonId in (0,19))c
|
||||
on JP.MID=c.MID
|
||||
left join
|
||||
OneApp_KelloggsMT.dbo.Master_NonWorkingReason nr on JP.ReasonId=nr.ReasonId
|
||||
Where 1=1 AND JP.MID in ({mid_list})
|
||||
|
||||
) ,
|
||||
Coverage
|
||||
(project_id,Mid,
|
||||
store_id,employee_id,visit_date,in_time,out_time,duration_in_minutes,
|
||||
is_covered,is_executed,reason_remarks,storetype_id,
|
||||
supervisor_id,coverage_type,distance_in_meters,reasonId,camera_allow,update_date,
|
||||
update_by,Unique_Id)
|
||||
|
||||
CASE
|
||||
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Category' THEN 'Category'
|
||||
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubCategory' THEN 'SubCategory'
|
||||
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Brand' THEN 'Brand'
|
||||
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubBrand' THEN 'SubBrand'
|
||||
END AS SOSHeaderDeatils,
|
||||
AS (
|
||||
|
||||
Select '40148' as ProjectId,JP.MID, sm.StoreId,JP.EmpId as EmpId,
|
||||
Convert(VARCHAR,Jp.VisitDate) AS VisitDate
|
||||
, Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time,
|
||||
Convert(Datetime,InTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time,
|
||||
Convert(Datetime,InTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage Where EmpId = Jp.EmpId And StoreId = Jp.StoreId
|
||||
And VisitDate = Jp.VisitDate),'') As [In Time]
|
||||
, Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time,
|
||||
Convert(Datetime,OutTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time,
|
||||
Convert(Datetime,OutTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage
|
||||
Where EmpId = Jp.EmpId And StoreId = Jp.StoreId
|
||||
And Convert(Date,VisitDate) = Jp.VisitDate),'') As [Out Time]
|
||||
,
|
||||
Isnull((Select Top 1 Case When convert(varchar,outtime) like '%00:00:00.00%' THEN 0
|
||||
Else Case When outtime<InTime THEN 0 ELSE DATEDIFF(SS,CONVERT(DATETIME, InTime)
|
||||
, CONVERT(DATETIME, outtime))/60 END END From
|
||||
OneApp_KelloggsMT.dbo.T_StoreCoverage sc
|
||||
Where sc.EmpId = Jp.EmpId And sc.StoreId = Jp.StoreId
|
||||
And Convert(Date,sc.visitdate,101) = convert(date,Jp.VisitDate,101)), '') As duration_in_minute,
|
||||
-- Isnull((Select Top 1 case when min(convert(varchar,InTime,108))='00:00:00' OR max(convert(varchar,CheckOutTime,108))='00:00:00'
|
||||
-- then '00:00:00' else Cast(Datediff(Ss,Min(Convert(Varchar,Convert(Datetime,InTime),108)),
|
||||
-- Max(Convert(Varchar,Convert(Datetime, CheckOutTime),108)) ) /3600 As Varchar(2))+':'+ Cast((Datediff(Ss,Min(Convert(Varchar,Convert(Datetime,InTime),108))
|
||||
|
||||
|
||||
-- ,Max(Convert(Varchar,Convert(Datetime, CheckOutTime),108)) ) /60 )%60 As Varchar(2)) +':'+ Cast(Datediff(Ss,Min(Convert(Varchar,Convert(Datetime,InTime),108)),
|
||||
|
||||
|
||||
-- Max(Convert(Varchar,Convert(Datetime, CheckOutTime),108)) ) %60 As Varchar(2)) end as time_taken From OneApp_KelloggsMT.dbo.T_StoreCoverage sc left join
|
||||
--OneApp_KelloggsMT.dbo.T_StoreCheckOut UD ON UD.EmpId = sc.EmpId AND CONVERT(VARCHAR,UD.CheckOutDate,101) = sc.VisitDate AND sc.StoreId = UD.StoreId
|
||||
-- Where sc.EmpId = Jp.EmpId And sc.StoreId = Jp.StoreId
|
||||
-- And VisitDate = Jp.VisitDate),'') As [Time Taken]
|
||||
Case When (Select Top 1 EmpId From OneApp_KelloggsMT.dbo.T_StoreCoverage Sc LEFT join
|
||||
OneApp_KelloggsMT.dbo.Master_NonWorkingReason nw on Sc.ReasonId= nw.ReasonId
|
||||
Where EmpId = Jp.EmpId And StoreId = Jp.StoreId And VisitDate = Jp.VisitDate
|
||||
And (sc.ReasonId =0 Or nw.ForCoverage=1) ) >0 Then 'Y'
|
||||
Else 'N' End As Covered
|
||||
, isnull(Exe.Executed,'') as Executed
|
||||
|
||||
,Isnull((Select Top 1 Case
|
||||
When D.ReasonId Is Null Or D.ReasonId =0 Then '' Else Reason+' - ' +
|
||||
Case When Replace(Replace(D.Remark,Char(13),''''),Char(10),' ')<>'' Then Replace(Replace(D.Remark,Char(13),''),Char(10),' ')
|
||||
Else Replace(Replace(D.Remark,Char(13),''),Char(10),' ') End End As Reason
|
||||
From OneApp_KelloggsMT.dbo.T_StoreCoverage D
|
||||
Inner Join OneApp_KelloggsMT.dbo.Master_NonWorkingReason N On N.ReasonId = D.ReasonId
|
||||
Where D.Isdel = 0
|
||||
And D.EmpId = Jp.EmpId And D.StoreId = Jp.StoreId And D.VisitDate = Jp.VisitDate),'''') As [Detailed Remarks],
|
||||
sm.StoreTypeId,Em.SupervisorId,
|
||||
Case When jp.Deviation=0 Then 'Planned' When jp.Deviation=1 Then 'Adhoc'
|
||||
When jp.Deviation=2 Then 'Beat Plan' When jp.Deviation=3 Then 'Non Merchandised'
|
||||
When jp.Deviation=4 Then 'Add New Store' When jp.Deviation=5 Then 'Non Program' else '' End as [PJP Status]
|
||||
,Isnull((Select Top 1 Case When (Sc.Latitude=0.00000000 Or Sc.Longitude Is Null) Then 0 Else Case When (sm.Latitude=0.00000000 Or sm.Latitude Is Null) Then 0
|
||||
Else Case When (sm.Longitude=0.00000000 Or sm.Longitude Is Null) Then 0
|
||||
Else SQRT(POWER(69.1 * ( Sc.Latitude - Sm.Latitude),2) + POWER(69.1 * ( Sm.Longitude - Sc.Longitude ) * COS(Sc.Latitude / 57.3), 2))*1000
|
||||
End End End As [Distance In Mtr] From OneApp_KelloggsMT.dbo.T_StoreCoverage Sc Where Sc.Isdel=0 And Sc.EmpId= Jp.EmpId
|
||||
And Sc.StoreId= Jp.StoreId And Sc.VisitDate= Jp.VisitDate),'') As [Distance In Mtr]
|
||||
,ISNULL(CAST (JP.ReasonId AS VARCHAR),'0'),sm.CameraAllow,
|
||||
|
||||
TS.SOSHeaderName,
|
||||
TS.SOSHeaderValue AS SOSHeaderID,
|
||||
GETDATE(),'SP-Pius' ,
|
||||
|
||||
'Header_Image' AS HDR1,
|
||||
CAST('40148' AS VARCHAR) + '_' + CAST(SM.storeid AS VARCHAR)
|
||||
+ '_' + CAST(EM.EMPID AS VARCHAR)
|
||||
|
||||
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) Inner Join
|
||||
Executed Exe on Jp.MID= Exe.MID Inner Join
|
||||
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on Jp.StoreId= sm.StoreId Inner Join
|
||||
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on JP.EmpId= Em.EmpId
|
||||
left join
|
||||
OneApp_KelloggsMT.dbo.T_StoreCoveragePositionPivot pv with (nolock)
|
||||
on jp.mid=pv.MID
|
||||
Where 1=1 and em.UserName not like 'test%' AND JP.MID in ({mid_list} )
|
||||
|
||||
CASE
|
||||
WHEN ISNULL(TSC.SOSChildTable,'')='Master_Category' THEN 'Category'
|
||||
WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubCategory' THEN 'SubCategory'
|
||||
WHEN ISNULL(TSC.SOSChildTable,'')='Master_Brand' THEN 'Brand'
|
||||
WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubBrand' THEN 'SubBrand'
|
||||
END AS SOSChildDeatils,
|
||||
|
||||
TSC.SOSChildName,
|
||||
TSC.SOSChildValue AS SOSChildID,
|
||||
TSC.ChildTotalFacing,
|
||||
TS.SOSHeaderFacing,
|
||||
TSC.ChildSelfFacing,
|
||||
|
||||
(
|
||||
SELECT TOP 1 SOSTarget
|
||||
FROM OneApp_KelloggsMT.dbo.Mapping_StoreShareOfShelfTarget a
|
||||
WHERE a.SOSDefinitionId = MSD.SOSDefinitionId
|
||||
AND a.StoreId = sm.StoreId
|
||||
AND a.FromDate <= sc.VisitDate
|
||||
AND a.ToDate >= sc.VisitDate
|
||||
) AS SOSTarget,
|
||||
|
||||
CASE
|
||||
WHEN ISNULL(SHI.SOSHeaderImage,'') = ''
|
||||
THEN ''
|
||||
ELSE
|
||||
'https://kimt1.parinaam.in/Upload/SOSImages/'
|
||||
+ SHI.SOSHeaderImage
|
||||
END AS SOSHeaderImg
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.T_ShareOfShelfHeader ts
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage sc
|
||||
ON ts.MID = sc.MID
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail sm
|
||||
ON sc.StoreId = sm.StoreId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em
|
||||
ON sc.EmpId = Em.EmpId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChild tsc
|
||||
ON ts.SOSId = tsc.SOSId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_ShareOfShelfDefinition msd
|
||||
ON msd.SOSDefinitionId = tsc.SOSDefinitionId
|
||||
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfHeaderImages SHI
|
||||
ON ts.SOSId = SHI.SOSId
|
||||
AND ts.SOSHeaderValue = SHI.SOSHeaderValue
|
||||
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChildImages SCI
|
||||
ON tsc.SOSId = SCI.SOSId
|
||||
AND tsc.SOSChildValue = SCI.SOSChildValue
|
||||
|
||||
WHERE Em.EmpName NOT LIKE 'test%'
|
||||
AND sc.MID IN ({mid_list})
|
||||
),
|
||||
|
||||
SOS_PIVOT AS
|
||||
(
|
||||
SELECT *
|
||||
FROM SOS_BASE
|
||||
PIVOT
|
||||
(
|
||||
MIN(SOSHeaderImg)
|
||||
FOR HDR1 IN ([Header_Image])
|
||||
) pvt
|
||||
)
|
||||
|
||||
SELECT
|
||||
'40148' AS ProjectId,
|
||||
MID,
|
||||
EmpId AS employee_id,
|
||||
StoreId AS store_id,
|
||||
VisitDate AS visit_date,
|
||||
StoreTypeid AS storetype_id,
|
||||
ChannelId AS channel_id,
|
||||
SOSDefinitionName,
|
||||
SOSHeaderDeatils,
|
||||
SOSHeaderName,
|
||||
SOSHeaderID,
|
||||
SOSChildDeatils,
|
||||
SOSChildName,
|
||||
SOSChildID,
|
||||
SOSHeaderFacing,
|
||||
ChildTotalFacing,
|
||||
ChildSelfFacing,
|
||||
SOSTarget,
|
||||
GETDATE() AS update_date,
|
||||
'SP-Pius' AS update_by
|
||||
|
||||
FROM SOS_PIVOT
|
||||
|
||||
GROUP BY
|
||||
CountryName,
|
||||
MID,
|
||||
RegionName,
|
||||
StateName,
|
||||
CityName,
|
||||
SupervisorName,
|
||||
EmpId,
|
||||
EmployeeName,
|
||||
Designation,
|
||||
StoreId,
|
||||
VisitDate,
|
||||
StoreCode,
|
||||
StoreName,
|
||||
Address,
|
||||
StoreTypeid,
|
||||
ChannelId,
|
||||
ChainName,
|
||||
SOSDefinitionName,
|
||||
SOSHeaderDeatils,
|
||||
SOSHeaderName,
|
||||
SOSHeaderID,
|
||||
SOSChildDeatils,
|
||||
SOSChildName,
|
||||
SOSChildID,
|
||||
ChildTotalFacing,
|
||||
SOSHeaderFacing,
|
||||
ChildSelfFacing,
|
||||
SOSTarget
|
||||
|
||||
ORDER BY
|
||||
RegionName,
|
||||
StateName,
|
||||
CityName,
|
||||
VisitDate;
|
||||
)
|
||||
select * from Coverage
|
||||
@@ -0,0 +1,10 @@
|
||||
with Journey_Plan
|
||||
(project_id,store_id,employee_id,visit_date,process_id,CreateDate,CreateBy,UpdateDate,
|
||||
UpdateBy)
|
||||
AS (
|
||||
select
|
||||
'40148' ,storeid,EmpId,cast(visitdate as DATE),Deviation,GETDATE(),'Pius',GETDATE(),'Pius' from OneApp_KelloggsMT.dbo.mapping_journeyplan
|
||||
where MONTH(VisitDate) = MONTH(CAST('{run_date}' AS DATE)) AND Year(VisitDate)=Year(CAST('{run_date}' AS DATE))
|
||||
AND EmpId NOT IN ( SELECT ID FROM OneApp_KelloggsMT.dbo.ASPNETUSERS
|
||||
WHERE USERNAME LIKE 'TEST%'))
|
||||
select * from Journey_Plan;
|
||||
@@ -0,0 +1,81 @@
|
||||
with employee_detail AS
|
||||
(
|
||||
SELECT rm.RegionId, rm.RegionName, st.StateId, st.StateName,
|
||||
cm.CityId, cm.CityName,
|
||||
em1.Id AS [Supervisor Code], ISNULL(em1.EmployeeName, '') AS Supervisor,
|
||||
em.Id AS [Employee code], ISNULL(em.EmployeeName, '') AS [Employee Name],
|
||||
dm.DesignationId, dm.DesignationName AS Designation
|
||||
FROM OneApp_KelloggsMT.dbo.AspNetUsers em
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.AspNetUsers em1 ON em1.Id = em.ManagerId
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_City cm ON em.CityId = cm.CityId
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_State st ON cm.StateId = st.StateId
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_Region rm ON st.RegionId = rm.RegionId
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_Designation dm ON Em.DesignationId = dm.DesignationId
|
||||
WHERE em.RightId IN (6)
|
||||
AND (Em.ResignDate IS NULL OR CONVERT(DATE, Em.ResignDate, 101) >= CAST('{start_date}' AS DATE))
|
||||
AND em.EmployeeName NOT LIKE '%test%'
|
||||
),
|
||||
Login (
|
||||
project_id,
|
||||
employee_id, login_date, login_time, process_id,
|
||||
first_store_in_time, last_store_out_time, update_date, update_by,
|
||||
Unique_Id
|
||||
) AS (
|
||||
SELECT '40148' AS ProjectId,
|
||||
Em.[Employee Code],
|
||||
[Login Date],
|
||||
[Login Time],
|
||||
'0',
|
||||
[1st Store In Time],
|
||||
[Last Store Out Time],
|
||||
GETDATE(),
|
||||
'SP-Pius',
|
||||
CAST(40148 AS VARCHAR) + '_' + CAST(Em.[Employee Code] AS VARCHAR)
|
||||
FROM employee_detail Em
|
||||
LEFT OUTER JOIN
|
||||
(
|
||||
SELECT SupervisorId AS [Supervisor Code],
|
||||
SupervisorName,
|
||||
EmpId AS [Employee Code],
|
||||
EmpName,
|
||||
DesignationName,
|
||||
[Login Date],
|
||||
[Login Time],
|
||||
[1st Store In Time],
|
||||
[Last Store Out Time],
|
||||
AppVersion AS [App Version]
|
||||
FROM (
|
||||
SELECT Em.SupervisorId,
|
||||
Em.SupervisorName,
|
||||
Em.EmpId,
|
||||
Em.EmpName,
|
||||
Em.DesignationName,
|
||||
CONVERT(VARCHAR, ud.LoginDate, 101) AS [Login Date],
|
||||
CONVERT(VARCHAR, ud.InTime, 108) AS [Login Time],
|
||||
ud.AppVersion,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY ud.Empid, CONVERT(VARCHAR, ud.LoginDate, 101)
|
||||
ORDER BY ud.LoginDate ASC
|
||||
) AS col,
|
||||
(SELECT MIN(CONVERT(NVARCHAR, sc.InTime, 108))
|
||||
FROM OneApp_KelloggsMT.dbo.T_StoreCoverage sc
|
||||
WHERE sc.Isdel = 0
|
||||
AND sc.EmpId = ud.EmpId
|
||||
AND CONVERT(VARCHAR, sc.VisitDate, 101) = CONVERT(VARCHAR, ud.LoginDate, 101)
|
||||
) AS [1st Store In Time],
|
||||
(SELECT MAX(CONVERT(NVARCHAR, OutTime, 108))
|
||||
FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC
|
||||
WHERE SC.Isdel = 0
|
||||
AND SC.EmpId = UD.EmpId
|
||||
AND CONVERT(VARCHAR, SC.VisitDate, 101) = CONVERT(VARCHAR, UD.LoginDate, 101)
|
||||
) AS [Last Store Out Time]
|
||||
FROM OneApp_KelloggsMT.dbo.T_DeviceLogin ud
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em ON ud.EmpId = Em.EmpId
|
||||
WHERE CAST(ud.LoginDate AS DATE) = CAST('{start_date}' AS DATE) -- fixed: direct date comparison
|
||||
) AS T1
|
||||
WHERE T1.col = 1
|
||||
) AS T ON Em.[Employee code] = T.[Employee Code]
|
||||
WHERE T.[Login Date] IS NOT NULL
|
||||
)
|
||||
|
||||
SELECT * FROM Login;
|
||||
@@ -0,0 +1,81 @@
|
||||
WITH MID_TABLE_COV1 AS
|
||||
(
|
||||
SELECT EmpId, VisitDate
|
||||
FROM OneApp_KelloggsMT.dbo.T_OQAD
|
||||
WHERE CreateDate >= {run_date}
|
||||
AND CreateDate < DATEADD(DAY,1,'{run_date}')
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT EmpId, VisitDate
|
||||
FROM OneApp_KelloggsMT.dbo.T_OQAD
|
||||
WHERE UpdateDate >= {run_date}
|
||||
AND UpdateDate < DATEADD(DAY,1, '{run_date}')
|
||||
),
|
||||
|
||||
|
||||
QUIZ AS
|
||||
(
|
||||
SELECT DISTINCT
|
||||
E.EmpId,
|
||||
E.EmpName,
|
||||
E.SupervisorId,
|
||||
E.SupervisorName,
|
||||
E.DesignationName,
|
||||
E.CityName,
|
||||
E.StateName,
|
||||
E.RegionName,
|
||||
CAST(DQ.VisitDate AS DATE) AS VisitDate,
|
||||
DQ.QuestionId,
|
||||
DQ.AnswerId,
|
||||
QC.QuestionCategoryId,
|
||||
QC.QuestionCategory
|
||||
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E
|
||||
ON DQ.EmpId = E.EmpId
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU
|
||||
ON DQ.QuestionId = QU.QuestionId
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC
|
||||
ON QU.QuestionCategoryId = QC.QuestionCategoryId
|
||||
WHERE E.EmpName NOT LIKE '%TEST%'
|
||||
AND E.RightId = 6
|
||||
AND (
|
||||
E.ResignDate IS NULL
|
||||
OR CAST(E.ResignDate AS DATE) >= '{run_date}'
|
||||
)
|
||||
AND EXISTS
|
||||
(
|
||||
SELECT 1
|
||||
FROM MID_TABLE_COV1 A
|
||||
WHERE A.EmpId = DQ.EmpId
|
||||
AND CAST(A.VisitDate AS DATE) = CAST(DQ.VisitDate AS DATE)
|
||||
)
|
||||
)
|
||||
|
||||
SELECT
|
||||
40148 AS project_id,
|
||||
Q.EmpId AS employee_id,
|
||||
0 AS process_id,
|
||||
Q.VisitDate AS visit_date,
|
||||
Q.QuestionCategoryId AS question_category_id,
|
||||
Q.QuestionCategory AS question_category,
|
||||
QM.QuestionId AS question_id,
|
||||
QM.Question AS question,
|
||||
ISNULL(QA.AnswerId,0) AS answer_id,
|
||||
ISNULL(QA.Answer,'') AS answer,
|
||||
CASE
|
||||
WHEN QA.AnswerId IS NULL THEN 'Not Answer'
|
||||
WHEN QA.RightAnswer = 1 THEN 'Y'
|
||||
WHEN QA.RightAnswer IS NULL THEN 'Not Answer'
|
||||
ELSE 'N'
|
||||
END AS correct_answer,
|
||||
GETDATE() AS update_date,
|
||||
'SP-Pius' AS update_by
|
||||
FROM QUIZ Q
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM
|
||||
ON Q.QuestionId = QM.QuestionId
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA
|
||||
ON Q.AnswerId = QA.AnswerId
|
||||
where Q.EmpId not in ({empid_list})
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
|
||||
with PaidVisibility(Mid,
|
||||
project_id,store_id,employee_id,visit_date,supervisor_id,channel_id,chain_id,storetype_id,Menuid,MenuName,
|
||||
Visibility_Id,
|
||||
Visibility_Name,
|
||||
Visibility_definition_id,Visibility_definition_name,
|
||||
Visibility_deatils,Visibility_deatils_id,Visibility_value_name,
|
||||
present,REASONID,reason,VisibilityQuestion,VisibilityAnswer,image1,image2,update_date,update_by)
|
||||
AS (
|
||||
|
||||
Select sc.MID, '40148' Projectid,sc.StoreId,Em.EmpId,sc.VisitDate,Em.SupervisorId,sm.ChannelId,sm.ChainId,sm.StoreTypeId, 0 as menuid ,'' as menuname,
|
||||
ts.visibilityid,mv.Visibilityname,msd.VisibilityDefinitionId,
|
||||
MSD.VisibilityDefinitionName, case when isnull(TS.VisibilityTable,'')='Master_Category' then 'Category'
|
||||
when isnull(TS.VisibilityTable,'')='Master_SubCategory' then 'SubCategory'
|
||||
when isnull(TS.VisibilityTable,'')='Master_Brand' then 'Brand'
|
||||
when isnull(TS.VisibilityTable,'')='Master_SubBrand' then 'SubBrand'
|
||||
end as PaidVisibilityDeatils ,ts.visibilityvalue[Paid_visiValueID],
|
||||
CASE when isnull(TS.VisibilityTable,'')='Master_Category' THEN (SELECT a.CategoryName from OneApp_KelloggsMT.dbo.Master_Category a where ts.VisibilityValue=a.CategoryId )
|
||||
when isnull(TS.VisibilityTable,'')='Master_SubCategory' THEN (SELECT a.SubCategoryName from OneApp_KelloggsMT.dbo.Master_SubCategory a where ts.VisibilityValue=a.SubCategoryId )
|
||||
when isnull(TS.VisibilityTable,'')='Master_Brand' THEN (SELECT a.BrandName from OneApp_KelloggsMT.dbo.Master_Brand a where ts.VisibilityValue=a.BrandId )
|
||||
when isnull(TS.VisibilityTable,'')='Master_SubBrand' THEN (SELECT a.SubBrandName from OneApp_KelloggsMT.dbo.Master_SubBrand a where ts.VisibilityValue=a.SubBrandId ) end as
|
||||
[Paid_VisiValueName],case when ts.Present=0 then 'N' else 'Y' end Present,
|
||||
CASE WHEN TS.PRESENT=0 THEN TS.VISIBILITYREASONID ELSE '' END AS VISIBILITYREASONID ,
|
||||
case when ts.Present=0 then isnull(mnp.VisibilityReason,'') else '' end Reason,
|
||||
ISNULL( MVQ.VisibilityQuestionName,'') AS Question, ISNULL(TVQ.VisibilityAnswerName,'') AS Answer ,
|
||||
case when ts.present=0 then '' else
|
||||
case when isnull(SHI.VisibilityImage1,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PaidVisibilityImages/'+SHI.VisibilityImage1
|
||||
end end as PaidVisibilityImg1 ,case when isnull(SHI.VisibilityImage2,'')='' then '' else 'https://kimt.parinaam.in/Upload/PaidVisibilityImages/'+SHI.VisibilityImage2
|
||||
end as PaidVisibilityImg2 ,GETDATE(),'SP-Pius'
|
||||
|
||||
|
||||
from
|
||||
OneApp_KelloggsMT.dbo.T_Visibility ts WITH (NOLOCK) Inner join
|
||||
OneApp_KelloggsMT.dbo.T_StoreCoverage sc WITH (NOLOCK) on ts.mid= sc.mid Inner Join
|
||||
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
|
||||
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId
|
||||
inner join
|
||||
OneApp_KelloggsMT.dbo.Master_VisibilityDefinition msd WITH (NOLOCK) on
|
||||
msd.VisibilityDefinitionId=ts.VisibilityDefinitionId --and msd.menuid=ts.menuid
|
||||
--inner join OneApp_KelloggsMT.dbo.Master_menu MM WITH (NOLOCK) ON ts.menuid=mm.menuid and msd.menuid=mm.menuid
|
||||
left join OneApp_KelloggsMT.DBO. master_visibility mv WITH (NOLOCK) on ts.visibilityid=mv.visibilityid
|
||||
|
||||
|
||||
LEFT join
|
||||
OneApp_KelloggsMT.dbo.T_VisibilityImages SHI WITH (NOLOCK) ON ts.VId=SHI.VId
|
||||
left join
|
||||
OneApp_KelloggsMT.dbo.Master_VisibilityReason mnp WITH (NOLOCK) on ts.VisibilityReasonId=mnp.VisibilityReasonId
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.T_VisibilityStock TVS WITH (NOLOCK) ON TS.VID=TVS.VID
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.T_VisibilityQuestion TVQ WITH (NOLOCK) ON TS.Vid=TVQ.Vid
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.Master_VisibilityQuestion MVQ WITH (NOLOCK) on TVQ.VisibilityQuestionId=MVQ.VisibilityQuestionId
|
||||
|
||||
Where 1=1 and Em.EmpName not like 'test%'
|
||||
--and sc.visitdate between '10/01/2023' and '10/31/2023'
|
||||
AND sc.MID in ({mid_list})
|
||||
)
|
||||
select * from PaidVisibility
|
||||
@@ -0,0 +1,43 @@
|
||||
with Promotion(Mid,
|
||||
project_id,store_id,employee_id,visit_date,supervisor_id,channel_id,chain_id,storetype_id,promo_definition_id,promo_definition_name,
|
||||
promotion_deatils,promotion_deatils_id,promotion_value_name,
|
||||
present,reason,PromoQuestion,PromoAnswer,image1,image2,update_date,update_by)
|
||||
as (
|
||||
Select
|
||||
sc.MID, '40148' Projectid,sc.StoreId,Em.EmpId,sc.VisitDate,Em.SupervisorId,sm.ChannelId,sm.ChainId,sm.StoreTypeId,msd.PromoDefinitionId,
|
||||
MSD.PromoDefinitionName, case when isnull(TS.PromoTable,'')='Master_Category' then 'Category'
|
||||
when isnull(TS.PromoTable,'')='Master_SubCategory' then 'SubCategory'
|
||||
when isnull(TS.PromoTable,'')='Master_Brand' then 'Brand'
|
||||
when isnull(TS.PromoTable,'')='Master_SubBrand' then 'SubBrand'
|
||||
end as PromotionDeatils ,
|
||||
TS.PromoValue[PromotionDeatilsId],
|
||||
CASE when isnull(TS.PromoTable,'')='Master_Category' THEN (SELECT a.CategoryName from OneApp_KelloggsMT.dbo.Master_Category a where ts.PromoValue=a.CategoryId )
|
||||
when isnull(TS.PromoTable,'')='Master_SubCategory' THEN (SELECT a.SubCategoryName from OneApp_KelloggsMT.dbo.Master_SubCategory a where ts.PromoValue=a.SubCategoryId )
|
||||
when isnull(TS.PromoTable,'')='Master_Brand' THEN (SELECT a.BrandName from OneApp_KelloggsMT.dbo.Master_Brand a where ts.PromoValue=a.BrandId )
|
||||
when isnull(TS.PromoTable,'')='Master_SubBrand' THEN (SELECT a.SubBrandName from OneApp_KelloggsMT.dbo.Master_SubBrand a where ts.PromoValue=a.SubBrandId ) end as
|
||||
[PromoValueName],case when ts.Present=0 then 'N' else 'Y' end Present,
|
||||
case when ts.Present=1 then '' else isnull(mnp.PromoReason,'') end as Reason, ISNULL(mpq.PromoQuestionName,'') AS Question,
|
||||
ISNULL(tpq.PromoAnswerName,'') AS Answer ,
|
||||
case when isnull(SHI.PromoImage1,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PromotionImages/'+SHI.PromoImage1
|
||||
end as Image1 ,case when isnull(SHI.PromoImage2,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PromotionImages/'+SHI.PromoImage2
|
||||
end as Image2 ,GETDATE(),'SP-Pius'
|
||||
|
||||
from
|
||||
OneApp_KelloggsMT.dbo.T_Promotion ts Inner join
|
||||
OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join
|
||||
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
|
||||
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId
|
||||
inner join
|
||||
OneApp_KelloggsMT.dbo.Master_PromotionDefinition msd on
|
||||
msd.PromoDefinitionId=ts.PromoDefinitionId
|
||||
LEFT join
|
||||
OneApp_KelloggsMT.dbo.T_PromotionImages SHI ON ts.PId=SHI.PId
|
||||
left join
|
||||
OneApp_KelloggsMT.dbo.Master_PromotionReason mnp on ts.PromoReasonId=mnp.PromoReasonId
|
||||
left join OneApp_KelloggsMT.dbo.t_promotionquestion tpq on ts.PId=tpq.PId
|
||||
left join OneApp_KelloggsMT.dbo.Master_PromotionQuestion mpq on tpq.PromoQuestionId=mpq.PromoQuestionId
|
||||
|
||||
Where 1=1 and Em.EmpName not like 'test%'
|
||||
AND sc.MID in ({mid_list})
|
||||
)
|
||||
select * from Promotion
|
||||
@@ -0,0 +1,70 @@
|
||||
with SOS_OneApp ( project_id,Mid,
|
||||
employee_id,store_id,visit_date,storetype_id,channel_id,SOSDefinitionName,SOSHeaderDeatils,SOSHeaderName,SOSHeaderID,SOSChildDeatils,SOSChildName,SOSChildID,
|
||||
SOSHeaderFacing,ChildTotalFacing,ChildSelfFacing,SOSTarget,update_date,update_by)
|
||||
as (
|
||||
Select '40148' AS ProjectId,MID,
|
||||
EmpId
|
||||
,StoreId,
|
||||
VisitDate,StoreTypeid,ChannelId ,
|
||||
--CameraAllow,
|
||||
SOSDefinitionName,SOSHeaderDeatils,SOSHeaderName,[SOSHeaderID] ,SOSChildDeatils , SOSChildName,SOSChildID,SOSHeaderFacing,ChildTotalFacing,
|
||||
ChildSelfFacing ,SOSTarget,getdate(),'SP-Pius' from
|
||||
(
|
||||
Select sm.CountryName,SC.MID, sm.RegionName, sm.StateName, sm.CityName, Em.SupervisorName, em.EmpId as EmpId
|
||||
, Em.EmpName as EmployeeName, Em.DesignationName as Designation, sm.StoreId,convert(varchar,sc.VisitDate,101)VisitDate, sm.StoreCode, sm.StoreName,
|
||||
sm.Address, sm.StoreTypeid ,sm.ChannelId,sm.ChainName,
|
||||
MSD.SOSDefinitionName, case when isnull(TS.SOSHeaderTable,'')='Master_Category' then 'Category'
|
||||
when isnull(TS.SOSHeaderTable,'')='Master_SubCategory' then 'SubCategory'
|
||||
when isnull(TS.SOSHeaderTable,'')='Master_Brand' then 'Brand'
|
||||
when isnull(TS.SOSHeaderTable,'')='Master_SubBrand' then 'SubBrand'
|
||||
end as SOSHeaderDeatils ,
|
||||
TS.SOSHeaderName,TS.SOSHeaderValue[SOSHeaderID], 'Header'+'_'+'Image' as HDR1,
|
||||
case when isnull(tsc.SOSChildTable,'')='Master_Category' then 'Category'
|
||||
when isnull(tsc.SOSChildTable,'')='Master_SubCategory' then 'SubCategory'
|
||||
when isnull(tsc.SOSChildTable,'')='Master_Brand' then 'Brand'
|
||||
when isnull(tsc.SOSChildTable,'')='Master_SubBrand' then 'SubBrand'
|
||||
end as SOSChildDeatils ,
|
||||
TSC.SOSChildName,TSC.SOSChildValue[SOSChildID],TSC.ChildTotalFacing,ts.SOSHeaderFacing,TSC.ChildSelfFacing
|
||||
,
|
||||
(select top 1 SOSTarget from OneApp_KelloggsMT.dbo.Mapping_StoreShareOfShelfTarget a where a.SOSDefinitionId=msd.SOSDefinitionId and
|
||||
a.Storeid=sm.StoreId and a.fromdate<=sc.visitdate and a.todate>=sc.visitdate ) as SOSTarget,
|
||||
case when isnull(SHI.SOSHeaderImage,'')='' then '' else 'https://kimt1.parinaam.in/Upload/SOSImages/'+SHI.SOSHeaderImage
|
||||
end as SOSHeaderImg
|
||||
--,case when isnull(SCI.SOSChildImage,'')='' then '' else 'https://di1.parinaam.in/Upload/SOSImages/'+SCI.SOSChildImage
|
||||
--end as SOSChildImage
|
||||
|
||||
from
|
||||
OneApp_KelloggsMT.dbo.T_ShareOfShelfHeader ts Inner join
|
||||
OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join
|
||||
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
|
||||
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId
|
||||
inner join
|
||||
OneApp_KelloggsMT.dbo.T_ShareOfShelfChild tsc on ts.sosid=tsc.SOSId
|
||||
inner join
|
||||
OneApp_KelloggsMT.dbo.Master_ShareOfShelfDefinition msd on
|
||||
msd.SOSDefinitionId=tsc.SOSDefinitionId
|
||||
|
||||
LEFT join
|
||||
OneApp_KelloggsMT.dbo.T_ShareOfShelfHeaderImages SHI ON ts.SOSId=SHI.SOSId AND TS.SOSHeaderValue=SHI.SOSHeaderValue
|
||||
LEFT JOIN
|
||||
OneApp_KelloggsMT.dbo.T_ShareOfShelfChildImages SCI ON TSC.SOSId=SCI.SOSId AND TSC.SOSChildValue=SCI.SOSChildValue
|
||||
Where 1=1 AND EM.EMPNAME NOT LIKE 'test%' AND sc.mid IN ({mid_list})
|
||||
-- AND sc.VisitDate BETWEEN '09/18/2023' AND '09/21/2023'
|
||||
--and sc.StoreId not in (
|
||||
--select store_id from SOS_OneApp a where a.project_id=40148 and a.store_id=sc.StoreId
|
||||
--and a.employee_id=sc.EmpId and convert(date,sc.VisitDate,101)=convert(date,a.visit_date,101) )
|
||||
|
||||
) as A
|
||||
|
||||
Pivot
|
||||
(
|
||||
min(SOSHeaderImg)
|
||||
for HDR1 in([Header_Image])
|
||||
) as pvt1
|
||||
|
||||
GROUP BY CountryName,MID, RegionName, StateName, CityName, SupervisorName, EmpId
|
||||
, EmployeeName, Designation, StoreId,VisitDate,StoreCode, StoreName,
|
||||
Address, StoreTypeid ,ChannelId,ChainName,
|
||||
SOSDefinitionName,SOSHeaderDeatils,SOSHeaderName,[SOSHeaderID] ,SOSChildDeatils ,
|
||||
SOSChildName,SOSChildID,ChildTotalFacing,SOSHeaderFacing,ChildSelfFacing,SOSTarget )
|
||||
select * from SOS_OneApp
|
||||
@@ -0,0 +1,29 @@
|
||||
with Stock_Details (project_id,
|
||||
Mid,supervisor_id,employee_id,store_id,
|
||||
visitdate,storetype_id,store_category_id,product_id,MSL, MBQ,Stock_Qty,
|
||||
damagedstock,loststock,expirystock,skuavailability,update_date,update_by,StockType)
|
||||
|
||||
AS (
|
||||
|
||||
Select '40148' AS ProjectId,sc.MID,Em.SupervisorId,Em.EmpId,sm.StoreId,sc.VisitDate,sc.StoreTypeId,sm.StoreCategoryId,
|
||||
vp.ProductId,ts.MSL,mp.MBQ, ISNULL(ts.OpeningStock,0)+ISNULL(TS.middaystock,0),0,0,0,
|
||||
Case WHEN ISNULL(ts.OpeningStock,0)+ISNULL(TS.middaystock,0)>=1THEN 'Y' ELSE 'N' END AS
|
||||
skuavailability, GETDATE(),'SP-Pius','Parinaam'
|
||||
from
|
||||
OneApp_KelloggsMT.dbo.T_Stock ts Inner join
|
||||
OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join
|
||||
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
|
||||
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId
|
||||
inner join OneApp_KelloggsMT.dbo.vw_product vp on ts.ProductId=vp.ProductId inner JOIN
|
||||
(SELECT MBQ,ProductId,StateId,ChainId,StoreTypeId,StoreCategoryId,StoreClassId FROM OneApp_KelloggsMT.dbo.Mapping_ProductAssortment where FromDate<='{run_date}' and ToDate>='{run_date}') mp on mp.StateId=sm.StateId and
|
||||
mp.ChainId=sm.ChainId
|
||||
and mp.StoreTypeId=sm.StoreTypeId
|
||||
and mp.StoreCategoryId=sm.StoreCategoryId
|
||||
and mp.StoreClassId=sm.StoreClassId
|
||||
and mp.ProductId=ts.ProductId
|
||||
|
||||
|
||||
Where Em.EmpName not like 'test%' AND sc.MID in ({mid_list})
|
||||
|
||||
)
|
||||
select * from Stock_Details
|
||||
@@ -0,0 +1,68 @@
|
||||
WITH QUIZ AS
|
||||
(
|
||||
SELECT
|
||||
SC.MID,
|
||||
SC.StoreId,
|
||||
EM.EmpId,
|
||||
EM.EmpName,
|
||||
EM.SupervisorId,
|
||||
EM.SupervisorName,
|
||||
EM.DesignationName,
|
||||
CAST(SC.VisitDate AS DATE) AS VisitDate,
|
||||
DQ.QuestionId,
|
||||
QU.Question,
|
||||
DQ.Answer,
|
||||
QC.CategoryId,
|
||||
QC.Category,
|
||||
MSS.SubCategoryId,
|
||||
MSS.SubCategory,
|
||||
DQ.ImageUrl
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.T_SURVEY DQ
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC
|
||||
ON DQ.MID = SC.MID
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveyQuestion QU
|
||||
ON DQ.QuestionId = QU.QuestionId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveySubCategory MSS
|
||||
ON QU.SubCategoryId = MSS.SubCategoryId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveyCategory QC
|
||||
ON MSS.CategoryId = QC.CategoryId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM
|
||||
ON SC.EmpId = EM.EmpId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_Survey MS
|
||||
ON DQ.SurveyId = MS.SurveyId
|
||||
AND QU.SurveyId = MS.SurveyId
|
||||
|
||||
WHERE EM.EmpName NOT LIKE 'test%'
|
||||
AND SC.MID IN ({mid_list})
|
||||
)
|
||||
|
||||
SELECT
|
||||
40148 AS Project_Id,
|
||||
Q.MID AS Mid,
|
||||
Q.SupervisorId AS SuperVisorId,
|
||||
Q.EmpId AS EmpID,
|
||||
SM.StoreId,
|
||||
Q.VisitDate,
|
||||
SM.StoreTypeId,
|
||||
SM.ChainId,
|
||||
Q.CategoryId,
|
||||
Q.Category,
|
||||
Q.SubCategoryId,
|
||||
Q.SubCategory,
|
||||
Q.QuestionId,
|
||||
Q.Question,
|
||||
Q.Answer,
|
||||
GETDATE() AS CreateDate,
|
||||
'SP-Pius' AS CreateBy
|
||||
|
||||
FROM QUIZ Q
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM
|
||||
ON Q.StoreId = SM.StoreId
|
||||
@@ -0,0 +1,23 @@
|
||||
|
||||
with Web_Logins (project_id,
|
||||
supervisor_id,supervisor_name,emp_id,employee_name,designation,date,time,activity_name,activity_type,right_name,CreateDate,CreateBy)
|
||||
as (
|
||||
Select Distinct '40148' as ProjectId
|
||||
, EM1.Id AS SupervisorId, EM1.EmployeeName AS Supervisor
|
||||
, EM.Id AS EmployeeId , EM.EmployeeName as Employee
|
||||
, DM.DesignationName as Designation
|
||||
, cast( CONVERT(date,AL.Date) as Date) AS Date,CONVERT(NVARCHAR,AL.Date ,108) AS Time
|
||||
, AL.Thread as ActivityName, AL.Level as ActivityType, R.RightName,GETDATE(),'Pius'
|
||||
FROM OneApp_KelloggsMT.dbo.T_User_Activity_Log AL LEFT JOIN
|
||||
OneApp_KelloggsMT.dbo.AspNetUsers EM ON AL.Logger=EM.USERNAME INNER JOIN
|
||||
OneApp_KelloggsMT.dbo.AspNetUsers EM1 ON EM1.Id = EM.ManagerId INNER JOIN
|
||||
OneApp_KelloggsMT.dbo.AspNetUsers EM2 ON EM2.Id = EM1.ManagerId INNER JOIN
|
||||
OneApp_KelloggsMT.dbo.Master_Designation DM ON EM.DesignationId = DM.DesignationId INNER JOIN
|
||||
OneApp_KelloggsMT.dbo.Master_City CM ON EM.CityId = CM.CityId INNER JOIN
|
||||
OneApp_KelloggsMT.dbo.Master_State ST ON ST.StateId = cm.StateId INNER JOIN
|
||||
OneApp_KelloggsMT.dbo.Master_Region RM ON RM.RegionId = ST.RegionId INNER JOIN
|
||||
OneApp_KelloggsMT.dbo.Right_Master R ON EM.RightId=R.RightId
|
||||
Where 1=1 and CONVERT(date,AL.Date,101) = CONVERT(DATE,CAST('{run_date}' AS DATE),101)
|
||||
)
|
||||
select * from Web_Logins
|
||||
ORDER BY employee_name
|
||||
@@ -0,0 +1,31 @@
|
||||
|
||||
|
||||
WITH
|
||||
additional_visibility (project_id,Mid,
|
||||
emp_id,store_id,storetype_id,channel_id,chain_id,camera_allowed,visit_date,is_present,brand_id,
|
||||
display_id,Remarks,image_url,created_date,created_by)
|
||||
as (
|
||||
|
||||
SELECT '40148' AS ProjectId,ts.mid, em.EmpId as EmpId
|
||||
, sm.StoreId,sm.storetypeid,sm.channelid,sm.chainid,case when sm.cameraallow=1 then 'Y' ELSE 'N' END AS
|
||||
cameraallow , convert(varchar,sc.VisitDate,101)VisitDate,
|
||||
case when ts.Present=0 then 'N' else 'Y' end Present,
|
||||
isnull(MB.BrandId,'')[BrandId],
|
||||
isnull(MD.DisplayId,'')[DisplayId],ISNULL(TS.REMARK,'')[Remark],
|
||||
case when isnull(ts.ImageUrl,'')='' then '' else 'https://kimt1.parinaam.in/Upload/VisibilityImages/'+ts.ImageUrl
|
||||
end as AdditionalImageUrl ,GETDATE(),'Pius'
|
||||
|
||||
from
|
||||
OneApp_KelloggsMT.dbo.T_AdditionalVisibility ts Inner join
|
||||
OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join
|
||||
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
|
||||
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId
|
||||
left join
|
||||
OneApp_KelloggsMT.dbo.Master_Brand MB ON TS.BrandId=MB.BrandId
|
||||
LEFT JOIN
|
||||
OneApp_KelloggsMT.dbo.Master_Display MD ON TS.DisplayId=MD.DisplayId
|
||||
Where Em.EmpName not like 'test%' and ts.Present=1 and sc.mid in ({mid_list})
|
||||
|
||||
)
|
||||
|
||||
select * from additional_visibility;
|
||||
@@ -62,32 +62,32 @@ tables:
|
||||
- name: Store_Master
|
||||
type: DIMENSION
|
||||
operation: DELETE+INSERT
|
||||
fetch_by: none
|
||||
fetch_by: master
|
||||
|
||||
- name: SKU_Master
|
||||
type: DIMENSION
|
||||
operation: DELETE+INSERT
|
||||
fetch_by: none
|
||||
fetch_by: master
|
||||
|
||||
- name: display_master
|
||||
type: DIMENSION
|
||||
operation: DELETE+INSERT
|
||||
fetch_by: none
|
||||
fetch_by: master
|
||||
|
||||
- name: Employee_Master
|
||||
type: DIMENSION
|
||||
operation: DELETE+INSERT
|
||||
fetch_by: none
|
||||
fetch_by: master
|
||||
|
||||
|
||||
- name: coverage_remarks
|
||||
type: DIMENSION
|
||||
operation: DELETE+INSERT
|
||||
fetch_by: none
|
||||
fetch_by: reason_id
|
||||
|
||||
- name: mapping_storevisibility
|
||||
type: BRIDGE
|
||||
operation: DELETE+INSERT
|
||||
operation: ONLY_INSERT
|
||||
fetch_by: run_date
|
||||
|
||||
- name: Master_VisibilityReason
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
tables:
|
||||
# - name: SOS_OneApp
|
||||
# type: FACT
|
||||
# operation: INSERT
|
||||
# fetch_by: mids
|
||||
|
||||
# # - name: OQaD
|
||||
# # type: FACT
|
||||
# # operation: INSERT
|
||||
# # fetch_by: run_date
|
||||
|
||||
# - name: additional_visibility
|
||||
# type: FACT
|
||||
# operation: INSERT
|
||||
# fetch_by: mids
|
||||
|
||||
# - name: Coverage
|
||||
# type: FACT
|
||||
# operation: INSERT
|
||||
# fetch_by: mids
|
||||
|
||||
# - name: Survey
|
||||
# type: FACT
|
||||
# operation: INSERT
|
||||
# fetch_by: mids
|
||||
|
||||
# - name: Login
|
||||
# type: FACT
|
||||
# operation: INSERT
|
||||
# fetch_by: run_date
|
||||
|
||||
# - name: Stock_Details
|
||||
# type: FACT
|
||||
# operation: INSERT
|
||||
# fetch_by: mids
|
||||
|
||||
# - name: Attendance
|
||||
# type: FACT
|
||||
# operation: DELETE+INSERT
|
||||
# fetch_by: run_date
|
||||
|
||||
# - name: Journey_Plan
|
||||
# type: FACT
|
||||
# operation: INSERT
|
||||
# fetch_by: run_date
|
||||
|
||||
# - name: PaidVisibility
|
||||
# type: FACT
|
||||
# operation: INSERT
|
||||
# fetch_by: mids
|
||||
|
||||
# - name: Web_Logins
|
||||
# type: FACT
|
||||
# operation: INSERT
|
||||
# fetch_by: run_date
|
||||
|
||||
|
||||
# - name: Store_Master
|
||||
# type: DIMENSION
|
||||
# operation: DELETE+INSERT
|
||||
# fetch_by: master
|
||||
|
||||
# - name: coverage_remarks
|
||||
# type: DIMENSION
|
||||
# operation: DELETE+INSERT
|
||||
# fetch_by: reason_id
|
||||
|
||||
# - name: SKU_Master
|
||||
# type: DIMENSION
|
||||
# operation: DELETE+INSERT
|
||||
# fetch_by: master
|
||||
|
||||
# - name: display_master
|
||||
# type: DIMENSION
|
||||
# operation: DELETE+INSERT
|
||||
# fetch_by: master
|
||||
|
||||
# - name: Employee_Master
|
||||
# type: DIMENSION
|
||||
# operation: DELETE+INSERT
|
||||
# fetch_by: master
|
||||
|
||||
# - name: mapping_storevisibility
|
||||
# type: BRIDGE
|
||||
# operation: ONLY_INSERT
|
||||
# fetch_by: run_date
|
||||
|
||||
- name: Master_VisibilityReason
|
||||
type: DIMENSION
|
||||
operation: DELETE+INSERT
|
||||
fetch_by: none
|
||||
|
||||
|
||||
- name: Master_VisibilityDefinition
|
||||
type: DIMENSION
|
||||
operation: DELETE+INSERT
|
||||
fetch_by: none
|
||||
|
||||
|
||||
- name: Master_Salesterritorylayer
|
||||
type: DIMENSION
|
||||
operation: DELETE+INSERT
|
||||
fetch_by: none
|
||||
|
||||
Reference in New Issue
Block a user