ankitm/data_pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: ankitm/data_pipeline:main
Branches Tags
ankitm/data_pipeline:main ankitm/data_pipeline:test ankitm/data_pipeline:backup
..
pull from: ankitm/data_pipeline:backup
Branches Tags
ankitm/data_pipeline:test ankitm/data_pipeline:backup ankitm/data_pipeline:main

5 Commits
main .. backup

Author SHA1 Message Date
Ankit Malik 2ead99553f 22-06-2026 1st commit 2026-06-22 12:05:25 +05:30
Ankit Malik 1ba1545ea8 19-06-2026 2nd commit 2026-06-19 18:15:27 +05:30
Ankit Malik 97b22ea0f8 19-06-2026 1st commit 2026-06-19 16:51:03 +05:30
Ankit Malik d60740ae48 18-06-2026 1st commit 2026-06-18 18:16:50 +05:30
Ankit Malik 9cd4135d7b 17-06-2026 1st commit 2026-06-17 15:40:13 +05:30
34 changed files with 6491 additions and 322 deletions
Expand all files Collapse all files
clickhouse_task/delete_task.py
+11 -14
View File
@@ -1,6 +1,7 @@
from __future__ import annotations from __future__ import annotations
import polars as pl import polars as pl
from datetime import date, datetime, timedelta
from log import log from log import log
@@ -38,10 +39,7 @@ def delete_rows(
WHERE {where_clause} WHERE {where_clause}
""" """
log.info( log.info(f"_ _ _ _ Deleting Data from ClickHouse for {table_name} _ _ _ _")
"Deleting from %s",
table_name,
)
client.command(query) client.command(query)
@@ -72,16 +70,14 @@ def delete_existing_data(
"Stock_Details", "Stock_Details",
} }
if table_name in mid_tables and mids: if table_name in mid_tables and mids :
mids_str = ",".join( mids_str = ",".join(str(mid) for mid in mids)
map(str, mids)
)
delete_rows( delete_rows(
client, client,
table_name, table_name,
f"MID IN ({mids_str})", f"Mid IN ({mids_str})",
) )
return return
@@ -96,9 +92,10 @@ def delete_existing_data(
client, client,
table_name, table_name,
f""" f"""
toMonth(visit_date) = {run_date.month} project_id = 40148
AND toYear(visit_date) = {run_date.year} AND toMonth(visit_date) = toMonth(toDate('2026-06-18'))
""", AND toYear(visit_date) = toYear(toDate('2026-06-18'))
""",
) )
return return
@@ -147,8 +144,8 @@ def delete_existing_data(
client, client,
table_name, table_name,
f""" f"""
toDate(attendance_date) toDate(visit_date) BETWEEN toDate('{run_date - timedelta(days=15) }') AND toDate('{run_date}')
= toDate('{run_date}') =
""", """,
) )
logs/etl_20260616.log
+71
View File
@@ -0,0 +1,71 @@
2026-06-16 17:56:17 | INFO | ================================================================================
2026-06-16 17:56:17 | INFO | Hello from data-move Python data pipeline!
2026-06-16 17:56:17 | INFO | Pipeline Run Date: 2026-06-15
2026-06-16 17:56:17 | INFO | Connecting to databases...
2026-06-16 17:56:19 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000017B644A81B0>
2026-06-16 17:56:23 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000017B6548EFD0>
2026-06-16 17:56:24 | INFO | Database connections established
2026-06-16 17:56:24 | INFO | Collecting MIDs for: 2026-06-15
2026-06-16 17:56:24 | INFO | Found 849 MIDs
2026-06-16 17:57:33 | INFO | ================================================================================
2026-06-16 17:57:33 | INFO | Hello from data-move Python data pipeline!
2026-06-16 17:57:33 | INFO | Pipeline Run Date: 2026-06-15
2026-06-16 17:57:33 | INFO | Connecting to databases...
2026-06-16 17:57:34 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DCA21AB750>
2026-06-16 17:57:36 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DCA3371590>
2026-06-16 17:57:36 | INFO | Database connections established
2026-06-16 17:57:36 | INFO | Collecting MIDs for: 2026-06-15
2026-06-16 17:57:37 | INFO | Found 849 MIDs
2026-06-16 17:58:52 | INFO | ================================================================================
2026-06-16 17:58:52 | INFO | Hello from data-move Python data pipeline!
2026-06-16 17:58:52 | INFO | Pipeline Run Date: 2026-06-15
2026-06-16 17:58:52 | INFO | Connecting to databases...
2026-06-16 17:58:53 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E5AB2CF750>
2026-06-16 17:58:54 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E5AC458910>
2026-06-16 17:58:55 | INFO | Database connections established
2026-06-16 17:58:55 | INFO | Collecting MIDs for: 2026-06-15
2026-06-16 17:58:55 | INFO | Found 849 MIDs
2026-06-16 18:00:31 | INFO | ================================================================================
2026-06-16 18:00:31 | INFO | Hello from data-move Python data pipeline!
2026-06-16 18:00:31 | INFO | Pipeline Run Date: 2026-06-15
2026-06-16 18:00:31 | INFO | Connecting to databases...
2026-06-16 18:00:38 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E8C854F750>
2026-06-16 18:00:40 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E8C9688910>
2026-06-16 18:00:40 | INFO | Database connections established
2026-06-16 18:00:40 | INFO | Collecting MIDs for: 2026-06-15
2026-06-16 18:00:42 | INFO | Found 849 MIDs
2026-06-16 18:00:44 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-16 18:02:58 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
2026-06-16 18:07:47 | INFO | ================================================================================
2026-06-16 18:07:47 | INFO | Hello from data-move Python data pipeline!
2026-06-16 18:07:47 | INFO | Pipeline Run Date: 2026-06-15
2026-06-16 18:07:47 | INFO | Connecting to databases...
2026-06-16 18:07:48 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DF4BB2F750>
2026-06-16 18:07:50 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DF4CCE8A50>
2026-06-16 18:07:50 | INFO | Database connections established
2026-06-16 18:07:50 | INFO | Collecting MIDs for: 2026-06-15
2026-06-16 18:07:50 | INFO | Found 849 MIDs
2026-06-16 18:07:51 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-16 18:08:09 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
2026-06-16 18:08:58 | INFO | ================================================================================
2026-06-16 18:08:58 | INFO | Hello from data-move Python data pipeline!
2026-06-16 18:08:58 | INFO | Pipeline Run Date: 2026-06-15
2026-06-16 18:08:58 | INFO | Connecting to databases...
2026-06-16 18:08:59 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001639269F750>
2026-06-16 18:09:01 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000016393858910>
2026-06-16 18:09:01 | INFO | Database connections established
2026-06-16 18:09:01 | INFO | Collecting MIDs for: 2026-06-15
2026-06-16 18:09:02 | INFO | Found 849 MIDs
2026-06-16 18:09:02 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-16 18:09:19 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
2026-06-16 18:11:21 | INFO | ================================================================================
2026-06-16 18:11:21 | INFO | Hello from data-move Python data pipeline!
2026-06-16 18:11:21 | INFO | Pipeline Run Date: 2026-06-15
2026-06-16 18:11:21 | INFO | Connecting to databases...
2026-06-16 18:11:23 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002240B7AF750>
2026-06-16 18:11:24 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002240C978910>
2026-06-16 18:11:24 | INFO | Database connections established
2026-06-16 18:11:24 | INFO | Collecting MIDs for: 2026-06-15
2026-06-16 18:11:25 | INFO | Found 849 MIDs
2026-06-16 18:11:25 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-16 18:13:05 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
logs/etl_20260617.log
+688
View File
@@ -0,0 +1,688 @@
2026-06-17 10:17:16 | INFO | ================================================================================
2026-06-17 10:17:16 | INFO | Hello from data-move Python data pipeline!
2026-06-17 10:17:16 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 10:17:16 | INFO | Connecting to databases...
2026-06-17 10:17:18 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000019A9A2CF750>
2026-06-17 10:17:21 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000019A9B778910>
2026-06-17 10:17:22 | INFO | Database connections established
2026-06-17 10:17:22 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 10:17:22 | INFO | Found 863 MIDs
2026-06-17 10:17:23 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 10:17:44 | INFO | Fetched 290,984 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 10:43:05 | INFO | ================================================================================
2026-06-17 10:43:05 | INFO | Hello from data-move Python data pipeline!
2026-06-17 10:43:05 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 10:43:05 | INFO | Connecting to databases...
2026-06-17 10:43:07 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002CE210FF750>
2026-06-17 10:43:08 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002CE2253CA50>
2026-06-17 10:43:09 | INFO | Database connections established
2026-06-17 10:43:09 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 10:43:10 | INFO | Found 863 MIDs
2026-06-17 10:43:11 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 10:43:55 | INFO | Fetched 291,001 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 10:45:51 | INFO | ================================================================================
2026-06-17 10:45:51 | INFO | Hello from data-move Python data pipeline!
2026-06-17 10:45:51 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 10:45:51 | INFO | Connecting to databases...
2026-06-17 10:45:52 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000214EC45F750>
2026-06-17 10:45:53 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000214ED908910>
2026-06-17 10:45:54 | INFO | Database connections established
2026-06-17 10:45:54 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 10:45:55 | INFO | Found 863 MIDs
2026-06-17 10:45:55 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 10:46:11 | INFO | Fetched 291,005 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 10:46:11 | INFO | ================================================================================
2026-06-17 10:46:11 | INFO | Processing Table: OQaD
2026-06-17 10:46:11 | ERROR | Failed processing table OQaD
Traceback (most recent call last):
File "D:\data_move\main2.py", line 212, in main
df=fetch_OQaD( engine=sql_engine,
table_name=table_name,
...<2 lines>...
run_date=run_date
)
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
with open(sql_file, "r", encoding="utf-8") as f:
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
2026-06-17 10:48:44 | INFO | ================================================================================
2026-06-17 10:48:44 | INFO | Hello from data-move Python data pipeline!
2026-06-17 10:48:44 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 10:48:44 | INFO | Connecting to databases...
2026-06-17 10:48:45 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000021A270378B0>
2026-06-17 10:48:46 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000021A28538910>
2026-06-17 10:48:47 | INFO | Database connections established
2026-06-17 10:48:47 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 10:48:48 | INFO | Found 863 MIDs
2026-06-17 10:48:48 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 10:49:00 | INFO | Fetched 291,005 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 10:49:01 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 10:49:01 | INFO | ================================================================================
2026-06-17 10:49:01 | INFO | Processing Table: OQaD
2026-06-17 10:49:01 | ERROR | Failed processing table OQaD
Traceback (most recent call last):
File "D:\data_move\main2.py", line 215, in main
df=fetch_OQaD( engine=sql_engine,
table_name=table_name,
...<2 lines>...
run_date=run_date
)
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
with open(sql_file, "r", encoding="utf-8") as f:
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
2026-06-17 10:53:50 | INFO | ================================================================================
2026-06-17 10:53:50 | INFO | Hello from data-move Python data pipeline!
2026-06-17 10:53:50 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 10:53:50 | INFO | Connecting to databases...
2026-06-17 10:53:51 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001D1D6DB78B0>
2026-06-17 10:53:52 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001D1D81F8910>
2026-06-17 10:53:53 | INFO | Database connections established
2026-06-17 10:53:53 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 10:53:54 | INFO | Found 863 MIDs
2026-06-17 10:53:54 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 10:54:08 | INFO | Fetched 291,006 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 10:54:09 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 10:54:09 | INFO | ================================================================================
2026-06-17 10:54:09 | INFO | Processing Table: OQaD
2026-06-17 10:54:09 | ERROR | Failed processing table OQaD
Traceback (most recent call last):
File "D:\data_move\main2.py", line 217, in main
df=fetch_OQaD( engine=sql_engine,
table_name=table_name,
...<2 lines>...
run_date=run_date
)
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
with open(sql_file, "r", encoding="utf-8") as f:
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
2026-06-17 10:57:28 | INFO | ================================================================================
2026-06-17 10:57:28 | INFO | Hello from data-move Python data pipeline!
2026-06-17 10:57:28 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 10:57:28 | INFO | Connecting to databases...
2026-06-17 10:57:30 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000197AB2778B0>
2026-06-17 10:57:31 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000197AC768910>
2026-06-17 10:57:32 | INFO | Database connections established
2026-06-17 10:57:32 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 10:57:32 | INFO | Found 863 MIDs
2026-06-17 10:57:33 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 10:57:45 | INFO | Fetched 291,010 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 10:57:46 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 10:57:46 | INFO | ================================================================================
2026-06-17 10:57:46 | INFO | Processing Table: OQaD
2026-06-17 10:57:46 | ERROR | Failed processing table OQaD
Traceback (most recent call last):
File "D:\data_move\main2.py", line 217, in main
df=fetch_OQaD( engine=sql_engine,
table_name=table_name,
...<2 lines>...
run_date=run_date
)
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
with open(sql_file, "r", encoding="utf-8") as f:
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
2026-06-17 10:59:57 | INFO | ================================================================================
2026-06-17 10:59:57 | INFO | Hello from data-move Python data pipeline!
2026-06-17 10:59:57 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 10:59:57 | INFO | Connecting to databases...
2026-06-17 10:59:58 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001A32C2078B0>
2026-06-17 11:00:00 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001A32D648910>
2026-06-17 11:00:00 | INFO | Database connections established
2026-06-17 11:00:00 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 11:00:01 | INFO | Found 863 MIDs
2026-06-17 11:00:02 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 11:00:24 | INFO | Fetched 291,011 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 11:00:25 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 11:00:25 | INFO | ================================================================================
2026-06-17 11:00:25 | INFO | Processing Table: OQaD
2026-06-17 11:00:25 | ERROR | Failed processing table OQaD
Traceback (most recent call last):
File "D:\data_move\main2.py", line 217, in main
df=fetch_OQaD( engine=sql_engine,
table_name=table_name,
...<2 lines>...
run_date=run_date
)
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
with open(sql_file, "r", encoding="utf-8") as f:
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\fact\\OQaD.sql'
2026-06-17 12:09:34 | INFO | ================================================================================
2026-06-17 12:09:34 | INFO | Hello from data-move Python data pipeline!
2026-06-17 12:09:34 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 12:09:34 | INFO | Connecting to databases...
2026-06-17 12:09:37 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000274B86078B0>
2026-06-17 12:09:39 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000274B9A18910>
2026-06-17 12:09:39 | INFO | Database connections established
2026-06-17 12:09:39 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 12:09:41 | INFO | Found 863 MIDs
2026-06-17 12:09:43 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 12:10:48 | INFO | Fetched 291,093 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 12:10:48 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 12:10:48 | INFO | ================================================================================
2026-06-17 12:10:48 | INFO | Processing Table: OQaD
2026-06-17 12:10:48 | INFO | Exists: False
2026-06-17 12:10:48 | INFO | Path: D:\data_move\sql\fact\OQaD.sql
2026-06-17 12:10:48 | ERROR | Failed processing table OQaD
Traceback (most recent call last):
File "D:\data_move\main2.py", line 217, in main
df=fetch_OQaD( engine=sql_engine,
table_name=table_name,
...<2 lines>...
run_date=run_date
)
File "D:\data_move\src\fact.py", line 62, in fetch_OQaD
with open(sql_file, "r", encoding="utf-8") as f:
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\fact\\OQaD.sql'
2026-06-17 12:11:47 | INFO | ================================================================================
2026-06-17 12:11:47 | INFO | Hello from data-move Python data pipeline!
2026-06-17 12:11:47 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 12:11:47 | INFO | Connecting to databases...
2026-06-17 12:11:48 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000241C8DF78B0>
2026-06-17 12:11:50 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000241CA238910>
2026-06-17 12:11:51 | INFO | Database connections established
2026-06-17 12:11:51 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 12:11:52 | INFO | Found 863 MIDs
2026-06-17 12:11:52 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 12:12:14 | INFO | Fetched 291,097 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 12:12:14 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 12:12:14 | INFO | ================================================================================
2026-06-17 12:12:14 | INFO | Processing Table: OQaD
2026-06-17 12:12:14 | INFO | Exists: True
2026-06-17 12:12:14 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
2026-06-17 12:12:14 | INFO | Fetching data for 893 EMPIDs
2026-06-17 12:12:14 | INFO | Fetching OQaD data for run_date=2026-06-16
2026-06-17 12:12:14 | ERROR | Failed processing table OQaD
Traceback (most recent call last):
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context
self.dialect.do_execute(
~~~~~~~~~~~~~~~~~~~~~~~^
cursor, str_statement, effective_parameters, context
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute
cursor.execute(statement, parameters)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near 'QUIZ'. (102) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'where'. (156)")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "D:\data_move\main2.py", line 217, in main
df=fetch_OQaD( engine=sql_engine,
table_name=table_name,
...<2 lines>...
run_date=run_date
)
File "D:\data_move\src\fact.py", line 74, in fetch_OQaD
df = pl.read_database(
query=sql,
connection=engine,
)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database
return cx.execute(
~~~~~~~~~~^
query=query,
^^^^^^^^^^^^
options=execute_options,
^^^^^^^^^^^^^^^^^^^^^^^^
).to_polars(
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute
result = cursor_execute(query, **options)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute
return meth(
self,
distilled_parameters,
execution_options or NO_OPTIONS,
)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection
return connection._execute_clauseelement(
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
self, distilled_params, execution_options
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement
ret = self._execute_context(
dialect,
...<8 lines>...
cache_hit=cache_hit,
)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context
return self._exec_single_context(
~~~~~~~~~~~~~~~~~~~~~~~~~^
dialect, context, statement, parameters
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context
self._handle_dbapi_exception(
~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
e, str_statement, effective_parameters, cursor, context
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context
self.dialect.do_execute(
~~~~~~~~~~~~~~~~~~~~~~~^
cursor, str_statement, effective_parameters, context
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute
cursor.execute(statement, parameters)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near 'QUIZ'. (102) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'where'. (156)")
[SQL: WITH MID_TABLE_COV1 AS
(
SELECT EmpId, VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE CreateDate >= 2026-06-16
AND CreateDate < DATEADD(DAY,1,'2026-06-16')
UNION ALL
SELECT EmpId, VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE UpdateDate >= 2026-06-16
AND UpdateDate < DATEADD(DAY,1, '2026-06-16')
),
OQaD
QUIZ AS
(
SELECT DISTINCT
E.EmpId,
E.EmpName,
E.SupervisorId,
E.SupervisorName,
E.DesignationName,
E.CityName,
E.StateName,
E.RegionName,
CAST(DQ.VisitDate AS DATE) AS VisitDate,
DQ.QuestionId,
DQ.AnswerId,
QC.QuestionCategoryId,
QC.QuestionCategory
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E
ON DQ.EmpId = E.EmpId
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU
ON DQ.QuestionId = QU.QuestionId
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC
ON QU.QuestionCategoryId = QC.QuestionCategoryId
WHERE E.EmpName NOT LIKE '%TEST%'
AND E.RightId = 6
AND (
E.ResignDate IS NULL
OR CAST(E.ResignDate AS DATE) >= '2026-06-16'
)
AND EXISTS
(
SELECT 1
FROM MID_TABLE_COV1 A
WHERE A.EmpId = DQ.EmpId
AND CAST(A.VisitDate AS DATE) = CAST(DQ.VisitDate AS DATE)
)
)
SELECT
40148 AS project_id,
Q.EmpId AS employee_id,
0 AS process_id,
Q.VisitDate AS visit_date,
Q.QuestionCategoryId AS question_category_id,
Q.QuestionCategory AS question_category,
QM.QuestionId AS question_id,
QM.Question AS question,
ISNULL(QA.AnswerId,0) AS answer_id,
ISNULL(QA.Answer,'') AS answer,
CASE
WHEN QA.AnswerId IS NULL THEN 'Not Answer'
WHEN QA.RightAnswer = 1 THEN 'Y'
WHEN QA.RightAnswer IS NULL THEN 'Not Answer'
ELSE 'N'
END AS correct_answer,
GETDATE() AS update_date,
'SP-Pius' AS update_by
FROM QUIZ Q
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM
ON Q.QuestionId = QM.QuestionId
LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA
ON Q.AnswerId = QA.AnswerId;
where Q.EmpId not in (3167,4724,4019,4902,4375,3352,3081,5244,3181,4810,5166,4895,4817,5173,5237,3608,4646,4126,3243,3948,3950,3599,3777,5235,5346,3606,3079,5175,5242,5064,4366,5071,5339,4812,4722,3169,5270,3326,3689,4062,3131,5054,5178,3371,4829,5185,4698,4836,3760,4665,5218,4451,4280,3269,3098,5327,4978,5261,3335,5090,5320,4214,4919,4570,3150,4221,3328,3112,3290,4950,4430,5104,3383,5213,4864,5199,5206,4439,3207,5033,4684,3979,5211,4774,3117,5102,5280,3309,4931,4753,5123,3155,5085,4914,5092,4387,5325,5154,4100,4805,3402,5187,3580,3409,3060,3962,4696,4831,3715,4710,4242,5282,5121,4933,5289,5282,3444,5104,3095,4933,4142,3437,3530,3088,3715,4840,5002,4831,5187,3361,3962,5113,3086,5106,5256,5085,4914,5092,3470,4387,5277,3309,5239,4712,4945,5246,5123,4774,3952,3603,4999,3727,3378,3207,5341,4992,5199,5334,4672,5206,3342,3520,3454,3171,4916,3105,4396,3112,3290,5265,3119,5227,3143,3150,5056,4529,3328,5313,5220,4693,5320,4615,3755,4444,3406,4978,4451,3746,4280,3399,5322,4617,4795,5218,3330,3159,4698,5144,5054,3473,3131,4726,4688,4743,3689,3812,3107,3169,5332,3347,5023,5022,4812,3380,3067,3950,4954,5132,3081,5296,5244,3140,4019,3492,4375,5210,3126,5289,3482,4847,5296,4954,4436,3380,5127,5305,3306,5291,5023,5120,4531,4346,5229,3107,3812,4353,3145,3152,3330,5322,4439,5144,4795,3363,4054,4232,3399,4239,3755,5279,4930,5213,3406,4864,5286,5220,4693,5101,5227,3985,3297,3636,4529,3420,5265,4916,3591,3335,4460,5343,4638,3342,5327,3520,3171,4467,3700,5336,4987,3515,4992,3760,5267,4562,3247,5082,4555,4733,5061,5239,3090,4712,5246,4904,5270,3059,4866,5277,4750,5106,3361,4968,5175,5310,3496,3608,4840,5248,4543,3615,4142,3437,3088,3444,3078,4902,3095,3567,4543,4450,3140,5319,4614,3496,4970,4443,5132,3916,3149,5310,3327,5312,3156,4968,4894,3394,5065,4538,4688,4866,3251,3090,3268,4671,4849,4904,4726,5082,4555,5191,4733,4562,3515,5300,4951,5115,3173,5293,4944,4460,4987,4638,4467,3591,3064,4526,3821,3232,3420,5056,5046,5008,3985,3636,5350,3109,5272,5101,5279,3358,4396,5172,5110,4054,4232,5281,3363,4999,3638,3603,3104,5027,5205,4856,4151,3083,4346,3764,3819,3415,4353,3470,3299,5167,3346,5113,3306,5331,5291,5160,5120,5127,4913,5091,4564,5262,5196,4847,5077,4666,4139,3612,3978,3078,5340,4139,3612,4991,5248,3170,5155,5333,4628,5077,4217,5084,4032,4913,4564,5257,5324,4381,5331,5250,5160,4134,4312,4141,3059,3764,3415,4305,5027,3220,3104,5321,4972,5307,5267,5136,5314,4787,3315,3322,4540,3835,3700,3486,4547,5172,5001,5008,3427,3379,5046,3372,3821,4526,3650,5124,5302,5286,3130,5115,4937,3132,5293,5117,3118,4239,5300,4951,4343,3897,5219,5191,4671,4849,5212,5229,5003,5181,4531,5065,5345,3538,4538,4189,4229,3531,5276,5305,4578,3106,3149,5312,5269,3113,4436,3099,4443,4970,3916,3567,3975,4153,4844,5210,5200,4844,3533,4666,5022,3184,5200,4402,4153,3704,3975,4922,3099,5262,4751,5091,3106,5269,5093,5276,4578,3346,5202,4675,4229,3531,3182,5195,5167,3203,5003,5205,5150,5328,3220,4979,4856,5157,4452,5335,3514,5219,3638,4343,4383,3289,4390,3118,3296,5281,5110,3438,3267,5074,5117,5252,3780,3431,5005,3372,3379,5176,3232,5316,4611,4967,3068,5309,3139,3495,3673,4540,3835,4891,3486,3488,3315,3322,5307,5136,3070,4877,4787,5314,5055,3939,3234,4072,5181,4305,3251,3374,5119,4134,3080,5297,4065,4894,3127,5250,4763,4941,5257,3298,4381,3327,3156,4687,4338,5221,4450,5155,3400,5333,5207,4628,5162,3742,4098,3742,4943,5121,3129,5207,5114,5292,4687,3122,4338,4696,3464,3113,4333,3298,3127,5031,4941,4763,5209,5119,3388,3566,3538,4065,3744,4072,3374,4753,3234,3939,5102,5088,5266,3070,5055,4877,3096,3979,3132,3488,3139,4841,5019,5302,3495,5124,5309,5012,4611,3369,3509,5176,5145,5323,5005,3153,4656,4371,3431,5254,5252,4547,3091,5074,4198,3782,3433,4383,4651,4829,4302,3953,5349,5321,3514,4972,5178,5150,5328,4979,5342,4452,5157,5304,3203,5195,3490,4312,3141,5126,4141,4675,4366,5057,5235,5064,3243,5093,4217,4922,3407,4751,4810,3170,4283,3756,3704,4974,3533,4991,3184,4974,3492,3143,4098,4283,3400,3756,3407,4710,5221,3067,3580,5057,3409,5196,3060,4805,4100,3402,5325,5297,3490,3141,5154,5126,3155,5304,4931,5109,5287,4117,5280,3473,4651,4302,3953,5033,5211,3782,4684,3433,3084,4891,3746,3383,4371,4022,5254,3673,5313,4207,3153,5316,3509,5145,5323,4430,4919,4570,3369,4221,3105,5090,4563,5012,3269,4841,3098,5019,3454,5261,3438,4672,3267,4836,3096,4665,5259,5088,3727,5266,3378,5185,4390,4397,3371,4062,4945,3528,3326,4722,3388,5256,3566,3744,5202,5242,3086,5071,5209,3777,4333,3606,3079,5002,3948,4126,3599,3464,5344,4817,5173,5351,4646,3122,3352,3530,5337,3181,3359,5166,5292,5114,4943)
-- select d.employee_id from OQaD d where project_id=40148 and d.employee_id=Q.EmpId and CONVERT(date,d.visit_date,101)=
-- CONVERT(date,q.VisitDate,101))
]
(Background on this error at: https://sqlalche.me/e/20/f405)
2026-06-17 12:19:35 | INFO | ================================================================================
2026-06-17 12:19:35 | INFO | Hello from data-move Python data pipeline!
2026-06-17 12:19:35 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 12:19:35 | INFO | Connecting to databases...
2026-06-17 12:19:37 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000210484378B0>
2026-06-17 12:19:38 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000021049928910>
2026-06-17 12:19:39 | INFO | Database connections established
2026-06-17 12:19:39 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 12:19:40 | INFO | Found 863 MIDs
2026-06-17 12:19:40 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 12:20:14 | INFO | Fetched 291,100 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 12:20:14 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 12:20:14 | INFO | ================================================================================
2026-06-17 12:20:14 | INFO | Processing Table: OQaD
2026-06-17 12:20:14 | INFO | Exists: True
2026-06-17 12:20:14 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
2026-06-17 12:20:14 | INFO | Fetching data for 893 EMPIDs
2026-06-17 12:20:14 | INFO | Fetching OQaD data for run_date=2026-06-16
2026-06-17 12:20:30 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
2026-06-17 12:20:30 | ERROR | Failed processing table OQaD
Traceback (most recent call last):
File "D:\data_move\main2.py", line 226, in main
df = get_dataframe(
fn_name=fn_name,
...<3 lines>...
run_date=run_date,
)
File "D:\data_move\main2.py", line 81, in get_dataframe
return fn(sql_engine, run_date)
TypeError: fetch_OQaD() missing 3 required positional arguments: 'table_type', 'empids', and 'run_date'
2026-06-17 12:38:43 | INFO | ================================================================================
2026-06-17 12:38:43 | INFO | Hello from data-move Python data pipeline!
2026-06-17 12:38:43 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 12:38:43 | INFO | Connecting to databases...
2026-06-17 12:38:45 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001C35AC2F750>
2026-06-17 12:38:46 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001C35BDF87D0>
2026-06-17 12:38:46 | INFO | Database connections established
2026-06-17 12:38:46 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 12:38:47 | INFO | Found 863 MIDs
2026-06-17 12:38:47 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 12:38:58 | INFO | Fetched 291,112 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 12:38:58 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 12:38:58 | INFO | ================================================================================
2026-06-17 12:38:58 | INFO | Processing Table: OQaD
2026-06-17 12:38:58 | INFO | Exists: True
2026-06-17 12:38:58 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
2026-06-17 12:38:58 | INFO | Fetching data for 893 EMPIDs
2026-06-17 12:38:58 | INFO | Fetching OQaD data for run_date=2026-06-16
2026-06-17 12:39:05 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
2026-06-17 12:39:05 | INFO | ================================================================================
2026-06-17 12:39:05 | INFO | Pipeline Completed Successfully
2026-06-17 12:39:05 | INFO | ================================================================================
2026-06-17 12:39:30 | INFO | ================================================================================
2026-06-17 12:39:30 | INFO | Hello from data-move Python data pipeline!
2026-06-17 12:39:30 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 12:39:30 | INFO | Connecting to databases...
2026-06-17 12:39:31 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000025D1633F750>
2026-06-17 12:39:32 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000025D175887D0>
2026-06-17 12:39:33 | INFO | Database connections established
2026-06-17 12:39:33 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 12:39:33 | INFO | Found 863 MIDs
2026-06-17 12:39:33 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 12:39:43 | INFO | Fetched 291,112 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 12:39:43 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 12:39:43 | INFO | ================================================================================
2026-06-17 12:39:43 | INFO | Processing Table: OQaD
2026-06-17 12:39:43 | INFO | Exists: True
2026-06-17 12:39:43 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
2026-06-17 12:39:43 | INFO | Fetching data for 893 EMPIDs
2026-06-17 12:39:43 | INFO | Fetching OQaD data for run_date=2026-06-16
2026-06-17 12:39:46 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
2026-06-17 12:39:46 | INFO | ================================================================================
2026-06-17 12:39:46 | INFO | Pipeline Completed Successfully
2026-06-17 12:39:46 | INFO | ================================================================================
2026-06-17 12:44:50 | INFO | ================================================================================
2026-06-17 12:44:50 | INFO | Hello from data-move Python data pipeline!
2026-06-17 12:44:50 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 12:44:50 | INFO | Connecting to databases...
2026-06-17 12:44:51 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E3647178B0>
2026-06-17 12:44:52 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E365BB8910>
2026-06-17 12:44:53 | INFO | Database connections established
2026-06-17 12:44:53 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 12:44:53 | INFO | Found 863 MIDs
2026-06-17 12:44:53 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 12:45:07 | INFO | Fetched 291,115 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 12:45:08 | INFO | Fetched 893 matched empids fetched for OQAD
2026-06-17 12:45:08 | INFO | ================================================================================
2026-06-17 12:45:08 | INFO | Processing Table: OQaD
2026-06-17 12:45:08 | INFO | Exists: True
2026-06-17 12:45:08 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
2026-06-17 12:45:08 | INFO | Fetching data for 893 EMPIDs
2026-06-17 12:45:08 | INFO | Fetching OQaD data for run_date=2026-06-16
2026-06-17 12:45:10 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
2026-06-17 12:45:10 | INFO | Fetched 4998 rows
2026-06-17 12:45:10 | INFO | Deleting from OQaD
2026-06-17 12:45:11 | INFO | OQaD: inserted 4,998 rows into ClickHouse
2026-06-17 12:45:11 | INFO | OQaD loaded successfully (4998 rows)
2026-06-17 12:45:11 | INFO | ================================================================================
2026-06-17 12:45:11 | INFO | Pipeline Completed Successfully
2026-06-17 12:45:11 | INFO | ================================================================================
2026-06-17 13:23:53 | INFO | ================================================================================
2026-06-17 13:23:53 | INFO | Hello from data-move Python data pipeline!
2026-06-17 13:23:53 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 13:23:53 | INFO | Connecting to databases...
2026-06-17 13:23:54 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000281306878B0>
2026-06-17 13:23:56 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000028131AD8910>
2026-06-17 13:23:56 | INFO | Database connections established
2026-06-17 13:23:56 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 13:23:57 | INFO | Found 863 MIDs
2026-06-17 13:23:58 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 13:24:15 | INFO | Fetched 291,128 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 13:24:15 | INFO | Fetched 4,819 matched empids fetched for OQAD
2026-06-17 13:24:15 | INFO | ================================================================================
2026-06-17 13:24:15 | INFO | Processing Table: OQaD
2026-06-17 13:24:15 | INFO | Exists: True
2026-06-17 13:24:15 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
2026-06-17 13:24:15 | INFO | Fetching data for 4,819 EMPIDs
2026-06-17 13:24:15 | INFO | Fetching OQaD data for run_date=2026-06-16
2026-06-17 13:24:22 | INFO | fn name is fetch_OQad ------Fetched 1072 rows
2026-06-17 13:24:22 | INFO | Fetched 1072 rows
2026-06-17 13:24:22 | INFO | Deleting from OQaD
2026-06-17 13:24:23 | INFO | OQaD: inserted 1,072 rows into ClickHouse
2026-06-17 13:24:23 | INFO | OQaD loaded successfully (1072 rows)
2026-06-17 13:24:23 | INFO | ================================================================================
2026-06-17 13:24:23 | INFO | Processing Table: Survey
2026-06-17 13:24:23 | INFO | Fetching data for 863 MIDs
2026-06-17 13:24:23 | INFO | Fetched 142 rows from SQL Server
2026-06-17 13:24:23 | INFO | Fetched 142 rows
2026-06-17 13:24:23 | INFO | Deleting from Survey
2026-06-17 13:24:24 | ERROR | Failed processing table Survey
Traceback (most recent call last):
File "D:\data_move\main2.py", line 299, in main
load_to_clickhouse(
~~~~~~~~~~~~~~~~~~^
client=client,
^^^^^^^^^^^^^^
table_name=table_name,
^^^^^^^^^^^^^^^^^^^^^^
df=df,
^^^^^^
)
^
File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse
client.insert_arrow(
~~~~~~~~~~~~~~~~~~~^
table=table_name,
^^^^^^^^^^^^^^^^^
arrow_table=arrow_table,
^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow
return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings)
~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert
response = self._raw_request(insert_block, params, headers, server_wait=False)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request
self._error_handler(response)
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler
raise err_type(err_str, code=code, name=name) from None
clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column Project_Id in table kelloggs_1.Survey (d8c44908-2ed7-4d29-bd10-8e70fa84dfba). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123)
2026-06-17 13:27:22 | INFO | ================================================================================
2026-06-17 13:27:22 | INFO | Hello from data-move Python data pipeline!
2026-06-17 13:27:22 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 13:27:22 | INFO | Connecting to databases...
2026-06-17 13:27:23 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000026D13D878B0>
2026-06-17 13:27:25 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000026D151D8910>
2026-06-17 13:27:25 | INFO | Database connections established
2026-06-17 13:27:25 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 13:27:26 | INFO | Found 863 MIDs
2026-06-17 13:27:26 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 13:27:37 | INFO | Fetched 291,128 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 13:27:38 | INFO | Fetched 4,814 matched empids fetched for OQAD
2026-06-17 13:27:38 | INFO | ================================================================================
2026-06-17 13:27:38 | INFO | Processing Table: OQaD
2026-06-17 13:27:38 | INFO | Exists: True
2026-06-17 13:27:38 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
2026-06-17 13:27:38 | INFO | Fetching data for 4,814 EMPIDs
2026-06-17 13:27:38 | INFO | Fetching OQaD data for run_date=2026-06-16
2026-06-17 13:27:44 | INFO | fn name is fetch_OQad ------Fetched 1072 rows
2026-06-17 13:27:44 | INFO | Fetched 1072 rows
2026-06-17 13:27:44 | INFO | Deleting from OQaD
2026-06-17 13:27:45 | INFO | OQaD: inserted 1,072 rows into ClickHouse
2026-06-17 13:27:45 | INFO | OQaD loaded successfully (1072 rows)
2026-06-17 13:27:45 | INFO | ================================================================================
2026-06-17 13:27:45 | INFO | Processing Table: Survey
2026-06-17 13:27:45 | INFO | Fetching data for 863 MIDs
2026-06-17 13:27:45 | INFO | Fetched 142 rows from SQL Server
2026-06-17 13:27:45 | INFO | Fetched 142 rows
2026-06-17 13:27:45 | INFO | Creating table Survey
2026-06-17 13:27:45 | INFO | Table ready: Survey
2026-06-17 13:27:45 | INFO | Survey: inserted 142 rows into ClickHouse
2026-06-17 13:27:45 | INFO | Survey loaded successfully (142 rows)
2026-06-17 13:27:45 | INFO | ================================================================================
2026-06-17 13:27:45 | INFO | Pipeline Completed Successfully
2026-06-17 13:27:45 | INFO | ================================================================================
2026-06-17 15:05:42 | INFO | ================================================================================
2026-06-17 15:05:42 | INFO | Hello from data-move Python data pipeline!
2026-06-17 15:05:42 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 15:05:42 | INFO | Connecting to databases...
2026-06-17 15:05:43 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002009FB778B0>
2026-06-17 15:05:45 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000200A0FB8910>
2026-06-17 15:05:45 | INFO | Database connections established
2026-06-17 15:05:45 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 15:05:45 | INFO | Found 863 MIDs
2026-06-17 15:05:46 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 15:05:57 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 15:05:57 | INFO | Fetched 4,814 matched empids fetched for OQAD
2026-06-17 15:05:57 | INFO | ================================================================================
2026-06-17 15:05:57 | INFO | Processing Table: additional_visibility
2026-06-17 15:05:57 | INFO | Fetching data for 863 MIDs
2026-06-17 15:05:58 | ERROR | Failed processing table additional_visibility
Traceback (most recent call last):
File "D:\data_move\main2.py", line 228, in main
df = fetch_data(
engine=sql_engine,
...<3 lines>...
run_date=run_date,
)
File "D:\data_move\src\fact.py", line 34, in fetch_data
df = pl.read_database(
query=sql,
connection=engine
)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 287, in read_database
).to_polars(
~~~~~~~~~^
batch_size=batch_size,
^^^^^^^^^^^^^^^^^^^^^^
...<2 lines>...
infer_schema_length=infer_schema_length,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 588, in to_polars
frame = frame_init(
batch_size=batch_size,
...<2 lines>...
infer_schema_length=infer_schema_length,
)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 285, in _from_rows
(d[0], d[1:]) for d in self.result.cursor.description
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AttributeError: 'NoneType' object has no attribute 'description'
2026-06-17 15:07:15 | INFO | ================================================================================
2026-06-17 15:07:15 | INFO | Hello from data-move Python data pipeline!
2026-06-17 15:07:15 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 15:07:15 | INFO | Connecting to databases...
2026-06-17 15:07:16 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001BD510978B0>
2026-06-17 15:07:17 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001BD52518910>
2026-06-17 15:07:18 | INFO | Database connections established
2026-06-17 15:07:18 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 15:07:18 | INFO | Found 863 MIDs
2026-06-17 15:07:18 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 15:07:31 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 15:07:31 | INFO | Fetched 4,814 matched empids fetched for OQAD
2026-06-17 15:07:31 | INFO | ================================================================================
2026-06-17 15:07:31 | INFO | Processing Table: additional_visibility
2026-06-17 15:07:31 | INFO | Fetching data for 863 MIDs
2026-06-17 15:07:32 | INFO | Fetched 1,773 rows from SQL Server
2026-06-17 15:07:32 | INFO | Fetched 1773 rows
2026-06-17 15:07:32 | INFO | Deleting from additional_visibility
2026-06-17 15:07:32 | ERROR | Failed processing table additional_visibility
Traceback (most recent call last):
File "D:\data_move\main2.py", line 299, in main
load_to_clickhouse(
~~~~~~~~~~~~~~~~~~^
client=client,
^^^^^^^^^^^^^^
table_name=table_name,
^^^^^^^^^^^^^^^^^^^^^^
df=df,
^^^^^^
)
^
File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse
client.insert_arrow(
~~~~~~~~~~~~~~~~~~~^
table=table_name,
^^^^^^^^^^^^^^^^^
arrow_table=arrow_table,
^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow
return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings)
~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert
response = self._raw_request(insert_block, params, headers, server_wait=False)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request
self._error_handler(response)
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler
raise err_type(err_str, code=code, name=name) from None
clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column project_id in table kelloggs_1.additional_visibility (9e03e625-028a-47db-8f60-e8ab93e907ea). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123)
2026-06-17 15:08:55 | INFO | ================================================================================
2026-06-17 15:08:55 | INFO | Hello from data-move Python data pipeline!
2026-06-17 15:08:55 | INFO | Pipeline Run Date: 2026-06-16
2026-06-17 15:08:55 | INFO | Connecting to databases...
2026-06-17 15:08:56 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001ED5A4C78B0>
2026-06-17 15:08:58 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001ED5B948910>
2026-06-17 15:08:58 | INFO | Database connections established
2026-06-17 15:08:58 | INFO | Collecting MIDs for: 2026-06-16
2026-06-17 15:08:59 | INFO | Found 863 MIDs
2026-06-17 15:08:59 | INFO | Fetching quiz_empids data for EMPID and Visitid
2026-06-17 15:09:12 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server
2026-06-17 15:09:12 | INFO | Fetched 4,814 matched empids fetched for OQAD
2026-06-17 15:09:12 | INFO | ================================================================================
2026-06-17 15:09:12 | INFO | Processing Table: additional_visibility
2026-06-17 15:09:12 | INFO | Fetching data for 863 MIDs
2026-06-17 15:09:14 | INFO | Fetched 1,773 rows from SQL Server
2026-06-17 15:09:14 | INFO | Fetched 1773 rows
2026-06-17 15:09:14 | INFO | Creating table additional_visibility
2026-06-17 15:09:14 | INFO | Table ready: additional_visibility
2026-06-17 15:09:15 | INFO | additional_visibility: inserted 1,773 rows into ClickHouse
2026-06-17 15:09:15 | INFO | additional_visibility loaded successfully (1773 rows)
2026-06-17 15:09:15 | INFO | ================================================================================
2026-06-17 15:09:15 | INFO | Pipeline Completed Successfully
2026-06-17 15:09:15 | INFO | ================================================================================
logs/etl_20260618.log
+2134
View File
File diff suppressed because one or more lines are too long
logs/etl_20260619.log
+1451
View File
File diff suppressed because it is too large Load Diff
logs/etl_20260622.log
+307
View File
@@ -0,0 +1,307 @@
2026-06-22 10:05:56 | INFO | ================================================================================
2026-06-22 10:05:56 | INFO | Hello from data-move Python data pipeline!
2026-06-22 10:05:56 | INFO | Pipeline Run Date: 2026-06-21
2026-06-22 10:05:56 | INFO | Connecting to databases...
2026-06-22 10:05:57 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001F93F950730>
2026-06-22 10:06:00 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001F9409FC7D0>
2026-06-22 10:06:03 | INFO | Database connections established
2026-06-22 10:06:03 | INFO | Collecting MIDs for: 2026-06-21
2026-06-22 10:06:05 | INFO | Found 569 MIDs
2026-06-22 10:06:07 | INFO | ================================================================================
2026-06-22 10:06:07 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
2026-06-22 10:06:07 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
2026-06-22 10:06:07 | INFO | Fetching data from sql server for Master table......
2026-06-22 10:06:07 | INFO | Fetching in progress ....
2026-06-22 10:06:31 | ERROR | Failed processing table Master_VisibilityReason
Traceback (most recent call last):
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context
self.dialect.do_execute(
~~~~~~~~~~~~~~~~~~~~~~~^
cursor, str_statement, effective_parameters, context
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute
cursor.execute(statement, parameters)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
pyodbc.OperationalError: ('08S01', '[08S01] [Microsoft][ODBC Driver 18 for SQL Server]TCP Provider: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond.\r\n (10060) (SQLExecDirectW); [08S01] [Microsoft][ODBC Driver 18 for SQL Server]Communication link failure (10060)')
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "D:\data_move\main2.py", line 161, in main
df = fetch_data(sql_engine ,table_name,table_type)
File "D:\data_move\src\dim.py", line 45, in fetch_data
df = pl.read_database(
query=sql,
connection=engine
)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database
return cx.execute(
~~~~~~~~~~^
query=query,
^^^^^^^^^^^^
options=execute_options,
^^^^^^^^^^^^^^^^^^^^^^^^
).to_polars(
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute
result = cursor_execute(query, **options)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute
return meth(
self,
distilled_parameters,
execution_options or NO_OPTIONS,
)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection
return connection._execute_clauseelement(
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
self, distilled_params, execution_options
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement
ret = self._execute_context(
dialect,
...<8 lines>...
cache_hit=cache_hit,
)
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context
return self._exec_single_context(
~~~~~~~~~~~~~~~~~~~~~~~~~^
dialect, context, statement, parameters
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context
self._handle_dbapi_exception(
~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
e, str_statement, effective_parameters, cursor, context
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context
self.dialect.do_execute(
~~~~~~~~~~~~~~~~~~~~~~~^
cursor, str_statement, effective_parameters, context
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute
cursor.execute(statement, parameters)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
sqlalchemy.exc.OperationalError: (pyodbc.OperationalError) ('08S01', '[08S01] [Microsoft][ODBC Driver 18 for SQL Server]TCP Provider: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond.\r\n (10060) (SQLExecDirectW); [08S01] [Microsoft][ODBC Driver 18 for SQL Server]Communication link failure (10060)')
[SQL: with Master_VisibilityReason (ProjectId,MenuId,ReasonId,Reason,CreateDate,Createby)
as (
select DISTINCT '40148' as ProjectId,MenuId,VisibilityReasonId,VisibilityReason,getdate(),'SP-Pius'
FROM OneApp_KelloggsMT.dbo.Master_VisibilityReason
)
select * from Master_VisibilityReason]
(Background on this error at: https://sqlalche.me/e/20/e3q8)
2026-06-22 10:10:52 | INFO | ================================================================================
2026-06-22 10:10:52 | INFO | Hello from data-move Python data pipeline!
2026-06-22 10:10:52 | INFO | Pipeline Run Date: 2026-06-21
2026-06-22 10:10:53 | INFO | Connecting to databases...
2026-06-22 10:10:53 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001500A41C730>
2026-06-22 10:10:55 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001500B56C7D0>
2026-06-22 10:10:56 | INFO | Database connections established
2026-06-22 10:10:56 | INFO | Collecting MIDs for: 2026-06-21
2026-06-22 10:10:56 | INFO | Found 569 MIDs
2026-06-22 10:10:57 | INFO | ================================================================================
2026-06-22 10:10:57 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
2026-06-22 10:10:57 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
2026-06-22 10:10:57 | INFO | Fetching data from sql server for Master table......
2026-06-22 10:10:57 | INFO | Fetching in progress ....
2026-06-22 10:10:57 | INFO | Fetched 17 rows from SQL Server
2026-06-22 10:10:57 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!!
2026-06-22 10:10:57 | INFO | Fetched 17 rows
2026-06-22 10:10:57 | INFO | Creating table Master_VisibilityReason
2026-06-22 10:10:57 | INFO | Table ready: Master_VisibilityReason
2026-06-22 10:10:57 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
2026-06-22 10:10:58 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse
2026-06-22 10:10:58 | INFO | Master_VisibilityReason loaded successfully (17 rows)
2026-06-22 10:10:58 | INFO | ================================================================================
2026-06-22 10:10:58 | INFO | Pipeline Completed Successfully
2026-06-22 10:10:58 | INFO | ================================================================================
2026-06-22 10:11:04 | INFO | ================================================================================
2026-06-22 10:11:04 | INFO | Hello from data-move Python data pipeline!
2026-06-22 10:11:04 | INFO | Pipeline Run Date: 2026-06-21
2026-06-22 10:11:04 | INFO | Connecting to databases...
2026-06-22 10:11:05 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002DFCC410730>
2026-06-22 10:11:06 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002DFCD4CC7D0>
2026-06-22 10:11:07 | INFO | Database connections established
2026-06-22 10:11:07 | INFO | Collecting MIDs for: 2026-06-21
2026-06-22 10:11:07 | INFO | Found 569 MIDs
2026-06-22 10:11:08 | INFO | ================================================================================
2026-06-22 10:11:08 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
2026-06-22 10:11:08 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
2026-06-22 10:11:08 | INFO | Fetching data from sql server for Master table......
2026-06-22 10:11:08 | INFO | Fetching in progress ....
2026-06-22 10:11:08 | INFO | Fetched 17 rows from SQL Server
2026-06-22 10:11:08 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!!
2026-06-22 10:11:08 | INFO | Fetched 17 rows
2026-06-22 10:11:08 | INFO | Truncated table Master_VisibilityReason
2026-06-22 10:11:08 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
2026-06-22 10:11:08 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse
2026-06-22 10:11:08 | INFO | Master_VisibilityReason loaded successfully (17 rows)
2026-06-22 10:11:08 | INFO | ================================================================================
2026-06-22 10:11:08 | INFO | Pipeline Completed Successfully
2026-06-22 10:11:08 | INFO | ================================================================================
2026-06-22 11:07:54 | INFO | ================================================================================
2026-06-22 11:07:54 | INFO | Hello from data-move Python data pipeline!
2026-06-22 11:07:54 | INFO | Pipeline Run Date: 2026-06-21
2026-06-22 11:07:54 | INFO | Connecting to databases...
2026-06-22 11:07:56 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001689BE04730>
2026-06-22 11:08:00 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001689CEBC7D0>
2026-06-22 11:08:01 | INFO | Database connections established
2026-06-22 11:08:01 | INFO | Collecting MIDs for: 2026-06-21
2026-06-22 11:08:02 | INFO | Found 569 MIDs
2026-06-22 11:08:42 | INFO | ================================================================================
2026-06-22 11:08:42 | INFO | Hello from data-move Python data pipeline!
2026-06-22 11:08:42 | INFO | Pipeline Run Date: 2026-06-21
2026-06-22 11:08:42 | INFO | Connecting to databases...
2026-06-22 11:08:44 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000015B77AE4730>
2026-06-22 11:08:46 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000015B78B8C7D0>
2026-06-22 11:08:47 | INFO | Database connections established
2026-06-22 11:08:47 | INFO | Collecting MIDs for: 2026-06-21
2026-06-22 11:08:48 | INFO | Found 569 MIDs
2026-06-22 11:08:48 | INFO | ================================================================================
2026-06-22 11:08:48 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
2026-06-22 11:08:48 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
2026-06-22 11:08:48 | INFO | Fetching data from sql server for Master table......
2026-06-22 11:08:48 | INFO | Fetching in progress ....
2026-06-22 11:08:48 | INFO | Fetched 17 rows from SQL Server
2026-06-22 11:08:48 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!!
2026-06-22 11:08:48 | INFO | Fetched 17 rows
2026-06-22 11:08:48 | INFO | Truncated table Master_VisibilityReason
2026-06-22 11:08:48 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
2026-06-22 11:08:49 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse
2026-06-22 11:08:49 | INFO | Master_VisibilityReason loaded successfully (17 rows)
2026-06-22 11:08:49 | INFO | ================================================================================
2026-06-22 11:08:49 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
2026-06-22 11:08:49 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition ..............
2026-06-22 11:08:49 | INFO | Fetching data from sql server for Master table......
2026-06-22 11:08:49 | INFO | Fetching in progress ....
2026-06-22 11:08:49 | INFO | Fetched 861 rows from SQL Server
2026-06-22 11:08:49 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!!
2026-06-22 11:08:49 | INFO | Fetched 861 rows
2026-06-22 11:08:49 | INFO | Creating table Master_VisibilityDefinition
2026-06-22 11:08:49 | INFO | Table ready: Master_VisibilityDefinition
2026-06-22 11:08:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
2026-06-22 11:08:49 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse
2026-06-22 11:08:49 | INFO | Master_VisibilityDefinition loaded successfully (861 rows)
2026-06-22 11:08:49 | INFO | ================================================================================
2026-06-22 11:08:49 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
2026-06-22 11:08:49 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer ..............
2026-06-22 11:08:49 | INFO | Fetching data from sql server for Master table......
2026-06-22 11:08:49 | INFO | Fetching in progress ....
2026-06-22 11:08:49 | INFO | Fetched 33 rows from SQL Server
2026-06-22 11:08:49 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!!
2026-06-22 11:08:49 | INFO | Fetched 33 rows
2026-06-22 11:08:50 | INFO | Creating table Master_Salesterritorylayer
2026-06-22 11:08:50 | INFO | Table ready: Master_Salesterritorylayer
2026-06-22 11:08:50 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
2026-06-22 11:08:50 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse
2026-06-22 11:08:50 | INFO | Master_Salesterritorylayer loaded successfully (33 rows)
2026-06-22 11:08:50 | INFO | ================================================================================
2026-06-22 11:08:50 | INFO | Pipeline Completed Successfully
2026-06-22 11:08:50 | INFO | ================================================================================
2026-06-22 11:08:56 | INFO | ================================================================================
2026-06-22 11:08:56 | INFO | Hello from data-move Python data pipeline!
2026-06-22 11:08:56 | INFO | Pipeline Run Date: 2026-06-21
2026-06-22 11:08:56 | INFO | Connecting to databases...
2026-06-22 11:08:57 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000214E3374730>
2026-06-22 11:08:59 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000214E44CC7D0>
2026-06-22 11:09:00 | INFO | Database connections established
2026-06-22 11:09:00 | INFO | Collecting MIDs for: 2026-06-21
2026-06-22 11:09:00 | INFO | Found 569 MIDs
2026-06-22 11:09:00 | INFO | ================================================================================
2026-06-22 11:09:00 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
2026-06-22 11:09:00 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
2026-06-22 11:09:00 | INFO | Fetching data from sql server for Master table......
2026-06-22 11:09:00 | INFO | Fetching in progress ....
2026-06-22 11:09:01 | INFO | Fetched 17 rows from SQL Server
2026-06-22 11:09:01 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!!
2026-06-22 11:09:01 | INFO | Fetched 17 rows
2026-06-22 11:09:01 | INFO | Truncated table Master_VisibilityReason
2026-06-22 11:09:01 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
2026-06-22 11:09:01 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse
2026-06-22 11:09:01 | INFO | Master_VisibilityReason loaded successfully (17 rows)
2026-06-22 11:09:01 | INFO | ================================================================================
2026-06-22 11:09:01 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
2026-06-22 11:09:01 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition ..............
2026-06-22 11:09:01 | INFO | Fetching data from sql server for Master table......
2026-06-22 11:09:01 | INFO | Fetching in progress ....
2026-06-22 11:09:02 | INFO | Fetched 861 rows from SQL Server
2026-06-22 11:09:02 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!!
2026-06-22 11:09:02 | INFO | Fetched 861 rows
2026-06-22 11:09:02 | INFO | Truncated table Master_VisibilityDefinition
2026-06-22 11:09:02 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
2026-06-22 11:09:02 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse
2026-06-22 11:09:02 | INFO | Master_VisibilityDefinition loaded successfully (861 rows)
2026-06-22 11:09:02 | INFO | ================================================================================
2026-06-22 11:09:02 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
2026-06-22 11:09:02 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer ..............
2026-06-22 11:09:02 | INFO | Fetching data from sql server for Master table......
2026-06-22 11:09:02 | INFO | Fetching in progress ....
2026-06-22 11:09:02 | INFO | Fetched 33 rows from SQL Server
2026-06-22 11:09:02 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!!
2026-06-22 11:09:02 | INFO | Fetched 33 rows
2026-06-22 11:09:02 | INFO | Truncated table Master_Salesterritorylayer
2026-06-22 11:09:02 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
2026-06-22 11:09:03 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse
2026-06-22 11:09:03 | INFO | Master_Salesterritorylayer loaded successfully (33 rows)
2026-06-22 11:09:03 | INFO | ================================================================================
2026-06-22 11:09:03 | INFO | Pipeline Completed Successfully
2026-06-22 11:09:03 | INFO | ================================================================================
2026-06-22 12:04:40 | INFO | ================================================================================
2026-06-22 12:04:40 | INFO | Hello from data-move Python data pipeline!
2026-06-22 12:04:40 | INFO | Pipeline Run Date: 2026-06-21
2026-06-22 12:04:40 | INFO | Connecting to databases...
2026-06-22 12:04:43 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000023A01CA4730>
2026-06-22 12:04:44 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000023A02D1C7D0>
2026-06-22 12:04:45 | INFO | Database connections established
2026-06-22 12:04:45 | INFO | Collecting MIDs for: 2026-06-21
2026-06-22 12:04:46 | INFO | Found 569 MIDs
2026-06-22 12:04:48 | INFO | ================================================================================
2026-06-22 12:04:48 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
2026-06-22 12:04:48 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
2026-06-22 12:04:48 | INFO | Fetching data from sql server for Master table......
2026-06-22 12:04:48 | INFO | Fetching in progress ....
2026-06-22 12:04:48 | INFO | Fetched 17 rows from SQL Server
2026-06-22 12:04:48 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!!
2026-06-22 12:04:48 | INFO | Fetched 17 rows
2026-06-22 12:04:48 | INFO | Truncated table Master_VisibilityReason
2026-06-22 12:04:48 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
2026-06-22 12:04:48 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse
2026-06-22 12:04:48 | INFO | Master_VisibilityReason loaded successfully (17 rows)
2026-06-22 12:04:48 | INFO | ================================================================================
2026-06-22 12:04:48 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
2026-06-22 12:04:48 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition ..............
2026-06-22 12:04:48 | INFO | Fetching data from sql server for Master table......
2026-06-22 12:04:48 | INFO | Fetching in progress ....
2026-06-22 12:04:49 | INFO | Fetched 861 rows from SQL Server
2026-06-22 12:04:49 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!!
2026-06-22 12:04:49 | INFO | Fetched 861 rows
2026-06-22 12:04:49 | INFO | Truncated table Master_VisibilityDefinition
2026-06-22 12:04:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
2026-06-22 12:04:49 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse
2026-06-22 12:04:49 | INFO | Master_VisibilityDefinition loaded successfully (861 rows)
2026-06-22 12:04:49 | INFO | ================================================================================
2026-06-22 12:04:49 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
2026-06-22 12:04:49 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer ..............
2026-06-22 12:04:49 | INFO | Fetching data from sql server for Master table......
2026-06-22 12:04:49 | INFO | Fetching in progress ....
2026-06-22 12:04:49 | INFO | Fetched 33 rows from SQL Server
2026-06-22 12:04:49 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!!
2026-06-22 12:04:49 | INFO | Fetched 33 rows
2026-06-22 12:04:49 | INFO | Truncated table Master_Salesterritorylayer
2026-06-22 12:04:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
2026-06-22 12:04:50 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse
2026-06-22 12:04:50 | INFO | Master_Salesterritorylayer loaded successfully (33 rows)
2026-06-22 12:04:50 | INFO | ================================================================================
2026-06-22 12:04:50 | INFO | Pipeline Completed Successfully
2026-06-22 12:04:50 | INFO | ================================================================================
main2.py
+19 -73
View File
@@ -43,9 +43,9 @@ from mids import (
MID_TABLE_COV1, MID_TABLE_COV1,
) )
from masters.dimensions import * from src.bridge import *
from masters.bridge import * from src.fact import *
from kpi.facts import * from src.dim import *
# ========================================================== # ==========================================================
@@ -64,53 +64,6 @@ def table_exists(
) )
def get_dataframe(
fn_name: str,
fetch_by: str,
sql_engine,
mids,
run_date,
):
fn = globals()[fn_name]
if fetch_by == "mids":
return fn(sql_engine, mids)
if fetch_by == "run_date":
return fn(sql_engine, run_date)
return fn(sql_engine)
def get_empids_clickhouse_OQAD(
client,
table_name: str = "OQaD",
) -> pl.DataFrame:
if not table_exists(client, table_name):
log.warning(f"Table '{table_name}' does not exist.")
return pl.DataFrame(
schema={
"EmpId": pl.Int64,
"VisitDate": pl.Date,
}
)
query = f"""
SELECT DISTINCT
employee_id AS EmpId,
toDate(visit_date) AS VisitDate
FROM {table_name}
"""
# ClickHouse -> PyArrow -> Polars
arrow_table = client.query_arrow(query)
return pl.from_arrow(arrow_table)
# ========================================================== # ==========================================================
# Main # Main
# ========================================================== # ==========================================================
@@ -166,20 +119,14 @@ def main():
sql_engine, sql_engine,
run_date, run_date,
) )
qf=fetch_quiz_empids()
db_df = get_empids_clickhouse_OQAD(client)
matched = qf.join(
db_df,
on=["EmpId", "VisitDate"],
how="inner",
)
# ------------------------------------------------------ # ------------------------------------------------------
# Config # Config
# ------------------------------------------------------ # ------------------------------------------------------
with open( with open(
"t.yml", "y.yml",
"r", "r",
) as file: ) as file:
@@ -188,19 +135,14 @@ def main():
# ------------------------------------------------------ # ------------------------------------------------------
# Process Tables # Process Tables
# ------------------------------------------------------ # ------------------------------------------------------
for table in config["tables"]: for table in config["tables"]:
table_name = table["name"] table_name = table["name"]
operation = table["operation"] operation = table["operation"]
fetch_by = table["fetch_by"] fetch_by = table["fetch_by"]
table_type=table["type"] table_type=table["type"]
log.info("=" * 80) log.info("=" * 80)
log.info( log.info(f"Processing Table: {table_name} | Table type is -: {table_type} | Based on -{fetch_by} and operation is used -{operation} " )
"Processing Table: %s",
table_name,
)
try: try:
@@ -208,15 +150,17 @@ def main():
# Fetch Data # Fetch Data
# ------------------------------------------ # ------------------------------------------
fn_name = f"fetch_{table_name}"
log.info(f"Fetching Data from sql server for table-: {table_name} ..............")
fetch_list=["mids" ,"run_date", "reason_id"]
if fetch_by in fetch_list :
fn_name = f"fetch_{table_name}"
fn = globals()[fn_name]
df=fn(sql_engine, table_name , table_type, mids, run_date)
else:
df = fetch_data(sql_engine ,table_name,table_type)
df = get_dataframe( log.info(f"Fetched total row -: {len(df)} from sql server for table-:{table_name} ...........!!!")
fn_name=fn_name,
fetch_by=fetch_by,
sql_engine=sql_engine,
mids=mids,
run_date=run_date,
)
if df.is_empty(): if df.is_empty():
@@ -267,6 +211,8 @@ def main():
table_name, table_name,
) )
elif operation =="ONLY_INSERT" :
continue
else: else:
delete_existing_data( delete_existing_data(
@@ -280,7 +226,7 @@ def main():
# ------------------------------------------ # ------------------------------------------
# Load Data # Load Data
# ------------------------------------------ # ------------------------------------------
log.info("_ _ _ _Inserting data into clickhouse db from sql server_ _ _ _")
load_to_clickhouse( load_to_clickhouse(
client=client, client=client,
table_name=table_name, table_name=table_name,
mids.py
-48
View File
@@ -72,51 +72,3 @@ def MID_TABLE_COV1(
def fetch_quiz_empids(engine: Engine, run_date : date) -> pl.DataFrame:
sql_template = f"""
WITH MID_TABLE_COV1 AS
(
SELECT EmpId, VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE CreateDate >= {run_date}
AND CreateDate < DATEADD(DAY,1,'{run_date}')
UNION ALL
SELECT EmpId, VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE UpdateDate >= {run_date}
AND UpdateDate < DATEADD(DAY,1, '{run_date}')
),
QUIZ AS
(
SELECT Distinct E.EmpId as empid
, CONVERT(VARCHAR,DQ.VisitDate) AS VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ INNER JOIN
OneApp_KelloggsMT.dbo.vw_Employee_Detail E ON DQ.EmpId = E.EmpId inner join
OneApp_KelloggsMT.dbo.Master_OQAD_Question QU on DQ.QuestionId= qu.QuestionId inner join
OneApp_KelloggsMT.dbo.Master_OQAD_Category qc on qu.QuestionCategoryId= qc.QuestionCategoryId
where e.EmpName not like 'test%' and e.RightId in (6)
and (E.ResignDate is null or E.ResignDate>=''+CONVERT(VARCHAR,'{run_date}')+'') AND E.EmpName NOT LIKE '%TEST%'
AND DQ.EmpId IN (SELECT EmpId FROM MID_TABLE_COV1 A WHERE
DQ.EmpId=A.EmpId AND CONVERT(date,VisitDate)=CONVERT(date,A.VisitDate) )
) select * from quiz
"""
sql = sql_template.format(
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching quiz_empids data for EMPID and Visitid")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} total empid and visitdate fetched for OQAD from SQL Server")
return df
src/bridge.py
+110
View File
@@ -0,0 +1,110 @@
from pathlib import Path
import polars as pl
from sqlalchemy import Engine
from datetime import date , timedelta
from log import log
from db_con.connection import (
build_sql_server_engine,
build_clickhouse_engine,
get_clickhouse_client,
)
def fetch_mapping_storevisibility(
sql_engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
run_date = run_date + timedelta(days=1)
client= get_clickhouse_client()
def table_exists(
client,
table_name: str,
) -> bool:
return bool(
client.command(
f"EXISTS TABLE {table_name}"
)
)
def get_reason_ids_mapping_storevisibility(
client,
run_date: date,
table_name: str = "mapping_storevisibility",
) -> list[int] :
if not table_exists(client, table_name):
log.warning(f"Table '{table_name}' does not exist. During collecting store_ids")
return [0]
query = f"""
SELECT DISTINCT StoreId
FROM mapping_storevisibility
WHERE toDate(Fromdate) <= toDate('{run_date + timedelta(days= 1)}')
AND toDate(Todate) >= toDate('{run_date + timedelta(days= 1)}')
AND project_Id = '40148'
"""
# ClickHouse -> PyArrow -> Polars
arrow_table = client.query_arrow(query)
df= pl.from_arrow(arrow_table)
list=df["reason_id"].to_list()
return list
def fetch_data(
engine: Engine,
table_name: str,
table_type: str,
run_date: date,
store_id: list[int]
) -> pl.DataFrame:
log.info(f"Fetching data from sql server for Master table......")
store_id_list = ",".join(str(sid) for sid in store_id)
sql_file = Path("src") / "sql" / f"bridge" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
store_id_list=store_id_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching in progress .... ")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
store_id=get_reason_ids_mapping_storevisibility(client, run_date, "mapping_storevisibility")
df=fetch_data(engine=sql_engine,
table_name=table_name,
table_type=table_type,
run_date=run_date,
store_id=store_id,
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
src/dim.py
+138
View File
@@ -0,0 +1,138 @@
from pathlib import Path
import polars as pl
from sqlalchemy import Engine
from datetime import date , timedelta
from log import log
from db_con.connection import (
build_sql_server_engine,
build_clickhouse_engine,
get_clickhouse_client,
)
def fetch_data(
engine: Engine,
table_name: str,
table_type: str
) -> pl.DataFrame:
log.info(f"Fetching data from sql server for Master table......")
sql_file = Path("src") / "sql" / f"dim" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format( )
log.info(f"Fetching in progress .... ")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
def fetch_coverage_remarks(
sql_engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
client= get_clickhouse_client()
def table_exists(
client,
table_name: str,
) -> bool:
return bool(
client.command(
f"EXISTS TABLE {table_name}"
)
)
def get_reason_ids_coverage_remarks(
client,
table_name: str = "coverage_remarks",
) -> list[int] :
if not table_exists(client, table_name):
log.warning(f"Table '{table_name}' does not exist. During collecting reason_ids")
return [0]
query = f"""
SELECT DISTINCT
reason_id
FROM {table_name}
"""
# ClickHouse -> PyArrow -> Polars
arrow_table = client.query_arrow(query)
df= pl.from_arrow(arrow_table)
list=df["reason_id"].to_list()
return list
def fetch_data(
engine: Engine,
table_name: str,
table_type: str,
reason_ids: list[int]
) -> pl.DataFrame:
log.info(f"Fetching data from sql server for Master table......")
resaon_id_list = ",".join(str(rid) for rid in reason_ids)
sql_file = Path("src") / "sql" / f"dim" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
resaon_id_list=resaon_id_list
)
log.info(f"Fetching in progress .... ")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
reason_ids=get_reason_ids_coverage_remarks(client, "coverage_remarks")
df=fetch_data(engine=sql_engine,
table_name=table_name,
table_type=table_type,
reason_ids=reason_ids,
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
src/fact.py
+621 -22
View File
@@ -1,25 +1,84 @@
from pathlib import Path from pathlib import Path
import polars as pl import polars as pl
from sqlalchemy import Engine from sqlalchemy import Engine
from datetime import date from datetime import date , timedelta
from log import log from log import log
def fetch_data(
engine: Engine,
from db_con.connection import (
build_sql_server_engine,
build_clickhouse_engine,
get_clickhouse_client,
)
# def fetch_data(
# engine: Engine,
# table_name: str,
# table_type: str,
# mids: list[int],
# run_date: date
# ) -> pl.DataFrame:
# if not mids:
# log.warning("No MIDs — nothing to fetch.")
# return pl.DataFrame()
# mid_list = ",".join(str(mid) for mid in mids)
# sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
# with open(sql_file, "r", encoding="utf-8") as f:
# sql_template = f.read()
# sql = sql_template.format(
# mid_list=mid_list,
# run_date=run_date.strftime("%Y-%m-%d")
# )
# log.info(f"Fetching data for {len(mids):,} MIDs")
# df = pl.read_database(
# query=sql,
# connection=engine
# )
# log.info(f"Fetched {len(df):,} rows from SQL Server")
# return df
def fetch_SOS_OneApp( engine: Engine,
table_name: str, table_name: str,
table_type: str, table_type: str,
mids: list[int], mids: list[int],
run_date: date run_date: date
) -> pl.DataFrame: ) -> pl.DataFrame:
if not mids: if not mids:
log.warning("No MIDs — nothing to fetch.") log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame() return pl.DataFrame()
log.info(f" Start Fetching data for these {len(mids):} MIDs ")
mid_list = ",".join(str(mid) for mid in mids) mid_list = ",".join(str(mid) for mid in mids)
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
sql_file = Path("sql") / f"{table_type}" / f"{table_name}.sql" sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f: with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read() sql_template = f.read()
@@ -28,14 +87,320 @@ def fetch_data(
mid_list=mid_list, mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d") run_date=run_date.strftime("%Y-%m-%d")
) )
log.info(f"Fetching data for {len(mids):,} MIDs")
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database( df = pl.read_database(
query=sql, query=sql,
connection=engine connection=engine
) )
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
def fetch_additional_visibility( engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
log.info(f" Start Fetching data for these {len(mids):} MIDs ")
mid_list = ",".join(str(mid) for mid in mids)
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
def fetch_OQaD(
sql_engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
client= get_clickhouse_client()
def table_exists(
client,
table_name: str,
) -> bool:
return bool(
client.command(
f"EXISTS TABLE {table_name}"
)
)
def fetch_quiz_empids(engine: Engine, run_date : date) -> pl.DataFrame:
sql_template = f"""
WITH MID_TABLE_COV1 AS
(
SELECT EmpId, VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE CreateDate >= {run_date}
AND CreateDate < DATEADD(DAY,1,'{run_date}')
UNION ALL
SELECT EmpId, VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE UpdateDate >= {run_date}
AND UpdateDate < DATEADD(DAY,1, '{run_date}')
),
QUIZ AS
(
SELECT Distinct E.EmpId as empid
, CONVERT(date,DQ.VisitDate) AS visitdate
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ INNER JOIN
OneApp_KelloggsMT.dbo.vw_Employee_Detail E ON DQ.EmpId = E.EmpId inner join
OneApp_KelloggsMT.dbo.Master_OQAD_Question QU on DQ.QuestionId= qu.QuestionId inner join
OneApp_KelloggsMT.dbo.Master_OQAD_Category qc on qu.QuestionCategoryId= qc.QuestionCategoryId
where e.EmpName not like 'test%' and e.RightId in (6)
and (E.ResignDate is null or E.ResignDate>=''+CONVERT(VARCHAR,'{run_date}')+'') AND E.EmpName NOT LIKE '%TEST%'
AND DQ.EmpId IN (SELECT EmpId FROM MID_TABLE_COV1 A WHERE
DQ.EmpId=A.EmpId AND CONVERT(date,VisitDate)=CONVERT(date,A.VisitDate) )
) select * from quiz
"""
sql = sql_template.format(
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching quiz_empids data for EMPID and Visitid")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} total empid and visitdate fetched for OQAD from SQL Server")
return df
def get_empids_clickhouse_OQAD(
client,
table_name: str = "OQaD",
) -> pl.DataFrame:
if not table_exists(client, table_name):
log.warning(f"Table '{table_name}' does not exist.")
return pl.DataFrame(
schema={
"empid": pl.Int64,
"visitdate": pl.Date,
}
)
query = f"""
SELECT DISTINCT
employee_id AS empid,
visit_date AS visitdate
FROM {table_name}
"""
# ClickHouse -> PyArrow -> Polars
arrow_table = client.query_arrow(query)
return pl.from_arrow(arrow_table)
qf=fetch_quiz_empids(sql_engine,run_date)
db_df = get_empids_clickhouse_OQAD(client)
matched = qf.join(
db_df,
on=["empid", "visitdate"],
how="inner",
)
empids=matched["empid"].to_list()
log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ")
def fetch_data(
engine: Engine,
table_name: str,
table_type: str,
empids: list[int],
run_date: date
) -> pl.DataFrame:
empid_list = ",".join(str(empid) for empid in empids)
sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql"
log.info(f"Exists: {sql_file.exists()}")
log.info(f"Path: {sql_file.resolve()}")
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
empid_list=empid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(empids):,} EMPIDs")
log.info("Fetching OQaD data for run_date=%s", run_date)
df = pl.read_database(
query=sql,
connection=engine,
)
log.info("fn name is fetch_OQad ------Fetched %s rows", len(df))
return df
df=fetch_data( engine=sql_engine,
table_name=table_name,
table_type=table_type,
empids=empids,
run_date=run_date
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
# def fetch_OQaD(
# engine: Engine,
# table_name: str,
# table_type: str,
# empids: list[int],
# run_date: date
# ) -> pl.DataFrame:
# empid_list = ",".join(str(empid) for empid in empids)
# sql_file = Path("src") / "sql" / "fact" / f"{table_name}.sql"
# log.info(f"Exists: {sql_file.exists()}")
# log.info(f"Path: {sql_file.resolve()}")
# with open(sql_file, "r", encoding="utf-8") as f:
# sql_template = f.read()
# sql = sql_template.format(
# empid_list=empid_list,
# run_date=run_date.strftime("%Y-%m-%d")
# )
# log.info(f"Fetching data for {len(empids):,} EMPIDs")
# log.info("Fetching OQaD data for run_date=%s", run_date)
# df = pl.read_database(
# query=sql,
# connection=engine,
# )
# log.info("fn name is fetch_OQad ------Fetched %s rows", len(df))
# return df
def fetch_Coverage( engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
mid_list = ",".join(str(mid) for mid in mids)
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
def fetch_Survey( engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
mid_list = ",".join(str(mid) for mid in mids)
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server") log.info(f"Fetched {len(df):,} rows from SQL Server")
return df return df
@@ -43,35 +408,269 @@ def fetch_data(
def fetch_OQaD(
engine: Engine, def fetch_Login( engine: Engine,
table_name: str, table_name: str,
table_type: str, table_type: str,
empids: list[int], mids: list[int],
run_date: date run_date: date
) -> pl.DataFrame: ) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
empid_list = ",".join(str(empid) for empid in empids) mid_list = ",".join(str(mid) for mid in mids)
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
sql_file = Path("sql") / table_type / f"{table_name}.sql" sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f: with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read() sql_template = f.read()
sql = sql_template.format( sql = sql_template.format(
empid_list=empid_list, mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d") run_date=run_date.strftime("%Y-%m-%d")
) )
log.info(f"Fetching data for {len(mids):,} MIDs")
log.info(f"Fetching data for {len(empids):,} EMPIDs")
log.info("Fetching OQaD data for run_date=%s", run_date)
df = pl.read_database( df = pl.read_database(
query=sql, query=sql,
connection=engine, connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
def fetch_Stock_Details( engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
mid_list = ",".join(str(mid) for mid in mids)
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
# def fetch_Coverage( engine: Engine,
# table_name: str,
# table_type: str,
# mids: list[int],
# run_date: date
# ) -> pl.DataFrame:
# if not mids:
# log.warning("No MIDs — nothing to fetch.")
# return pl.DataFrame()
# mid_list = ",".join(str(mid) for mid in mids)
# log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
# sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
# with open(sql_file, "r", encoding="utf-8") as f:
# sql_template = f.read()
# sql = sql_template.format(
# mid_list=mid_list,
# run_date=run_date.strftime("%Y-%m-%d")
# )
# log.info(f"Fetching data for {len(mids):,} MIDs")
# df = pl.read_database(
# query=sql,
# connection=engine
# )
# log.info(f"Fetched {len(df):,} rows from SQL Server")
# return df
def fetch_Attendance(
engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date | None = None,
days_back: int = 15
) -> pl.DataFrame:
"""
Fetch attendance source data.
Default:
end_date = yesterday
start_date = yesterday - 15 days
"""
if run_date is None:
run_date = date.today() - timedelta(days=1)
start_date = run_date - timedelta(days=days_back)
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
start_date=start_date.strftime("%Y-%m-%d"),
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(
f"Fetching Attendance data from {start_date} to {run_date}"
) )
log.info("Fetched %s rows", len(df)) df = pl.read_database(
query=sql,
connection=engine
)
log.info(
f"Fetched {len(df):,} attendance rows "
f"for {df['employee_id'].n_unique():,} employees"
)
return df
def fetch_Journey_Plan( engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
log.info(f" Start Fetching data for these {len(mids):} MIDs ")
mid_list = ",".join(str(mid) for mid in mids)
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
def fetch_PaidVisibility( engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
log.info(f" Start Fetching data for these {len(mids):} MIDs ")
mid_list = ",".join(str(mid) for mid in mids)
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df
def fetch_Web_Logins( engine: Engine,
table_name: str,
table_type: str,
mids: list[int],
run_date: date
) -> pl.DataFrame:
if not mids:
log.warning("No MIDs — nothing to fetch.")
return pl.DataFrame()
log.info(f" Start Fetching data for these {len(mids):} MIDs ")
mid_list = ",".join(str(mid) for mid in mids)
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
with open(sql_file, "r", encoding="utf-8") as f:
sql_template = f.read()
sql = sql_template.format(
mid_list=mid_list,
run_date=run_date.strftime("%Y-%m-%d")
)
log.info(f"Fetching data for {len(mids):,} MIDs")
df = pl.read_database(
query=sql,
connection=engine
)
log.info(f"Fetched {len(df):,} rows from SQL Server")
return df return df
src/sql/bridge/mapping_storevisibility.sql
+11
View File
@@ -0,0 +1,11 @@
with mapping_storevisibility
(Project_Id,StoreId,VisibilityDefinitionid,Fromdate,Todate,CreateDate,CreateBy)
AS (
select DISTINCT '40148' as Project_Id,StoreId,VisibilityDefinitionid,Fromdate,Todate,getdate(),'SP-Pius'
FROM OneApp_KelloggsMT.dbo.mapping_storevisibility z WHERE
convert(date,FROMDATE,101)<=convert(Date,getdate(),101) AND CONVERT(DATE,ToDate,101)>=convert(Date,getdate(),101)
AND z.VisibilityDefinitionid IN
(SELECT DISTINCT VisibilityDefinitionid FROM OneApp_KelloggsMT.dbo.MASTER_VISIBILITYDEFINITION WHERE MENUID=22 )
AND z.StoreId NOT IN ({store_id_list})
)
select * from mapping_storevisibility
src/sql/dim/Employee_Master.sql
+17
View File
@@ -0,0 +1,17 @@
with Employee_Master (project_id,region_id,region,state_id,state,city_id,
city,employee_id,employee_name,gender,designation_id,designation,manager_id,
manager_name,employee_joining_date,employee_resign_date,position_code,employee_legacy_code,employee_role,EMPLOYEE_TYPE)
as (
select '40148' as ProjectId, RegionId,
RegionName,StateId,StateName, CityId,CityName,
a.EmpId,EmpName,Gender,a.DesignationId,DesignationName,SupervisorId,SupervisorName,JoinDate,ResignDate,c.PositionCode ,EmpCode ,RIGHTNAME,
case when RIGHTNAME in ('Client','Client HO') Then 'NON CPM' else 'CPM' END AS EMPLOYEE_TYPE
from OneApp_KelloggsMT.dbo.vw_Employee_Detail a Left join (Select distinct PositionId, EmpId from OneApp_KelloggsMT.dbo.Mapping_PositionUser b
Where DATEdiff(d,FromDate, GETDATE())>= 0 and DATEDIFF(d,ToDate,getdate())<=0) b on a.EmpId=b.EmpId
LEFT join (select PositionId,PositionCode from OneApp_KelloggsMT.dbo.Master_Position) c on b.PositionId=c.PositionId
where 1=1
-- and EmpId not in (
--select employee_id from [dbo].[Employee_Master] where project_id='40148')
)
select * from Employee_Master
src/sql/dim/Master_Salesterritorylayer.sql
+14
View File
@@ -0,0 +1,14 @@
with Master_Salesterritorylayer(ProjectId,StLayerOneId,StLayerOneName,StLayerTwoId,StLayerTwoName,StLayerThreeId,StLayerThreeName,StLayerFourId,
StLayerFourName,CreateDate,CreateBy)
as (
select DISTINCT '40148' as Project_Id,d.StLayerOneId,d.StLayerOneName,c.StLayerTwoId,c.StLayerTwoName ,
b.StLayerThreeId,b.StLayerThreeName,a.StLayerFourId,a.StLayerFourName ,
getdate(),'SP-Pius'
FROM OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerFour] A
INNER JOIN OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerThree]B ON
A.StLayerThreeId=B.StLayerThreeId
INNER JOIN OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerTWo]C ON
B.StLayerTwoId=C.StLayerTwoId
INNER JOIN OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerOne]D ON
C.StLayerOneId=D.StLayerOneId)
select * from Master_Salesterritorylayer
src/sql/dim/Master_VisibilityDefinition.sql
+5
View File
@@ -0,0 +1,5 @@
with Master_VisibilityDefinition(Project_Id,VisibilityDefinitionId,VisibilityDefinitionName,CreateDate,CreateBy)
as (
select DISTINCT '40148' as Project_Id,VisibilityDefinitionid,VisibilityDefinitionname,getdate(),'SP-Pius'
FROM OneApp_KelloggsMT.dbo.Master_VisibilityDefinition)
select * from Master_VisibilityDefinition
src/sql/dim/Master_VisibilityReason.sql
+7
View File
@@ -0,0 +1,7 @@
with Master_VisibilityReason (ProjectId,MenuId,ReasonId,Reason,CreateDate,Createby)
as (
select DISTINCT '40148' as ProjectId,MenuId,VisibilityReasonId,VisibilityReason,getdate(),'SP-Pius'
FROM OneApp_KelloggsMT.dbo.Master_VisibilityReason
)
select * from Master_VisibilityReason
src/sql/dim/SKU_Master.sql
+18
View File
@@ -0,0 +1,18 @@
with SKU_Master (project_id,category_id,category_code,Category_name,sub_category_id,sub_category_code,sub_category_name,brand_id,
brand_code,brand_name,sub_brand_id,sub_brand_code,sub_brand_name,product_id,product_name,product_code,mrp,flavour_id,
flavour,grammage,product_sequence,case_size,company_name,is_competitor,ptr,update_date,update_by)
as (
Select '40148' as ProjectId, cm.CategoryId,
cm.CategoryCode,cm.CategoryName,sca.SubCategoryId,sca.SubCategoryCode,sca.SubCategoryName,br.BrandId,br.BrandCode,br.BrandName
,sb.SubBrandId,sb.SubBrandCode,sb.SubBrandName,p.ProductId,p.ProductName,p.ProductCode,p.Mrp,fl.FlavourId,
fl.Flavour,p.Grammage,p.ProductSequence,p.CaseSize,mc.Company,mc.IsCompetitor,p.ptr,GETDATE(),'MB'
from OneApp_KelloggsMT.dbo.Master_Product p Right join
OneApp_KelloggsMT.dbo.Master_Flavour fl on p.FlavourId= Fl.FlavourId Right join
OneApp_KelloggsMT.dbo.Master_subbrand sb on p.SubBrandId= sb.SubBrandId Right join
OneApp_KelloggsMT.dbo.Master_Brand br on sb.BrandId = br.BrandId Right join
OneApp_KelloggsMT.dbo.Master_SubCategory sca on br.SubCategoryId= sca.SubCategoryId Right join
OneApp_KelloggsMT.dbo.Master_Category cm on sca.CategoryId= cm.CategoryId Right join
OneApp_KelloggsMT.dbo.Master_Company mc on mc.companyId=br.companyId )
select * from SKU_Master
src/sql/dim/Store_Master.sql
+14
View File
@@ -0,0 +1,14 @@
with Store_Master (project_id,
region_id,region,state_id,state,city_id,city,cpm_city_id,channel_id,channel,distributor_id,distributor_name, keyaccount_id, keyaccount,
insight_store_id,client_store_code,latitude,longitude,store_category_id,store_category,store_type_id,store_type,store_classification_id,store_classification, StLayerFourId,
store_id,store_name,address)
AS (
select '40148',RegionId,RegionName,StateId,StateName,CityId,
CityName,CityCode,ChannelId,ChannelName,DistributorId,Distributor,ChainId,ChainName,StoreUniqueCode,StoreCode,Latitude,
Longitude, StoreCategoryId,StoreCategory,StoreTypeId,
StoreType,StoreClassId,StoreClass,StLayerFourId,StoreId,
StoreName,Address from OneApp_KelloggsMT.dbo.vw_storedetail where 1=1
--and storeid not in (
--select store_id from Store_Master where project_id='40148')
)
select * from Store_Master
src/sql/dim/coverage_remarks.sql
+8
View File
@@ -0,0 +1,8 @@
with coverage_remarks
(project_id,reason_id,reason_remarks)
as(
select
'40148' ,reasonid,reason from OneApp_KelloggsMT.dbo.master_nonworkingreason
where reasonid not in ( {resaon_id_list})
)
select * from coverage_remarks
src/sql/dim/display_master.sql
+8
View File
@@ -0,0 +1,8 @@
with display_master (project_id,display_id,display_code,display_name,display_ref_url,created_date,
created_by) as (
select '40148',displayid,displaycode,displayname,displayrefImage,getdate(),'Pius'
from OneApp_KelloggsMT.dbo.Master_display
-- and ProductId not in (
--select product_id from [dbo].[SKU Master] where project_id='40148')
)
select * from display_master
src/sql/fact/Attendance.sql
+134
View File
@@ -0,0 +1,134 @@
with Emp_cte as
(Select T1.*, ISNULL(ps.PositionCode ,'') PositionCode,FromDate,ToDate from(
select a.*,b.date from
(Select rm.RegionId, rm.RegionName as Region, st.StateId, st.StateName , cm.CityId, cm.CityName
, em2.Id as [Area Manager Id], em2.EmployeeName AS [Area Manager]
, em1.Id as [Supervisor Id], isnull(em1.EmployeeName,'') as [Supervisor Name]
, em.Id as [Employee Id], isnull(em.EmployeeName,'') as [Employee Name]
, dm.DesignationId, dm.DesignationName as Designation
, CAST(EM.ResignDate AS DATE) as DOR, CAST(EM.JoinDate AS DATE) as DOJ, em.LegacyCode
--,ISNULL(ps.PositionCode ,'') PositionCode--, FromDate, ToDate
from
OneApp_KelloggsMT.dbo.AspNetUsers em inner join
OneApp_KelloggsMT.dbo.AspNetUsers em1 on em1.Id = em.ManagerId inner join
OneApp_KelloggsMT.dbo.AspNetUsers em2 on em2.Id = em1.ManagerId inner join
OneApp_KelloggsMT.dbo.Master_City cm on em.CityId= cm.CityId inner join
OneApp_KelloggsMT.dbo.Master_State st on cm.StateId= st.StateId inner join
OneApp_KelloggsMT.dbo.Master_Region rm on st.RegionId= rm.RegionId inner join
OneApp_KelloggsMT.dbo.Master_Designation dm on Em.DesignationId= dm.DesignationId
where em.RightId in (6) and em.EmployeeName not like '%test%' and
(Em.ResignDate is null or Em.ResignDate>=CAST('{start_date}' AS DATE)) and CONVERT(date,em.JoinDate,101)<=CONVERT(date,CAST('{run_date}' AS DATE),101)
--and em.Id=2290
)a
cross join (select distinct DATE from DBO.GET_ALL_DAYS(CAST('{start_date}' AS DATE),CAST('{run_date}' AS DATE))) b
) AS T1 LEFT JOIN
(Select distinct EmpId, PositionId, Date,FromDate,ToDate from
(Select distinct EmpId,PositionId,FromDate,ToDate from OneApp_KelloggsMT.dbo.Mapping_PositionUser
Where convert(Date,FromDate)<= CONVERT(date,CAST('{run_date}' AS DATE),101) and convert(Date,ToDate)>=CONVERT(date,CAST('{start_date}' AS DATE),101)) as m Inner Join
(Select Date from OneApp_KelloggsMT.dbo.Master_Calender Where 1=1 and Date between CONVERT(date,CAST('{start_date}' AS DATE),101) and CONVERT(date,CAST('{run_date}' AS DATE),101))
cal on cal.Date >= m.FromDate AND cal.Date <= m.ToDate) pu
on T1.[Employee Id]=pu.EmpId and T1.date= pu.date Left Join
OneApp_KelloggsMT.dbo.Master_Position ps WITH (NOLOCK) ON pu.PositionId=ps.PositionId
) ,
Attendance
(project_id,
employee_id,position_code,legacy_code,supervisor_id,date_of_join,date_of_resign,visit_date,parinaam_attendance,
update_date,update_by,Unique_Id)
AS (
select '40148', emp.[Employee Id], Case When emp.DATE <= ToDate and emp.DATE>= FromDate
then emp.PositionCode Else '' End as PositionCode ,emp.LegacyCode, emp.[Supervisor Id] ,emp.DOJ,emp.DOR,
emp.DATE, case when emp.DATE>=emp.DOR then 'R' WHEN emp.DATE< emp.DOJ THEN '-'
WHEN b.COL IS NULL AND (emp.DATE<=emp.DOR OR emp.DOR IS NULL )AND (emp.DATE>=emp.DOJ OR emp.DOJ IS NULL) THEN 'W' ELSE b.COL
END AS Attendance,GETDATE(),'SP-Pius',CAST('40148' AS VARCHAR) + '_' + CAST(emp.[Employee Id] AS VARCHAR)
from Emp_cte as emp
left join
(select a.*
FROM
(
SELECT A.DATE,TT.[MERCHANDISER CD],TT.VisitDate,TT.DOJ1,TT.DOR1,TT.COL from DBO.GET_ALL_DAYS(CAST('{start_date}' AS DATE),CAST('{run_date}' AS DATE)) a
left join
(
SELECT distinct [EMPLOYEE ID] AS [MERCHANDISER CD], VISITDATE
, CASE WHEN COL=1 THEN 'H' WHEN COL=2 THEN 'L' WHEN COL=4 THEN 'M' WHEN COL=5 THEN 'CO' WHEN COL=6 THEN 'WO'
WHEN COL=7 THEN 'P' WHEN COL=10 THEN 'EH' WHEN COL=8 THEN 'HD' WHEN COL IS NULL THEN 'A' END AS COL
, DOJ AS DOJ1, DOR AS DOR1
FROM
(
SELECT DOJ, DOR, Region, [AREA MANAGER], SUPERVISOR_CD, [SUPERVISOR], City, [EMPLOYEE ID], MERCHANDISER, VISITDATE
, REMARKS, COL, ROW_NUMBER()OVER (PARTITION BY [EMPLOYEE ID], VISITDATE ORDER BY COL DESC,Intime, VISITDATE, ROW ) AS COL1
FROM
(
SELECT DOJ, DOR, Region, [AREA MANAGER], SUPERVISOR_CD, [SUPERVISOR], City, [EMPLOYEE ID], MERCHANDISER, InTime, VISITDATE
, REMARKS, COL, ROW_NUMBER()OVER (PARTITION BY [EMPLOYEE ID], VISITDATE ORDER BY COL) AS ROW
FROM
(
SELECT RM.RegionName as Region, em2.EmployeeName AS [Area Manager]
, em1.Id as Supervisor_cd, em1.EmployeeName AS [Supervisor]
, CM.CityName as City, EM.Id as [Employee Id], em.EmployeeName AS Merchandiser
,JP.VisitDate as VisitDate, sc.InTime
,ISNULL((SELECT top 1 case
when d.ReasonId IS NULL OR d.ReasonId =0 then '' else REASON+' - ' +
case when replace(replace(D.REMARK,CHAR(13),''),CHAR(10),' ')<>'' then replace(replace(D.REMARK,CHAR(13),''),CHAR(10),' ')
else replace(replace(D.REMARK,CHAR(13),''),CHAR(10),' ') end end as REASON FROM OneApp_KelloggsMT.dbo.T_StoreCoverage D
INNER JOIN OneApp_KelloggsMT.dbo.Master_NonWorkingReason N ON N.ReasonId = D.ReasonId
WHERE D.ISDEL = 0
AND D.EmpId = JP.EmpId AND D.StoreId = JP.StoreId AND d.VisitDate = JP.VisitDate),'') AS Remarks
, CASE WHEN (Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId = SC.EmpId AND JP.StoreId = SC.StoreId AND JP.VisitDate = SC.VisitDate
AND SC.ReasonId=11) >=1 THEN 1
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
AND SC.ReasonId=2) >=1 THEN 2
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
AND SC.ReasonId=6) >=1 THEN 4
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
AND SC.ReasonId=12) >=1 THEN 5
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
AND SC.ReasonId=13) >=1 THEN 6
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
AND SC.ReasonId=7) >=1 THEN 8
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
AND SC.ReasonId=26) >=1 THEN 10
ELSE CASE WHEN(Select Top 1 EmpId FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC WHERE JP.EmpId= SC.EmpId AND JP.StoreId= SC.StoreId AND JP.VisitDate= SC.VisitDate
AND SC.ReasonId IN(0,1,3,4,5,8,9,10,14,15,17,18,19,20,21,22,24)) >=1 THEN 7
END END END END END END END END AS COL, em.ResignDate as DOR, em.JoinDate as DOJ
FROM OneApp_KelloggsMT.dbo.Mapping_JourneyPlan JP LEFT OUTER JOIN
OneApp_KelloggsMT.dbo.T_StoreCoverage SC ON JP.StoreId= SC.StoreId AND JP.EmpId= SC.EmpId AND JP.VisitDate= SC.VisitDate INNER JOIN
OneApp_KelloggsMT.dbo.Master_Store SM ON JP.StoreId= SM.StoreId INNER JOIN
OneApp_KelloggsMT.dbo.AspNetUsers em ON JP.EmpId= em.Id inner join
OneApp_KelloggsMT.dbo.AspNetUsers em1 on em1.Id = em.ManagerId inner join
OneApp_KelloggsMT.dbo.AspNetUsers em2 on em2.Id = em1.ManagerId inner join
OneApp_KelloggsMT.dbo.Master_City cm on em.CityId= cm.CityId inner join
OneApp_KelloggsMT.dbo.Master_State st on cm.StateId= st.StateId inner join
OneApp_KelloggsMT.dbo.Master_Region rm on st.RegionId= rm.RegionId --Left Join
WHERE EM.EMPLOYEENAME not like 'teststore%' AND JP.VisitDate BETWEEN CAST('{start_date}' AS DATE) AND CAST('{run_date}' AS DATE)
) AS T1
) AS T2
) AS T
WHERE COL1=1
) AS TT
on a.date=tt.visitdate)A)b
on emp.[Employee Id]=b.[MERCHANDISER CD] and (
emp.DOR IS NULL
OR emp.DATE >= emp.DOR
) )
select * from Attendance where
parinaam_attendance !='-' and
project_id=40148
AND visit_date > date_of_join
order by employee_id, visit_date
src/sql/fact/Coverage.sql
+103 -159
View File
@@ -1,164 +1,108 @@
WITH SOS_BASE AS
(
SELECT
sm.CountryName,
sc.MID,
sm.RegionName,
sm.StateName,
sm.CityName,
Em.SupervisorName,
em.EmpId,
Em.EmpName AS EmployeeName,
Em.DesignationName AS Designation,
sm.StoreId,
CONVERT(varchar, sc.VisitDate, 101) AS VisitDate,
sm.StoreCode,
sm.StoreName,
sm.Address,
sm.StoreTypeid,
sm.ChannelId,
sm.ChainName,
MSD.SOSDefinitionName, with Executed as
(select JP.mid, CASE WHEN isnull(B.MID,'')<>'' then 'Y'
ELSE 'N' END AS COVERED,
CASE WHEN isnull(C.MID,'')<>'' then 'Y'
ELSE 'N' END AS EXECUTED,isnull(NR.REASON,'') AS REASON
from OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) left join
(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock)
inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B
ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101)
AND JP.ReasonId in (0,1,3,9,10,19,20))b
on JP.MID=b.MID
left join
(SELECT jp.mid FROM OneApp_KelloggsMT.dbo.T_StoreCoverage jp with (nolock)
inner join OneApp_KelloggsMT.dbo.Mapping_JourneyPlan B
ON JP.STOREID=B.STOREID AND JP.EMPID=B.EMPID AND CONVERT(VARCHAR,JP.VISITDATE,101)=CONVERT(VARCHAR,B.VISITDATE,101)
AND JP.ReasonId in (0,19))c
on JP.MID=c.MID
left join
OneApp_KelloggsMT.dbo.Master_NonWorkingReason nr on JP.ReasonId=nr.ReasonId
Where 1=1 AND JP.MID in ({mid_list})
) ,
Coverage
(project_id,Mid,
store_id,employee_id,visit_date,in_time,out_time,duration_in_minutes,
is_covered,is_executed,reason_remarks,storetype_id,
supervisor_id,coverage_type,distance_in_meters,reasonId,camera_allow,update_date,
update_by,Unique_Id)
CASE AS (
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Category' THEN 'Category'
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubCategory' THEN 'SubCategory' Select '40148' as ProjectId,JP.MID, sm.StoreId,JP.EmpId as EmpId,
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Brand' THEN 'Brand' Convert(VARCHAR,Jp.VisitDate) AS VisitDate
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubBrand' THEN 'SubBrand' , Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time,
END AS SOSHeaderDeatils, Convert(Datetime,InTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time,
Convert(Datetime,InTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage Where EmpId = Jp.EmpId And StoreId = Jp.StoreId
And VisitDate = Jp.VisitDate),'') As [In Time]
, Isnull((Select Top 1 Case When Convert(Varchar(15),Convert(Time,
Convert(Datetime,OutTime,101)),100) = 'Null' Then '' Else Convert(Varchar(15),Convert(Time,
Convert(Datetime,OutTime,101)),100)End From OneApp_KelloggsMT.dbo.T_StoreCoverage
Where EmpId = Jp.EmpId And StoreId = Jp.StoreId
And Convert(Date,VisitDate) = Jp.VisitDate),'') As [Out Time]
,
Isnull((Select Top 1 Case When convert(varchar,outtime) like '%00:00:00.00%' THEN 0
Else Case When outtime<InTime THEN 0 ELSE DATEDIFF(SS,CONVERT(DATETIME, InTime)
, CONVERT(DATETIME, outtime))/60 END END From
OneApp_KelloggsMT.dbo.T_StoreCoverage sc
Where sc.EmpId = Jp.EmpId And sc.StoreId = Jp.StoreId
And Convert(Date,sc.visitdate,101) = convert(date,Jp.VisitDate,101)), '') As duration_in_minute,
-- Isnull((Select Top 1 case when min(convert(varchar,InTime,108))='00:00:00' OR max(convert(varchar,CheckOutTime,108))='00:00:00'
-- then '00:00:00' else Cast(Datediff(Ss,Min(Convert(Varchar,Convert(Datetime,InTime),108)),
-- Max(Convert(Varchar,Convert(Datetime, CheckOutTime),108)) ) /3600 As Varchar(2))+':'+ Cast((Datediff(Ss,Min(Convert(Varchar,Convert(Datetime,InTime),108))
-- ,Max(Convert(Varchar,Convert(Datetime, CheckOutTime),108)) ) /60 )%60 As Varchar(2)) +':'+ Cast(Datediff(Ss,Min(Convert(Varchar,Convert(Datetime,InTime),108)),
-- Max(Convert(Varchar,Convert(Datetime, CheckOutTime),108)) ) %60 As Varchar(2)) end as time_taken From OneApp_KelloggsMT.dbo.T_StoreCoverage sc left join
--OneApp_KelloggsMT.dbo.T_StoreCheckOut UD ON UD.EmpId = sc.EmpId AND CONVERT(VARCHAR,UD.CheckOutDate,101) = sc.VisitDate AND sc.StoreId = UD.StoreId
-- Where sc.EmpId = Jp.EmpId And sc.StoreId = Jp.StoreId
-- And VisitDate = Jp.VisitDate),'') As [Time Taken]
Case When (Select Top 1 EmpId From OneApp_KelloggsMT.dbo.T_StoreCoverage Sc LEFT join
OneApp_KelloggsMT.dbo.Master_NonWorkingReason nw on Sc.ReasonId= nw.ReasonId
Where EmpId = Jp.EmpId And StoreId = Jp.StoreId And VisitDate = Jp.VisitDate
And (sc.ReasonId =0 Or nw.ForCoverage=1) ) >0 Then 'Y'
Else 'N' End As Covered
, isnull(Exe.Executed,'') as Executed
,Isnull((Select Top 1 Case
When D.ReasonId Is Null Or D.ReasonId =0 Then '' Else Reason+' - ' +
Case When Replace(Replace(D.Remark,Char(13),''''),Char(10),' ')<>'' Then Replace(Replace(D.Remark,Char(13),''),Char(10),' ')
Else Replace(Replace(D.Remark,Char(13),''),Char(10),' ') End End As Reason
From OneApp_KelloggsMT.dbo.T_StoreCoverage D
Inner Join OneApp_KelloggsMT.dbo.Master_NonWorkingReason N On N.ReasonId = D.ReasonId
Where D.Isdel = 0
And D.EmpId = Jp.EmpId And D.StoreId = Jp.StoreId And D.VisitDate = Jp.VisitDate),'''') As [Detailed Remarks],
sm.StoreTypeId,Em.SupervisorId,
Case When jp.Deviation=0 Then 'Planned' When jp.Deviation=1 Then 'Adhoc'
When jp.Deviation=2 Then 'Beat Plan' When jp.Deviation=3 Then 'Non Merchandised'
When jp.Deviation=4 Then 'Add New Store' When jp.Deviation=5 Then 'Non Program' else '' End as [PJP Status]
,Isnull((Select Top 1 Case When (Sc.Latitude=0.00000000 Or Sc.Longitude Is Null) Then 0 Else Case When (sm.Latitude=0.00000000 Or sm.Latitude Is Null) Then 0
Else Case When (sm.Longitude=0.00000000 Or sm.Longitude Is Null) Then 0
Else SQRT(POWER(69.1 * ( Sc.Latitude - Sm.Latitude),2) + POWER(69.1 * ( Sm.Longitude - Sc.Longitude ) * COS(Sc.Latitude / 57.3), 2))*1000
End End End As [Distance In Mtr] From OneApp_KelloggsMT.dbo.T_StoreCoverage Sc Where Sc.Isdel=0 And Sc.EmpId= Jp.EmpId
And Sc.StoreId= Jp.StoreId And Sc.VisitDate= Jp.VisitDate),'') As [Distance In Mtr]
,ISNULL(CAST (JP.ReasonId AS VARCHAR),'0'),sm.CameraAllow,
TS.SOSHeaderName, GETDATE(),'SP-Pius' ,
TS.SOSHeaderValue AS SOSHeaderID,
'Header_Image' AS HDR1, CAST('40148' AS VARCHAR) + '_' + CAST(SM.storeid AS VARCHAR)
+ '_' + CAST(EM.EMPID AS VARCHAR)
FROM OneApp_KelloggsMT.dbo.T_StoreCoverage JP with (nolock) Inner Join
Executed Exe on Jp.MID= Exe.MID Inner Join
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on Jp.StoreId= sm.StoreId Inner Join
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on JP.EmpId= Em.EmpId
left join
OneApp_KelloggsMT.dbo.T_StoreCoveragePositionPivot pv with (nolock)
on jp.mid=pv.MID
Where 1=1 and em.UserName not like 'test%' AND JP.MID in ({mid_list} )
CASE )
WHEN ISNULL(TSC.SOSChildTable,'')='Master_Category' THEN 'Category' select * from Coverage
WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubCategory' THEN 'SubCategory'
WHEN ISNULL(TSC.SOSChildTable,'')='Master_Brand' THEN 'Brand'
WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubBrand' THEN 'SubBrand'
END AS SOSChildDeatils,
TSC.SOSChildName,
TSC.SOSChildValue AS SOSChildID,
TSC.ChildTotalFacing,
TS.SOSHeaderFacing,
TSC.ChildSelfFacing,
(
SELECT TOP 1 SOSTarget
FROM OneApp_KelloggsMT.dbo.Mapping_StoreShareOfShelfTarget a
WHERE a.SOSDefinitionId = MSD.SOSDefinitionId
AND a.StoreId = sm.StoreId
AND a.FromDate <= sc.VisitDate
AND a.ToDate >= sc.VisitDate
) AS SOSTarget,
CASE
WHEN ISNULL(SHI.SOSHeaderImage,'') = ''
THEN ''
ELSE
'https://kimt1.parinaam.in/Upload/SOSImages/'
+ SHI.SOSHeaderImage
END AS SOSHeaderImg
FROM OneApp_KelloggsMT.dbo.T_ShareOfShelfHeader ts
INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage sc
ON ts.MID = sc.MID
INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail sm
ON sc.StoreId = sm.StoreId
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em
ON sc.EmpId = Em.EmpId
INNER JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChild tsc
ON ts.SOSId = tsc.SOSId
INNER JOIN OneApp_KelloggsMT.dbo.Master_ShareOfShelfDefinition msd
ON msd.SOSDefinitionId = tsc.SOSDefinitionId
LEFT JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfHeaderImages SHI
ON ts.SOSId = SHI.SOSId
AND ts.SOSHeaderValue = SHI.SOSHeaderValue
LEFT JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChildImages SCI
ON tsc.SOSId = SCI.SOSId
AND tsc.SOSChildValue = SCI.SOSChildValue
WHERE Em.EmpName NOT LIKE 'test%'
AND sc.MID IN ({mid_list})
),
SOS_PIVOT AS
(
SELECT *
FROM SOS_BASE
PIVOT
(
MIN(SOSHeaderImg)
FOR HDR1 IN ([Header_Image])
) pvt
)
SELECT
'40148' AS ProjectId,
MID,
EmpId AS employee_id,
StoreId AS store_id,
VisitDate AS visit_date,
StoreTypeid AS storetype_id,
ChannelId AS channel_id,
SOSDefinitionName,
SOSHeaderDeatils,
SOSHeaderName,
SOSHeaderID,
SOSChildDeatils,
SOSChildName,
SOSChildID,
SOSHeaderFacing,
ChildTotalFacing,
ChildSelfFacing,
SOSTarget,
GETDATE() AS update_date,
'SP-Pius' AS update_by
FROM SOS_PIVOT
GROUP BY
CountryName,
MID,
RegionName,
StateName,
CityName,
SupervisorName,
EmpId,
EmployeeName,
Designation,
StoreId,
VisitDate,
StoreCode,
StoreName,
Address,
StoreTypeid,
ChannelId,
ChainName,
SOSDefinitionName,
SOSHeaderDeatils,
SOSHeaderName,
SOSHeaderID,
SOSChildDeatils,
SOSChildName,
SOSChildID,
ChildTotalFacing,
SOSHeaderFacing,
ChildSelfFacing,
SOSTarget
ORDER BY
RegionName,
StateName,
CityName,
VisitDate;
src/sql/fact/Journey_Plan.sql
+10
View File
@@ -0,0 +1,10 @@
with Journey_Plan
(project_id,store_id,employee_id,visit_date,process_id,CreateDate,CreateBy,UpdateDate,
UpdateBy)
AS (
select
'40148' ,storeid,EmpId,cast(visitdate as DATE),Deviation,GETDATE(),'Pius',GETDATE(),'Pius' from OneApp_KelloggsMT.dbo.mapping_journeyplan
where MONTH(VisitDate) = MONTH(CAST('{run_date}' AS DATE)) AND Year(VisitDate)=Year(CAST('{run_date}' AS DATE))
AND EmpId NOT IN ( SELECT ID FROM OneApp_KelloggsMT.dbo.ASPNETUSERS
WHERE USERNAME LIKE 'TEST%'))
select * from Journey_Plan;
src/sql/fact/Login.sql
+81
View File
@@ -0,0 +1,81 @@
with employee_detail AS
(
SELECT rm.RegionId, rm.RegionName, st.StateId, st.StateName,
cm.CityId, cm.CityName,
em1.Id AS [Supervisor Code], ISNULL(em1.EmployeeName, '') AS Supervisor,
em.Id AS [Employee code], ISNULL(em.EmployeeName, '') AS [Employee Name],
dm.DesignationId, dm.DesignationName AS Designation
FROM OneApp_KelloggsMT.dbo.AspNetUsers em
INNER JOIN OneApp_KelloggsMT.dbo.AspNetUsers em1 ON em1.Id = em.ManagerId
INNER JOIN OneApp_KelloggsMT.dbo.Master_City cm ON em.CityId = cm.CityId
INNER JOIN OneApp_KelloggsMT.dbo.Master_State st ON cm.StateId = st.StateId
INNER JOIN OneApp_KelloggsMT.dbo.Master_Region rm ON st.RegionId = rm.RegionId
INNER JOIN OneApp_KelloggsMT.dbo.Master_Designation dm ON Em.DesignationId = dm.DesignationId
WHERE em.RightId IN (6)
AND (Em.ResignDate IS NULL OR CONVERT(DATE, Em.ResignDate, 101) >= CAST('{start_date}' AS DATE))
AND em.EmployeeName NOT LIKE '%test%'
),
Login (
project_id,
employee_id, login_date, login_time, process_id,
first_store_in_time, last_store_out_time, update_date, update_by,
Unique_Id
) AS (
SELECT '40148' AS ProjectId,
Em.[Employee Code],
[Login Date],
[Login Time],
'0',
[1st Store In Time],
[Last Store Out Time],
GETDATE(),
'SP-Pius',
CAST(40148 AS VARCHAR) + '_' + CAST(Em.[Employee Code] AS VARCHAR)
FROM employee_detail Em
LEFT OUTER JOIN
(
SELECT SupervisorId AS [Supervisor Code],
SupervisorName,
EmpId AS [Employee Code],
EmpName,
DesignationName,
[Login Date],
[Login Time],
[1st Store In Time],
[Last Store Out Time],
AppVersion AS [App Version]
FROM (
SELECT Em.SupervisorId,
Em.SupervisorName,
Em.EmpId,
Em.EmpName,
Em.DesignationName,
CONVERT(VARCHAR, ud.LoginDate, 101) AS [Login Date],
CONVERT(VARCHAR, ud.InTime, 108) AS [Login Time],
ud.AppVersion,
ROW_NUMBER() OVER (
PARTITION BY ud.Empid, CONVERT(VARCHAR, ud.LoginDate, 101)
ORDER BY ud.LoginDate ASC
) AS col,
(SELECT MIN(CONVERT(NVARCHAR, sc.InTime, 108))
FROM OneApp_KelloggsMT.dbo.T_StoreCoverage sc
WHERE sc.Isdel = 0
AND sc.EmpId = ud.EmpId
AND CONVERT(VARCHAR, sc.VisitDate, 101) = CONVERT(VARCHAR, ud.LoginDate, 101)
) AS [1st Store In Time],
(SELECT MAX(CONVERT(NVARCHAR, OutTime, 108))
FROM OneApp_KelloggsMT.dbo.T_StoreCoverage SC
WHERE SC.Isdel = 0
AND SC.EmpId = UD.EmpId
AND CONVERT(VARCHAR, SC.VisitDate, 101) = CONVERT(VARCHAR, UD.LoginDate, 101)
) AS [Last Store Out Time]
FROM OneApp_KelloggsMT.dbo.T_DeviceLogin ud
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em ON ud.EmpId = Em.EmpId
WHERE CAST(ud.LoginDate AS DATE) = CAST('{start_date}' AS DATE) -- fixed: direct date comparison
) AS T1
WHERE T1.col = 1
) AS T ON Em.[Employee code] = T.[Employee Code]
WHERE T.[Login Date] IS NOT NULL
)
SELECT * FROM Login;
src/sql/fact/OQaD.sql
+81
View File
@@ -0,0 +1,81 @@
WITH MID_TABLE_COV1 AS
(
SELECT EmpId, VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE CreateDate >= {run_date}
AND CreateDate < DATEADD(DAY,1,'{run_date}')
UNION ALL
SELECT EmpId, VisitDate
FROM OneApp_KelloggsMT.dbo.T_OQAD
WHERE UpdateDate >= {run_date}
AND UpdateDate < DATEADD(DAY,1, '{run_date}')
),
QUIZ AS
(
SELECT DISTINCT
E.EmpId,
E.EmpName,
E.SupervisorId,
E.SupervisorName,
E.DesignationName,
E.CityName,
E.StateName,
E.RegionName,
CAST(DQ.VisitDate AS DATE) AS VisitDate,
DQ.QuestionId,
DQ.AnswerId,
QC.QuestionCategoryId,
QC.QuestionCategory
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E
ON DQ.EmpId = E.EmpId
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU
ON DQ.QuestionId = QU.QuestionId
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC
ON QU.QuestionCategoryId = QC.QuestionCategoryId
WHERE E.EmpName NOT LIKE '%TEST%'
AND E.RightId = 6
AND (
E.ResignDate IS NULL
OR CAST(E.ResignDate AS DATE) >= '{run_date}'
)
AND EXISTS
(
SELECT 1
FROM MID_TABLE_COV1 A
WHERE A.EmpId = DQ.EmpId
AND CAST(A.VisitDate AS DATE) = CAST(DQ.VisitDate AS DATE)
)
)
SELECT
40148 AS project_id,
Q.EmpId AS employee_id,
0 AS process_id,
Q.VisitDate AS visit_date,
Q.QuestionCategoryId AS question_category_id,
Q.QuestionCategory AS question_category,
QM.QuestionId AS question_id,
QM.Question AS question,
ISNULL(QA.AnswerId,0) AS answer_id,
ISNULL(QA.Answer,'') AS answer,
CASE
WHEN QA.AnswerId IS NULL THEN 'Not Answer'
WHEN QA.RightAnswer = 1 THEN 'Y'
WHEN QA.RightAnswer IS NULL THEN 'Not Answer'
ELSE 'N'
END AS correct_answer,
GETDATE() AS update_date,
'SP-Pius' AS update_by
FROM QUIZ Q
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM
ON Q.QuestionId = QM.QuestionId
LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA
ON Q.AnswerId = QA.AnswerId
where Q.EmpId not in ({empid_list})
src/sql/fact/PaidVisibility.sql
+56
View File
@@ -0,0 +1,56 @@
with PaidVisibility(Mid,
project_id,store_id,employee_id,visit_date,supervisor_id,channel_id,chain_id,storetype_id,Menuid,MenuName,
Visibility_Id,
Visibility_Name,
Visibility_definition_id,Visibility_definition_name,
Visibility_deatils,Visibility_deatils_id,Visibility_value_name,
present,REASONID,reason,VisibilityQuestion,VisibilityAnswer,image1,image2,update_date,update_by)
AS (
Select sc.MID, '40148' Projectid,sc.StoreId,Em.EmpId,sc.VisitDate,Em.SupervisorId,sm.ChannelId,sm.ChainId,sm.StoreTypeId, 0 as menuid ,'' as menuname,
ts.visibilityid,mv.Visibilityname,msd.VisibilityDefinitionId,
MSD.VisibilityDefinitionName, case when isnull(TS.VisibilityTable,'')='Master_Category' then 'Category'
when isnull(TS.VisibilityTable,'')='Master_SubCategory' then 'SubCategory'
when isnull(TS.VisibilityTable,'')='Master_Brand' then 'Brand'
when isnull(TS.VisibilityTable,'')='Master_SubBrand' then 'SubBrand'
end as PaidVisibilityDeatils ,ts.visibilityvalue[Paid_visiValueID],
CASE when isnull(TS.VisibilityTable,'')='Master_Category' THEN (SELECT a.CategoryName from OneApp_KelloggsMT.dbo.Master_Category a where ts.VisibilityValue=a.CategoryId )
when isnull(TS.VisibilityTable,'')='Master_SubCategory' THEN (SELECT a.SubCategoryName from OneApp_KelloggsMT.dbo.Master_SubCategory a where ts.VisibilityValue=a.SubCategoryId )
when isnull(TS.VisibilityTable,'')='Master_Brand' THEN (SELECT a.BrandName from OneApp_KelloggsMT.dbo.Master_Brand a where ts.VisibilityValue=a.BrandId )
when isnull(TS.VisibilityTable,'')='Master_SubBrand' THEN (SELECT a.SubBrandName from OneApp_KelloggsMT.dbo.Master_SubBrand a where ts.VisibilityValue=a.SubBrandId ) end as
[Paid_VisiValueName],case when ts.Present=0 then 'N' else 'Y' end Present,
CASE WHEN TS.PRESENT=0 THEN TS.VISIBILITYREASONID ELSE '' END AS VISIBILITYREASONID ,
case when ts.Present=0 then isnull(mnp.VisibilityReason,'') else '' end Reason,
ISNULL( MVQ.VisibilityQuestionName,'') AS Question, ISNULL(TVQ.VisibilityAnswerName,'') AS Answer ,
case when ts.present=0 then '' else
case when isnull(SHI.VisibilityImage1,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PaidVisibilityImages/'+SHI.VisibilityImage1
end end as PaidVisibilityImg1 ,case when isnull(SHI.VisibilityImage2,'')='' then '' else 'https://kimt.parinaam.in/Upload/PaidVisibilityImages/'+SHI.VisibilityImage2
end as PaidVisibilityImg2 ,GETDATE(),'SP-Pius'
from
OneApp_KelloggsMT.dbo.T_Visibility ts WITH (NOLOCK) Inner join
OneApp_KelloggsMT.dbo.T_StoreCoverage sc WITH (NOLOCK) on ts.mid= sc.mid Inner Join
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId
inner join
OneApp_KelloggsMT.dbo.Master_VisibilityDefinition msd WITH (NOLOCK) on
msd.VisibilityDefinitionId=ts.VisibilityDefinitionId --and msd.menuid=ts.menuid
--inner join OneApp_KelloggsMT.dbo.Master_menu MM WITH (NOLOCK) ON ts.menuid=mm.menuid and msd.menuid=mm.menuid
left join OneApp_KelloggsMT.DBO. master_visibility mv WITH (NOLOCK) on ts.visibilityid=mv.visibilityid
LEFT join
OneApp_KelloggsMT.dbo.T_VisibilityImages SHI WITH (NOLOCK) ON ts.VId=SHI.VId
left join
OneApp_KelloggsMT.dbo.Master_VisibilityReason mnp WITH (NOLOCK) on ts.VisibilityReasonId=mnp.VisibilityReasonId
LEFT JOIN OneApp_KelloggsMT.dbo.T_VisibilityStock TVS WITH (NOLOCK) ON TS.VID=TVS.VID
LEFT JOIN OneApp_KelloggsMT.dbo.T_VisibilityQuestion TVQ WITH (NOLOCK) ON TS.Vid=TVQ.Vid
LEFT JOIN OneApp_KelloggsMT.dbo.Master_VisibilityQuestion MVQ WITH (NOLOCK) on TVQ.VisibilityQuestionId=MVQ.VisibilityQuestionId
Where 1=1 and Em.EmpName not like 'test%'
--and sc.visitdate between '10/01/2023' and '10/31/2023'
AND sc.MID in ({mid_list})
)
select * from PaidVisibility
src/sql/fact/Promotion.sql
+43
View File
@@ -0,0 +1,43 @@
with Promotion(Mid,
project_id,store_id,employee_id,visit_date,supervisor_id,channel_id,chain_id,storetype_id,promo_definition_id,promo_definition_name,
promotion_deatils,promotion_deatils_id,promotion_value_name,
present,reason,PromoQuestion,PromoAnswer,image1,image2,update_date,update_by)
as (
Select
sc.MID, '40148' Projectid,sc.StoreId,Em.EmpId,sc.VisitDate,Em.SupervisorId,sm.ChannelId,sm.ChainId,sm.StoreTypeId,msd.PromoDefinitionId,
MSD.PromoDefinitionName, case when isnull(TS.PromoTable,'')='Master_Category' then 'Category'
when isnull(TS.PromoTable,'')='Master_SubCategory' then 'SubCategory'
when isnull(TS.PromoTable,'')='Master_Brand' then 'Brand'
when isnull(TS.PromoTable,'')='Master_SubBrand' then 'SubBrand'
end as PromotionDeatils ,
TS.PromoValue[PromotionDeatilsId],
CASE when isnull(TS.PromoTable,'')='Master_Category' THEN (SELECT a.CategoryName from OneApp_KelloggsMT.dbo.Master_Category a where ts.PromoValue=a.CategoryId )
when isnull(TS.PromoTable,'')='Master_SubCategory' THEN (SELECT a.SubCategoryName from OneApp_KelloggsMT.dbo.Master_SubCategory a where ts.PromoValue=a.SubCategoryId )
when isnull(TS.PromoTable,'')='Master_Brand' THEN (SELECT a.BrandName from OneApp_KelloggsMT.dbo.Master_Brand a where ts.PromoValue=a.BrandId )
when isnull(TS.PromoTable,'')='Master_SubBrand' THEN (SELECT a.SubBrandName from OneApp_KelloggsMT.dbo.Master_SubBrand a where ts.PromoValue=a.SubBrandId ) end as
[PromoValueName],case when ts.Present=0 then 'N' else 'Y' end Present,
case when ts.Present=1 then '' else isnull(mnp.PromoReason,'') end as Reason, ISNULL(mpq.PromoQuestionName,'') AS Question,
ISNULL(tpq.PromoAnswerName,'') AS Answer ,
case when isnull(SHI.PromoImage1,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PromotionImages/'+SHI.PromoImage1
end as Image1 ,case when isnull(SHI.PromoImage2,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PromotionImages/'+SHI.PromoImage2
end as Image2 ,GETDATE(),'SP-Pius'
from
OneApp_KelloggsMT.dbo.T_Promotion ts Inner join
OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId
inner join
OneApp_KelloggsMT.dbo.Master_PromotionDefinition msd on
msd.PromoDefinitionId=ts.PromoDefinitionId
LEFT join
OneApp_KelloggsMT.dbo.T_PromotionImages SHI ON ts.PId=SHI.PId
left join
OneApp_KelloggsMT.dbo.Master_PromotionReason mnp on ts.PromoReasonId=mnp.PromoReasonId
left join OneApp_KelloggsMT.dbo.t_promotionquestion tpq on ts.PId=tpq.PId
left join OneApp_KelloggsMT.dbo.Master_PromotionQuestion mpq on tpq.PromoQuestionId=mpq.PromoQuestionId
Where 1=1 and Em.EmpName not like 'test%'
AND sc.MID in ({mid_list})
)
select * from Promotion
src/sql/fact/SOS_OneApp.sql
+70
View File
@@ -0,0 +1,70 @@
with SOS_OneApp ( project_id,Mid,
employee_id,store_id,visit_date,storetype_id,channel_id,SOSDefinitionName,SOSHeaderDeatils,SOSHeaderName,SOSHeaderID,SOSChildDeatils,SOSChildName,SOSChildID,
SOSHeaderFacing,ChildTotalFacing,ChildSelfFacing,SOSTarget,update_date,update_by)
as (
Select '40148' AS ProjectId,MID,
EmpId
,StoreId,
VisitDate,StoreTypeid,ChannelId ,
--CameraAllow,
SOSDefinitionName,SOSHeaderDeatils,SOSHeaderName,[SOSHeaderID] ,SOSChildDeatils , SOSChildName,SOSChildID,SOSHeaderFacing,ChildTotalFacing,
ChildSelfFacing ,SOSTarget,getdate(),'SP-Pius' from
(
Select sm.CountryName,SC.MID, sm.RegionName, sm.StateName, sm.CityName, Em.SupervisorName, em.EmpId as EmpId
, Em.EmpName as EmployeeName, Em.DesignationName as Designation, sm.StoreId,convert(varchar,sc.VisitDate,101)VisitDate, sm.StoreCode, sm.StoreName,
sm.Address, sm.StoreTypeid ,sm.ChannelId,sm.ChainName,
MSD.SOSDefinitionName, case when isnull(TS.SOSHeaderTable,'')='Master_Category' then 'Category'
when isnull(TS.SOSHeaderTable,'')='Master_SubCategory' then 'SubCategory'
when isnull(TS.SOSHeaderTable,'')='Master_Brand' then 'Brand'
when isnull(TS.SOSHeaderTable,'')='Master_SubBrand' then 'SubBrand'
end as SOSHeaderDeatils ,
TS.SOSHeaderName,TS.SOSHeaderValue[SOSHeaderID], 'Header'+'_'+'Image' as HDR1,
case when isnull(tsc.SOSChildTable,'')='Master_Category' then 'Category'
when isnull(tsc.SOSChildTable,'')='Master_SubCategory' then 'SubCategory'
when isnull(tsc.SOSChildTable,'')='Master_Brand' then 'Brand'
when isnull(tsc.SOSChildTable,'')='Master_SubBrand' then 'SubBrand'
end as SOSChildDeatils ,
TSC.SOSChildName,TSC.SOSChildValue[SOSChildID],TSC.ChildTotalFacing,ts.SOSHeaderFacing,TSC.ChildSelfFacing
,
(select top 1 SOSTarget from OneApp_KelloggsMT.dbo.Mapping_StoreShareOfShelfTarget a where a.SOSDefinitionId=msd.SOSDefinitionId and
a.Storeid=sm.StoreId and a.fromdate<=sc.visitdate and a.todate>=sc.visitdate ) as SOSTarget,
case when isnull(SHI.SOSHeaderImage,'')='' then '' else 'https://kimt1.parinaam.in/Upload/SOSImages/'+SHI.SOSHeaderImage
end as SOSHeaderImg
--,case when isnull(SCI.SOSChildImage,'')='' then '' else 'https://di1.parinaam.in/Upload/SOSImages/'+SCI.SOSChildImage
--end as SOSChildImage
from
OneApp_KelloggsMT.dbo.T_ShareOfShelfHeader ts Inner join
OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId
inner join
OneApp_KelloggsMT.dbo.T_ShareOfShelfChild tsc on ts.sosid=tsc.SOSId
inner join
OneApp_KelloggsMT.dbo.Master_ShareOfShelfDefinition msd on
msd.SOSDefinitionId=tsc.SOSDefinitionId
LEFT join
OneApp_KelloggsMT.dbo.T_ShareOfShelfHeaderImages SHI ON ts.SOSId=SHI.SOSId AND TS.SOSHeaderValue=SHI.SOSHeaderValue
LEFT JOIN
OneApp_KelloggsMT.dbo.T_ShareOfShelfChildImages SCI ON TSC.SOSId=SCI.SOSId AND TSC.SOSChildValue=SCI.SOSChildValue
Where 1=1 AND EM.EMPNAME NOT LIKE 'test%' AND sc.mid IN ({mid_list})
-- AND sc.VisitDate BETWEEN '09/18/2023' AND '09/21/2023'
--and sc.StoreId not in (
--select store_id from SOS_OneApp a where a.project_id=40148 and a.store_id=sc.StoreId
--and a.employee_id=sc.EmpId and convert(date,sc.VisitDate,101)=convert(date,a.visit_date,101) )
) as A
Pivot
(
min(SOSHeaderImg)
for HDR1 in([Header_Image])
) as pvt1
GROUP BY CountryName,MID, RegionName, StateName, CityName, SupervisorName, EmpId
, EmployeeName, Designation, StoreId,VisitDate,StoreCode, StoreName,
Address, StoreTypeid ,ChannelId,ChainName,
SOSDefinitionName,SOSHeaderDeatils,SOSHeaderName,[SOSHeaderID] ,SOSChildDeatils ,
SOSChildName,SOSChildID,ChildTotalFacing,SOSHeaderFacing,ChildSelfFacing,SOSTarget )
select * from SOS_OneApp
src/sql/fact/Stock_Details.sql
+29
View File
@@ -0,0 +1,29 @@
with Stock_Details (project_id,
Mid,supervisor_id,employee_id,store_id,
visitdate,storetype_id,store_category_id,product_id,MSL, MBQ,Stock_Qty,
damagedstock,loststock,expirystock,skuavailability,update_date,update_by,StockType)
AS (
Select '40148' AS ProjectId,sc.MID,Em.SupervisorId,Em.EmpId,sm.StoreId,sc.VisitDate,sc.StoreTypeId,sm.StoreCategoryId,
vp.ProductId,ts.MSL,mp.MBQ, ISNULL(ts.OpeningStock,0)+ISNULL(TS.middaystock,0),0,0,0,
Case WHEN ISNULL(ts.OpeningStock,0)+ISNULL(TS.middaystock,0)>=1THEN 'Y' ELSE 'N' END AS
skuavailability, GETDATE(),'SP-Pius','Parinaam'
from
OneApp_KelloggsMT.dbo.T_Stock ts Inner join
OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId
inner join OneApp_KelloggsMT.dbo.vw_product vp on ts.ProductId=vp.ProductId inner JOIN
(SELECT MBQ,ProductId,StateId,ChainId,StoreTypeId,StoreCategoryId,StoreClassId FROM OneApp_KelloggsMT.dbo.Mapping_ProductAssortment where FromDate<='{run_date}' and ToDate>='{run_date}') mp on mp.StateId=sm.StateId and
mp.ChainId=sm.ChainId
and mp.StoreTypeId=sm.StoreTypeId
and mp.StoreCategoryId=sm.StoreCategoryId
and mp.StoreClassId=sm.StoreClassId
and mp.ProductId=ts.ProductId
Where Em.EmpName not like 'test%' AND sc.MID in ({mid_list})
)
select * from Stock_Details
src/sql/fact/Survey.sql
+68
View File
@@ -0,0 +1,68 @@
WITH QUIZ AS
(
SELECT
SC.MID,
SC.StoreId,
EM.EmpId,
EM.EmpName,
EM.SupervisorId,
EM.SupervisorName,
EM.DesignationName,
CAST(SC.VisitDate AS DATE) AS VisitDate,
DQ.QuestionId,
QU.Question,
DQ.Answer,
QC.CategoryId,
QC.Category,
MSS.SubCategoryId,
MSS.SubCategory,
DQ.ImageUrl
FROM OneApp_KelloggsMT.dbo.T_SURVEY DQ
INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage SC
ON DQ.MID = SC.MID
INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveyQuestion QU
ON DQ.QuestionId = QU.QuestionId
INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveySubCategory MSS
ON QU.SubCategoryId = MSS.SubCategoryId
INNER JOIN OneApp_KelloggsMT.dbo.Master_SurveyCategory QC
ON MSS.CategoryId = QC.CategoryId
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail EM
ON SC.EmpId = EM.EmpId
INNER JOIN OneApp_KelloggsMT.dbo.Master_Survey MS
ON DQ.SurveyId = MS.SurveyId
AND QU.SurveyId = MS.SurveyId
WHERE EM.EmpName NOT LIKE 'test%'
AND SC.MID IN ({mid_list})
)
SELECT
40148 AS Project_Id,
Q.MID AS Mid,
Q.SupervisorId AS SuperVisorId,
Q.EmpId AS EmpID,
SM.StoreId,
Q.VisitDate,
SM.StoreTypeId,
SM.ChainId,
Q.CategoryId,
Q.Category,
Q.SubCategoryId,
Q.SubCategory,
Q.QuestionId,
Q.Question,
Q.Answer,
GETDATE() AS CreateDate,
'SP-Pius' AS CreateBy
FROM QUIZ Q
INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail SM
ON Q.StoreId = SM.StoreId
src/sql/fact/Web_Logins.sql
+23
View File
@@ -0,0 +1,23 @@
with Web_Logins (project_id,
supervisor_id,supervisor_name,emp_id,employee_name,designation,date,time,activity_name,activity_type,right_name,CreateDate,CreateBy)
as (
Select Distinct '40148' as ProjectId
, EM1.Id AS SupervisorId, EM1.EmployeeName AS Supervisor
, EM.Id AS EmployeeId , EM.EmployeeName as Employee
, DM.DesignationName as Designation
, cast( CONVERT(date,AL.Date) as Date) AS Date,CONVERT(NVARCHAR,AL.Date ,108) AS Time
, AL.Thread as ActivityName, AL.Level as ActivityType, R.RightName,GETDATE(),'Pius'
FROM OneApp_KelloggsMT.dbo.T_User_Activity_Log AL LEFT JOIN
OneApp_KelloggsMT.dbo.AspNetUsers EM ON AL.Logger=EM.USERNAME INNER JOIN
OneApp_KelloggsMT.dbo.AspNetUsers EM1 ON EM1.Id = EM.ManagerId INNER JOIN
OneApp_KelloggsMT.dbo.AspNetUsers EM2 ON EM2.Id = EM1.ManagerId INNER JOIN
OneApp_KelloggsMT.dbo.Master_Designation DM ON EM.DesignationId = DM.DesignationId INNER JOIN
OneApp_KelloggsMT.dbo.Master_City CM ON EM.CityId = CM.CityId INNER JOIN
OneApp_KelloggsMT.dbo.Master_State ST ON ST.StateId = cm.StateId INNER JOIN
OneApp_KelloggsMT.dbo.Master_Region RM ON RM.RegionId = ST.RegionId INNER JOIN
OneApp_KelloggsMT.dbo.Right_Master R ON EM.RightId=R.RightId
Where 1=1 and CONVERT(date,AL.Date,101) = CONVERT(DATE,CAST('{run_date}' AS DATE),101)
)
select * from Web_Logins
ORDER BY employee_name
src/sql/fact/additional_visibility.sql
+31
View File
@@ -0,0 +1,31 @@
WITH
additional_visibility (project_id,Mid,
emp_id,store_id,storetype_id,channel_id,chain_id,camera_allowed,visit_date,is_present,brand_id,
display_id,Remarks,image_url,created_date,created_by)
as (
SELECT '40148' AS ProjectId,ts.mid, em.EmpId as EmpId
, sm.StoreId,sm.storetypeid,sm.channelid,sm.chainid,case when sm.cameraallow=1 then 'Y' ELSE 'N' END AS
cameraallow , convert(varchar,sc.VisitDate,101)VisitDate,
case when ts.Present=0 then 'N' else 'Y' end Present,
isnull(MB.BrandId,'')[BrandId],
isnull(MD.DisplayId,'')[DisplayId],ISNULL(TS.REMARK,'')[Remark],
case when isnull(ts.ImageUrl,'')='' then '' else 'https://kimt1.parinaam.in/Upload/VisibilityImages/'+ts.ImageUrl
end as AdditionalImageUrl ,GETDATE(),'Pius'
from
OneApp_KelloggsMT.dbo.T_AdditionalVisibility ts Inner join
OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
OneApp_KelloggsMT.dbo.vw_Employee_Detail Em on sc.EmpId= Em.EmpId
left join
OneApp_KelloggsMT.dbo.Master_Brand MB ON TS.BrandId=MB.BrandId
LEFT JOIN
OneApp_KelloggsMT.dbo.Master_Display MD ON TS.DisplayId=MD.DisplayId
Where Em.EmpName not like 'test%' and ts.Present=1 and sc.mid in ({mid_list})
)
select * from additional_visibility;
t.yml
+6 -6
View File
@@ -62,32 +62,32 @@ tables:
- name: Store_Master - name: Store_Master
type: DIMENSION type: DIMENSION
operation: DELETE+INSERT operation: DELETE+INSERT
fetch_by: none fetch_by: master
- name: SKU_Master - name: SKU_Master
type: DIMENSION type: DIMENSION
operation: DELETE+INSERT operation: DELETE+INSERT
fetch_by: none fetch_by: master
- name: display_master - name: display_master
type: DIMENSION type: DIMENSION
operation: DELETE+INSERT operation: DELETE+INSERT
fetch_by: none fetch_by: master
- name: Employee_Master - name: Employee_Master
type: DIMENSION type: DIMENSION
operation: DELETE+INSERT operation: DELETE+INSERT
fetch_by: none fetch_by: master
- name: coverage_remarks - name: coverage_remarks
type: DIMENSION type: DIMENSION
operation: DELETE+INSERT operation: DELETE+INSERT
fetch_by: none fetch_by: reason_id
- name: mapping_storevisibility - name: mapping_storevisibility
type: BRIDGE type: BRIDGE
operation: DELETE+INSERT operation: ONLY_INSERT
fetch_by: run_date fetch_by: run_date
- name: Master_VisibilityReason - name: Master_VisibilityReason
y.yml
+104
View File
@@ -0,0 +1,104 @@
tables:
# - name: SOS_OneApp
# type: FACT
# operation: INSERT
# fetch_by: mids
# # - name: OQaD
# # type: FACT
# # operation: INSERT
# # fetch_by: run_date
# - name: additional_visibility
# type: FACT
# operation: INSERT
# fetch_by: mids
# - name: Coverage
# type: FACT
# operation: INSERT
# fetch_by: mids
# - name: Survey
# type: FACT
# operation: INSERT
# fetch_by: mids
# - name: Login
# type: FACT
# operation: INSERT
# fetch_by: run_date
# - name: Stock_Details
# type: FACT
# operation: INSERT
# fetch_by: mids
# - name: Attendance
# type: FACT
# operation: DELETE+INSERT
# fetch_by: run_date
# - name: Journey_Plan
# type: FACT
# operation: INSERT
# fetch_by: run_date
# - name: PaidVisibility
# type: FACT
# operation: INSERT
# fetch_by: mids
# - name: Web_Logins
# type: FACT
# operation: INSERT
# fetch_by: run_date
# - name: Store_Master
# type: DIMENSION
# operation: DELETE+INSERT
# fetch_by: master
# - name: coverage_remarks
# type: DIMENSION
# operation: DELETE+INSERT
# fetch_by: reason_id
# - name: SKU_Master
# type: DIMENSION
# operation: DELETE+INSERT
# fetch_by: master
# - name: display_master
# type: DIMENSION
# operation: DELETE+INSERT
# fetch_by: master
# - name: Employee_Master
# type: DIMENSION
# operation: DELETE+INSERT
# fetch_by: master
# - name: mapping_storevisibility
# type: BRIDGE
# operation: ONLY_INSERT
# fetch_by: run_date
- name: Master_VisibilityReason
type: DIMENSION
operation: DELETE+INSERT
fetch_by: none
- name: Master_VisibilityDefinition
type: DIMENSION
operation: DELETE+INSERT
fetch_by: none
- name: Master_Salesterritorylayer
type: DIMENSION
operation: DELETE+INSERT
fetch_by: none