Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Ankit Malik
|
75fc088c68 | ||
|
Ankit Malik
|
e1bd387849 |
-1301
File diff suppressed because it is too large
Load Diff
@@ -1,164 +0,0 @@
|
||||
WITH SOS_BASE AS
|
||||
(
|
||||
SELECT
|
||||
sm.CountryName,
|
||||
sc.MID,
|
||||
sm.RegionName,
|
||||
sm.StateName,
|
||||
sm.CityName,
|
||||
Em.SupervisorName,
|
||||
em.EmpId,
|
||||
Em.EmpName AS EmployeeName,
|
||||
Em.DesignationName AS Designation,
|
||||
sm.StoreId,
|
||||
CONVERT(varchar, sc.VisitDate, 101) AS VisitDate,
|
||||
sm.StoreCode,
|
||||
sm.StoreName,
|
||||
sm.Address,
|
||||
sm.StoreTypeid,
|
||||
sm.ChannelId,
|
||||
sm.ChainName,
|
||||
|
||||
MSD.SOSDefinitionName,
|
||||
|
||||
CASE
|
||||
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Category' THEN 'Category'
|
||||
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubCategory' THEN 'SubCategory'
|
||||
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Brand' THEN 'Brand'
|
||||
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubBrand' THEN 'SubBrand'
|
||||
END AS SOSHeaderDeatils,
|
||||
|
||||
TS.SOSHeaderName,
|
||||
TS.SOSHeaderValue AS SOSHeaderID,
|
||||
|
||||
'Header_Image' AS HDR1,
|
||||
|
||||
CASE
|
||||
WHEN ISNULL(TSC.SOSChildTable,'')='Master_Category' THEN 'Category'
|
||||
WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubCategory' THEN 'SubCategory'
|
||||
WHEN ISNULL(TSC.SOSChildTable,'')='Master_Brand' THEN 'Brand'
|
||||
WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubBrand' THEN 'SubBrand'
|
||||
END AS SOSChildDeatils,
|
||||
|
||||
TSC.SOSChildName,
|
||||
TSC.SOSChildValue AS SOSChildID,
|
||||
TSC.ChildTotalFacing,
|
||||
TS.SOSHeaderFacing,
|
||||
TSC.ChildSelfFacing,
|
||||
|
||||
(
|
||||
SELECT TOP 1 SOSTarget
|
||||
FROM OneApp_KelloggsMT.dbo.Mapping_StoreShareOfShelfTarget a
|
||||
WHERE a.SOSDefinitionId = MSD.SOSDefinitionId
|
||||
AND a.StoreId = sm.StoreId
|
||||
AND a.FromDate <= sc.VisitDate
|
||||
AND a.ToDate >= sc.VisitDate
|
||||
) AS SOSTarget,
|
||||
|
||||
CASE
|
||||
WHEN ISNULL(SHI.SOSHeaderImage,'') = ''
|
||||
THEN ''
|
||||
ELSE
|
||||
'https://kimt1.parinaam.in/Upload/SOSImages/'
|
||||
+ SHI.SOSHeaderImage
|
||||
END AS SOSHeaderImg
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.T_ShareOfShelfHeader ts
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage sc
|
||||
ON ts.MID = sc.MID
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail sm
|
||||
ON sc.StoreId = sm.StoreId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em
|
||||
ON sc.EmpId = Em.EmpId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChild tsc
|
||||
ON ts.SOSId = tsc.SOSId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_ShareOfShelfDefinition msd
|
||||
ON msd.SOSDefinitionId = tsc.SOSDefinitionId
|
||||
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfHeaderImages SHI
|
||||
ON ts.SOSId = SHI.SOSId
|
||||
AND ts.SOSHeaderValue = SHI.SOSHeaderValue
|
||||
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChildImages SCI
|
||||
ON tsc.SOSId = SCI.SOSId
|
||||
AND tsc.SOSChildValue = SCI.SOSChildValue
|
||||
|
||||
WHERE Em.EmpName NOT LIKE 'test%'
|
||||
AND sc.MID IN ({mid_list})
|
||||
),
|
||||
|
||||
SOS_PIVOT AS
|
||||
(
|
||||
SELECT *
|
||||
FROM SOS_BASE
|
||||
PIVOT
|
||||
(
|
||||
MIN(SOSHeaderImg)
|
||||
FOR HDR1 IN ([Header_Image])
|
||||
) pvt
|
||||
)
|
||||
|
||||
SELECT
|
||||
'40148' AS ProjectId,
|
||||
MID,
|
||||
EmpId AS employee_id,
|
||||
StoreId AS store_id,
|
||||
VisitDate AS visit_date,
|
||||
StoreTypeid AS storetype_id,
|
||||
ChannelId AS channel_id,
|
||||
SOSDefinitionName,
|
||||
SOSHeaderDeatils,
|
||||
SOSHeaderName,
|
||||
SOSHeaderID,
|
||||
SOSChildDeatils,
|
||||
SOSChildName,
|
||||
SOSChildID,
|
||||
SOSHeaderFacing,
|
||||
ChildTotalFacing,
|
||||
ChildSelfFacing,
|
||||
SOSTarget,
|
||||
GETDATE() AS update_date,
|
||||
'SP-Pius' AS update_by
|
||||
|
||||
FROM SOS_PIVOT
|
||||
|
||||
GROUP BY
|
||||
CountryName,
|
||||
MID,
|
||||
RegionName,
|
||||
StateName,
|
||||
CityName,
|
||||
SupervisorName,
|
||||
EmpId,
|
||||
EmployeeName,
|
||||
Designation,
|
||||
StoreId,
|
||||
VisitDate,
|
||||
StoreCode,
|
||||
StoreName,
|
||||
Address,
|
||||
StoreTypeid,
|
||||
ChannelId,
|
||||
ChainName,
|
||||
SOSDefinitionName,
|
||||
SOSHeaderDeatils,
|
||||
SOSHeaderName,
|
||||
SOSHeaderID,
|
||||
SOSChildDeatils,
|
||||
SOSChildName,
|
||||
SOSChildID,
|
||||
ChildTotalFacing,
|
||||
SOSHeaderFacing,
|
||||
ChildSelfFacing,
|
||||
SOSTarget
|
||||
|
||||
ORDER BY
|
||||
RegionName,
|
||||
StateName,
|
||||
CityName,
|
||||
VisitDate;
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@@ -1,71 +0,0 @@
|
||||
2026-06-16 17:56:17 | INFO | ================================================================================
|
||||
2026-06-16 17:56:17 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-16 17:56:17 | INFO | Pipeline Run Date: 2026-06-15
|
||||
2026-06-16 17:56:17 | INFO | Connecting to databases...
|
||||
2026-06-16 17:56:19 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000017B644A81B0>
|
||||
2026-06-16 17:56:23 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000017B6548EFD0>
|
||||
2026-06-16 17:56:24 | INFO | Database connections established
|
||||
2026-06-16 17:56:24 | INFO | Collecting MIDs for: 2026-06-15
|
||||
2026-06-16 17:56:24 | INFO | Found 849 MIDs
|
||||
2026-06-16 17:57:33 | INFO | ================================================================================
|
||||
2026-06-16 17:57:33 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-16 17:57:33 | INFO | Pipeline Run Date: 2026-06-15
|
||||
2026-06-16 17:57:33 | INFO | Connecting to databases...
|
||||
2026-06-16 17:57:34 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DCA21AB750>
|
||||
2026-06-16 17:57:36 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DCA3371590>
|
||||
2026-06-16 17:57:36 | INFO | Database connections established
|
||||
2026-06-16 17:57:36 | INFO | Collecting MIDs for: 2026-06-15
|
||||
2026-06-16 17:57:37 | INFO | Found 849 MIDs
|
||||
2026-06-16 17:58:52 | INFO | ================================================================================
|
||||
2026-06-16 17:58:52 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-16 17:58:52 | INFO | Pipeline Run Date: 2026-06-15
|
||||
2026-06-16 17:58:52 | INFO | Connecting to databases...
|
||||
2026-06-16 17:58:53 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E5AB2CF750>
|
||||
2026-06-16 17:58:54 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E5AC458910>
|
||||
2026-06-16 17:58:55 | INFO | Database connections established
|
||||
2026-06-16 17:58:55 | INFO | Collecting MIDs for: 2026-06-15
|
||||
2026-06-16 17:58:55 | INFO | Found 849 MIDs
|
||||
2026-06-16 18:00:31 | INFO | ================================================================================
|
||||
2026-06-16 18:00:31 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-16 18:00:31 | INFO | Pipeline Run Date: 2026-06-15
|
||||
2026-06-16 18:00:31 | INFO | Connecting to databases...
|
||||
2026-06-16 18:00:38 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E8C854F750>
|
||||
2026-06-16 18:00:40 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E8C9688910>
|
||||
2026-06-16 18:00:40 | INFO | Database connections established
|
||||
2026-06-16 18:00:40 | INFO | Collecting MIDs for: 2026-06-15
|
||||
2026-06-16 18:00:42 | INFO | Found 849 MIDs
|
||||
2026-06-16 18:00:44 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-16 18:02:58 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-16 18:07:47 | INFO | ================================================================================
|
||||
2026-06-16 18:07:47 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-16 18:07:47 | INFO | Pipeline Run Date: 2026-06-15
|
||||
2026-06-16 18:07:47 | INFO | Connecting to databases...
|
||||
2026-06-16 18:07:48 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DF4BB2F750>
|
||||
2026-06-16 18:07:50 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DF4CCE8A50>
|
||||
2026-06-16 18:07:50 | INFO | Database connections established
|
||||
2026-06-16 18:07:50 | INFO | Collecting MIDs for: 2026-06-15
|
||||
2026-06-16 18:07:50 | INFO | Found 849 MIDs
|
||||
2026-06-16 18:07:51 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-16 18:08:09 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-16 18:08:58 | INFO | ================================================================================
|
||||
2026-06-16 18:08:58 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-16 18:08:58 | INFO | Pipeline Run Date: 2026-06-15
|
||||
2026-06-16 18:08:58 | INFO | Connecting to databases...
|
||||
2026-06-16 18:08:59 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001639269F750>
|
||||
2026-06-16 18:09:01 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000016393858910>
|
||||
2026-06-16 18:09:01 | INFO | Database connections established
|
||||
2026-06-16 18:09:01 | INFO | Collecting MIDs for: 2026-06-15
|
||||
2026-06-16 18:09:02 | INFO | Found 849 MIDs
|
||||
2026-06-16 18:09:02 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-16 18:09:19 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-16 18:11:21 | INFO | ================================================================================
|
||||
2026-06-16 18:11:21 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-16 18:11:21 | INFO | Pipeline Run Date: 2026-06-15
|
||||
2026-06-16 18:11:21 | INFO | Connecting to databases...
|
||||
2026-06-16 18:11:23 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002240B7AF750>
|
||||
2026-06-16 18:11:24 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002240C978910>
|
||||
2026-06-16 18:11:24 | INFO | Database connections established
|
||||
2026-06-16 18:11:24 | INFO | Collecting MIDs for: 2026-06-15
|
||||
2026-06-16 18:11:25 | INFO | Found 849 MIDs
|
||||
2026-06-16 18:11:25 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-16 18:13:05 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
|
||||
@@ -1,688 +0,0 @@
|
||||
2026-06-17 10:17:16 | INFO | ================================================================================
|
||||
2026-06-17 10:17:16 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 10:17:16 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 10:17:16 | INFO | Connecting to databases...
|
||||
2026-06-17 10:17:18 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000019A9A2CF750>
|
||||
2026-06-17 10:17:21 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000019A9B778910>
|
||||
2026-06-17 10:17:22 | INFO | Database connections established
|
||||
2026-06-17 10:17:22 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 10:17:22 | INFO | Found 863 MIDs
|
||||
2026-06-17 10:17:23 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 10:17:44 | INFO | Fetched 290,984 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 10:43:05 | INFO | ================================================================================
|
||||
2026-06-17 10:43:05 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 10:43:05 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 10:43:05 | INFO | Connecting to databases...
|
||||
2026-06-17 10:43:07 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002CE210FF750>
|
||||
2026-06-17 10:43:08 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002CE2253CA50>
|
||||
2026-06-17 10:43:09 | INFO | Database connections established
|
||||
2026-06-17 10:43:09 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 10:43:10 | INFO | Found 863 MIDs
|
||||
2026-06-17 10:43:11 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 10:43:55 | INFO | Fetched 291,001 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 10:45:51 | INFO | ================================================================================
|
||||
2026-06-17 10:45:51 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 10:45:51 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 10:45:51 | INFO | Connecting to databases...
|
||||
2026-06-17 10:45:52 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000214EC45F750>
|
||||
2026-06-17 10:45:53 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000214ED908910>
|
||||
2026-06-17 10:45:54 | INFO | Database connections established
|
||||
2026-06-17 10:45:54 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 10:45:55 | INFO | Found 863 MIDs
|
||||
2026-06-17 10:45:55 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 10:46:11 | INFO | Fetched 291,005 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 10:46:11 | INFO | ================================================================================
|
||||
2026-06-17 10:46:11 | INFO | Processing Table: OQaD
|
||||
2026-06-17 10:46:11 | ERROR | Failed processing table OQaD
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 212, in main
|
||||
df=fetch_OQaD( engine=sql_engine,
|
||||
table_name=table_name,
|
||||
...<2 lines>...
|
||||
run_date=run_date
|
||||
)
|
||||
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
|
||||
2026-06-17 10:48:44 | INFO | ================================================================================
|
||||
2026-06-17 10:48:44 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 10:48:44 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 10:48:44 | INFO | Connecting to databases...
|
||||
2026-06-17 10:48:45 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000021A270378B0>
|
||||
2026-06-17 10:48:46 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000021A28538910>
|
||||
2026-06-17 10:48:47 | INFO | Database connections established
|
||||
2026-06-17 10:48:47 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 10:48:48 | INFO | Found 863 MIDs
|
||||
2026-06-17 10:48:48 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 10:49:00 | INFO | Fetched 291,005 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 10:49:01 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 10:49:01 | INFO | ================================================================================
|
||||
2026-06-17 10:49:01 | INFO | Processing Table: OQaD
|
||||
2026-06-17 10:49:01 | ERROR | Failed processing table OQaD
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 215, in main
|
||||
df=fetch_OQaD( engine=sql_engine,
|
||||
table_name=table_name,
|
||||
...<2 lines>...
|
||||
run_date=run_date
|
||||
)
|
||||
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
|
||||
2026-06-17 10:53:50 | INFO | ================================================================================
|
||||
2026-06-17 10:53:50 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 10:53:50 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 10:53:50 | INFO | Connecting to databases...
|
||||
2026-06-17 10:53:51 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001D1D6DB78B0>
|
||||
2026-06-17 10:53:52 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001D1D81F8910>
|
||||
2026-06-17 10:53:53 | INFO | Database connections established
|
||||
2026-06-17 10:53:53 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 10:53:54 | INFO | Found 863 MIDs
|
||||
2026-06-17 10:53:54 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 10:54:08 | INFO | Fetched 291,006 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 10:54:09 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 10:54:09 | INFO | ================================================================================
|
||||
2026-06-17 10:54:09 | INFO | Processing Table: OQaD
|
||||
2026-06-17 10:54:09 | ERROR | Failed processing table OQaD
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 217, in main
|
||||
df=fetch_OQaD( engine=sql_engine,
|
||||
table_name=table_name,
|
||||
...<2 lines>...
|
||||
run_date=run_date
|
||||
)
|
||||
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
|
||||
2026-06-17 10:57:28 | INFO | ================================================================================
|
||||
2026-06-17 10:57:28 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 10:57:28 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 10:57:28 | INFO | Connecting to databases...
|
||||
2026-06-17 10:57:30 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000197AB2778B0>
|
||||
2026-06-17 10:57:31 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000197AC768910>
|
||||
2026-06-17 10:57:32 | INFO | Database connections established
|
||||
2026-06-17 10:57:32 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 10:57:32 | INFO | Found 863 MIDs
|
||||
2026-06-17 10:57:33 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 10:57:45 | INFO | Fetched 291,010 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 10:57:46 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 10:57:46 | INFO | ================================================================================
|
||||
2026-06-17 10:57:46 | INFO | Processing Table: OQaD
|
||||
2026-06-17 10:57:46 | ERROR | Failed processing table OQaD
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 217, in main
|
||||
df=fetch_OQaD( engine=sql_engine,
|
||||
table_name=table_name,
|
||||
...<2 lines>...
|
||||
run_date=run_date
|
||||
)
|
||||
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
|
||||
2026-06-17 10:59:57 | INFO | ================================================================================
|
||||
2026-06-17 10:59:57 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 10:59:57 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 10:59:57 | INFO | Connecting to databases...
|
||||
2026-06-17 10:59:58 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001A32C2078B0>
|
||||
2026-06-17 11:00:00 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001A32D648910>
|
||||
2026-06-17 11:00:00 | INFO | Database connections established
|
||||
2026-06-17 11:00:00 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 11:00:01 | INFO | Found 863 MIDs
|
||||
2026-06-17 11:00:02 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 11:00:24 | INFO | Fetched 291,011 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 11:00:25 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 11:00:25 | INFO | ================================================================================
|
||||
2026-06-17 11:00:25 | INFO | Processing Table: OQaD
|
||||
2026-06-17 11:00:25 | ERROR | Failed processing table OQaD
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 217, in main
|
||||
df=fetch_OQaD( engine=sql_engine,
|
||||
table_name=table_name,
|
||||
...<2 lines>...
|
||||
run_date=run_date
|
||||
)
|
||||
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\fact\\OQaD.sql'
|
||||
2026-06-17 12:09:34 | INFO | ================================================================================
|
||||
2026-06-17 12:09:34 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 12:09:34 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 12:09:34 | INFO | Connecting to databases...
|
||||
2026-06-17 12:09:37 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000274B86078B0>
|
||||
2026-06-17 12:09:39 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000274B9A18910>
|
||||
2026-06-17 12:09:39 | INFO | Database connections established
|
||||
2026-06-17 12:09:39 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 12:09:41 | INFO | Found 863 MIDs
|
||||
2026-06-17 12:09:43 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 12:10:48 | INFO | Fetched 291,093 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 12:10:48 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 12:10:48 | INFO | ================================================================================
|
||||
2026-06-17 12:10:48 | INFO | Processing Table: OQaD
|
||||
2026-06-17 12:10:48 | INFO | Exists: False
|
||||
2026-06-17 12:10:48 | INFO | Path: D:\data_move\sql\fact\OQaD.sql
|
||||
2026-06-17 12:10:48 | ERROR | Failed processing table OQaD
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 217, in main
|
||||
df=fetch_OQaD( engine=sql_engine,
|
||||
table_name=table_name,
|
||||
...<2 lines>...
|
||||
run_date=run_date
|
||||
)
|
||||
File "D:\data_move\src\fact.py", line 62, in fetch_OQaD
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\fact\\OQaD.sql'
|
||||
2026-06-17 12:11:47 | INFO | ================================================================================
|
||||
2026-06-17 12:11:47 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 12:11:47 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 12:11:47 | INFO | Connecting to databases...
|
||||
2026-06-17 12:11:48 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000241C8DF78B0>
|
||||
2026-06-17 12:11:50 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000241CA238910>
|
||||
2026-06-17 12:11:51 | INFO | Database connections established
|
||||
2026-06-17 12:11:51 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 12:11:52 | INFO | Found 863 MIDs
|
||||
2026-06-17 12:11:52 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 12:12:14 | INFO | Fetched 291,097 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 12:12:14 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 12:12:14 | INFO | ================================================================================
|
||||
2026-06-17 12:12:14 | INFO | Processing Table: OQaD
|
||||
2026-06-17 12:12:14 | INFO | Exists: True
|
||||
2026-06-17 12:12:14 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
||||
2026-06-17 12:12:14 | INFO | Fetching data for 893 EMPIDs
|
||||
2026-06-17 12:12:14 | INFO | Fetching OQaD data for run_date=2026-06-16
|
||||
2026-06-17 12:12:14 | ERROR | Failed processing table OQaD
|
||||
Traceback (most recent call last):
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context
|
||||
self.dialect.do_execute(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
cursor, str_statement, effective_parameters, context
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute
|
||||
cursor.execute(statement, parameters)
|
||||
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near 'QUIZ'. (102) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'where'. (156)")
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 217, in main
|
||||
df=fetch_OQaD( engine=sql_engine,
|
||||
table_name=table_name,
|
||||
...<2 lines>...
|
||||
run_date=run_date
|
||||
)
|
||||
File "D:\data_move\src\fact.py", line 74, in fetch_OQaD
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine,
|
||||
)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database
|
||||
return cx.execute(
|
||||
~~~~~~~~~~^
|
||||
query=query,
|
||||
^^^^^^^^^^^^
|
||||
options=execute_options,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
).to_polars(
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute
|
||||
result = cursor_execute(query, **options)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute
|
||||
return meth(
|
||||
self,
|
||||
distilled_parameters,
|
||||
execution_options or NO_OPTIONS,
|
||||
)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection
|
||||
return connection._execute_clauseelement(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
self, distilled_params, execution_options
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement
|
||||
ret = self._execute_context(
|
||||
dialect,
|
||||
...<8 lines>...
|
||||
cache_hit=cache_hit,
|
||||
)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context
|
||||
return self._exec_single_context(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
dialect, context, statement, parameters
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context
|
||||
self._handle_dbapi_exception(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
e, str_statement, effective_parameters, cursor, context
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception
|
||||
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context
|
||||
self.dialect.do_execute(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
cursor, str_statement, effective_parameters, context
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute
|
||||
cursor.execute(statement, parameters)
|
||||
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near 'QUIZ'. (102) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'where'. (156)")
|
||||
[SQL: WITH MID_TABLE_COV1 AS
|
||||
(
|
||||
SELECT EmpId, VisitDate
|
||||
FROM OneApp_KelloggsMT.dbo.T_OQAD
|
||||
WHERE CreateDate >= 2026-06-16
|
||||
AND CreateDate < DATEADD(DAY,1,'2026-06-16')
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT EmpId, VisitDate
|
||||
FROM OneApp_KelloggsMT.dbo.T_OQAD
|
||||
WHERE UpdateDate >= 2026-06-16
|
||||
AND UpdateDate < DATEADD(DAY,1, '2026-06-16')
|
||||
),
|
||||
|
||||
OQaD
|
||||
|
||||
|
||||
QUIZ AS
|
||||
(
|
||||
SELECT DISTINCT
|
||||
E.EmpId,
|
||||
E.EmpName,
|
||||
E.SupervisorId,
|
||||
E.SupervisorName,
|
||||
E.DesignationName,
|
||||
E.CityName,
|
||||
E.StateName,
|
||||
E.RegionName,
|
||||
CAST(DQ.VisitDate AS DATE) AS VisitDate,
|
||||
DQ.QuestionId,
|
||||
DQ.AnswerId,
|
||||
QC.QuestionCategoryId,
|
||||
QC.QuestionCategory
|
||||
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E
|
||||
ON DQ.EmpId = E.EmpId
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU
|
||||
ON DQ.QuestionId = QU.QuestionId
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC
|
||||
ON QU.QuestionCategoryId = QC.QuestionCategoryId
|
||||
WHERE E.EmpName NOT LIKE '%TEST%'
|
||||
AND E.RightId = 6
|
||||
AND (
|
||||
E.ResignDate IS NULL
|
||||
OR CAST(E.ResignDate AS DATE) >= '2026-06-16'
|
||||
)
|
||||
AND EXISTS
|
||||
(
|
||||
SELECT 1
|
||||
FROM MID_TABLE_COV1 A
|
||||
WHERE A.EmpId = DQ.EmpId
|
||||
AND CAST(A.VisitDate AS DATE) = CAST(DQ.VisitDate AS DATE)
|
||||
)
|
||||
)
|
||||
|
||||
SELECT
|
||||
40148 AS project_id,
|
||||
Q.EmpId AS employee_id,
|
||||
0 AS process_id,
|
||||
Q.VisitDate AS visit_date,
|
||||
Q.QuestionCategoryId AS question_category_id,
|
||||
Q.QuestionCategory AS question_category,
|
||||
QM.QuestionId AS question_id,
|
||||
QM.Question AS question,
|
||||
ISNULL(QA.AnswerId,0) AS answer_id,
|
||||
ISNULL(QA.Answer,'') AS answer,
|
||||
CASE
|
||||
WHEN QA.AnswerId IS NULL THEN 'Not Answer'
|
||||
WHEN QA.RightAnswer = 1 THEN 'Y'
|
||||
WHEN QA.RightAnswer IS NULL THEN 'Not Answer'
|
||||
ELSE 'N'
|
||||
END AS correct_answer,
|
||||
GETDATE() AS update_date,
|
||||
'SP-Pius' AS update_by
|
||||
FROM QUIZ Q
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM
|
||||
ON Q.QuestionId = QM.QuestionId
|
||||
LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA
|
||||
ON Q.AnswerId = QA.AnswerId;
|
||||
where Q.EmpId not in (3167,4724,4019,4902,4375,3352,3081,5244,3181,4810,5166,4895,4817,5173,5237,3608,4646,4126,3243,3948,3950,3599,3777,5235,5346,3606,3079,5175,5242,5064,4366,5071,5339,4812,4722,3169,5270,3326,3689,4062,3131,5054,5178,3371,4829,5185,4698,4836,3760,4665,5218,4451,4280,3269,3098,5327,4978,5261,3335,5090,5320,4214,4919,4570,3150,4221,3328,3112,3290,4950,4430,5104,3383,5213,4864,5199,5206,4439,3207,5033,4684,3979,5211,4774,3117,5102,5280,3309,4931,4753,5123,3155,5085,4914,5092,4387,5325,5154,4100,4805,3402,5187,3580,3409,3060,3962,4696,4831,3715,4710,4242,5282,5121,4933,5289,5282,3444,5104,3095,4933,4142,3437,3530,3088,3715,4840,5002,4831,5187,3361,3962,5113,3086,5106,5256,5085,4914,5092,3470,4387,5277,3309,5239,4712,4945,5246,5123,4774,3952,3603,4999,3727,3378,3207,5341,4992,5199,5334,4672,5206,3342,3520,3454,3171,4916,3105,4396,3112,3290,5265,3119,5227,3143,3150,5056,4529,3328,5313,5220,4693,5320,4615,3755,4444,3406,4978,4451,3746,4280,3399,5322,4617,4795,5218,3330,3159,4698,5144,5054,3473,3131,4726,4688,4743,3689,3812,3107,3169,5332,3347,5023,5022,4812,3380,3067,3950,4954,5132,3081,5296,5244,3140,4019,3492,4375,5210,3126,5289,3482,4847,5296,4954,4436,3380,5127,5305,3306,5291,5023,5120,4531,4346,5229,3107,3812,4353,3145,3152,3330,5322,4439,5144,4795,3363,4054,4232,3399,4239,3755,5279,4930,5213,3406,4864,5286,5220,4693,5101,5227,3985,3297,3636,4529,3420,5265,4916,3591,3335,4460,5343,4638,3342,5327,3520,3171,4467,3700,5336,4987,3515,4992,3760,5267,4562,3247,5082,4555,4733,5061,5239,3090,4712,5246,4904,5270,3059,4866,5277,4750,5106,3361,4968,5175,5310,3496,3608,4840,5248,4543,3615,4142,3437,3088,3444,3078,4902,3095,3567,4543,4450,3140,5319,4614,3496,4970,4443,5132,3916,3149,5310,3327,5312,3156,4968,4894,3394,5065,4538,4688,4866,3251,3090,3268,4671,4849,4904,4726,5082,4555,5191,4733,4562,3515,5300,4951,5115,3173,5293,4944,4460,4987,4638,4467,3591,3064,4526,3821,3232,3420,5056,5046,5008,3985,3636,5350,3109,5272,5101,5279,3358,4396,5172,5110,4054,4232,5281,3363,4999,3638,3603,3104,5027,5205,4856,4151,3083,4346,3764,3819,3415,4353,3470,3299,5167,3346,5113,3306,5331,5291,5160,5120,5127,4913,5091,4564,5262,5196,4847,5077,4666,4139,3612,3978,3078,5340,4139,3612,4991,5248,3170,5155,5333,4628,5077,4217,5084,4032,4913,4564,5257,5324,4381,5331,5250,5160,4134,4312,4141,3059,3764,3415,4305,5027,3220,3104,5321,4972,5307,5267,5136,5314,4787,3315,3322,4540,3835,3700,3486,4547,5172,5001,5008,3427,3379,5046,3372,3821,4526,3650,5124,5302,5286,3130,5115,4937,3132,5293,5117,3118,4239,5300,4951,4343,3897,5219,5191,4671,4849,5212,5229,5003,5181,4531,5065,5345,3538,4538,4189,4229,3531,5276,5305,4578,3106,3149,5312,5269,3113,4436,3099,4443,4970,3916,3567,3975,4153,4844,5210,5200,4844,3533,4666,5022,3184,5200,4402,4153,3704,3975,4922,3099,5262,4751,5091,3106,5269,5093,5276,4578,3346,5202,4675,4229,3531,3182,5195,5167,3203,5003,5205,5150,5328,3220,4979,4856,5157,4452,5335,3514,5219,3638,4343,4383,3289,4390,3118,3296,5281,5110,3438,3267,5074,5117,5252,3780,3431,5005,3372,3379,5176,3232,5316,4611,4967,3068,5309,3139,3495,3673,4540,3835,4891,3486,3488,3315,3322,5307,5136,3070,4877,4787,5314,5055,3939,3234,4072,5181,4305,3251,3374,5119,4134,3080,5297,4065,4894,3127,5250,4763,4941,5257,3298,4381,3327,3156,4687,4338,5221,4450,5155,3400,5333,5207,4628,5162,3742,4098,3742,4943,5121,3129,5207,5114,5292,4687,3122,4338,4696,3464,3113,4333,3298,3127,5031,4941,4763,5209,5119,3388,3566,3538,4065,3744,4072,3374,4753,3234,3939,5102,5088,5266,3070,5055,4877,3096,3979,3132,3488,3139,4841,5019,5302,3495,5124,5309,5012,4611,3369,3509,5176,5145,5323,5005,3153,4656,4371,3431,5254,5252,4547,3091,5074,4198,3782,3433,4383,4651,4829,4302,3953,5349,5321,3514,4972,5178,5150,5328,4979,5342,4452,5157,5304,3203,5195,3490,4312,3141,5126,4141,4675,4366,5057,5235,5064,3243,5093,4217,4922,3407,4751,4810,3170,4283,3756,3704,4974,3533,4991,3184,4974,3492,3143,4098,4283,3400,3756,3407,4710,5221,3067,3580,5057,3409,5196,3060,4805,4100,3402,5325,5297,3490,3141,5154,5126,3155,5304,4931,5109,5287,4117,5280,3473,4651,4302,3953,5033,5211,3782,4684,3433,3084,4891,3746,3383,4371,4022,5254,3673,5313,4207,3153,5316,3509,5145,5323,4430,4919,4570,3369,4221,3105,5090,4563,5012,3269,4841,3098,5019,3454,5261,3438,4672,3267,4836,3096,4665,5259,5088,3727,5266,3378,5185,4390,4397,3371,4062,4945,3528,3326,4722,3388,5256,3566,3744,5202,5242,3086,5071,5209,3777,4333,3606,3079,5002,3948,4126,3599,3464,5344,4817,5173,5351,4646,3122,3352,3530,5337,3181,3359,5166,5292,5114,4943)
|
||||
-- select d.employee_id from OQaD d where project_id=40148 and d.employee_id=Q.EmpId and CONVERT(date,d.visit_date,101)=
|
||||
-- CONVERT(date,q.VisitDate,101))
|
||||
|
||||
]
|
||||
(Background on this error at: https://sqlalche.me/e/20/f405)
|
||||
2026-06-17 12:19:35 | INFO | ================================================================================
|
||||
2026-06-17 12:19:35 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 12:19:35 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 12:19:35 | INFO | Connecting to databases...
|
||||
2026-06-17 12:19:37 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000210484378B0>
|
||||
2026-06-17 12:19:38 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000021049928910>
|
||||
2026-06-17 12:19:39 | INFO | Database connections established
|
||||
2026-06-17 12:19:39 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 12:19:40 | INFO | Found 863 MIDs
|
||||
2026-06-17 12:19:40 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 12:20:14 | INFO | Fetched 291,100 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 12:20:14 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 12:20:14 | INFO | ================================================================================
|
||||
2026-06-17 12:20:14 | INFO | Processing Table: OQaD
|
||||
2026-06-17 12:20:14 | INFO | Exists: True
|
||||
2026-06-17 12:20:14 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
||||
2026-06-17 12:20:14 | INFO | Fetching data for 893 EMPIDs
|
||||
2026-06-17 12:20:14 | INFO | Fetching OQaD data for run_date=2026-06-16
|
||||
2026-06-17 12:20:30 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
|
||||
2026-06-17 12:20:30 | ERROR | Failed processing table OQaD
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 226, in main
|
||||
df = get_dataframe(
|
||||
fn_name=fn_name,
|
||||
...<3 lines>...
|
||||
run_date=run_date,
|
||||
)
|
||||
File "D:\data_move\main2.py", line 81, in get_dataframe
|
||||
return fn(sql_engine, run_date)
|
||||
TypeError: fetch_OQaD() missing 3 required positional arguments: 'table_type', 'empids', and 'run_date'
|
||||
2026-06-17 12:38:43 | INFO | ================================================================================
|
||||
2026-06-17 12:38:43 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 12:38:43 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 12:38:43 | INFO | Connecting to databases...
|
||||
2026-06-17 12:38:45 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001C35AC2F750>
|
||||
2026-06-17 12:38:46 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001C35BDF87D0>
|
||||
2026-06-17 12:38:46 | INFO | Database connections established
|
||||
2026-06-17 12:38:46 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 12:38:47 | INFO | Found 863 MIDs
|
||||
2026-06-17 12:38:47 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 12:38:58 | INFO | Fetched 291,112 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 12:38:58 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 12:38:58 | INFO | ================================================================================
|
||||
2026-06-17 12:38:58 | INFO | Processing Table: OQaD
|
||||
2026-06-17 12:38:58 | INFO | Exists: True
|
||||
2026-06-17 12:38:58 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
||||
2026-06-17 12:38:58 | INFO | Fetching data for 893 EMPIDs
|
||||
2026-06-17 12:38:58 | INFO | Fetching OQaD data for run_date=2026-06-16
|
||||
2026-06-17 12:39:05 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
|
||||
2026-06-17 12:39:05 | INFO | ================================================================================
|
||||
2026-06-17 12:39:05 | INFO | Pipeline Completed Successfully
|
||||
2026-06-17 12:39:05 | INFO | ================================================================================
|
||||
2026-06-17 12:39:30 | INFO | ================================================================================
|
||||
2026-06-17 12:39:30 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 12:39:30 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 12:39:30 | INFO | Connecting to databases...
|
||||
2026-06-17 12:39:31 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000025D1633F750>
|
||||
2026-06-17 12:39:32 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000025D175887D0>
|
||||
2026-06-17 12:39:33 | INFO | Database connections established
|
||||
2026-06-17 12:39:33 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 12:39:33 | INFO | Found 863 MIDs
|
||||
2026-06-17 12:39:33 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 12:39:43 | INFO | Fetched 291,112 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 12:39:43 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 12:39:43 | INFO | ================================================================================
|
||||
2026-06-17 12:39:43 | INFO | Processing Table: OQaD
|
||||
2026-06-17 12:39:43 | INFO | Exists: True
|
||||
2026-06-17 12:39:43 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
||||
2026-06-17 12:39:43 | INFO | Fetching data for 893 EMPIDs
|
||||
2026-06-17 12:39:43 | INFO | Fetching OQaD data for run_date=2026-06-16
|
||||
2026-06-17 12:39:46 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
|
||||
2026-06-17 12:39:46 | INFO | ================================================================================
|
||||
2026-06-17 12:39:46 | INFO | Pipeline Completed Successfully
|
||||
2026-06-17 12:39:46 | INFO | ================================================================================
|
||||
2026-06-17 12:44:50 | INFO | ================================================================================
|
||||
2026-06-17 12:44:50 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 12:44:50 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 12:44:50 | INFO | Connecting to databases...
|
||||
2026-06-17 12:44:51 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E3647178B0>
|
||||
2026-06-17 12:44:52 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E365BB8910>
|
||||
2026-06-17 12:44:53 | INFO | Database connections established
|
||||
2026-06-17 12:44:53 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 12:44:53 | INFO | Found 863 MIDs
|
||||
2026-06-17 12:44:53 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 12:45:07 | INFO | Fetched 291,115 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 12:45:08 | INFO | Fetched 893 matched empids fetched for OQAD
|
||||
2026-06-17 12:45:08 | INFO | ================================================================================
|
||||
2026-06-17 12:45:08 | INFO | Processing Table: OQaD
|
||||
2026-06-17 12:45:08 | INFO | Exists: True
|
||||
2026-06-17 12:45:08 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
||||
2026-06-17 12:45:08 | INFO | Fetching data for 893 EMPIDs
|
||||
2026-06-17 12:45:08 | INFO | Fetching OQaD data for run_date=2026-06-16
|
||||
2026-06-17 12:45:10 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
|
||||
2026-06-17 12:45:10 | INFO | Fetched 4998 rows
|
||||
2026-06-17 12:45:10 | INFO | Deleting from OQaD
|
||||
2026-06-17 12:45:11 | INFO | OQaD: inserted 4,998 rows into ClickHouse
|
||||
2026-06-17 12:45:11 | INFO | OQaD loaded successfully (4998 rows)
|
||||
2026-06-17 12:45:11 | INFO | ================================================================================
|
||||
2026-06-17 12:45:11 | INFO | Pipeline Completed Successfully
|
||||
2026-06-17 12:45:11 | INFO | ================================================================================
|
||||
2026-06-17 13:23:53 | INFO | ================================================================================
|
||||
2026-06-17 13:23:53 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 13:23:53 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 13:23:53 | INFO | Connecting to databases...
|
||||
2026-06-17 13:23:54 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000281306878B0>
|
||||
2026-06-17 13:23:56 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000028131AD8910>
|
||||
2026-06-17 13:23:56 | INFO | Database connections established
|
||||
2026-06-17 13:23:56 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 13:23:57 | INFO | Found 863 MIDs
|
||||
2026-06-17 13:23:58 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 13:24:15 | INFO | Fetched 291,128 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 13:24:15 | INFO | Fetched 4,819 matched empids fetched for OQAD
|
||||
2026-06-17 13:24:15 | INFO | ================================================================================
|
||||
2026-06-17 13:24:15 | INFO | Processing Table: OQaD
|
||||
2026-06-17 13:24:15 | INFO | Exists: True
|
||||
2026-06-17 13:24:15 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
||||
2026-06-17 13:24:15 | INFO | Fetching data for 4,819 EMPIDs
|
||||
2026-06-17 13:24:15 | INFO | Fetching OQaD data for run_date=2026-06-16
|
||||
2026-06-17 13:24:22 | INFO | fn name is fetch_OQad ------Fetched 1072 rows
|
||||
2026-06-17 13:24:22 | INFO | Fetched 1072 rows
|
||||
2026-06-17 13:24:22 | INFO | Deleting from OQaD
|
||||
2026-06-17 13:24:23 | INFO | OQaD: inserted 1,072 rows into ClickHouse
|
||||
2026-06-17 13:24:23 | INFO | OQaD loaded successfully (1072 rows)
|
||||
2026-06-17 13:24:23 | INFO | ================================================================================
|
||||
2026-06-17 13:24:23 | INFO | Processing Table: Survey
|
||||
2026-06-17 13:24:23 | INFO | Fetching data for 863 MIDs
|
||||
2026-06-17 13:24:23 | INFO | Fetched 142 rows from SQL Server
|
||||
2026-06-17 13:24:23 | INFO | Fetched 142 rows
|
||||
2026-06-17 13:24:23 | INFO | Deleting from Survey
|
||||
2026-06-17 13:24:24 | ERROR | Failed processing table Survey
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 299, in main
|
||||
load_to_clickhouse(
|
||||
~~~~~~~~~~~~~~~~~~^
|
||||
client=client,
|
||||
^^^^^^^^^^^^^^
|
||||
table_name=table_name,
|
||||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
df=df,
|
||||
^^^^^^
|
||||
)
|
||||
^
|
||||
File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse
|
||||
client.insert_arrow(
|
||||
~~~~~~~~~~~~~~~~~~~^
|
||||
table=table_name,
|
||||
^^^^^^^^^^^^^^^^^
|
||||
arrow_table=arrow_table,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow
|
||||
return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings)
|
||||
~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert
|
||||
response = self._raw_request(insert_block, params, headers, server_wait=False)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request
|
||||
self._error_handler(response)
|
||||
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler
|
||||
raise err_type(err_str, code=code, name=name) from None
|
||||
clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column Project_Id in table kelloggs_1.Survey (d8c44908-2ed7-4d29-bd10-8e70fa84dfba). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123)
|
||||
2026-06-17 13:27:22 | INFO | ================================================================================
|
||||
2026-06-17 13:27:22 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 13:27:22 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 13:27:22 | INFO | Connecting to databases...
|
||||
2026-06-17 13:27:23 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000026D13D878B0>
|
||||
2026-06-17 13:27:25 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000026D151D8910>
|
||||
2026-06-17 13:27:25 | INFO | Database connections established
|
||||
2026-06-17 13:27:25 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 13:27:26 | INFO | Found 863 MIDs
|
||||
2026-06-17 13:27:26 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 13:27:37 | INFO | Fetched 291,128 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 13:27:38 | INFO | Fetched 4,814 matched empids fetched for OQAD
|
||||
2026-06-17 13:27:38 | INFO | ================================================================================
|
||||
2026-06-17 13:27:38 | INFO | Processing Table: OQaD
|
||||
2026-06-17 13:27:38 | INFO | Exists: True
|
||||
2026-06-17 13:27:38 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
||||
2026-06-17 13:27:38 | INFO | Fetching data for 4,814 EMPIDs
|
||||
2026-06-17 13:27:38 | INFO | Fetching OQaD data for run_date=2026-06-16
|
||||
2026-06-17 13:27:44 | INFO | fn name is fetch_OQad ------Fetched 1072 rows
|
||||
2026-06-17 13:27:44 | INFO | Fetched 1072 rows
|
||||
2026-06-17 13:27:44 | INFO | Deleting from OQaD
|
||||
2026-06-17 13:27:45 | INFO | OQaD: inserted 1,072 rows into ClickHouse
|
||||
2026-06-17 13:27:45 | INFO | OQaD loaded successfully (1072 rows)
|
||||
2026-06-17 13:27:45 | INFO | ================================================================================
|
||||
2026-06-17 13:27:45 | INFO | Processing Table: Survey
|
||||
2026-06-17 13:27:45 | INFO | Fetching data for 863 MIDs
|
||||
2026-06-17 13:27:45 | INFO | Fetched 142 rows from SQL Server
|
||||
2026-06-17 13:27:45 | INFO | Fetched 142 rows
|
||||
2026-06-17 13:27:45 | INFO | Creating table Survey
|
||||
2026-06-17 13:27:45 | INFO | Table ready: Survey
|
||||
2026-06-17 13:27:45 | INFO | Survey: inserted 142 rows into ClickHouse
|
||||
2026-06-17 13:27:45 | INFO | Survey loaded successfully (142 rows)
|
||||
2026-06-17 13:27:45 | INFO | ================================================================================
|
||||
2026-06-17 13:27:45 | INFO | Pipeline Completed Successfully
|
||||
2026-06-17 13:27:45 | INFO | ================================================================================
|
||||
2026-06-17 15:05:42 | INFO | ================================================================================
|
||||
2026-06-17 15:05:42 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 15:05:42 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 15:05:42 | INFO | Connecting to databases...
|
||||
2026-06-17 15:05:43 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002009FB778B0>
|
||||
2026-06-17 15:05:45 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000200A0FB8910>
|
||||
2026-06-17 15:05:45 | INFO | Database connections established
|
||||
2026-06-17 15:05:45 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 15:05:45 | INFO | Found 863 MIDs
|
||||
2026-06-17 15:05:46 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 15:05:57 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 15:05:57 | INFO | Fetched 4,814 matched empids fetched for OQAD
|
||||
2026-06-17 15:05:57 | INFO | ================================================================================
|
||||
2026-06-17 15:05:57 | INFO | Processing Table: additional_visibility
|
||||
2026-06-17 15:05:57 | INFO | Fetching data for 863 MIDs
|
||||
2026-06-17 15:05:58 | ERROR | Failed processing table additional_visibility
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 228, in main
|
||||
df = fetch_data(
|
||||
engine=sql_engine,
|
||||
...<3 lines>...
|
||||
run_date=run_date,
|
||||
)
|
||||
File "D:\data_move\src\fact.py", line 34, in fetch_data
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 287, in read_database
|
||||
).to_polars(
|
||||
~~~~~~~~~^
|
||||
batch_size=batch_size,
|
||||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
...<2 lines>...
|
||||
infer_schema_length=infer_schema_length,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 588, in to_polars
|
||||
frame = frame_init(
|
||||
batch_size=batch_size,
|
||||
...<2 lines>...
|
||||
infer_schema_length=infer_schema_length,
|
||||
)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 285, in _from_rows
|
||||
(d[0], d[1:]) for d in self.result.cursor.description
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
AttributeError: 'NoneType' object has no attribute 'description'
|
||||
2026-06-17 15:07:15 | INFO | ================================================================================
|
||||
2026-06-17 15:07:15 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 15:07:15 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 15:07:15 | INFO | Connecting to databases...
|
||||
2026-06-17 15:07:16 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001BD510978B0>
|
||||
2026-06-17 15:07:17 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001BD52518910>
|
||||
2026-06-17 15:07:18 | INFO | Database connections established
|
||||
2026-06-17 15:07:18 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 15:07:18 | INFO | Found 863 MIDs
|
||||
2026-06-17 15:07:18 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 15:07:31 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 15:07:31 | INFO | Fetched 4,814 matched empids fetched for OQAD
|
||||
2026-06-17 15:07:31 | INFO | ================================================================================
|
||||
2026-06-17 15:07:31 | INFO | Processing Table: additional_visibility
|
||||
2026-06-17 15:07:31 | INFO | Fetching data for 863 MIDs
|
||||
2026-06-17 15:07:32 | INFO | Fetched 1,773 rows from SQL Server
|
||||
2026-06-17 15:07:32 | INFO | Fetched 1773 rows
|
||||
2026-06-17 15:07:32 | INFO | Deleting from additional_visibility
|
||||
2026-06-17 15:07:32 | ERROR | Failed processing table additional_visibility
|
||||
Traceback (most recent call last):
|
||||
File "D:\data_move\main2.py", line 299, in main
|
||||
load_to_clickhouse(
|
||||
~~~~~~~~~~~~~~~~~~^
|
||||
client=client,
|
||||
^^^^^^^^^^^^^^
|
||||
table_name=table_name,
|
||||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
df=df,
|
||||
^^^^^^
|
||||
)
|
||||
^
|
||||
File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse
|
||||
client.insert_arrow(
|
||||
~~~~~~~~~~~~~~~~~~~^
|
||||
table=table_name,
|
||||
^^^^^^^^^^^^^^^^^
|
||||
arrow_table=arrow_table,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow
|
||||
return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings)
|
||||
~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert
|
||||
response = self._raw_request(insert_block, params, headers, server_wait=False)
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request
|
||||
self._error_handler(response)
|
||||
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
|
||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler
|
||||
raise err_type(err_str, code=code, name=name) from None
|
||||
clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column project_id in table kelloggs_1.additional_visibility (9e03e625-028a-47db-8f60-e8ab93e907ea). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123)
|
||||
2026-06-17 15:08:55 | INFO | ================================================================================
|
||||
2026-06-17 15:08:55 | INFO | Hello from data-move Python data pipeline!
|
||||
2026-06-17 15:08:55 | INFO | Pipeline Run Date: 2026-06-16
|
||||
2026-06-17 15:08:55 | INFO | Connecting to databases...
|
||||
2026-06-17 15:08:56 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001ED5A4C78B0>
|
||||
2026-06-17 15:08:58 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001ED5B948910>
|
||||
2026-06-17 15:08:58 | INFO | Database connections established
|
||||
2026-06-17 15:08:58 | INFO | Collecting MIDs for: 2026-06-16
|
||||
2026-06-17 15:08:59 | INFO | Found 863 MIDs
|
||||
2026-06-17 15:08:59 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
||||
2026-06-17 15:09:12 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server
|
||||
2026-06-17 15:09:12 | INFO | Fetched 4,814 matched empids fetched for OQAD
|
||||
2026-06-17 15:09:12 | INFO | ================================================================================
|
||||
2026-06-17 15:09:12 | INFO | Processing Table: additional_visibility
|
||||
2026-06-17 15:09:12 | INFO | Fetching data for 863 MIDs
|
||||
2026-06-17 15:09:14 | INFO | Fetched 1,773 rows from SQL Server
|
||||
2026-06-17 15:09:14 | INFO | Fetched 1773 rows
|
||||
2026-06-17 15:09:14 | INFO | Creating table additional_visibility
|
||||
2026-06-17 15:09:14 | INFO | Table ready: additional_visibility
|
||||
2026-06-17 15:09:15 | INFO | additional_visibility: inserted 1,773 rows into ClickHouse
|
||||
2026-06-17 15:09:15 | INFO | additional_visibility loaded successfully (1773 rows)
|
||||
2026-06-17 15:09:15 | INFO | ================================================================================
|
||||
2026-06-17 15:09:15 | INFO | Pipeline Completed Successfully
|
||||
2026-06-17 15:09:15 | INFO | ================================================================================
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
+1129
-243
File diff suppressed because it is too large
Load Diff
@@ -1,276 +0,0 @@
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# dependencies = [
|
||||
# "polars>=0.20.0",
|
||||
# "pyarrow>=18.0.0",
|
||||
# "sqlalchemy>=2.0.0",
|
||||
# "pyodbc>=5.0.0",
|
||||
# "clickhouse-connect>=0.7.0",
|
||||
# "clickhouse-sqlalchemy>=0.3.2",
|
||||
# "pyyaml>=6.0.3",
|
||||
# "python-dotenv>=1.0.0",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from datetime import date, datetime, timedelta
|
||||
|
||||
import polars as pl
|
||||
import yaml
|
||||
|
||||
|
||||
from log import log
|
||||
|
||||
from clickhouse_task.create_table import create_clickhouse_table
|
||||
from clickhouse_task.delete_task import (
|
||||
delete_existing_data,
|
||||
truncate_table,
|
||||
)
|
||||
|
||||
from clickhouse_task.load_table import load_to_clickhouse
|
||||
|
||||
from db_con.connection import (
|
||||
build_sql_server_engine,
|
||||
build_clickhouse_engine,
|
||||
get_clickhouse_client,
|
||||
)
|
||||
|
||||
from mids import (
|
||||
MID_TABLE_COV,
|
||||
MID_TABLE_COV1,
|
||||
)
|
||||
|
||||
from masters.dimensions import *
|
||||
from masters.bridge import *
|
||||
from kpi.facts import *
|
||||
|
||||
|
||||
# ==========================================================
|
||||
# Helpers
|
||||
# ==========================================================
|
||||
|
||||
def table_exists(
|
||||
client,
|
||||
table_name: str,
|
||||
) -> bool:
|
||||
|
||||
return bool(
|
||||
client.command(
|
||||
f"EXISTS TABLE {table_name}"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def get_dataframe(
|
||||
fn_name: str,
|
||||
fetch_by: str,
|
||||
sql_engine,
|
||||
mids,
|
||||
run_date,
|
||||
):
|
||||
|
||||
fn = globals()[fn_name]
|
||||
|
||||
if fetch_by == "mids":
|
||||
return fn(sql_engine, mids)
|
||||
|
||||
if fetch_by == "run_date":
|
||||
return fn(sql_engine, run_date)
|
||||
|
||||
return fn(sql_engine)
|
||||
|
||||
|
||||
# ==========================================================
|
||||
# Main
|
||||
# ==========================================================
|
||||
|
||||
def main():
|
||||
|
||||
log.info("=" * 80)
|
||||
log.info("Hello from data-move Python data pipeline!")
|
||||
|
||||
# ------------------------------------------------------
|
||||
# Run Date
|
||||
# ------------------------------------------------------
|
||||
|
||||
if len(sys.argv) > 1:
|
||||
run_date = datetime.strptime(
|
||||
sys.argv[1],
|
||||
"%Y-%m-%d",
|
||||
).date()
|
||||
else:
|
||||
run_date = date.today() - timedelta(days=1)
|
||||
|
||||
log.info(
|
||||
"Pipeline Run Date: %s",
|
||||
run_date,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------
|
||||
# Connections
|
||||
# ------------------------------------------------------
|
||||
|
||||
log.info(
|
||||
"Connecting to databases..."
|
||||
)
|
||||
|
||||
sql_engine = build_sql_server_engine()
|
||||
clickhouse_engine = build_clickhouse_engine()
|
||||
client = get_clickhouse_client()
|
||||
|
||||
log.info(
|
||||
"Database connections established"
|
||||
)
|
||||
|
||||
# ------------------------------------------------------
|
||||
# Delete Keys
|
||||
# ------------------------------------------------------
|
||||
|
||||
mids = MID_TABLE_COV(
|
||||
sql_engine,
|
||||
run_date,
|
||||
)
|
||||
|
||||
emp_visit_df = MID_TABLE_COV1(
|
||||
sql_engine,
|
||||
run_date,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------
|
||||
# Config
|
||||
# ------------------------------------------------------
|
||||
|
||||
with open(
|
||||
"t.yml",
|
||||
"r",
|
||||
) as file:
|
||||
|
||||
config = yaml.safe_load(file)
|
||||
|
||||
# ------------------------------------------------------
|
||||
# Process Tables
|
||||
# ------------------------------------------------------
|
||||
|
||||
for table in config["tables"]:
|
||||
|
||||
table_name = table["name"]
|
||||
operation = table["operation"]
|
||||
fetch_by = table["fetch_by"]
|
||||
table_type=table["type"]
|
||||
|
||||
log.info("=" * 80)
|
||||
log.info(
|
||||
"Processing Table: %s",
|
||||
table_name,
|
||||
)
|
||||
|
||||
try:
|
||||
|
||||
# ------------------------------------------
|
||||
# Fetch Data
|
||||
# ------------------------------------------
|
||||
|
||||
fn_name = f"fetch_{table_name}"
|
||||
|
||||
df = get_dataframe(
|
||||
fn_name=fn_name,
|
||||
fetch_by=fetch_by,
|
||||
sql_engine=sql_engine,
|
||||
mids=mids,
|
||||
run_date=run_date,
|
||||
)
|
||||
|
||||
if df.is_empty():
|
||||
|
||||
log.warning(
|
||||
"%s returned no rows",
|
||||
table_name,
|
||||
)
|
||||
|
||||
continue
|
||||
|
||||
log.info(
|
||||
"Fetched %s rows",
|
||||
len(df),
|
||||
)
|
||||
|
||||
# ------------------------------------------
|
||||
# Create Table If Missing
|
||||
# ------------------------------------------
|
||||
|
||||
exists = table_exists(
|
||||
client,
|
||||
table_name,
|
||||
)
|
||||
|
||||
if not exists:
|
||||
|
||||
log.info(
|
||||
"Creating table %s",
|
||||
table_name,
|
||||
)
|
||||
|
||||
create_clickhouse_table(
|
||||
df=df,
|
||||
table_name=table_name,
|
||||
clickhouse_engine=clickhouse_engine,
|
||||
)
|
||||
|
||||
# ------------------------------------------
|
||||
# Existing Table Logic
|
||||
# ------------------------------------------
|
||||
|
||||
else:
|
||||
|
||||
if operation == "DELETE+INSERT":
|
||||
|
||||
truncate_table(
|
||||
client,
|
||||
table_name,
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
delete_existing_data(
|
||||
client=client,
|
||||
table_name=table_name,
|
||||
run_date=run_date,
|
||||
mids=mids,
|
||||
emp_visit_df=emp_visit_df,
|
||||
)
|
||||
|
||||
# ------------------------------------------
|
||||
# Load Data
|
||||
# ------------------------------------------
|
||||
|
||||
load_to_clickhouse(
|
||||
client=client,
|
||||
table_name=table_name,
|
||||
df=df,
|
||||
)
|
||||
|
||||
log.info(
|
||||
"%s loaded successfully (%s rows)",
|
||||
table_name,
|
||||
len(df),
|
||||
)
|
||||
|
||||
except Exception:
|
||||
|
||||
log.exception(
|
||||
"Failed processing table %s",
|
||||
table_name,
|
||||
)
|
||||
|
||||
raise
|
||||
|
||||
log.info("=" * 80)
|
||||
log.info("Pipeline Completed Successfully")
|
||||
log.info("=" * 80)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from time import sleep
|
||||
import sys
|
||||
from datetime import date, datetime, timedelta
|
||||
|
||||
@@ -126,7 +126,7 @@ def main():
|
||||
# ------------------------------------------------------
|
||||
|
||||
with open(
|
||||
"y.yml",
|
||||
"t.yml",
|
||||
"r",
|
||||
) as file:
|
||||
|
||||
@@ -254,4 +254,42 @@ def main():
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
|
||||
for attempt in range(3):
|
||||
try:
|
||||
main()
|
||||
|
||||
with open("Pipeline_config.yml", "r") as f:
|
||||
config = yaml.safe_load(f)
|
||||
|
||||
config["pipeline"]["last_successful_run_date"] = str(date.today())
|
||||
|
||||
with open("Pipeline_config.yml", "w") as f:
|
||||
yaml.safe_dump(config, f, sort_keys=False)
|
||||
|
||||
log.info(
|
||||
f"Pipeline completed successfully. "
|
||||
f"last_successful_run_date={date.today()}"
|
||||
)
|
||||
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
|
||||
with open("Pipeline_config.yml", "r") as f:
|
||||
config = yaml.safe_load(f)
|
||||
|
||||
config["pipeline"]["run_date"] = str(date.today())
|
||||
|
||||
with open("Pipeline_config.yml", "w") as f:
|
||||
yaml.safe_dump(config, f, sort_keys=False)
|
||||
|
||||
if attempt == 2:
|
||||
raise
|
||||
|
||||
log.warning(
|
||||
f"Pipeline failed. Retry {attempt + 1}/3. Error: {e}"
|
||||
)
|
||||
|
||||
sleep(5)
|
||||
@@ -1,64 +0,0 @@
|
||||
import os
|
||||
# import pyarrow
|
||||
import sys
|
||||
import logging
|
||||
from datetime import date, timedelta
|
||||
import polars as pl
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.engine import Engine, URL
|
||||
import clickhouse_connect
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from log import log
|
||||
from clickhouse_task.create_table import *
|
||||
from db_con.connection import *
|
||||
from mids import *
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def fetch_mapping_storevisibility(
|
||||
engine: Engine,
|
||||
run_date: date
|
||||
) -> pl.DataFrame:
|
||||
|
||||
sql = f"""
|
||||
SELECT DISTINCT
|
||||
40148 AS project_id,
|
||||
Z.StoreId AS store_id,
|
||||
Z.VisibilityDefinitionId AS visibility_definition_id,
|
||||
Z.FromDate AS from_date,
|
||||
Z.ToDate AS to_date
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.Mapping_StoreVisibility Z
|
||||
|
||||
WHERE CAST(Z.FromDate AS DATE) <= '{run_date}'
|
||||
AND CAST(Z.ToDate AS DATE) >= '{run_date}'
|
||||
|
||||
AND Z.VisibilityDefinitionId IN
|
||||
(
|
||||
SELECT DISTINCT VisibilityDefinitionId
|
||||
FROM OneApp_KelloggsMT.dbo.Master_VisibilityDefinition
|
||||
WHERE MenuId = 22
|
||||
)
|
||||
"""
|
||||
|
||||
log.info(
|
||||
f"Fetching Mapping Store Visibility for {run_date}"
|
||||
)
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(
|
||||
f"Fetched {len(df):,} Mapping Store Visibility records"
|
||||
)
|
||||
|
||||
return df
|
||||
@@ -1,515 +0,0 @@
|
||||
import os
|
||||
# import pyarrow
|
||||
import sys
|
||||
import logging
|
||||
from datetime import date, timedelta
|
||||
import polars as pl
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.engine import Engine, URL
|
||||
import clickhouse_connect
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from log import log
|
||||
from clickhouse_task.create_table import *
|
||||
from db_con.connection import *
|
||||
from mids import *
|
||||
|
||||
|
||||
|
||||
#PROJECT_ID = 40148
|
||||
p=40148
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def fetch_Store_Master(engine: Engine) -> pl.DataFrame:
|
||||
sql = """
|
||||
SELECT
|
||||
|
||||
RegionName AS region,
|
||||
StateId AS state_id,
|
||||
StateName AS state,
|
||||
CityId AS city_id,
|
||||
CityName AS city,
|
||||
CityCode AS cpm_city_id,
|
||||
|
||||
ChannelId AS channel_id,
|
||||
ChannelName AS channel,
|
||||
|
||||
DistributorId AS distributor_id,
|
||||
Distributor AS distributor_name,
|
||||
|
||||
ChainId AS keyaccount_id,
|
||||
ChainName AS keyaccount,
|
||||
|
||||
StoreUniqueCode AS insight_store_id,
|
||||
StoreCode AS client_store_code,
|
||||
|
||||
Latitude AS latitude,
|
||||
Longitude AS longitude,
|
||||
|
||||
StoreCategoryId AS store_category_id,
|
||||
StoreCategory AS store_category,
|
||||
|
||||
StoreTypeId AS store_type_id,
|
||||
StoreType AS store_type,
|
||||
|
||||
StoreClassId AS store_classification_id,
|
||||
StoreClass AS store_classification,
|
||||
|
||||
StLayerFourId,
|
||||
|
||||
StoreId AS store_id,
|
||||
StoreName AS store_name,
|
||||
Address AS address
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.vw_storedetail
|
||||
"""
|
||||
|
||||
log.info("Fetching Store Master data")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(
|
||||
f"Fetched {len(df):,} stores"
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
|
||||
def fetch_SKU_Master(engine: Engine) -> pl.DataFrame:
|
||||
|
||||
sql = """
|
||||
SELECT
|
||||
CM.CategoryId AS category_id,
|
||||
CM.CategoryCode AS category_code,
|
||||
CM.CategoryName AS category_name,
|
||||
|
||||
SCA.SubCategoryId AS sub_category_id,
|
||||
SCA.SubCategoryCode AS sub_category_code,
|
||||
SCA.SubCategoryName AS sub_category_name,
|
||||
|
||||
BR.BrandId AS brand_id,
|
||||
BR.BrandCode AS brand_code,
|
||||
BR.BrandName AS brand_name,
|
||||
|
||||
SB.SubBrandId AS sub_brand_id,
|
||||
SB.SubBrandCode AS sub_brand_code,
|
||||
SB.SubBrandName AS sub_brand_name,
|
||||
|
||||
P.ProductId AS product_id,
|
||||
P.ProductName AS product_name,
|
||||
P.ProductCode AS product_code,
|
||||
|
||||
P.MRP AS mrp,
|
||||
|
||||
FL.FlavourId AS flavour_id,
|
||||
FL.Flavour AS flavour,
|
||||
|
||||
P.Grammage AS grammage,
|
||||
P.ProductSequence AS product_sequence,
|
||||
P.CaseSize AS case_size,
|
||||
|
||||
MC.Company AS company_name,
|
||||
MC.IsCompetitor AS is_competitor,
|
||||
|
||||
P.PTR AS ptr
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.Master_Product P
|
||||
|
||||
RIGHT JOIN OneApp_KelloggsMT.dbo.Master_Flavour FL
|
||||
ON P.FlavourId = FL.FlavourId
|
||||
|
||||
RIGHT JOIN OneApp_KelloggsMT.dbo.Master_SubBrand SB
|
||||
ON P.SubBrandId = SB.SubBrandId
|
||||
|
||||
RIGHT JOIN OneApp_KelloggsMT.dbo.Master_Brand BR
|
||||
ON SB.BrandId = BR.BrandId
|
||||
|
||||
RIGHT JOIN OneApp_KelloggsMT.dbo.Master_SubCategory SCA
|
||||
ON BR.SubCategoryId = SCA.SubCategoryId
|
||||
|
||||
RIGHT JOIN OneApp_KelloggsMT.dbo.Master_Category CM
|
||||
ON SCA.CategoryId = CM.CategoryId
|
||||
|
||||
RIGHT JOIN OneApp_KelloggsMT.dbo.Master_Company MC
|
||||
ON MC.CompanyId = BR.CompanyId
|
||||
"""
|
||||
|
||||
log.info("Fetching SKU Master data")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(
|
||||
f"Fetched {len(df):,} SKU Master rows"
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
|
||||
def fetch_display_master(engine: Engine) -> pl.DataFrame:
|
||||
"""
|
||||
Fetch Display Master data.
|
||||
Source: Master_Display
|
||||
Target: display_master
|
||||
"""
|
||||
|
||||
sql = """
|
||||
SELECT
|
||||
DisplayId AS display_id,
|
||||
DisplayCode AS display_code,
|
||||
DisplayName AS display_name,
|
||||
DisplayRefImage AS display_ref_url
|
||||
FROM OneApp_KelloggsMT.dbo.Master_Display
|
||||
"""
|
||||
|
||||
log.info("Fetching Display Master data")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(
|
||||
f"Fetched {len(df):,} Display Master records"
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
def fetch_Employee_Master(engine: Engine) -> pl.DataFrame:
|
||||
"""
|
||||
Fetch Employee Master data.
|
||||
Source: vw_Employee_Detail + Mapping_PositionUser + Master_Position
|
||||
Target: Employee_Master
|
||||
"""
|
||||
|
||||
sql = """
|
||||
SELECT
|
||||
RegionId AS region_id,
|
||||
RegionName AS region,
|
||||
|
||||
StateId AS state_id,
|
||||
StateName AS state,
|
||||
|
||||
CityId AS city_id,
|
||||
CityName AS city,
|
||||
|
||||
A.EmpId AS employee_id,
|
||||
EmpName AS employee_name,
|
||||
Gender AS gender,
|
||||
|
||||
A.DesignationId AS designation_id,
|
||||
DesignationName AS designation,
|
||||
|
||||
SupervisorId AS manager_id,
|
||||
SupervisorName AS manager_name,
|
||||
|
||||
JoinDate AS employee_joining_date,
|
||||
ResignDate AS employee_resign_date,
|
||||
|
||||
C.PositionCode AS position_code,
|
||||
|
||||
EmpCode AS employee_legacy_code,
|
||||
|
||||
RIGHTNAME AS employee_role,
|
||||
|
||||
CASE
|
||||
WHEN RIGHTNAME IN ('Client', 'Client HO')
|
||||
THEN 'NON CPM'
|
||||
ELSE 'CPM'
|
||||
END AS employee_type
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.vw_Employee_Detail A
|
||||
|
||||
LEFT JOIN
|
||||
(
|
||||
SELECT DISTINCT
|
||||
PositionId,
|
||||
EmpId
|
||||
FROM OneApp_KelloggsMT.dbo.Mapping_PositionUser
|
||||
WHERE DATEDIFF(DAY, FromDate, GETDATE()) >= 0
|
||||
AND DATEDIFF(DAY, ToDate, GETDATE()) <= 0
|
||||
) B
|
||||
ON A.EmpId = B.EmpId
|
||||
|
||||
LEFT JOIN
|
||||
(
|
||||
SELECT
|
||||
PositionId,
|
||||
PositionCode
|
||||
FROM OneApp_KelloggsMT.dbo.Master_Position
|
||||
) C
|
||||
ON B.PositionId = C.PositionId
|
||||
"""
|
||||
|
||||
log.info("Fetching Employee Master data")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(
|
||||
f"Fetched {len(df):,} Employee Master records"
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def fetch_Employee_Master(engine: Engine) -> pl.DataFrame:
|
||||
"""
|
||||
Fetch Employee Master data.
|
||||
Source: vw_Employee_Detail + Mapping_PositionUser + Master_Position
|
||||
Target: Employee_Master
|
||||
"""
|
||||
|
||||
sql = """
|
||||
SELECT
|
||||
RegionId AS region_id,
|
||||
RegionName AS region,
|
||||
|
||||
StateId AS state_id,
|
||||
StateName AS state,
|
||||
|
||||
CityId AS city_id,
|
||||
CityName AS city,
|
||||
|
||||
A.EmpId AS employee_id,
|
||||
EmpName AS employee_name,
|
||||
Gender AS gender,
|
||||
|
||||
A.DesignationId AS designation_id,
|
||||
DesignationName AS designation,
|
||||
|
||||
SupervisorId AS manager_id,
|
||||
SupervisorName AS manager_name,
|
||||
|
||||
JoinDate AS employee_joining_date,
|
||||
ResignDate AS employee_resign_date,
|
||||
|
||||
C.PositionCode AS position_code,
|
||||
|
||||
EmpCode AS employee_legacy_code,
|
||||
|
||||
RIGHTNAME AS employee_role,
|
||||
|
||||
CASE
|
||||
WHEN RIGHTNAME IN ('Client', 'Client HO')
|
||||
THEN 'NON CPM'
|
||||
ELSE 'CPM'
|
||||
END AS employee_type
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.vw_Employee_Detail A
|
||||
|
||||
LEFT JOIN
|
||||
(
|
||||
SELECT DISTINCT
|
||||
PositionId,
|
||||
EmpId
|
||||
FROM OneApp_KelloggsMT.dbo.Mapping_PositionUser
|
||||
WHERE DATEDIFF(DAY, FromDate, GETDATE()) >= 0
|
||||
AND DATEDIFF(DAY, ToDate, GETDATE()) <= 0
|
||||
) B
|
||||
ON A.EmpId = B.EmpId
|
||||
|
||||
LEFT JOIN
|
||||
(
|
||||
SELECT
|
||||
PositionId,
|
||||
PositionCode
|
||||
FROM OneApp_KelloggsMT.dbo.Master_Position
|
||||
) C
|
||||
ON B.PositionId = C.PositionId
|
||||
"""
|
||||
|
||||
log.info("Fetching Employee Master data")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(
|
||||
f"Fetched {len(df):,} Employee Master records"
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
def fetch_Master_VisibilityReason(engine: Engine) -> pl.DataFrame:
|
||||
"""
|
||||
Source:
|
||||
Master_VisibilityReason
|
||||
|
||||
Target:
|
||||
Master_VisibilityReason
|
||||
"""
|
||||
|
||||
sql = """
|
||||
SELECT DISTINCT
|
||||
40148 AS project_id,
|
||||
MenuId AS menu_id,
|
||||
VisibilityReasonId AS reason_id,
|
||||
VisibilityReason AS reason
|
||||
FROM OneApp_KelloggsMT.dbo.Master_VisibilityReason
|
||||
"""
|
||||
|
||||
log.info("Fetching Master Visibility Reason data")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(
|
||||
f"Fetched {len(df):,} Master Visibility Reason records"
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
def fetch_Master_VisibilityDefinition(engine: Engine) -> pl.DataFrame:
|
||||
"""
|
||||
Source:
|
||||
OneApp_KelloggsMT.dbo.Master_VisibilityDefinition
|
||||
|
||||
Target:
|
||||
Master_VisibilityDefinition
|
||||
"""
|
||||
|
||||
sql = """
|
||||
SELECT DISTINCT
|
||||
40148 AS project_id,
|
||||
VisibilityDefinitionId AS visibility_definition_id,
|
||||
VisibilityDefinitionName AS visibility_definition_name,
|
||||
GETDATE() AS create_date,
|
||||
'SP-Pius' AS create_by
|
||||
FROM OneApp_KelloggsMT.dbo.Master_VisibilityDefinition
|
||||
"""
|
||||
|
||||
log.info("Fetching Master Visibility Definition data")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(
|
||||
f"Fetched {len(df):,} Master Visibility Definition records"
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
|
||||
def fetch_Master_Salesterritorylayer(
|
||||
engine: Engine
|
||||
) -> pl.DataFrame:
|
||||
"""
|
||||
Source:
|
||||
Master_SalesTerritoryLayerOne
|
||||
Master_SalesTerritoryLayerTwo
|
||||
Master_SalesTerritoryLayerThree
|
||||
Master_SalesTerritoryLayerFour
|
||||
|
||||
Target:
|
||||
Master_Salesterritorylayer
|
||||
"""
|
||||
|
||||
sql = """
|
||||
SELECT DISTINCT
|
||||
40148 AS project_id,
|
||||
|
||||
D.StLayerOneId AS st_layer_one_id,
|
||||
D.StLayerOneName AS st_layer_one_name,
|
||||
|
||||
C.StLayerTwoId AS st_layer_two_id,
|
||||
C.StLayerTwoName AS st_layer_two_name,
|
||||
|
||||
B.StLayerThreeId AS st_layer_three_id,
|
||||
B.StLayerThreeName AS st_layer_three_name,
|
||||
|
||||
A.StLayerFourId AS st_layer_four_id,
|
||||
A.StLayerFourName AS st_layer_four_name,
|
||||
|
||||
GETDATE() AS create_date,
|
||||
'SP-Pius' AS create_by
|
||||
|
||||
FROM OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerFour A
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerThree B
|
||||
ON A.StLayerThreeId = B.StLayerThreeId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerTwo C
|
||||
ON B.StLayerTwoId = C.StLayerTwoId
|
||||
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerOne D
|
||||
ON C.StLayerOneId = D.StLayerOneId
|
||||
"""
|
||||
|
||||
log.info("Fetching Master Sales Territory Layer data")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(
|
||||
f"Fetched {len(df):,} Master Sales Territory Layer records"
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def fetch_coverage_remarks(engine: Engine ) -> pl.DataFrame:
|
||||
"""
|
||||
Source:
|
||||
OneApp_KelloggsMT.dbo.Master_NonWorkingReason
|
||||
|
||||
Target:
|
||||
coverage_remarks
|
||||
"""
|
||||
|
||||
sql = """
|
||||
SELECT
|
||||
40148 AS project_id,
|
||||
ReasonId AS reason_id,
|
||||
Reason AS reason_remarks
|
||||
FROM OneApp_KelloggsMT.dbo.Master_NonWorkingReason
|
||||
"""
|
||||
|
||||
log.info("Fetching Coverage Remarks")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(
|
||||
f"Fetched {len(df):,} Coverage Remark records"
|
||||
)
|
||||
|
||||
return df
|
||||
+51
-4
@@ -242,8 +242,19 @@ def fetch_OQaD(
|
||||
on=["empid", "visitdate"],
|
||||
how="inner",
|
||||
)
|
||||
|
||||
if matched.is_empty():
|
||||
|
||||
empids=[0]
|
||||
log.warning(
|
||||
"%s Matched df in OQaD returned no rows",
|
||||
table_name,
|
||||
)
|
||||
|
||||
else:
|
||||
empids=matched["empid"].to_list()
|
||||
|
||||
|
||||
log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ")
|
||||
|
||||
def fetch_data(
|
||||
@@ -418,9 +429,9 @@ def fetch_Login( engine: Engine,
|
||||
|
||||
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
# if not mids:
|
||||
# log.warning("No MIDs — nothing to fetch.")
|
||||
# return pl.DataFrame()
|
||||
|
||||
mid_list = ",".join(str(mid) for mid in mids)
|
||||
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
|
||||
@@ -430,7 +441,7 @@ def fetch_Login( engine: Engine,
|
||||
sql_template = f.read()
|
||||
|
||||
sql = sql_template.format(
|
||||
mid_list=mid_list,
|
||||
# mid_list=mid_list,
|
||||
run_date=run_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||
@@ -674,3 +685,39 @@ def fetch_Web_Logins( engine: Engine,
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def fetch_Promotion(
|
||||
engine: Engine,
|
||||
table_name: str,
|
||||
table_type: str,
|
||||
mids: list[int],
|
||||
run_date: date
|
||||
) -> pl.DataFrame:
|
||||
|
||||
if not mids:
|
||||
log.warning("No MIDs — nothing to fetch.")
|
||||
return pl.DataFrame()
|
||||
|
||||
mid_list = ",".join(str(mid) for mid in mids)
|
||||
|
||||
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
|
||||
|
||||
with open(sql_file, "r", encoding="utf-8") as f:
|
||||
sql_template = f.read()
|
||||
|
||||
sql = sql_template.format(
|
||||
mid_list=mid_list,
|
||||
run_date=run_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
|
||||
log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||
|
||||
df = pl.read_database(
|
||||
query=sql,
|
||||
connection=engine
|
||||
)
|
||||
|
||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||
|
||||
return df
|
||||
@@ -12,7 +12,7 @@ with employee_detail AS
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_Region rm ON st.RegionId = rm.RegionId
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_Designation dm ON Em.DesignationId = dm.DesignationId
|
||||
WHERE em.RightId IN (6)
|
||||
AND (Em.ResignDate IS NULL OR CONVERT(DATE, Em.ResignDate, 101) >= CAST('{start_date}' AS DATE))
|
||||
AND (Em.ResignDate IS NULL OR CONVERT(DATE, Em.ResignDate, 101) >= CAST('{run_date}' AS DATE))
|
||||
AND em.EmployeeName NOT LIKE '%test%'
|
||||
),
|
||||
Login (
|
||||
@@ -71,7 +71,7 @@ Login (
|
||||
) AS [Last Store Out Time]
|
||||
FROM OneApp_KelloggsMT.dbo.T_DeviceLogin ud
|
||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em ON ud.EmpId = Em.EmpId
|
||||
WHERE CAST(ud.LoginDate AS DATE) = CAST('{start_date}' AS DATE) -- fixed: direct date comparison
|
||||
WHERE CAST(ud.LoginDate AS DATE) = CAST('{run_date}' AS DATE) -- fixed: direct date comparison
|
||||
) AS T1
|
||||
WHERE T1.col = 1
|
||||
) AS T ON Em.[Employee code] = T.[Employee Code]
|
||||
|
||||
@@ -3,6 +3,7 @@ with Promotion(Mid,
|
||||
promotion_deatils,promotion_deatils_id,promotion_value_name,
|
||||
present,reason,PromoQuestion,PromoAnswer,image1,image2,update_date,update_by)
|
||||
as (
|
||||
|
||||
Select
|
||||
sc.MID, '40148' Projectid,sc.StoreId,Em.EmpId,sc.VisitDate,Em.SupervisorId,sm.ChannelId,sm.ChainId,sm.StoreTypeId,msd.PromoDefinitionId,
|
||||
MSD.PromoDefinitionName, case when isnull(TS.PromoTable,'')='Master_Category' then 'Category'
|
||||
@@ -38,6 +39,5 @@ with Promotion(Mid,
|
||||
left join OneApp_KelloggsMT.dbo.Master_PromotionQuestion mpq on tpq.PromoQuestionId=mpq.PromoQuestionId
|
||||
|
||||
Where 1=1 and Em.EmpName not like 'test%'
|
||||
AND sc.MID in ({mid_list})
|
||||
)
|
||||
AND sc.MID in ({mid_list}) )
|
||||
select * from Promotion
|
||||
@@ -85,20 +85,25 @@ tables:
|
||||
# operation: ONLY_INSERT
|
||||
# fetch_by: run_date
|
||||
|
||||
- name: Master_VisibilityReason
|
||||
type: DIMENSION
|
||||
operation: DELETE+INSERT
|
||||
fetch_by: none
|
||||
# - name: Master_VisibilityReason
|
||||
# type: DIMENSION
|
||||
# operation: DELETE+INSERT
|
||||
# fetch_by: none
|
||||
|
||||
|
||||
- name: Master_VisibilityDefinition
|
||||
type: DIMENSION
|
||||
operation: DELETE+INSERT
|
||||
fetch_by: none
|
||||
# - name: Master_VisibilityDefinition
|
||||
# type: DIMENSION
|
||||
# operation: DELETE+INSERT
|
||||
# fetch_by: none
|
||||
|
||||
|
||||
- name: Master_Salesterritorylayer
|
||||
type: DIMENSION
|
||||
operation: DELETE+INSERT
|
||||
fetch_by: none
|
||||
# - name: Master_Salesterritorylayer
|
||||
# type: DIMENSION
|
||||
# operation: DELETE+INSERT
|
||||
# fetch_by: none
|
||||
|
||||
- name: Promotion
|
||||
type: FACT
|
||||
operation: INSERT
|
||||
fetch_by: mids
|
||||
|
||||
|
||||
Reference in New Issue
Block a user