Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Ankit Malik
|
75fc088c68 | ||
|
Ankit Malik
|
e1bd387849 |
-1301
File diff suppressed because it is too large
Load Diff
@@ -1,164 +0,0 @@
|
|||||||
WITH SOS_BASE AS
|
|
||||||
(
|
|
||||||
SELECT
|
|
||||||
sm.CountryName,
|
|
||||||
sc.MID,
|
|
||||||
sm.RegionName,
|
|
||||||
sm.StateName,
|
|
||||||
sm.CityName,
|
|
||||||
Em.SupervisorName,
|
|
||||||
em.EmpId,
|
|
||||||
Em.EmpName AS EmployeeName,
|
|
||||||
Em.DesignationName AS Designation,
|
|
||||||
sm.StoreId,
|
|
||||||
CONVERT(varchar, sc.VisitDate, 101) AS VisitDate,
|
|
||||||
sm.StoreCode,
|
|
||||||
sm.StoreName,
|
|
||||||
sm.Address,
|
|
||||||
sm.StoreTypeid,
|
|
||||||
sm.ChannelId,
|
|
||||||
sm.ChainName,
|
|
||||||
|
|
||||||
MSD.SOSDefinitionName,
|
|
||||||
|
|
||||||
CASE
|
|
||||||
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Category' THEN 'Category'
|
|
||||||
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubCategory' THEN 'SubCategory'
|
|
||||||
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_Brand' THEN 'Brand'
|
|
||||||
WHEN ISNULL(TS.SOSHeaderTable,'')='Master_SubBrand' THEN 'SubBrand'
|
|
||||||
END AS SOSHeaderDeatils,
|
|
||||||
|
|
||||||
TS.SOSHeaderName,
|
|
||||||
TS.SOSHeaderValue AS SOSHeaderID,
|
|
||||||
|
|
||||||
'Header_Image' AS HDR1,
|
|
||||||
|
|
||||||
CASE
|
|
||||||
WHEN ISNULL(TSC.SOSChildTable,'')='Master_Category' THEN 'Category'
|
|
||||||
WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubCategory' THEN 'SubCategory'
|
|
||||||
WHEN ISNULL(TSC.SOSChildTable,'')='Master_Brand' THEN 'Brand'
|
|
||||||
WHEN ISNULL(TSC.SOSChildTable,'')='Master_SubBrand' THEN 'SubBrand'
|
|
||||||
END AS SOSChildDeatils,
|
|
||||||
|
|
||||||
TSC.SOSChildName,
|
|
||||||
TSC.SOSChildValue AS SOSChildID,
|
|
||||||
TSC.ChildTotalFacing,
|
|
||||||
TS.SOSHeaderFacing,
|
|
||||||
TSC.ChildSelfFacing,
|
|
||||||
|
|
||||||
(
|
|
||||||
SELECT TOP 1 SOSTarget
|
|
||||||
FROM OneApp_KelloggsMT.dbo.Mapping_StoreShareOfShelfTarget a
|
|
||||||
WHERE a.SOSDefinitionId = MSD.SOSDefinitionId
|
|
||||||
AND a.StoreId = sm.StoreId
|
|
||||||
AND a.FromDate <= sc.VisitDate
|
|
||||||
AND a.ToDate >= sc.VisitDate
|
|
||||||
) AS SOSTarget,
|
|
||||||
|
|
||||||
CASE
|
|
||||||
WHEN ISNULL(SHI.SOSHeaderImage,'') = ''
|
|
||||||
THEN ''
|
|
||||||
ELSE
|
|
||||||
'https://kimt1.parinaam.in/Upload/SOSImages/'
|
|
||||||
+ SHI.SOSHeaderImage
|
|
||||||
END AS SOSHeaderImg
|
|
||||||
|
|
||||||
FROM OneApp_KelloggsMT.dbo.T_ShareOfShelfHeader ts
|
|
||||||
|
|
||||||
INNER JOIN OneApp_KelloggsMT.dbo.T_StoreCoverage sc
|
|
||||||
ON ts.MID = sc.MID
|
|
||||||
|
|
||||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_StoreDetail sm
|
|
||||||
ON sc.StoreId = sm.StoreId
|
|
||||||
|
|
||||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em
|
|
||||||
ON sc.EmpId = Em.EmpId
|
|
||||||
|
|
||||||
INNER JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChild tsc
|
|
||||||
ON ts.SOSId = tsc.SOSId
|
|
||||||
|
|
||||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_ShareOfShelfDefinition msd
|
|
||||||
ON msd.SOSDefinitionId = tsc.SOSDefinitionId
|
|
||||||
|
|
||||||
LEFT JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfHeaderImages SHI
|
|
||||||
ON ts.SOSId = SHI.SOSId
|
|
||||||
AND ts.SOSHeaderValue = SHI.SOSHeaderValue
|
|
||||||
|
|
||||||
LEFT JOIN OneApp_KelloggsMT.dbo.T_ShareOfShelfChildImages SCI
|
|
||||||
ON tsc.SOSId = SCI.SOSId
|
|
||||||
AND tsc.SOSChildValue = SCI.SOSChildValue
|
|
||||||
|
|
||||||
WHERE Em.EmpName NOT LIKE 'test%'
|
|
||||||
AND sc.MID IN ({mid_list})
|
|
||||||
),
|
|
||||||
|
|
||||||
SOS_PIVOT AS
|
|
||||||
(
|
|
||||||
SELECT *
|
|
||||||
FROM SOS_BASE
|
|
||||||
PIVOT
|
|
||||||
(
|
|
||||||
MIN(SOSHeaderImg)
|
|
||||||
FOR HDR1 IN ([Header_Image])
|
|
||||||
) pvt
|
|
||||||
)
|
|
||||||
|
|
||||||
SELECT
|
|
||||||
'40148' AS ProjectId,
|
|
||||||
MID,
|
|
||||||
EmpId AS employee_id,
|
|
||||||
StoreId AS store_id,
|
|
||||||
VisitDate AS visit_date,
|
|
||||||
StoreTypeid AS storetype_id,
|
|
||||||
ChannelId AS channel_id,
|
|
||||||
SOSDefinitionName,
|
|
||||||
SOSHeaderDeatils,
|
|
||||||
SOSHeaderName,
|
|
||||||
SOSHeaderID,
|
|
||||||
SOSChildDeatils,
|
|
||||||
SOSChildName,
|
|
||||||
SOSChildID,
|
|
||||||
SOSHeaderFacing,
|
|
||||||
ChildTotalFacing,
|
|
||||||
ChildSelfFacing,
|
|
||||||
SOSTarget,
|
|
||||||
GETDATE() AS update_date,
|
|
||||||
'SP-Pius' AS update_by
|
|
||||||
|
|
||||||
FROM SOS_PIVOT
|
|
||||||
|
|
||||||
GROUP BY
|
|
||||||
CountryName,
|
|
||||||
MID,
|
|
||||||
RegionName,
|
|
||||||
StateName,
|
|
||||||
CityName,
|
|
||||||
SupervisorName,
|
|
||||||
EmpId,
|
|
||||||
EmployeeName,
|
|
||||||
Designation,
|
|
||||||
StoreId,
|
|
||||||
VisitDate,
|
|
||||||
StoreCode,
|
|
||||||
StoreName,
|
|
||||||
Address,
|
|
||||||
StoreTypeid,
|
|
||||||
ChannelId,
|
|
||||||
ChainName,
|
|
||||||
SOSDefinitionName,
|
|
||||||
SOSHeaderDeatils,
|
|
||||||
SOSHeaderName,
|
|
||||||
SOSHeaderID,
|
|
||||||
SOSChildDeatils,
|
|
||||||
SOSChildName,
|
|
||||||
SOSChildID,
|
|
||||||
ChildTotalFacing,
|
|
||||||
SOSHeaderFacing,
|
|
||||||
ChildSelfFacing,
|
|
||||||
SOSTarget
|
|
||||||
|
|
||||||
ORDER BY
|
|
||||||
RegionName,
|
|
||||||
StateName,
|
|
||||||
CityName,
|
|
||||||
VisitDate;
|
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@@ -1,71 +0,0 @@
|
|||||||
2026-06-16 17:56:17 | INFO | ================================================================================
|
|
||||||
2026-06-16 17:56:17 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-16 17:56:17 | INFO | Pipeline Run Date: 2026-06-15
|
|
||||||
2026-06-16 17:56:17 | INFO | Connecting to databases...
|
|
||||||
2026-06-16 17:56:19 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000017B644A81B0>
|
|
||||||
2026-06-16 17:56:23 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000017B6548EFD0>
|
|
||||||
2026-06-16 17:56:24 | INFO | Database connections established
|
|
||||||
2026-06-16 17:56:24 | INFO | Collecting MIDs for: 2026-06-15
|
|
||||||
2026-06-16 17:56:24 | INFO | Found 849 MIDs
|
|
||||||
2026-06-16 17:57:33 | INFO | ================================================================================
|
|
||||||
2026-06-16 17:57:33 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-16 17:57:33 | INFO | Pipeline Run Date: 2026-06-15
|
|
||||||
2026-06-16 17:57:33 | INFO | Connecting to databases...
|
|
||||||
2026-06-16 17:57:34 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DCA21AB750>
|
|
||||||
2026-06-16 17:57:36 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DCA3371590>
|
|
||||||
2026-06-16 17:57:36 | INFO | Database connections established
|
|
||||||
2026-06-16 17:57:36 | INFO | Collecting MIDs for: 2026-06-15
|
|
||||||
2026-06-16 17:57:37 | INFO | Found 849 MIDs
|
|
||||||
2026-06-16 17:58:52 | INFO | ================================================================================
|
|
||||||
2026-06-16 17:58:52 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-16 17:58:52 | INFO | Pipeline Run Date: 2026-06-15
|
|
||||||
2026-06-16 17:58:52 | INFO | Connecting to databases...
|
|
||||||
2026-06-16 17:58:53 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E5AB2CF750>
|
|
||||||
2026-06-16 17:58:54 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E5AC458910>
|
|
||||||
2026-06-16 17:58:55 | INFO | Database connections established
|
|
||||||
2026-06-16 17:58:55 | INFO | Collecting MIDs for: 2026-06-15
|
|
||||||
2026-06-16 17:58:55 | INFO | Found 849 MIDs
|
|
||||||
2026-06-16 18:00:31 | INFO | ================================================================================
|
|
||||||
2026-06-16 18:00:31 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-16 18:00:31 | INFO | Pipeline Run Date: 2026-06-15
|
|
||||||
2026-06-16 18:00:31 | INFO | Connecting to databases...
|
|
||||||
2026-06-16 18:00:38 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E8C854F750>
|
|
||||||
2026-06-16 18:00:40 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E8C9688910>
|
|
||||||
2026-06-16 18:00:40 | INFO | Database connections established
|
|
||||||
2026-06-16 18:00:40 | INFO | Collecting MIDs for: 2026-06-15
|
|
||||||
2026-06-16 18:00:42 | INFO | Found 849 MIDs
|
|
||||||
2026-06-16 18:00:44 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-16 18:02:58 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-16 18:07:47 | INFO | ================================================================================
|
|
||||||
2026-06-16 18:07:47 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-16 18:07:47 | INFO | Pipeline Run Date: 2026-06-15
|
|
||||||
2026-06-16 18:07:47 | INFO | Connecting to databases...
|
|
||||||
2026-06-16 18:07:48 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DF4BB2F750>
|
|
||||||
2026-06-16 18:07:50 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001DF4CCE8A50>
|
|
||||||
2026-06-16 18:07:50 | INFO | Database connections established
|
|
||||||
2026-06-16 18:07:50 | INFO | Collecting MIDs for: 2026-06-15
|
|
||||||
2026-06-16 18:07:50 | INFO | Found 849 MIDs
|
|
||||||
2026-06-16 18:07:51 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-16 18:08:09 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-16 18:08:58 | INFO | ================================================================================
|
|
||||||
2026-06-16 18:08:58 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-16 18:08:58 | INFO | Pipeline Run Date: 2026-06-15
|
|
||||||
2026-06-16 18:08:58 | INFO | Connecting to databases...
|
|
||||||
2026-06-16 18:08:59 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001639269F750>
|
|
||||||
2026-06-16 18:09:01 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000016393858910>
|
|
||||||
2026-06-16 18:09:01 | INFO | Database connections established
|
|
||||||
2026-06-16 18:09:01 | INFO | Collecting MIDs for: 2026-06-15
|
|
||||||
2026-06-16 18:09:02 | INFO | Found 849 MIDs
|
|
||||||
2026-06-16 18:09:02 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-16 18:09:19 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-16 18:11:21 | INFO | ================================================================================
|
|
||||||
2026-06-16 18:11:21 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-16 18:11:21 | INFO | Pipeline Run Date: 2026-06-15
|
|
||||||
2026-06-16 18:11:21 | INFO | Connecting to databases...
|
|
||||||
2026-06-16 18:11:23 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002240B7AF750>
|
|
||||||
2026-06-16 18:11:24 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002240C978910>
|
|
||||||
2026-06-16 18:11:24 | INFO | Database connections established
|
|
||||||
2026-06-16 18:11:24 | INFO | Collecting MIDs for: 2026-06-15
|
|
||||||
2026-06-16 18:11:25 | INFO | Found 849 MIDs
|
|
||||||
2026-06-16 18:11:25 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-16 18:13:05 | INFO | Fetched 291,776 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
@@ -1,688 +0,0 @@
|
|||||||
2026-06-17 10:17:16 | INFO | ================================================================================
|
|
||||||
2026-06-17 10:17:16 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 10:17:16 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 10:17:16 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 10:17:18 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000019A9A2CF750>
|
|
||||||
2026-06-17 10:17:21 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000019A9B778910>
|
|
||||||
2026-06-17 10:17:22 | INFO | Database connections established
|
|
||||||
2026-06-17 10:17:22 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 10:17:22 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 10:17:23 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 10:17:44 | INFO | Fetched 290,984 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 10:43:05 | INFO | ================================================================================
|
|
||||||
2026-06-17 10:43:05 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 10:43:05 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 10:43:05 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 10:43:07 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002CE210FF750>
|
|
||||||
2026-06-17 10:43:08 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002CE2253CA50>
|
|
||||||
2026-06-17 10:43:09 | INFO | Database connections established
|
|
||||||
2026-06-17 10:43:09 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 10:43:10 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 10:43:11 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 10:43:55 | INFO | Fetched 291,001 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 10:45:51 | INFO | ================================================================================
|
|
||||||
2026-06-17 10:45:51 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 10:45:51 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 10:45:51 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 10:45:52 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000214EC45F750>
|
|
||||||
2026-06-17 10:45:53 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000214ED908910>
|
|
||||||
2026-06-17 10:45:54 | INFO | Database connections established
|
|
||||||
2026-06-17 10:45:54 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 10:45:55 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 10:45:55 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 10:46:11 | INFO | Fetched 291,005 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 10:46:11 | INFO | ================================================================================
|
|
||||||
2026-06-17 10:46:11 | INFO | Processing Table: OQaD
|
|
||||||
2026-06-17 10:46:11 | ERROR | Failed processing table OQaD
|
|
||||||
Traceback (most recent call last):
|
|
||||||
File "D:\data_move\main2.py", line 212, in main
|
|
||||||
df=fetch_OQaD( engine=sql_engine,
|
|
||||||
table_name=table_name,
|
|
||||||
...<2 lines>...
|
|
||||||
run_date=run_date
|
|
||||||
)
|
|
||||||
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
|
|
||||||
with open(sql_file, "r", encoding="utf-8") as f:
|
|
||||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
|
|
||||||
2026-06-17 10:48:44 | INFO | ================================================================================
|
|
||||||
2026-06-17 10:48:44 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 10:48:44 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 10:48:44 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 10:48:45 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000021A270378B0>
|
|
||||||
2026-06-17 10:48:46 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000021A28538910>
|
|
||||||
2026-06-17 10:48:47 | INFO | Database connections established
|
|
||||||
2026-06-17 10:48:47 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 10:48:48 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 10:48:48 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 10:49:00 | INFO | Fetched 291,005 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 10:49:01 | INFO | Fetched 893 matched empids fetched for OQAD
|
|
||||||
2026-06-17 10:49:01 | INFO | ================================================================================
|
|
||||||
2026-06-17 10:49:01 | INFO | Processing Table: OQaD
|
|
||||||
2026-06-17 10:49:01 | ERROR | Failed processing table OQaD
|
|
||||||
Traceback (most recent call last):
|
|
||||||
File "D:\data_move\main2.py", line 215, in main
|
|
||||||
df=fetch_OQaD( engine=sql_engine,
|
|
||||||
table_name=table_name,
|
|
||||||
...<2 lines>...
|
|
||||||
run_date=run_date
|
|
||||||
)
|
|
||||||
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
|
|
||||||
with open(sql_file, "r", encoding="utf-8") as f:
|
|
||||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
|
|
||||||
2026-06-17 10:53:50 | INFO | ================================================================================
|
|
||||||
2026-06-17 10:53:50 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 10:53:50 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 10:53:50 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 10:53:51 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001D1D6DB78B0>
|
|
||||||
2026-06-17 10:53:52 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001D1D81F8910>
|
|
||||||
2026-06-17 10:53:53 | INFO | Database connections established
|
|
||||||
2026-06-17 10:53:53 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 10:53:54 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 10:53:54 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 10:54:08 | INFO | Fetched 291,006 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 10:54:09 | INFO | Fetched 893 matched empids fetched for OQAD
|
|
||||||
2026-06-17 10:54:09 | INFO | ================================================================================
|
|
||||||
2026-06-17 10:54:09 | INFO | Processing Table: OQaD
|
|
||||||
2026-06-17 10:54:09 | ERROR | Failed processing table OQaD
|
|
||||||
Traceback (most recent call last):
|
|
||||||
File "D:\data_move\main2.py", line 217, in main
|
|
||||||
df=fetch_OQaD( engine=sql_engine,
|
|
||||||
table_name=table_name,
|
|
||||||
...<2 lines>...
|
|
||||||
run_date=run_date
|
|
||||||
)
|
|
||||||
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
|
|
||||||
with open(sql_file, "r", encoding="utf-8") as f:
|
|
||||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
|
|
||||||
2026-06-17 10:57:28 | INFO | ================================================================================
|
|
||||||
2026-06-17 10:57:28 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 10:57:28 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 10:57:28 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 10:57:30 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000197AB2778B0>
|
|
||||||
2026-06-17 10:57:31 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000197AC768910>
|
|
||||||
2026-06-17 10:57:32 | INFO | Database connections established
|
|
||||||
2026-06-17 10:57:32 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 10:57:32 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 10:57:33 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 10:57:45 | INFO | Fetched 291,010 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 10:57:46 | INFO | Fetched 893 matched empids fetched for OQAD
|
|
||||||
2026-06-17 10:57:46 | INFO | ================================================================================
|
|
||||||
2026-06-17 10:57:46 | INFO | Processing Table: OQaD
|
|
||||||
2026-06-17 10:57:46 | ERROR | Failed processing table OQaD
|
|
||||||
Traceback (most recent call last):
|
|
||||||
File "D:\data_move\main2.py", line 217, in main
|
|
||||||
df=fetch_OQaD( engine=sql_engine,
|
|
||||||
table_name=table_name,
|
|
||||||
...<2 lines>...
|
|
||||||
run_date=run_date
|
|
||||||
)
|
|
||||||
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
|
|
||||||
with open(sql_file, "r", encoding="utf-8") as f:
|
|
||||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\FACT\\OQaD.sql'
|
|
||||||
2026-06-17 10:59:57 | INFO | ================================================================================
|
|
||||||
2026-06-17 10:59:57 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 10:59:57 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 10:59:57 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 10:59:58 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001A32C2078B0>
|
|
||||||
2026-06-17 11:00:00 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001A32D648910>
|
|
||||||
2026-06-17 11:00:00 | INFO | Database connections established
|
|
||||||
2026-06-17 11:00:00 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 11:00:01 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 11:00:02 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 11:00:24 | INFO | Fetched 291,011 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 11:00:25 | INFO | Fetched 893 matched empids fetched for OQAD
|
|
||||||
2026-06-17 11:00:25 | INFO | ================================================================================
|
|
||||||
2026-06-17 11:00:25 | INFO | Processing Table: OQaD
|
|
||||||
2026-06-17 11:00:25 | ERROR | Failed processing table OQaD
|
|
||||||
Traceback (most recent call last):
|
|
||||||
File "D:\data_move\main2.py", line 217, in main
|
|
||||||
df=fetch_OQaD( engine=sql_engine,
|
|
||||||
table_name=table_name,
|
|
||||||
...<2 lines>...
|
|
||||||
run_date=run_date
|
|
||||||
)
|
|
||||||
File "D:\data_move\src\fact.py", line 58, in fetch_OQaD
|
|
||||||
with open(sql_file, "r", encoding="utf-8") as f:
|
|
||||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\fact\\OQaD.sql'
|
|
||||||
2026-06-17 12:09:34 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:09:34 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 12:09:34 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 12:09:34 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 12:09:37 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000274B86078B0>
|
|
||||||
2026-06-17 12:09:39 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000274B9A18910>
|
|
||||||
2026-06-17 12:09:39 | INFO | Database connections established
|
|
||||||
2026-06-17 12:09:39 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 12:09:41 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 12:09:43 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 12:10:48 | INFO | Fetched 291,093 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 12:10:48 | INFO | Fetched 893 matched empids fetched for OQAD
|
|
||||||
2026-06-17 12:10:48 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:10:48 | INFO | Processing Table: OQaD
|
|
||||||
2026-06-17 12:10:48 | INFO | Exists: False
|
|
||||||
2026-06-17 12:10:48 | INFO | Path: D:\data_move\sql\fact\OQaD.sql
|
|
||||||
2026-06-17 12:10:48 | ERROR | Failed processing table OQaD
|
|
||||||
Traceback (most recent call last):
|
|
||||||
File "D:\data_move\main2.py", line 217, in main
|
|
||||||
df=fetch_OQaD( engine=sql_engine,
|
|
||||||
table_name=table_name,
|
|
||||||
...<2 lines>...
|
|
||||||
run_date=run_date
|
|
||||||
)
|
|
||||||
File "D:\data_move\src\fact.py", line 62, in fetch_OQaD
|
|
||||||
with open(sql_file, "r", encoding="utf-8") as f:
|
|
||||||
~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
FileNotFoundError: [Errno 2] No such file or directory: 'sql\\fact\\OQaD.sql'
|
|
||||||
2026-06-17 12:11:47 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:11:47 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 12:11:47 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 12:11:47 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 12:11:48 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000241C8DF78B0>
|
|
||||||
2026-06-17 12:11:50 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000241CA238910>
|
|
||||||
2026-06-17 12:11:51 | INFO | Database connections established
|
|
||||||
2026-06-17 12:11:51 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 12:11:52 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 12:11:52 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 12:12:14 | INFO | Fetched 291,097 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 12:12:14 | INFO | Fetched 893 matched empids fetched for OQAD
|
|
||||||
2026-06-17 12:12:14 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:12:14 | INFO | Processing Table: OQaD
|
|
||||||
2026-06-17 12:12:14 | INFO | Exists: True
|
|
||||||
2026-06-17 12:12:14 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
|
||||||
2026-06-17 12:12:14 | INFO | Fetching data for 893 EMPIDs
|
|
||||||
2026-06-17 12:12:14 | INFO | Fetching OQaD data for run_date=2026-06-16
|
|
||||||
2026-06-17 12:12:14 | ERROR | Failed processing table OQaD
|
|
||||||
Traceback (most recent call last):
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context
|
|
||||||
self.dialect.do_execute(
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~^
|
|
||||||
cursor, str_statement, effective_parameters, context
|
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
)
|
|
||||||
^
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute
|
|
||||||
cursor.execute(statement, parameters)
|
|
||||||
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near 'QUIZ'. (102) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'where'. (156)")
|
|
||||||
|
|
||||||
The above exception was the direct cause of the following exception:
|
|
||||||
|
|
||||||
Traceback (most recent call last):
|
|
||||||
File "D:\data_move\main2.py", line 217, in main
|
|
||||||
df=fetch_OQaD( engine=sql_engine,
|
|
||||||
table_name=table_name,
|
|
||||||
...<2 lines>...
|
|
||||||
run_date=run_date
|
|
||||||
)
|
|
||||||
File "D:\data_move\src\fact.py", line 74, in fetch_OQaD
|
|
||||||
df = pl.read_database(
|
|
||||||
query=sql,
|
|
||||||
connection=engine,
|
|
||||||
)
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database
|
|
||||||
return cx.execute(
|
|
||||||
~~~~~~~~~~^
|
|
||||||
query=query,
|
|
||||||
^^^^^^^^^^^^
|
|
||||||
options=execute_options,
|
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
).to_polars(
|
|
||||||
^
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute
|
|
||||||
result = cursor_execute(query, **options)
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute
|
|
||||||
return meth(
|
|
||||||
self,
|
|
||||||
distilled_parameters,
|
|
||||||
execution_options or NO_OPTIONS,
|
|
||||||
)
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection
|
|
||||||
return connection._execute_clauseelement(
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
|
||||||
self, distilled_params, execution_options
|
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
)
|
|
||||||
^
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement
|
|
||||||
ret = self._execute_context(
|
|
||||||
dialect,
|
|
||||||
...<8 lines>...
|
|
||||||
cache_hit=cache_hit,
|
|
||||||
)
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context
|
|
||||||
return self._exec_single_context(
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~^
|
|
||||||
dialect, context, statement, parameters
|
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
)
|
|
||||||
^
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context
|
|
||||||
self._handle_dbapi_exception(
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
|
||||||
e, str_statement, effective_parameters, cursor, context
|
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
)
|
|
||||||
^
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception
|
|
||||||
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context
|
|
||||||
self.dialect.do_execute(
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~^
|
|
||||||
cursor, str_statement, effective_parameters, context
|
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
)
|
|
||||||
^
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute
|
|
||||||
cursor.execute(statement, parameters)
|
|
||||||
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near 'QUIZ'. (102) (SQLExecDirectW); [42000] [Microsoft][ODBC Driver 18 for SQL Server][SQL Server]Incorrect syntax near the keyword 'where'. (156)")
|
|
||||||
[SQL: WITH MID_TABLE_COV1 AS
|
|
||||||
(
|
|
||||||
SELECT EmpId, VisitDate
|
|
||||||
FROM OneApp_KelloggsMT.dbo.T_OQAD
|
|
||||||
WHERE CreateDate >= 2026-06-16
|
|
||||||
AND CreateDate < DATEADD(DAY,1,'2026-06-16')
|
|
||||||
|
|
||||||
UNION ALL
|
|
||||||
|
|
||||||
SELECT EmpId, VisitDate
|
|
||||||
FROM OneApp_KelloggsMT.dbo.T_OQAD
|
|
||||||
WHERE UpdateDate >= 2026-06-16
|
|
||||||
AND UpdateDate < DATEADD(DAY,1, '2026-06-16')
|
|
||||||
),
|
|
||||||
|
|
||||||
OQaD
|
|
||||||
|
|
||||||
|
|
||||||
QUIZ AS
|
|
||||||
(
|
|
||||||
SELECT DISTINCT
|
|
||||||
E.EmpId,
|
|
||||||
E.EmpName,
|
|
||||||
E.SupervisorId,
|
|
||||||
E.SupervisorName,
|
|
||||||
E.DesignationName,
|
|
||||||
E.CityName,
|
|
||||||
E.StateName,
|
|
||||||
E.RegionName,
|
|
||||||
CAST(DQ.VisitDate AS DATE) AS VisitDate,
|
|
||||||
DQ.QuestionId,
|
|
||||||
DQ.AnswerId,
|
|
||||||
QC.QuestionCategoryId,
|
|
||||||
QC.QuestionCategory
|
|
||||||
FROM OneApp_KelloggsMT.dbo.T_OQAD DQ
|
|
||||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail E
|
|
||||||
ON DQ.EmpId = E.EmpId
|
|
||||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QU
|
|
||||||
ON DQ.QuestionId = QU.QuestionId
|
|
||||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Category QC
|
|
||||||
ON QU.QuestionCategoryId = QC.QuestionCategoryId
|
|
||||||
WHERE E.EmpName NOT LIKE '%TEST%'
|
|
||||||
AND E.RightId = 6
|
|
||||||
AND (
|
|
||||||
E.ResignDate IS NULL
|
|
||||||
OR CAST(E.ResignDate AS DATE) >= '2026-06-16'
|
|
||||||
)
|
|
||||||
AND EXISTS
|
|
||||||
(
|
|
||||||
SELECT 1
|
|
||||||
FROM MID_TABLE_COV1 A
|
|
||||||
WHERE A.EmpId = DQ.EmpId
|
|
||||||
AND CAST(A.VisitDate AS DATE) = CAST(DQ.VisitDate AS DATE)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
SELECT
|
|
||||||
40148 AS project_id,
|
|
||||||
Q.EmpId AS employee_id,
|
|
||||||
0 AS process_id,
|
|
||||||
Q.VisitDate AS visit_date,
|
|
||||||
Q.QuestionCategoryId AS question_category_id,
|
|
||||||
Q.QuestionCategory AS question_category,
|
|
||||||
QM.QuestionId AS question_id,
|
|
||||||
QM.Question AS question,
|
|
||||||
ISNULL(QA.AnswerId,0) AS answer_id,
|
|
||||||
ISNULL(QA.Answer,'') AS answer,
|
|
||||||
CASE
|
|
||||||
WHEN QA.AnswerId IS NULL THEN 'Not Answer'
|
|
||||||
WHEN QA.RightAnswer = 1 THEN 'Y'
|
|
||||||
WHEN QA.RightAnswer IS NULL THEN 'Not Answer'
|
|
||||||
ELSE 'N'
|
|
||||||
END AS correct_answer,
|
|
||||||
GETDATE() AS update_date,
|
|
||||||
'SP-Pius' AS update_by
|
|
||||||
FROM QUIZ Q
|
|
||||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Question QM
|
|
||||||
ON Q.QuestionId = QM.QuestionId
|
|
||||||
LEFT JOIN OneApp_KelloggsMT.dbo.Master_OQAD_Answer QA
|
|
||||||
ON Q.AnswerId = QA.AnswerId;
|
|
||||||
where Q.EmpId not in (3167,4724,4019,4902,4375,3352,3081,5244,3181,4810,5166,4895,4817,5173,5237,3608,4646,4126,3243,3948,3950,3599,3777,5235,5346,3606,3079,5175,5242,5064,4366,5071,5339,4812,4722,3169,5270,3326,3689,4062,3131,5054,5178,3371,4829,5185,4698,4836,3760,4665,5218,4451,4280,3269,3098,5327,4978,5261,3335,5090,5320,4214,4919,4570,3150,4221,3328,3112,3290,4950,4430,5104,3383,5213,4864,5199,5206,4439,3207,5033,4684,3979,5211,4774,3117,5102,5280,3309,4931,4753,5123,3155,5085,4914,5092,4387,5325,5154,4100,4805,3402,5187,3580,3409,3060,3962,4696,4831,3715,4710,4242,5282,5121,4933,5289,5282,3444,5104,3095,4933,4142,3437,3530,3088,3715,4840,5002,4831,5187,3361,3962,5113,3086,5106,5256,5085,4914,5092,3470,4387,5277,3309,5239,4712,4945,5246,5123,4774,3952,3603,4999,3727,3378,3207,5341,4992,5199,5334,4672,5206,3342,3520,3454,3171,4916,3105,4396,3112,3290,5265,3119,5227,3143,3150,5056,4529,3328,5313,5220,4693,5320,4615,3755,4444,3406,4978,4451,3746,4280,3399,5322,4617,4795,5218,3330,3159,4698,5144,5054,3473,3131,4726,4688,4743,3689,3812,3107,3169,5332,3347,5023,5022,4812,3380,3067,3950,4954,5132,3081,5296,5244,3140,4019,3492,4375,5210,3126,5289,3482,4847,5296,4954,4436,3380,5127,5305,3306,5291,5023,5120,4531,4346,5229,3107,3812,4353,3145,3152,3330,5322,4439,5144,4795,3363,4054,4232,3399,4239,3755,5279,4930,5213,3406,4864,5286,5220,4693,5101,5227,3985,3297,3636,4529,3420,5265,4916,3591,3335,4460,5343,4638,3342,5327,3520,3171,4467,3700,5336,4987,3515,4992,3760,5267,4562,3247,5082,4555,4733,5061,5239,3090,4712,5246,4904,5270,3059,4866,5277,4750,5106,3361,4968,5175,5310,3496,3608,4840,5248,4543,3615,4142,3437,3088,3444,3078,4902,3095,3567,4543,4450,3140,5319,4614,3496,4970,4443,5132,3916,3149,5310,3327,5312,3156,4968,4894,3394,5065,4538,4688,4866,3251,3090,3268,4671,4849,4904,4726,5082,4555,5191,4733,4562,3515,5300,4951,5115,3173,5293,4944,4460,4987,4638,4467,3591,3064,4526,3821,3232,3420,5056,5046,5008,3985,3636,5350,3109,5272,5101,5279,3358,4396,5172,5110,4054,4232,5281,3363,4999,3638,3603,3104,5027,5205,4856,4151,3083,4346,3764,3819,3415,4353,3470,3299,5167,3346,5113,3306,5331,5291,5160,5120,5127,4913,5091,4564,5262,5196,4847,5077,4666,4139,3612,3978,3078,5340,4139,3612,4991,5248,3170,5155,5333,4628,5077,4217,5084,4032,4913,4564,5257,5324,4381,5331,5250,5160,4134,4312,4141,3059,3764,3415,4305,5027,3220,3104,5321,4972,5307,5267,5136,5314,4787,3315,3322,4540,3835,3700,3486,4547,5172,5001,5008,3427,3379,5046,3372,3821,4526,3650,5124,5302,5286,3130,5115,4937,3132,5293,5117,3118,4239,5300,4951,4343,3897,5219,5191,4671,4849,5212,5229,5003,5181,4531,5065,5345,3538,4538,4189,4229,3531,5276,5305,4578,3106,3149,5312,5269,3113,4436,3099,4443,4970,3916,3567,3975,4153,4844,5210,5200,4844,3533,4666,5022,3184,5200,4402,4153,3704,3975,4922,3099,5262,4751,5091,3106,5269,5093,5276,4578,3346,5202,4675,4229,3531,3182,5195,5167,3203,5003,5205,5150,5328,3220,4979,4856,5157,4452,5335,3514,5219,3638,4343,4383,3289,4390,3118,3296,5281,5110,3438,3267,5074,5117,5252,3780,3431,5005,3372,3379,5176,3232,5316,4611,4967,3068,5309,3139,3495,3673,4540,3835,4891,3486,3488,3315,3322,5307,5136,3070,4877,4787,5314,5055,3939,3234,4072,5181,4305,3251,3374,5119,4134,3080,5297,4065,4894,3127,5250,4763,4941,5257,3298,4381,3327,3156,4687,4338,5221,4450,5155,3400,5333,5207,4628,5162,3742,4098,3742,4943,5121,3129,5207,5114,5292,4687,3122,4338,4696,3464,3113,4333,3298,3127,5031,4941,4763,5209,5119,3388,3566,3538,4065,3744,4072,3374,4753,3234,3939,5102,5088,5266,3070,5055,4877,3096,3979,3132,3488,3139,4841,5019,5302,3495,5124,5309,5012,4611,3369,3509,5176,5145,5323,5005,3153,4656,4371,3431,5254,5252,4547,3091,5074,4198,3782,3433,4383,4651,4829,4302,3953,5349,5321,3514,4972,5178,5150,5328,4979,5342,4452,5157,5304,3203,5195,3490,4312,3141,5126,4141,4675,4366,5057,5235,5064,3243,5093,4217,4922,3407,4751,4810,3170,4283,3756,3704,4974,3533,4991,3184,4974,3492,3143,4098,4283,3400,3756,3407,4710,5221,3067,3580,5057,3409,5196,3060,4805,4100,3402,5325,5297,3490,3141,5154,5126,3155,5304,4931,5109,5287,4117,5280,3473,4651,4302,3953,5033,5211,3782,4684,3433,3084,4891,3746,3383,4371,4022,5254,3673,5313,4207,3153,5316,3509,5145,5323,4430,4919,4570,3369,4221,3105,5090,4563,5012,3269,4841,3098,5019,3454,5261,3438,4672,3267,4836,3096,4665,5259,5088,3727,5266,3378,5185,4390,4397,3371,4062,4945,3528,3326,4722,3388,5256,3566,3744,5202,5242,3086,5071,5209,3777,4333,3606,3079,5002,3948,4126,3599,3464,5344,4817,5173,5351,4646,3122,3352,3530,5337,3181,3359,5166,5292,5114,4943)
|
|
||||||
-- select d.employee_id from OQaD d where project_id=40148 and d.employee_id=Q.EmpId and CONVERT(date,d.visit_date,101)=
|
|
||||||
-- CONVERT(date,q.VisitDate,101))
|
|
||||||
|
|
||||||
]
|
|
||||||
(Background on this error at: https://sqlalche.me/e/20/f405)
|
|
||||||
2026-06-17 12:19:35 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:19:35 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 12:19:35 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 12:19:35 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 12:19:37 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000210484378B0>
|
|
||||||
2026-06-17 12:19:38 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000021049928910>
|
|
||||||
2026-06-17 12:19:39 | INFO | Database connections established
|
|
||||||
2026-06-17 12:19:39 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 12:19:40 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 12:19:40 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 12:20:14 | INFO | Fetched 291,100 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 12:20:14 | INFO | Fetched 893 matched empids fetched for OQAD
|
|
||||||
2026-06-17 12:20:14 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:20:14 | INFO | Processing Table: OQaD
|
|
||||||
2026-06-17 12:20:14 | INFO | Exists: True
|
|
||||||
2026-06-17 12:20:14 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
|
||||||
2026-06-17 12:20:14 | INFO | Fetching data for 893 EMPIDs
|
|
||||||
2026-06-17 12:20:14 | INFO | Fetching OQaD data for run_date=2026-06-16
|
|
||||||
2026-06-17 12:20:30 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
|
|
||||||
2026-06-17 12:20:30 | ERROR | Failed processing table OQaD
|
|
||||||
Traceback (most recent call last):
|
|
||||||
File "D:\data_move\main2.py", line 226, in main
|
|
||||||
df = get_dataframe(
|
|
||||||
fn_name=fn_name,
|
|
||||||
...<3 lines>...
|
|
||||||
run_date=run_date,
|
|
||||||
)
|
|
||||||
File "D:\data_move\main2.py", line 81, in get_dataframe
|
|
||||||
return fn(sql_engine, run_date)
|
|
||||||
TypeError: fetch_OQaD() missing 3 required positional arguments: 'table_type', 'empids', and 'run_date'
|
|
||||||
2026-06-17 12:38:43 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:38:43 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 12:38:43 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 12:38:43 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 12:38:45 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001C35AC2F750>
|
|
||||||
2026-06-17 12:38:46 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001C35BDF87D0>
|
|
||||||
2026-06-17 12:38:46 | INFO | Database connections established
|
|
||||||
2026-06-17 12:38:46 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 12:38:47 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 12:38:47 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 12:38:58 | INFO | Fetched 291,112 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 12:38:58 | INFO | Fetched 893 matched empids fetched for OQAD
|
|
||||||
2026-06-17 12:38:58 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:38:58 | INFO | Processing Table: OQaD
|
|
||||||
2026-06-17 12:38:58 | INFO | Exists: True
|
|
||||||
2026-06-17 12:38:58 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
|
||||||
2026-06-17 12:38:58 | INFO | Fetching data for 893 EMPIDs
|
|
||||||
2026-06-17 12:38:58 | INFO | Fetching OQaD data for run_date=2026-06-16
|
|
||||||
2026-06-17 12:39:05 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
|
|
||||||
2026-06-17 12:39:05 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:39:05 | INFO | Pipeline Completed Successfully
|
|
||||||
2026-06-17 12:39:05 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:39:30 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:39:30 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 12:39:30 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 12:39:30 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 12:39:31 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000025D1633F750>
|
|
||||||
2026-06-17 12:39:32 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000025D175887D0>
|
|
||||||
2026-06-17 12:39:33 | INFO | Database connections established
|
|
||||||
2026-06-17 12:39:33 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 12:39:33 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 12:39:33 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 12:39:43 | INFO | Fetched 291,112 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 12:39:43 | INFO | Fetched 893 matched empids fetched for OQAD
|
|
||||||
2026-06-17 12:39:43 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:39:43 | INFO | Processing Table: OQaD
|
|
||||||
2026-06-17 12:39:43 | INFO | Exists: True
|
|
||||||
2026-06-17 12:39:43 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
|
||||||
2026-06-17 12:39:43 | INFO | Fetching data for 893 EMPIDs
|
|
||||||
2026-06-17 12:39:43 | INFO | Fetching OQaD data for run_date=2026-06-16
|
|
||||||
2026-06-17 12:39:46 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
|
|
||||||
2026-06-17 12:39:46 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:39:46 | INFO | Pipeline Completed Successfully
|
|
||||||
2026-06-17 12:39:46 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:44:50 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:44:50 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 12:44:50 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 12:44:50 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 12:44:51 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E3647178B0>
|
|
||||||
2026-06-17 12:44:52 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001E365BB8910>
|
|
||||||
2026-06-17 12:44:53 | INFO | Database connections established
|
|
||||||
2026-06-17 12:44:53 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 12:44:53 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 12:44:53 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 12:45:07 | INFO | Fetched 291,115 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 12:45:08 | INFO | Fetched 893 matched empids fetched for OQAD
|
|
||||||
2026-06-17 12:45:08 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:45:08 | INFO | Processing Table: OQaD
|
|
||||||
2026-06-17 12:45:08 | INFO | Exists: True
|
|
||||||
2026-06-17 12:45:08 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
|
||||||
2026-06-17 12:45:08 | INFO | Fetching data for 893 EMPIDs
|
|
||||||
2026-06-17 12:45:08 | INFO | Fetching OQaD data for run_date=2026-06-16
|
|
||||||
2026-06-17 12:45:10 | INFO | fn name is fetch_OQad ------Fetched 4998 rows
|
|
||||||
2026-06-17 12:45:10 | INFO | Fetched 4998 rows
|
|
||||||
2026-06-17 12:45:10 | INFO | Deleting from OQaD
|
|
||||||
2026-06-17 12:45:11 | INFO | OQaD: inserted 4,998 rows into ClickHouse
|
|
||||||
2026-06-17 12:45:11 | INFO | OQaD loaded successfully (4998 rows)
|
|
||||||
2026-06-17 12:45:11 | INFO | ================================================================================
|
|
||||||
2026-06-17 12:45:11 | INFO | Pipeline Completed Successfully
|
|
||||||
2026-06-17 12:45:11 | INFO | ================================================================================
|
|
||||||
2026-06-17 13:23:53 | INFO | ================================================================================
|
|
||||||
2026-06-17 13:23:53 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 13:23:53 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 13:23:53 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 13:23:54 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000281306878B0>
|
|
||||||
2026-06-17 13:23:56 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000028131AD8910>
|
|
||||||
2026-06-17 13:23:56 | INFO | Database connections established
|
|
||||||
2026-06-17 13:23:56 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 13:23:57 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 13:23:58 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 13:24:15 | INFO | Fetched 291,128 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 13:24:15 | INFO | Fetched 4,819 matched empids fetched for OQAD
|
|
||||||
2026-06-17 13:24:15 | INFO | ================================================================================
|
|
||||||
2026-06-17 13:24:15 | INFO | Processing Table: OQaD
|
|
||||||
2026-06-17 13:24:15 | INFO | Exists: True
|
|
||||||
2026-06-17 13:24:15 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
|
||||||
2026-06-17 13:24:15 | INFO | Fetching data for 4,819 EMPIDs
|
|
||||||
2026-06-17 13:24:15 | INFO | Fetching OQaD data for run_date=2026-06-16
|
|
||||||
2026-06-17 13:24:22 | INFO | fn name is fetch_OQad ------Fetched 1072 rows
|
|
||||||
2026-06-17 13:24:22 | INFO | Fetched 1072 rows
|
|
||||||
2026-06-17 13:24:22 | INFO | Deleting from OQaD
|
|
||||||
2026-06-17 13:24:23 | INFO | OQaD: inserted 1,072 rows into ClickHouse
|
|
||||||
2026-06-17 13:24:23 | INFO | OQaD loaded successfully (1072 rows)
|
|
||||||
2026-06-17 13:24:23 | INFO | ================================================================================
|
|
||||||
2026-06-17 13:24:23 | INFO | Processing Table: Survey
|
|
||||||
2026-06-17 13:24:23 | INFO | Fetching data for 863 MIDs
|
|
||||||
2026-06-17 13:24:23 | INFO | Fetched 142 rows from SQL Server
|
|
||||||
2026-06-17 13:24:23 | INFO | Fetched 142 rows
|
|
||||||
2026-06-17 13:24:23 | INFO | Deleting from Survey
|
|
||||||
2026-06-17 13:24:24 | ERROR | Failed processing table Survey
|
|
||||||
Traceback (most recent call last):
|
|
||||||
File "D:\data_move\main2.py", line 299, in main
|
|
||||||
load_to_clickhouse(
|
|
||||||
~~~~~~~~~~~~~~~~~~^
|
|
||||||
client=client,
|
|
||||||
^^^^^^^^^^^^^^
|
|
||||||
table_name=table_name,
|
|
||||||
^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
df=df,
|
|
||||||
^^^^^^
|
|
||||||
)
|
|
||||||
^
|
|
||||||
File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse
|
|
||||||
client.insert_arrow(
|
|
||||||
~~~~~~~~~~~~~~~~~~~^
|
|
||||||
table=table_name,
|
|
||||||
^^^^^^^^^^^^^^^^^
|
|
||||||
arrow_table=arrow_table,
|
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
)
|
|
||||||
^
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow
|
|
||||||
return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings)
|
|
||||||
~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert
|
|
||||||
response = self._raw_request(insert_block, params, headers, server_wait=False)
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request
|
|
||||||
self._error_handler(response)
|
|
||||||
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler
|
|
||||||
raise err_type(err_str, code=code, name=name) from None
|
|
||||||
clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column Project_Id in table kelloggs_1.Survey (d8c44908-2ed7-4d29-bd10-8e70fa84dfba). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123)
|
|
||||||
2026-06-17 13:27:22 | INFO | ================================================================================
|
|
||||||
2026-06-17 13:27:22 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 13:27:22 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 13:27:22 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 13:27:23 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000026D13D878B0>
|
|
||||||
2026-06-17 13:27:25 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000026D151D8910>
|
|
||||||
2026-06-17 13:27:25 | INFO | Database connections established
|
|
||||||
2026-06-17 13:27:25 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 13:27:26 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 13:27:26 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 13:27:37 | INFO | Fetched 291,128 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 13:27:38 | INFO | Fetched 4,814 matched empids fetched for OQAD
|
|
||||||
2026-06-17 13:27:38 | INFO | ================================================================================
|
|
||||||
2026-06-17 13:27:38 | INFO | Processing Table: OQaD
|
|
||||||
2026-06-17 13:27:38 | INFO | Exists: True
|
|
||||||
2026-06-17 13:27:38 | INFO | Path: D:\data_move\src\sql\fact\OQaD.sql
|
|
||||||
2026-06-17 13:27:38 | INFO | Fetching data for 4,814 EMPIDs
|
|
||||||
2026-06-17 13:27:38 | INFO | Fetching OQaD data for run_date=2026-06-16
|
|
||||||
2026-06-17 13:27:44 | INFO | fn name is fetch_OQad ------Fetched 1072 rows
|
|
||||||
2026-06-17 13:27:44 | INFO | Fetched 1072 rows
|
|
||||||
2026-06-17 13:27:44 | INFO | Deleting from OQaD
|
|
||||||
2026-06-17 13:27:45 | INFO | OQaD: inserted 1,072 rows into ClickHouse
|
|
||||||
2026-06-17 13:27:45 | INFO | OQaD loaded successfully (1072 rows)
|
|
||||||
2026-06-17 13:27:45 | INFO | ================================================================================
|
|
||||||
2026-06-17 13:27:45 | INFO | Processing Table: Survey
|
|
||||||
2026-06-17 13:27:45 | INFO | Fetching data for 863 MIDs
|
|
||||||
2026-06-17 13:27:45 | INFO | Fetched 142 rows from SQL Server
|
|
||||||
2026-06-17 13:27:45 | INFO | Fetched 142 rows
|
|
||||||
2026-06-17 13:27:45 | INFO | Creating table Survey
|
|
||||||
2026-06-17 13:27:45 | INFO | Table ready: Survey
|
|
||||||
2026-06-17 13:27:45 | INFO | Survey: inserted 142 rows into ClickHouse
|
|
||||||
2026-06-17 13:27:45 | INFO | Survey loaded successfully (142 rows)
|
|
||||||
2026-06-17 13:27:45 | INFO | ================================================================================
|
|
||||||
2026-06-17 13:27:45 | INFO | Pipeline Completed Successfully
|
|
||||||
2026-06-17 13:27:45 | INFO | ================================================================================
|
|
||||||
2026-06-17 15:05:42 | INFO | ================================================================================
|
|
||||||
2026-06-17 15:05:42 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 15:05:42 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 15:05:42 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 15:05:43 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002009FB778B0>
|
|
||||||
2026-06-17 15:05:45 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000200A0FB8910>
|
|
||||||
2026-06-17 15:05:45 | INFO | Database connections established
|
|
||||||
2026-06-17 15:05:45 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 15:05:45 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 15:05:46 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 15:05:57 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 15:05:57 | INFO | Fetched 4,814 matched empids fetched for OQAD
|
|
||||||
2026-06-17 15:05:57 | INFO | ================================================================================
|
|
||||||
2026-06-17 15:05:57 | INFO | Processing Table: additional_visibility
|
|
||||||
2026-06-17 15:05:57 | INFO | Fetching data for 863 MIDs
|
|
||||||
2026-06-17 15:05:58 | ERROR | Failed processing table additional_visibility
|
|
||||||
Traceback (most recent call last):
|
|
||||||
File "D:\data_move\main2.py", line 228, in main
|
|
||||||
df = fetch_data(
|
|
||||||
engine=sql_engine,
|
|
||||||
...<3 lines>...
|
|
||||||
run_date=run_date,
|
|
||||||
)
|
|
||||||
File "D:\data_move\src\fact.py", line 34, in fetch_data
|
|
||||||
df = pl.read_database(
|
|
||||||
query=sql,
|
|
||||||
connection=engine
|
|
||||||
)
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 287, in read_database
|
|
||||||
).to_polars(
|
|
||||||
~~~~~~~~~^
|
|
||||||
batch_size=batch_size,
|
|
||||||
^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
...<2 lines>...
|
|
||||||
infer_schema_length=infer_schema_length,
|
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
)
|
|
||||||
^
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 588, in to_polars
|
|
||||||
frame = frame_init(
|
|
||||||
batch_size=batch_size,
|
|
||||||
...<2 lines>...
|
|
||||||
infer_schema_length=infer_schema_length,
|
|
||||||
)
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 285, in _from_rows
|
|
||||||
(d[0], d[1:]) for d in self.result.cursor.description
|
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
AttributeError: 'NoneType' object has no attribute 'description'
|
|
||||||
2026-06-17 15:07:15 | INFO | ================================================================================
|
|
||||||
2026-06-17 15:07:15 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 15:07:15 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 15:07:15 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 15:07:16 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001BD510978B0>
|
|
||||||
2026-06-17 15:07:17 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001BD52518910>
|
|
||||||
2026-06-17 15:07:18 | INFO | Database connections established
|
|
||||||
2026-06-17 15:07:18 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 15:07:18 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 15:07:18 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 15:07:31 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 15:07:31 | INFO | Fetched 4,814 matched empids fetched for OQAD
|
|
||||||
2026-06-17 15:07:31 | INFO | ================================================================================
|
|
||||||
2026-06-17 15:07:31 | INFO | Processing Table: additional_visibility
|
|
||||||
2026-06-17 15:07:31 | INFO | Fetching data for 863 MIDs
|
|
||||||
2026-06-17 15:07:32 | INFO | Fetched 1,773 rows from SQL Server
|
|
||||||
2026-06-17 15:07:32 | INFO | Fetched 1773 rows
|
|
||||||
2026-06-17 15:07:32 | INFO | Deleting from additional_visibility
|
|
||||||
2026-06-17 15:07:32 | ERROR | Failed processing table additional_visibility
|
|
||||||
Traceback (most recent call last):
|
|
||||||
File "D:\data_move\main2.py", line 299, in main
|
|
||||||
load_to_clickhouse(
|
|
||||||
~~~~~~~~~~~~~~~~~~^
|
|
||||||
client=client,
|
|
||||||
^^^^^^^^^^^^^^
|
|
||||||
table_name=table_name,
|
|
||||||
^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
df=df,
|
|
||||||
^^^^^^
|
|
||||||
)
|
|
||||||
^
|
|
||||||
File "D:\data_move\clickhouse_task\load_table.py", line 25, in load_to_clickhouse
|
|
||||||
client.insert_arrow(
|
|
||||||
~~~~~~~~~~~~~~~~~~~^
|
|
||||||
table=table_name,
|
|
||||||
^^^^^^^^^^^^^^^^^
|
|
||||||
arrow_table=arrow_table,
|
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
)
|
|
||||||
^
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 1054, in insert_arrow
|
|
||||||
return self.raw_insert(full_table, column_names, insert_block, settings, "Arrow", transport_settings)
|
|
||||||
~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 432, in raw_insert
|
|
||||||
response = self._raw_request(insert_block, params, headers, server_wait=False)
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request
|
|
||||||
self._error_handler(response)
|
|
||||||
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
|
|
||||||
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler
|
|
||||||
raise err_type(err_str, code=code, name=name) from None
|
|
||||||
clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 16, server response: Code: 16. DB::Exception: No such column project_id in table kelloggs_1.additional_visibility (9e03e625-028a-47db-8f60-e8ab93e907ea). (NO_SUCH_COLUMN_IN_TABLE) (for url http://172.188.12.194:8123)
|
|
||||||
2026-06-17 15:08:55 | INFO | ================================================================================
|
|
||||||
2026-06-17 15:08:55 | INFO | Hello from data-move Python data pipeline!
|
|
||||||
2026-06-17 15:08:55 | INFO | Pipeline Run Date: 2026-06-16
|
|
||||||
2026-06-17 15:08:55 | INFO | Connecting to databases...
|
|
||||||
2026-06-17 15:08:56 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001ED5A4C78B0>
|
|
||||||
2026-06-17 15:08:58 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001ED5B948910>
|
|
||||||
2026-06-17 15:08:58 | INFO | Database connections established
|
|
||||||
2026-06-17 15:08:58 | INFO | Collecting MIDs for: 2026-06-16
|
|
||||||
2026-06-17 15:08:59 | INFO | Found 863 MIDs
|
|
||||||
2026-06-17 15:08:59 | INFO | Fetching quiz_empids data for EMPID and Visitid
|
|
||||||
2026-06-17 15:09:12 | INFO | Fetched 291,151 total empid and visitdate fetched for OQAD from SQL Server
|
|
||||||
2026-06-17 15:09:12 | INFO | Fetched 4,814 matched empids fetched for OQAD
|
|
||||||
2026-06-17 15:09:12 | INFO | ================================================================================
|
|
||||||
2026-06-17 15:09:12 | INFO | Processing Table: additional_visibility
|
|
||||||
2026-06-17 15:09:12 | INFO | Fetching data for 863 MIDs
|
|
||||||
2026-06-17 15:09:14 | INFO | Fetched 1,773 rows from SQL Server
|
|
||||||
2026-06-17 15:09:14 | INFO | Fetched 1773 rows
|
|
||||||
2026-06-17 15:09:14 | INFO | Creating table additional_visibility
|
|
||||||
2026-06-17 15:09:14 | INFO | Table ready: additional_visibility
|
|
||||||
2026-06-17 15:09:15 | INFO | additional_visibility: inserted 1,773 rows into ClickHouse
|
|
||||||
2026-06-17 15:09:15 | INFO | additional_visibility loaded successfully (1773 rows)
|
|
||||||
2026-06-17 15:09:15 | INFO | ================================================================================
|
|
||||||
2026-06-17 15:09:15 | INFO | Pipeline Completed Successfully
|
|
||||||
2026-06-17 15:09:15 | INFO | ================================================================================
|
|
||||||
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
+1129
-243
File diff suppressed because it is too large
Load Diff
@@ -1,276 +0,0 @@
|
|||||||
# /// script
|
|
||||||
# requires-python = ">=3.11"
|
|
||||||
# dependencies = [
|
|
||||||
# "polars>=0.20.0",
|
|
||||||
# "pyarrow>=18.0.0",
|
|
||||||
# "sqlalchemy>=2.0.0",
|
|
||||||
# "pyodbc>=5.0.0",
|
|
||||||
# "clickhouse-connect>=0.7.0",
|
|
||||||
# "clickhouse-sqlalchemy>=0.3.2",
|
|
||||||
# "pyyaml>=6.0.3",
|
|
||||||
# "python-dotenv>=1.0.0",
|
|
||||||
# ]
|
|
||||||
# ///
|
|
||||||
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import sys
|
|
||||||
from datetime import date, datetime, timedelta
|
|
||||||
|
|
||||||
import polars as pl
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
|
|
||||||
from log import log
|
|
||||||
|
|
||||||
from clickhouse_task.create_table import create_clickhouse_table
|
|
||||||
from clickhouse_task.delete_task import (
|
|
||||||
delete_existing_data,
|
|
||||||
truncate_table,
|
|
||||||
)
|
|
||||||
|
|
||||||
from clickhouse_task.load_table import load_to_clickhouse
|
|
||||||
|
|
||||||
from db_con.connection import (
|
|
||||||
build_sql_server_engine,
|
|
||||||
build_clickhouse_engine,
|
|
||||||
get_clickhouse_client,
|
|
||||||
)
|
|
||||||
|
|
||||||
from mids import (
|
|
||||||
MID_TABLE_COV,
|
|
||||||
MID_TABLE_COV1,
|
|
||||||
)
|
|
||||||
|
|
||||||
from masters.dimensions import *
|
|
||||||
from masters.bridge import *
|
|
||||||
from kpi.facts import *
|
|
||||||
|
|
||||||
|
|
||||||
# ==========================================================
|
|
||||||
# Helpers
|
|
||||||
# ==========================================================
|
|
||||||
|
|
||||||
def table_exists(
|
|
||||||
client,
|
|
||||||
table_name: str,
|
|
||||||
) -> bool:
|
|
||||||
|
|
||||||
return bool(
|
|
||||||
client.command(
|
|
||||||
f"EXISTS TABLE {table_name}"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def get_dataframe(
|
|
||||||
fn_name: str,
|
|
||||||
fetch_by: str,
|
|
||||||
sql_engine,
|
|
||||||
mids,
|
|
||||||
run_date,
|
|
||||||
):
|
|
||||||
|
|
||||||
fn = globals()[fn_name]
|
|
||||||
|
|
||||||
if fetch_by == "mids":
|
|
||||||
return fn(sql_engine, mids)
|
|
||||||
|
|
||||||
if fetch_by == "run_date":
|
|
||||||
return fn(sql_engine, run_date)
|
|
||||||
|
|
||||||
return fn(sql_engine)
|
|
||||||
|
|
||||||
|
|
||||||
# ==========================================================
|
|
||||||
# Main
|
|
||||||
# ==========================================================
|
|
||||||
|
|
||||||
def main():
|
|
||||||
|
|
||||||
log.info("=" * 80)
|
|
||||||
log.info("Hello from data-move Python data pipeline!")
|
|
||||||
|
|
||||||
# ------------------------------------------------------
|
|
||||||
# Run Date
|
|
||||||
# ------------------------------------------------------
|
|
||||||
|
|
||||||
if len(sys.argv) > 1:
|
|
||||||
run_date = datetime.strptime(
|
|
||||||
sys.argv[1],
|
|
||||||
"%Y-%m-%d",
|
|
||||||
).date()
|
|
||||||
else:
|
|
||||||
run_date = date.today() - timedelta(days=1)
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
"Pipeline Run Date: %s",
|
|
||||||
run_date,
|
|
||||||
)
|
|
||||||
|
|
||||||
# ------------------------------------------------------
|
|
||||||
# Connections
|
|
||||||
# ------------------------------------------------------
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
"Connecting to databases..."
|
|
||||||
)
|
|
||||||
|
|
||||||
sql_engine = build_sql_server_engine()
|
|
||||||
clickhouse_engine = build_clickhouse_engine()
|
|
||||||
client = get_clickhouse_client()
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
"Database connections established"
|
|
||||||
)
|
|
||||||
|
|
||||||
# ------------------------------------------------------
|
|
||||||
# Delete Keys
|
|
||||||
# ------------------------------------------------------
|
|
||||||
|
|
||||||
mids = MID_TABLE_COV(
|
|
||||||
sql_engine,
|
|
||||||
run_date,
|
|
||||||
)
|
|
||||||
|
|
||||||
emp_visit_df = MID_TABLE_COV1(
|
|
||||||
sql_engine,
|
|
||||||
run_date,
|
|
||||||
)
|
|
||||||
|
|
||||||
# ------------------------------------------------------
|
|
||||||
# Config
|
|
||||||
# ------------------------------------------------------
|
|
||||||
|
|
||||||
with open(
|
|
||||||
"t.yml",
|
|
||||||
"r",
|
|
||||||
) as file:
|
|
||||||
|
|
||||||
config = yaml.safe_load(file)
|
|
||||||
|
|
||||||
# ------------------------------------------------------
|
|
||||||
# Process Tables
|
|
||||||
# ------------------------------------------------------
|
|
||||||
|
|
||||||
for table in config["tables"]:
|
|
||||||
|
|
||||||
table_name = table["name"]
|
|
||||||
operation = table["operation"]
|
|
||||||
fetch_by = table["fetch_by"]
|
|
||||||
table_type=table["type"]
|
|
||||||
|
|
||||||
log.info("=" * 80)
|
|
||||||
log.info(
|
|
||||||
"Processing Table: %s",
|
|
||||||
table_name,
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
|
|
||||||
# ------------------------------------------
|
|
||||||
# Fetch Data
|
|
||||||
# ------------------------------------------
|
|
||||||
|
|
||||||
fn_name = f"fetch_{table_name}"
|
|
||||||
|
|
||||||
df = get_dataframe(
|
|
||||||
fn_name=fn_name,
|
|
||||||
fetch_by=fetch_by,
|
|
||||||
sql_engine=sql_engine,
|
|
||||||
mids=mids,
|
|
||||||
run_date=run_date,
|
|
||||||
)
|
|
||||||
|
|
||||||
if df.is_empty():
|
|
||||||
|
|
||||||
log.warning(
|
|
||||||
"%s returned no rows",
|
|
||||||
table_name,
|
|
||||||
)
|
|
||||||
|
|
||||||
continue
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
"Fetched %s rows",
|
|
||||||
len(df),
|
|
||||||
)
|
|
||||||
|
|
||||||
# ------------------------------------------
|
|
||||||
# Create Table If Missing
|
|
||||||
# ------------------------------------------
|
|
||||||
|
|
||||||
exists = table_exists(
|
|
||||||
client,
|
|
||||||
table_name,
|
|
||||||
)
|
|
||||||
|
|
||||||
if not exists:
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
"Creating table %s",
|
|
||||||
table_name,
|
|
||||||
)
|
|
||||||
|
|
||||||
create_clickhouse_table(
|
|
||||||
df=df,
|
|
||||||
table_name=table_name,
|
|
||||||
clickhouse_engine=clickhouse_engine,
|
|
||||||
)
|
|
||||||
|
|
||||||
# ------------------------------------------
|
|
||||||
# Existing Table Logic
|
|
||||||
# ------------------------------------------
|
|
||||||
|
|
||||||
else:
|
|
||||||
|
|
||||||
if operation == "DELETE+INSERT":
|
|
||||||
|
|
||||||
truncate_table(
|
|
||||||
client,
|
|
||||||
table_name,
|
|
||||||
)
|
|
||||||
|
|
||||||
else:
|
|
||||||
|
|
||||||
delete_existing_data(
|
|
||||||
client=client,
|
|
||||||
table_name=table_name,
|
|
||||||
run_date=run_date,
|
|
||||||
mids=mids,
|
|
||||||
emp_visit_df=emp_visit_df,
|
|
||||||
)
|
|
||||||
|
|
||||||
# ------------------------------------------
|
|
||||||
# Load Data
|
|
||||||
# ------------------------------------------
|
|
||||||
|
|
||||||
load_to_clickhouse(
|
|
||||||
client=client,
|
|
||||||
table_name=table_name,
|
|
||||||
df=df,
|
|
||||||
)
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
"%s loaded successfully (%s rows)",
|
|
||||||
table_name,
|
|
||||||
len(df),
|
|
||||||
)
|
|
||||||
|
|
||||||
except Exception:
|
|
||||||
|
|
||||||
log.exception(
|
|
||||||
"Failed processing table %s",
|
|
||||||
table_name,
|
|
||||||
)
|
|
||||||
|
|
||||||
raise
|
|
||||||
|
|
||||||
log.info("=" * 80)
|
|
||||||
log.info("Pipeline Completed Successfully")
|
|
||||||
log.info("=" * 80)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
from time import sleep
|
||||||
import sys
|
import sys
|
||||||
from datetime import date, datetime, timedelta
|
from datetime import date, datetime, timedelta
|
||||||
|
|
||||||
@@ -126,7 +126,7 @@ def main():
|
|||||||
# ------------------------------------------------------
|
# ------------------------------------------------------
|
||||||
|
|
||||||
with open(
|
with open(
|
||||||
"y.yml",
|
"t.yml",
|
||||||
"r",
|
"r",
|
||||||
) as file:
|
) as file:
|
||||||
|
|
||||||
@@ -254,4 +254,42 @@ def main():
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
||||||
|
|
||||||
|
for attempt in range(3):
|
||||||
|
try:
|
||||||
main()
|
main()
|
||||||
|
|
||||||
|
with open("Pipeline_config.yml", "r") as f:
|
||||||
|
config = yaml.safe_load(f)
|
||||||
|
|
||||||
|
config["pipeline"]["last_successful_run_date"] = str(date.today())
|
||||||
|
|
||||||
|
with open("Pipeline_config.yml", "w") as f:
|
||||||
|
yaml.safe_dump(config, f, sort_keys=False)
|
||||||
|
|
||||||
|
log.info(
|
||||||
|
f"Pipeline completed successfully. "
|
||||||
|
f"last_successful_run_date={date.today()}"
|
||||||
|
)
|
||||||
|
|
||||||
|
break
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
|
||||||
|
with open("Pipeline_config.yml", "r") as f:
|
||||||
|
config = yaml.safe_load(f)
|
||||||
|
|
||||||
|
config["pipeline"]["run_date"] = str(date.today())
|
||||||
|
|
||||||
|
with open("Pipeline_config.yml", "w") as f:
|
||||||
|
yaml.safe_dump(config, f, sort_keys=False)
|
||||||
|
|
||||||
|
if attempt == 2:
|
||||||
|
raise
|
||||||
|
|
||||||
|
log.warning(
|
||||||
|
f"Pipeline failed. Retry {attempt + 1}/3. Error: {e}"
|
||||||
|
)
|
||||||
|
|
||||||
|
sleep(5)
|
||||||
@@ -1,64 +0,0 @@
|
|||||||
import os
|
|
||||||
# import pyarrow
|
|
||||||
import sys
|
|
||||||
import logging
|
|
||||||
from datetime import date, timedelta
|
|
||||||
import polars as pl
|
|
||||||
from sqlalchemy import create_engine, text
|
|
||||||
from sqlalchemy.engine import Engine, URL
|
|
||||||
import clickhouse_connect
|
|
||||||
from dotenv import load_dotenv
|
|
||||||
|
|
||||||
from log import log
|
|
||||||
from clickhouse_task.create_table import *
|
|
||||||
from db_con.connection import *
|
|
||||||
from mids import *
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_mapping_storevisibility(
|
|
||||||
engine: Engine,
|
|
||||||
run_date: date
|
|
||||||
) -> pl.DataFrame:
|
|
||||||
|
|
||||||
sql = f"""
|
|
||||||
SELECT DISTINCT
|
|
||||||
40148 AS project_id,
|
|
||||||
Z.StoreId AS store_id,
|
|
||||||
Z.VisibilityDefinitionId AS visibility_definition_id,
|
|
||||||
Z.FromDate AS from_date,
|
|
||||||
Z.ToDate AS to_date
|
|
||||||
|
|
||||||
FROM OneApp_KelloggsMT.dbo.Mapping_StoreVisibility Z
|
|
||||||
|
|
||||||
WHERE CAST(Z.FromDate AS DATE) <= '{run_date}'
|
|
||||||
AND CAST(Z.ToDate AS DATE) >= '{run_date}'
|
|
||||||
|
|
||||||
AND Z.VisibilityDefinitionId IN
|
|
||||||
(
|
|
||||||
SELECT DISTINCT VisibilityDefinitionId
|
|
||||||
FROM OneApp_KelloggsMT.dbo.Master_VisibilityDefinition
|
|
||||||
WHERE MenuId = 22
|
|
||||||
)
|
|
||||||
"""
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
f"Fetching Mapping Store Visibility for {run_date}"
|
|
||||||
)
|
|
||||||
|
|
||||||
df = pl.read_database(
|
|
||||||
query=sql,
|
|
||||||
connection=engine
|
|
||||||
)
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
f"Fetched {len(df):,} Mapping Store Visibility records"
|
|
||||||
)
|
|
||||||
|
|
||||||
return df
|
|
||||||
@@ -1,515 +0,0 @@
|
|||||||
import os
|
|
||||||
# import pyarrow
|
|
||||||
import sys
|
|
||||||
import logging
|
|
||||||
from datetime import date, timedelta
|
|
||||||
import polars as pl
|
|
||||||
from sqlalchemy import create_engine, text
|
|
||||||
from sqlalchemy.engine import Engine, URL
|
|
||||||
import clickhouse_connect
|
|
||||||
from dotenv import load_dotenv
|
|
||||||
|
|
||||||
from log import log
|
|
||||||
from clickhouse_task.create_table import *
|
|
||||||
from db_con.connection import *
|
|
||||||
from mids import *
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#PROJECT_ID = 40148
|
|
||||||
p=40148
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_Store_Master(engine: Engine) -> pl.DataFrame:
|
|
||||||
sql = """
|
|
||||||
SELECT
|
|
||||||
|
|
||||||
RegionName AS region,
|
|
||||||
StateId AS state_id,
|
|
||||||
StateName AS state,
|
|
||||||
CityId AS city_id,
|
|
||||||
CityName AS city,
|
|
||||||
CityCode AS cpm_city_id,
|
|
||||||
|
|
||||||
ChannelId AS channel_id,
|
|
||||||
ChannelName AS channel,
|
|
||||||
|
|
||||||
DistributorId AS distributor_id,
|
|
||||||
Distributor AS distributor_name,
|
|
||||||
|
|
||||||
ChainId AS keyaccount_id,
|
|
||||||
ChainName AS keyaccount,
|
|
||||||
|
|
||||||
StoreUniqueCode AS insight_store_id,
|
|
||||||
StoreCode AS client_store_code,
|
|
||||||
|
|
||||||
Latitude AS latitude,
|
|
||||||
Longitude AS longitude,
|
|
||||||
|
|
||||||
StoreCategoryId AS store_category_id,
|
|
||||||
StoreCategory AS store_category,
|
|
||||||
|
|
||||||
StoreTypeId AS store_type_id,
|
|
||||||
StoreType AS store_type,
|
|
||||||
|
|
||||||
StoreClassId AS store_classification_id,
|
|
||||||
StoreClass AS store_classification,
|
|
||||||
|
|
||||||
StLayerFourId,
|
|
||||||
|
|
||||||
StoreId AS store_id,
|
|
||||||
StoreName AS store_name,
|
|
||||||
Address AS address
|
|
||||||
|
|
||||||
FROM OneApp_KelloggsMT.dbo.vw_storedetail
|
|
||||||
"""
|
|
||||||
|
|
||||||
log.info("Fetching Store Master data")
|
|
||||||
|
|
||||||
df = pl.read_database(
|
|
||||||
query=sql,
|
|
||||||
connection=engine
|
|
||||||
)
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
f"Fetched {len(df):,} stores"
|
|
||||||
)
|
|
||||||
|
|
||||||
return df
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_SKU_Master(engine: Engine) -> pl.DataFrame:
|
|
||||||
|
|
||||||
sql = """
|
|
||||||
SELECT
|
|
||||||
CM.CategoryId AS category_id,
|
|
||||||
CM.CategoryCode AS category_code,
|
|
||||||
CM.CategoryName AS category_name,
|
|
||||||
|
|
||||||
SCA.SubCategoryId AS sub_category_id,
|
|
||||||
SCA.SubCategoryCode AS sub_category_code,
|
|
||||||
SCA.SubCategoryName AS sub_category_name,
|
|
||||||
|
|
||||||
BR.BrandId AS brand_id,
|
|
||||||
BR.BrandCode AS brand_code,
|
|
||||||
BR.BrandName AS brand_name,
|
|
||||||
|
|
||||||
SB.SubBrandId AS sub_brand_id,
|
|
||||||
SB.SubBrandCode AS sub_brand_code,
|
|
||||||
SB.SubBrandName AS sub_brand_name,
|
|
||||||
|
|
||||||
P.ProductId AS product_id,
|
|
||||||
P.ProductName AS product_name,
|
|
||||||
P.ProductCode AS product_code,
|
|
||||||
|
|
||||||
P.MRP AS mrp,
|
|
||||||
|
|
||||||
FL.FlavourId AS flavour_id,
|
|
||||||
FL.Flavour AS flavour,
|
|
||||||
|
|
||||||
P.Grammage AS grammage,
|
|
||||||
P.ProductSequence AS product_sequence,
|
|
||||||
P.CaseSize AS case_size,
|
|
||||||
|
|
||||||
MC.Company AS company_name,
|
|
||||||
MC.IsCompetitor AS is_competitor,
|
|
||||||
|
|
||||||
P.PTR AS ptr
|
|
||||||
|
|
||||||
FROM OneApp_KelloggsMT.dbo.Master_Product P
|
|
||||||
|
|
||||||
RIGHT JOIN OneApp_KelloggsMT.dbo.Master_Flavour FL
|
|
||||||
ON P.FlavourId = FL.FlavourId
|
|
||||||
|
|
||||||
RIGHT JOIN OneApp_KelloggsMT.dbo.Master_SubBrand SB
|
|
||||||
ON P.SubBrandId = SB.SubBrandId
|
|
||||||
|
|
||||||
RIGHT JOIN OneApp_KelloggsMT.dbo.Master_Brand BR
|
|
||||||
ON SB.BrandId = BR.BrandId
|
|
||||||
|
|
||||||
RIGHT JOIN OneApp_KelloggsMT.dbo.Master_SubCategory SCA
|
|
||||||
ON BR.SubCategoryId = SCA.SubCategoryId
|
|
||||||
|
|
||||||
RIGHT JOIN OneApp_KelloggsMT.dbo.Master_Category CM
|
|
||||||
ON SCA.CategoryId = CM.CategoryId
|
|
||||||
|
|
||||||
RIGHT JOIN OneApp_KelloggsMT.dbo.Master_Company MC
|
|
||||||
ON MC.CompanyId = BR.CompanyId
|
|
||||||
"""
|
|
||||||
|
|
||||||
log.info("Fetching SKU Master data")
|
|
||||||
|
|
||||||
df = pl.read_database(
|
|
||||||
query=sql,
|
|
||||||
connection=engine
|
|
||||||
)
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
f"Fetched {len(df):,} SKU Master rows"
|
|
||||||
)
|
|
||||||
|
|
||||||
return df
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_display_master(engine: Engine) -> pl.DataFrame:
|
|
||||||
"""
|
|
||||||
Fetch Display Master data.
|
|
||||||
Source: Master_Display
|
|
||||||
Target: display_master
|
|
||||||
"""
|
|
||||||
|
|
||||||
sql = """
|
|
||||||
SELECT
|
|
||||||
DisplayId AS display_id,
|
|
||||||
DisplayCode AS display_code,
|
|
||||||
DisplayName AS display_name,
|
|
||||||
DisplayRefImage AS display_ref_url
|
|
||||||
FROM OneApp_KelloggsMT.dbo.Master_Display
|
|
||||||
"""
|
|
||||||
|
|
||||||
log.info("Fetching Display Master data")
|
|
||||||
|
|
||||||
df = pl.read_database(
|
|
||||||
query=sql,
|
|
||||||
connection=engine
|
|
||||||
)
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
f"Fetched {len(df):,} Display Master records"
|
|
||||||
)
|
|
||||||
|
|
||||||
return df
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_Employee_Master(engine: Engine) -> pl.DataFrame:
|
|
||||||
"""
|
|
||||||
Fetch Employee Master data.
|
|
||||||
Source: vw_Employee_Detail + Mapping_PositionUser + Master_Position
|
|
||||||
Target: Employee_Master
|
|
||||||
"""
|
|
||||||
|
|
||||||
sql = """
|
|
||||||
SELECT
|
|
||||||
RegionId AS region_id,
|
|
||||||
RegionName AS region,
|
|
||||||
|
|
||||||
StateId AS state_id,
|
|
||||||
StateName AS state,
|
|
||||||
|
|
||||||
CityId AS city_id,
|
|
||||||
CityName AS city,
|
|
||||||
|
|
||||||
A.EmpId AS employee_id,
|
|
||||||
EmpName AS employee_name,
|
|
||||||
Gender AS gender,
|
|
||||||
|
|
||||||
A.DesignationId AS designation_id,
|
|
||||||
DesignationName AS designation,
|
|
||||||
|
|
||||||
SupervisorId AS manager_id,
|
|
||||||
SupervisorName AS manager_name,
|
|
||||||
|
|
||||||
JoinDate AS employee_joining_date,
|
|
||||||
ResignDate AS employee_resign_date,
|
|
||||||
|
|
||||||
C.PositionCode AS position_code,
|
|
||||||
|
|
||||||
EmpCode AS employee_legacy_code,
|
|
||||||
|
|
||||||
RIGHTNAME AS employee_role,
|
|
||||||
|
|
||||||
CASE
|
|
||||||
WHEN RIGHTNAME IN ('Client', 'Client HO')
|
|
||||||
THEN 'NON CPM'
|
|
||||||
ELSE 'CPM'
|
|
||||||
END AS employee_type
|
|
||||||
|
|
||||||
FROM OneApp_KelloggsMT.dbo.vw_Employee_Detail A
|
|
||||||
|
|
||||||
LEFT JOIN
|
|
||||||
(
|
|
||||||
SELECT DISTINCT
|
|
||||||
PositionId,
|
|
||||||
EmpId
|
|
||||||
FROM OneApp_KelloggsMT.dbo.Mapping_PositionUser
|
|
||||||
WHERE DATEDIFF(DAY, FromDate, GETDATE()) >= 0
|
|
||||||
AND DATEDIFF(DAY, ToDate, GETDATE()) <= 0
|
|
||||||
) B
|
|
||||||
ON A.EmpId = B.EmpId
|
|
||||||
|
|
||||||
LEFT JOIN
|
|
||||||
(
|
|
||||||
SELECT
|
|
||||||
PositionId,
|
|
||||||
PositionCode
|
|
||||||
FROM OneApp_KelloggsMT.dbo.Master_Position
|
|
||||||
) C
|
|
||||||
ON B.PositionId = C.PositionId
|
|
||||||
"""
|
|
||||||
|
|
||||||
log.info("Fetching Employee Master data")
|
|
||||||
|
|
||||||
df = pl.read_database(
|
|
||||||
query=sql,
|
|
||||||
connection=engine
|
|
||||||
)
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
f"Fetched {len(df):,} Employee Master records"
|
|
||||||
)
|
|
||||||
|
|
||||||
return df
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_Employee_Master(engine: Engine) -> pl.DataFrame:
|
|
||||||
"""
|
|
||||||
Fetch Employee Master data.
|
|
||||||
Source: vw_Employee_Detail + Mapping_PositionUser + Master_Position
|
|
||||||
Target: Employee_Master
|
|
||||||
"""
|
|
||||||
|
|
||||||
sql = """
|
|
||||||
SELECT
|
|
||||||
RegionId AS region_id,
|
|
||||||
RegionName AS region,
|
|
||||||
|
|
||||||
StateId AS state_id,
|
|
||||||
StateName AS state,
|
|
||||||
|
|
||||||
CityId AS city_id,
|
|
||||||
CityName AS city,
|
|
||||||
|
|
||||||
A.EmpId AS employee_id,
|
|
||||||
EmpName AS employee_name,
|
|
||||||
Gender AS gender,
|
|
||||||
|
|
||||||
A.DesignationId AS designation_id,
|
|
||||||
DesignationName AS designation,
|
|
||||||
|
|
||||||
SupervisorId AS manager_id,
|
|
||||||
SupervisorName AS manager_name,
|
|
||||||
|
|
||||||
JoinDate AS employee_joining_date,
|
|
||||||
ResignDate AS employee_resign_date,
|
|
||||||
|
|
||||||
C.PositionCode AS position_code,
|
|
||||||
|
|
||||||
EmpCode AS employee_legacy_code,
|
|
||||||
|
|
||||||
RIGHTNAME AS employee_role,
|
|
||||||
|
|
||||||
CASE
|
|
||||||
WHEN RIGHTNAME IN ('Client', 'Client HO')
|
|
||||||
THEN 'NON CPM'
|
|
||||||
ELSE 'CPM'
|
|
||||||
END AS employee_type
|
|
||||||
|
|
||||||
FROM OneApp_KelloggsMT.dbo.vw_Employee_Detail A
|
|
||||||
|
|
||||||
LEFT JOIN
|
|
||||||
(
|
|
||||||
SELECT DISTINCT
|
|
||||||
PositionId,
|
|
||||||
EmpId
|
|
||||||
FROM OneApp_KelloggsMT.dbo.Mapping_PositionUser
|
|
||||||
WHERE DATEDIFF(DAY, FromDate, GETDATE()) >= 0
|
|
||||||
AND DATEDIFF(DAY, ToDate, GETDATE()) <= 0
|
|
||||||
) B
|
|
||||||
ON A.EmpId = B.EmpId
|
|
||||||
|
|
||||||
LEFT JOIN
|
|
||||||
(
|
|
||||||
SELECT
|
|
||||||
PositionId,
|
|
||||||
PositionCode
|
|
||||||
FROM OneApp_KelloggsMT.dbo.Master_Position
|
|
||||||
) C
|
|
||||||
ON B.PositionId = C.PositionId
|
|
||||||
"""
|
|
||||||
|
|
||||||
log.info("Fetching Employee Master data")
|
|
||||||
|
|
||||||
df = pl.read_database(
|
|
||||||
query=sql,
|
|
||||||
connection=engine
|
|
||||||
)
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
f"Fetched {len(df):,} Employee Master records"
|
|
||||||
)
|
|
||||||
|
|
||||||
return df
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_Master_VisibilityReason(engine: Engine) -> pl.DataFrame:
|
|
||||||
"""
|
|
||||||
Source:
|
|
||||||
Master_VisibilityReason
|
|
||||||
|
|
||||||
Target:
|
|
||||||
Master_VisibilityReason
|
|
||||||
"""
|
|
||||||
|
|
||||||
sql = """
|
|
||||||
SELECT DISTINCT
|
|
||||||
40148 AS project_id,
|
|
||||||
MenuId AS menu_id,
|
|
||||||
VisibilityReasonId AS reason_id,
|
|
||||||
VisibilityReason AS reason
|
|
||||||
FROM OneApp_KelloggsMT.dbo.Master_VisibilityReason
|
|
||||||
"""
|
|
||||||
|
|
||||||
log.info("Fetching Master Visibility Reason data")
|
|
||||||
|
|
||||||
df = pl.read_database(
|
|
||||||
query=sql,
|
|
||||||
connection=engine
|
|
||||||
)
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
f"Fetched {len(df):,} Master Visibility Reason records"
|
|
||||||
)
|
|
||||||
|
|
||||||
return df
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_Master_VisibilityDefinition(engine: Engine) -> pl.DataFrame:
|
|
||||||
"""
|
|
||||||
Source:
|
|
||||||
OneApp_KelloggsMT.dbo.Master_VisibilityDefinition
|
|
||||||
|
|
||||||
Target:
|
|
||||||
Master_VisibilityDefinition
|
|
||||||
"""
|
|
||||||
|
|
||||||
sql = """
|
|
||||||
SELECT DISTINCT
|
|
||||||
40148 AS project_id,
|
|
||||||
VisibilityDefinitionId AS visibility_definition_id,
|
|
||||||
VisibilityDefinitionName AS visibility_definition_name,
|
|
||||||
GETDATE() AS create_date,
|
|
||||||
'SP-Pius' AS create_by
|
|
||||||
FROM OneApp_KelloggsMT.dbo.Master_VisibilityDefinition
|
|
||||||
"""
|
|
||||||
|
|
||||||
log.info("Fetching Master Visibility Definition data")
|
|
||||||
|
|
||||||
df = pl.read_database(
|
|
||||||
query=sql,
|
|
||||||
connection=engine
|
|
||||||
)
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
f"Fetched {len(df):,} Master Visibility Definition records"
|
|
||||||
)
|
|
||||||
|
|
||||||
return df
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_Master_Salesterritorylayer(
|
|
||||||
engine: Engine
|
|
||||||
) -> pl.DataFrame:
|
|
||||||
"""
|
|
||||||
Source:
|
|
||||||
Master_SalesTerritoryLayerOne
|
|
||||||
Master_SalesTerritoryLayerTwo
|
|
||||||
Master_SalesTerritoryLayerThree
|
|
||||||
Master_SalesTerritoryLayerFour
|
|
||||||
|
|
||||||
Target:
|
|
||||||
Master_Salesterritorylayer
|
|
||||||
"""
|
|
||||||
|
|
||||||
sql = """
|
|
||||||
SELECT DISTINCT
|
|
||||||
40148 AS project_id,
|
|
||||||
|
|
||||||
D.StLayerOneId AS st_layer_one_id,
|
|
||||||
D.StLayerOneName AS st_layer_one_name,
|
|
||||||
|
|
||||||
C.StLayerTwoId AS st_layer_two_id,
|
|
||||||
C.StLayerTwoName AS st_layer_two_name,
|
|
||||||
|
|
||||||
B.StLayerThreeId AS st_layer_three_id,
|
|
||||||
B.StLayerThreeName AS st_layer_three_name,
|
|
||||||
|
|
||||||
A.StLayerFourId AS st_layer_four_id,
|
|
||||||
A.StLayerFourName AS st_layer_four_name,
|
|
||||||
|
|
||||||
GETDATE() AS create_date,
|
|
||||||
'SP-Pius' AS create_by
|
|
||||||
|
|
||||||
FROM OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerFour A
|
|
||||||
|
|
||||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerThree B
|
|
||||||
ON A.StLayerThreeId = B.StLayerThreeId
|
|
||||||
|
|
||||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerTwo C
|
|
||||||
ON B.StLayerTwoId = C.StLayerTwoId
|
|
||||||
|
|
||||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_SalesTerritoryLayerOne D
|
|
||||||
ON C.StLayerOneId = D.StLayerOneId
|
|
||||||
"""
|
|
||||||
|
|
||||||
log.info("Fetching Master Sales Territory Layer data")
|
|
||||||
|
|
||||||
df = pl.read_database(
|
|
||||||
query=sql,
|
|
||||||
connection=engine
|
|
||||||
)
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
f"Fetched {len(df):,} Master Sales Territory Layer records"
|
|
||||||
)
|
|
||||||
|
|
||||||
return df
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_coverage_remarks(engine: Engine ) -> pl.DataFrame:
|
|
||||||
"""
|
|
||||||
Source:
|
|
||||||
OneApp_KelloggsMT.dbo.Master_NonWorkingReason
|
|
||||||
|
|
||||||
Target:
|
|
||||||
coverage_remarks
|
|
||||||
"""
|
|
||||||
|
|
||||||
sql = """
|
|
||||||
SELECT
|
|
||||||
40148 AS project_id,
|
|
||||||
ReasonId AS reason_id,
|
|
||||||
Reason AS reason_remarks
|
|
||||||
FROM OneApp_KelloggsMT.dbo.Master_NonWorkingReason
|
|
||||||
"""
|
|
||||||
|
|
||||||
log.info("Fetching Coverage Remarks")
|
|
||||||
|
|
||||||
df = pl.read_database(
|
|
||||||
query=sql,
|
|
||||||
connection=engine
|
|
||||||
)
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
f"Fetched {len(df):,} Coverage Remark records"
|
|
||||||
)
|
|
||||||
|
|
||||||
return df
|
|
||||||
+51
-4
@@ -242,8 +242,19 @@ def fetch_OQaD(
|
|||||||
on=["empid", "visitdate"],
|
on=["empid", "visitdate"],
|
||||||
how="inner",
|
how="inner",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if matched.is_empty():
|
||||||
|
|
||||||
|
empids=[0]
|
||||||
|
log.warning(
|
||||||
|
"%s Matched df in OQaD returned no rows",
|
||||||
|
table_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
else:
|
||||||
empids=matched["empid"].to_list()
|
empids=matched["empid"].to_list()
|
||||||
|
|
||||||
|
|
||||||
log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ")
|
log.info(f"Fetched {len(empids):,} matched empids fetched for OQAD ")
|
||||||
|
|
||||||
def fetch_data(
|
def fetch_data(
|
||||||
@@ -418,9 +429,9 @@ def fetch_Login( engine: Engine,
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
if not mids:
|
# if not mids:
|
||||||
log.warning("No MIDs — nothing to fetch.")
|
# log.warning("No MIDs — nothing to fetch.")
|
||||||
return pl.DataFrame()
|
# return pl.DataFrame()
|
||||||
|
|
||||||
mid_list = ",".join(str(mid) for mid in mids)
|
mid_list = ",".join(str(mid) for mid in mids)
|
||||||
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
|
log.info(f"Start Fetching data for these {len(mids):} MIDs or based on this date {run_date}")
|
||||||
@@ -430,7 +441,7 @@ def fetch_Login( engine: Engine,
|
|||||||
sql_template = f.read()
|
sql_template = f.read()
|
||||||
|
|
||||||
sql = sql_template.format(
|
sql = sql_template.format(
|
||||||
mid_list=mid_list,
|
# mid_list=mid_list,
|
||||||
run_date=run_date.strftime("%Y-%m-%d")
|
run_date=run_date.strftime("%Y-%m-%d")
|
||||||
)
|
)
|
||||||
log.info(f"Fetching data for {len(mids):,} MIDs")
|
log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||||
@@ -674,3 +685,39 @@ def fetch_Web_Logins( engine: Engine,
|
|||||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||||
|
|
||||||
return df
|
return df
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_Promotion(
|
||||||
|
engine: Engine,
|
||||||
|
table_name: str,
|
||||||
|
table_type: str,
|
||||||
|
mids: list[int],
|
||||||
|
run_date: date
|
||||||
|
) -> pl.DataFrame:
|
||||||
|
|
||||||
|
if not mids:
|
||||||
|
log.warning("No MIDs — nothing to fetch.")
|
||||||
|
return pl.DataFrame()
|
||||||
|
|
||||||
|
mid_list = ",".join(str(mid) for mid in mids)
|
||||||
|
|
||||||
|
sql_file = Path("src") / "sql" / f"{table_type.lower()}" / f"{table_name}.sql"
|
||||||
|
|
||||||
|
with open(sql_file, "r", encoding="utf-8") as f:
|
||||||
|
sql_template = f.read()
|
||||||
|
|
||||||
|
sql = sql_template.format(
|
||||||
|
mid_list=mid_list,
|
||||||
|
run_date=run_date.strftime("%Y-%m-%d")
|
||||||
|
)
|
||||||
|
|
||||||
|
log.info(f"Fetching data for {len(mids):,} MIDs")
|
||||||
|
|
||||||
|
df = pl.read_database(
|
||||||
|
query=sql,
|
||||||
|
connection=engine
|
||||||
|
)
|
||||||
|
|
||||||
|
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||||
|
|
||||||
|
return df
|
||||||
@@ -12,7 +12,7 @@ with employee_detail AS
|
|||||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_Region rm ON st.RegionId = rm.RegionId
|
INNER JOIN OneApp_KelloggsMT.dbo.Master_Region rm ON st.RegionId = rm.RegionId
|
||||||
INNER JOIN OneApp_KelloggsMT.dbo.Master_Designation dm ON Em.DesignationId = dm.DesignationId
|
INNER JOIN OneApp_KelloggsMT.dbo.Master_Designation dm ON Em.DesignationId = dm.DesignationId
|
||||||
WHERE em.RightId IN (6)
|
WHERE em.RightId IN (6)
|
||||||
AND (Em.ResignDate IS NULL OR CONVERT(DATE, Em.ResignDate, 101) >= CAST('{start_date}' AS DATE))
|
AND (Em.ResignDate IS NULL OR CONVERT(DATE, Em.ResignDate, 101) >= CAST('{run_date}' AS DATE))
|
||||||
AND em.EmployeeName NOT LIKE '%test%'
|
AND em.EmployeeName NOT LIKE '%test%'
|
||||||
),
|
),
|
||||||
Login (
|
Login (
|
||||||
@@ -71,7 +71,7 @@ Login (
|
|||||||
) AS [Last Store Out Time]
|
) AS [Last Store Out Time]
|
||||||
FROM OneApp_KelloggsMT.dbo.T_DeviceLogin ud
|
FROM OneApp_KelloggsMT.dbo.T_DeviceLogin ud
|
||||||
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em ON ud.EmpId = Em.EmpId
|
INNER JOIN OneApp_KelloggsMT.dbo.vw_Employee_Detail Em ON ud.EmpId = Em.EmpId
|
||||||
WHERE CAST(ud.LoginDate AS DATE) = CAST('{start_date}' AS DATE) -- fixed: direct date comparison
|
WHERE CAST(ud.LoginDate AS DATE) = CAST('{run_date}' AS DATE) -- fixed: direct date comparison
|
||||||
) AS T1
|
) AS T1
|
||||||
WHERE T1.col = 1
|
WHERE T1.col = 1
|
||||||
) AS T ON Em.[Employee code] = T.[Employee Code]
|
) AS T ON Em.[Employee code] = T.[Employee Code]
|
||||||
|
|||||||
@@ -1,10 +1,11 @@
|
|||||||
with Promotion(Mid,
|
with Promotion (Mid,
|
||||||
project_id,store_id,employee_id,visit_date,supervisor_id,channel_id,chain_id,storetype_id,promo_definition_id,promo_definition_name,
|
project_id,store_id,employee_id,visit_date,supervisor_id,channel_id,chain_id,storetype_id,promo_definition_id,promo_definition_name,
|
||||||
promotion_deatils,promotion_deatils_id,promotion_value_name,
|
promotion_deatils,promotion_deatils_id,promotion_value_name,
|
||||||
present,reason,PromoQuestion,PromoAnswer,image1,image2,update_date,update_by)
|
present,reason,PromoQuestion,PromoAnswer,image1,image2,update_date,update_by)
|
||||||
as (
|
as (
|
||||||
Select
|
|
||||||
sc.MID, '40148' Projectid,sc.StoreId,Em.EmpId,sc.VisitDate,Em.SupervisorId,sm.ChannelId,sm.ChainId,sm.StoreTypeId,msd.PromoDefinitionId,
|
Select
|
||||||
|
sc.MID, '40148' Projectid,sc.StoreId,Em.EmpId,sc.VisitDate,Em.SupervisorId,sm.ChannelId,sm.ChainId,sm.StoreTypeId,msd.PromoDefinitionId,
|
||||||
MSD.PromoDefinitionName, case when isnull(TS.PromoTable,'')='Master_Category' then 'Category'
|
MSD.PromoDefinitionName, case when isnull(TS.PromoTable,'')='Master_Category' then 'Category'
|
||||||
when isnull(TS.PromoTable,'')='Master_SubCategory' then 'SubCategory'
|
when isnull(TS.PromoTable,'')='Master_SubCategory' then 'SubCategory'
|
||||||
when isnull(TS.PromoTable,'')='Master_Brand' then 'Brand'
|
when isnull(TS.PromoTable,'')='Master_Brand' then 'Brand'
|
||||||
@@ -15,14 +16,14 @@ with Promotion(Mid,
|
|||||||
when isnull(TS.PromoTable,'')='Master_SubCategory' THEN (SELECT a.SubCategoryName from OneApp_KelloggsMT.dbo.Master_SubCategory a where ts.PromoValue=a.SubCategoryId )
|
when isnull(TS.PromoTable,'')='Master_SubCategory' THEN (SELECT a.SubCategoryName from OneApp_KelloggsMT.dbo.Master_SubCategory a where ts.PromoValue=a.SubCategoryId )
|
||||||
when isnull(TS.PromoTable,'')='Master_Brand' THEN (SELECT a.BrandName from OneApp_KelloggsMT.dbo.Master_Brand a where ts.PromoValue=a.BrandId )
|
when isnull(TS.PromoTable,'')='Master_Brand' THEN (SELECT a.BrandName from OneApp_KelloggsMT.dbo.Master_Brand a where ts.PromoValue=a.BrandId )
|
||||||
when isnull(TS.PromoTable,'')='Master_SubBrand' THEN (SELECT a.SubBrandName from OneApp_KelloggsMT.dbo.Master_SubBrand a where ts.PromoValue=a.SubBrandId ) end as
|
when isnull(TS.PromoTable,'')='Master_SubBrand' THEN (SELECT a.SubBrandName from OneApp_KelloggsMT.dbo.Master_SubBrand a where ts.PromoValue=a.SubBrandId ) end as
|
||||||
[PromoValueName],case when ts.Present=0 then 'N' else 'Y' end Present,
|
[PromoValueName],case when ts.Present=0 then 'N' else 'Y' end Present,
|
||||||
case when ts.Present=1 then '' else isnull(mnp.PromoReason,'') end as Reason, ISNULL(mpq.PromoQuestionName,'') AS Question,
|
case when ts.Present=1 then '' else isnull(mnp.PromoReason,'') end as Reason, ISNULL(mpq.PromoQuestionName,'') AS Question,
|
||||||
ISNULL(tpq.PromoAnswerName,'') AS Answer ,
|
ISNULL(tpq.PromoAnswerName,'') AS Answer ,
|
||||||
case when isnull(SHI.PromoImage1,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PromotionImages/'+SHI.PromoImage1
|
case when isnull(SHI.PromoImage1,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PromotionImages/'+SHI.PromoImage1
|
||||||
end as Image1 ,case when isnull(SHI.PromoImage2,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PromotionImages/'+SHI.PromoImage2
|
end as Image1 ,case when isnull(SHI.PromoImage2,'')='' then '' else 'https://kimt1.parinaam.in/Upload/PromotionImages/'+SHI.PromoImage2
|
||||||
end as Image2 ,GETDATE(),'SP-Pius'
|
end as Image2 ,GETDATE(),'SP-Pius'
|
||||||
|
|
||||||
from
|
from
|
||||||
OneApp_KelloggsMT.dbo.T_Promotion ts Inner join
|
OneApp_KelloggsMT.dbo.T_Promotion ts Inner join
|
||||||
OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join
|
OneApp_KelloggsMT.dbo.T_StoreCoverage sc on ts.mid= sc.mid Inner Join
|
||||||
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
|
OneApp_KelloggsMT.dbo.vw_StoreDetail sm on sc.StoreId= sm.StoreId Inner Join
|
||||||
@@ -38,6 +39,5 @@ with Promotion(Mid,
|
|||||||
left join OneApp_KelloggsMT.dbo.Master_PromotionQuestion mpq on tpq.PromoQuestionId=mpq.PromoQuestionId
|
left join OneApp_KelloggsMT.dbo.Master_PromotionQuestion mpq on tpq.PromoQuestionId=mpq.PromoQuestionId
|
||||||
|
|
||||||
Where 1=1 and Em.EmpName not like 'test%'
|
Where 1=1 and Em.EmpName not like 'test%'
|
||||||
AND sc.MID in ({mid_list})
|
AND sc.MID in ({mid_list}) )
|
||||||
)
|
|
||||||
select * from Promotion
|
select * from Promotion
|
||||||
@@ -85,20 +85,25 @@ tables:
|
|||||||
# operation: ONLY_INSERT
|
# operation: ONLY_INSERT
|
||||||
# fetch_by: run_date
|
# fetch_by: run_date
|
||||||
|
|
||||||
- name: Master_VisibilityReason
|
# - name: Master_VisibilityReason
|
||||||
type: DIMENSION
|
# type: DIMENSION
|
||||||
operation: DELETE+INSERT
|
# operation: DELETE+INSERT
|
||||||
fetch_by: none
|
# fetch_by: none
|
||||||
|
|
||||||
|
|
||||||
- name: Master_VisibilityDefinition
|
# - name: Master_VisibilityDefinition
|
||||||
type: DIMENSION
|
# type: DIMENSION
|
||||||
operation: DELETE+INSERT
|
# operation: DELETE+INSERT
|
||||||
fetch_by: none
|
# fetch_by: none
|
||||||
|
|
||||||
|
|
||||||
- name: Master_Salesterritorylayer
|
# - name: Master_Salesterritorylayer
|
||||||
type: DIMENSION
|
# type: DIMENSION
|
||||||
operation: DELETE+INSERT
|
# operation: DELETE+INSERT
|
||||||
fetch_by: none
|
# fetch_by: none
|
||||||
|
|
||||||
|
- name: Promotion
|
||||||
|
type: FACT
|
||||||
|
operation: INSERT
|
||||||
|
fetch_by: mids
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user