From 2ead99553f3e92539f444eefbd880ae3076dbe01 Mon Sep 17 00:00:00 2001 From: Ankit Malik Date: Mon, 22 Jun 2026 12:05:25 +0530 Subject: [PATCH] 22-06-2026 1st commit --- logs/etl_20260619.log | 156 ++++++++++ logs/etl_20260622.log | 307 ++++++++++++++++++++ main2.py | 4 +- src/bridge.py | 5 +- src/sql/dim/Master_Salesterritorylayer.sql | 14 + src/sql/dim/Master_VisibilityDefinition.sql | 5 + src/sql/dim/Master_VisibilityReason.sql | 7 + y.yml | 26 +- 8 files changed, 519 insertions(+), 5 deletions(-) create mode 100644 logs/etl_20260622.log diff --git a/logs/etl_20260619.log b/logs/etl_20260619.log index 49f298f..3fee27f 100644 --- a/logs/etl_20260619.log +++ b/logs/etl_20260619.log @@ -1293,3 +1293,159 @@ KeyError: 'fetch_Store_Master' 2026-06-19 16:00:39 | INFO | ================================================================================ 2026-06-19 16:00:39 | INFO | Pipeline Completed Successfully 2026-06-19 16:00:39 | INFO | ================================================================================ +2026-06-19 18:19:29 | INFO | ================================================================================ +2026-06-19 18:19:29 | INFO | Hello from data-move Python data pipeline! +2026-06-19 18:19:29 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 18:19:29 | INFO | Connecting to databases... +2026-06-19 18:19:30 | INFO | +2026-06-19 18:19:31 | INFO | +2026-06-19 18:19:31 | INFO | Database connections established +2026-06-19 18:19:31 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 18:19:31 | INFO | Found 859 MIDs +2026-06-19 18:19:32 | INFO | ================================================================================ +2026-06-19 18:19:32 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT +2026-06-19 18:19:32 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-19 18:19:32 | ERROR | Failed processing table mapping_storevisibility +Traceback (most recent call last): + File "D:\data_move\main2.py", line 159, in main + df=fn(sql_engine, table_name , table_type, mids, run_date) +TypeError: fetch_mapping_storevisibility() takes 2 positional arguments but 5 were given +2026-06-19 18:20:42 | INFO | ================================================================================ +2026-06-19 18:20:42 | INFO | Hello from data-move Python data pipeline! +2026-06-19 18:20:42 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 18:20:42 | INFO | Connecting to databases... +2026-06-19 18:20:42 | INFO | +2026-06-19 18:20:43 | INFO | +2026-06-19 18:20:44 | INFO | Database connections established +2026-06-19 18:20:44 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 18:20:44 | INFO | Found 859 MIDs +2026-06-19 18:20:44 | INFO | ================================================================================ +2026-06-19 18:20:44 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT +2026-06-19 18:20:44 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-19 18:20:44 | ERROR | Failed processing table mapping_storevisibility +Traceback (most recent call last): + File "D:\data_move\main2.py", line 159, in main + df=fn(sql_engine, table_name , table_type, mids, run_date) +TypeError: fetch_mapping_storevisibility() takes 2 positional arguments but 5 were given +2026-06-19 18:25:10 | INFO | ================================================================================ +2026-06-19 18:25:10 | INFO | Hello from data-move Python data pipeline! +2026-06-19 18:25:10 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 18:25:10 | INFO | Connecting to databases... +2026-06-19 18:25:11 | INFO | +2026-06-19 18:25:12 | INFO | +2026-06-19 18:25:13 | INFO | Database connections established +2026-06-19 18:25:13 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 18:25:13 | INFO | Found 859 MIDs +2026-06-19 18:25:13 | INFO | ================================================================================ +2026-06-19 18:25:13 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT +2026-06-19 18:25:13 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-19 18:25:13 | ERROR | Failed processing table mapping_storevisibility +Traceback (most recent call last): + File "D:\data_move\main2.py", line 159, in main + df=fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\data_move\src\bridge.py", line 100, in fetch_mapping_storevisibility + store_id=get_reason_ids_mapping_storevisibility(client, run_date, "coverage_remarks") + File "D:\data_move\src\bridge.py", line 61, in get_reason_ids_mapping_storevisibility + arrow_table = client.query_arrow(query) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 736, in query_arrow + self.raw_query( + ~~~~~~~~~~~~~~^ + query, + ^^^^^^ + ...<4 lines>... + transport_settings=transport_settings, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 661, in raw_query + return self._raw_request(body, params, fields=fields, headers=transport_settings, retries=self.query_retries).data + ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 60, server response: Code: 60. DB::Exception: Unknown table expression identifier 'mapping_storevisibility' in scope SELECT DISTINCT StoreId FROM mapping_storevisibility WHERE (toDate(Fromdate) <= toDate('2026-06-20')) AND (toDate(Todate) >= toDate('2026-06-20')) AND (project_Id = '40148'). (UNKNOWN_TABLE) (for url http://172.188.12.194:8123) +2026-06-19 18:25:58 | INFO | ================================================================================ +2026-06-19 18:25:58 | INFO | Hello from data-move Python data pipeline! +2026-06-19 18:25:58 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 18:25:58 | INFO | Connecting to databases... +2026-06-19 18:25:59 | INFO | +2026-06-19 18:26:00 | INFO | +2026-06-19 18:26:01 | INFO | Database connections established +2026-06-19 18:26:01 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 18:26:01 | INFO | Found 859 MIDs +2026-06-19 18:26:01 | INFO | ================================================================================ +2026-06-19 18:26:01 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT +2026-06-19 18:26:01 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-19 18:26:01 | ERROR | Failed processing table mapping_storevisibility +Traceback (most recent call last): + File "D:\data_move\main2.py", line 159, in main + df=fn(sql_engine, table_name , table_type, mids, run_date) + File "D:\data_move\src\bridge.py", line 100, in fetch_mapping_storevisibility + store_id=get_reason_ids_mapping_storevisibility(client, run_date, "coverage_remarks") + File "D:\data_move\src\bridge.py", line 61, in get_reason_ids_mapping_storevisibility + arrow_table = client.query_arrow(query) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 736, in query_arrow + self.raw_query( + ~~~~~~~~~~~~~~^ + query, + ^^^^^^ + ...<4 lines>... + transport_settings=transport_settings, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 661, in raw_query + return self._raw_request(body, params, fields=fields, headers=transport_settings, retries=self.query_retries).data + ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request + self._error_handler(response) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler + raise err_type(err_str, code=code, name=name) from None +clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 60, server response: Code: 60. DB::Exception: Unknown table expression identifier 'mapping_storevisibility' in scope SELECT DISTINCT StoreId FROM mapping_storevisibility WHERE (toDate(Fromdate) <= toDate('2026-06-20')) AND (toDate(Todate) >= toDate('2026-06-20')) AND (project_Id = '40148'). (UNKNOWN_TABLE) (for url http://172.188.12.194:8123) +2026-06-19 18:27:24 | INFO | ================================================================================ +2026-06-19 18:27:24 | INFO | Hello from data-move Python data pipeline! +2026-06-19 18:27:24 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 18:27:24 | INFO | Connecting to databases... +2026-06-19 18:27:25 | INFO | +2026-06-19 18:27:26 | INFO | +2026-06-19 18:27:26 | INFO | Database connections established +2026-06-19 18:27:26 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 18:27:27 | INFO | Found 859 MIDs +2026-06-19 18:27:27 | INFO | ================================================================================ +2026-06-19 18:27:27 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT +2026-06-19 18:27:27 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-19 18:27:27 | WARNING | Table 'mapping_storevisibility' does not exist. During collecting store_ids +2026-06-19 18:27:27 | INFO | Fetching data from sql server for Master table...... +2026-06-19 18:27:27 | INFO | Fetching in progress .... +2026-06-19 18:27:27 | INFO | Fetched 0 rows from SQL Server +2026-06-19 18:27:27 | INFO | Fetched 0 rows from SQL Server +2026-06-19 18:27:27 | INFO | Fetched total row -: 0 from sql server for table-:mapping_storevisibility ...........!!! +2026-06-19 18:27:27 | WARNING | mapping_storevisibility returned no rows +2026-06-19 18:27:27 | INFO | ================================================================================ +2026-06-19 18:27:27 | INFO | Pipeline Completed Successfully +2026-06-19 18:27:27 | INFO | ================================================================================ +2026-06-19 18:27:32 | INFO | ================================================================================ +2026-06-19 18:27:32 | INFO | Hello from data-move Python data pipeline! +2026-06-19 18:27:32 | INFO | Pipeline Run Date: 2026-06-18 +2026-06-19 18:27:32 | INFO | Connecting to databases... +2026-06-19 18:27:33 | INFO | +2026-06-19 18:27:34 | INFO | +2026-06-19 18:27:35 | INFO | Database connections established +2026-06-19 18:27:35 | INFO | Collecting MIDs for: 2026-06-18 +2026-06-19 18:27:35 | INFO | Found 859 MIDs +2026-06-19 18:27:35 | INFO | ================================================================================ +2026-06-19 18:27:35 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT +2026-06-19 18:27:35 | INFO | Fetching Data from sql server for table-: mapping_storevisibility .............. +2026-06-19 18:27:35 | WARNING | Table 'mapping_storevisibility' does not exist. During collecting store_ids +2026-06-19 18:27:35 | INFO | Fetching data from sql server for Master table...... +2026-06-19 18:27:35 | INFO | Fetching in progress .... +2026-06-19 18:27:35 | INFO | Fetched 0 rows from SQL Server +2026-06-19 18:27:35 | INFO | Fetched 0 rows from SQL Server +2026-06-19 18:27:35 | INFO | Fetched total row -: 0 from sql server for table-:mapping_storevisibility ...........!!! +2026-06-19 18:27:35 | WARNING | mapping_storevisibility returned no rows +2026-06-19 18:27:35 | INFO | ================================================================================ +2026-06-19 18:27:35 | INFO | Pipeline Completed Successfully +2026-06-19 18:27:35 | INFO | ================================================================================ diff --git a/logs/etl_20260622.log b/logs/etl_20260622.log new file mode 100644 index 0000000..169b564 --- /dev/null +++ b/logs/etl_20260622.log @@ -0,0 +1,307 @@ +2026-06-22 10:05:56 | INFO | ================================================================================ +2026-06-22 10:05:56 | INFO | Hello from data-move Python data pipeline! +2026-06-22 10:05:56 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 10:05:56 | INFO | Connecting to databases... +2026-06-22 10:05:57 | INFO | +2026-06-22 10:06:00 | INFO | +2026-06-22 10:06:03 | INFO | Database connections established +2026-06-22 10:06:03 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 10:06:05 | INFO | Found 569 MIDs +2026-06-22 10:06:07 | INFO | ================================================================================ +2026-06-22 10:06:07 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 10:06:07 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-22 10:06:07 | INFO | Fetching data from sql server for Master table...... +2026-06-22 10:06:07 | INFO | Fetching in progress .... +2026-06-22 10:06:31 | ERROR | Failed processing table Master_VisibilityReason +Traceback (most recent call last): + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +pyodbc.OperationalError: ('08S01', '[08S01] [Microsoft][ODBC Driver 18 for SQL Server]TCP Provider: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond.\r\n (10060) (SQLExecDirectW); [08S01] [Microsoft][ODBC Driver 18 for SQL Server]Communication link failure (10060)') + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "D:\data_move\main2.py", line 161, in main + df = fetch_data(sql_engine ,table_name,table_type) + File "D:\data_move\src\dim.py", line 45, in fetch_data + df = pl.read_database( + query=sql, + connection=engine + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database + return cx.execute( + ~~~~~~~~~~^ + query=query, + ^^^^^^^^^^^^ + options=execute_options, + ^^^^^^^^^^^^^^^^^^^^^^^^ + ).to_polars( + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute + result = cursor_execute(query, **options) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute + return meth( + self, + distilled_parameters, + execution_options or NO_OPTIONS, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection + return connection._execute_clauseelement( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + self, distilled_params, execution_options + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement + ret = self._execute_context( + dialect, + ...<8 lines>... + cache_hit=cache_hit, + ) + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context + return self._exec_single_context( + ~~~~~~~~~~~~~~~~~~~~~~~~~^ + dialect, context, statement, parameters + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context + self._handle_dbapi_exception( + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ + e, str_statement, effective_parameters, cursor, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context + self.dialect.do_execute( + ~~~~~~~~~~~~~~~~~~~~~~~^ + cursor, str_statement, effective_parameters, context + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +sqlalchemy.exc.OperationalError: (pyodbc.OperationalError) ('08S01', '[08S01] [Microsoft][ODBC Driver 18 for SQL Server]TCP Provider: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond.\r\n (10060) (SQLExecDirectW); [08S01] [Microsoft][ODBC Driver 18 for SQL Server]Communication link failure (10060)') +[SQL: with Master_VisibilityReason (ProjectId,MenuId,ReasonId,Reason,CreateDate,Createby) + as ( + +select DISTINCT '40148' as ProjectId,MenuId,VisibilityReasonId,VisibilityReason,getdate(),'SP-Pius' +FROM OneApp_KelloggsMT.dbo.Master_VisibilityReason + ) +select * from Master_VisibilityReason] +(Background on this error at: https://sqlalche.me/e/20/e3q8) +2026-06-22 10:10:52 | INFO | ================================================================================ +2026-06-22 10:10:52 | INFO | Hello from data-move Python data pipeline! +2026-06-22 10:10:52 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 10:10:53 | INFO | Connecting to databases... +2026-06-22 10:10:53 | INFO | +2026-06-22 10:10:55 | INFO | +2026-06-22 10:10:56 | INFO | Database connections established +2026-06-22 10:10:56 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 10:10:56 | INFO | Found 569 MIDs +2026-06-22 10:10:57 | INFO | ================================================================================ +2026-06-22 10:10:57 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 10:10:57 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-22 10:10:57 | INFO | Fetching data from sql server for Master table...... +2026-06-22 10:10:57 | INFO | Fetching in progress .... +2026-06-22 10:10:57 | INFO | Fetched 17 rows from SQL Server +2026-06-22 10:10:57 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-22 10:10:57 | INFO | Fetched 17 rows +2026-06-22 10:10:57 | INFO | Creating table Master_VisibilityReason +2026-06-22 10:10:57 | INFO | Table ready: Master_VisibilityReason +2026-06-22 10:10:57 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 10:10:58 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-22 10:10:58 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-22 10:10:58 | INFO | ================================================================================ +2026-06-22 10:10:58 | INFO | Pipeline Completed Successfully +2026-06-22 10:10:58 | INFO | ================================================================================ +2026-06-22 10:11:04 | INFO | ================================================================================ +2026-06-22 10:11:04 | INFO | Hello from data-move Python data pipeline! +2026-06-22 10:11:04 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 10:11:04 | INFO | Connecting to databases... +2026-06-22 10:11:05 | INFO | +2026-06-22 10:11:06 | INFO | +2026-06-22 10:11:07 | INFO | Database connections established +2026-06-22 10:11:07 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 10:11:07 | INFO | Found 569 MIDs +2026-06-22 10:11:08 | INFO | ================================================================================ +2026-06-22 10:11:08 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 10:11:08 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-22 10:11:08 | INFO | Fetching data from sql server for Master table...... +2026-06-22 10:11:08 | INFO | Fetching in progress .... +2026-06-22 10:11:08 | INFO | Fetched 17 rows from SQL Server +2026-06-22 10:11:08 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-22 10:11:08 | INFO | Fetched 17 rows +2026-06-22 10:11:08 | INFO | Truncated table Master_VisibilityReason +2026-06-22 10:11:08 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 10:11:08 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-22 10:11:08 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-22 10:11:08 | INFO | ================================================================================ +2026-06-22 10:11:08 | INFO | Pipeline Completed Successfully +2026-06-22 10:11:08 | INFO | ================================================================================ +2026-06-22 11:07:54 | INFO | ================================================================================ +2026-06-22 11:07:54 | INFO | Hello from data-move Python data pipeline! +2026-06-22 11:07:54 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 11:07:54 | INFO | Connecting to databases... +2026-06-22 11:07:56 | INFO | +2026-06-22 11:08:00 | INFO | +2026-06-22 11:08:01 | INFO | Database connections established +2026-06-22 11:08:01 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 11:08:02 | INFO | Found 569 MIDs +2026-06-22 11:08:42 | INFO | ================================================================================ +2026-06-22 11:08:42 | INFO | Hello from data-move Python data pipeline! +2026-06-22 11:08:42 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 11:08:42 | INFO | Connecting to databases... +2026-06-22 11:08:44 | INFO | +2026-06-22 11:08:46 | INFO | +2026-06-22 11:08:47 | INFO | Database connections established +2026-06-22 11:08:47 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 11:08:48 | INFO | Found 569 MIDs +2026-06-22 11:08:48 | INFO | ================================================================================ +2026-06-22 11:08:48 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 11:08:48 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-22 11:08:48 | INFO | Fetching data from sql server for Master table...... +2026-06-22 11:08:48 | INFO | Fetching in progress .... +2026-06-22 11:08:48 | INFO | Fetched 17 rows from SQL Server +2026-06-22 11:08:48 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-22 11:08:48 | INFO | Fetched 17 rows +2026-06-22 11:08:48 | INFO | Truncated table Master_VisibilityReason +2026-06-22 11:08:48 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 11:08:49 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-22 11:08:49 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-22 11:08:49 | INFO | ================================================================================ +2026-06-22 11:08:49 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 11:08:49 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. +2026-06-22 11:08:49 | INFO | Fetching data from sql server for Master table...... +2026-06-22 11:08:49 | INFO | Fetching in progress .... +2026-06-22 11:08:49 | INFO | Fetched 861 rows from SQL Server +2026-06-22 11:08:49 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! +2026-06-22 11:08:49 | INFO | Fetched 861 rows +2026-06-22 11:08:49 | INFO | Creating table Master_VisibilityDefinition +2026-06-22 11:08:49 | INFO | Table ready: Master_VisibilityDefinition +2026-06-22 11:08:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 11:08:49 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-22 11:08:49 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-22 11:08:49 | INFO | ================================================================================ +2026-06-22 11:08:49 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 11:08:49 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. +2026-06-22 11:08:49 | INFO | Fetching data from sql server for Master table...... +2026-06-22 11:08:49 | INFO | Fetching in progress .... +2026-06-22 11:08:49 | INFO | Fetched 33 rows from SQL Server +2026-06-22 11:08:49 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! +2026-06-22 11:08:49 | INFO | Fetched 33 rows +2026-06-22 11:08:50 | INFO | Creating table Master_Salesterritorylayer +2026-06-22 11:08:50 | INFO | Table ready: Master_Salesterritorylayer +2026-06-22 11:08:50 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 11:08:50 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-22 11:08:50 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-22 11:08:50 | INFO | ================================================================================ +2026-06-22 11:08:50 | INFO | Pipeline Completed Successfully +2026-06-22 11:08:50 | INFO | ================================================================================ +2026-06-22 11:08:56 | INFO | ================================================================================ +2026-06-22 11:08:56 | INFO | Hello from data-move Python data pipeline! +2026-06-22 11:08:56 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 11:08:56 | INFO | Connecting to databases... +2026-06-22 11:08:57 | INFO | +2026-06-22 11:08:59 | INFO | +2026-06-22 11:09:00 | INFO | Database connections established +2026-06-22 11:09:00 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 11:09:00 | INFO | Found 569 MIDs +2026-06-22 11:09:00 | INFO | ================================================================================ +2026-06-22 11:09:00 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 11:09:00 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-22 11:09:00 | INFO | Fetching data from sql server for Master table...... +2026-06-22 11:09:00 | INFO | Fetching in progress .... +2026-06-22 11:09:01 | INFO | Fetched 17 rows from SQL Server +2026-06-22 11:09:01 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-22 11:09:01 | INFO | Fetched 17 rows +2026-06-22 11:09:01 | INFO | Truncated table Master_VisibilityReason +2026-06-22 11:09:01 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 11:09:01 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-22 11:09:01 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-22 11:09:01 | INFO | ================================================================================ +2026-06-22 11:09:01 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 11:09:01 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. +2026-06-22 11:09:01 | INFO | Fetching data from sql server for Master table...... +2026-06-22 11:09:01 | INFO | Fetching in progress .... +2026-06-22 11:09:02 | INFO | Fetched 861 rows from SQL Server +2026-06-22 11:09:02 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! +2026-06-22 11:09:02 | INFO | Fetched 861 rows +2026-06-22 11:09:02 | INFO | Truncated table Master_VisibilityDefinition +2026-06-22 11:09:02 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 11:09:02 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-22 11:09:02 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-22 11:09:02 | INFO | ================================================================================ +2026-06-22 11:09:02 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 11:09:02 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. +2026-06-22 11:09:02 | INFO | Fetching data from sql server for Master table...... +2026-06-22 11:09:02 | INFO | Fetching in progress .... +2026-06-22 11:09:02 | INFO | Fetched 33 rows from SQL Server +2026-06-22 11:09:02 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! +2026-06-22 11:09:02 | INFO | Fetched 33 rows +2026-06-22 11:09:02 | INFO | Truncated table Master_Salesterritorylayer +2026-06-22 11:09:02 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 11:09:03 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-22 11:09:03 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-22 11:09:03 | INFO | ================================================================================ +2026-06-22 11:09:03 | INFO | Pipeline Completed Successfully +2026-06-22 11:09:03 | INFO | ================================================================================ +2026-06-22 12:04:40 | INFO | ================================================================================ +2026-06-22 12:04:40 | INFO | Hello from data-move Python data pipeline! +2026-06-22 12:04:40 | INFO | Pipeline Run Date: 2026-06-21 +2026-06-22 12:04:40 | INFO | Connecting to databases... +2026-06-22 12:04:43 | INFO | +2026-06-22 12:04:44 | INFO | +2026-06-22 12:04:45 | INFO | Database connections established +2026-06-22 12:04:45 | INFO | Collecting MIDs for: 2026-06-21 +2026-06-22 12:04:46 | INFO | Found 569 MIDs +2026-06-22 12:04:48 | INFO | ================================================================================ +2026-06-22 12:04:48 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 12:04:48 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason .............. +2026-06-22 12:04:48 | INFO | Fetching data from sql server for Master table...... +2026-06-22 12:04:48 | INFO | Fetching in progress .... +2026-06-22 12:04:48 | INFO | Fetched 17 rows from SQL Server +2026-06-22 12:04:48 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!! +2026-06-22 12:04:48 | INFO | Fetched 17 rows +2026-06-22 12:04:48 | INFO | Truncated table Master_VisibilityReason +2026-06-22 12:04:48 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:04:48 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse +2026-06-22 12:04:48 | INFO | Master_VisibilityReason loaded successfully (17 rows) +2026-06-22 12:04:48 | INFO | ================================================================================ +2026-06-22 12:04:48 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 12:04:48 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition .............. +2026-06-22 12:04:48 | INFO | Fetching data from sql server for Master table...... +2026-06-22 12:04:48 | INFO | Fetching in progress .... +2026-06-22 12:04:49 | INFO | Fetched 861 rows from SQL Server +2026-06-22 12:04:49 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!! +2026-06-22 12:04:49 | INFO | Fetched 861 rows +2026-06-22 12:04:49 | INFO | Truncated table Master_VisibilityDefinition +2026-06-22 12:04:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:04:49 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse +2026-06-22 12:04:49 | INFO | Master_VisibilityDefinition loaded successfully (861 rows) +2026-06-22 12:04:49 | INFO | ================================================================================ +2026-06-22 12:04:49 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT +2026-06-22 12:04:49 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer .............. +2026-06-22 12:04:49 | INFO | Fetching data from sql server for Master table...... +2026-06-22 12:04:49 | INFO | Fetching in progress .... +2026-06-22 12:04:49 | INFO | Fetched 33 rows from SQL Server +2026-06-22 12:04:49 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!! +2026-06-22 12:04:49 | INFO | Fetched 33 rows +2026-06-22 12:04:49 | INFO | Truncated table Master_Salesterritorylayer +2026-06-22 12:04:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _ +2026-06-22 12:04:50 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse +2026-06-22 12:04:50 | INFO | Master_Salesterritorylayer loaded successfully (33 rows) +2026-06-22 12:04:50 | INFO | ================================================================================ +2026-06-22 12:04:50 | INFO | Pipeline Completed Successfully +2026-06-22 12:04:50 | INFO | ================================================================================ diff --git a/main2.py b/main2.py index 1571d79..54ad901 100644 --- a/main2.py +++ b/main2.py @@ -43,7 +43,7 @@ from mids import ( MID_TABLE_COV1, ) -from masters.bridge import * +from src.bridge import * from src.fact import * from src.dim import * @@ -211,6 +211,8 @@ def main(): table_name, ) + elif operation =="ONLY_INSERT" : + continue else: delete_existing_data( diff --git a/src/bridge.py b/src/bridge.py index eabd8af..8edb255 100644 --- a/src/bridge.py +++ b/src/bridge.py @@ -75,7 +75,7 @@ def fetch_mapping_storevisibility( store_id_list = ",".join(str(sid) for sid in store_id) - sql_file = Path("src") / "sql" / f"dim" / f"{table_name}.sql" + sql_file = Path("src") / "sql" / f"bridge" / f"{table_name}.sql" with open(sql_file, "r", encoding="utf-8") as f: sql_template = f.read() @@ -97,10 +97,11 @@ def fetch_mapping_storevisibility( return df - store_id=get_reason_ids_mapping_storevisibility(client, run_date, "coverage_remarks") + store_id=get_reason_ids_mapping_storevisibility(client, run_date, "mapping_storevisibility") df=fetch_data(engine=sql_engine, table_name=table_name, table_type=table_type, + run_date=run_date, store_id=store_id, ) log.info(f"Fetched {len(df):,} rows from SQL Server") diff --git a/src/sql/dim/Master_Salesterritorylayer.sql b/src/sql/dim/Master_Salesterritorylayer.sql index e69de29..853c238 100644 --- a/src/sql/dim/Master_Salesterritorylayer.sql +++ b/src/sql/dim/Master_Salesterritorylayer.sql @@ -0,0 +1,14 @@ +with Master_Salesterritorylayer(ProjectId,StLayerOneId,StLayerOneName,StLayerTwoId,StLayerTwoName,StLayerThreeId,StLayerThreeName,StLayerFourId, +StLayerFourName,CreateDate,CreateBy) +as ( +select DISTINCT '40148' as Project_Id,d.StLayerOneId,d.StLayerOneName,c.StLayerTwoId,c.StLayerTwoName , +b.StLayerThreeId,b.StLayerThreeName,a.StLayerFourId,a.StLayerFourName , +getdate(),'SP-Pius' +FROM OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerFour] A +INNER JOIN OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerThree]B ON + A.StLayerThreeId=B.StLayerThreeId + INNER JOIN OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerTWo]C ON + B.StLayerTwoId=C.StLayerTwoId + INNER JOIN OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerOne]D ON + C.StLayerOneId=D.StLayerOneId) + select * from Master_Salesterritorylayer \ No newline at end of file diff --git a/src/sql/dim/Master_VisibilityDefinition.sql b/src/sql/dim/Master_VisibilityDefinition.sql index e69de29..78d7731 100644 --- a/src/sql/dim/Master_VisibilityDefinition.sql +++ b/src/sql/dim/Master_VisibilityDefinition.sql @@ -0,0 +1,5 @@ +with Master_VisibilityDefinition(Project_Id,VisibilityDefinitionId,VisibilityDefinitionName,CreateDate,CreateBy) +as ( +select DISTINCT '40148' as Project_Id,VisibilityDefinitionid,VisibilityDefinitionname,getdate(),'SP-Pius' +FROM OneApp_KelloggsMT.dbo.Master_VisibilityDefinition) +select * from Master_VisibilityDefinition \ No newline at end of file diff --git a/src/sql/dim/Master_VisibilityReason.sql b/src/sql/dim/Master_VisibilityReason.sql index e69de29..02f6453 100644 --- a/src/sql/dim/Master_VisibilityReason.sql +++ b/src/sql/dim/Master_VisibilityReason.sql @@ -0,0 +1,7 @@ +with Master_VisibilityReason (ProjectId,MenuId,ReasonId,Reason,CreateDate,Createby) + as ( + +select DISTINCT '40148' as ProjectId,MenuId,VisibilityReasonId,VisibilityReason,getdate(),'SP-Pius' +FROM OneApp_KelloggsMT.dbo.Master_VisibilityReason + ) +select * from Master_VisibilityReason \ No newline at end of file diff --git a/y.yml b/y.yml index f8e0634..e3cf9f8 100644 --- a/y.yml +++ b/y.yml @@ -75,8 +75,30 @@ tables: # operation: DELETE+INSERT # fetch_by: master - - name: Employee_Master + # - name: Employee_Master + # type: DIMENSION + # operation: DELETE+INSERT + # fetch_by: master + + # - name: mapping_storevisibility + # type: BRIDGE + # operation: ONLY_INSERT + # fetch_by: run_date + + - name: Master_VisibilityReason type: DIMENSION operation: DELETE+INSERT - fetch_by: master + fetch_by: none + + + - name: Master_VisibilityDefinition + type: DIMENSION + operation: DELETE+INSERT + fetch_by: none + + + - name: Master_Salesterritorylayer + type: DIMENSION + operation: DELETE+INSERT + fetch_by: none