22-06-2026 1st commit
This commit is contained in:
Ankit Malik
@@ -1293,3 +1293,159 @@ KeyError: 'fetch_Store_Master'
|
|||||||
2026-06-19 16:00:39 | INFO | ================================================================================
|
2026-06-19 16:00:39 | INFO | ================================================================================
|
||||||
2026-06-19 16:00:39 | INFO | Pipeline Completed Successfully
|
2026-06-19 16:00:39 | INFO | Pipeline Completed Successfully
|
||||||
2026-06-19 16:00:39 | INFO | ================================================================================
|
2026-06-19 16:00:39 | INFO | ================================================================================
|
||||||
|
2026-06-19 18:19:29 | INFO | ================================================================================
|
||||||
|
2026-06-19 18:19:29 | INFO | Hello from data-move Python data pipeline!
|
||||||
|
2026-06-19 18:19:29 | INFO | Pipeline Run Date: 2026-06-18
|
||||||
|
2026-06-19 18:19:29 | INFO | Connecting to databases...
|
||||||
|
2026-06-19 18:19:30 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000016658E305D0>
|
||||||
|
2026-06-19 18:19:31 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000016659E587D0>
|
||||||
|
2026-06-19 18:19:31 | INFO | Database connections established
|
||||||
|
2026-06-19 18:19:31 | INFO | Collecting MIDs for: 2026-06-18
|
||||||
|
2026-06-19 18:19:31 | INFO | Found 859 MIDs
|
||||||
|
2026-06-19 18:19:32 | INFO | ================================================================================
|
||||||
|
2026-06-19 18:19:32 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT
|
||||||
|
2026-06-19 18:19:32 | INFO | Fetching Data from sql server for table-: mapping_storevisibility ..............
|
||||||
|
2026-06-19 18:19:32 | ERROR | Failed processing table mapping_storevisibility
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "D:\data_move\main2.py", line 159, in main
|
||||||
|
df=fn(sql_engine, table_name , table_type, mids, run_date)
|
||||||
|
TypeError: fetch_mapping_storevisibility() takes 2 positional arguments but 5 were given
|
||||||
|
2026-06-19 18:20:42 | INFO | ================================================================================
|
||||||
|
2026-06-19 18:20:42 | INFO | Hello from data-move Python data pipeline!
|
||||||
|
2026-06-19 18:20:42 | INFO | Pipeline Run Date: 2026-06-18
|
||||||
|
2026-06-19 18:20:42 | INFO | Connecting to databases...
|
||||||
|
2026-06-19 18:20:42 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000028FE2FD05D0>
|
||||||
|
2026-06-19 18:20:43 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000028FE40787D0>
|
||||||
|
2026-06-19 18:20:44 | INFO | Database connections established
|
||||||
|
2026-06-19 18:20:44 | INFO | Collecting MIDs for: 2026-06-18
|
||||||
|
2026-06-19 18:20:44 | INFO | Found 859 MIDs
|
||||||
|
2026-06-19 18:20:44 | INFO | ================================================================================
|
||||||
|
2026-06-19 18:20:44 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT
|
||||||
|
2026-06-19 18:20:44 | INFO | Fetching Data from sql server for table-: mapping_storevisibility ..............
|
||||||
|
2026-06-19 18:20:44 | ERROR | Failed processing table mapping_storevisibility
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "D:\data_move\main2.py", line 159, in main
|
||||||
|
df=fn(sql_engine, table_name , table_type, mids, run_date)
|
||||||
|
TypeError: fetch_mapping_storevisibility() takes 2 positional arguments but 5 were given
|
||||||
|
2026-06-19 18:25:10 | INFO | ================================================================================
|
||||||
|
2026-06-19 18:25:10 | INFO | Hello from data-move Python data pipeline!
|
||||||
|
2026-06-19 18:25:10 | INFO | Pipeline Run Date: 2026-06-18
|
||||||
|
2026-06-19 18:25:10 | INFO | Connecting to databases...
|
||||||
|
2026-06-19 18:25:11 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000270F1B7C890>
|
||||||
|
2026-06-19 18:25:12 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000270F2C3C7D0>
|
||||||
|
2026-06-19 18:25:13 | INFO | Database connections established
|
||||||
|
2026-06-19 18:25:13 | INFO | Collecting MIDs for: 2026-06-18
|
||||||
|
2026-06-19 18:25:13 | INFO | Found 859 MIDs
|
||||||
|
2026-06-19 18:25:13 | INFO | ================================================================================
|
||||||
|
2026-06-19 18:25:13 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT
|
||||||
|
2026-06-19 18:25:13 | INFO | Fetching Data from sql server for table-: mapping_storevisibility ..............
|
||||||
|
2026-06-19 18:25:13 | ERROR | Failed processing table mapping_storevisibility
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "D:\data_move\main2.py", line 159, in main
|
||||||
|
df=fn(sql_engine, table_name , table_type, mids, run_date)
|
||||||
|
File "D:\data_move\src\bridge.py", line 100, in fetch_mapping_storevisibility
|
||||||
|
store_id=get_reason_ids_mapping_storevisibility(client, run_date, "coverage_remarks")
|
||||||
|
File "D:\data_move\src\bridge.py", line 61, in get_reason_ids_mapping_storevisibility
|
||||||
|
arrow_table = client.query_arrow(query)
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 736, in query_arrow
|
||||||
|
self.raw_query(
|
||||||
|
~~~~~~~~~~~~~~^
|
||||||
|
query,
|
||||||
|
^^^^^^
|
||||||
|
...<4 lines>...
|
||||||
|
transport_settings=transport_settings,
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
)
|
||||||
|
^
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 661, in raw_query
|
||||||
|
return self._raw_request(body, params, fields=fields, headers=transport_settings, retries=self.query_retries).data
|
||||||
|
~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request
|
||||||
|
self._error_handler(response)
|
||||||
|
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler
|
||||||
|
raise err_type(err_str, code=code, name=name) from None
|
||||||
|
clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 60, server response: Code: 60. DB::Exception: Unknown table expression identifier 'mapping_storevisibility' in scope SELECT DISTINCT StoreId FROM mapping_storevisibility WHERE (toDate(Fromdate) <= toDate('2026-06-20')) AND (toDate(Todate) >= toDate('2026-06-20')) AND (project_Id = '40148'). (UNKNOWN_TABLE) (for url http://172.188.12.194:8123)
|
||||||
|
2026-06-19 18:25:58 | INFO | ================================================================================
|
||||||
|
2026-06-19 18:25:58 | INFO | Hello from data-move Python data pipeline!
|
||||||
|
2026-06-19 18:25:58 | INFO | Pipeline Run Date: 2026-06-18
|
||||||
|
2026-06-19 18:25:58 | INFO | Connecting to databases...
|
||||||
|
2026-06-19 18:25:59 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000029019974890>
|
||||||
|
2026-06-19 18:26:00 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002901AA7C7D0>
|
||||||
|
2026-06-19 18:26:01 | INFO | Database connections established
|
||||||
|
2026-06-19 18:26:01 | INFO | Collecting MIDs for: 2026-06-18
|
||||||
|
2026-06-19 18:26:01 | INFO | Found 859 MIDs
|
||||||
|
2026-06-19 18:26:01 | INFO | ================================================================================
|
||||||
|
2026-06-19 18:26:01 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT
|
||||||
|
2026-06-19 18:26:01 | INFO | Fetching Data from sql server for table-: mapping_storevisibility ..............
|
||||||
|
2026-06-19 18:26:01 | ERROR | Failed processing table mapping_storevisibility
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "D:\data_move\main2.py", line 159, in main
|
||||||
|
df=fn(sql_engine, table_name , table_type, mids, run_date)
|
||||||
|
File "D:\data_move\src\bridge.py", line 100, in fetch_mapping_storevisibility
|
||||||
|
store_id=get_reason_ids_mapping_storevisibility(client, run_date, "coverage_remarks")
|
||||||
|
File "D:\data_move\src\bridge.py", line 61, in get_reason_ids_mapping_storevisibility
|
||||||
|
arrow_table = client.query_arrow(query)
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\client.py", line 736, in query_arrow
|
||||||
|
self.raw_query(
|
||||||
|
~~~~~~~~~~~~~~^
|
||||||
|
query,
|
||||||
|
^^^^^^
|
||||||
|
...<4 lines>...
|
||||||
|
transport_settings=transport_settings,
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
)
|
||||||
|
^
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 661, in raw_query
|
||||||
|
return self._raw_request(body, params, fields=fields, headers=transport_settings, retries=self.query_retries).data
|
||||||
|
~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 645, in _raw_request
|
||||||
|
self._error_handler(response)
|
||||||
|
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 537, in _error_handler
|
||||||
|
raise err_type(err_str, code=code, name=name) from None
|
||||||
|
clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exception, code: 60, server response: Code: 60. DB::Exception: Unknown table expression identifier 'mapping_storevisibility' in scope SELECT DISTINCT StoreId FROM mapping_storevisibility WHERE (toDate(Fromdate) <= toDate('2026-06-20')) AND (toDate(Todate) >= toDate('2026-06-20')) AND (project_Id = '40148'). (UNKNOWN_TABLE) (for url http://172.188.12.194:8123)
|
||||||
|
2026-06-19 18:27:24 | INFO | ================================================================================
|
||||||
|
2026-06-19 18:27:24 | INFO | Hello from data-move Python data pipeline!
|
||||||
|
2026-06-19 18:27:24 | INFO | Pipeline Run Date: 2026-06-18
|
||||||
|
2026-06-19 18:27:24 | INFO | Connecting to databases...
|
||||||
|
2026-06-19 18:27:25 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000216D0A0C890>
|
||||||
|
2026-06-19 18:27:26 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000216D1ABC7D0>
|
||||||
|
2026-06-19 18:27:26 | INFO | Database connections established
|
||||||
|
2026-06-19 18:27:26 | INFO | Collecting MIDs for: 2026-06-18
|
||||||
|
2026-06-19 18:27:27 | INFO | Found 859 MIDs
|
||||||
|
2026-06-19 18:27:27 | INFO | ================================================================================
|
||||||
|
2026-06-19 18:27:27 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT
|
||||||
|
2026-06-19 18:27:27 | INFO | Fetching Data from sql server for table-: mapping_storevisibility ..............
|
||||||
|
2026-06-19 18:27:27 | WARNING | Table 'mapping_storevisibility' does not exist. During collecting store_ids
|
||||||
|
2026-06-19 18:27:27 | INFO | Fetching data from sql server for Master table......
|
||||||
|
2026-06-19 18:27:27 | INFO | Fetching in progress ....
|
||||||
|
2026-06-19 18:27:27 | INFO | Fetched 0 rows from SQL Server
|
||||||
|
2026-06-19 18:27:27 | INFO | Fetched 0 rows from SQL Server
|
||||||
|
2026-06-19 18:27:27 | INFO | Fetched total row -: 0 from sql server for table-:mapping_storevisibility ...........!!!
|
||||||
|
2026-06-19 18:27:27 | WARNING | mapping_storevisibility returned no rows
|
||||||
|
2026-06-19 18:27:27 | INFO | ================================================================================
|
||||||
|
2026-06-19 18:27:27 | INFO | Pipeline Completed Successfully
|
||||||
|
2026-06-19 18:27:27 | INFO | ================================================================================
|
||||||
|
2026-06-19 18:27:32 | INFO | ================================================================================
|
||||||
|
2026-06-19 18:27:32 | INFO | Hello from data-move Python data pipeline!
|
||||||
|
2026-06-19 18:27:32 | INFO | Pipeline Run Date: 2026-06-18
|
||||||
|
2026-06-19 18:27:32 | INFO | Connecting to databases...
|
||||||
|
2026-06-19 18:27:33 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000261CC384730>
|
||||||
|
2026-06-19 18:27:34 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000261CD42C7D0>
|
||||||
|
2026-06-19 18:27:35 | INFO | Database connections established
|
||||||
|
2026-06-19 18:27:35 | INFO | Collecting MIDs for: 2026-06-18
|
||||||
|
2026-06-19 18:27:35 | INFO | Found 859 MIDs
|
||||||
|
2026-06-19 18:27:35 | INFO | ================================================================================
|
||||||
|
2026-06-19 18:27:35 | INFO | Processing Table: mapping_storevisibility | Table type is -: BRIDGE | Based on -run_date and operation is used -ONLY_INSERT
|
||||||
|
2026-06-19 18:27:35 | INFO | Fetching Data from sql server for table-: mapping_storevisibility ..............
|
||||||
|
2026-06-19 18:27:35 | WARNING | Table 'mapping_storevisibility' does not exist. During collecting store_ids
|
||||||
|
2026-06-19 18:27:35 | INFO | Fetching data from sql server for Master table......
|
||||||
|
2026-06-19 18:27:35 | INFO | Fetching in progress ....
|
||||||
|
2026-06-19 18:27:35 | INFO | Fetched 0 rows from SQL Server
|
||||||
|
2026-06-19 18:27:35 | INFO | Fetched 0 rows from SQL Server
|
||||||
|
2026-06-19 18:27:35 | INFO | Fetched total row -: 0 from sql server for table-:mapping_storevisibility ...........!!!
|
||||||
|
2026-06-19 18:27:35 | WARNING | mapping_storevisibility returned no rows
|
||||||
|
2026-06-19 18:27:35 | INFO | ================================================================================
|
||||||
|
2026-06-19 18:27:35 | INFO | Pipeline Completed Successfully
|
||||||
|
2026-06-19 18:27:35 | INFO | ================================================================================
|
||||||
|
|||||||
@@ -0,0 +1,307 @@
|
|||||||
|
2026-06-22 10:05:56 | INFO | ================================================================================
|
||||||
|
2026-06-22 10:05:56 | INFO | Hello from data-move Python data pipeline!
|
||||||
|
2026-06-22 10:05:56 | INFO | Pipeline Run Date: 2026-06-21
|
||||||
|
2026-06-22 10:05:56 | INFO | Connecting to databases...
|
||||||
|
2026-06-22 10:05:57 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001F93F950730>
|
||||||
|
2026-06-22 10:06:00 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001F9409FC7D0>
|
||||||
|
2026-06-22 10:06:03 | INFO | Database connections established
|
||||||
|
2026-06-22 10:06:03 | INFO | Collecting MIDs for: 2026-06-21
|
||||||
|
2026-06-22 10:06:05 | INFO | Found 569 MIDs
|
||||||
|
2026-06-22 10:06:07 | INFO | ================================================================================
|
||||||
|
2026-06-22 10:06:07 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||||
|
2026-06-22 10:06:07 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
|
||||||
|
2026-06-22 10:06:07 | INFO | Fetching data from sql server for Master table......
|
||||||
|
2026-06-22 10:06:07 | INFO | Fetching in progress ....
|
||||||
|
2026-06-22 10:06:31 | ERROR | Failed processing table Master_VisibilityReason
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context
|
||||||
|
self.dialect.do_execute(
|
||||||
|
~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
cursor, str_statement, effective_parameters, context
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
)
|
||||||
|
^
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute
|
||||||
|
cursor.execute(statement, parameters)
|
||||||
|
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
pyodbc.OperationalError: ('08S01', '[08S01] [Microsoft][ODBC Driver 18 for SQL Server]TCP Provider: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond.\r\n (10060) (SQLExecDirectW); [08S01] [Microsoft][ODBC Driver 18 for SQL Server]Communication link failure (10060)')
|
||||||
|
|
||||||
|
The above exception was the direct cause of the following exception:
|
||||||
|
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "D:\data_move\main2.py", line 161, in main
|
||||||
|
df = fetch_data(sql_engine ,table_name,table_type)
|
||||||
|
File "D:\data_move\src\dim.py", line 45, in fetch_data
|
||||||
|
df = pl.read_database(
|
||||||
|
query=sql,
|
||||||
|
connection=engine
|
||||||
|
)
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\functions.py", line 284, in read_database
|
||||||
|
return cx.execute(
|
||||||
|
~~~~~~~~~~^
|
||||||
|
query=query,
|
||||||
|
^^^^^^^^^^^^
|
||||||
|
options=execute_options,
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
).to_polars(
|
||||||
|
^
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\polars\io\database\_executor.py", line 546, in execute
|
||||||
|
result = cursor_execute(query, **options)
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1421, in execute
|
||||||
|
return meth(
|
||||||
|
self,
|
||||||
|
distilled_parameters,
|
||||||
|
execution_options or NO_OPTIONS,
|
||||||
|
)
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\sql\elements.py", line 526, in _execute_on_connection
|
||||||
|
return connection._execute_clauseelement(
|
||||||
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
self, distilled_params, execution_options
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
)
|
||||||
|
^
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1643, in _execute_clauseelement
|
||||||
|
ret = self._execute_context(
|
||||||
|
dialect,
|
||||||
|
...<8 lines>...
|
||||||
|
cache_hit=cache_hit,
|
||||||
|
)
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1848, in _execute_context
|
||||||
|
return self._exec_single_context(
|
||||||
|
~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
dialect, context, statement, parameters
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
)
|
||||||
|
^
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1988, in _exec_single_context
|
||||||
|
self._handle_dbapi_exception(
|
||||||
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
e, str_statement, effective_parameters, cursor, context
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
)
|
||||||
|
^
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 2365, in _handle_dbapi_exception
|
||||||
|
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\base.py", line 1969, in _exec_single_context
|
||||||
|
self.dialect.do_execute(
|
||||||
|
~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
cursor, str_statement, effective_parameters, context
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
)
|
||||||
|
^
|
||||||
|
File "C:\Users\ankitm\AppData\Local\uv\cache\environments-v2\main2-f1515603462f26e4\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute
|
||||||
|
cursor.execute(statement, parameters)
|
||||||
|
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
sqlalchemy.exc.OperationalError: (pyodbc.OperationalError) ('08S01', '[08S01] [Microsoft][ODBC Driver 18 for SQL Server]TCP Provider: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond.\r\n (10060) (SQLExecDirectW); [08S01] [Microsoft][ODBC Driver 18 for SQL Server]Communication link failure (10060)')
|
||||||
|
[SQL: with Master_VisibilityReason (ProjectId,MenuId,ReasonId,Reason,CreateDate,Createby)
|
||||||
|
as (
|
||||||
|
|
||||||
|
select DISTINCT '40148' as ProjectId,MenuId,VisibilityReasonId,VisibilityReason,getdate(),'SP-Pius'
|
||||||
|
FROM OneApp_KelloggsMT.dbo.Master_VisibilityReason
|
||||||
|
)
|
||||||
|
select * from Master_VisibilityReason]
|
||||||
|
(Background on this error at: https://sqlalche.me/e/20/e3q8)
|
||||||
|
2026-06-22 10:10:52 | INFO | ================================================================================
|
||||||
|
2026-06-22 10:10:52 | INFO | Hello from data-move Python data pipeline!
|
||||||
|
2026-06-22 10:10:52 | INFO | Pipeline Run Date: 2026-06-21
|
||||||
|
2026-06-22 10:10:53 | INFO | Connecting to databases...
|
||||||
|
2026-06-22 10:10:53 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001500A41C730>
|
||||||
|
2026-06-22 10:10:55 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001500B56C7D0>
|
||||||
|
2026-06-22 10:10:56 | INFO | Database connections established
|
||||||
|
2026-06-22 10:10:56 | INFO | Collecting MIDs for: 2026-06-21
|
||||||
|
2026-06-22 10:10:56 | INFO | Found 569 MIDs
|
||||||
|
2026-06-22 10:10:57 | INFO | ================================================================================
|
||||||
|
2026-06-22 10:10:57 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||||
|
2026-06-22 10:10:57 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
|
||||||
|
2026-06-22 10:10:57 | INFO | Fetching data from sql server for Master table......
|
||||||
|
2026-06-22 10:10:57 | INFO | Fetching in progress ....
|
||||||
|
2026-06-22 10:10:57 | INFO | Fetched 17 rows from SQL Server
|
||||||
|
2026-06-22 10:10:57 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!!
|
||||||
|
2026-06-22 10:10:57 | INFO | Fetched 17 rows
|
||||||
|
2026-06-22 10:10:57 | INFO | Creating table Master_VisibilityReason
|
||||||
|
2026-06-22 10:10:57 | INFO | Table ready: Master_VisibilityReason
|
||||||
|
2026-06-22 10:10:57 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||||
|
2026-06-22 10:10:58 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse
|
||||||
|
2026-06-22 10:10:58 | INFO | Master_VisibilityReason loaded successfully (17 rows)
|
||||||
|
2026-06-22 10:10:58 | INFO | ================================================================================
|
||||||
|
2026-06-22 10:10:58 | INFO | Pipeline Completed Successfully
|
||||||
|
2026-06-22 10:10:58 | INFO | ================================================================================
|
||||||
|
2026-06-22 10:11:04 | INFO | ================================================================================
|
||||||
|
2026-06-22 10:11:04 | INFO | Hello from data-move Python data pipeline!
|
||||||
|
2026-06-22 10:11:04 | INFO | Pipeline Run Date: 2026-06-21
|
||||||
|
2026-06-22 10:11:04 | INFO | Connecting to databases...
|
||||||
|
2026-06-22 10:11:05 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002DFCC410730>
|
||||||
|
2026-06-22 10:11:06 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000002DFCD4CC7D0>
|
||||||
|
2026-06-22 10:11:07 | INFO | Database connections established
|
||||||
|
2026-06-22 10:11:07 | INFO | Collecting MIDs for: 2026-06-21
|
||||||
|
2026-06-22 10:11:07 | INFO | Found 569 MIDs
|
||||||
|
2026-06-22 10:11:08 | INFO | ================================================================================
|
||||||
|
2026-06-22 10:11:08 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||||
|
2026-06-22 10:11:08 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
|
||||||
|
2026-06-22 10:11:08 | INFO | Fetching data from sql server for Master table......
|
||||||
|
2026-06-22 10:11:08 | INFO | Fetching in progress ....
|
||||||
|
2026-06-22 10:11:08 | INFO | Fetched 17 rows from SQL Server
|
||||||
|
2026-06-22 10:11:08 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!!
|
||||||
|
2026-06-22 10:11:08 | INFO | Fetched 17 rows
|
||||||
|
2026-06-22 10:11:08 | INFO | Truncated table Master_VisibilityReason
|
||||||
|
2026-06-22 10:11:08 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||||
|
2026-06-22 10:11:08 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse
|
||||||
|
2026-06-22 10:11:08 | INFO | Master_VisibilityReason loaded successfully (17 rows)
|
||||||
|
2026-06-22 10:11:08 | INFO | ================================================================================
|
||||||
|
2026-06-22 10:11:08 | INFO | Pipeline Completed Successfully
|
||||||
|
2026-06-22 10:11:08 | INFO | ================================================================================
|
||||||
|
2026-06-22 11:07:54 | INFO | ================================================================================
|
||||||
|
2026-06-22 11:07:54 | INFO | Hello from data-move Python data pipeline!
|
||||||
|
2026-06-22 11:07:54 | INFO | Pipeline Run Date: 2026-06-21
|
||||||
|
2026-06-22 11:07:54 | INFO | Connecting to databases...
|
||||||
|
2026-06-22 11:07:56 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001689BE04730>
|
||||||
|
2026-06-22 11:08:00 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x000001689CEBC7D0>
|
||||||
|
2026-06-22 11:08:01 | INFO | Database connections established
|
||||||
|
2026-06-22 11:08:01 | INFO | Collecting MIDs for: 2026-06-21
|
||||||
|
2026-06-22 11:08:02 | INFO | Found 569 MIDs
|
||||||
|
2026-06-22 11:08:42 | INFO | ================================================================================
|
||||||
|
2026-06-22 11:08:42 | INFO | Hello from data-move Python data pipeline!
|
||||||
|
2026-06-22 11:08:42 | INFO | Pipeline Run Date: 2026-06-21
|
||||||
|
2026-06-22 11:08:42 | INFO | Connecting to databases...
|
||||||
|
2026-06-22 11:08:44 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000015B77AE4730>
|
||||||
|
2026-06-22 11:08:46 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000015B78B8C7D0>
|
||||||
|
2026-06-22 11:08:47 | INFO | Database connections established
|
||||||
|
2026-06-22 11:08:47 | INFO | Collecting MIDs for: 2026-06-21
|
||||||
|
2026-06-22 11:08:48 | INFO | Found 569 MIDs
|
||||||
|
2026-06-22 11:08:48 | INFO | ================================================================================
|
||||||
|
2026-06-22 11:08:48 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||||
|
2026-06-22 11:08:48 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
|
||||||
|
2026-06-22 11:08:48 | INFO | Fetching data from sql server for Master table......
|
||||||
|
2026-06-22 11:08:48 | INFO | Fetching in progress ....
|
||||||
|
2026-06-22 11:08:48 | INFO | Fetched 17 rows from SQL Server
|
||||||
|
2026-06-22 11:08:48 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!!
|
||||||
|
2026-06-22 11:08:48 | INFO | Fetched 17 rows
|
||||||
|
2026-06-22 11:08:48 | INFO | Truncated table Master_VisibilityReason
|
||||||
|
2026-06-22 11:08:48 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||||
|
2026-06-22 11:08:49 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse
|
||||||
|
2026-06-22 11:08:49 | INFO | Master_VisibilityReason loaded successfully (17 rows)
|
||||||
|
2026-06-22 11:08:49 | INFO | ================================================================================
|
||||||
|
2026-06-22 11:08:49 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||||
|
2026-06-22 11:08:49 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition ..............
|
||||||
|
2026-06-22 11:08:49 | INFO | Fetching data from sql server for Master table......
|
||||||
|
2026-06-22 11:08:49 | INFO | Fetching in progress ....
|
||||||
|
2026-06-22 11:08:49 | INFO | Fetched 861 rows from SQL Server
|
||||||
|
2026-06-22 11:08:49 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!!
|
||||||
|
2026-06-22 11:08:49 | INFO | Fetched 861 rows
|
||||||
|
2026-06-22 11:08:49 | INFO | Creating table Master_VisibilityDefinition
|
||||||
|
2026-06-22 11:08:49 | INFO | Table ready: Master_VisibilityDefinition
|
||||||
|
2026-06-22 11:08:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||||
|
2026-06-22 11:08:49 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse
|
||||||
|
2026-06-22 11:08:49 | INFO | Master_VisibilityDefinition loaded successfully (861 rows)
|
||||||
|
2026-06-22 11:08:49 | INFO | ================================================================================
|
||||||
|
2026-06-22 11:08:49 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||||
|
2026-06-22 11:08:49 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer ..............
|
||||||
|
2026-06-22 11:08:49 | INFO | Fetching data from sql server for Master table......
|
||||||
|
2026-06-22 11:08:49 | INFO | Fetching in progress ....
|
||||||
|
2026-06-22 11:08:49 | INFO | Fetched 33 rows from SQL Server
|
||||||
|
2026-06-22 11:08:49 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!!
|
||||||
|
2026-06-22 11:08:49 | INFO | Fetched 33 rows
|
||||||
|
2026-06-22 11:08:50 | INFO | Creating table Master_Salesterritorylayer
|
||||||
|
2026-06-22 11:08:50 | INFO | Table ready: Master_Salesterritorylayer
|
||||||
|
2026-06-22 11:08:50 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||||
|
2026-06-22 11:08:50 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse
|
||||||
|
2026-06-22 11:08:50 | INFO | Master_Salesterritorylayer loaded successfully (33 rows)
|
||||||
|
2026-06-22 11:08:50 | INFO | ================================================================================
|
||||||
|
2026-06-22 11:08:50 | INFO | Pipeline Completed Successfully
|
||||||
|
2026-06-22 11:08:50 | INFO | ================================================================================
|
||||||
|
2026-06-22 11:08:56 | INFO | ================================================================================
|
||||||
|
2026-06-22 11:08:56 | INFO | Hello from data-move Python data pipeline!
|
||||||
|
2026-06-22 11:08:56 | INFO | Pipeline Run Date: 2026-06-21
|
||||||
|
2026-06-22 11:08:56 | INFO | Connecting to databases...
|
||||||
|
2026-06-22 11:08:57 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000214E3374730>
|
||||||
|
2026-06-22 11:08:59 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x00000214E44CC7D0>
|
||||||
|
2026-06-22 11:09:00 | INFO | Database connections established
|
||||||
|
2026-06-22 11:09:00 | INFO | Collecting MIDs for: 2026-06-21
|
||||||
|
2026-06-22 11:09:00 | INFO | Found 569 MIDs
|
||||||
|
2026-06-22 11:09:00 | INFO | ================================================================================
|
||||||
|
2026-06-22 11:09:00 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||||
|
2026-06-22 11:09:00 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
|
||||||
|
2026-06-22 11:09:00 | INFO | Fetching data from sql server for Master table......
|
||||||
|
2026-06-22 11:09:00 | INFO | Fetching in progress ....
|
||||||
|
2026-06-22 11:09:01 | INFO | Fetched 17 rows from SQL Server
|
||||||
|
2026-06-22 11:09:01 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!!
|
||||||
|
2026-06-22 11:09:01 | INFO | Fetched 17 rows
|
||||||
|
2026-06-22 11:09:01 | INFO | Truncated table Master_VisibilityReason
|
||||||
|
2026-06-22 11:09:01 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||||
|
2026-06-22 11:09:01 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse
|
||||||
|
2026-06-22 11:09:01 | INFO | Master_VisibilityReason loaded successfully (17 rows)
|
||||||
|
2026-06-22 11:09:01 | INFO | ================================================================================
|
||||||
|
2026-06-22 11:09:01 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||||
|
2026-06-22 11:09:01 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition ..............
|
||||||
|
2026-06-22 11:09:01 | INFO | Fetching data from sql server for Master table......
|
||||||
|
2026-06-22 11:09:01 | INFO | Fetching in progress ....
|
||||||
|
2026-06-22 11:09:02 | INFO | Fetched 861 rows from SQL Server
|
||||||
|
2026-06-22 11:09:02 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!!
|
||||||
|
2026-06-22 11:09:02 | INFO | Fetched 861 rows
|
||||||
|
2026-06-22 11:09:02 | INFO | Truncated table Master_VisibilityDefinition
|
||||||
|
2026-06-22 11:09:02 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||||
|
2026-06-22 11:09:02 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse
|
||||||
|
2026-06-22 11:09:02 | INFO | Master_VisibilityDefinition loaded successfully (861 rows)
|
||||||
|
2026-06-22 11:09:02 | INFO | ================================================================================
|
||||||
|
2026-06-22 11:09:02 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||||
|
2026-06-22 11:09:02 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer ..............
|
||||||
|
2026-06-22 11:09:02 | INFO | Fetching data from sql server for Master table......
|
||||||
|
2026-06-22 11:09:02 | INFO | Fetching in progress ....
|
||||||
|
2026-06-22 11:09:02 | INFO | Fetched 33 rows from SQL Server
|
||||||
|
2026-06-22 11:09:02 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!!
|
||||||
|
2026-06-22 11:09:02 | INFO | Fetched 33 rows
|
||||||
|
2026-06-22 11:09:02 | INFO | Truncated table Master_Salesterritorylayer
|
||||||
|
2026-06-22 11:09:02 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||||
|
2026-06-22 11:09:03 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse
|
||||||
|
2026-06-22 11:09:03 | INFO | Master_Salesterritorylayer loaded successfully (33 rows)
|
||||||
|
2026-06-22 11:09:03 | INFO | ================================================================================
|
||||||
|
2026-06-22 11:09:03 | INFO | Pipeline Completed Successfully
|
||||||
|
2026-06-22 11:09:03 | INFO | ================================================================================
|
||||||
|
2026-06-22 12:04:40 | INFO | ================================================================================
|
||||||
|
2026-06-22 12:04:40 | INFO | Hello from data-move Python data pipeline!
|
||||||
|
2026-06-22 12:04:40 | INFO | Pipeline Run Date: 2026-06-21
|
||||||
|
2026-06-22 12:04:40 | INFO | Connecting to databases...
|
||||||
|
2026-06-22 12:04:43 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000023A01CA4730>
|
||||||
|
2026-06-22 12:04:44 | INFO | <sqlalchemy.engine.cursor.CursorResult object at 0x0000023A02D1C7D0>
|
||||||
|
2026-06-22 12:04:45 | INFO | Database connections established
|
||||||
|
2026-06-22 12:04:45 | INFO | Collecting MIDs for: 2026-06-21
|
||||||
|
2026-06-22 12:04:46 | INFO | Found 569 MIDs
|
||||||
|
2026-06-22 12:04:48 | INFO | ================================================================================
|
||||||
|
2026-06-22 12:04:48 | INFO | Processing Table: Master_VisibilityReason | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||||
|
2026-06-22 12:04:48 | INFO | Fetching Data from sql server for table-: Master_VisibilityReason ..............
|
||||||
|
2026-06-22 12:04:48 | INFO | Fetching data from sql server for Master table......
|
||||||
|
2026-06-22 12:04:48 | INFO | Fetching in progress ....
|
||||||
|
2026-06-22 12:04:48 | INFO | Fetched 17 rows from SQL Server
|
||||||
|
2026-06-22 12:04:48 | INFO | Fetched total row -: 17 from sql server for table-:Master_VisibilityReason ...........!!!
|
||||||
|
2026-06-22 12:04:48 | INFO | Fetched 17 rows
|
||||||
|
2026-06-22 12:04:48 | INFO | Truncated table Master_VisibilityReason
|
||||||
|
2026-06-22 12:04:48 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||||
|
2026-06-22 12:04:48 | INFO | Master_VisibilityReason: inserted 17 rows into ClickHouse
|
||||||
|
2026-06-22 12:04:48 | INFO | Master_VisibilityReason loaded successfully (17 rows)
|
||||||
|
2026-06-22 12:04:48 | INFO | ================================================================================
|
||||||
|
2026-06-22 12:04:48 | INFO | Processing Table: Master_VisibilityDefinition | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||||
|
2026-06-22 12:04:48 | INFO | Fetching Data from sql server for table-: Master_VisibilityDefinition ..............
|
||||||
|
2026-06-22 12:04:48 | INFO | Fetching data from sql server for Master table......
|
||||||
|
2026-06-22 12:04:48 | INFO | Fetching in progress ....
|
||||||
|
2026-06-22 12:04:49 | INFO | Fetched 861 rows from SQL Server
|
||||||
|
2026-06-22 12:04:49 | INFO | Fetched total row -: 861 from sql server for table-:Master_VisibilityDefinition ...........!!!
|
||||||
|
2026-06-22 12:04:49 | INFO | Fetched 861 rows
|
||||||
|
2026-06-22 12:04:49 | INFO | Truncated table Master_VisibilityDefinition
|
||||||
|
2026-06-22 12:04:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||||
|
2026-06-22 12:04:49 | INFO | Master_VisibilityDefinition: inserted 861 rows into ClickHouse
|
||||||
|
2026-06-22 12:04:49 | INFO | Master_VisibilityDefinition loaded successfully (861 rows)
|
||||||
|
2026-06-22 12:04:49 | INFO | ================================================================================
|
||||||
|
2026-06-22 12:04:49 | INFO | Processing Table: Master_Salesterritorylayer | Table type is -: DIMENSION | Based on -none and operation is used -DELETE+INSERT
|
||||||
|
2026-06-22 12:04:49 | INFO | Fetching Data from sql server for table-: Master_Salesterritorylayer ..............
|
||||||
|
2026-06-22 12:04:49 | INFO | Fetching data from sql server for Master table......
|
||||||
|
2026-06-22 12:04:49 | INFO | Fetching in progress ....
|
||||||
|
2026-06-22 12:04:49 | INFO | Fetched 33 rows from SQL Server
|
||||||
|
2026-06-22 12:04:49 | INFO | Fetched total row -: 33 from sql server for table-:Master_Salesterritorylayer ...........!!!
|
||||||
|
2026-06-22 12:04:49 | INFO | Fetched 33 rows
|
||||||
|
2026-06-22 12:04:49 | INFO | Truncated table Master_Salesterritorylayer
|
||||||
|
2026-06-22 12:04:49 | INFO | _ _ _ _Inserting data into clickhouse db from sql server_ _ _ _
|
||||||
|
2026-06-22 12:04:50 | INFO | Master_Salesterritorylayer: inserted 33 rows into ClickHouse
|
||||||
|
2026-06-22 12:04:50 | INFO | Master_Salesterritorylayer loaded successfully (33 rows)
|
||||||
|
2026-06-22 12:04:50 | INFO | ================================================================================
|
||||||
|
2026-06-22 12:04:50 | INFO | Pipeline Completed Successfully
|
||||||
|
2026-06-22 12:04:50 | INFO | ================================================================================
|
||||||
@@ -43,7 +43,7 @@ from mids import (
|
|||||||
MID_TABLE_COV1,
|
MID_TABLE_COV1,
|
||||||
)
|
)
|
||||||
|
|
||||||
from masters.bridge import *
|
from src.bridge import *
|
||||||
from src.fact import *
|
from src.fact import *
|
||||||
from src.dim import *
|
from src.dim import *
|
||||||
|
|
||||||
@@ -211,6 +211,8 @@ def main():
|
|||||||
table_name,
|
table_name,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
elif operation =="ONLY_INSERT" :
|
||||||
|
continue
|
||||||
else:
|
else:
|
||||||
|
|
||||||
delete_existing_data(
|
delete_existing_data(
|
||||||
|
|||||||
+3
-2
@@ -75,7 +75,7 @@ def fetch_mapping_storevisibility(
|
|||||||
|
|
||||||
store_id_list = ",".join(str(sid) for sid in store_id)
|
store_id_list = ",".join(str(sid) for sid in store_id)
|
||||||
|
|
||||||
sql_file = Path("src") / "sql" / f"dim" / f"{table_name}.sql"
|
sql_file = Path("src") / "sql" / f"bridge" / f"{table_name}.sql"
|
||||||
|
|
||||||
with open(sql_file, "r", encoding="utf-8") as f:
|
with open(sql_file, "r", encoding="utf-8") as f:
|
||||||
sql_template = f.read()
|
sql_template = f.read()
|
||||||
@@ -97,10 +97,11 @@ def fetch_mapping_storevisibility(
|
|||||||
|
|
||||||
return df
|
return df
|
||||||
|
|
||||||
store_id=get_reason_ids_mapping_storevisibility(client, run_date, "coverage_remarks")
|
store_id=get_reason_ids_mapping_storevisibility(client, run_date, "mapping_storevisibility")
|
||||||
df=fetch_data(engine=sql_engine,
|
df=fetch_data(engine=sql_engine,
|
||||||
table_name=table_name,
|
table_name=table_name,
|
||||||
table_type=table_type,
|
table_type=table_type,
|
||||||
|
run_date=run_date,
|
||||||
store_id=store_id,
|
store_id=store_id,
|
||||||
)
|
)
|
||||||
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
log.info(f"Fetched {len(df):,} rows from SQL Server")
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
with Master_Salesterritorylayer(ProjectId,StLayerOneId,StLayerOneName,StLayerTwoId,StLayerTwoName,StLayerThreeId,StLayerThreeName,StLayerFourId,
|
||||||
|
StLayerFourName,CreateDate,CreateBy)
|
||||||
|
as (
|
||||||
|
select DISTINCT '40148' as Project_Id,d.StLayerOneId,d.StLayerOneName,c.StLayerTwoId,c.StLayerTwoName ,
|
||||||
|
b.StLayerThreeId,b.StLayerThreeName,a.StLayerFourId,a.StLayerFourName ,
|
||||||
|
getdate(),'SP-Pius'
|
||||||
|
FROM OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerFour] A
|
||||||
|
INNER JOIN OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerThree]B ON
|
||||||
|
A.StLayerThreeId=B.StLayerThreeId
|
||||||
|
INNER JOIN OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerTWo]C ON
|
||||||
|
B.StLayerTwoId=C.StLayerTwoId
|
||||||
|
INNER JOIN OneApp_KelloggsMT.dbo.[Master_SalesTerritoryLayerOne]D ON
|
||||||
|
C.StLayerOneId=D.StLayerOneId)
|
||||||
|
select * from Master_Salesterritorylayer
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
with Master_VisibilityDefinition(Project_Id,VisibilityDefinitionId,VisibilityDefinitionName,CreateDate,CreateBy)
|
||||||
|
as (
|
||||||
|
select DISTINCT '40148' as Project_Id,VisibilityDefinitionid,VisibilityDefinitionname,getdate(),'SP-Pius'
|
||||||
|
FROM OneApp_KelloggsMT.dbo.Master_VisibilityDefinition)
|
||||||
|
select * from Master_VisibilityDefinition
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
with Master_VisibilityReason (ProjectId,MenuId,ReasonId,Reason,CreateDate,Createby)
|
||||||
|
as (
|
||||||
|
|
||||||
|
select DISTINCT '40148' as ProjectId,MenuId,VisibilityReasonId,VisibilityReason,getdate(),'SP-Pius'
|
||||||
|
FROM OneApp_KelloggsMT.dbo.Master_VisibilityReason
|
||||||
|
)
|
||||||
|
select * from Master_VisibilityReason
|
||||||
@@ -75,8 +75,30 @@ tables:
|
|||||||
# operation: DELETE+INSERT
|
# operation: DELETE+INSERT
|
||||||
# fetch_by: master
|
# fetch_by: master
|
||||||
|
|
||||||
- name: Employee_Master
|
# - name: Employee_Master
|
||||||
|
# type: DIMENSION
|
||||||
|
# operation: DELETE+INSERT
|
||||||
|
# fetch_by: master
|
||||||
|
|
||||||
|
# - name: mapping_storevisibility
|
||||||
|
# type: BRIDGE
|
||||||
|
# operation: ONLY_INSERT
|
||||||
|
# fetch_by: run_date
|
||||||
|
|
||||||
|
- name: Master_VisibilityReason
|
||||||
type: DIMENSION
|
type: DIMENSION
|
||||||
operation: DELETE+INSERT
|
operation: DELETE+INSERT
|
||||||
fetch_by: master
|
fetch_by: none
|
||||||
|
|
||||||
|
|
||||||
|
- name: Master_VisibilityDefinition
|
||||||
|
type: DIMENSION
|
||||||
|
operation: DELETE+INSERT
|
||||||
|
fetch_by: none
|
||||||
|
|
||||||
|
|
||||||
|
- name: Master_Salesterritorylayer
|
||||||
|
type: DIMENSION
|
||||||
|
operation: DELETE+INSERT
|
||||||
|
fetch_by: none
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user