Ankit Malik
157 lines
3.0 KiB
Python
157 lines
3.0 KiB
Python
# /// script
|
|
# requires-python = ">=3.11"
|
|
# dependencies = [
|
|
# "polars>=0.20.0",
|
|
# "pyarrow>=18.0.0",
|
|
# "sqlalchemy>=2.0.0",
|
|
# "pyodbc>=5.0.0",
|
|
# "clickhouse-connect>=0.7.0",
|
|
# "clickhouse-sqlalchemy>=0.3.2",
|
|
# "pyyaml>=6.0.3",
|
|
# "python-dotenv>=1.0.0",
|
|
# ]
|
|
# ///
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
# import pyarrow
|
|
import sys
|
|
from datetime import date, timedelta
|
|
|
|
import polars as pl
|
|
import yaml
|
|
from dotenv import load_dotenv
|
|
|
|
from sqlalchemy import create_engine, text
|
|
from sqlalchemy.engine import Engine, URL
|
|
|
|
import clickhouse_connect
|
|
|
|
from log import log
|
|
|
|
|
|
|
|
from clickhouse_task.create_table import *
|
|
from clickhouse_task.delete_task import *
|
|
from clickhouse_task.load_table import *
|
|
from db_con.connection import *
|
|
from mids import *
|
|
from masters.dimensions import *
|
|
from masters.bridge import *
|
|
from kpi.facts import *
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main():
|
|
|
|
log.info("=" * 80)
|
|
log.info("Hello from data-move Python data pipeline !")
|
|
|
|
if len(sys.argv) > 1:
|
|
run_date = datetime.strptime(sys.argv[1], "%Y-%m-%d").date()
|
|
else:
|
|
run_date = date.today() - timedelta(days=1)
|
|
|
|
log.info(f"Data-pipeline running Date is -:{run_date}")
|
|
# connecting with both db servers sql-server
|
|
|
|
|
|
log.info("connecting with both db servers sql-serveras well as clickhouse DB")
|
|
|
|
sql_engine = build_sql_server_engine()
|
|
clickhouse_engine = build_clickhouse_engine()
|
|
client=get_clickhouse_client()
|
|
|
|
log.info("Both databases connected successfully")
|
|
|
|
mids=collect_mids(sql_engine , run_date)
|
|
|
|
|
|
# fetching polar df from sql-server
|
|
|
|
|
|
|
|
|
|
|
|
with open("tables.yml", "r") as file:
|
|
config = yaml.safe_load(file)
|
|
|
|
for table in config["tables"]:
|
|
|
|
table_name=table["name"]
|
|
table_type=table["type"]
|
|
operation=table["operation"]
|
|
|
|
log.info("=" * 80)
|
|
log.info("TABLE=%s | TYPE=%s | OPERATION=%s",
|
|
table_name,
|
|
table_type,
|
|
operation)
|
|
fn=f"fetch_{table_name}"
|
|
list=["Attendance", "Journey_Plan", "Web_Logins"]
|
|
|
|
if table_type =="FACT" :
|
|
|
|
if table_name in list :
|
|
df = globals()[fn](sql_engine, run_date)
|
|
|
|
else:
|
|
df = globals()[fn](sql_engine, mids)
|
|
|
|
|
|
elif table_type =="BRIDGE" :
|
|
|
|
df = globals()[fn](sql_engine, run_date)
|
|
|
|
else:
|
|
|
|
df = globals()[fn](sql_engine)
|
|
|
|
|
|
|
|
# Step 2
|
|
|
|
if operation == "DELETE+INSERT" :
|
|
truncate_table(client , table_name )
|
|
log.info(f"Truncate a ClickHouse table - {table_name}")
|
|
|
|
load_to_clickhouse(
|
|
client=client,
|
|
table_name=table_name,
|
|
df=df,
|
|
)
|
|
|
|
else:
|
|
print("table is fact ")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#fetch table details
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|