ankitm/data_pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

19-06-2026 1st commit

Browse Source
This commit is contained in:
Ankit Malik
2026-06-19 16:51:03 +05:30
parent d60740ae48
commit 97b22ea0f8
23 changed files with 2026 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main2.py
+9 -36
View File
@@ -43,9 +43,9 @@ from mids import (
MID_TABLE_COV1,
)
from masters.dimensions import *
from masters.bridge import *
from src.fact import *
from src.dim import *
# ==========================================================
@@ -64,29 +64,6 @@ def table_exists(
)
def get_dataframe(
sql_engine,
fn_name: str,
fetch_by: str,
table_name: str,
table_type: str ,
mids,
run_date,
):
fn = globals()[fn_name]
if fetch_by == "mids" or "run_date":
return fn(sql_engine, table_name , table_type, mids, run_date)
return fn(sql_engine ,table_name,table_type)
# ==========================================================
# Main
# ==========================================================
@@ -158,9 +135,7 @@ def main():
# ------------------------------------------------------
# Process Tables
# ------------------------------------------------------
for table in config["tables"]:
table_name = table["name"]
operation = table["operation"]
fetch_by = table["fetch_by"]
@@ -177,16 +152,14 @@ def main():
log.info(f"Fetching Data from sql server for table-: {table_name} ..............")
fn_name = f"fetch_{table_name}"
df = get_dataframe(
sql_engine,
fn_name=fn_name,
fetch_by=fetch_by,
table_name=table_name,
table_type=table_type,
mids=mids,
run_date=run_date,
)
fetch_list=["mids" ,"run_date", "reason_id"]
if fetch_by in fetch_list :
fn_name = f"fetch_{table_name}"
fn = globals()[fn_name]
df=fn(sql_engine, table_name , table_type, mids, run_date)
else:
df = fetch_data(sql_engine ,table_name,table_type)
log.info(f"Fetched total row -: {len(df)} from sql server for table-:{table_name} ...........!!!")
if df.is_empty():