Ankit Malik
21 lines
543 B
Python
21 lines
543 B
Python
from airflow import DAG
|
|
from airflow.operators.bash import BashOperator
|
|
from datetime import datetime, timedelta
|
|
|
|
with DAG(
|
|
dag_id="sqlserver_clickhouse",
|
|
start_date=datetime(2023, 1, 1),
|
|
schedule="@daily",
|
|
catchup=True, # Important
|
|
max_active_runs=1, # Process one date at a time
|
|
):
|
|
|
|
run_pipeline = BashOperator(
|
|
task_id="run_pipeline",
|
|
bash_command="""
|
|
cd /opt/airflow/project &&
|
|
uv run main.py {{ ds }}
|
|
""",
|
|
retries=3,
|
|
retry_delay=timedelta(minutes=5),
|
|
) |