Journey_Plan data Import
This commit is contained in:
Dipanshu Kumar
+257
-107
@@ -1,5 +1,3 @@
|
||||
|
||||
|
||||
import pyodbc
|
||||
import pandas as pd
|
||||
import clickhouse_connect
|
||||
@@ -7,6 +5,7 @@ import numpy as np
|
||||
from datetime import datetime
|
||||
import traceback
|
||||
import warnings
|
||||
import time
|
||||
|
||||
# =========================================================
|
||||
# IGNORE WARNINGS
|
||||
@@ -16,10 +15,12 @@ warnings.filterwarnings(
|
||||
'pandas only supports SQLAlchemy connectable'
|
||||
)
|
||||
|
||||
print("ETL Started :", datetime.now())
|
||||
print("\n================================================")
|
||||
print("ETL STARTED :", datetime.now())
|
||||
print("================================================")
|
||||
|
||||
# =========================================================
|
||||
# SQL SERVER CONNECTION
|
||||
# SQL SERVER CONNECTION STRING
|
||||
# =========================================================
|
||||
SQL_CONN_STR = (
|
||||
'DRIVER={ODBC Driver 17 for SQL Server};'
|
||||
@@ -28,6 +29,7 @@ SQL_CONN_STR = (
|
||||
'UID=bsgteam_test;'
|
||||
'PWD=B$gt3@m#00512;'
|
||||
'TrustServerCertificate=yes;'
|
||||
'Connection Timeout=60;'
|
||||
)
|
||||
|
||||
# =========================================================
|
||||
@@ -42,11 +44,67 @@ CH_CONFIG = {
|
||||
}
|
||||
|
||||
# =========================================================
|
||||
# TABLE NAME
|
||||
# TABLE DETAILS
|
||||
# =========================================================
|
||||
TABLE_NAME = 'Journey_Plan'
|
||||
PROJECT_ID = 41654
|
||||
|
||||
# =========================================================
|
||||
# SETTINGS
|
||||
# =========================================================
|
||||
TRUNCATE_BEFORE_LOAD = True
|
||||
table_truncated = False
|
||||
|
||||
# =========================================================
|
||||
# CHUNK SIZE
|
||||
# =========================================================
|
||||
chunk_size = 20000
|
||||
|
||||
# =========================================================
|
||||
# CONNECT SQL SERVER
|
||||
# =========================================================
|
||||
def connect_sql():
|
||||
|
||||
try:
|
||||
|
||||
conn = pyodbc.connect(
|
||||
SQL_CONN_STR,
|
||||
autocommit=True
|
||||
)
|
||||
|
||||
print("Connected SQL Server")
|
||||
|
||||
return conn
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("SQL CONNECTION FAILED")
|
||||
print(str(e))
|
||||
|
||||
raise
|
||||
|
||||
# =========================================================
|
||||
# CONNECT CLICKHOUSE
|
||||
# =========================================================
|
||||
def connect_clickhouse():
|
||||
|
||||
try:
|
||||
|
||||
client = clickhouse_connect.get_client(
|
||||
**CH_CONFIG
|
||||
)
|
||||
|
||||
print("Connected ClickHouse")
|
||||
|
||||
return client
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("CLICKHOUSE CONNECTION FAILED")
|
||||
print(str(e))
|
||||
|
||||
raise
|
||||
|
||||
# =========================================================
|
||||
# CLEAN DATAFRAME
|
||||
# =========================================================
|
||||
@@ -66,39 +124,46 @@ def clean_dataframe(df):
|
||||
|
||||
try:
|
||||
|
||||
col_lower = col.lower()
|
||||
print(f"\nCleaning Column : {col}")
|
||||
|
||||
# =====================================
|
||||
# DATE COLUMNS
|
||||
# AUTO DETECT DATE / TIME COLUMNS
|
||||
# =====================================
|
||||
if 'date' in col_lower:
|
||||
if (
|
||||
'date' in col.lower()
|
||||
or 'time' in col.lower()
|
||||
):
|
||||
|
||||
print(f"Cleaning Date Column : {col}")
|
||||
print(f"Date Column : {col}")
|
||||
|
||||
df[col] = pd.to_datetime(
|
||||
df[col],
|
||||
errors='coerce'
|
||||
)
|
||||
|
||||
# Remove invalid dates
|
||||
df[col] = df[col].where(
|
||||
(df[col].dt.year >= 1970) &
|
||||
(df[col].dt.year <= 2100)
|
||||
cleaned_dates = []
|
||||
|
||||
for val in df[col]:
|
||||
|
||||
if pd.isnull(val):
|
||||
|
||||
cleaned_dates.append(None)
|
||||
|
||||
else:
|
||||
|
||||
cleaned_dates.append(
|
||||
val.to_pydatetime()
|
||||
)
|
||||
|
||||
# Convert to Python Date
|
||||
df[col] = df[col].apply(
|
||||
lambda x:
|
||||
x.date()
|
||||
if pd.notnull(x)
|
||||
else None
|
||||
)
|
||||
df[col] = cleaned_dates
|
||||
|
||||
# =====================================
|
||||
# INTEGER COLUMNS
|
||||
# =====================================
|
||||
elif pd.api.types.is_integer_dtype(df[col]):
|
||||
|
||||
print(f"Integer Column : {col}")
|
||||
|
||||
df[col] = pd.to_numeric(
|
||||
df[col],
|
||||
errors='coerce'
|
||||
@@ -116,26 +181,13 @@ def clean_dataframe(df):
|
||||
# =====================================
|
||||
elif pd.api.types.is_float_dtype(df[col]):
|
||||
|
||||
non_null = df[col].dropna()
|
||||
print(f"Float Column : {col}")
|
||||
|
||||
# ---------------------------------
|
||||
# Convert whole float to int
|
||||
# Example:
|
||||
# 3240.0 -> 3240
|
||||
# ---------------------------------
|
||||
if len(non_null) > 0 and (
|
||||
(non_null % 1 == 0).all()
|
||||
):
|
||||
|
||||
df[col] = df[col].apply(
|
||||
lambda x:
|
||||
int(x)
|
||||
if pd.notnull(x)
|
||||
else None
|
||||
df[col] = pd.to_numeric(
|
||||
df[col],
|
||||
errors='coerce'
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
df[col] = df[col].apply(
|
||||
lambda x:
|
||||
float(x)
|
||||
@@ -148,6 +200,8 @@ def clean_dataframe(df):
|
||||
# =====================================
|
||||
else:
|
||||
|
||||
print(f"String/Object Column : {col}")
|
||||
|
||||
cleaned = []
|
||||
|
||||
for val in df[col]:
|
||||
@@ -157,6 +211,28 @@ def clean_dataframe(df):
|
||||
|
||||
cleaned.append(None)
|
||||
|
||||
# DATETIME
|
||||
elif isinstance(
|
||||
val,
|
||||
(
|
||||
datetime,
|
||||
pd.Timestamp
|
||||
)
|
||||
):
|
||||
|
||||
if isinstance(
|
||||
val,
|
||||
pd.Timestamp
|
||||
):
|
||||
|
||||
cleaned.append(
|
||||
val.to_pydatetime()
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
cleaned.append(val)
|
||||
|
||||
# INTEGER
|
||||
elif isinstance(
|
||||
val,
|
||||
@@ -166,7 +242,9 @@ def clean_dataframe(df):
|
||||
)
|
||||
):
|
||||
|
||||
cleaned.append(int(val))
|
||||
cleaned.append(
|
||||
int(val)
|
||||
)
|
||||
|
||||
# FLOAT
|
||||
elif isinstance(
|
||||
@@ -183,41 +261,26 @@ def clean_dataframe(df):
|
||||
|
||||
else:
|
||||
|
||||
# IMPORTANT FIX
|
||||
# Avoid '3240.0' string issue
|
||||
if val.is_integer():
|
||||
|
||||
cleaned.append(int(val))
|
||||
|
||||
else:
|
||||
|
||||
cleaned.append(float(val))
|
||||
|
||||
# STRING
|
||||
elif isinstance(val, str):
|
||||
|
||||
cleaned.append(val.strip())
|
||||
cleaned.append(
|
||||
float(val)
|
||||
)
|
||||
|
||||
# BOOLEAN
|
||||
elif isinstance(val, bool):
|
||||
|
||||
cleaned.append(int(val))
|
||||
|
||||
# DATETIME
|
||||
elif isinstance(
|
||||
val,
|
||||
(
|
||||
datetime,
|
||||
pd.Timestamp
|
||||
)
|
||||
bool
|
||||
):
|
||||
|
||||
cleaned.append(str(val))
|
||||
cleaned.append(
|
||||
int(val)
|
||||
)
|
||||
|
||||
# OTHER
|
||||
# STRING
|
||||
else:
|
||||
|
||||
cleaned.append(str(val))
|
||||
cleaned.append(
|
||||
str(val).strip()
|
||||
)
|
||||
|
||||
df[col] = cleaned
|
||||
|
||||
@@ -247,16 +310,12 @@ try:
|
||||
# =====================================================
|
||||
# CONNECT SQL SERVER
|
||||
# =====================================================
|
||||
sql_conn = pyodbc.connect(SQL_CONN_STR)
|
||||
|
||||
print("Connected to SQL Server")
|
||||
sql_conn = connect_sql()
|
||||
|
||||
# =====================================================
|
||||
# CONNECT CLICKHOUSE
|
||||
# =====================================================
|
||||
ch_client = clickhouse_connect.get_client(**CH_CONFIG)
|
||||
|
||||
print("Connected to ClickHouse")
|
||||
ch_client = connect_clickhouse()
|
||||
|
||||
# =====================================================
|
||||
# QUERY
|
||||
@@ -267,19 +326,25 @@ try:
|
||||
WHERE Project_Id = {PROJECT_ID}
|
||||
"""
|
||||
|
||||
print("\nExecuting Query:")
|
||||
print("\nExecuting Query")
|
||||
print(query)
|
||||
|
||||
# =====================================================
|
||||
# CHUNK SIZE
|
||||
# RETRY SETTINGS
|
||||
# =====================================================
|
||||
chunk_size = 100000
|
||||
|
||||
total_rows = 0
|
||||
retry_count = 0
|
||||
max_retry = 5
|
||||
|
||||
# =====================================================
|
||||
# READ DATA
|
||||
# MAIN RETRY LOOP
|
||||
# =====================================================
|
||||
while retry_count < max_retry:
|
||||
|
||||
try:
|
||||
|
||||
# =============================================
|
||||
# READ SQL DATA
|
||||
# =============================================
|
||||
for chunk in pd.read_sql(
|
||||
query,
|
||||
sql_conn,
|
||||
@@ -289,18 +354,24 @@ try:
|
||||
try:
|
||||
|
||||
print("\n================================")
|
||||
print(f"Processing {len(chunk)} Rows")
|
||||
print(
|
||||
f"Processing Rows : "
|
||||
f"{len(chunk)}"
|
||||
)
|
||||
print("================================")
|
||||
|
||||
# =================================================
|
||||
# =====================================
|
||||
# CLEAN DATA
|
||||
# =================================================
|
||||
# =====================================
|
||||
chunk = clean_dataframe(chunk)
|
||||
|
||||
# =================================================
|
||||
# =====================================
|
||||
# DEBUG COLUMN TYPES
|
||||
# =================================================
|
||||
print("\nCOLUMN TYPES")
|
||||
# =====================================
|
||||
print("\nCOLUMN DATATYPES")
|
||||
print(chunk.dtypes)
|
||||
|
||||
print("\nCOLUMN SAMPLE TYPES")
|
||||
|
||||
for col in chunk.columns:
|
||||
|
||||
@@ -314,16 +385,44 @@ try:
|
||||
sample.iloc[0]
|
||||
)
|
||||
|
||||
# =================================================
|
||||
# SAMPLE DATA
|
||||
# =================================================
|
||||
print("\nSAMPLE DATA")
|
||||
print(chunk.head(2))
|
||||
# =====================================
|
||||
# TRUNCATE TABLE FIRST TIME ONLY
|
||||
# =====================================
|
||||
if (
|
||||
TRUNCATE_BEFORE_LOAD
|
||||
and
|
||||
not table_truncated
|
||||
):
|
||||
|
||||
# =================================================
|
||||
print("\n================================")
|
||||
print(
|
||||
f"TRUNCATING TABLE : "
|
||||
f"{TABLE_NAME}"
|
||||
)
|
||||
print("================================")
|
||||
|
||||
# IMPORTANT FIX
|
||||
truncate_query = f"""
|
||||
TRUNCATE TABLE
|
||||
`{CH_CONFIG['database']}`.`{TABLE_NAME}`
|
||||
"""
|
||||
|
||||
ch_client.command(
|
||||
truncate_query
|
||||
)
|
||||
|
||||
print(
|
||||
"TABLE TRUNCATED SUCCESSFULLY"
|
||||
)
|
||||
|
||||
table_truncated = True
|
||||
|
||||
# =====================================
|
||||
# INSERT INTO CLICKHOUSE
|
||||
# =================================================
|
||||
print("\nInserting into ClickHouse...")
|
||||
# =====================================
|
||||
print(
|
||||
"\nINSERTING INTO CLICKHOUSE..."
|
||||
)
|
||||
|
||||
ch_client.insert_df(
|
||||
table=TABLE_NAME,
|
||||
@@ -331,27 +430,26 @@ try:
|
||||
database=CH_CONFIG['database']
|
||||
)
|
||||
|
||||
total_rows += len(chunk)
|
||||
|
||||
print(
|
||||
f"\nInserted Total Rows : {total_rows}"
|
||||
f"INSERTED : "
|
||||
f"{len(chunk)} ROWS"
|
||||
)
|
||||
|
||||
except Exception as chunk_error:
|
||||
except Exception as insert_error:
|
||||
|
||||
print("\n================================")
|
||||
print("CHUNK INSERT FAILED")
|
||||
print("INSERT FAILED")
|
||||
print("================================")
|
||||
|
||||
print(str(chunk_error))
|
||||
print(str(insert_error))
|
||||
|
||||
traceback.print_exc()
|
||||
|
||||
# =============================================
|
||||
# =================================
|
||||
# SAVE ERROR LOG
|
||||
# =============================================
|
||||
# =================================
|
||||
with open(
|
||||
"clickhouse_chunk_error.log",
|
||||
"insert_error.log",
|
||||
"a",
|
||||
encoding="utf-8"
|
||||
) as log:
|
||||
@@ -361,15 +459,18 @@ try:
|
||||
)
|
||||
|
||||
log.write(
|
||||
f"\nTIME : {datetime.now()}"
|
||||
f"\nTIME : "
|
||||
f"{datetime.now()}"
|
||||
)
|
||||
|
||||
log.write(
|
||||
f"\nTABLE : {TABLE_NAME}"
|
||||
f"\nTABLE : "
|
||||
f"{TABLE_NAME}"
|
||||
)
|
||||
|
||||
log.write(
|
||||
f"\nERROR : {str(chunk_error)}"
|
||||
f"\nERROR : "
|
||||
f"{str(insert_error)}"
|
||||
)
|
||||
|
||||
log.write(
|
||||
@@ -383,9 +484,56 @@ try:
|
||||
|
||||
continue
|
||||
|
||||
# =============================================
|
||||
# SUCCESS
|
||||
# =============================================
|
||||
break
|
||||
|
||||
# =================================================
|
||||
# SQL CONNECTION FAILURE
|
||||
# =================================================
|
||||
except pyodbc.OperationalError as op_error:
|
||||
|
||||
retry_count += 1
|
||||
|
||||
print("\n================================")
|
||||
print(
|
||||
f"SQL CONNECTION LOST "
|
||||
f"- RETRY {retry_count}"
|
||||
)
|
||||
print("================================")
|
||||
|
||||
print(str(op_error))
|
||||
|
||||
time.sleep(10)
|
||||
|
||||
try:
|
||||
|
||||
sql_conn.close()
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
# RECONNECT SQL
|
||||
sql_conn = connect_sql()
|
||||
|
||||
# =================================================
|
||||
# OTHER ERROR
|
||||
# =================================================
|
||||
except Exception as loop_error:
|
||||
|
||||
print("\n================================")
|
||||
print("MAIN LOOP ERROR")
|
||||
print("================================")
|
||||
|
||||
print(str(loop_error))
|
||||
|
||||
traceback.print_exc()
|
||||
|
||||
break
|
||||
|
||||
print("\n================================")
|
||||
print("ETL COMPLETED SUCCESSFULLY")
|
||||
print(f"TOTAL ROWS INSERTED : {total_rows}")
|
||||
print("================================")
|
||||
|
||||
# =========================================================
|
||||
@@ -402,7 +550,7 @@ except Exception as main_error:
|
||||
traceback.print_exc()
|
||||
|
||||
with open(
|
||||
"clickhouse_main_error.log",
|
||||
"main_error.log",
|
||||
"a",
|
||||
encoding="utf-8"
|
||||
) as log:
|
||||
@@ -437,7 +585,7 @@ finally:
|
||||
|
||||
sql_conn.close()
|
||||
|
||||
print("\nSQL Server Connection Closed")
|
||||
print("\nSQL SERVER CONNECTION CLOSED")
|
||||
|
||||
except:
|
||||
pass
|
||||
@@ -446,9 +594,11 @@ finally:
|
||||
|
||||
ch_client.close()
|
||||
|
||||
print("ClickHouse Connection Closed")
|
||||
print("CLICKHOUSE CONNECTION CLOSED")
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
print("\nETL Finished :", datetime.now())
|
||||
print("\n================================================")
|
||||
print("ETL FINISHED :", datetime.now())
|
||||
print("================================================")
|
||||
|
||||
+209
-177
@@ -5,6 +5,7 @@ import numpy as np
|
||||
from datetime import datetime
|
||||
import traceback
|
||||
import warnings
|
||||
import time
|
||||
|
||||
# =========================================================
|
||||
# IGNORE WARNINGS
|
||||
@@ -14,10 +15,12 @@ warnings.filterwarnings(
|
||||
'pandas only supports SQLAlchemy connectable'
|
||||
)
|
||||
|
||||
print("ETL Started :", datetime.now())
|
||||
print("\n================================================")
|
||||
print("ETL STARTED :", datetime.now())
|
||||
print("================================================")
|
||||
|
||||
# =========================================================
|
||||
# SQL SERVER CONNECTION
|
||||
# SQL SERVER CONNECTION STRING
|
||||
# =========================================================
|
||||
SQL_CONN_STR = (
|
||||
'DRIVER={ODBC Driver 17 for SQL Server};'
|
||||
@@ -26,6 +29,7 @@ SQL_CONN_STR = (
|
||||
'UID=bsgteam_test;'
|
||||
'PWD=B$gt3@m#00512;'
|
||||
'TrustServerCertificate=yes;'
|
||||
'Connection Timeout=60;'
|
||||
)
|
||||
|
||||
# =========================================================
|
||||
@@ -40,31 +44,66 @@ CH_CONFIG = {
|
||||
}
|
||||
|
||||
# =========================================================
|
||||
# TABLE NAME
|
||||
# TABLE DETAILS
|
||||
# =========================================================
|
||||
TABLE_NAME = 'PaidVisibility_Compliance'
|
||||
TABLE_NAME = 'Sales'
|
||||
PROJECT_ID = 41654
|
||||
|
||||
# =========================================================
|
||||
# LOAD SETTINGS
|
||||
# SETTINGS
|
||||
# =========================================================
|
||||
TRUNCATE_BEFORE_LOAD = True
|
||||
table_truncated = False
|
||||
|
||||
# =========================================================
|
||||
# CLICKHOUSE DATE COLUMNS
|
||||
# CHUNK SIZE
|
||||
# =========================================================
|
||||
DATE_COLUMNS = [
|
||||
'visit_date'
|
||||
]
|
||||
chunk_size = 20000
|
||||
|
||||
# =========================================================
|
||||
# CLICKHOUSE DATETIME COLUMNS
|
||||
# CONNECT SQL SERVER
|
||||
# =========================================================
|
||||
DATETIME_COLUMNS = [
|
||||
'create_date',
|
||||
'update_date'
|
||||
]
|
||||
def connect_sql():
|
||||
|
||||
try:
|
||||
|
||||
conn = pyodbc.connect(
|
||||
SQL_CONN_STR,
|
||||
autocommit=True
|
||||
)
|
||||
|
||||
print("Connected SQL Server")
|
||||
|
||||
return conn
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("SQL CONNECTION FAILED")
|
||||
print(str(e))
|
||||
|
||||
raise
|
||||
|
||||
# =========================================================
|
||||
# CONNECT CLICKHOUSE
|
||||
# =========================================================
|
||||
def connect_clickhouse():
|
||||
|
||||
try:
|
||||
|
||||
client = clickhouse_connect.get_client(
|
||||
**CH_CONFIG
|
||||
)
|
||||
|
||||
print("Connected ClickHouse")
|
||||
|
||||
return client
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("CLICKHOUSE CONNECTION FAILED")
|
||||
print(str(e))
|
||||
|
||||
raise
|
||||
|
||||
# =========================================================
|
||||
# CLEAN DATAFRAME
|
||||
@@ -79,7 +118,7 @@ def clean_dataframe(df):
|
||||
df = df.replace({np.nan: None})
|
||||
|
||||
# ---------------------------------------------
|
||||
# Process Columns
|
||||
# Process Column Wise
|
||||
# ---------------------------------------------
|
||||
for col in df.columns:
|
||||
|
||||
@@ -88,13 +127,14 @@ def clean_dataframe(df):
|
||||
print(f"\nCleaning Column : {col}")
|
||||
|
||||
# =====================================
|
||||
# DATE32 COLUMNS
|
||||
# AUTO DETECT DATE / TIME COLUMNS
|
||||
# =====================================
|
||||
if col.lower() in [
|
||||
x.lower() for x in DATE_COLUMNS
|
||||
]:
|
||||
if (
|
||||
'date' in col.lower()
|
||||
or 'time' in col.lower()
|
||||
):
|
||||
|
||||
print(f"Date32 Column : {col}")
|
||||
print(f"Date Column : {col}")
|
||||
|
||||
df[col] = pd.to_datetime(
|
||||
df[col],
|
||||
@@ -111,43 +151,11 @@ def clean_dataframe(df):
|
||||
|
||||
else:
|
||||
|
||||
# IMPORTANT FIX
|
||||
cleaned_dates.append(
|
||||
val.date()
|
||||
)
|
||||
|
||||
df[col] = cleaned_dates
|
||||
|
||||
# =====================================
|
||||
# DATETIME COLUMNS
|
||||
# =====================================
|
||||
elif col.lower() in [
|
||||
x.lower() for x in DATETIME_COLUMNS
|
||||
]:
|
||||
|
||||
print(f"DateTime Column : {col}")
|
||||
|
||||
df[col] = pd.to_datetime(
|
||||
df[col],
|
||||
errors='coerce'
|
||||
)
|
||||
|
||||
cleaned_datetime = []
|
||||
|
||||
for val in df[col]:
|
||||
|
||||
if pd.isnull(val):
|
||||
|
||||
cleaned_datetime.append(None)
|
||||
|
||||
else:
|
||||
|
||||
# IMPORTANT FIX
|
||||
cleaned_datetime.append(
|
||||
val.to_pydatetime()
|
||||
)
|
||||
|
||||
df[col] = cleaned_datetime
|
||||
df[col] = cleaned_dates
|
||||
|
||||
# =====================================
|
||||
# INTEGER COLUMNS
|
||||
@@ -180,22 +188,6 @@ def clean_dataframe(df):
|
||||
errors='coerce'
|
||||
)
|
||||
|
||||
non_null = df[col].dropna()
|
||||
|
||||
# Convert float to int if possible
|
||||
if len(non_null) > 0 and (
|
||||
(non_null % 1 == 0).all()
|
||||
):
|
||||
|
||||
df[col] = df[col].apply(
|
||||
lambda x:
|
||||
int(x)
|
||||
if pd.notnull(x)
|
||||
else None
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
df[col] = df[col].apply(
|
||||
lambda x:
|
||||
float(x)
|
||||
@@ -219,19 +211,27 @@ def clean_dataframe(df):
|
||||
|
||||
cleaned.append(None)
|
||||
|
||||
# STRING
|
||||
elif isinstance(val, str):
|
||||
# DATETIME
|
||||
elif isinstance(
|
||||
val,
|
||||
(
|
||||
datetime,
|
||||
pd.Timestamp
|
||||
)
|
||||
):
|
||||
|
||||
if isinstance(
|
||||
val,
|
||||
pd.Timestamp
|
||||
):
|
||||
|
||||
cleaned.append(
|
||||
val.strip()
|
||||
val.to_pydatetime()
|
||||
)
|
||||
|
||||
# BOOLEAN
|
||||
elif isinstance(val, bool):
|
||||
else:
|
||||
|
||||
cleaned.append(
|
||||
int(val)
|
||||
)
|
||||
cleaned.append(val)
|
||||
|
||||
# INTEGER
|
||||
elif isinstance(
|
||||
@@ -261,50 +261,25 @@ def clean_dataframe(df):
|
||||
|
||||
else:
|
||||
|
||||
if val.is_integer():
|
||||
cleaned.append(
|
||||
float(val)
|
||||
)
|
||||
|
||||
# BOOLEAN
|
||||
elif isinstance(
|
||||
val,
|
||||
bool
|
||||
):
|
||||
|
||||
cleaned.append(
|
||||
int(val)
|
||||
)
|
||||
|
||||
# STRING
|
||||
else:
|
||||
|
||||
cleaned.append(
|
||||
float(val)
|
||||
)
|
||||
|
||||
# DATETIME
|
||||
elif isinstance(
|
||||
val,
|
||||
(
|
||||
datetime,
|
||||
pd.Timestamp
|
||||
)
|
||||
):
|
||||
|
||||
if isinstance(
|
||||
val,
|
||||
pd.Timestamp
|
||||
):
|
||||
|
||||
cleaned.append(
|
||||
val.to_pydatetime()
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
cleaned.append(val)
|
||||
|
||||
# DATE
|
||||
elif hasattr(val, 'year') and hasattr(val, 'month'):
|
||||
|
||||
cleaned.append(val)
|
||||
|
||||
# OTHER
|
||||
else:
|
||||
|
||||
cleaned.append(
|
||||
str(val)
|
||||
str(val).strip()
|
||||
)
|
||||
|
||||
df[col] = cleaned
|
||||
@@ -335,16 +310,12 @@ try:
|
||||
# =====================================================
|
||||
# CONNECT SQL SERVER
|
||||
# =====================================================
|
||||
sql_conn = pyodbc.connect(SQL_CONN_STR)
|
||||
|
||||
print("Connected to SQL Server")
|
||||
sql_conn = connect_sql()
|
||||
|
||||
# =====================================================
|
||||
# CONNECT CLICKHOUSE
|
||||
# =====================================================
|
||||
ch_client = clickhouse_connect.get_client(**CH_CONFIG)
|
||||
|
||||
print("Connected to ClickHouse")
|
||||
ch_client = connect_clickhouse()
|
||||
|
||||
# =====================================================
|
||||
# QUERY
|
||||
@@ -355,19 +326,25 @@ try:
|
||||
WHERE Project_Id = {PROJECT_ID}
|
||||
"""
|
||||
|
||||
print("\nExecuting Query:")
|
||||
print("\nExecuting Query")
|
||||
print(query)
|
||||
|
||||
# =====================================================
|
||||
# CHUNK SIZE
|
||||
# RETRY SETTINGS
|
||||
# =====================================================
|
||||
chunk_size = 100000
|
||||
|
||||
total_rows = 0
|
||||
retry_count = 0
|
||||
max_retry = 5
|
||||
|
||||
# =====================================================
|
||||
# READ DATA
|
||||
# MAIN RETRY LOOP
|
||||
# =====================================================
|
||||
while retry_count < max_retry:
|
||||
|
||||
try:
|
||||
|
||||
# =============================================
|
||||
# READ SQL DATA
|
||||
# =============================================
|
||||
for chunk in pd.read_sql(
|
||||
query,
|
||||
sql_conn,
|
||||
@@ -377,18 +354,24 @@ try:
|
||||
try:
|
||||
|
||||
print("\n================================")
|
||||
print(f"Processing {len(chunk)} Rows")
|
||||
print(
|
||||
f"Processing Rows : "
|
||||
f"{len(chunk)}"
|
||||
)
|
||||
print("================================")
|
||||
|
||||
# =================================================
|
||||
# =====================================
|
||||
# CLEAN DATA
|
||||
# =================================================
|
||||
# =====================================
|
||||
chunk = clean_dataframe(chunk)
|
||||
|
||||
# =================================================
|
||||
# =====================================
|
||||
# DEBUG COLUMN TYPES
|
||||
# =================================================
|
||||
print("\nCOLUMN TYPES")
|
||||
# =====================================
|
||||
print("\nCOLUMN DATATYPES")
|
||||
print(chunk.dtypes)
|
||||
|
||||
print("\nCOLUMN SAMPLE TYPES")
|
||||
|
||||
for col in chunk.columns:
|
||||
|
||||
@@ -402,47 +385,44 @@ try:
|
||||
sample.iloc[0]
|
||||
)
|
||||
|
||||
# =================================================
|
||||
# DEBUG DATE COLUMN
|
||||
# =================================================
|
||||
if 'visit_date' in chunk.columns:
|
||||
|
||||
print("\nvisit_date Sample")
|
||||
print(chunk['visit_date'].head())
|
||||
|
||||
sample = chunk['visit_date'].dropna()
|
||||
|
||||
if len(sample) > 0:
|
||||
|
||||
print(
|
||||
"visit_date datatype:",
|
||||
type(sample.iloc[0])
|
||||
)
|
||||
|
||||
# =================================================
|
||||
# TRUNCATE TABLE
|
||||
# =================================================
|
||||
if TRUNCATE_BEFORE_LOAD and not table_truncated:
|
||||
# =====================================
|
||||
# TRUNCATE TABLE FIRST TIME ONLY
|
||||
# =====================================
|
||||
if (
|
||||
TRUNCATE_BEFORE_LOAD
|
||||
and
|
||||
not table_truncated
|
||||
):
|
||||
|
||||
print("\n================================")
|
||||
print(f"TRUNCATING TABLE : {TABLE_NAME}")
|
||||
print(
|
||||
f"TRUNCATING TABLE : "
|
||||
f"{TABLE_NAME}"
|
||||
)
|
||||
print("================================")
|
||||
|
||||
# IMPORTANT FIX
|
||||
truncate_query = f"""
|
||||
TRUNCATE TABLE
|
||||
{CH_CONFIG['database']}.{TABLE_NAME}
|
||||
`{CH_CONFIG['database']}`.`{TABLE_NAME}`
|
||||
"""
|
||||
|
||||
ch_client.command(truncate_query)
|
||||
ch_client.command(
|
||||
truncate_query
|
||||
)
|
||||
|
||||
print("TABLE TRUNCATED SUCCESSFULLY")
|
||||
print(
|
||||
"TABLE TRUNCATED SUCCESSFULLY"
|
||||
)
|
||||
|
||||
table_truncated = True
|
||||
|
||||
# =================================================
|
||||
# =====================================
|
||||
# INSERT INTO CLICKHOUSE
|
||||
# =================================================
|
||||
print("\nInserting into ClickHouse...")
|
||||
# =====================================
|
||||
print(
|
||||
"\nINSERTING INTO CLICKHOUSE..."
|
||||
)
|
||||
|
||||
ch_client.insert_df(
|
||||
table=TABLE_NAME,
|
||||
@@ -450,27 +430,26 @@ try:
|
||||
database=CH_CONFIG['database']
|
||||
)
|
||||
|
||||
total_rows += len(chunk)
|
||||
|
||||
print(
|
||||
f"\nInserted Total Rows : {total_rows}"
|
||||
f"INSERTED : "
|
||||
f"{len(chunk)} ROWS"
|
||||
)
|
||||
|
||||
except Exception as chunk_error:
|
||||
except Exception as insert_error:
|
||||
|
||||
print("\n================================")
|
||||
print("CHUNK INSERT FAILED")
|
||||
print("INSERT FAILED")
|
||||
print("================================")
|
||||
|
||||
print(str(chunk_error))
|
||||
print(str(insert_error))
|
||||
|
||||
traceback.print_exc()
|
||||
|
||||
# =============================================
|
||||
# =================================
|
||||
# SAVE ERROR LOG
|
||||
# =============================================
|
||||
# =================================
|
||||
with open(
|
||||
"clickhouse_chunk_error.log",
|
||||
"insert_error.log",
|
||||
"a",
|
||||
encoding="utf-8"
|
||||
) as log:
|
||||
@@ -480,15 +459,18 @@ try:
|
||||
)
|
||||
|
||||
log.write(
|
||||
f"\nTIME : {datetime.now()}"
|
||||
f"\nTIME : "
|
||||
f"{datetime.now()}"
|
||||
)
|
||||
|
||||
log.write(
|
||||
f"\nTABLE : {TABLE_NAME}"
|
||||
f"\nTABLE : "
|
||||
f"{TABLE_NAME}"
|
||||
)
|
||||
|
||||
log.write(
|
||||
f"\nERROR : {str(chunk_error)}"
|
||||
f"\nERROR : "
|
||||
f"{str(insert_error)}"
|
||||
)
|
||||
|
||||
log.write(
|
||||
@@ -502,9 +484,56 @@ try:
|
||||
|
||||
continue
|
||||
|
||||
# =============================================
|
||||
# SUCCESS
|
||||
# =============================================
|
||||
break
|
||||
|
||||
# =================================================
|
||||
# SQL CONNECTION FAILURE
|
||||
# =================================================
|
||||
except pyodbc.OperationalError as op_error:
|
||||
|
||||
retry_count += 1
|
||||
|
||||
print("\n================================")
|
||||
print(
|
||||
f"SQL CONNECTION LOST "
|
||||
f"- RETRY {retry_count}"
|
||||
)
|
||||
print("================================")
|
||||
|
||||
print(str(op_error))
|
||||
|
||||
time.sleep(10)
|
||||
|
||||
try:
|
||||
|
||||
sql_conn.close()
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
# RECONNECT SQL
|
||||
sql_conn = connect_sql()
|
||||
|
||||
# =================================================
|
||||
# OTHER ERROR
|
||||
# =================================================
|
||||
except Exception as loop_error:
|
||||
|
||||
print("\n================================")
|
||||
print("MAIN LOOP ERROR")
|
||||
print("================================")
|
||||
|
||||
print(str(loop_error))
|
||||
|
||||
traceback.print_exc()
|
||||
|
||||
break
|
||||
|
||||
print("\n================================")
|
||||
print("ETL COMPLETED SUCCESSFULLY")
|
||||
print(f"TOTAL ROWS INSERTED : {total_rows}")
|
||||
print("================================")
|
||||
|
||||
# =========================================================
|
||||
@@ -521,7 +550,7 @@ except Exception as main_error:
|
||||
traceback.print_exc()
|
||||
|
||||
with open(
|
||||
"clickhouse_main_error.log",
|
||||
"main_error.log",
|
||||
"a",
|
||||
encoding="utf-8"
|
||||
) as log:
|
||||
@@ -555,7 +584,8 @@ finally:
|
||||
try:
|
||||
|
||||
sql_conn.close()
|
||||
print("\nSQL Server Connection Closed")
|
||||
|
||||
print("\nSQL SERVER CONNECTION CLOSED")
|
||||
|
||||
except:
|
||||
pass
|
||||
@@ -563,10 +593,12 @@ finally:
|
||||
try:
|
||||
|
||||
ch_client.close()
|
||||
print("ClickHouse Connection Closed")
|
||||
|
||||
print("CLICKHOUSE CONNECTION CLOSED")
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
print("\nETL Finished :", datetime.now())
|
||||
|
||||
print("\n================================================")
|
||||
print("ETL FINISHED :", datetime.now())
|
||||
print("================================================")
|
||||
|
||||
+604
@@ -0,0 +1,604 @@
|
||||
import pyodbc
|
||||
import pandas as pd
|
||||
import clickhouse_connect
|
||||
import numpy as np
|
||||
from datetime import datetime
|
||||
import traceback
|
||||
import warnings
|
||||
import time
|
||||
|
||||
# =========================================================
|
||||
# IGNORE WARNINGS
|
||||
# =========================================================
|
||||
warnings.filterwarnings(
|
||||
'ignore',
|
||||
'pandas only supports SQLAlchemy connectable'
|
||||
)
|
||||
|
||||
print("\n================================================")
|
||||
print("ETL STARTED :", datetime.now())
|
||||
print("================================================")
|
||||
|
||||
# =========================================================
|
||||
# SQL SERVER CONNECTION STRING
|
||||
# =========================================================
|
||||
SQL_CONN_STR = (
|
||||
'DRIVER={ODBC Driver 17 for SQL Server};'
|
||||
'SERVER=10.200.25.65;'
|
||||
'DATABASE=CPMIndiaBusinessInsight;'
|
||||
'UID=bsgteam_test;'
|
||||
'PWD=B$gt3@m#00512;'
|
||||
'TrustServerCertificate=yes;'
|
||||
'Connection Timeout=60;'
|
||||
)
|
||||
|
||||
# =========================================================
|
||||
# CLICKHOUSE CONFIG
|
||||
# =========================================================
|
||||
CH_CONFIG = {
|
||||
'host': '172.188.12.194',
|
||||
'port': 8123,
|
||||
'username': 'default',
|
||||
'password': 'dipanshu_k',
|
||||
'database': 'DaburIndia_BI'
|
||||
}
|
||||
|
||||
# =========================================================
|
||||
# TABLE DETAILS
|
||||
# =========================================================
|
||||
TABLE_NAME = 'Sales'
|
||||
PROJECT_ID = 41654
|
||||
|
||||
# =========================================================
|
||||
# SETTINGS
|
||||
# =========================================================
|
||||
TRUNCATE_BEFORE_LOAD = True
|
||||
table_truncated = False
|
||||
|
||||
# =========================================================
|
||||
# CHUNK SIZE
|
||||
# =========================================================
|
||||
chunk_size = 10000
|
||||
|
||||
# =========================================================
|
||||
# CONNECT SQL SERVER
|
||||
# =========================================================
|
||||
def connect_sql():
|
||||
|
||||
try:
|
||||
|
||||
conn = pyodbc.connect(
|
||||
SQL_CONN_STR,
|
||||
autocommit=True
|
||||
)
|
||||
|
||||
print("Connected SQL Server")
|
||||
|
||||
return conn
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("SQL CONNECTION FAILED")
|
||||
print(str(e))
|
||||
|
||||
raise
|
||||
|
||||
# =========================================================
|
||||
# CONNECT CLICKHOUSE
|
||||
# =========================================================
|
||||
def connect_clickhouse():
|
||||
|
||||
try:
|
||||
|
||||
client = clickhouse_connect.get_client(
|
||||
**CH_CONFIG
|
||||
)
|
||||
|
||||
print("Connected ClickHouse")
|
||||
|
||||
return client
|
||||
|
||||
except Exception as e:
|
||||
|
||||
print("CLICKHOUSE CONNECTION FAILED")
|
||||
print(str(e))
|
||||
|
||||
raise
|
||||
|
||||
# =========================================================
|
||||
# CLEAN DATAFRAME
|
||||
# =========================================================
|
||||
def clean_dataframe(df):
|
||||
|
||||
try:
|
||||
|
||||
# ---------------------------------------------
|
||||
# Replace NaN
|
||||
# ---------------------------------------------
|
||||
df = df.replace({np.nan: None})
|
||||
|
||||
# ---------------------------------------------
|
||||
# Process Column Wise
|
||||
# ---------------------------------------------
|
||||
for col in df.columns:
|
||||
|
||||
try:
|
||||
|
||||
print(f"\nCleaning Column : {col}")
|
||||
|
||||
# =====================================
|
||||
# AUTO DETECT DATE / TIME COLUMNS
|
||||
# =====================================
|
||||
if (
|
||||
'date' in col.lower()
|
||||
or 'time' in col.lower()
|
||||
):
|
||||
|
||||
print(f"Date Column : {col}")
|
||||
|
||||
df[col] = pd.to_datetime(
|
||||
df[col],
|
||||
errors='coerce'
|
||||
)
|
||||
|
||||
cleaned_dates = []
|
||||
|
||||
for val in df[col]:
|
||||
|
||||
if pd.isnull(val):
|
||||
|
||||
cleaned_dates.append(None)
|
||||
|
||||
else:
|
||||
|
||||
cleaned_dates.append(
|
||||
val.to_pydatetime()
|
||||
)
|
||||
|
||||
df[col] = cleaned_dates
|
||||
|
||||
# =====================================
|
||||
# INTEGER COLUMNS
|
||||
# =====================================
|
||||
elif pd.api.types.is_integer_dtype(df[col]):
|
||||
|
||||
print(f"Integer Column : {col}")
|
||||
|
||||
df[col] = pd.to_numeric(
|
||||
df[col],
|
||||
errors='coerce'
|
||||
)
|
||||
|
||||
df[col] = df[col].apply(
|
||||
lambda x:
|
||||
int(x)
|
||||
if pd.notnull(x)
|
||||
else None
|
||||
)
|
||||
|
||||
# =====================================
|
||||
# FLOAT COLUMNS
|
||||
# =====================================
|
||||
elif pd.api.types.is_float_dtype(df[col]):
|
||||
|
||||
print(f"Float Column : {col}")
|
||||
|
||||
df[col] = pd.to_numeric(
|
||||
df[col],
|
||||
errors='coerce'
|
||||
)
|
||||
|
||||
df[col] = df[col].apply(
|
||||
lambda x:
|
||||
float(x)
|
||||
if pd.notnull(x)
|
||||
else None
|
||||
)
|
||||
|
||||
# =====================================
|
||||
# OBJECT / STRING COLUMNS
|
||||
# =====================================
|
||||
else:
|
||||
|
||||
print(f"String/Object Column : {col}")
|
||||
|
||||
cleaned = []
|
||||
|
||||
for val in df[col]:
|
||||
|
||||
# NULL
|
||||
if pd.isnull(val):
|
||||
|
||||
cleaned.append(None)
|
||||
|
||||
# DATETIME
|
||||
elif isinstance(
|
||||
val,
|
||||
(
|
||||
datetime,
|
||||
pd.Timestamp
|
||||
)
|
||||
):
|
||||
|
||||
if isinstance(
|
||||
val,
|
||||
pd.Timestamp
|
||||
):
|
||||
|
||||
cleaned.append(
|
||||
val.to_pydatetime()
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
cleaned.append(val)
|
||||
|
||||
# INTEGER
|
||||
elif isinstance(
|
||||
val,
|
||||
(
|
||||
int,
|
||||
np.integer
|
||||
)
|
||||
):
|
||||
|
||||
cleaned.append(
|
||||
int(val)
|
||||
)
|
||||
|
||||
# FLOAT
|
||||
elif isinstance(
|
||||
val,
|
||||
(
|
||||
float,
|
||||
np.floating
|
||||
)
|
||||
):
|
||||
|
||||
if np.isnan(val):
|
||||
|
||||
cleaned.append(None)
|
||||
|
||||
else:
|
||||
|
||||
cleaned.append(
|
||||
float(val)
|
||||
)
|
||||
|
||||
# BOOLEAN
|
||||
elif isinstance(
|
||||
val,
|
||||
bool
|
||||
):
|
||||
|
||||
cleaned.append(
|
||||
int(val)
|
||||
)
|
||||
|
||||
# STRING
|
||||
else:
|
||||
|
||||
cleaned.append(
|
||||
str(val).strip()
|
||||
)
|
||||
|
||||
df[col] = cleaned
|
||||
|
||||
except Exception as col_error:
|
||||
|
||||
print("\n================================")
|
||||
print(f"COLUMN FAILED : {col}")
|
||||
print(str(col_error))
|
||||
print("================================")
|
||||
|
||||
return df
|
||||
|
||||
except Exception as clean_error:
|
||||
|
||||
print("\n================================")
|
||||
print("DATA CLEAN FAILED")
|
||||
print(str(clean_error))
|
||||
print("================================")
|
||||
|
||||
return df
|
||||
|
||||
# =========================================================
|
||||
# MAIN PROCESS
|
||||
# =========================================================
|
||||
try:
|
||||
|
||||
# =====================================================
|
||||
# CONNECT SQL SERVER
|
||||
# =====================================================
|
||||
sql_conn = connect_sql()
|
||||
|
||||
# =====================================================
|
||||
# CONNECT CLICKHOUSE
|
||||
# =====================================================
|
||||
ch_client = connect_clickhouse()
|
||||
|
||||
# =====================================================
|
||||
# QUERY
|
||||
# =====================================================
|
||||
query = f"""
|
||||
SELECT *
|
||||
FROM dbo.[{TABLE_NAME}]
|
||||
WHERE Project_Id = {PROJECT_ID}
|
||||
"""
|
||||
|
||||
print("\nExecuting Query")
|
||||
print(query)
|
||||
|
||||
# =====================================================
|
||||
# RETRY SETTINGS
|
||||
# =====================================================
|
||||
retry_count = 0
|
||||
max_retry = 5
|
||||
|
||||
# =====================================================
|
||||
# MAIN RETRY LOOP
|
||||
# =====================================================
|
||||
while retry_count < max_retry:
|
||||
|
||||
try:
|
||||
|
||||
# =============================================
|
||||
# READ SQL DATA
|
||||
# =============================================
|
||||
for chunk in pd.read_sql(
|
||||
query,
|
||||
sql_conn,
|
||||
chunksize=chunk_size
|
||||
):
|
||||
|
||||
try:
|
||||
|
||||
print("\n================================")
|
||||
print(
|
||||
f"Processing Rows : "
|
||||
f"{len(chunk)}"
|
||||
)
|
||||
print("================================")
|
||||
|
||||
# =====================================
|
||||
# CLEAN DATA
|
||||
# =====================================
|
||||
chunk = clean_dataframe(chunk)
|
||||
|
||||
# =====================================
|
||||
# DEBUG COLUMN TYPES
|
||||
# =====================================
|
||||
print("\nCOLUMN DATATYPES")
|
||||
print(chunk.dtypes)
|
||||
|
||||
print("\nCOLUMN SAMPLE TYPES")
|
||||
|
||||
for col in chunk.columns:
|
||||
|
||||
sample = chunk[col].dropna()
|
||||
|
||||
if len(sample) > 0:
|
||||
|
||||
print(
|
||||
col,
|
||||
type(sample.iloc[0]),
|
||||
sample.iloc[0]
|
||||
)
|
||||
|
||||
# =====================================
|
||||
# TRUNCATE TABLE FIRST TIME ONLY
|
||||
# =====================================
|
||||
if (
|
||||
TRUNCATE_BEFORE_LOAD
|
||||
and
|
||||
not table_truncated
|
||||
):
|
||||
|
||||
print("\n================================")
|
||||
print(
|
||||
f"TRUNCATING TABLE : "
|
||||
f"{TABLE_NAME}"
|
||||
)
|
||||
print("================================")
|
||||
|
||||
# IMPORTANT FIX
|
||||
truncate_query = f"""
|
||||
TRUNCATE TABLE
|
||||
`{CH_CONFIG['database']}`.`{TABLE_NAME}`
|
||||
"""
|
||||
|
||||
ch_client.command(
|
||||
truncate_query
|
||||
)
|
||||
|
||||
print(
|
||||
"TABLE TRUNCATED SUCCESSFULLY"
|
||||
)
|
||||
|
||||
table_truncated = True
|
||||
|
||||
# =====================================
|
||||
# INSERT INTO CLICKHOUSE
|
||||
# =====================================
|
||||
print(
|
||||
"\nINSERTING INTO CLICKHOUSE..."
|
||||
)
|
||||
|
||||
ch_client.insert_df(
|
||||
table=TABLE_NAME,
|
||||
df=chunk,
|
||||
database=CH_CONFIG['database']
|
||||
)
|
||||
|
||||
print(
|
||||
f"INSERTED : "
|
||||
f"{len(chunk)} ROWS"
|
||||
)
|
||||
|
||||
except Exception as insert_error:
|
||||
|
||||
print("\n================================")
|
||||
print("INSERT FAILED")
|
||||
print("================================")
|
||||
|
||||
print(str(insert_error))
|
||||
|
||||
traceback.print_exc()
|
||||
|
||||
# =================================
|
||||
# SAVE ERROR LOG
|
||||
# =================================
|
||||
with open(
|
||||
"insert_error.log",
|
||||
"a",
|
||||
encoding="utf-8"
|
||||
) as log:
|
||||
|
||||
log.write(
|
||||
"\n\n================================"
|
||||
)
|
||||
|
||||
log.write(
|
||||
f"\nTIME : "
|
||||
f"{datetime.now()}"
|
||||
)
|
||||
|
||||
log.write(
|
||||
f"\nTABLE : "
|
||||
f"{TABLE_NAME}"
|
||||
)
|
||||
|
||||
log.write(
|
||||
f"\nERROR : "
|
||||
f"{str(insert_error)}"
|
||||
)
|
||||
|
||||
log.write(
|
||||
f"\nTRACEBACK :\n"
|
||||
f"{traceback.format_exc()}"
|
||||
)
|
||||
|
||||
log.write(
|
||||
"\n================================"
|
||||
)
|
||||
|
||||
continue
|
||||
|
||||
# =============================================
|
||||
# SUCCESS
|
||||
# =============================================
|
||||
break
|
||||
|
||||
# =================================================
|
||||
# SQL CONNECTION FAILURE
|
||||
# =================================================
|
||||
except pyodbc.OperationalError as op_error:
|
||||
|
||||
retry_count += 1
|
||||
|
||||
print("\n================================")
|
||||
print(
|
||||
f"SQL CONNECTION LOST "
|
||||
f"- RETRY {retry_count}"
|
||||
)
|
||||
print("================================")
|
||||
|
||||
print(str(op_error))
|
||||
|
||||
time.sleep(10)
|
||||
|
||||
try:
|
||||
|
||||
sql_conn.close()
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
# RECONNECT SQL
|
||||
sql_conn = connect_sql()
|
||||
|
||||
# =================================================
|
||||
# OTHER ERROR
|
||||
# =================================================
|
||||
except Exception as loop_error:
|
||||
|
||||
print("\n================================")
|
||||
print("MAIN LOOP ERROR")
|
||||
print("================================")
|
||||
|
||||
print(str(loop_error))
|
||||
|
||||
traceback.print_exc()
|
||||
|
||||
break
|
||||
|
||||
print("\n================================")
|
||||
print("ETL COMPLETED SUCCESSFULLY")
|
||||
print("================================")
|
||||
|
||||
# =========================================================
|
||||
# MAIN ERROR
|
||||
# =========================================================
|
||||
except Exception as main_error:
|
||||
|
||||
print("\n================================")
|
||||
print("MAIN ERROR")
|
||||
print("================================")
|
||||
|
||||
print(str(main_error))
|
||||
|
||||
traceback.print_exc()
|
||||
|
||||
with open(
|
||||
"main_error.log",
|
||||
"a",
|
||||
encoding="utf-8"
|
||||
) as log:
|
||||
|
||||
log.write(
|
||||
"\n\n================================"
|
||||
)
|
||||
|
||||
log.write(
|
||||
f"\nTIME : {datetime.now()}"
|
||||
)
|
||||
|
||||
log.write(
|
||||
f"\nERROR : {str(main_error)}"
|
||||
)
|
||||
|
||||
log.write(
|
||||
f"\nTRACEBACK :\n"
|
||||
f"{traceback.format_exc()}"
|
||||
)
|
||||
|
||||
log.write(
|
||||
"\n================================"
|
||||
)
|
||||
|
||||
# =========================================================
|
||||
# CLOSE CONNECTIONS
|
||||
# =========================================================
|
||||
finally:
|
||||
|
||||
try:
|
||||
|
||||
sql_conn.close()
|
||||
|
||||
print("\nSQL SERVER CONNECTION CLOSED")
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
|
||||
ch_client.close()
|
||||
|
||||
print("CLICKHOUSE CONNECTION CLOSED")
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
print("\n================================================")
|
||||
print("ETL FINISHED :", datetime.now())
|
||||
print("================================================")
|
||||
@@ -10006,3 +10006,286 @@ clickhouse_connect.driver.exceptions.DatabaseError: Received ClickHouse exceptio
|
||||
. Expected one of: ParserArrayOfJSONIdentifierDelimiter, token sequence, OpeningSquareBracket, Dot, token, UUID, Comma, ON, NO DELAY, SYNC, INTO OUTFILE, FORMAT, SETTINGS, ParallelWithClause, PARALLEL WITH, end of query. (SYNTAX_ERROR) (for url http://172.188.12.194:8123)
|
||||
|
||||
================================
|
||||
|
||||
================================
|
||||
TIME : 2026-05-19 17:34:04.760816
|
||||
TABLE : Sales
|
||||
ERROR : unsupported operand type(s) for -: 'str' and 'datetime.date'
|
||||
TRACEBACK :
|
||||
Traceback (most recent call last):
|
||||
File "d:\Python Code\Sales_Import.py", line 426, in <module>
|
||||
ch_client.insert_df(
|
||||
~~~~~~~~~~~~~~~~~~~^
|
||||
table=f"`{TABLE_NAME}`",
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
df=chunk,
|
||||
^^^^^^^^^
|
||||
database=CH_CONFIG['database']
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\client.py", line 1013, in insert_df
|
||||
return self.insert(table,
|
||||
~~~~~~~~~~~^^^^^^^
|
||||
df,
|
||||
^^^
|
||||
...<5 lines>...
|
||||
transport_settings=transport_settings,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
context=context)
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\client.py", line 978, in insert
|
||||
return self.data_insert(context)
|
||||
~~~~~~~~~~~~~~~~^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 347, in data_insert
|
||||
response = self._raw_request(block_gen, params, headers, error_handler=error_handler, server_wait=False)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 569, in _raw_request
|
||||
error_handler(response)
|
||||
~~~~~~~~~~~~~^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 331, in error_handler
|
||||
raise ex
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\transform.py", line 114, in chunk_gen
|
||||
col_type.write_column(data, output, context)
|
||||
~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\datatypes\base.py", line 216, in write_column
|
||||
self.write_column_data(column, dest, ctx)
|
||||
~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\datatypes\base.py", line 231, in write_column_data
|
||||
self._write_column_binary(column, dest, ctx)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\datatypes\temporal.py", line 60, in _write_column_binary
|
||||
column = [(x - esd).days for x in column]
|
||||
~~^~~~~
|
||||
TypeError: unsupported operand type(s) for -: 'str' and 'datetime.date'
|
||||
|
||||
================================
|
||||
|
||||
================================
|
||||
TIME : 2026-05-20 10:42:59.946046
|
||||
TABLE : PaidVisibility_Compliance
|
||||
ERROR : Error ('Connection aborted.', TimeoutError('timed out')) executing HTTP request attempt 1 (http://172.188.12.194:8123)
|
||||
TRACEBACK :
|
||||
Traceback (most recent call last):
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 788, in urlopen
|
||||
response = self._make_request(
|
||||
conn,
|
||||
...<10 lines>...
|
||||
**response_kw,
|
||||
)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 493, in _make_request
|
||||
conn.request(
|
||||
~~~~~~~~~~~~^
|
||||
method,
|
||||
^^^^^^^
|
||||
...<6 lines>...
|
||||
enforce_content_length=enforce_content_length,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connection.py", line 512, in request
|
||||
self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk))
|
||||
~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\http\client.py", line 1086, in send
|
||||
self.sock.sendall(data)
|
||||
~~~~~~~~~~~~~~~~~^^^^^^
|
||||
TimeoutError: timed out
|
||||
|
||||
During handling of the above exception, another exception occurred:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 546, in _raw_request
|
||||
response = self.http.request(method, url, **kwargs)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\_request_methods.py", line 143, in request
|
||||
return self.request_encode_body(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
method, url, fields=fields, headers=headers, **urlopen_kw
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\_request_methods.py", line 278, in request_encode_body
|
||||
return self.urlopen(method, url, **extra_kw)
|
||||
~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\poolmanager.py", line 457, in urlopen
|
||||
response = conn.urlopen(method, u.request_uri, **kw)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 842, in urlopen
|
||||
retries = retries.increment(
|
||||
method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]
|
||||
)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\util\retry.py", line 498, in increment
|
||||
raise reraise(type(error), error, _stacktrace)
|
||||
~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\util\util.py", line 38, in reraise
|
||||
raise value.with_traceback(tb)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 788, in urlopen
|
||||
response = self._make_request(
|
||||
conn,
|
||||
...<10 lines>...
|
||||
**response_kw,
|
||||
)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 493, in _make_request
|
||||
conn.request(
|
||||
~~~~~~~~~~~~^
|
||||
method,
|
||||
^^^^^^^
|
||||
...<6 lines>...
|
||||
enforce_content_length=enforce_content_length,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connection.py", line 512, in request
|
||||
self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk))
|
||||
~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\http\client.py", line 1086, in send
|
||||
self.sock.sendall(data)
|
||||
~~~~~~~~~~~~~~~~~^^^^^^
|
||||
urllib3.exceptions.ProtocolError: ('Connection aborted.', TimeoutError('timed out'))
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "d:\Python Code\PaidVisibility_Compliance Import.py", line 447, in <module>
|
||||
ch_client.insert_df(
|
||||
~~~~~~~~~~~~~~~~~~~^
|
||||
table=TABLE_NAME,
|
||||
^^^^^^^^^^^^^^^^^
|
||||
df=chunk,
|
||||
^^^^^^^^^
|
||||
database=CH_CONFIG['database']
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\client.py", line 1013, in insert_df
|
||||
return self.insert(table,
|
||||
~~~~~~~~~~~^^^^^^^
|
||||
df,
|
||||
^^^
|
||||
...<5 lines>...
|
||||
transport_settings=transport_settings,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
context=context)
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\client.py", line 978, in insert
|
||||
return self.data_insert(context)
|
||||
~~~~~~~~~~~~~~~~^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 347, in data_insert
|
||||
response = self._raw_request(block_gen, params, headers, error_handler=error_handler, server_wait=False)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 558, in _raw_request
|
||||
raise OperationalError(f'Error {ex} executing HTTP request attempt {attempts}{err_url}') from ex
|
||||
clickhouse_connect.driver.exceptions.OperationalError: Error ('Connection aborted.', TimeoutError('timed out')) executing HTTP request attempt 1 (http://172.188.12.194:8123)
|
||||
|
||||
================================
|
||||
|
||||
================================
|
||||
TIME : 2026-05-20 10:43:26.819553
|
||||
TABLE : PaidVisibility_Compliance
|
||||
ERROR : Error ('Connection aborted.', TimeoutError('timed out')) executing HTTP request attempt 1 (http://172.188.12.194:8123)
|
||||
TRACEBACK :
|
||||
Traceback (most recent call last):
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 788, in urlopen
|
||||
response = self._make_request(
|
||||
conn,
|
||||
...<10 lines>...
|
||||
**response_kw,
|
||||
)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 493, in _make_request
|
||||
conn.request(
|
||||
~~~~~~~~~~~~^
|
||||
method,
|
||||
^^^^^^^
|
||||
...<6 lines>...
|
||||
enforce_content_length=enforce_content_length,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connection.py", line 512, in request
|
||||
self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk))
|
||||
~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\http\client.py", line 1086, in send
|
||||
self.sock.sendall(data)
|
||||
~~~~~~~~~~~~~~~~~^^^^^^
|
||||
TimeoutError: timed out
|
||||
|
||||
During handling of the above exception, another exception occurred:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 546, in _raw_request
|
||||
response = self.http.request(method, url, **kwargs)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\_request_methods.py", line 143, in request
|
||||
return self.request_encode_body(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
method, url, fields=fields, headers=headers, **urlopen_kw
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\_request_methods.py", line 278, in request_encode_body
|
||||
return self.urlopen(method, url, **extra_kw)
|
||||
~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\poolmanager.py", line 457, in urlopen
|
||||
response = conn.urlopen(method, u.request_uri, **kw)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 842, in urlopen
|
||||
retries = retries.increment(
|
||||
method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]
|
||||
)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\util\retry.py", line 498, in increment
|
||||
raise reraise(type(error), error, _stacktrace)
|
||||
~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\util\util.py", line 38, in reraise
|
||||
raise value.with_traceback(tb)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 788, in urlopen
|
||||
response = self._make_request(
|
||||
conn,
|
||||
...<10 lines>...
|
||||
**response_kw,
|
||||
)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 493, in _make_request
|
||||
conn.request(
|
||||
~~~~~~~~~~~~^
|
||||
method,
|
||||
^^^^^^^
|
||||
...<6 lines>...
|
||||
enforce_content_length=enforce_content_length,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connection.py", line 512, in request
|
||||
self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk))
|
||||
~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\http\client.py", line 1086, in send
|
||||
self.sock.sendall(data)
|
||||
~~~~~~~~~~~~~~~~~^^^^^^
|
||||
urllib3.exceptions.ProtocolError: ('Connection aborted.', TimeoutError('timed out'))
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "d:\Python Code\PaidVisibility_Compliance Import.py", line 447, in <module>
|
||||
ch_client.insert_df(
|
||||
~~~~~~~~~~~~~~~~~~~^
|
||||
table=TABLE_NAME,
|
||||
^^^^^^^^^^^^^^^^^
|
||||
df=chunk,
|
||||
^^^^^^^^^
|
||||
database=CH_CONFIG['database']
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\client.py", line 1013, in insert_df
|
||||
return self.insert(table,
|
||||
~~~~~~~~~~~^^^^^^^
|
||||
df,
|
||||
^^^
|
||||
...<5 lines>...
|
||||
transport_settings=transport_settings,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
context=context)
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\client.py", line 978, in insert
|
||||
return self.data_insert(context)
|
||||
~~~~~~~~~~~~~~~~^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 347, in data_insert
|
||||
response = self._raw_request(block_gen, params, headers, error_handler=error_handler, server_wait=False)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 558, in _raw_request
|
||||
raise OperationalError(f'Error {ex} executing HTTP request attempt {attempts}{err_url}') from ex
|
||||
clickhouse_connect.driver.exceptions.OperationalError: Error ('Connection aborted.', TimeoutError('timed out')) executing HTTP request attempt 1 (http://172.188.12.194:8123)
|
||||
|
||||
================================
|
||||
@@ -0,0 +1,231 @@
|
||||
|
||||
|
||||
================================
|
||||
TIME : 2026-05-19 17:13:53.695971
|
||||
TABLE : Sales
|
||||
ERROR : Error ('Connection aborted.', TimeoutError('timed out')) executing HTTP request attempt 1 (http://172.188.12.194:8123)
|
||||
TRACEBACK :
|
||||
Traceback (most recent call last):
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 788, in urlopen
|
||||
response = self._make_request(
|
||||
conn,
|
||||
...<10 lines>...
|
||||
**response_kw,
|
||||
)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 493, in _make_request
|
||||
conn.request(
|
||||
~~~~~~~~~~~~^
|
||||
method,
|
||||
^^^^^^^
|
||||
...<6 lines>...
|
||||
enforce_content_length=enforce_content_length,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connection.py", line 512, in request
|
||||
self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk))
|
||||
~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\http\client.py", line 1086, in send
|
||||
self.sock.sendall(data)
|
||||
~~~~~~~~~~~~~~~~~^^^^^^
|
||||
TimeoutError: timed out
|
||||
|
||||
During handling of the above exception, another exception occurred:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 546, in _raw_request
|
||||
response = self.http.request(method, url, **kwargs)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\_request_methods.py", line 143, in request
|
||||
return self.request_encode_body(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
method, url, fields=fields, headers=headers, **urlopen_kw
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\_request_methods.py", line 278, in request_encode_body
|
||||
return self.urlopen(method, url, **extra_kw)
|
||||
~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\poolmanager.py", line 457, in urlopen
|
||||
response = conn.urlopen(method, u.request_uri, **kw)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 842, in urlopen
|
||||
retries = retries.increment(
|
||||
method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]
|
||||
)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\util\retry.py", line 498, in increment
|
||||
raise reraise(type(error), error, _stacktrace)
|
||||
~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\util\util.py", line 38, in reraise
|
||||
raise value.with_traceback(tb)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 788, in urlopen
|
||||
response = self._make_request(
|
||||
conn,
|
||||
...<10 lines>...
|
||||
**response_kw,
|
||||
)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 493, in _make_request
|
||||
conn.request(
|
||||
~~~~~~~~~~~~^
|
||||
method,
|
||||
^^^^^^^
|
||||
...<6 lines>...
|
||||
enforce_content_length=enforce_content_length,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connection.py", line 512, in request
|
||||
self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk))
|
||||
~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\http\client.py", line 1086, in send
|
||||
self.sock.sendall(data)
|
||||
~~~~~~~~~~~~~~~~~^^^^^^
|
||||
urllib3.exceptions.ProtocolError: ('Connection aborted.', TimeoutError('timed out'))
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "d:\Python Code\Sales_Import.py", line 494, in <module>
|
||||
ch_client.insert_df(
|
||||
~~~~~~~~~~~~~~~~~~~^
|
||||
table=f"`{TABLE_NAME}`",
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
df=chunk,
|
||||
^^^^^^^^^
|
||||
database=CH_CONFIG['database']
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\client.py", line 1013, in insert_df
|
||||
return self.insert(table,
|
||||
~~~~~~~~~~~^^^^^^^
|
||||
df,
|
||||
^^^
|
||||
...<5 lines>...
|
||||
transport_settings=transport_settings,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
context=context)
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\client.py", line 978, in insert
|
||||
return self.data_insert(context)
|
||||
~~~~~~~~~~~~~~~~^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 347, in data_insert
|
||||
response = self._raw_request(block_gen, params, headers, error_handler=error_handler, server_wait=False)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 558, in _raw_request
|
||||
raise OperationalError(f'Error {ex} executing HTTP request attempt {attempts}{err_url}') from ex
|
||||
clickhouse_connect.driver.exceptions.OperationalError: Error ('Connection aborted.', TimeoutError('timed out')) executing HTTP request attempt 1 (http://172.188.12.194:8123)
|
||||
|
||||
================================
|
||||
|
||||
================================
|
||||
TIME : 2026-05-19 17:15:26.425862
|
||||
TABLE : Sales
|
||||
ERROR : Error ('Connection aborted.', TimeoutError('timed out')) executing HTTP request attempt 1 (http://172.188.12.194:8123)
|
||||
TRACEBACK :
|
||||
Traceback (most recent call last):
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 788, in urlopen
|
||||
response = self._make_request(
|
||||
conn,
|
||||
...<10 lines>...
|
||||
**response_kw,
|
||||
)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 493, in _make_request
|
||||
conn.request(
|
||||
~~~~~~~~~~~~^
|
||||
method,
|
||||
^^^^^^^
|
||||
...<6 lines>...
|
||||
enforce_content_length=enforce_content_length,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connection.py", line 512, in request
|
||||
self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk))
|
||||
~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\http\client.py", line 1086, in send
|
||||
self.sock.sendall(data)
|
||||
~~~~~~~~~~~~~~~~~^^^^^^
|
||||
TimeoutError: timed out
|
||||
|
||||
During handling of the above exception, another exception occurred:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 546, in _raw_request
|
||||
response = self.http.request(method, url, **kwargs)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\_request_methods.py", line 143, in request
|
||||
return self.request_encode_body(
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
method, url, fields=fields, headers=headers, **urlopen_kw
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\_request_methods.py", line 278, in request_encode_body
|
||||
return self.urlopen(method, url, **extra_kw)
|
||||
~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\poolmanager.py", line 457, in urlopen
|
||||
response = conn.urlopen(method, u.request_uri, **kw)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 842, in urlopen
|
||||
retries = retries.increment(
|
||||
method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]
|
||||
)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\util\retry.py", line 498, in increment
|
||||
raise reraise(type(error), error, _stacktrace)
|
||||
~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\util\util.py", line 38, in reraise
|
||||
raise value.with_traceback(tb)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 788, in urlopen
|
||||
response = self._make_request(
|
||||
conn,
|
||||
...<10 lines>...
|
||||
**response_kw,
|
||||
)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connectionpool.py", line 493, in _make_request
|
||||
conn.request(
|
||||
~~~~~~~~~~~~^
|
||||
method,
|
||||
^^^^^^^
|
||||
...<6 lines>...
|
||||
enforce_content_length=enforce_content_length,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\urllib3\connection.py", line 512, in request
|
||||
self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk))
|
||||
~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\http\client.py", line 1086, in send
|
||||
self.sock.sendall(data)
|
||||
~~~~~~~~~~~~~~~~~^^^^^^
|
||||
urllib3.exceptions.ProtocolError: ('Connection aborted.', TimeoutError('timed out'))
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "d:\Python Code\Sales_Import.py", line 494, in <module>
|
||||
ch_client.insert_df(
|
||||
~~~~~~~~~~~~~~~~~~~^
|
||||
table=f"`{TABLE_NAME}`",
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
df=chunk,
|
||||
^^^^^^^^^
|
||||
database=CH_CONFIG['database']
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\client.py", line 1013, in insert_df
|
||||
return self.insert(table,
|
||||
~~~~~~~~~~~^^^^^^^
|
||||
df,
|
||||
^^^
|
||||
...<5 lines>...
|
||||
transport_settings=transport_settings,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
context=context)
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\client.py", line 978, in insert
|
||||
return self.data_insert(context)
|
||||
~~~~~~~~~~~~~~~~^^^^^^^^^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 347, in data_insert
|
||||
response = self._raw_request(block_gen, params, headers, error_handler=error_handler, server_wait=False)
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\clickhouse_connect\driver\httpclient.py", line 558, in _raw_request
|
||||
raise OperationalError(f'Error {ex} executing HTTP request attempt {attempts}{err_url}') from ex
|
||||
clickhouse_connect.driver.exceptions.OperationalError: Error ('Connection aborted.', TimeoutError('timed out')) executing HTTP request attempt 1 (http://172.188.12.194:8123)
|
||||
|
||||
================================
|
||||
@@ -167,3 +167,25 @@ Traceback (most recent call last):
|
||||
pyodbc.OperationalError: ('08S01', '[08S01] [Microsoft][ODBC Driver 17 for SQL Server]TCP Provider: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond.\r\n (10060) (SQLGetData); [08S01] [Microsoft][ODBC Driver 17 for SQL Server]Communication link failure (10060)')
|
||||
|
||||
================================
|
||||
|
||||
================================
|
||||
TIME : 2026-05-19 17:16:08.366780
|
||||
ERROR : ('08S01', '[08S01] [Microsoft][ODBC Driver 17 for SQL Server]TCP Provider: An existing connection was forcibly closed by the remote host.\r\n (10054) (SQLGetData); [08S01] [Microsoft][ODBC Driver 17 for SQL Server]Communication link failure (10054)')
|
||||
TRACEBACK :
|
||||
Traceback (most recent call last):
|
||||
File "d:\Python Code\Sales_Import.py", line 411, in <module>
|
||||
for chunk in pd.read_sql(
|
||||
~~~~~~~~~~~^
|
||||
query,
|
||||
^^^^^^
|
||||
sql_conn,
|
||||
^^^^^^^^^
|
||||
chunksize=chunk_size
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
):
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\pandas\io\sql.py", line 2730, in _query_iterator
|
||||
data = cursor.fetchmany(chunksize)
|
||||
pyodbc.OperationalError: ('08S01', '[08S01] [Microsoft][ODBC Driver 17 for SQL Server]TCP Provider: An existing connection was forcibly closed by the remote host.\r\n (10054) (SQLGetData); [08S01] [Microsoft][ODBC Driver 17 for SQL Server]Communication link failure (10054)')
|
||||
|
||||
================================
|
||||
+13248
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,85 @@
|
||||
|
||||
|
||||
================================
|
||||
TIME : 2026-05-20 09:30:27.532149
|
||||
ERROR : ('08001', '[08001] [Microsoft][ODBC Driver 17 for SQL Server]Named Pipes Provider: Could not open a connection to SQL Server [2]. (2) (SQLDriverConnect); [08001] [Microsoft][ODBC Driver 17 for SQL Server]Login timeout expired (0); [08001] [Microsoft][ODBC Driver 17 for SQL Server]Invalid connection string attribute (0); [08001] [Microsoft][ODBC Driver 17 for SQL Server]A network-related or instance-specific error has occurred while establishing a connection to SQL Server. Server is not found or not accessible. Check if instance name is correct and if SQL Server is configured to allow remote connections. For more information see SQL Server Books Online. (2)')
|
||||
TRACEBACK :
|
||||
Traceback (most recent call last):
|
||||
File "d:\Python Code\PaidVisibility_Compliance Import.py", line 313, in <module>
|
||||
sql_conn = connect_sql()
|
||||
File "d:\Python Code\PaidVisibility_Compliance Import.py", line 70, in connect_sql
|
||||
conn = pyodbc.connect(
|
||||
SQL_CONN_STR,
|
||||
autocommit=True
|
||||
)
|
||||
pyodbc.OperationalError: ('08001', '[08001] [Microsoft][ODBC Driver 17 for SQL Server]Named Pipes Provider: Could not open a connection to SQL Server [2]. (2) (SQLDriverConnect); [08001] [Microsoft][ODBC Driver 17 for SQL Server]Login timeout expired (0); [08001] [Microsoft][ODBC Driver 17 for SQL Server]Invalid connection string attribute (0); [08001] [Microsoft][ODBC Driver 17 for SQL Server]A network-related or instance-specific error has occurred while establishing a connection to SQL Server. Server is not found or not accessible. Check if instance name is correct and if SQL Server is configured to allow remote connections. For more information see SQL Server Books Online. (2)')
|
||||
|
||||
================================
|
||||
|
||||
================================
|
||||
TIME : 2026-05-20 10:38:12.600484
|
||||
ERROR : ('08001', '[08001] [Microsoft][ODBC Driver 17 for SQL Server]Named Pipes Provider: Could not open a connection to SQL Server [64]. (64) (SQLDriverConnect); [08001] [Microsoft][ODBC Driver 17 for SQL Server]Login timeout expired (0); [08001] [Microsoft][ODBC Driver 17 for SQL Server]Invalid connection string attribute (0); [08001] [Microsoft][ODBC Driver 17 for SQL Server]A network-related or instance-specific error has occurred while establishing a connection to SQL Server. Server is not found or not accessible. Check if instance name is correct and if SQL Server is configured to allow remote connections. For more information see SQL Server Books Online. (64)')
|
||||
TRACEBACK :
|
||||
Traceback (most recent call last):
|
||||
File "d:\Python Code\PaidVisibility_Compliance Import.py", line 348, in <module>
|
||||
for chunk in pd.read_sql(
|
||||
~~~~~~~~~~~^
|
||||
query,
|
||||
^^^^^^
|
||||
sql_conn,
|
||||
^^^^^^^^^
|
||||
chunksize=chunk_size
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
):
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\pandas\io\sql.py", line 2730, in _query_iterator
|
||||
data = cursor.fetchmany(chunksize)
|
||||
pyodbc.OperationalError: ('08S01', '[08S01] [Microsoft][ODBC Driver 17 for SQL Server]TCP Provider: An existing connection was forcibly closed by the remote host.\r\n (10054) (SQLGetData); [08S01] [Microsoft][ODBC Driver 17 for SQL Server]Communication link failure (10054)')
|
||||
|
||||
During handling of the above exception, another exception occurred:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "d:\Python Code\PaidVisibility_Compliance Import.py", line 518, in <module>
|
||||
sql_conn = connect_sql()
|
||||
File "d:\Python Code\PaidVisibility_Compliance Import.py", line 70, in connect_sql
|
||||
conn = pyodbc.connect(
|
||||
SQL_CONN_STR,
|
||||
autocommit=True
|
||||
)
|
||||
pyodbc.OperationalError: ('08001', '[08001] [Microsoft][ODBC Driver 17 for SQL Server]Named Pipes Provider: Could not open a connection to SQL Server [64]. (64) (SQLDriverConnect); [08001] [Microsoft][ODBC Driver 17 for SQL Server]Login timeout expired (0); [08001] [Microsoft][ODBC Driver 17 for SQL Server]Invalid connection string attribute (0); [08001] [Microsoft][ODBC Driver 17 for SQL Server]A network-related or instance-specific error has occurred while establishing a connection to SQL Server. Server is not found or not accessible. Check if instance name is correct and if SQL Server is configured to allow remote connections. For more information see SQL Server Books Online. (64)')
|
||||
|
||||
================================
|
||||
|
||||
================================
|
||||
TIME : 2026-05-20 12:28:25.483277
|
||||
ERROR : ('08001', '[08001] [Microsoft][ODBC Driver 17 for SQL Server]Named Pipes Provider: Could not open a connection to SQL Server [53]. (53) (SQLDriverConnect); [08001] [Microsoft][ODBC Driver 17 for SQL Server]Login timeout expired (0); [08001] [Microsoft][ODBC Driver 17 for SQL Server]Invalid connection string attribute (0); [08001] [Microsoft][ODBC Driver 17 for SQL Server]A network-related or instance-specific error has occurred while establishing a connection to SQL Server. Server is not found or not accessible. Check if instance name is correct and if SQL Server is configured to allow remote connections. For more information see SQL Server Books Online. (53)')
|
||||
TRACEBACK :
|
||||
Traceback (most recent call last):
|
||||
File "d:\Python Code\PaidVisibility_Compliance Import.py", line 348, in <module>
|
||||
for chunk in pd.read_sql(
|
||||
~~~~~~~~~~~^
|
||||
query,
|
||||
^^^^^^
|
||||
sql_conn,
|
||||
^^^^^^^^^
|
||||
chunksize=chunk_size
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
):
|
||||
^
|
||||
File "C:\Users\dipanshuk\AppData\Local\Python\pythoncore-3.14-64\Lib\site-packages\pandas\io\sql.py", line 2730, in _query_iterator
|
||||
data = cursor.fetchmany(chunksize)
|
||||
pyodbc.OperationalError: ('08S01', '[08S01] [Microsoft][ODBC Driver 17 for SQL Server]TCP Provider: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond.\r\n (10060) (SQLGetData); [08S01] [Microsoft][ODBC Driver 17 for SQL Server]Communication link failure (10060)')
|
||||
|
||||
During handling of the above exception, another exception occurred:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "d:\Python Code\PaidVisibility_Compliance Import.py", line 518, in <module>
|
||||
sql_conn = connect_sql()
|
||||
File "d:\Python Code\PaidVisibility_Compliance Import.py", line 70, in connect_sql
|
||||
conn = pyodbc.connect(
|
||||
SQL_CONN_STR,
|
||||
autocommit=True
|
||||
)
|
||||
pyodbc.OperationalError: ('08001', '[08001] [Microsoft][ODBC Driver 17 for SQL Server]Named Pipes Provider: Could not open a connection to SQL Server [53]. (53) (SQLDriverConnect); [08001] [Microsoft][ODBC Driver 17 for SQL Server]Login timeout expired (0); [08001] [Microsoft][ODBC Driver 17 for SQL Server]Invalid connection string attribute (0); [08001] [Microsoft][ODBC Driver 17 for SQL Server]A network-related or instance-specific error has occurred while establishing a connection to SQL Server. Server is not found or not accessible. Check if instance name is correct and if SQL Server is configured to allow remote connections. For more information see SQL Server Books Online. (53)')
|
||||
|
||||
================================
|
||||
Reference in New Issue
Block a user