NishantRajputRN
Executable
+328
@@ -0,0 +1,328 @@
|
||||
#!/usr/bin/env bash
|
||||
# Migrate SQL Server tables into ClickHouse using sqlcmd + clickhouse-client.
|
||||
#
|
||||
# Recommended paths:
|
||||
# - Full database (creates Nullable(String) columns, safe TSV): migrate-db
|
||||
# - One table via temp file (same safety as migrate-db): migrate-table
|
||||
#
|
||||
# Optional "stream" mode pipes sqlcmd straight into ClickHouse (like a manual
|
||||
# one-liner). Use only when data has no tabs/newlines and NULL handling matches
|
||||
# your expectations; prefer migrate-table otherwise.
|
||||
#
|
||||
# Environment (examples — set in your shell or a sourced file):
|
||||
# export MSSQL_HOST="43.242.212.54"
|
||||
# export MSSQL_PORT="21443"
|
||||
# export MSSQL_USER="Nishant_Dev"
|
||||
# export MSSQL_PASSWORD='nishant@dev'
|
||||
# export MSSQL_DATABASE="CPMIndiaBusinessInsight_test"
|
||||
# export MSSQL_TRUST_SERVER_CERT="1"
|
||||
# # export MSSQL_ENCRYPT="optional"
|
||||
#
|
||||
# export CH_DOCKER_CONTAINER="clickhouse"
|
||||
# export CH_USER="default"
|
||||
# export CH_PASSWORD=""
|
||||
# export CH_PORT="9000"
|
||||
# export CH_HOST="127.0.0.1"
|
||||
# export CH_DATABASE="cpm"
|
||||
#
|
||||
# export MSSQL_CH_EXPORT_DIR="./mssql_export_all"
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_NAME=$(basename "$0")
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
CORE="${ROOT_DIR}/mssql_to_clickhouse.sh"
|
||||
|
||||
die() {
|
||||
echo "$SCRIPT_NAME: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
$SCRIPT_NAME — MSSQL → ClickHouse migration (wraps ${CORE##*/}).
|
||||
|
||||
Commands:
|
||||
migrate-db [--mssql-schema SCHEMA] [--tables-file PATH] [--out-dir DIR]
|
||||
Create ClickHouse database/tables and copy all discoverable MSSQL base tables.
|
||||
Uses safe export (NULL as \\N, tabs/newlines stripped in text).
|
||||
|
||||
migrate-table --mssql-table SCHEMA.TABLE [--ch-database DB] [--out-dir DIR]
|
||||
Create table if needed and copy one MSSQL table (delegates to migrate-db with a one-line list).
|
||||
|
||||
export-import --mssql-table SCHEMA.TABLE [--ch-database DB] [--ch-table NAME] [--out FILE]
|
||||
Export then import one table (ClickHouse table must already exist with matching columns).
|
||||
|
||||
list-tables [--mssql-schema SCHEMA]
|
||||
Print SCHEMA<TAB>TABLE rows from SQL Server discovery.
|
||||
|
||||
stream-table --mssql-table SCHEMA.TABLE [--ch-database DB] [--ch-table NAME]
|
||||
Pipe: sqlcmd SELECT * | clickhouse-client INSERT FORMAT TabSeparated.
|
||||
Less safe than migrate-table; for quick tests or clean numeric/text data.
|
||||
|
||||
print-env
|
||||
Print current MSSQL/CH-related environment (passwords masked).
|
||||
|
||||
Environment defaults this wrapper applies before delegating:
|
||||
MSSQL_PORT=\${MSSQL_PORT:-1433}
|
||||
CH_PORT=\${CH_PORT:-9000}
|
||||
CH_USER=\${CH_USER:-default}
|
||||
CH_HOST=\${CH_HOST:-127.0.0.1}
|
||||
CH_DATABASE (or pass --ch-database) — ClickHouse database name, e.g. cpm
|
||||
|
||||
Examples:
|
||||
export CH_DATABASE=cpm CH_DOCKER_CONTAINER=clickhouse MSSQL_TRUST_SERVER_CERT=1
|
||||
$SCRIPT_NAME migrate-table --mssql-table dbo.Category_Execution
|
||||
$SCRIPT_NAME migrate-db --ch-database cpm --out-dir ./ch_export
|
||||
EOF
|
||||
}
|
||||
|
||||
apply_defaults() {
|
||||
export MSSQL_PORT="${MSSQL_PORT:-1433}"
|
||||
export CH_PORT="${CH_PORT:-9000}"
|
||||
export CH_USER="${CH_USER:-default}"
|
||||
export CH_HOST="${CH_HOST:-127.0.0.1}"
|
||||
}
|
||||
|
||||
ensure_core() {
|
||||
[[ -x "$CORE" || -f "$CORE" ]] || die "Missing core script: $CORE"
|
||||
}
|
||||
|
||||
delegate() {
|
||||
apply_defaults
|
||||
ensure_core
|
||||
bash "$CORE" "$@"
|
||||
}
|
||||
|
||||
mask() {
|
||||
local v="${1:-}"
|
||||
if [[ -z "$v" ]]; then
|
||||
printf "(empty)"
|
||||
else
|
||||
printf "***"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_print_env() {
|
||||
apply_defaults
|
||||
cat <<EOF
|
||||
MSSQL_HOST=${MSSQL_HOST:-}
|
||||
MSSQL_PORT=${MSSQL_PORT:-}
|
||||
MSSQL_USER=${MSSQL_USER:-}
|
||||
MSSQL_PASSWORD=$(mask "${MSSQL_PASSWORD:-}")
|
||||
MSSQL_DATABASE=${MSSQL_DATABASE:-}
|
||||
MSSQL_TRUST_SERVER_CERT=${MSSQL_TRUST_SERVER_CERT:-}
|
||||
MSSQL_ENCRYPT=${MSSQL_ENCRYPT:-}
|
||||
MSSQL_TABLES_FILE=${MSSQL_TABLES_FILE:-}
|
||||
|
||||
CH_DOCKER_CONTAINER=${CH_DOCKER_CONTAINER:-}
|
||||
CH_HOST=${CH_HOST:-}
|
||||
CH_PORT=${CH_PORT:-}
|
||||
CH_USER=${CH_USER:-}
|
||||
CH_PASSWORD=$(mask "${CH_PASSWORD:-}")
|
||||
CH_SECURE=${CH_SECURE:-}
|
||||
CH_DATABASE=${CH_DATABASE:-}
|
||||
|
||||
MSSQL_CH_EXPORT_DIR=${MSSQL_CH_EXPORT_DIR:-}
|
||||
EOF
|
||||
}
|
||||
|
||||
ensure_sqlcmd() {
|
||||
if command -v sqlcmd >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
if [[ -x /opt/mssql-tools18/bin/sqlcmd ]]; then
|
||||
export PATH="/opt/mssql-tools18/bin:$PATH"
|
||||
return 0
|
||||
fi
|
||||
die "sqlcmd not found (install mssql-tools18 or add to PATH)"
|
||||
}
|
||||
|
||||
ensure_mssql_env() {
|
||||
[[ -n "${MSSQL_HOST:-}" ]] || die "Set MSSQL_HOST"
|
||||
[[ -n "${MSSQL_USER:-}" ]] || die "Set MSSQL_USER"
|
||||
[[ -n "${MSSQL_PASSWORD:-}" ]] || die "Set MSSQL_PASSWORD"
|
||||
[[ -n "${MSSQL_DATABASE:-}" ]] || die "Set MSSQL_DATABASE"
|
||||
}
|
||||
|
||||
sqlcmd_conn_args() {
|
||||
local port="${MSSQL_PORT:-1433}"
|
||||
local server="${MSSQL_HOST},${port}"
|
||||
local args=(
|
||||
-S "$server"
|
||||
-d "$MSSQL_DATABASE"
|
||||
-U "$MSSQL_USER"
|
||||
-P "$MSSQL_PASSWORD"
|
||||
)
|
||||
if [[ "${MSSQL_TRUST_SERVER_CERT:-0}" == "1" ]]; then
|
||||
args+=(-C)
|
||||
fi
|
||||
if [[ -n "${MSSQL_ENCRYPT:-}" ]]; then
|
||||
args+=(-N "$MSSQL_ENCRYPT")
|
||||
fi
|
||||
printf '%s\n' "${args[@]}"
|
||||
}
|
||||
|
||||
ch_default_database() {
|
||||
printf '%s' "${CH_DATABASE:-default}"
|
||||
}
|
||||
|
||||
escape_ch_ident() {
|
||||
local ident="${1:-}"
|
||||
ident="${ident//\`/\`\`}"
|
||||
printf "%s" "$ident"
|
||||
}
|
||||
|
||||
cmd_stream_table() {
|
||||
apply_defaults
|
||||
ensure_sqlcmd
|
||||
ensure_mssql_env
|
||||
|
||||
local mssql_table=""
|
||||
local ch_db=""
|
||||
local ch_table=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--mssql-table)
|
||||
mssql_table="$2"
|
||||
shift 2
|
||||
;;
|
||||
--ch-database)
|
||||
ch_db="$2"
|
||||
shift 2
|
||||
;;
|
||||
--ch-table)
|
||||
ch_table="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
die "Unknown option: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "$mssql_table" ]] || die "stream-table requires --mssql-table SCHEMA.TABLE"
|
||||
ch_db="${ch_db:-$(ch_default_database)}"
|
||||
if [[ -z "$ch_table" ]]; then
|
||||
if [[ "$mssql_table" == *.* ]]; then
|
||||
ch_table="${mssql_table#*.}"
|
||||
else
|
||||
ch_table="$mssql_table"
|
||||
fi
|
||||
fi
|
||||
|
||||
[[ -n "${CH_DOCKER_CONTAINER:-}" ]] || die "stream-table expects CH_DOCKER_CONTAINER for docker exec -i"
|
||||
command -v docker >/dev/null 2>&1 || die "docker not found"
|
||||
|
||||
local conn_args=()
|
||||
mapfile -t conn_args < <(sqlcmd_conn_args)
|
||||
|
||||
local insert_q
|
||||
insert_q="INSERT INTO \`$(escape_ch_ident "$ch_db")\`.\`$(escape_ch_ident "$ch_table")\` FORMAT TabSeparated"
|
||||
|
||||
echo "$SCRIPT_NAME: streaming SELECT * from $mssql_table → ${ch_db}.${ch_table} (ensure table exists and types match)." >&2
|
||||
|
||||
local ch_args=(
|
||||
--host "${CH_HOST:-127.0.0.1}"
|
||||
--port "${CH_PORT:-9000}"
|
||||
--user "${CH_USER:-default}"
|
||||
--query "$insert_q"
|
||||
)
|
||||
[[ -n "${CH_PASSWORD:-}" ]] && ch_args+=(--password "$CH_PASSWORD")
|
||||
[[ "${CH_SECURE:-0}" == "1" ]] && ch_args+=(--secure)
|
||||
|
||||
sqlcmd "${conn_args[@]}" \
|
||||
-Q "SET NOCOUNT ON; SELECT * FROM ${mssql_table}" \
|
||||
-h -1 -W -s "$(printf '\t')" -w 65535 -f i:65001,o:65001 -b \
|
||||
| docker exec -i "$CH_DOCKER_CONTAINER" clickhouse-client "${ch_args[@]}"
|
||||
}
|
||||
|
||||
cmd_migrate_table() {
|
||||
apply_defaults
|
||||
ensure_core
|
||||
|
||||
local mssql_table=""
|
||||
local ch_db=""
|
||||
local out_dir=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--mssql-table)
|
||||
mssql_table="$2"
|
||||
shift 2
|
||||
;;
|
||||
--ch-database)
|
||||
ch_db="$2"
|
||||
shift 2
|
||||
;;
|
||||
--out-dir)
|
||||
out_dir="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
die "Unknown option: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "$mssql_table" ]] || die "migrate-table requires --mssql-table SCHEMA.TABLE"
|
||||
ch_db="${ch_db:-$(ch_default_database)}"
|
||||
out_dir="${out_dir:-${MSSQL_CH_EXPORT_DIR:-./mssql_export_all}}"
|
||||
|
||||
local tmp
|
||||
tmp="$(mktemp "${TMPDIR:-/tmp}/${SCRIPT_NAME}.tables.XXXXXX")"
|
||||
trap 'rm -f "$tmp"' RETURN
|
||||
|
||||
printf '%s\n' "$mssql_table" >"$tmp"
|
||||
|
||||
bash "$CORE" migrate-db \
|
||||
--ch-database "$ch_db" \
|
||||
--out-dir "$out_dir" \
|
||||
--tables-file "$tmp"
|
||||
}
|
||||
|
||||
main() {
|
||||
[[ $# -ge 1 ]] || {
|
||||
usage
|
||||
exit 1
|
||||
}
|
||||
|
||||
local cmd="$1"
|
||||
shift || true
|
||||
|
||||
if [[ "$cmd" == "-h" || "$cmd" == "--help" ]]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
case "$cmd" in
|
||||
print-env)
|
||||
cmd_print_env
|
||||
;;
|
||||
migrate-db)
|
||||
delegate migrate-db \
|
||||
--ch-database "$(ch_default_database)" \
|
||||
--out-dir "${MSSQL_CH_EXPORT_DIR:-./mssql_export_all}" \
|
||||
"$@"
|
||||
;;
|
||||
list-tables)
|
||||
delegate list-tables "$@"
|
||||
;;
|
||||
migrate-table)
|
||||
cmd_migrate_table "$@"
|
||||
;;
|
||||
export-import)
|
||||
delegate export-import \
|
||||
--ch-database "$(ch_default_database)" \
|
||||
"$@"
|
||||
;;
|
||||
stream-table)
|
||||
cmd_stream_table "$@"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
die "Unknown command: $cmd"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user