diff --git a/scripts/mssql_clickhouse_migrate.sh b/scripts/mssql_clickhouse_migrate.sh new file mode 100755 index 0000000..7919ec1 --- /dev/null +++ b/scripts/mssql_clickhouse_migrate.sh @@ -0,0 +1,328 @@ +#!/usr/bin/env bash +# Migrate SQL Server tables into ClickHouse using sqlcmd + clickhouse-client. +# +# Recommended paths: +# - Full database (creates Nullable(String) columns, safe TSV): migrate-db +# - One table via temp file (same safety as migrate-db): migrate-table +# +# Optional "stream" mode pipes sqlcmd straight into ClickHouse (like a manual +# one-liner). Use only when data has no tabs/newlines and NULL handling matches +# your expectations; prefer migrate-table otherwise. +# +# Environment (examples — set in your shell or a sourced file): +# export MSSQL_HOST="43.242.212.54" +# export MSSQL_PORT="21443" +# export MSSQL_USER="Nishant_Dev" +# export MSSQL_PASSWORD='nishant@dev' +# export MSSQL_DATABASE="CPMIndiaBusinessInsight_test" +# export MSSQL_TRUST_SERVER_CERT="1" +# # export MSSQL_ENCRYPT="optional" +# +# export CH_DOCKER_CONTAINER="clickhouse" +# export CH_USER="default" +# export CH_PASSWORD="" +# export CH_PORT="9000" +# export CH_HOST="127.0.0.1" +# export CH_DATABASE="cpm" +# +# export MSSQL_CH_EXPORT_DIR="./mssql_export_all" +# +set -euo pipefail + +SCRIPT_NAME=$(basename "$0") +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CORE="${ROOT_DIR}/mssql_to_clickhouse.sh" + +die() { + echo "$SCRIPT_NAME: $*" >&2 + exit 1 +} + +usage() { + cat <TABLE rows from SQL Server discovery. + + stream-table --mssql-table SCHEMA.TABLE [--ch-database DB] [--ch-table NAME] + Pipe: sqlcmd SELECT * | clickhouse-client INSERT FORMAT TabSeparated. + Less safe than migrate-table; for quick tests or clean numeric/text data. + + print-env + Print current MSSQL/CH-related environment (passwords masked). + +Environment defaults this wrapper applies before delegating: + MSSQL_PORT=\${MSSQL_PORT:-1433} + CH_PORT=\${CH_PORT:-9000} + CH_USER=\${CH_USER:-default} + CH_HOST=\${CH_HOST:-127.0.0.1} + CH_DATABASE (or pass --ch-database) — ClickHouse database name, e.g. cpm + +Examples: + export CH_DATABASE=cpm CH_DOCKER_CONTAINER=clickhouse MSSQL_TRUST_SERVER_CERT=1 + $SCRIPT_NAME migrate-table --mssql-table dbo.Category_Execution + $SCRIPT_NAME migrate-db --ch-database cpm --out-dir ./ch_export +EOF +} + +apply_defaults() { + export MSSQL_PORT="${MSSQL_PORT:-1433}" + export CH_PORT="${CH_PORT:-9000}" + export CH_USER="${CH_USER:-default}" + export CH_HOST="${CH_HOST:-127.0.0.1}" +} + +ensure_core() { + [[ -x "$CORE" || -f "$CORE" ]] || die "Missing core script: $CORE" +} + +delegate() { + apply_defaults + ensure_core + bash "$CORE" "$@" +} + +mask() { + local v="${1:-}" + if [[ -z "$v" ]]; then + printf "(empty)" + else + printf "***" + fi +} + +cmd_print_env() { + apply_defaults + cat </dev/null 2>&1; then + return 0 + fi + if [[ -x /opt/mssql-tools18/bin/sqlcmd ]]; then + export PATH="/opt/mssql-tools18/bin:$PATH" + return 0 + fi + die "sqlcmd not found (install mssql-tools18 or add to PATH)" +} + +ensure_mssql_env() { + [[ -n "${MSSQL_HOST:-}" ]] || die "Set MSSQL_HOST" + [[ -n "${MSSQL_USER:-}" ]] || die "Set MSSQL_USER" + [[ -n "${MSSQL_PASSWORD:-}" ]] || die "Set MSSQL_PASSWORD" + [[ -n "${MSSQL_DATABASE:-}" ]] || die "Set MSSQL_DATABASE" +} + +sqlcmd_conn_args() { + local port="${MSSQL_PORT:-1433}" + local server="${MSSQL_HOST},${port}" + local args=( + -S "$server" + -d "$MSSQL_DATABASE" + -U "$MSSQL_USER" + -P "$MSSQL_PASSWORD" + ) + if [[ "${MSSQL_TRUST_SERVER_CERT:-0}" == "1" ]]; then + args+=(-C) + fi + if [[ -n "${MSSQL_ENCRYPT:-}" ]]; then + args+=(-N "$MSSQL_ENCRYPT") + fi + printf '%s\n' "${args[@]}" +} + +ch_default_database() { + printf '%s' "${CH_DATABASE:-default}" +} + +escape_ch_ident() { + local ident="${1:-}" + ident="${ident//\`/\`\`}" + printf "%s" "$ident" +} + +cmd_stream_table() { + apply_defaults + ensure_sqlcmd + ensure_mssql_env + + local mssql_table="" + local ch_db="" + local ch_table="" + while [[ $# -gt 0 ]]; do + case "$1" in + --mssql-table) + mssql_table="$2" + shift 2 + ;; + --ch-database) + ch_db="$2" + shift 2 + ;; + --ch-table) + ch_table="$2" + shift 2 + ;; + *) + die "Unknown option: $1" + ;; + esac + done + + [[ -n "$mssql_table" ]] || die "stream-table requires --mssql-table SCHEMA.TABLE" + ch_db="${ch_db:-$(ch_default_database)}" + if [[ -z "$ch_table" ]]; then + if [[ "$mssql_table" == *.* ]]; then + ch_table="${mssql_table#*.}" + else + ch_table="$mssql_table" + fi + fi + + [[ -n "${CH_DOCKER_CONTAINER:-}" ]] || die "stream-table expects CH_DOCKER_CONTAINER for docker exec -i" + command -v docker >/dev/null 2>&1 || die "docker not found" + + local conn_args=() + mapfile -t conn_args < <(sqlcmd_conn_args) + + local insert_q + insert_q="INSERT INTO \`$(escape_ch_ident "$ch_db")\`.\`$(escape_ch_ident "$ch_table")\` FORMAT TabSeparated" + + echo "$SCRIPT_NAME: streaming SELECT * from $mssql_table → ${ch_db}.${ch_table} (ensure table exists and types match)." >&2 + + local ch_args=( + --host "${CH_HOST:-127.0.0.1}" + --port "${CH_PORT:-9000}" + --user "${CH_USER:-default}" + --query "$insert_q" + ) + [[ -n "${CH_PASSWORD:-}" ]] && ch_args+=(--password "$CH_PASSWORD") + [[ "${CH_SECURE:-0}" == "1" ]] && ch_args+=(--secure) + + sqlcmd "${conn_args[@]}" \ + -Q "SET NOCOUNT ON; SELECT * FROM ${mssql_table}" \ + -h -1 -W -s "$(printf '\t')" -w 65535 -f i:65001,o:65001 -b \ + | docker exec -i "$CH_DOCKER_CONTAINER" clickhouse-client "${ch_args[@]}" +} + +cmd_migrate_table() { + apply_defaults + ensure_core + + local mssql_table="" + local ch_db="" + local out_dir="" + while [[ $# -gt 0 ]]; do + case "$1" in + --mssql-table) + mssql_table="$2" + shift 2 + ;; + --ch-database) + ch_db="$2" + shift 2 + ;; + --out-dir) + out_dir="$2" + shift 2 + ;; + *) + die "Unknown option: $1" + ;; + esac + done + + [[ -n "$mssql_table" ]] || die "migrate-table requires --mssql-table SCHEMA.TABLE" + ch_db="${ch_db:-$(ch_default_database)}" + out_dir="${out_dir:-${MSSQL_CH_EXPORT_DIR:-./mssql_export_all}}" + + local tmp + tmp="$(mktemp "${TMPDIR:-/tmp}/${SCRIPT_NAME}.tables.XXXXXX")" + trap 'rm -f "$tmp"' RETURN + + printf '%s\n' "$mssql_table" >"$tmp" + + bash "$CORE" migrate-db \ + --ch-database "$ch_db" \ + --out-dir "$out_dir" \ + --tables-file "$tmp" +} + +main() { + [[ $# -ge 1 ]] || { + usage + exit 1 + } + + local cmd="$1" + shift || true + + if [[ "$cmd" == "-h" || "$cmd" == "--help" ]]; then + usage + exit 0 + fi + + case "$cmd" in + print-env) + cmd_print_env + ;; + migrate-db) + delegate migrate-db \ + --ch-database "$(ch_default_database)" \ + --out-dir "${MSSQL_CH_EXPORT_DIR:-./mssql_export_all}" \ + "$@" + ;; + list-tables) + delegate list-tables "$@" + ;; + migrate-table) + cmd_migrate_table "$@" + ;; + export-import) + delegate export-import \ + --ch-database "$(ch_default_database)" \ + "$@" + ;; + stream-table) + cmd_stream_table "$@" + ;; + *) + usage + die "Unknown command: $cmd" + ;; + esac +} + +main "$@"