diff --git a/scripts/mssql_clickhouse_migrate.sh b/scripts/mssql_clickhouse_migrate.sh index dd429a6..dc355f2 100755 --- a/scripts/mssql_clickhouse_migrate.sh +++ b/scripts/mssql_clickhouse_migrate.sh @@ -2,8 +2,9 @@ # Bulk copy one SQL Server table to ClickHouse on Ubuntu (or Debian-based). # # Prerequisites (or run with --install-deps): -# - sqlcmd: Microsoft mssql-tools18 (/opt/mssql-tools18/bin/sqlcmd) +# - sqlcmd: Microsoft mssql-tools18 (/opt/mssql-tools18/bin/sqlcmd) or mssql-tools (/opt/mssql-tools/bin) # - clickhouse-client +# Optional: SQLCMD_PATH=/full/path/to/sqlcmd if installed outside PATH # # Environment (required for export-import): # MSSQL_HOST e.g. db.example.com @@ -19,6 +20,7 @@ # CH_USER default default # CH_PASSWORD optional # CH_SECURE set to 1 to use TLS (--secure) +# SQLCMD_PATH optional full path to sqlcmd if not on PATH # # Example: # export MSSQL_HOST=sql.mycompany.internal MSSQL_USER=ro MSSQL_PASSWORD='***' MSSQL_DATABASE=sales @@ -46,7 +48,7 @@ usage() { $SCRIPT_NAME — export SQL Server data to TSV and load into ClickHouse (Ubuntu/Debian). Commands: - --install-deps Install mssql-tools18 + clickhouse-client (needs sudo) + --install-deps Install mssql-tools18 + clickhouse-client (needs sudo). Run alone FIRST, then migrate-db. export Export MSSQL table/query to TSV (TabSeparated) import Load TSV into ClickHouse (INSERT FORMAT TabSeparated) export-import Run export then import @@ -72,6 +74,7 @@ Environment: MSSQL_TABLES_FILE optional explicit table list path for migrate-db CH_HOST, CH_PORT, CH_USER, CH_PASSWORD, CH_SECURE=1 for TLS CH_DOCKER_CONTAINER optional; run clickhouse-client inside this container + SQLCMD_PATH optional; full path to sqlcmd when not on PATH Notes: - ClickHouse columns must match export order and types. @@ -81,14 +84,23 @@ EOF } ensure_sqlcmd() { + if [[ -n "${SQLCMD_PATH:-}" ]]; then + [[ -x "$SQLCMD_PATH" ]] || die "SQLCMD_PATH is set but not executable: $SQLCMD_PATH" + export PATH="$(dirname "$SQLCMD_PATH"):$PATH" + fi if command -v sqlcmd >/dev/null 2>&1; then return 0 fi - if [[ -x /opt/mssql-tools18/bin/sqlcmd ]]; then - export PATH="/opt/mssql-tools18/bin:$PATH" - return 0 - fi - die "sqlcmd not found. Install mssql-tools18 or run: $0 --install-deps" + local d + for d in /opt/mssql-tools18/bin /opt/mssql-tools/bin; do + if [[ -x "$d/sqlcmd" ]]; then + export PATH="$d:$PATH" + return 0 + fi + done + die "sqlcmd not found. On this host run once (needs sudo), then open a new shell or re-login: + $0 --install-deps +If sqlcmd is already installed, add it to PATH or set SQLCMD_PATH=/path/to/sqlcmd" } ensure_clickhouse_client() { @@ -502,17 +514,28 @@ install_deps() { [[ "${ID:-}" == "ubuntu" || "${ID:-}" == "debian" || "${ID_LIKE:-}" == *"debian"* ]] \ || die "This installer expects Ubuntu or Debian." + export DEBIAN_FRONTEND=noninteractive $SUDO apt-get update $SUDO apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release - # Microsoft ODBC + sqlcmd + # Microsoft ODBC + sqlcmd (Ubuntu codename: use current release; fall back if MS has no prod.list yet) curl -fsSL https://packages.microsoft.com/keys/microsoft.asc \ | $SUDO gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg - curl -fsSL "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list" \ - | $SUDO tee /etc/apt/sources.list.d/mssql-release.list >/dev/null + local ms_ub ms_url + ms_ub="$(lsb_release -rs)" + ms_url="https://packages.microsoft.com/config/ubuntu/${ms_ub}/prod.list" + if ! curl -fsSL "$ms_url" -o /tmp/mssql-release.list 2>/dev/null; then + echo "$SCRIPT_NAME: No Microsoft prod.list for Ubuntu ${ms_ub}; trying 22.04 repo (packages often install on newer LTS)." >&2 + ms_ub="22.04" + ms_url="https://packages.microsoft.com/config/ubuntu/${ms_ub}/prod.list" + curl -fsSL "$ms_url" -o /tmp/mssql-release.list + fi + $SUDO mv /tmp/mssql-release.list /etc/apt/sources.list.d/mssql-release.list $SUDO apt-get update ACCEPT_EULA=Y $SUDO apt-get install -y msodbcsql18 mssql-tools18 unixodbc-dev - echo 'export PATH="$PATH:/opt/mssql-tools18/bin"' >>"$HOME/.bashrc" || true + local path_line='export PATH="$PATH:/opt/mssql-tools18/bin"' + grep -qF 'mssql-tools18/bin' "$HOME/.bashrc" 2>/dev/null || echo "$path_line" >>"$HOME/.bashrc" || true + grep -qF 'mssql-tools18/bin' "$HOME/.profile" 2>/dev/null || echo "$path_line" >>"$HOME/.profile" || true export PATH="$PATH:/opt/mssql-tools18/bin" # ClickHouse client