diff --git a/.gitea/workflows/azure-pipelines.yml b/.gitea/workflows/azure-pipelines.yml index 3a83191..06570c9 100644 --- a/.gitea/workflows/azure-pipelines.yml +++ b/.gitea/workflows/azure-pipelines.yml @@ -15,45 +15,13 @@ jobs: with: fetch-depth: 0 - # Host runner: no ~/.bashrc. Jobs may run as root while nvm lives under a normal user - # (e.g. /home/azureuser/.nvm). Prefer system Node, then that user's nvm. - # - name: Put Node on PATH (host runner) - # run: | - # set -euo pipefail - # if command -v node >/dev/null 2>&1; then - # echo "Using node already on PATH: $(command -v node)" - # echo "PATH=$PATH" >> "$GITHUB_ENV" - # node -v - # exit 0 - # fi - # NVM_DIR_RESOLVED="" - # for dir in "${NVM_DIR:-}" "${HOME}/.nvm" "/home/azureuser/.nvm" "/home/ubuntu/.nvm"; do - # [ -z "$dir" ] && continue - # if [ -s "$dir/nvm.sh" ]; then - # NVM_DIR_RESOLVED="$dir" - # break - # fi - # done - # if [ -z "$NVM_DIR_RESOLVED" ]; then - # echo "Node not found. Either symlink node to /usr/local/bin or install nvm under HOME," >&2 - # echo "or under /home/azureuser/.nvm for this runner." >&2 - # exit 1 - # fi - # export NVM_DIR="$NVM_DIR_RESOLVED" - # # shellcheck source=/dev/null - # . "$NVM_DIR/nvm.sh" - # nvm use 20 2>/dev/null || nvm use default - # echo "PATH=$PATH" >> "$GITHUB_ENV" - # command -v node - # node -v - # ---------------- JAVA (Gradle + SonarScanner need JDK on Ubuntu) ---------------- # - name: Setup Java - # uses: actions/setup-java@v4 - # with: - # distribution: temurin - # java-version: 17 - # cache: gradle + # uses: actions/setup-java@v4 + # with: + # distribution: temurin + # java-version: 17 + # cache: gradle # ---------------- NODE ---------------- # - name: Setup Node @@ -74,40 +42,26 @@ jobs: # "${HOME}/bin/trivy" --version # Use "${HOME}/bin/trivy" — act/Gitea may not prepend GITHUB_PATH before the next step. - # Gitea only orchestrates the job; output path is on the runner. With ubuntu-latest:host this is the VM; with Docker jobs, mount /home/azureuser or use artifacts. + # Gitea only orchestrates the job; checkout + report.json live on the runner machine (this server), not on the Gitea host. - name: Trivy filesystem scan run: | - TRIVY_REPORT="/home/azureuser/Trivy/report.json" - mkdir -p /home/azureuser/Trivy - "${HOME}/bin/trivy" fs -f json -o "${TRIVY_REPORT}" \ + "${HOME}/bin/trivy" fs -f json -o report.json \ --skip-dirs node_modules,android/.gradle,android/build,ios/Pods,ios/build,.git \ --exit-code 0 \ . + report_path="${GITHUB_WORKSPACE:-$(pwd)}/report.json" echo "Runner host: $(hostname)" - echo "Trivy JSON report: ${TRIVY_REPORT}" - ls -la "${TRIVY_REPORT}" + echo "report.json (on this runner, under job workspace): ${report_path}" + ls -la report.json + mkdir -p /home/azureuser/builds + cp -f report.json /home/azureuser/builds/trivy-report.json + echo "Persistent copy (survives after job workspace is removed): /home/azureuser/builds/trivy-report.json" - # Download this artifact from the Gitea run UI — file leaves the ephemeral job container without docker cp. - # - name: Upload Trivy report (artifact) + # - name: Upload Trivy report # uses: actions/upload-artifact@v3 # with: # name: trivy-fs-report - # path: /home/azureuser/Trivy/report.json - - # Optional: persist on the VM host. In act_runner config.yaml set (then restart runner): - # container: - # options: "-v /home/azureuser/gitea-reports:/gitea-reports" - # If your config uses valid_volumes, allow that host path (see act_runner config.example.yaml). - # - name: Copy Trivy report to host bind mount (if configured) - # run: | - # TRIVY_REPORT="/home/azureuser/Trivy/report.json" - # if [ -d /gitea-reports ] && [ -w /gitea-reports ]; then - # out="/gitea-reports/trivy-report-${GITHUB_RUN_ID:-$(date +%s)}.json" - # cp -f "${TRIVY_REPORT}" "${out}" - # echo "Copied to bind mount (see host dir mapped to /gitea-reports): ${out}" - # else - # echo "Skip host copy: no /gitea-reports volume. Use artifact above, or add runner container.options volume — see workflow comment." - # fi + # path: report.json # ---------------- SONARQUBE ---------------- # In Gitea: Settings → Secrets → SONAR_TOKEN (and optionally SONAR_URL). @@ -127,10 +81,10 @@ jobs: build-tools;36.0.0 ndk;27.1.12297006 - # - name: Point Gradle to the SDK - # run: | - # printf 'sdk.dir=%s\n' "${ANDROID_SDK_ROOT}" > android/local.properties - # cat android/local.properties + - name: Point Gradle to the SDK + run: | + printf 'sdk.dir=%s\n' "${ANDROID_SDK_ROOT}" > android/local.properties + cat android/local.properties - name: Grant Gradle execute permission run: chmod +x android/gradlew @@ -158,8 +112,8 @@ jobs: ls -l /home/azureuser/builds # ---------------- (OPTIONAL) ARTIFACT ---------------- - # - name: Upload APK (optional) - # uses: actions/upload-artifact@v3 - # with: - # name: app-release - # path: android/app/build/outputs/apk/release/*.apk + - name: Upload APK (optional) + uses: actions/upload-artifact@v3 + with: + name: app-release + path: android/app/build/outputs/apk/release/*.apk diff --git a/scripts/mssql_clickhouse_migrate.sh b/scripts/mssql_clickhouse_migrate.sh index 6e5f64a..ce56b01 100755 --- a/scripts/mssql_clickhouse_migrate.sh +++ b/scripts/mssql_clickhouse_migrate.sh @@ -20,7 +20,7 @@ # CH_USER default default # CH_PASSWORD optional # CH_SECURE set to 1 to use TLS (--secure) -# SQLCMD_PATH optional full path to sqlcmd if not on PATH +# MSSQL_INSTALL_DEPS_IF_MISSING set to 1 to auto-run Microsoft ODBC + sqlcmd install when missing (sudo) # # Example: # export MSSQL_HOST=sql.mycompany.internal MSSQL_USER=ro MSSQL_PASSWORD='***' MSSQL_DATABASE=sales @@ -70,6 +70,8 @@ Options: --ch-table TABLE ClickHouse table name --out PATH File path (default: ./mssql_export.tsv) --out-dir PATH Directory for migrate-db TSV files (default: ./mssql_export_all) + --install-deps-if-missing If sqlcmd is not installed, run the same steps as --install-deps first (needs sudo). + Or set env MSSQL_INSTALL_DEPS_IF_MISSING=1 for the same behavior. Environment: MSSQL_HOST, MSSQL_PORT, MSSQL_USER, MSSQL_PASSWORD, MSSQL_DATABASE @@ -79,6 +81,7 @@ Environment: CH_HOST, CH_PORT, CH_USER, CH_PASSWORD, CH_SECURE=1 for TLS CH_DOCKER_CONTAINER optional; run clickhouse-client inside this container SQLCMD_PATH optional; full path to sqlcmd when not on PATH + MSSQL_INSTALL_DEPS_IF_MISSING set to 1 to install mssql-tools18 when sqlcmd is missing (same as --install-deps-if-missing) Notes: - ClickHouse columns must match export order and types. @@ -95,17 +98,23 @@ ensure_sqlcmd() { if command -v sqlcmd >/dev/null 2>&1; then return 0 fi - die "sqlcmd not found (no Microsoft mssql-tools18 on this machine yet). + die "sqlcmd not found (Microsoft mssql-tools18 is not installed). -Install it with (needs sudo; ~2 minutes): +Option A — one combined command (will prompt for sudo if needed): + + $0 migrate-db --install-deps-if-missing --ch-database ... --out-dir ... + +Option B — two steps: $0 --install-deps + $0 migrate-db ... -Then re-run migrate-db. Quick check: +Option C — same shell, no flag: - ls -la /opt/mssql-tools18/bin/sqlcmd + export MSSQL_INSTALL_DEPS_IF_MISSING=1 + $0 migrate-db ... -Or set SQLCMD_PATH=/full/path/to/sqlcmd if you installed sqlcmd elsewhere." +Or set SQLCMD_PATH=/full/path/to/sqlcmd" } ensure_clickhouse_client() { @@ -640,9 +649,14 @@ main() { local out_file="./mssql_export.tsv" local out_dir="./mssql_export_all" local tables_file="${MSSQL_TABLES_FILE:-}" + local install_deps_if_missing=0 while [[ $# -gt 0 ]]; do case "$1" in + --install-deps-if-missing) + install_deps_if_missing=1 + shift + ;; --mssql-table) mssql_table="$2" shift 2 @@ -681,6 +695,20 @@ main() { esac done + local needs_sqlcmd=0 + case "$cmd" in + migrate-db | list-tables | export | export-import) needs_sqlcmd=1 ;; + esac + + if [[ "$needs_sqlcmd" -eq 1 ]]; then + if [[ "${MSSQL_INSTALL_DEPS_IF_MISSING:-0}" == "1" || "$install_deps_if_missing" -eq 1 ]]; then + if ! command -v sqlcmd >/dev/null 2>&1; then + echo "$SCRIPT_NAME: sqlcmd not found; running install_deps (Microsoft ODBC + mssql-tools18; needs sudo)..." >&2 + install_deps + fi + fi + fi + case "$cmd" in export) run_export "$mssql_table" "$mssql_query" "$out_file"