Compare commits

...

95 Commits

Author SHA1 Message Date
Gitea b7bfc286d9 test
RN APK Build / build (push) Has been cancelled
2026-05-15 11:59:46 +05:30
Gitea bce35205bd test
RN APK Build / build (push) Has been cancelled
2026-05-15 11:07:53 +05:30
NishantRajputRN d850157421 asd
RN APK Build / build (push) Failing after 30s
2026-05-13 18:00:11 +05:30
NishantRajputRN 4eec254629 test
RN APK Build / build (push) Has been cancelled
2026-05-13 17:46:29 +05:30
NishantRajputRN fb767d19ce sa
RN APK Build / build (push) Has been cancelled
2026-05-13 17:33:38 +05:30
NishantRajputRN 2b744c1b0b xas
RN APK Build / build (push) Has been cancelled
2026-05-13 17:31:07 +05:30
NishantRajputRN 6d7dca7916 asd
RN APK Build / build (push) Has been cancelled
2026-05-13 17:27:26 +05:30
NishantRajputRN 99ddc3de84 script
RN APK Build / build (push) Has been cancelled
2026-05-13 17:23:01 +05:30
NishantRajputRN 5d3ed02ef5 asd
RN APK Build / build (push) Has been cancelled
2026-05-13 17:07:05 +05:30
NishantRajputRN a5c3ae1343 mssql to clickhouse
RN APK Build / build (push) Has been cancelled
2026-05-13 16:49:56 +05:30
NishantRajputRN e4349e8459 as
RN APK Build / build (push) Successful in 20m48s
2026-05-13 16:28:30 +05:30
NishantRajputRN 92d04988f4 as
RN APK Build / build (push) Has been cancelled
2026-05-13 16:15:22 +05:30
NishantRajputRN 55c3cbdc66 scipt
RN APK Build / build (push) Has been cancelled
2026-05-13 16:06:30 +05:30
NishantRajputRN 2d2250df12 clickhouse
RN APK Build / build (push) Has been cancelled
2026-05-13 16:00:52 +05:30
NishantRajputRN ff8550713e sadsada
RN APK Build / build (push) Successful in 20m37s
2026-05-13 15:26:35 +05:30
NishantRajputRN 8b235111b9 asA
RN APK Build / build (push) Successful in 22m14s
2026-05-13 14:35:08 +05:30
NishantRajputRN 7837fe55a0 asssddsaas
RN APK Build / build (push) Failing after 3m10s
2026-05-13 14:30:38 +05:30
NishantRajputRN 0ff2dd9ce9 das
RN APK Build / build (push) Has been cancelled
2026-05-13 14:27:57 +05:30
NishantRajputRN b6d5718db0 sda
RN APK Build / build (push) Successful in 31m12s
2026-05-13 13:31:49 +05:30
NishantRajputRN 7b6785df8a tres
RN APK Build / build (push) Failing after 5s
2026-05-13 13:29:33 +05:30
NishantRajputRN 071f31ae59 tswe
RN APK Build / build (push) Failing after 5s
2026-05-13 13:08:20 +05:30
NishantRajputRN e58d45cac4 node test
RN APK Build / build (push) Failing after 4s
2026-05-13 13:05:13 +05:30
NishantRajputRN 6866d650fe test with new runner
RN APK Build / build (push) Failing after 1m50s
2026-05-13 13:01:06 +05:30
NishantRajputRN 65ac716177 trviy check
RN APK Build / build (push) Failing after 56m7s
2026-05-13 11:04:11 +05:30
NishantRajputRN 67fc69ee0e report file
RN APK Build / build (push) Waiting to run
2026-05-13 10:23:56 +05:30
NishantRajputRN 9bc637d43d report.json
RN APK Build / build (push) Has been cancelled
2026-05-13 10:19:15 +05:30
NishantRajputRN 22bca573ce trviy file
RN APK Build / build (push) Has been cancelled
2026-05-13 10:13:22 +05:30
NishantRajputRN d815cdb373 sda
RN APK Build / build (push) Failing after 27m8s
2026-05-12 18:14:22 +05:30
NishantRajputRN fc0b063154 sdsa
RN APK Build / build (push) Has been cancelled
2026-05-12 18:13:04 +05:30
NishantRajputRN 8a6e583e57 tes
RN APK Build / build (push) Failing after 1m29s
2026-05-12 18:10:16 +05:30
NishantRajputRN fe45b3532d Trivy filesystem
RN APK Build / build (push) Failing after 1m22s
2026-05-12 18:07:05 +05:30
NishantRajputRN 98bea55844 trivy test
RN APK Build / build (push) Failing after 1m25s
2026-05-12 17:11:27 +05:30
NishantRajputRN 7abeeed9d8 asdsa
RN APK Build / build (push) Waiting to run
2026-05-12 16:37:28 +05:30
NishantRajputRN 5603b173a5 sfs
RN APK Build / build (push) Failing after 1m45s
2026-05-12 16:34:03 +05:30
NishantRajputRN 668a18f62b node 20 2026-05-12 16:30:28 +05:30
NishantRajputRN db56945979 node version change 24
RN APK Build / build (push) Failing after 5s
2026-05-12 16:27:49 +05:30
NishantRajputRN ca29c1d7c1 apk test
RN APK Build / build (push) Failing after 5s
2026-05-12 16:23:33 +05:30
NishantRajputRN 5de9d04dd3 Sonar url change
RN APK Build / build (push) Has been cancelled
2026-05-12 16:18:22 +05:30
NishantRajputRN 1919716676 performics
RN APK Build / build (push) Failing after 9m56s
2026-05-12 16:04:50 +05:30
NishantRajputRN fff4b3bfe4 test sonar
RN APK Build / build (push) Failing after 5m10s
2026-05-12 15:53:39 +05:30
NishantRajputRN 4515112b63 sonar testing
RN APK Build / build (push) Failing after 3m2s
2026-05-12 15:41:45 +05:30
NishantRajputRN b43612fca1 Test runner
RN APK Build / build (push) Failing after 15m55s
2026-05-11 12:54:12 +05:30
NishantRajputRN 194c9bc52b bunty test
RN APK Build / build (push) Failing after 39m48s
2026-04-28 17:15:03 +05:30
NishantRajputRN e6bb3ea725 ubuntu-latest
RN APK Build / build (push) Waiting to run
2026-04-28 15:52:34 +05:30
NishantRajputRN e5dc1f8506 sd
RN APK Build / build (push) Has been cancelled
2026-04-28 15:30:47 +05:30
NishantRajputRN ba68b13458 wd
RN APK Build / build (push) Has been cancelled
2026-04-28 15:23:51 +05:30
NishantRajputRN 582c436d46 ad
RN APK Build / build (push) Has been cancelled
2026-04-28 15:22:23 +05:30
NishantRajputRN 1edfe42d6c asd
RN APK Build / build (push) Has been cancelled
2026-04-28 15:18:26 +05:30
NishantRajputRN dec30524f0 test
RN APK Build / build (push) Has been cancelled
2026-04-28 15:14:52 +05:30
NishantRajputRN 39b44e9b2c twsf
RN APK Build / build (push) Has been cancelled
2026-04-28 14:59:52 +05:30
NishantRajputRN 804a38cd24 yes
RN APK Build / build (push) Has been cancelled
2026-04-28 14:56:51 +05:30
NishantRajputRN 43fb4f746f tsf
RN APK Build / build (push) Has been cancelled
2026-04-28 14:51:07 +05:30
NishantRajputRN ab96a4ae6e teest
RN APK Build / build (push) Has been cancelled
2026-04-28 14:49:36 +05:30
NishantRajputRN 91ad7de977 twefs
RN APK Build / build (push) Has been cancelled
2026-04-28 14:33:39 +05:30
NishantRajputRN e343974695 test
RN APK Build / build (push) Has been cancelled
2026-04-28 14:29:06 +05:30
NishantRajputRN 039d02685d test2
RN APK Build / build (push) Has been cancelled
2026-04-28 14:28:11 +05:30
NishantRajputRN 79278e0b43 retest
RN APK Build / build (push) Has been cancelled
2026-04-28 14:26:29 +05:30
NishantRajputRN 2535059f09 ubuntu-latest
RN APK Build / build (push) Failing after 36m0s
2026-04-28 13:43:52 +05:30
NishantRajputRN d61f21076a novaNewVm
RN APK Build / build (push) Has been cancelled
2026-04-28 13:25:07 +05:30
NishantRajputRN 976adab33a sda
RN APK Build / build (push) Has been cancelled
2026-04-28 13:17:10 +05:30
NishantRajputRN 935d6f0051 sfa
Performics DevOps / Performics DevOps Pipeline (push) Has been cancelled
2026-04-28 13:16:06 +05:30
NishantRajputRN 48269ff7d7 label name update
Performics DevOps / Performics DevOps Pipeline (push) Has been cancelled
2026-04-28 13:02:36 +05:30
NishantRajputRN d89d19aa0e g\\sd
Performics DevOps / Performics DevOps Pipeline (push) Has been cancelled
2026-04-28 13:00:44 +05:30
NishantRajputRN acfb933c80 novaNewVm
Performics DevOps / Performics DevOps Pipeline (push) Has been cancelled
2026-04-28 12:55:14 +05:30
NishantRajputRN fdea77d651 ewrtyui
Performics DevOps / Performics DevOps Pipeline (push) Has been cancelled
2026-04-28 12:48:41 +05:30
NishantRajputRN 1a0c48618a tregh
Performics DevOps / Performics DevOps Pipeline (push) Has been cancelled
2026-04-28 12:27:43 +05:30
NishantRajputRN f6d9fcd29e test
Performics DevOps / Performics DevOps Pipeline (push) Has been cancelled
2026-04-28 12:26:13 +05:30
NishantRajputRN 81d9700dcf test
Performics DevOps / Performics DevOps Pipeline (push) Has been cancelled
2026-04-28 12:24:24 +05:30
NishantRajputRN a177e7e961 test
Performics DevOps / Performics DevOps Pipeline (push) Has been cancelled
2026-04-28 12:17:56 +05:30
NishantRajputRN 9db084c1c2 test 13
Performics DevOps / Performics DevOps Pipeline (push) Has been cancelled
2026-04-28 12:16:43 +05:30
NishantRajputRN a8d87eec00 label name change
Performics DevOps / Performics DevOps Pipeline (push) Has been cancelled
2026-04-28 12:15:32 +05:30
NishantRajputRN d93efc0ac4 test23
Performics DevOps / Performics DevOps Pipeline (push) Has been cancelled
2026-04-28 12:09:24 +05:30
NishantRajputRN 2eec4d38e9 test
Performics DevOps / Performics DevOps Pipeline (push) Has been cancelled
2026-04-28 12:05:53 +05:30
NishantRajputRN 0db474c4bc test2
Performics DevOps / Performics DevOps Pipeline (push) Has been cancelled
2026-04-28 12:01:34 +05:30
NishantRajputRN cac5795c34 test
Performics DevOps / Performics DevOps Pipeline (push) Has been cancelled
2026-04-28 11:24:30 +05:30
NishantRajputRN 473c0ad66f test
Performics DevOps / Performics DevOps Pipeline (push) Has been cancelled
2026-04-28 11:23:46 +05:30
NishantRajputRN d47fc70c39 test 2026-04-28 11:20:34 +05:30
NishantRajputRN d1b3a045f1 script update 2026-04-22 15:26:01 +05:30
NishantRajputRN 8d2f5b645c Add Ubuntu setup script and set configureOnDemand=false for New Architecture
- scripts/setup-ubuntu.sh: JDK 17, nvm/Node, Android SDK/NDK/CMake
- android/gradle.properties: avoid CMake/codegen race with New Arch

Made-with: Cursor
2026-04-22 13:48:17 +05:30
Gitea 0310beb610 second 2026-04-22 12:27:46 +05:30
NishantRajputRN 846055c536 clean
RN APK Build / build (push) Has been cancelled
2026-04-22 12:19:56 +05:30
Gitea cbb9ade94a second
RN APK Build / build (push) Failing after 4m18s
2026-04-22 01:04:39 +05:30
Gitea e5cd0315d5 first
RN APK Build / build (push) Has been cancelled
2026-04-22 00:11:15 +05:30
Gitea e3a0b2e921 first
RN APK Build / build (push) Failing after 51m21s
2026-04-21 22:45:45 +05:30
Gitea a87b0df3a4 first
RN APK Build / build (push) Failing after 38m57s
2026-04-21 22:00:40 +05:30
Gitea 2b6c10d95a first 2026-04-21 16:31:27 +05:30
NishantRajputRN bb5371b719 test 2026-04-20 10:37:48 +05:30
NishantRajputRN 462d424a58 test 2026-04-20 10:09:10 +05:30
NishantRajputRN b6905f30c9 test 2026-04-16 17:57:40 +05:30
NishantRajputRN 795c0addc3 test 2 2026-04-16 17:43:39 +05:30
nishantr 6868b61f82 Set up CI with Azure Pipelines
test [skip ci]
2026-04-16 12:06:26 +00:00
NishantRajputRN 8d653670ce test 2026-04-16 17:30:35 +05:30
NishantRajputRN c00ffbbb76 test 2026-04-16 17:26:13 +05:30
nishantr 290094e797 Merged PR 216: test
test
2026-04-16 17:05:56 +05:30
NishantRajputRN 82d41cda26 test 2026-04-16 17:04:08 +05:30
13 changed files with 1257 additions and 67 deletions
+122
View File
@@ -0,0 +1,122 @@
name: RN APK Build
on:
push:
branches:
- main
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
# ---------------- JAVA (Gradle + SonarScanner need JDK on Ubuntu) ----------------
# - name: Setup Java
# uses: actions/setup-java@v4
# with:
# distribution: temurin
# java-version: 17
# cache: gradle
# ---------------- NODE ----------------
# - name: Setup Node
# uses: actions/setup-node@v4
# with:
# node-version: 20
# cache: npm
- name: Install dependencies
run: npm ci
# ---------------- TRIVY (filesystem scan; avoid trivy-action — it pulls actions/cache node24) ----------------
# - name: Install Trivy
# run: |
# mkdir -p "${HOME}/bin"
# curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b "${HOME}/bin" v0.70.0
# echo "${HOME}/bin" >> "${GITHUB_PATH}"
# "${HOME}/bin/trivy" --version
# Use "${HOME}/bin/trivy" — act/Gitea may not prepend GITHUB_PATH before the next step.
# Gitea only orchestrates the job; checkout + report.json live on the runner machine (this server), not on the Gitea host.
- name: Trivy filesystem scan
run: |
"${HOME}/bin/trivy" fs -f json -o report.json \
--skip-dirs node_modules,android/.gradle,android/build,ios/Pods,ios/build,.git \
--exit-code 0 \
.
report_path="${GITHUB_WORKSPACE:-$(pwd)}/report.json"
echo "Runner host: $(hostname)"
echo "report.json (on this runner, under job workspace): ${report_path}"
ls -la report.json
mkdir -p /home/azureuser/builds
cp -f report.json /home/azureuser/builds/trivy-report.json
echo "Persistent copy (survives after job workspace is removed): /home/azureuser/builds/trivy-report.json"
- name: Upload Trivy report to MongoDB
run: node /home/azureuser/uploadJSONMongoDB/scripts/upload-report-to-mongodb.js
# - name: Upload Trivy report
# uses: actions/upload-artifact@v3
# with:
# name: trivy-fs-report
# path: report.json
# ---------------- SONARQUBE ----------------
# In Gitea: Settings → Secrets → SONAR_TOKEN (and optionally SONAR_URL).
- name: SonarQube Scan
uses: SonarSource/sonarqube-scan-action@v6
env:
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
SONAR_HOST_URL: ${{ secrets.SONAR_URL }}
# ---------------- ANDROID SDK (required on Ubuntu: ANDROID_HOME / sdk.dir) ----------------
- name: Setup Android SDK
uses: android-actions/setup-android@v3
with:
packages: >-
tools platform-tools
platforms;android-36
build-tools;36.0.0
ndk;27.1.12297006
- name: Point Gradle to the SDK
run: |
printf 'sdk.dir=%s\n' "${ANDROID_SDK_ROOT}" > android/local.properties
cat android/local.properties
- name: Grant Gradle execute permission
run: chmod +x android/gradlew
# ---------------- BUILD APK ----------------
- name: Build Release APK
run: |
cd android
./gradlew assembleRelease --stacktrace --info
# ---------------- VERIFY APK ----------------
- name: Check APK Output
run: |
ls -R android/app/build/outputs/apk
# ---------------- SAVE TO VM ----------------
- name: Save APK to VM folder
run: |
mkdir -p /home/azureuser/builds
cp android/app/build/outputs/apk/release/*.apk /home/azureuser/builds/
# ---------------- VERIFY FINAL ----------------
- name: Verify APK in VM
run: |
ls -l /home/azureuser/builds
# ---------------- (OPTIONAL) ARTIFACT ----------------
- name: Upload APK (optional)
uses: actions/upload-artifact@v3
with:
name: app-release
path: android/app/build/outputs/apk/release/*.apk
+1 -1
View File
@@ -2,4 +2,4 @@ module.exports = {
arrowParens: 'avoid',
singleQuote: true,
trailingComma: 'all',
};
};
+8 -1
View File
@@ -1 +1,8 @@
console.log("trest")
console.log("test1")
console.log("trest2")
console.log("trest3")
console.log("trest4")
console.log("trest4")
console.log("trest4")
console.log("trest4")
console.log("trest4")
-63
View File
@@ -1,63 +0,0 @@
# Azure Pipelines — React Native (JS CI)
# https://learn.microsoft.com/azure/devops/pipelines/yaml-schema
# Run on every push directly to main (and when merging into main via PR).
trigger:
branches:
include:
- main
pr:
branches:
include:
- main
variables:
NODE_VERSION: '22.x'
jobs:
- job: BuildAndTest
displayName: Performics DevOps Pipeline
# Self-hosted pool "PerformicsCICD" (agent "myagent" listens via ./run.sh).
pool:
name: PerformicsCICD
steps:
- checkout: self
displayName: Checkout
continueOnError: false
- task: NodeTool@0
displayName: Use Node.js
continueOnError: false
inputs:
versionSpec: $(NODE_VERSION)
- script: |
set -euo pipefail
node -v
npm -v
pwd
displayName: Diagnostics
continueOnError: false
- script: |
set -euo pipefail
npm ci
displayName: npm ci
continueOnError: false
# - script: |
# set -euo pipefail
# npm run lint
# displayName: ESLint
# continueOnError: false
# - script: |
# set -euo pipefail
# cd android
# ./gradlew assembleRelease
# displayName: Android assembleRelease
# continueOnError: false
# - script: npm test -- --ci --coverage
# displayName: Jest
+1
View File
@@ -134,6 +134,7 @@ function WindowIR(props) {
useEffect(() => {
console.log("window IR page init")
console.log("window IR page init")
let params = props.route.params ? props.route.params : {};
let storeData1 = params.storeData || {};
+4 -2
View File
@@ -22,8 +22,10 @@ org.gradle.jvmargs=-Xmx4g -Dfile.encoding=UTF-8
# Enable parallel builds
org.gradle.parallel=true
# Optional: Enable configuration on demand
org.gradle.configureondemand=true
# Must stay false for React Native New Architecture: configure-on-demand can run app
# CMake (Android-autolinking) before library projects run codegen, so
# .../node_modules/.../android/build/generated/source/codegen/jni/ never exists (CXX1429).
org.gradle.configureondemand=false
# When configured, Gradle will run in incubating parallel mode.
# This option should only be used with decoupled projects. More details, visit
+2
View File
@@ -8,7 +8,9 @@
"lint": "eslint .",
"start": "react-native start",
"test": "jest",
"sonar": "npx --yes sonarqube-scanner",
"postinstall": "patch-package",
"clean": "react-native-clean-project",
"start:poll": "react-native start --watchFolders . --reset-cache --no-interactive --config metro.config.js"
},
"dependencies": {
+737
View File
@@ -0,0 +1,737 @@
#!/usr/bin/env bash
# Bulk copy one SQL Server table to ClickHouse on Ubuntu (or Debian-based).
#
# Prerequisites (or run with --install-deps):
# - sqlcmd: Microsoft mssql-tools18 (/opt/mssql-tools18/bin/sqlcmd) or mssql-tools (/opt/mssql-tools/bin)
# - clickhouse-client
# Optional: SQLCMD_PATH=/full/path/to/sqlcmd if installed outside PATH
#
# Environment (required for export-import):
# MSSQL_HOST e.g. db.example.com
# MSSQL_PORT default 1433
# MSSQL_USER
# MSSQL_PASSWORD
# MSSQL_DATABASE
# MSSQL_TRUST_SERVER_CERT set to 1 to pass -C (trust self-signed server cert)
# MSSQL_ENCRYPT optional; passed to sqlcmd as -N (e.g. optional|mandatory|strict)
#
# CH_HOST default localhost
# CH_PORT native TCP port, default 9000
# CH_USER default default
# CH_PASSWORD optional
# CH_SECURE set to 1 to use TLS (--secure)
# MSSQL_INSTALL_DEPS_IF_MISSING set to 1 to auto-run Microsoft ODBC + sqlcmd install when missing (sudo)
#
# Example:
# export MSSQL_HOST=sql.mycompany.internal MSSQL_USER=ro MSSQL_PASSWORD='***' MSSQL_DATABASE=sales
# export CH_HOST=ch.mycompany.internal CH_PASSWORD='***'
# ./mssql_to_clickhouse.sh export-import \
# --mssql-table "dbo.Orders" \
# --ch-database analytics \
# --ch-table orders_raw \
# --out /tmp/orders.tsv
# ./mssql_to_clickhouse.sh migrate-db \
# --ch-database cpm \
# --out-dir /tmp/mssql_full_export
#
set -euo pipefail
# sqlcmd is installed under /opt by mssql-tools18; that path is usually added only in login shells.
# Prepend here so migrate-db works in the same non-login SSH session right after --install-deps.
export PATH="/opt/mssql-tools18/bin:/opt/mssql-tools/bin:${PATH:-}"
SCRIPT_NAME=$(basename "$0")
die() {
echo "$SCRIPT_NAME: $*" >&2
exit 1
}
usage() {
cat <<EOF
$SCRIPT_NAME — export SQL Server data to TSV and load into ClickHouse (Ubuntu/Debian).
Commands:
--install-deps Install mssql-tools18 + clickhouse-client (needs sudo). Run alone FIRST, then migrate-db.
export Export MSSQL table/query to TSV (TabSeparated)
import Load TSV into ClickHouse (INSERT FORMAT TabSeparated)
export-import Run export then import
migrate-db Export and import all MSSQL base tables
list-tables Print table names MSSQL discovery returns (for debugging metadata permissions)
Options:
--mssql-table SCHEMA.TABLE Table for SELECT * (export / export-import)
--mssql-query 'SQL' Raw SELECT (overrides --mssql-table)
--mssql-schema SCHEMA Limit migrate-db to one MSSQL schema
--tables-file PATH Migrate only these SCHEMA.TABLE rows (also env MSSQL_TABLES_FILE).
One per line: SCHEMA.TABLE or SCHEMA<TAB>TABLE (# comments allowed).
Use when MSSQL catalogs hide most tables unless VIEW DEFINITION is granted.
--ch-database DB ClickHouse database (default: \$CH_DATABASE or default)
--ch-table TABLE ClickHouse table name
--out PATH File path (default: ./mssql_export.tsv)
--out-dir PATH Directory for migrate-db TSV files (default: ./mssql_export_all)
--install-deps-if-missing If sqlcmd is not installed, run the same steps as --install-deps first (needs sudo).
Or set env MSSQL_INSTALL_DEPS_IF_MISSING=1 for the same behavior.
Environment:
MSSQL_HOST, MSSQL_PORT, MSSQL_USER, MSSQL_PASSWORD, MSSQL_DATABASE
MSSQL_TRUST_SERVER_CERT=1 (optional; trust self-signed SQL Server cert with sqlcmd -C)
MSSQL_ENCRYPT (optional; forwarded to sqlcmd -N)
MSSQL_TABLES_FILE optional explicit table list path for migrate-db
CH_HOST, CH_PORT, CH_USER, CH_PASSWORD, CH_SECURE=1 for TLS
CH_DOCKER_CONTAINER optional; run clickhouse-client inside this container
SQLCMD_PATH optional; full path to sqlcmd when not on PATH
MSSQL_INSTALL_DEPS_IF_MISSING set to 1 to install mssql-tools18 when sqlcmd is missing (same as --install-deps-if-missing)
Notes:
- ClickHouse columns must match export order and types.
- TabSeparated fails if fields contain tab characters; use --mssql-query to sanitize or export CSV elsewhere.
- Very large tables: run multiple exports with --mssql-query and ranges, then import each file.
EOF
}
ensure_sqlcmd() {
if [[ -n "${SQLCMD_PATH:-}" ]]; then
[[ -f "$SQLCMD_PATH" && -x "$SQLCMD_PATH" ]] || die "SQLCMD_PATH is set but not an executable file: $SQLCMD_PATH"
export PATH="$(dirname "$SQLCMD_PATH"):$PATH"
fi
if command -v sqlcmd >/dev/null 2>&1; then
return 0
fi
die "sqlcmd not found (Microsoft mssql-tools18 is not installed).
Option A — one combined command (will prompt for sudo if needed):
$0 migrate-db --install-deps-if-missing --ch-database ... --out-dir ...
Option B — two steps:
$0 --install-deps
$0 migrate-db ...
Option C — same shell, no flag:
export MSSQL_INSTALL_DEPS_IF_MISSING=1
$0 migrate-db ...
Or set SQLCMD_PATH=/full/path/to/sqlcmd"
}
ensure_clickhouse_client() {
if [[ -n "${CH_DOCKER_CONTAINER:-}" ]]; then
command -v docker >/dev/null 2>&1 || die "docker not found, but CH_DOCKER_CONTAINER is set"
return 0
fi
command -v clickhouse-client >/dev/null 2>&1 || die "clickhouse-client not found. Run: $0 --install-deps or set CH_DOCKER_CONTAINER"
}
ensure_mssql_env() {
[[ -n "${MSSQL_HOST:-}" ]] || die "Set MSSQL_HOST"
[[ -n "${MSSQL_USER:-}" ]] || die "Set MSSQL_USER"
[[ -n "${MSSQL_PASSWORD:-}" ]] || die "Set MSSQL_PASSWORD"
[[ -n "${MSSQL_DATABASE:-}" ]] || die "Set MSSQL_DATABASE"
}
sqlcmd_conn_args() {
local port="${MSSQL_PORT:-1433}"
local server="${MSSQL_HOST},${port}"
local args=(
-S "$server"
-d "$MSSQL_DATABASE"
-U "$MSSQL_USER"
-P "$MSSQL_PASSWORD"
)
# ODBC 18 validates cert chains by default; -C is commonly needed for self-signed certs.
if [[ "${MSSQL_TRUST_SERVER_CERT:-0}" == "1" ]]; then
args+=(-C)
fi
if [[ -n "${MSSQL_ENCRYPT:-}" ]]; then
args+=(-N "$MSSQL_ENCRYPT")
fi
printf '%s\n' "${args[@]}"
}
sqlcmd_query() {
ensure_sqlcmd
ensure_mssql_env
local sql="${1:-}"
mapfile -t conn_args < <(sqlcmd_conn_args)
sqlcmd "${conn_args[@]}" \
-Q "SET NOCOUNT ON; ${sql}" \
-h -1 -W -s "$(printf '\t')" -w 65535 -f i:65001,o:65001 -b
}
run_clickhouse_query() {
ensure_clickhouse_client
local query="${1:-}"
local host="${CH_HOST:-localhost}"
local port="${CH_PORT:-9000}"
local user="${CH_USER:-default}"
local args=(
--host "$host"
--port "$port"
--user "$user"
--query "$query"
)
[[ -n "${CH_PASSWORD:-}" ]] && args+=(--password "$CH_PASSWORD")
[[ "${CH_SECURE:-0}" == "1" ]] && args+=(--secure)
if [[ -n "${CH_DOCKER_CONTAINER:-}" ]]; then
# Important: do not pass -i here, otherwise docker can consume caller stdin
# (e.g. migrate-db table loop) and stop after the first row.
docker exec "$CH_DOCKER_CONTAINER" clickhouse-client "${args[@]}"
else
clickhouse-client "${args[@]}"
fi
}
run_clickhouse_insert() {
ensure_clickhouse_client
local query="${1:-}"
local in_file="${2:-}"
local host="${CH_HOST:-localhost}"
local port="${CH_PORT:-9000}"
local user="${CH_USER:-default}"
local args=(
--host "$host"
--port "$port"
--user "$user"
--query "$query"
)
[[ -n "${CH_PASSWORD:-}" ]] && args+=(--password "$CH_PASSWORD")
[[ "${CH_SECURE:-0}" == "1" ]] && args+=(--secure)
if [[ -n "${CH_DOCKER_CONTAINER:-}" ]]; then
docker exec -i "$CH_DOCKER_CONTAINER" clickhouse-client "${args[@]}" <"$in_file"
else
clickhouse-client "${args[@]}" <"$in_file"
fi
}
escape_ch_ident() {
local ident="${1:-}"
ident="${ident//\`/\`\`}"
printf "%s" "$ident"
}
escape_mssql_ident() {
local ident="${1:-}"
ident="${ident//]/]]}"
printf "%s" "$ident"
}
# Escape single quotes for literals embedded in MSSQL dynamically built strings.
escape_mssql_sql_literal() {
local s="${1:-}"
s="${s//\'/\'\'}"
printf "%s" "$s"
}
strip_cr() {
printf '%s' "${1//$'\r'/}"
}
mssql_discovery_sql_union() {
local schema_filter="${1:-}"
local where_schema_sys=""
local where_schema_is=""
local where_schema_obj=""
local esc=""
if [[ -n "$schema_filter" ]]; then
esc="$(escape_mssql_sql_literal "$schema_filter")"
where_schema_sys="AND s.name = N'${esc}'"
where_schema_is="AND TABLE_SCHEMA = N'${esc}'"
where_schema_obj="AND SCHEMA_NAME(o.schema_id) = N'${esc}'"
fi
printf "%s" "
SELECT DISTINCT
CAST(x.TABLE_SCHEMA AS nvarchar(256)) COLLATE DATABASE_DEFAULT AS TABLE_SCHEMA,
CAST(x.TABLE_NAME AS nvarchar(256)) COLLATE DATABASE_DEFAULT AS TABLE_NAME
FROM (
SELECT s.name AS TABLE_SCHEMA, t.name AS TABLE_NAME
FROM sys.tables t
INNER JOIN sys.schemas s ON s.schema_id = t.schema_id
WHERE t.is_ms_shipped = 0 ${where_schema_sys}
UNION ALL
SELECT TABLE_SCHEMA, TABLE_NAME
FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_TYPE = N'BASE TABLE' ${where_schema_is}
UNION ALL
SELECT SCHEMA_NAME(o.schema_id) AS TABLE_SCHEMA, o.name AS TABLE_NAME
FROM sys.objects o
WHERE o.type = N'U' AND o.is_ms_shipped = 0 ${where_schema_obj}
) AS x
WHERE NULLIF(LTRIM(RTRIM(TABLE_SCHEMA)), N'') IS NOT NULL
AND NULLIF(LTRIM(RTRIM(TABLE_NAME)), N'') IS NOT NULL
ORDER BY TABLE_SCHEMA, TABLE_NAME;
"
}
list_mssql_tables() {
ensure_sqlcmd
ensure_mssql_env
local schema_filter="${1:-}"
sqlcmd_query "$(mssql_discovery_sql_union "$schema_filter")"
}
migrate_permission_hint() {
cat <<EOF >&2
$SCRIPT_NAME: MSSQL catalogs only expose tables your login may see metadata for.
If migrate-db discovers too few rows, grant read + metadata visibility (run as dbo/sa):
USE [$MSSQL_DATABASE];
GRANT VIEW DEFINITION ON DATABASE::[$MSSQL_DATABASE] TO [$MSSQL_USER];
EXEC sp_addrolemember N'db_datareader', N'$MSSQL_USER';
Or migrate using an explicit list from a dbo export:
$SCRIPT_NAME migrate-db --tables-file ./all_tables.txt --ch-database ...
all_tables.txt format (one table per line):
dbo.MyTable
sales.Orders
EOF
}
mssql_column_expr() {
local col_name="${1:-}"
local data_type="${2:-}"
local col_escaped
local col_bracketed
col_escaped="$(escape_mssql_ident "$col_name")"
col_bracketed="[${col_escaped}]"
case "${data_type,,}" in
binary|varbinary|image|rowversion|timestamp)
printf "CASE WHEN %s IS NULL THEN '\\\\N' ELSE master.dbo.fn_varbintohexstr(%s) END AS [%s]" \
"$col_bracketed" "$col_bracketed" "$col_escaped"
;;
*)
printf "CASE WHEN %s IS NULL THEN '\\\\N' ELSE REPLACE(REPLACE(REPLACE(CONVERT(nvarchar(max), %s), CHAR(9), ' '), CHAR(10), ' '), CHAR(13), ' ') END AS [%s]" \
"$col_bracketed" "$col_bracketed" "$col_escaped"
;;
esac
}
build_mssql_export_query() {
local schema_name="${1:-}"
local table_name="${2:-}"
local schema_escaped table_escaped
local exprs=()
local line col_name data_type
schema_escaped="$(escape_mssql_ident "$schema_name")"
table_escaped="$(escape_mssql_ident "$table_name")"
while IFS= read -r line; do
[[ -n "$line" ]] || continue
col_name="${line%%$'\t'*}"
data_type="${line#*$'\t'}"
exprs+=("$(mssql_column_expr "$col_name" "$data_type")")
done < <(
sqlcmd_query "
SELECT COLUMN_NAME, DATA_TYPE
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = '$(escape_mssql_sql_literal "$schema_name")' AND TABLE_NAME = '$(escape_mssql_sql_literal "$table_name")'
ORDER BY ORDINAL_POSITION;
"
)
[[ ${#exprs[@]} -gt 0 ]] || die "No columns found for ${schema_name}.${table_name}"
local select_list
local IFS=", "
select_list="${exprs[*]}"
printf "SELECT %s FROM [%s].[%s]" "$select_list" "$schema_escaped" "$table_escaped"
}
# Parse one line from --tables-file: SCHEMA.TABLE, SCHEMA<TAB>TABLE, or bare name (dbo default).
parse_mssql_table_line() {
local raw="$1"
local -n _sch="$2"
local -n _tbl="$3"
raw="$(strip_cr "$raw")"
[[ -z "$raw" ]] && return 1
[[ "$raw" =~ ^[[:space:]]*# ]] && return 1
raw="${raw#"${raw%%[![:space:]]*}"}"
raw="${raw%"${raw##*[![:space:]]}"}"
[[ -z "$raw" ]] && return 1
if [[ "$raw" == *$'\t'* ]]; then
_sch="${raw%%$'\t'*}"
_tbl="${raw#*$'\t'}"
elif [[ "$raw" == *.* ]]; then
_sch="${raw%%.*}"
_tbl="${raw#*.}"
else
_sch="dbo"
_tbl="$raw"
fi
_sch="${_sch#"${_sch%%[![:space:]]*}"}"
_sch="${_sch%"${_sch##*[![:space:]]}"}"
_tbl="${_tbl#"${_tbl%%[![:space:]]*}"}"
_tbl="${_tbl%"${_tbl##*[![:space:]]}"}"
[[ -n "$_sch" && -n "$_tbl" ]] || return 1
return 0
}
migrate_one_mssql_table() {
local ch_db="${1:?}"
local out_dir="${2:?}"
local schema_name="${3:?}"
local table_name="${4:?}"
local ch_table columns_ddl column_line column_name exported_file export_query
# Use the same table name in ClickHouse as in SQL Server (no schema__ prefix).
# If two schemas contain the same table name, the second migrate would target the same CH table.
ch_table="${table_name}"
columns_ddl=""
while IFS= read -r column_line || [[ -n "$column_line" ]]; do
column_line="$(strip_cr "$column_line")"
[[ -n "$column_line" ]] || continue
column_name="$column_line"
if [[ -n "$columns_ddl" ]]; then
columns_ddl+=", "
fi
columns_ddl+="\`$(escape_ch_ident "$column_name")\` Nullable(String)"
done < <(
sqlcmd_query "
SELECT COLUMN_NAME
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = '$(escape_mssql_sql_literal "$schema_name")' AND TABLE_NAME = '$(escape_mssql_sql_literal "$table_name")'
ORDER BY ORDINAL_POSITION;
"
)
[[ -n "$columns_ddl" ]] || {
echo "Skipping ${schema_name}.${table_name}: no columns readable in INFORMATION_SCHEMA"
return 1
}
run_clickhouse_query "CREATE TABLE IF NOT EXISTS \`$(escape_ch_ident "$ch_db")\`.\`$(escape_ch_ident "$ch_table")\` (${columns_ddl}) ENGINE = MergeTree ORDER BY tuple()"
export_query="$(build_mssql_export_query "$schema_name" "$table_name")"
exported_file="${out_dir}/${schema_name}.${table_name}.tsv"
run_export "" "$export_query" "$exported_file"
run_import "$ch_db" "$ch_table" "$exported_file"
echo "Migrated ${schema_name}.${table_name} -> ${ch_db}.${ch_table}"
return 0
}
run_migrate_db() {
ensure_sqlcmd
ensure_clickhouse_client
ensure_mssql_env
local ch_db="${1:-${CH_DATABASE:-default}}"
local out_dir="${2:-./mssql_export_all}"
local schema_filter="${3:-}"
local tables_file="${4:-}"
local line schema_name table_name tables_rows
local discovered=0
local migrated=0
local skipped=0
local use_file=0
[[ -n "$ch_db" ]] || die "Provide --ch-database or CH_DATABASE"
mkdir -p "$out_dir"
run_clickhouse_query "CREATE DATABASE IF NOT EXISTS \`$(escape_ch_ident "$ch_db")\`"
if [[ -n "$tables_file" ]]; then
[[ -f "$tables_file" ]] || die "Tables list file not found: $tables_file"
use_file=1
while IFS= read -r line || [[ -n "$line" ]]; do
schema_name=""
table_name=""
parse_mssql_table_line "$line" schema_name table_name || continue
discovered=$((discovered + 1))
if migrate_one_mssql_table "$ch_db" "$out_dir" "$schema_name" "$table_name"; then
migrated=$((migrated + 1))
else
skipped=$((skipped + 1))
fi
done <"$tables_file"
else
tables_rows="$(sqlcmd_query "$(mssql_discovery_sql_union "$schema_filter")")"
while IFS= read -r line || [[ -n "$line" ]]; do
line="$(strip_cr "$line")"
[[ -n "$line" ]] || continue
schema_name="${line%%$'\t'*}"
table_name="${line#*$'\t'}"
schema_name="$(strip_cr "$schema_name")"
table_name="$(strip_cr "$table_name")"
schema_name="${schema_name#"${schema_name%%[![:space:]]*}"}"
schema_name="${schema_name%"${schema_name##*[![:space:]]}"}"
table_name="${table_name#"${table_name%%[![:space:]]*}"}"
table_name="${table_name%"${table_name##*[![:space:]]}"}"
[[ -n "$schema_name" && -n "$table_name" ]] || continue
discovered=$((discovered + 1))
if migrate_one_mssql_table "$ch_db" "$out_dir" "$schema_name" "$table_name"; then
migrated=$((migrated + 1))
else
skipped=$((skipped + 1))
fi
done <<<"$tables_rows"
fi
if [[ "$discovered" -le 0 ]]; then
if [[ "$use_file" -eq 1 ]]; then
die "No valid SCHEMA.TABLE rows in tables file (after skipping blanks/comments): ${tables_file}"
else
die "No SQL Server tables discovered in ${MSSQL_DATABASE}. Run: ${SCRIPT_NAME} list-tables (or grant VIEW DEFINITION + db_datareader)."
fi
fi
if [[ "$migrated" -eq 0 ]]; then
migrate_permission_hint
die "No tables migrated successfully (had $discovered candidate(s), $skipped skipped)."
fi
if [[ "$use_file" -eq 0 && "$discovered" -eq 1 ]]; then
migrate_permission_hint
fi
echo "Done. Queued $discovered table(s), migrated $migrated, skipped/failed-metadata $skipped into ClickHouse database ${ch_db}."
}
install_deps() {
[[ $(id -u) -eq 0 ]] || command -v sudo >/dev/null 2>&1 || die "Need sudo for --install-deps"
local SUDO=""
[[ $(id -u) -ne 0 ]] && SUDO="sudo"
. /etc/os-release || die "Cannot read /etc/os-release"
[[ "${ID:-}" == "ubuntu" || "${ID:-}" == "debian" || "${ID_LIKE:-}" == *"debian"* ]] \
|| die "This installer expects Ubuntu or Debian."
export DEBIAN_FRONTEND=noninteractive
$SUDO apt-get update
$SUDO apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release
# Microsoft ODBC + sqlcmd
curl -fsSL https://packages.microsoft.com/keys/microsoft.asc \
| $SUDO gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg
local ms_rel ms_url
ms_rel="$(lsb_release -rs)"
if [[ "${ID:-}" == "debian" ]]; then
ms_url="https://packages.microsoft.com/config/debian/${ms_rel}/prod.list"
if ! curl -fsSL "$ms_url" -o /tmp/mssql-release.list 2>/dev/null; then
echo "$SCRIPT_NAME: No Microsoft prod.list for Debian ${ms_rel}; trying 12 (bookworm)." >&2
ms_url="https://packages.microsoft.com/config/debian/12/prod.list"
curl -fsSL "$ms_url" -o /tmp/mssql-release.list
fi
else
ms_url="https://packages.microsoft.com/config/ubuntu/${ms_rel}/prod.list"
if ! curl -fsSL "$ms_url" -o /tmp/mssql-release.list 2>/dev/null; then
echo "$SCRIPT_NAME: No Microsoft prod.list for Ubuntu ${ms_rel}; trying 22.04 repo." >&2
ms_url="https://packages.microsoft.com/config/ubuntu/22.04/prod.list"
curl -fsSL "$ms_url" -o /tmp/mssql-release.list
fi
fi
$SUDO mv /tmp/mssql-release.list /etc/apt/sources.list.d/mssql-release.list
$SUDO apt-get update
ACCEPT_EULA=Y $SUDO apt-get install -y msodbcsql18 mssql-tools18 unixodbc-dev
export PATH="/opt/mssql-tools18/bin:/opt/mssql-tools/bin:${PATH:-}"
if ! command -v sqlcmd >/dev/null 2>&1; then
die "apt reported success but sqlcmd is still missing. Inspect package contents:
dpkg -L mssql-tools18 2>/dev/null | grep -E '/sqlcmd$' || true
Try: sudo apt-get install -f && sudo apt-get install -y msodbcsql18 mssql-tools18"
fi
local path_line='export PATH="$PATH:/opt/mssql-tools18/bin"'
grep -qF 'mssql-tools18/bin' "$HOME/.bashrc" 2>/dev/null || echo "$path_line" >>"$HOME/.bashrc" || true
grep -qF 'mssql-tools18/bin' "$HOME/.profile" 2>/dev/null || echo "$path_line" >>"$HOME/.profile" || true
# ClickHouse client
$SUDO mkdir -p /etc/apt/keyrings
curl -fsSL https://packages.clickhouse.com/rpm/lifecyclePolicies/policy.json >/dev/null 2>&1 || true
curl -fsSL https://packages.clickhouse.com/deb/pubkey.gpg \
| $SUDO gpg --dearmor -o /etc/apt/keyrings/clickhouse.gpg
echo "deb [signed-by=/etc/apt/keyrings/clickhouse.gpg] https://packages.clickhouse.com/deb stable main" \
| $SUDO tee /etc/apt/sources.list.d/clickhouse.list >/dev/null
$SUDO apt-get update
$SUDO apt-get install -y clickhouse-client
echo "Installed. Open a new shell or run: export PATH=\"\$PATH:/opt/mssql-tools18/bin\""
}
run_export() {
ensure_sqlcmd
ensure_mssql_env
local mssql_table="${1:-}"
local mssql_query="${2:-}"
local out_file="${3:-./mssql_export.tsv}"
local conn_args=()
mapfile -t conn_args < <(sqlcmd_conn_args)
local sql
if [[ -n "$mssql_query" ]]; then
sql="$mssql_query"
else
[[ -n "$mssql_table" ]] || die "Provide --mssql-table or --mssql-query"
sql="SELECT * FROM ${mssql_table}"
fi
mkdir -p "$(dirname "$out_file")"
# UTF-8 in/out; tab separator; no column headers (-h -1); trim trailing spaces (-W).
sqlcmd "${conn_args[@]}" \
-Q "SET NOCOUNT ON; ${sql}" \
-h -1 -W -s "$(printf '\t')" -w 65535 -f i:65001,o:65001 -o "$out_file" -b
echo "Exported to $out_file ($(wc -l <"$out_file") lines)"
}
run_import() {
local ch_db="${1:-${CH_DATABASE:-default}}"
local ch_table="${2:-}"
local in_file="${3:-./mssql_export.tsv}"
[[ -n "$ch_table" ]] || die "Provide --ch-table"
[[ -f "$in_file" ]] || die "File not found: $in_file"
run_clickhouse_insert "INSERT INTO \`$(escape_ch_ident "$ch_db")\`.\`$(escape_ch_ident "$ch_table")\` FORMAT TabSeparated" "$in_file"
echo "Imported $in_file into ${ch_db}.${ch_table}"
}
main() {
[[ $# -ge 1 ]] || {
usage
exit 1
}
local cmd="$1"
shift || true
if [[ "$cmd" == "-h" || "$cmd" == "--help" ]]; then
usage
exit 0
fi
if [[ "$cmd" == "--install-deps" ]]; then
install_deps
exit 0
fi
local mssql_table=""
local mssql_query=""
local mssql_schema=""
local ch_database=""
local ch_table=""
local out_file="./mssql_export.tsv"
local out_dir="./mssql_export_all"
local tables_file="${MSSQL_TABLES_FILE:-}"
local install_deps_if_missing=0
while [[ $# -gt 0 ]]; do
case "$1" in
--install-deps-if-missing)
install_deps_if_missing=1
shift
;;
--mssql-table)
mssql_table="$2"
shift 2
;;
--mssql-query)
mssql_query="$2"
shift 2
;;
--ch-database)
ch_database="$2"
shift 2
;;
--mssql-schema)
mssql_schema="$2"
shift 2
;;
--ch-table)
ch_table="$2"
shift 2
;;
--out)
out_file="$2"
shift 2
;;
--out-dir)
out_dir="$2"
shift 2
;;
--tables-file)
tables_file="$2"
shift 2
;;
*)
die "Unknown option: $1"
;;
esac
done
local needs_sqlcmd=0
case "$cmd" in
migrate-db | list-tables | export | export-import) needs_sqlcmd=1 ;;
esac
if [[ "$needs_sqlcmd" -eq 1 ]]; then
if [[ "${MSSQL_INSTALL_DEPS_IF_MISSING:-0}" == "1" || "$install_deps_if_missing" -eq 1 ]]; then
if ! command -v sqlcmd >/dev/null 2>&1; then
echo "$SCRIPT_NAME: sqlcmd not found; running install_deps (Microsoft ODBC + mssql-tools18; needs sudo)..." >&2
install_deps
fi
fi
fi
case "$cmd" in
export)
run_export "$mssql_table" "$mssql_query" "$out_file"
;;
import)
run_import "${ch_database:-${CH_DATABASE:-default}}" "$ch_table" "$out_file"
;;
export-import)
[[ -n "$ch_table" ]] || die "export-import requires --ch-table"
run_export "$mssql_table" "$mssql_query" "$out_file"
run_import "${ch_database:-${CH_DATABASE:-default}}" "$ch_table" "$out_file"
;;
list-tables)
list_mssql_tables "$mssql_schema"
;;
migrate-db)
run_migrate_db "${ch_database:-${CH_DATABASE:-default}}" "$out_dir" "$mssql_schema" "$tables_file"
;;
*)
usage
die "Unknown command: $cmd"
;;
esac
}
main "$@"
+174
View File
@@ -0,0 +1,174 @@
#!/usr/bin/env bash
#
# Install ClickHouse server + client on Debian/Ubuntu from packages.clickhouse.com.
# Avoids Ubuntu "universe" ClickHouse 18.x, which is outdated and often misconfigured.
#
# Usage (from repo root or any path):
# bash scripts/setup-clickhouse.sh
# bash scripts/setup-clickhouse.sh --purge-existing # one line; do not split after "--"
# bash scripts/setup-clickhouse.sh --listen-all-interfaces # HTTP 8123 / native 9000 on 0.0.0.0 (see firewall)
# CLICKHOUSE_CHANNEL=lts bash scripts/setup-clickhouse.sh
#
# Env:
# CLICKHOUSE_CHANNEL stable (default) or lts — see https://clickhouse.com/docs/knowledgebase/production
# WAIT_FOR_APT_LOCK_SEC max seconds to wait for apt/dpkg lock (default: 900)
#
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
WAIT_FOR_APT_LOCK_SEC="${WAIT_FOR_APT_LOCK_SEC:-900}"
APT_LOCK_SLEEP_SEC="${APT_LOCK_SLEEP_SEC:-5}"
# Wait until dpkg/apt can run (avoids E: Could not get lock ... lock-frontend).
# Uses non-blocking flock(1) on the same lock files apt/dpkg use — not fuser on lists/archives
# locks, which can look "busy" for a long time and cause endless waits.
wait_for_apt_lock() {
local waited=0
local last_holder_line=-1
while true; do
if _dpkg_locks_available; then
return 0
fi
if (( waited >= WAIT_FOR_APT_LOCK_SEC )); then
echo "Error: timed out after ${WAIT_FOR_APT_LOCK_SEC}s waiting for apt/dpkg lock." >&2
echo "Check what holds the lock:" >&2
echo " sudo fuser -v /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock 2>&1 | head -20" >&2
echo " sudo ps aux | egrep '[a]pt|[d]pkg'" >&2
echo " sudo systemctl status unattended-upgrades" >&2
echo "Then re-run, or set WAIT_FOR_APT_LOCK_SEC=3600 to wait longer." >&2
exit 1
fi
echo "==> Waiting for apt/dpkg lock to clear (${waited}s / ${WAIT_FOR_APT_LOCK_SEC}s max)..."
if (( waited >= 60 && waited != last_holder_line && waited % 60 == 0 )); then
last_holder_line=$waited
if command -v fuser >/dev/null 2>&1; then
echo " (lock holders: $(sudo fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock 2>/dev/null | tr -s ' ' || true))" >&2
fi
fi
sleep "$APT_LOCK_SLEEP_SEC"
waited=$((waited + APT_LOCK_SLEEP_SEC))
done
}
# True if we can take an exclusive flock on dpkg locks right now (same mechanism apt uses).
_dpkg_locks_available() {
if ! command -v flock >/dev/null 2>&1; then
echo "Error: flock(1) not found (install util-linux)." >&2
exit 1
fi
local f
for f in /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock; do
[[ -e "$f" ]] || continue
if ! sudo flock -n "$f" true 2>/dev/null; then
return 1
fi
done
return 0
}
apt_get() {
wait_for_apt_lock
sudo apt-get "$@"
}
CLICKHOUSE_LIST="/etc/apt/sources.list.d/clickhouse.list"
KEYRING="/usr/share/keyrings/clickhouse-keyring.gpg"
CHANNEL="${CLICKHOUSE_CHANNEL:-stable}"
PURGE_EXISTING=false
LISTEN_ALL_INTERFACES=false
for arg in "$@"; do
case "$arg" in
--purge-existing) PURGE_EXISTING=true ;;
--listen-all-interfaces) LISTEN_ALL_INTERFACES=true ;;
-h|--help)
echo "Usage: $0 [--purge-existing] [--listen-all-interfaces]"
echo " --purge-existing apt purge existing clickhouse* packages before installing (destructive for old installs)."
echo " --listen-all-interfaces bind HTTP (8123) and native (9000) on 0.0.0.0 — open cloud NSG/ufw + use strong passwords."
echo " Env: CLICKHOUSE_CHANNEL=stable|lts (default: stable), WAIT_FOR_APT_LOCK_SEC (default: 900)"
exit 0
;;
*)
echo "Unknown option: $arg (try --help)" >&2
exit 1
;;
esac
done
if [[ ! -f /etc/os-release ]]; then
echo "Error: /etc/os-release not found. This script supports Debian/Ubuntu only." >&2
exit 1
fi
# shellcheck source=/dev/null
source /etc/os-release
case "${ID:-},${ID_LIKE:-}" in
debian,*|ubuntu,*|*,*debian*|*,*ubuntu*) ;;
*)
echo "Warning: OS ID=${ID:-unknown} may not be Debian/Ubuntu. Continuing if apt works." >&2
;;
esac
if [[ "$CHANNEL" != "stable" && "$CHANNEL" != "lts" ]]; then
echo "Error: CLICKHOUSE_CHANNEL must be stable or lts, got: $CHANNEL" >&2
exit 1
fi
if [[ "$PURGE_EXISTING" == "true" ]]; then
echo "==> Purging existing ClickHouse packages (--purge-existing)..."
sudo systemctl stop clickhouse-server 2>/dev/null || true
sudo systemctl stop clickhouse-keeper 2>/dev/null || true
apt_get purge -y 'clickhouse*' || true
apt_get autoremove -y || true
fi
echo "==> Installing prerequisites..."
export DEBIAN_FRONTEND=noninteractive
apt_get update -qq
apt_get install -y apt-transport-https ca-certificates curl gnupg
echo "==> Adding ClickHouse APT repo (channel: $CHANNEL)..."
sudo mkdir -p /usr/share/keyrings
if [[ ! -f "$KEYRING" ]]; then
curl -fsSL 'https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key' \
| sudo gpg --dearmor -o "$KEYRING"
fi
ARCH="$(dpkg --print-architecture)"
echo "deb [signed-by=${KEYRING} arch=${ARCH}] https://packages.clickhouse.com/deb ${CHANNEL} main" \
| sudo tee "$CLICKHOUSE_LIST" >/dev/null
echo "==> Installing clickhouse-server and clickhouse-client..."
apt_get update -qq
apt_get install -y clickhouse-server clickhouse-client
echo "==> Enabling and starting clickhouse-server..."
sudo systemctl enable --now clickhouse-server
sudo systemctl --no-pager --full status clickhouse-server || true
if [[ "$LISTEN_ALL_INTERFACES" == "true" ]]; then
echo "==> Binding ClickHouse to all IPv4 interfaces (0.0.0.0) for remote HTTP/native..."
sudo tee /etc/clickhouse-server/config.d/99-listen-all-interfaces.xml >/dev/null <<'EOF'
<clickhouse>
<!-- Default install often listens only on loopback; this allows e.g. http://SERVER_IP:8123 from other hosts. -->
<listen_host>0.0.0.0</listen_host>
</clickhouse>
EOF
sudo systemctl restart clickhouse-server
sudo systemctl --no-pager --full status clickhouse-server || true
echo " Open inbound TCP 8123 (and 9000 if you use native clients) in your cloud NSG / security group and ufw."
fi
echo "==> Verifying..."
if clickhouse-client -q "SELECT version(), 1" 2>/dev/null; then
echo "ClickHouse responded OK."
else
echo "Note: clickhouse-client check failed (password required?). Try: clickhouse-client --password -q \"SELECT 1\"" >&2
echo "Logs: sudo tail -50 /var/log/clickhouse-server/clickhouse-server.err.log" >&2
exit 1
fi
echo ""
echo "Done. Client: clickhouse-client | HTTP default :8123 | native default :9000"
if [[ "$LISTEN_ALL_INTERFACES" != "true" ]]; then
echo "Remote browser/API to this host: re-run with --listen-all-interfaces or add listen_host in /etc/clickhouse-server/config.d/"
echo " (default is often localhost-only, so http://PUBLIC_IP:8123 will not work until you do)."
fi
+164
View File
@@ -0,0 +1,164 @@
#!/usr/bin/env bash
#
# Ubuntu (22.04+) setup for this React Native 0.81 / Android build (headless or desktop).
# Run: bash scripts/setup-ubuntu.sh
# or: bash scripts/setup-ubuntu.sh --skip-android # if ANDROID_HOME is already set up
# Fix "SDK location not found" on an existing box: bash scripts/write-android-local-properties.sh
#
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
# Match android/build.gradle (update here if the project changes)
COMPILE_SDK_VERSION=36
BUILD_TOOLS_VERSION="36.0.0"
NDK_VERSION="27.1.12297006"
CMAKE_VERSION="3.22.1"
# Android command-line tools (update periodically): https://developer.android.com/studio#command-line-tools-only
CMDLINE_TOOLS_ZIP_URL="${CMDLINE_TOOLS_ZIP_URL:-https://dl.google.com/android/repository/commandlinetools-linux-12266719_latest.zip}"
# Node: use 20 LTS (compatible with RN 0.81). Override: NODE_VERSION=22 bash ...
NODE_VERSION="${NODE_VERSION:-20}"
ANDROID_HOME_DEFAULT="${HOME}/Android/Sdk"
ANDROID_HOME="${ANDROID_HOME:-$ANDROID_HOME_DEFAULT}"
SKIP_ANDROID=false
INSTALL_NVM=true
for arg in "$@"; do
case "$arg" in
--skip-android) SKIP_ANDROID=true ;;
--no-nvm) INSTALL_NVM=false ;;
-h|--help)
echo "Usage: $0 [--skip-android] [--no-nvm]"
echo " Env: ANDROID_HOME (default: $ANDROID_HOME_DEFAULT), NODE_VERSION (default: $NODE_VERSION), CMDLINE_TOOLS_ZIP_URL"
exit 0
;;
esac
done
echo "==> Installing apt packages (needs sudo)..."
sudo apt-get update
sudo apt-get install -y \
openjdk-17-jdk \
build-essential \
git \
curl \
unzip \
libssl-dev \
pkg-config \
ca-certificates
# Watchman (Metro file watching). Optional; skip if the package is missing on your release.
if apt-cache show watchman &>/dev/null; then
sudo apt-get install -y watchman || true
else
echo " (Optional) Install watchman from source or use Metro polling if needed."
fi
if [[ "$INSTALL_NVM" == "true" ]]; then
echo "==> Installing nvm + Node $NODE_VERSION..."
export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"
if [[ ! -s "$NVM_DIR/nvm.sh" ]]; then
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
fi
# shellcheck source=/dev/null
source "$NVM_DIR/nvm.sh"
nvm install "$NODE_VERSION"
nvm use "$NODE_VERSION"
nvm alias default "$NODE_VERSION"
node -v
corepack enable 2>/dev/null || true
else
echo "==> Skipping nvm (--no-nvm). Ensure Node 18+ is on PATH."
fi
if [[ "$SKIP_ANDROID" == "true" ]]; then
echo "==> Skipping Android SDK (--skip-android). Set ANDROID_HOME and ensure SDK/NDK match the project."
echo " Required: platforms;android-$COMPILE_SDK_VERSION, build-tools;$BUILD_TOOLS_VERSION, ndk;$NDK_VERSION, cmake;$CMAKE_VERSION"
if [[ -n "${ANDROID_HOME:-}" && -d "${ANDROID_HOME}" ]]; then
bash "$SCRIPT_DIR/write-android-local-properties.sh"
else
echo " After setting ANDROID_HOME, run: bash $SCRIPT_DIR/write-android-local-properties.sh"
fi
exit 0
fi
echo "==> Installing Android SDK to $ANDROID_HOME..."
if [[ -z "${JAVA_HOME:-}" ]]; then
if [[ -d /usr/lib/jvm/java-17-openjdk-amd64 ]]; then
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
else
JAVA_17=$(dirname "$(dirname "$(readlink -f "$(command -v java)")")")
export JAVA_HOME="$JAVA_17"
fi
fi
echo " JAVA_HOME=$JAVA_HOME"
WORKDIR="$(mktemp -d)"
TMP_ZIP="${WORKDIR}/android-cmdline-tools.zip"
curl -L -f -o "$TMP_ZIP" "$CMDLINE_TOOLS_ZIP_URL"
unzip -q -o "$TMP_ZIP" -d "$WORKDIR"
# Google zip contains a top-level `cmdline-tools` dir; it must end up as .../cmdline-tools/latest/...
if [[ -d "$WORKDIR/cmdline-tools" ]]; then
rm -rf "$ANDROID_HOME/cmdline-tools/latest"
mkdir -p "$ANDROID_HOME/cmdline-tools"
mv "$WORKDIR/cmdline-tools" "$ANDROID_HOME/cmdline-tools/latest"
else
echo "Error: expected folder cmdline-tools/ inside the downloaded zip. Update CMDLINE_TOOLS_ZIP_URL."
rm -rf "$WORKDIR"
exit 1
fi
rm -rf "$WORKDIR"
export PATH="$PATH:$ANDROID_HOME/cmdline-tools/latest/bin:$ANDROID_HOME/platform-tools"
SDKMGR=("$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager")
echo " Accepting licenses..."
yes | "${SDKMGR[@]}" --licenses --sdk_root="$ANDROID_HOME" >/dev/null 2>&1 || yes | "${SDKMGR[@]}" --licenses
echo " Installing packages (this may take a while)..."
"${SDKMGR[@]}" --sdk_root="$ANDROID_HOME" \
"platforms;android-${COMPILE_SDK_VERSION}" \
"build-tools;${BUILD_TOOLS_VERSION}" \
"platform-tools" \
"ndk;${NDK_VERSION}" \
"cmake;${CMAKE_VERSION}"
# Gradle requires sdk.dir in android/local.properties (or ANDROID_HOME in every session).
if [[ -f "$SCRIPT_DIR/write-android-local-properties.sh" ]]; then
export ANDROID_HOME
bash "$SCRIPT_DIR/write-android-local-properties.sh" || {
echo " (Could not auto-write local.properties; run: bash $SCRIPT_DIR/write-android-local-properties.sh )"
}
else
printf '## This file is machine-specific (not committed).\nsdk.dir=%s\n' "$ANDROID_HOME" > "${REPO_ROOT}/android/local.properties"
echo "==> Wrote ${REPO_ROOT}/android/local.properties"
fi
echo ""
echo "==> Shell exports (add to ~/.bashrc or ~/.zshrc):"
{
echo ""
echo "# React Native / Android"
echo "export JAVA_HOME=\"${JAVA_HOME}\""
echo "export ANDROID_HOME=\"${ANDROID_HOME}\""
echo "export PATH=\"\$PATH:\$ANDROID_HOME/platform-tools:\$ANDROID_HOME/emulator\""
echo "export PATH=\"\$PATH:\$ANDROID_HOME/cmdline-tools/latest/bin\""
if [[ "$INSTALL_NVM" == "true" ]]; then
echo "export NVM_DIR=\"\$HOME/.nvm\""
echo "[ -s \"\$NVM_DIR/nvm.sh\" ] && . \"\$NVM_DIR/nvm.sh\""
fi
}
echo ""
echo "==> Next: open a new shell (or source your rc file), then from the repo:"
echo " npm ci # or npm install / yarn / pnpm"
echo " cd android && ./gradlew :app:assembleRelease"
echo " (If you still see missing codegen/jni, use: ./gradlew :app:assembleRelease --no-configure-on-demand --no-parallel)"
echo ""
echo "Done. This project needs New Architecture: keep org.gradle.configureondemand=false in android/gradle.properties."
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env bash
# Create android/local.properties with sdk.dir so Gradle finds the Android SDK.
# Run from repo root: ./scripts/write-android-local-properties.sh
# or: ANDROID_HOME=/path/to/Sdk ./scripts/write-android-local-properties.sh
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
ANDROID_DIR="${REPO_ROOT}/android"
LOCAL_PROPS="${ANDROID_DIR}/local.properties"
SDK_DIR="${ANDROID_HOME:-${HOME}/Android/Sdk}"
if [[ ! -d "$SDK_DIR" ]]; then
echo "Error: Android SDK not found at: $SDK_DIR" >&2
echo "Set ANDROID_HOME to your SDK path (e.g. export ANDROID_HOME=\"\$HOME/Android/Sdk\")." >&2
exit 1
fi
if command -v realpath &>/dev/null; then
SDK_DIR=$(realpath "$SDK_DIR")
else
SDK_DIR=$(readlink -f "$SDK_DIR" 2>/dev/null || echo "$SDK_DIR")
fi
if [[ ! -d "$ANDROID_DIR" ]]; then
echo "Error: expected android/ at: $ANDROID_DIR" >&2
exit 1
fi
printf '## This file is machine-specific (not committed). Points Gradle to the Android SDK.\nsdk.dir=%s\n' "$SDK_DIR" > "$LOCAL_PROPS"
echo "Wrote $LOCAL_PROPS"
echo "sdk.dir=$SDK_DIR"
+6
View File
@@ -0,0 +1,6 @@
# Auth: use SONAR_TOKEN (env) or -Dsonar.token=... — never commit tokens or sonar.login here.
sonar.projectKey=performics
sonar.sources=.
sonar.host.url=http://98.70.31.53:9000
# From CLI intent: skip Java/Kotlin and the Android tree; **/ forms cover the repo (not only repo root).
sonar.exclusions=**/*.java,**/*.kt,**/android/**,**/node_modules/**,**/ios/Pods/**,**/ios/build/**
+5
View File
@@ -5577,6 +5577,11 @@ fs.realpath@^1.0.0:
resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz"
integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==
fsevents@^2.3.2:
version "2.3.3"
resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz"
integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==
function-bind@^1.1.2:
version "1.1.2"
resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz"