diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 64972b44..c150bbf0 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -105,6 +105,66 @@ jobs: - name: Build run: make linux + # Fans the DEFAULT pure-Go driver (CGO_ENABLED=0, Thrift backend) out to macOS + # and Windows so the cross-platform surface is exercised per-PR — the Linux + # build-and-test job above already covers linux/amd64 on the protected runners. + # The pure-Go build has no cgo and no kernel .a, so it needs no Rust toolchain, + # no C compiler, and no private-repo access; it just has to compile and pass its + # unit tests on each OS. The SEA-via-kernel backend is NOT built here (it stays + # Linux-only for now — see build-and-test-kernel below and the distribution + # design doc's per-OS rollout). + # + # These run on GitHub-hosted macos-latest (arm64) / windows-latest (amd64) via + # runs-on: ${{ matrix.os }}, NOT the databricks-protected-runner-group (which is + # linux-only). That mirrors the sibling databricks-adbc Go driver's go_test.yaml, + # which already tests on hosted macos-latest + windows-latest. Hosted runners have + # open egress and every module dependency is public (go.mod has no private + # databricks modules), so setup-jfrog / the JFrog Go proxy is deliberately omitted + # — the default public proxy resolves everything. + build-and-test-cross-os: + name: Test (${{ matrix.os }}, Go ${{ matrix.go-version }}) + strategy: + # Don't let one OS's failure cancel the others — we want the full + # cross-platform signal on every run while this job stabilizes. + fail-fast: false + matrix: + os: [macos-latest, windows-latest] + # Same support window as build-and-test. Hosted runners don't pin + # GOTOOLCHAIN=local, but keep the versions identical for parity. + go-version: ['1.25.x', '1.26.x'] + runs-on: ${{ matrix.os }} + # bash on every OS (git-bash on Windows) so the run: blocks below are portable. + defaults: + run: + shell: bash + + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Go Toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: ${{ matrix.go-version }} + # Let setup-go manage the module/build cache here — the bespoke + # actions/cache step in build-and-test is tuned to the Linux runner's + # paths; the built-in cache is portable across macOS/Windows. + cache: true + cache-dependency-path: go.sum + + # Build + unit-test the pure-Go driver directly with the go tool rather than + # `make test`: the Makefile's test target shells out to gotestsum + writes + # JUnit XML and is oriented at the Linux CI, whereas `go build`/`go test` are + # uniform across macOS and Windows and need no extra tooling installed. Race + # detection is intentionally NOT run here — it requires cgo + a C toolchain, + # which cuts against this job's toolchain-free goal, and it's already covered + # on Linux by build-and-test's Test-Race step. + - name: Build + run: CGO_ENABLED=0 go build -v ./... + + - name: Test + run: CGO_ENABLED=0 go test -count=1 ./... + build-and-test-kernel: name: Test (kernel backend) # Exercises the opt-in SEA-via-kernel backend: builds the Rust kernel static @@ -249,3 +309,252 @@ jobs: # the separate nightly-e2e workflow. - name: Build kernel lib + run tagged tests run: make test-kernel + + # Exercises the SEA-via-kernel backend on macOS (darwin/arm64) — the second + # kernel OS after linux/amd64, per the distribution design's one-OS-at-a-time + # rollout. A green run validates cgo_darwin.go's link flags (positional .a for + # Apple ld64, -lc++, @loader_path), which are otherwise never linked in CI. + # + # GATED OFF by default (vars.KERNEL_MACOS_CI_ENABLED == 'true'), because it is + # blocked by org infrastructure, NOT by the driver code: + # + # The kernel repo (databricks/databricks-sql-kernel) is PRIVATE, so building + # its .a requires a GitHub App token to clone it. The databricks org enforces + # an IP allow list, and GitHub-HOSTED macos-latest runners have public IPs that + # are NOT allow-listed, so create-github-app-token fails with "your IP address + # is not permitted to access this resource" before cargo/cgo ever run. The + # linux kernel job avoids this only because it runs on the self-hosted, + # allow-listed databricks-protected-runner-group — which offers a + # windows-server-latest label (see the Windows kernel job) but NO macOS label + # anywhere in the org today. + # + # Two ways to turn this on (then set the repo var to 'true'): + # 1. An allow-listed self-hosted macOS/arm64 runner becomes available — point + # runs-on at it (the App-token step then succeeds); OR + # 2. The kernel publishes a downloadable darwin_arm64 .a (the distribution + # design's kernel-lib-download path) — then drop the App-token + private + # clone entirely and this runs on hosted macos-latest with open egress. + # + # The job body below is otherwise ready: structurally the linux kernel job MINUS + # the two JFrog steps (kernel deps are all public crates.io and go.mod deps are + # all public, so a hosted runner needs no JFrog Go proxy / cargo shim), KEEPING + # the App-token step for the private clone. macos-latest is arm64 = cgo_darwin.go's + # `darwin && arm64` tag, so host == target (no --target); aarch64-apple-darwin is + # Rust tier-1 (bundled with 1.96.1) and Xcode CLT supplies cc + libc++. + build-and-test-kernel-macos: + name: Test (kernel backend, macOS) + # Default-off: see the block comment above. Flip vars.KERNEL_MACOS_CI_ENABLED + # to 'true' once an allow-listed macOS runner or a downloadable kernel .a exists. + if: ${{ vars.KERNEL_MACOS_CI_ENABLED == 'true' }} + timeout-minutes: 30 + concurrency: + group: kernel-build-macos-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + runs-on: macos-latest + defaults: + run: + shell: bash + + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Go Toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: '1.25.x' + cache: false + + # Keep in lockstep with rust-toolchain.toml's channel (it governs the archive + # under a fixed KERNEL_REV; a floating `stable` here would drift it). + - name: Set up Rust Toolchain + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 + with: + toolchain: 1.96.1 + + # The kernel repo (databricks/databricks-sql-kernel) is private and the hosted + # runner has no ambient git credentials, so kernel-lib.sh's fetch would fail + # with "could not read Username". Mint a repo-scoped token from the + # INTEGRATION_TEST_APP GitHub App and rewrite the kernel HTTPS URL to carry it + # (same mechanism as the linux kernel job). This is repo auth only — crate + # resolution goes to public crates.io, so no cargo→JFrog shim is needed here. + - name: Generate GitHub App token for databricks-sql-kernel + id: kernel-token + uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 + with: + app-id: ${{ secrets.INTEGRATION_TEST_APP_ID }} + private-key: ${{ secrets.INTEGRATION_TEST_PRIVATE_KEY }} + owner: databricks + repositories: databricks-sql-kernel + + - name: Rewrite kernel repo URL to authenticated HTTPS + env: + TOKEN: ${{ steps.kernel-token.outputs.token }} + run: | + git config --global \ + url."https://x-access-token:${TOKEN}@github.com/databricks/".insteadOf \ + "https://github.com/databricks/" + + # Cache keyed on runner.os (macOS) so it never collides with the linux job's + # cache; the .a is platform-specific. + - name: Cache kernel static lib + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + internal/backend/kernel/lib + internal/backend/kernel/include + key: ${{ runner.os }}-kernellib-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + + - name: Cache cargo + kernel build tree + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + build/kernel-src/target + key: ${{ runner.os }}-kernel-cargo-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + restore-keys: | + ${{ runner.os }}-kernel-cargo- + + # make test-kernel builds the kernel lib (make kernel-lib) if the cache missed + # — cargo build on darwin/arm64 (host==target) — then runs + # `CGO_ENABLED=1 go test -tags databricks_kernel ./...`, linking via + # cgo_darwin.go. make + cc + libc++ are preinstalled on macos-latest (Xcode CLT). + - name: Build kernel lib + run tagged tests + run: make test-kernel + + # Exercises the SEA-via-kernel backend on Windows (windows/amd64) — the third + # kernel OS. A green run validates cgo_windows.go's link flags + # (-l:libdatabricks_sql_kernel.a -lws2_32 -lwsock32 -lrstrtmgr), otherwise never + # linked in CI, AND the Makefile/kernel-lib.sh windows-gnu target plumbing. + # + # Runs on the SELF-HOSTED, allow-listed databricks-protected-runner-group with the + # windows-server-latest label (as databricks-jdbc does). This is deliberate, not a + # hosted runner: the kernel repo is private, so create-github-app-token must run + # from an allow-listed IP — GitHub-hosted windows-latest is NOT allow-listed and + # fails the org IP allow list (the same block that gates the macOS kernel job). + # Being on the protected runner also means crates.io is blocked (go/hardened-gha), + # so — unlike the hosted-macOS job — this KEEPS the JFrog Go proxy + cargo→JFrog + # shim, exactly like the linux kernel job. + # + # Windows-specific vs the linux job: + # - cgo links with the mingw/gcc toolchain, which needs a GNU archive (.a), not + # the MSVC import lib that the default x86_64-pc-windows-MSVC host triple emits. + # So the Rust gnu target is installed and the Makefile passes + # --target x86_64-pc-windows-gnu (KERNEL_CARGO_TARGET) for GOOS=windows. + # - mingw-w64 gcc provides both the gnu-target link and cgo's own C compiler. + # - core.longpaths avoids checkout/cargo failures on deep dependency paths. + build-and-test-kernel-windows: + name: Test (kernel backend, Windows) + timeout-minutes: 40 + concurrency: + group: kernel-build-windows-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + runs-on: + group: databricks-protected-runner-group + labels: windows-server-latest + defaults: + run: + shell: bash + + steps: + # Deep cargo dependency paths can exceed the legacy 260-char Windows MAX_PATH; + # enable long paths before checkout so neither the clone nor the Rust build + # trips on it. + - name: Enable Git long paths + run: git config --system core.longpaths true + + - name: Check out code into the Go module directory + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + # Go module proxy (GOPROXY + ~/.netrc) via JFrog OIDC. Also exports + # JFROG_ACCESS_TOKEN, which the cargo step below reuses. Kept for Windows + # because the protected runner blocks direct crates.io / public proxy egress. + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + + # Point cargo at the JFrog crates proxy (protected runner blocks crates.io), + # reusing setup-jfrog's JFROG_ACCESS_TOKEN — same as the linux kernel job. The + # ~/.cargo paths resolve under git-bash on the Windows runner (HOME is set). + - name: Configure cargo to use JFrog + run: | + set -euo pipefail + mkdir -p ~/.cargo + cat > ~/.cargo/config.toml << 'EOF' + [source.crates-io] + replace-with = "jfrog" + + [source.jfrog] + registry = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" + + [registries.jfrog] + index = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" + credential-provider = ["cargo:token"] + EOF + cat > ~/.cargo/credentials.toml << EOF + [registries.jfrog] + token = "Bearer ${JFROG_ACCESS_TOKEN}" + EOF + chmod 600 ~/.cargo/credentials.toml + + - name: Set up Go Toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: '1.25.x' + cache: false + + # Install 1.96.1 (lockstep with rust-toolchain.toml) WITH the windows-gnu + # target: the default host triple is -msvc, but cgo needs a -gnu archive. + - name: Set up Rust Toolchain (windows-gnu target) + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 + with: + toolchain: 1.96.1 + target: x86_64-pc-windows-gnu + + # mingw-w64 supplies the gcc that links the -gnu Rust archive AND is cgo's own + # C compiler on Windows. choco is preinstalled on the runner image. + - name: Install mingw-w64 (gcc for cgo + gnu link) + run: choco install mingw --no-progress -y + + - name: Generate GitHub App token for databricks-sql-kernel + id: kernel-token + uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 + with: + app-id: ${{ secrets.INTEGRATION_TEST_APP_ID }} + private-key: ${{ secrets.INTEGRATION_TEST_PRIVATE_KEY }} + owner: databricks + repositories: databricks-sql-kernel + + - name: Rewrite kernel repo URL to authenticated HTTPS + env: + TOKEN: ${{ steps.kernel-token.outputs.token }} + run: | + git config --global \ + url."https://x-access-token:${TOKEN}@github.com/databricks/".insteadOf \ + "https://github.com/databricks/" + + - name: Cache kernel static lib + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + internal/backend/kernel/lib + internal/backend/kernel/include + key: ${{ runner.os }}-kernellib-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + + - name: Cache cargo + kernel build tree + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + build/kernel-src/target + key: ${{ runner.os }}-kernel-cargo-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + restore-keys: | + ${{ runner.os }}-kernel-cargo- + + # make test-kernel builds the kernel lib (make kernel-lib) — cargo build with + # --target x86_64-pc-windows-gnu (set by the Makefile for GOOS=windows), reading + # the .a from target//release/ — then runs the tagged unit tests, + # linking via cgo_windows.go. No warehouse creds, so live e2e/parity self-skip. + - name: Build kernel lib + run tagged tests + run: make test-kernel diff --git a/.github/workflows/on-demand-e2e.yml b/.github/workflows/on-demand-e2e.yml new file mode 100644 index 00000000..63f17cc1 --- /dev/null +++ b/.github/workflows/on-demand-e2e.yml @@ -0,0 +1,351 @@ +name: On-Demand E2E (all reachable OSes) + +# EXPERIMENTAL / testing aid for PR #421. Runs the live-warehouse E2E suites +# across every OS we can actually reach, ON DEMAND — triggered by adding the +# `run-e2e` label to a PR (re-add to re-run). Unlike schedule/workflow_dispatch +# (which run the file from the default branch), a `pull_request` trigger runs the +# file from the PR BRANCH, so this exercises the branch's code + workflow. #421 is +# a same-repo (non-fork) branch, so it receives the DATABRICKS_PECOTESTING_* +# secrets a fork PR would not. +# +# Reachable live matrix (see the per-job comments for why each is/ isn't here): +# Thrift E2E (pure-Go, needs only creds): linux + macOS + Windows (hosted) +# Kernel E2E (needs the PRIVATE kernel repo): linux + Windows (protected runners) +# macOS kernel E2E is intentionally ABSENT: a hosted macOS runner cannot mint +# the GitHub App token for the private kernel repo (org IP allow list), and no +# allow-listed macOS runner exists in the org — the same block that gates the +# macOS kernel unit job in go.yml. +# +# This is a testing scaffold, not a required gate; it is expected to be removed (or +# folded into nightly-e2e) before #421 is considered done. + +on: + pull_request: + types: [labeled] + +permissions: + contents: read + id-token: write + +jobs: + # ── Thrift (pure-Go) E2E on every OS ──────────────────────────────────────── + # Pure-Go (CGO_ENABLED=0), so no private kernel, no JFrog, no Rust/C toolchain — + # it just needs the warehouse creds. That makes all three hosted OSes reachable. + thrift-e2e: + if: github.event.label.name == 'run-e2e' + name: Thrift E2E (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + defaults: + run: + shell: bash + steps: + - name: Check out the PR branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Go Toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: '1.25.x' + cache: true + cache-dependency-path: go.sum + + # Fail loud if a credential is missing rather than letting the tests silently + # t.Skip() into a misleading green (same guard as nightly-e2e). Accept _TOKEN + # or _TOKEN_PERSONAL. + - name: Verify warehouse credentials are present + env: + HOST: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }} + HTTP_PATH: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }} + TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} + TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} + run: | + if [ -z "$HOST" ] || [ -z "$HTTP_PATH" ] || { [ -z "$TOKEN" ] && [ -z "$TOKEN_PERSONAL" ]; }; then + echo "::error::E2E warehouse secrets not reachable on this run (need DATABRICKS_PECOTESTING_SERVER_HOSTNAME, _HTTP_PATH2, and _TOKEN or _TOKEN_PERSONAL). If this is a PR-context restriction, in-repo live E2E is not possible here." >&2 + exit 1 + fi + + - name: Run Thrift E2E + env: + CGO_ENABLED: 0 + DATABRICKS_PECOTESTING_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }} + DATABRICKS_PECOTESTING_HTTP_PATH2: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }} + DATABRICKS_PECOTESTING_TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} + DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} + # The HTTP-path secret starts with "/" (e.g. /sql/1.0/warehouses/...). + # Under git-bash on Windows, MSYS auto-converts leading-slash values into + # a Windows path ("/sql/..." -> "C:/Program Files/Git/sql/..."), so the + # driver POSTed to a mangled path and got 404 (linux/macOS are unaffected). + # Disable MSYS arg/path conversion so the path reaches Go verbatim. These + # vars are MSYS-only, so they are harmless no-ops on ubuntu/macOS. + MSYS2_ARG_CONV_EXCL: '*' + MSYS_NO_PATHCONV: '1' + run: | + go test -count=1 -v -timeout 15m \ + -run 'TestE2EArrowBatchesSurviveQueryContextCancellation|TestE2ECloudFetchExactRowCount' . + + # ── Kernel (SEA) E2E on linux ─────────────────────────────────────────────── + # Protected runner (allow-listed): App token for the private kernel works, and + # crates.io is blocked so cargo uses the JFrog proxy. Mirrors nightly-e2e's + # kernel-e2e job. + kernel-e2e-linux: + if: github.event.label.name == 'run-e2e' + name: Kernel E2E (linux) + timeout-minutes: 65 + concurrency: + group: ondemand-kernel-e2e-linux-${{ github.ref }} + cancel-in-progress: true + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest + steps: + - name: Check out the PR branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + + - name: Configure cargo to use JFrog + shell: bash + run: | + set -euo pipefail + mkdir -p ~/.cargo + cat > ~/.cargo/config.toml << 'EOF' + [source.crates-io] + replace-with = "jfrog" + + [source.jfrog] + registry = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" + + [registries.jfrog] + index = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" + credential-provider = ["cargo:token"] + EOF + cat > ~/.cargo/credentials.toml << EOF + [registries.jfrog] + token = "Bearer ${JFROG_ACCESS_TOKEN}" + EOF + chmod 600 ~/.cargo/credentials.toml + + - name: Set up Go Toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: '1.25.x' + cache: false + + - name: Set up Rust Toolchain + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 + with: + toolchain: 1.96.1 + + - name: Generate GitHub App token for databricks-sql-kernel + id: kernel-token + uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 + with: + app-id: ${{ secrets.INTEGRATION_TEST_APP_ID }} + private-key: ${{ secrets.INTEGRATION_TEST_PRIVATE_KEY }} + owner: databricks + repositories: databricks-sql-kernel + + - name: Rewrite kernel repo URL to authenticated HTTPS + env: + TOKEN: ${{ steps.kernel-token.outputs.token }} + run: | + git config --global \ + url."https://x-access-token:${TOKEN}@github.com/databricks/".insteadOf \ + "https://github.com/databricks/" + + - name: Cache kernel static lib + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + internal/backend/kernel/lib + internal/backend/kernel/include + key: ${{ runner.os }}-kernellib-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + + - name: Cache cargo + kernel build tree + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + build/kernel-src/target + key: ${{ runner.os }}-kernel-cargo-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + restore-keys: | + ${{ runner.os }}-kernel-cargo- + + - name: Install build prerequisites (make, C compiler) + run: | + if ! command -v make &> /dev/null ; then + apt-get update && apt-get install -y make + fi + if ! command -v cc &> /dev/null ; then + apt-get update && apt-get install -y build-essential + fi + + - name: Verify warehouse credentials are present + env: + HOST: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }} + HTTP_PATH: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }} + TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} + TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} + run: | + if [ -z "$HOST" ] || [ -z "$HTTP_PATH" ] || { [ -z "$TOKEN" ] && [ -z "$TOKEN_PERSONAL" ]; }; then + echo "::error::E2E warehouse secrets not reachable on this run." >&2 + exit 1 + fi + + - name: Build kernel lib + timeout-minutes: 25 + run: make kernel-lib + + - name: Run kernel E2E + env: + DATABRICKS_PECOTESTING_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }} + DATABRICKS_PECOTESTING_HTTP_PATH2: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }} + DATABRICKS_PECOTESTING_TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} + DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} + # Same MSYS leading-slash path-mangling guard as the Thrift E2E job (the + # HTTP-path secret starts with "/", which git-bash would rewrite to a + # Windows path). MSYS-only vars, so a harmless no-op on the linux job. + MSYS2_ARG_CONV_EXCL: '*' + MSYS_NO_PATHCONV: '1' + run: | + CGO_ENABLED=1 go test -tags databricks_kernel -count=1 -v -timeout 30m \ + -run 'TestKernelE2E|TestKernelThriftParity|TestKernelParamsVsThrift' \ + -skip 'TestKernelE2EM2M' . + + # ── Kernel (SEA) E2E on Windows ───────────────────────────────────────────── + # Protected windows-server-latest (allow-listed → App token works). Same + # windows-gnu build plumbing as go.yml's build-and-test-kernel-windows, plus the + # warehouse creds + the kernel E2E filter. + kernel-e2e-windows: + if: github.event.label.name == 'run-e2e' + name: Kernel E2E (Windows) + timeout-minutes: 65 + concurrency: + group: ondemand-kernel-e2e-windows-${{ github.ref }} + cancel-in-progress: true + runs-on: + group: databricks-protected-runner-group + labels: windows-server-latest + defaults: + run: + shell: bash + steps: + - name: Enable Git long paths + run: git config --system core.longpaths true + + - name: Check out the PR branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + + - name: Configure cargo to use JFrog + run: | + set -euo pipefail + mkdir -p ~/.cargo + cat > ~/.cargo/config.toml << 'EOF' + [source.crates-io] + replace-with = "jfrog" + + [source.jfrog] + registry = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" + + [registries.jfrog] + index = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" + credential-provider = ["cargo:token"] + EOF + cat > ~/.cargo/credentials.toml << EOF + [registries.jfrog] + token = "Bearer ${JFROG_ACCESS_TOKEN}" + EOF + chmod 600 ~/.cargo/credentials.toml + + - name: Set up Go Toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: '1.25.x' + cache: false + + - name: Set up Rust Toolchain (windows-gnu target) + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 + with: + toolchain: 1.96.1 + target: x86_64-pc-windows-gnu + + - name: Install mingw-w64 (gcc for cgo + gnu link) + run: choco install mingw --no-progress -y + + - name: Generate GitHub App token for databricks-sql-kernel + id: kernel-token + uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 + with: + app-id: ${{ secrets.INTEGRATION_TEST_APP_ID }} + private-key: ${{ secrets.INTEGRATION_TEST_PRIVATE_KEY }} + owner: databricks + repositories: databricks-sql-kernel + + - name: Rewrite kernel repo URL to authenticated HTTPS + env: + TOKEN: ${{ steps.kernel-token.outputs.token }} + run: | + git config --global \ + url."https://x-access-token:${TOKEN}@github.com/databricks/".insteadOf \ + "https://github.com/databricks/" + + - name: Cache kernel static lib + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + internal/backend/kernel/lib + internal/backend/kernel/include + key: ${{ runner.os }}-kernellib-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + + - name: Cache cargo + kernel build tree + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + build/kernel-src/target + key: ${{ runner.os }}-kernel-cargo-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + restore-keys: | + ${{ runner.os }}-kernel-cargo- + + - name: Verify warehouse credentials are present + env: + HOST: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }} + HTTP_PATH: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }} + TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} + TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} + run: | + if [ -z "$HOST" ] || [ -z "$HTTP_PATH" ] || { [ -z "$TOKEN" ] && [ -z "$TOKEN_PERSONAL" ]; }; then + echo "::error::E2E warehouse secrets not reachable on this run." >&2 + exit 1 + fi + + - name: Build kernel lib + timeout-minutes: 25 + run: make kernel-lib + + - name: Run kernel E2E + env: + DATABRICKS_PECOTESTING_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }} + DATABRICKS_PECOTESTING_HTTP_PATH2: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }} + DATABRICKS_PECOTESTING_TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} + DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} + # Same MSYS leading-slash path-mangling guard as the Thrift E2E job (the + # HTTP-path secret starts with "/", which git-bash would rewrite to a + # Windows path). MSYS-only vars, so a harmless no-op on the linux job. + MSYS2_ARG_CONV_EXCL: '*' + MSYS_NO_PATHCONV: '1' + run: | + CGO_ENABLED=1 go test -tags databricks_kernel -count=1 -v -timeout 30m \ + -run 'TestKernelE2E|TestKernelThriftParity|TestKernelParamsVsThrift' \ + -skip 'TestKernelE2EM2M' . diff --git a/Makefile b/Makefile index 6744c9f9..e2e87aa6 100644 --- a/Makefile +++ b/Makefile @@ -101,6 +101,13 @@ KERNEL_LIB_DIR = internal/backend/kernel/lib/$(KERNEL_GOOS)_$(KERNEL_GOARCH) KERNEL_INC_DIR = internal/backend/kernel/include KERNEL_GO = CGO_ENABLED=1 go KERNEL_TAGS = -tags databricks_kernel +# Explicit cargo target triple. Empty on linux/macOS (cargo's default host triple +# is what cgo_.go expects, and kernel-lib.sh omits --target). On Windows the +# default host triple is x86_64-pc-windows-MSVC, which emits an MSVC import lib, +# but cgo links via mingw/gcc and needs a GNU archive — so force the -gnu triple. +# This is an ABI choice on the same os/arch (host==target still holds), not a +# cross-OS build; kernel-lib.sh reads the .a from target//release/ when set. +KERNEL_CARGO_TARGET = $(if $(filter windows,$(KERNEL_GOOS)),x86_64-pc-windows-gnu,) .PHONY: kernel-lib kernel-lib: ## Build the pinned kernel static lib + header into the cgo link dir (source build). @@ -108,6 +115,7 @@ kernel-lib: ## Build the pinned kernel static lib + header into the cgo link di KERNEL_LIB_DIR="$(KERNEL_LIB_DIR)" KERNEL_INC_DIR="$(KERNEL_INC_DIR)" \ KERNEL_GOOS="$(KERNEL_GOOS)" KERNEL_GOARCH="$(KERNEL_GOARCH)" \ KERNEL_GOHOSTOS="$(KERNEL_GOHOSTOS)" KERNEL_GOHOSTARCH="$(KERNEL_GOHOSTARCH)" \ + KERNEL_CARGO_TARGET="$(KERNEL_CARGO_TARGET)" \ KERNEL_LOCAL_A="$(KERNEL_LOCAL_A)" KERNEL_LOCAL_HEADER="$(KERNEL_LOCAL_HEADER)" \ ./build/kernel-lib.sh diff --git a/build/kernel-lib.sh b/build/kernel-lib.sh index ffc17e6f..379e2a47 100755 --- a/build/kernel-lib.sh +++ b/build/kernel-lib.sh @@ -157,6 +157,23 @@ if ! git -C "$KERNEL_SRC" checkout -f --quiet "$KERNEL_REV" 2>/dev/null; then fi log "kernel at $(git -C "$KERNEL_SRC" rev-parse --short HEAD)" +# Optional explicit cargo --target (KERNEL_CARGO_TARGET). Needed on Windows: the +# default host triple there is x86_64-pc-windows-MSVC, which emits an MSVC-style +# import lib, but cgo links with the mingw/gcc toolchain and needs a GNU archive — +# so the Makefile sets KERNEL_CARGO_TARGET=x86_64-pc-windows-gnu for GOOS=windows. +# This is an ABI selection on the SAME os/arch (host==target still holds, so the +# cross-build guard above is not tripped), NOT a cross-OS build. With --target, +# cargo writes to target//release/ instead of target/release/, so the +# archive path is scoped accordingly. Unset on linux/macOS → the original +# no-`--target`, target/release/ behavior is preserved byte-for-byte. +target_flag="" +target_subdir="" +if [ -n "${KERNEL_CARGO_TARGET:-}" ]; then + target_flag="--target ${KERNEL_CARGO_TARGET}" + target_subdir="${KERNEL_CARGO_TARGET}/" + log "cargo target override: ${KERNEL_CARGO_TARGET}" +fi + # --no-default-features --features tls-rustls: pure-Rust TLS (no system OpenSSL) # keeps the archive self-contained and cross-compile-tractable. The kernel's # default is tls-native, so the override is required. @@ -164,10 +181,10 @@ log "kernel at $(git -C "$KERNEL_SRC" rev-parse --short HEAD)" # re-resolving, so a fixed KERNEL_REV yields a fixed dependency graph (paired # with the pinned rustc in rust-toolchain.toml, the .a is reproducible). Fails # loud if the lock is stale instead of silently pulling newer deps. -log "cargo build --release --locked --no-default-features --features tls-rustls" -( cd "$KERNEL_SRC" && cargo build --release --locked --no-default-features --features tls-rustls ) +log "cargo build --release --locked ${target_flag} --no-default-features --features tls-rustls" +( cd "$KERNEL_SRC" && cargo build --release --locked ${target_flag} --no-default-features --features tls-rustls ) -src_a="$KERNEL_SRC/target/release/$LIB_NAME" +src_a="$KERNEL_SRC/target/${target_subdir}release/$LIB_NAME" src_h="$KERNEL_SRC/include/$HEADER_NAME" [ -f "$src_a" ] || { log "expected archive not produced: $src_a"; exit 1; } [ -f "$src_h" ] || { log "expected header not found: $src_h"; exit 1; } diff --git a/connection.go b/connection.go index 6f5a9cee..61bfb183 100644 --- a/connection.go +++ b/connection.go @@ -716,7 +716,15 @@ func localPathIsAllowed(stagingAllowedLocalPaths []string, localFile string) boo if err != nil { return false } - if !strings.Contains(relativePath, "../") { + // localFile is inside the allowed dir iff the relative path does not climb + // out of it, i.e. it is neither ".." itself nor starts with "..". + // filepath.Rel returns OS-native separators ("../x" on unix, "..\\x" on + // windows), so this must compare against filepath.Separator rather than a + // hard-coded "../" — the latter never matches the windows "..\\" form and + // silently allows an escaping path (a fail-open path-traversal check on + // windows). Using the "../" prefix (not a Contains) also avoids treating a + // sibling like "..foo" as an escape. + if relativePath != ".." && !strings.HasPrefix(relativePath, ".."+string(filepath.Separator)) { return true } } diff --git a/internal/backend/kernel/cgo_linux_arm64.go b/internal/backend/kernel/cgo_linux_arm64.go new file mode 100644 index 00000000..b7fdec91 --- /dev/null +++ b/internal/backend/kernel/cgo_linux_arm64.go @@ -0,0 +1,20 @@ +//go:build cgo && databricks_kernel && linux && arm64 + +package kernel + +// Link flags for linux/arm64 (e.g. AWS Graviton). Identical in form to +// cgo_linux.go's amd64 flags: the same GNU ld runs on both Linux arches, so the +// -l:.a static-forcing form (which stops the linker preferring a +// same-named .so and baking in an rpath) and the -lstdc++/-lm/-ldl transitive +// system deps carry over unchanged. Only the ${SRCDIR}-relative lib dir differs +// (lib/linux_arm64), where `make kernel-lib` drops a natively built arm64 archive. +// +// NOTE: not yet exercised in CI — no allow-listed linux/arm64 runner exists in the +// org today, so the arm64 kernel .a is built and linked only on a native arm64 host +// (host == target for kernel-lib.sh). The flags are the intended shape; validate on +// an arm64 runner before relying on this in a release. + +/* +#cgo LDFLAGS: -L${SRCDIR}/lib/linux_arm64 -l:libdatabricks_sql_kernel.a -lstdc++ -lm -ldl +*/ +import "C" diff --git a/internal/backend/kernel/cgo_unsupported.go b/internal/backend/kernel/cgo_unsupported.go index bcc96910..f53541d7 100644 --- a/internal/backend/kernel/cgo_unsupported.go +++ b/internal/backend/kernel/cgo_unsupported.go @@ -1,19 +1,18 @@ -//go:build cgo && databricks_kernel && !(linux && amd64) && !(darwin && arm64) && !(windows && amd64) +//go:build cgo && databricks_kernel && !(linux && amd64) && !(linux && arm64) && !(darwin && arm64) && !(windows && amd64) package kernel // This file is compiled only on GOOS/GOARCH combinations the kernel backend does // not support. Per-platform link flags (cgo_.go) exist only for linux/amd64, -// darwin/arm64, and windows/amd64; on any other target there is no static archive -// to link, so cgo.go's C ABI calls would otherwise fail at the LINK step with an -// opaque "undefined reference to kernel_*". The Makefile's host==target guard -// does not catch this — it happily source-builds a host .a on e.g. linux/arm64 -// (Graviton) or an Intel Mac, only for the link to fall over with no matching -// LDFLAGS file. Referencing an undefined identifier here fails earlier, at -// COMPILE time, with a message that names the supported targets — a legible build -// error instead of a linker dump. +// linux/arm64, darwin/arm64, and windows/amd64; on any other target there is no +// static archive to link, so cgo.go's C ABI calls would otherwise fail at the LINK +// step with an opaque "undefined reference to kernel_*". The Makefile's host==target +// guard does not catch this — it happily source-builds a host .a on e.g. an Intel +// Mac, only for the link to fall over with no matching LDFLAGS file. Referencing an +// undefined identifier here fails earlier, at COMPILE time, with a message that +// names the supported targets — a legible build error instead of a linker dump. // // Broader OS/arch coverage is tracked in the distribution design (native per-OS // runners or a staged prebuilt .a); until then this guard makes the supported // boundary explicit rather than latent. -const _ = kernel_backend_supports_only_linux_amd64_darwin_arm64_and_windows_amd64 +const _ = kernel_backend_supports_only_linux_amd64_linux_arm64_darwin_arm64_and_windows_amd64 diff --git a/internal/backend/kernel/cgo_windows.go b/internal/backend/kernel/cgo_windows.go index ae650b9b..a9f19fb0 100644 --- a/internal/backend/kernel/cgo_windows.go +++ b/internal/backend/kernel/cgo_windows.go @@ -2,19 +2,24 @@ package kernel -// Link flags for windows/amd64. NOTE: this platform is not yet exercised in CI -// (M0 is linux/amd64); the flags below are the intended shape but must be -// validated on windows before it is enabled. +// Link flags for windows/amd64, validated on a windows-server-latest CI runner +// (see the build-and-test-kernel-windows job). // // cgo on windows uses the mingw/gcc toolchain, which links GNU archives (.a) — // NOT the MSVC .lib that `cargo build --target x86_64-pc-windows-msvc` emits. // So the kernel must be built for the windows-gnu target // (`--target x86_64-pc-windows-gnu`) to produce a mingw-compatible .a; that -// target selection is the build step's responsibility. -lws2_32/-lwsock32 are -// the kernel's Winsock deps and -lrstrtmgr is Restart Manager (pulled in by the -// Rust std/dep graph on windows). +// target selection is the build step's responsibility. +// +// System libs the kernel's Rust std/dep graph pulls in on windows (mingw does +// NOT auto-link these transitively, so each must be named explicitly or the +// final cgo link fails with `undefined reference`): +// -lws2_32 / -lwsock32 Winsock (sockets) +// -lrstrtmgr Restart Manager +// -lbcrypt BCryptGenRandom — Rust's getrandom crate (RNG) +// -lntdll NtCreateNamedPipeFile — Rust std child_pipe (native NT API) /* -#cgo LDFLAGS: -L${SRCDIR}/lib/windows_amd64 -l:libdatabricks_sql_kernel.a -lws2_32 -lwsock32 -lrstrtmgr +#cgo LDFLAGS: -L${SRCDIR}/lib/windows_amd64 -l:libdatabricks_sql_kernel.a -lws2_32 -lwsock32 -lrstrtmgr -lbcrypt -lntdll */ import "C"