From 397d1e8a2e813009684db68cf12f3194882061c5 Mon Sep 17 00:00:00 2001 From: Mani Kaustubh Mathur Date: Wed, 22 Jul 2026 20:52:20 +0000 Subject: [PATCH 01/10] ci: fan out pure-Go driver tests to macOS and Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a build-and-test-cross-os job that builds and unit-tests the default pure-Go driver (CGO_ENABLED=0, Thrift backend) on hosted macos-latest (arm64) and windows-latest (amd64), across Go 1.25.x/1.26.x. The Linux build-and-test job already covers linux/amd64 on the protected runners; this extends the cross-platform surface without touching it. The pure-Go build has no cgo, no kernel .a, and no private module deps, so it needs no Rust/C toolchain, no JFrog proxy, and no protected runner — it runs on plain hosted runners, mirroring the sibling databricks-adbc Go driver. The SEA-via-kernel backend stays Linux-only for now (added per-OS later). Co-authored-by: Isaac Signed-off-by: Mani Kaustubh Mathur --- .github/workflows/go.yml | 60 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 64972b44..ee53e794 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -105,6 +105,66 @@ jobs: - name: Build run: make linux + # Fans the DEFAULT pure-Go driver (CGO_ENABLED=0, Thrift backend) out to macOS + # and Windows so the cross-platform surface is exercised per-PR — the Linux + # build-and-test job above already covers linux/amd64 on the protected runners. + # The pure-Go build has no cgo and no kernel .a, so it needs no Rust toolchain, + # no C compiler, and no private-repo access; it just has to compile and pass its + # unit tests on each OS. The SEA-via-kernel backend is NOT built here (it stays + # Linux-only for now — see build-and-test-kernel below and the distribution + # design doc's per-OS rollout). + # + # These run on GitHub-hosted macos-latest (arm64) / windows-latest (amd64) via + # runs-on: ${{ matrix.os }}, NOT the databricks-protected-runner-group (which is + # linux-only). That mirrors the sibling databricks-adbc Go driver's go_test.yaml, + # which already tests on hosted macos-latest + windows-latest. Hosted runners have + # open egress and every module dependency is public (go.mod has no private + # databricks modules), so setup-jfrog / the JFrog Go proxy is deliberately omitted + # — the default public proxy resolves everything. + build-and-test-cross-os: + name: Test (${{ matrix.os }}, Go ${{ matrix.go-version }}) + strategy: + # Don't let one OS's failure cancel the others — we want the full + # cross-platform signal on every run while this job stabilizes. + fail-fast: false + matrix: + os: [macos-latest, windows-latest] + # Same support window as build-and-test. Hosted runners don't pin + # GOTOOLCHAIN=local, but keep the versions identical for parity. + go-version: ['1.25.x', '1.26.x'] + runs-on: ${{ matrix.os }} + # bash on every OS (git-bash on Windows) so the run: blocks below are portable. + defaults: + run: + shell: bash + + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Go Toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: ${{ matrix.go-version }} + # Let setup-go manage the module/build cache here — the bespoke + # actions/cache step in build-and-test is tuned to the Linux runner's + # paths; the built-in cache is portable across macOS/Windows. + cache: true + cache-dependency-path: go.sum + + # Build + unit-test the pure-Go driver directly with the go tool rather than + # `make test`: the Makefile's test target shells out to gotestsum + writes + # JUnit XML and is oriented at the Linux CI, whereas `go build`/`go test` are + # uniform across macOS and Windows and need no extra tooling installed. Race + # detection is intentionally NOT run here — it requires cgo + a C toolchain, + # which cuts against this job's toolchain-free goal, and it's already covered + # on Linux by build-and-test's Test-Race step. + - name: Build + run: CGO_ENABLED=0 go build -v ./... + + - name: Test + run: CGO_ENABLED=0 go test -count=1 ./... + build-and-test-kernel: name: Test (kernel backend) # Exercises the opt-in SEA-via-kernel backend: builds the Rust kernel static From 312c34d4824bccbe569b0c9c9e933183c60b0659 Mon Sep 17 00:00:00 2001 From: Mani Kaustubh Mathur Date: Wed, 22 Jul 2026 21:13:02 +0000 Subject: [PATCH 02/10] fix: make staging path-traversal check separator-aware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit localPathIsAllowed guarded against staging PUT/GET escaping the configured stagingAllowedLocalPaths by rejecting a relative path containing the literal "../". filepath.Rel returns OS-native separators, so on Windows an escaping path is "..\\x", which the "../" substring never matches — the guard returned true (allowed) for a path outside the allowlist. This is a fail-open path-traversal check on Windows, present since the driver's first commit. Compare the relative path against filepath.Separator instead: a path is inside the allowed dir iff it is neither ".." nor starts with ".."+separator. Using a prefix check (not Contains) also stops a sibling like "..foo" from being mistaken for an escape. Only the default Thrift path reaches this code; the kernel/SEA backend rejects staging statements before execStagingOperation. Surfaced by the new macOS/Windows CI matrix — the existing TestPathAllowed now passes on Windows. Co-authored-by: Isaac Signed-off-by: Mani Kaustubh Mathur --- connection.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/connection.go b/connection.go index 6f5a9cee..61bfb183 100644 --- a/connection.go +++ b/connection.go @@ -716,7 +716,15 @@ func localPathIsAllowed(stagingAllowedLocalPaths []string, localFile string) boo if err != nil { return false } - if !strings.Contains(relativePath, "../") { + // localFile is inside the allowed dir iff the relative path does not climb + // out of it, i.e. it is neither ".." itself nor starts with "..". + // filepath.Rel returns OS-native separators ("../x" on unix, "..\\x" on + // windows), so this must compare against filepath.Separator rather than a + // hard-coded "../" — the latter never matches the windows "..\\" form and + // silently allows an escaping path (a fail-open path-traversal check on + // windows). Using the "../" prefix (not a Contains) also avoids treating a + // sibling like "..foo" as an escape. + if relativePath != ".." && !strings.HasPrefix(relativePath, ".."+string(filepath.Separator)) { return true } } From 23cd7a4d36ec993f36d5448a8adf3ed129094b37 Mon Sep 17 00:00:00 2001 From: Mani Kaustubh Mathur Date: Wed, 22 Jul 2026 21:33:34 +0000 Subject: [PATCH 03/10] ci: add SEA/kernel backend test job on macOS (darwin/arm64) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds build-and-test-kernel-macos, extending kernel/SEA CI coverage to its second OS (after linux/amd64). This is the first time cgo_darwin.go's link flags (positional .a for Apple ld64, -lc++, @loader_path) are compiled and linked in CI — a green run validates the darwin link contract. Structurally it is the linux build-and-test-kernel job minus the two JFrog steps: it runs on a hosted macos-latest (arm64) runner with open egress, and the kernel's deps plus the driver's go.mod deps are all public, so cargo uses crates.io and go uses the default proxy directly. The GitHub App token step is kept because the kernel repo is private (repo auth, not crate resolution). macos-latest is arm64 (matches cgo_darwin.go's `darwin && arm64` tag), so host == target and kernel-lib.sh needs no --target; aarch64-apple-darwin is a Rust tier-1 target and Xcode CLT supplies cc + libc++. No warehouse creds, so only the tagged unit tests run (live e2e/parity self-skip, as on linux). Co-authored-by: Isaac Signed-off-by: Mani Kaustubh Mathur --- .github/workflows/go.yml | 100 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index ee53e794..8c8331d7 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -309,3 +309,103 @@ jobs: # the separate nightly-e2e workflow. - name: Build kernel lib + run tagged tests run: make test-kernel + + # Exercises the SEA-via-kernel backend on macOS (darwin/arm64) — the second + # kernel OS after linux/amd64, per the distribution design's one-OS-at-a-time + # rollout. This is the FIRST time cgo_darwin.go's link flags (positional .a for + # Apple ld64, -lc++, @loader_path) are compiled+linked in CI; a green run here + # is the validation that the darwin link contract is correct. + # + # Structurally this is the linux build-and-test-kernel job MINUS the two JFrog + # steps. It runs on a GitHub-hosted macos-latest runner (arm64), which has open + # egress: the kernel's ~300 deps are all public crates.io (no git/private + # sources) and the driver's own go.mod deps are all public, so cargo hits + # crates.io directly and go hits the default public proxy — no JFrog Go proxy, + # no cargo→JFrog credential shim. The GitHub App token step IS kept: the kernel + # repo is still private, so kernel-lib.sh's clone needs authenticated HTTPS + # (that is repo auth, unrelated to crate resolution). + # + # macos-latest is arm64, matching cgo_darwin.go's `darwin && arm64` build tag, + # so host == target and kernel-lib.sh's cross-build guard passes with no + # --target. aarch64-apple-darwin is a Rust tier-1 target bundled with the pinned + # 1.96.1 toolchain (no `rustup target add`), and Xcode CLT on the runner supplies + # cc + libc++. No warehouse creds here, so only the tagged UNIT tests run; the + # live e2e / parity suites self-skip (they run in nightly-e2e). + build-and-test-kernel-macos: + name: Test (kernel backend, macOS) + timeout-minutes: 30 + concurrency: + group: kernel-build-macos-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + runs-on: macos-latest + defaults: + run: + shell: bash + + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Go Toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: '1.25.x' + cache: false + + # Keep in lockstep with rust-toolchain.toml's channel (it governs the archive + # under a fixed KERNEL_REV; a floating `stable` here would drift it). + - name: Set up Rust Toolchain + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 + with: + toolchain: 1.96.1 + + # The kernel repo (databricks/databricks-sql-kernel) is private and the hosted + # runner has no ambient git credentials, so kernel-lib.sh's fetch would fail + # with "could not read Username". Mint a repo-scoped token from the + # INTEGRATION_TEST_APP GitHub App and rewrite the kernel HTTPS URL to carry it + # (same mechanism as the linux kernel job). This is repo auth only — crate + # resolution goes to public crates.io, so no cargo→JFrog shim is needed here. + - name: Generate GitHub App token for databricks-sql-kernel + id: kernel-token + uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 + with: + app-id: ${{ secrets.INTEGRATION_TEST_APP_ID }} + private-key: ${{ secrets.INTEGRATION_TEST_PRIVATE_KEY }} + owner: databricks + repositories: databricks-sql-kernel + + - name: Rewrite kernel repo URL to authenticated HTTPS + env: + TOKEN: ${{ steps.kernel-token.outputs.token }} + run: | + git config --global \ + url."https://x-access-token:${TOKEN}@github.com/databricks/".insteadOf \ + "https://github.com/databricks/" + + # Cache keyed on runner.os (macOS) so it never collides with the linux job's + # cache; the .a is platform-specific. + - name: Cache kernel static lib + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + internal/backend/kernel/lib + internal/backend/kernel/include + key: ${{ runner.os }}-kernellib-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + + - name: Cache cargo + kernel build tree + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + build/kernel-src/target + key: ${{ runner.os }}-kernel-cargo-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + restore-keys: | + ${{ runner.os }}-kernel-cargo- + + # make test-kernel builds the kernel lib (make kernel-lib) if the cache missed + # — cargo build on darwin/arm64 (host==target) — then runs + # `CGO_ENABLED=1 go test -tags databricks_kernel ./...`, linking via + # cgo_darwin.go. make + cc + libc++ are preinstalled on macos-latest (Xcode CLT). + - name: Build kernel lib + run tagged tests + run: make test-kernel From 59e48a5e85a6a383b0788295e40add10aed70e26 Mon Sep 17 00:00:00 2001 From: Mani Kaustubh Mathur Date: Wed, 22 Jul 2026 21:46:44 +0000 Subject: [PATCH 04/10] feat(kernel): enable linux/arm64 target for the SEA backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add cgo_linux_arm64.go with the linux/arm64 link flags (same GNU ld form as cgo_linux.go: -l:libdatabricks_sql_kernel.a -lstdc++ -lm -ldl, from lib/linux_arm64), and narrow cgo_unsupported.go's build constraint to admit linux/arm64 so it no longer compile-blocks that target. Verified via `go list -tags databricks_kernel` that each supported target now selects exactly one link file (linux/amd64, linux/arm64, darwin/arm64, windows/amd64) with no overlap, and darwin/amd64 still falls through to the unsupported guard. Not yet exercised in CI — no allow-listed linux/arm64 runner exists in the org today; the arm64 .a is built and linked only on a native arm64 host. Flags are the intended shape; validate on an arm64 runner before a release relies on it. Co-authored-by: Isaac Signed-off-by: Mani Kaustubh Mathur --- internal/backend/kernel/cgo_linux_arm64.go | 20 ++++++++++++++++++++ internal/backend/kernel/cgo_unsupported.go | 19 +++++++++---------- 2 files changed, 29 insertions(+), 10 deletions(-) create mode 100644 internal/backend/kernel/cgo_linux_arm64.go diff --git a/internal/backend/kernel/cgo_linux_arm64.go b/internal/backend/kernel/cgo_linux_arm64.go new file mode 100644 index 00000000..b7fdec91 --- /dev/null +++ b/internal/backend/kernel/cgo_linux_arm64.go @@ -0,0 +1,20 @@ +//go:build cgo && databricks_kernel && linux && arm64 + +package kernel + +// Link flags for linux/arm64 (e.g. AWS Graviton). Identical in form to +// cgo_linux.go's amd64 flags: the same GNU ld runs on both Linux arches, so the +// -l:.a static-forcing form (which stops the linker preferring a +// same-named .so and baking in an rpath) and the -lstdc++/-lm/-ldl transitive +// system deps carry over unchanged. Only the ${SRCDIR}-relative lib dir differs +// (lib/linux_arm64), where `make kernel-lib` drops a natively built arm64 archive. +// +// NOTE: not yet exercised in CI — no allow-listed linux/arm64 runner exists in the +// org today, so the arm64 kernel .a is built and linked only on a native arm64 host +// (host == target for kernel-lib.sh). The flags are the intended shape; validate on +// an arm64 runner before relying on this in a release. + +/* +#cgo LDFLAGS: -L${SRCDIR}/lib/linux_arm64 -l:libdatabricks_sql_kernel.a -lstdc++ -lm -ldl +*/ +import "C" diff --git a/internal/backend/kernel/cgo_unsupported.go b/internal/backend/kernel/cgo_unsupported.go index bcc96910..f53541d7 100644 --- a/internal/backend/kernel/cgo_unsupported.go +++ b/internal/backend/kernel/cgo_unsupported.go @@ -1,19 +1,18 @@ -//go:build cgo && databricks_kernel && !(linux && amd64) && !(darwin && arm64) && !(windows && amd64) +//go:build cgo && databricks_kernel && !(linux && amd64) && !(linux && arm64) && !(darwin && arm64) && !(windows && amd64) package kernel // This file is compiled only on GOOS/GOARCH combinations the kernel backend does // not support. Per-platform link flags (cgo_.go) exist only for linux/amd64, -// darwin/arm64, and windows/amd64; on any other target there is no static archive -// to link, so cgo.go's C ABI calls would otherwise fail at the LINK step with an -// opaque "undefined reference to kernel_*". The Makefile's host==target guard -// does not catch this — it happily source-builds a host .a on e.g. linux/arm64 -// (Graviton) or an Intel Mac, only for the link to fall over with no matching -// LDFLAGS file. Referencing an undefined identifier here fails earlier, at -// COMPILE time, with a message that names the supported targets — a legible build -// error instead of a linker dump. +// linux/arm64, darwin/arm64, and windows/amd64; on any other target there is no +// static archive to link, so cgo.go's C ABI calls would otherwise fail at the LINK +// step with an opaque "undefined reference to kernel_*". The Makefile's host==target +// guard does not catch this — it happily source-builds a host .a on e.g. an Intel +// Mac, only for the link to fall over with no matching LDFLAGS file. Referencing an +// undefined identifier here fails earlier, at COMPILE time, with a message that +// names the supported targets — a legible build error instead of a linker dump. // // Broader OS/arch coverage is tracked in the distribution design (native per-OS // runners or a staged prebuilt .a); until then this guard makes the supported // boundary explicit rather than latent. -const _ = kernel_backend_supports_only_linux_amd64_darwin_arm64_and_windows_amd64 +const _ = kernel_backend_supports_only_linux_amd64_linux_arm64_darwin_arm64_and_windows_amd64 From 80c373b25a03cdde948ca6a0b0eb26ed7a786c80 Mon Sep 17 00:00:00 2001 From: Mani Kaustubh Mathur Date: Wed, 22 Jul 2026 21:46:55 +0000 Subject: [PATCH 05/10] build(kernel): support an explicit cargo --target for the windows-gnu build kernel-lib.sh gains an optional KERNEL_CARGO_TARGET: when set it passes `cargo build --target ` and reads the archive from target//release/ instead of target/release/. The Makefile sets it to x86_64-pc-windows-gnu for GOOS=windows and leaves it empty elsewhere. Windows cgo links via the mingw/gcc toolchain, which needs a GNU archive (.a), but the default x86_64-pc-windows-MSVC host triple emits an MSVC import lib. This is an ABI selection on the same os/arch (host==target still holds, so the cross-build guard is not tripped), not a cross-OS build. Linux/macOS keep the original no-`--target`, target/release/ behavior byte-for-byte (empty override). Co-authored-by: Isaac Signed-off-by: Mani Kaustubh Mathur --- Makefile | 8 ++++++++ build/kernel-lib.sh | 23 ++++++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 6744c9f9..e2e87aa6 100644 --- a/Makefile +++ b/Makefile @@ -101,6 +101,13 @@ KERNEL_LIB_DIR = internal/backend/kernel/lib/$(KERNEL_GOOS)_$(KERNEL_GOARCH) KERNEL_INC_DIR = internal/backend/kernel/include KERNEL_GO = CGO_ENABLED=1 go KERNEL_TAGS = -tags databricks_kernel +# Explicit cargo target triple. Empty on linux/macOS (cargo's default host triple +# is what cgo_.go expects, and kernel-lib.sh omits --target). On Windows the +# default host triple is x86_64-pc-windows-MSVC, which emits an MSVC import lib, +# but cgo links via mingw/gcc and needs a GNU archive — so force the -gnu triple. +# This is an ABI choice on the same os/arch (host==target still holds), not a +# cross-OS build; kernel-lib.sh reads the .a from target//release/ when set. +KERNEL_CARGO_TARGET = $(if $(filter windows,$(KERNEL_GOOS)),x86_64-pc-windows-gnu,) .PHONY: kernel-lib kernel-lib: ## Build the pinned kernel static lib + header into the cgo link dir (source build). @@ -108,6 +115,7 @@ kernel-lib: ## Build the pinned kernel static lib + header into the cgo link di KERNEL_LIB_DIR="$(KERNEL_LIB_DIR)" KERNEL_INC_DIR="$(KERNEL_INC_DIR)" \ KERNEL_GOOS="$(KERNEL_GOOS)" KERNEL_GOARCH="$(KERNEL_GOARCH)" \ KERNEL_GOHOSTOS="$(KERNEL_GOHOSTOS)" KERNEL_GOHOSTARCH="$(KERNEL_GOHOSTARCH)" \ + KERNEL_CARGO_TARGET="$(KERNEL_CARGO_TARGET)" \ KERNEL_LOCAL_A="$(KERNEL_LOCAL_A)" KERNEL_LOCAL_HEADER="$(KERNEL_LOCAL_HEADER)" \ ./build/kernel-lib.sh diff --git a/build/kernel-lib.sh b/build/kernel-lib.sh index ffc17e6f..379e2a47 100755 --- a/build/kernel-lib.sh +++ b/build/kernel-lib.sh @@ -157,6 +157,23 @@ if ! git -C "$KERNEL_SRC" checkout -f --quiet "$KERNEL_REV" 2>/dev/null; then fi log "kernel at $(git -C "$KERNEL_SRC" rev-parse --short HEAD)" +# Optional explicit cargo --target (KERNEL_CARGO_TARGET). Needed on Windows: the +# default host triple there is x86_64-pc-windows-MSVC, which emits an MSVC-style +# import lib, but cgo links with the mingw/gcc toolchain and needs a GNU archive — +# so the Makefile sets KERNEL_CARGO_TARGET=x86_64-pc-windows-gnu for GOOS=windows. +# This is an ABI selection on the SAME os/arch (host==target still holds, so the +# cross-build guard above is not tripped), NOT a cross-OS build. With --target, +# cargo writes to target//release/ instead of target/release/, so the +# archive path is scoped accordingly. Unset on linux/macOS → the original +# no-`--target`, target/release/ behavior is preserved byte-for-byte. +target_flag="" +target_subdir="" +if [ -n "${KERNEL_CARGO_TARGET:-}" ]; then + target_flag="--target ${KERNEL_CARGO_TARGET}" + target_subdir="${KERNEL_CARGO_TARGET}/" + log "cargo target override: ${KERNEL_CARGO_TARGET}" +fi + # --no-default-features --features tls-rustls: pure-Rust TLS (no system OpenSSL) # keeps the archive self-contained and cross-compile-tractable. The kernel's # default is tls-native, so the override is required. @@ -164,10 +181,10 @@ log "kernel at $(git -C "$KERNEL_SRC" rev-parse --short HEAD)" # re-resolving, so a fixed KERNEL_REV yields a fixed dependency graph (paired # with the pinned rustc in rust-toolchain.toml, the .a is reproducible). Fails # loud if the lock is stale instead of silently pulling newer deps. -log "cargo build --release --locked --no-default-features --features tls-rustls" -( cd "$KERNEL_SRC" && cargo build --release --locked --no-default-features --features tls-rustls ) +log "cargo build --release --locked ${target_flag} --no-default-features --features tls-rustls" +( cd "$KERNEL_SRC" && cargo build --release --locked ${target_flag} --no-default-features --features tls-rustls ) -src_a="$KERNEL_SRC/target/release/$LIB_NAME" +src_a="$KERNEL_SRC/target/${target_subdir}release/$LIB_NAME" src_h="$KERNEL_SRC/include/$HEADER_NAME" [ -f "$src_a" ] || { log "expected archive not produced: $src_a"; exit 1; } [ -f "$src_h" ] || { log "expected header not found: $src_h"; exit 1; } From c79afb86ba334d1cbed15ef481a4c8dc374b031c Mon Sep 17 00:00:00 2001 From: Mani Kaustubh Mathur Date: Wed, 22 Jul 2026 21:47:36 +0000 Subject: [PATCH 06/10] ci: add gated SEA/kernel test job on macOS (darwin/arm64) Adds build-and-test-kernel-macos, extending kernel/SEA CI toward its second OS. A green run validates cgo_darwin.go's link flags (positional .a for Apple ld64, -lc++, @loader_path), otherwise never linked in CI. GATED OFF by default (vars.KERNEL_MACOS_CI_ENABLED == 'true') because it is blocked by org infrastructure, not the driver code: the kernel repo is private, so building its .a needs a GitHub App token, but GitHub-hosted macos-latest runners have public IPs that are not on the databricks org IP allow list, so create-github-app-token fails before cargo/cgo run. No allow-listed macOS runner exists in the org today (the protected group offers windows-server-latest but no macOS label). Turn on once either an allow-listed macOS runner exists or the kernel publishes a downloadable darwin_arm64 .a (the distribution download path). The job body is otherwise ready: the linux kernel job minus the two JFrog steps (hosted mac has open egress; kernel + go deps are all public), keeping the App token for the private clone. macos-latest is arm64 = cgo_darwin.go's tag, so host==target (no --target); aarch64-apple-darwin is Rust tier-1; Xcode CLT gives cc + libc++. Co-authored-by: Isaac Signed-off-by: Mani Kaustubh Mathur --- .github/workflows/go.yml | 47 +++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 8c8331d7..58d5a025 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -312,27 +312,40 @@ jobs: # Exercises the SEA-via-kernel backend on macOS (darwin/arm64) — the second # kernel OS after linux/amd64, per the distribution design's one-OS-at-a-time - # rollout. This is the FIRST time cgo_darwin.go's link flags (positional .a for - # Apple ld64, -lc++, @loader_path) are compiled+linked in CI; a green run here - # is the validation that the darwin link contract is correct. + # rollout. A green run validates cgo_darwin.go's link flags (positional .a for + # Apple ld64, -lc++, @loader_path), which are otherwise never linked in CI. # - # Structurally this is the linux build-and-test-kernel job MINUS the two JFrog - # steps. It runs on a GitHub-hosted macos-latest runner (arm64), which has open - # egress: the kernel's ~300 deps are all public crates.io (no git/private - # sources) and the driver's own go.mod deps are all public, so cargo hits - # crates.io directly and go hits the default public proxy — no JFrog Go proxy, - # no cargo→JFrog credential shim. The GitHub App token step IS kept: the kernel - # repo is still private, so kernel-lib.sh's clone needs authenticated HTTPS - # (that is repo auth, unrelated to crate resolution). + # GATED OFF by default (vars.KERNEL_MACOS_CI_ENABLED == 'true'), because it is + # blocked by org infrastructure, NOT by the driver code: # - # macos-latest is arm64, matching cgo_darwin.go's `darwin && arm64` build tag, - # so host == target and kernel-lib.sh's cross-build guard passes with no - # --target. aarch64-apple-darwin is a Rust tier-1 target bundled with the pinned - # 1.96.1 toolchain (no `rustup target add`), and Xcode CLT on the runner supplies - # cc + libc++. No warehouse creds here, so only the tagged UNIT tests run; the - # live e2e / parity suites self-skip (they run in nightly-e2e). + # The kernel repo (databricks/databricks-sql-kernel) is PRIVATE, so building + # its .a requires a GitHub App token to clone it. The databricks org enforces + # an IP allow list, and GitHub-HOSTED macos-latest runners have public IPs that + # are NOT allow-listed, so create-github-app-token fails with "your IP address + # is not permitted to access this resource" before cargo/cgo ever run. The + # linux kernel job avoids this only because it runs on the self-hosted, + # allow-listed databricks-protected-runner-group — which offers a + # windows-server-latest label (see the Windows kernel job) but NO macOS label + # anywhere in the org today. + # + # Two ways to turn this on (then set the repo var to 'true'): + # 1. An allow-listed self-hosted macOS/arm64 runner becomes available — point + # runs-on at it (the App-token step then succeeds); OR + # 2. The kernel publishes a downloadable darwin_arm64 .a (the distribution + # design's kernel-lib-download path) — then drop the App-token + private + # clone entirely and this runs on hosted macos-latest with open egress. + # + # The job body below is otherwise ready: structurally the linux kernel job MINUS + # the two JFrog steps (kernel deps are all public crates.io and go.mod deps are + # all public, so a hosted runner needs no JFrog Go proxy / cargo shim), KEEPING + # the App-token step for the private clone. macos-latest is arm64 = cgo_darwin.go's + # `darwin && arm64` tag, so host == target (no --target); aarch64-apple-darwin is + # Rust tier-1 (bundled with 1.96.1) and Xcode CLT supplies cc + libc++. build-and-test-kernel-macos: name: Test (kernel backend, macOS) + # Default-off: see the block comment above. Flip vars.KERNEL_MACOS_CI_ENABLED + # to 'true' once an allow-listed macOS runner or a downloadable kernel .a exists. + if: ${{ vars.KERNEL_MACOS_CI_ENABLED == 'true' }} timeout-minutes: 30 concurrency: group: kernel-build-macos-${{ github.workflow }}-${{ github.ref }} From 521f0777b4ee2d4cce2b2d772d236e34be2f8a54 Mon Sep 17 00:00:00 2001 From: Mani Kaustubh Mathur Date: Wed, 22 Jul 2026 21:47:51 +0000 Subject: [PATCH 07/10] ci: add SEA/kernel test job on Windows (windows/amd64) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds build-and-test-kernel-windows, the third kernel OS. A green run validates cgo_windows.go's link flags (-l:libdatabricks_sql_kernel.a -lws2_32 -lwsock32 -lrstrtmgr), otherwise never linked in CI, plus the windows-gnu target plumbing. Runs on the self-hosted, allow-listed databricks-protected-runner-group with the windows-server-latest label (as databricks-jdbc does), NOT a hosted runner: create-github-app-token for the private kernel repo must run from an allow-listed IP, and hosted windows-latest is not allow-listed (same org IP-allow-list block that gates the macOS kernel job). Being on the protected runner, crates.io is blocked, so — unlike the hosted-macOS job — this keeps the JFrog Go proxy + cargo→JFrog shim, like the linux kernel job. Windows specifics: install the x86_64-pc-windows-gnu Rust target + mingw-w64 (cgo links GNU archives via gcc, not the MSVC lib the default -msvc target emits; the Makefile passes --target for GOOS=windows), and enable core.longpaths before checkout. No warehouse creds, so live e2e/parity self-skip. Co-authored-by: Isaac Signed-off-by: Mani Kaustubh Mathur --- .github/workflows/go.yml | 136 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 58d5a025..c150bbf0 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -422,3 +422,139 @@ jobs: # cgo_darwin.go. make + cc + libc++ are preinstalled on macos-latest (Xcode CLT). - name: Build kernel lib + run tagged tests run: make test-kernel + + # Exercises the SEA-via-kernel backend on Windows (windows/amd64) — the third + # kernel OS. A green run validates cgo_windows.go's link flags + # (-l:libdatabricks_sql_kernel.a -lws2_32 -lwsock32 -lrstrtmgr), otherwise never + # linked in CI, AND the Makefile/kernel-lib.sh windows-gnu target plumbing. + # + # Runs on the SELF-HOSTED, allow-listed databricks-protected-runner-group with the + # windows-server-latest label (as databricks-jdbc does). This is deliberate, not a + # hosted runner: the kernel repo is private, so create-github-app-token must run + # from an allow-listed IP — GitHub-hosted windows-latest is NOT allow-listed and + # fails the org IP allow list (the same block that gates the macOS kernel job). + # Being on the protected runner also means crates.io is blocked (go/hardened-gha), + # so — unlike the hosted-macOS job — this KEEPS the JFrog Go proxy + cargo→JFrog + # shim, exactly like the linux kernel job. + # + # Windows-specific vs the linux job: + # - cgo links with the mingw/gcc toolchain, which needs a GNU archive (.a), not + # the MSVC import lib that the default x86_64-pc-windows-MSVC host triple emits. + # So the Rust gnu target is installed and the Makefile passes + # --target x86_64-pc-windows-gnu (KERNEL_CARGO_TARGET) for GOOS=windows. + # - mingw-w64 gcc provides both the gnu-target link and cgo's own C compiler. + # - core.longpaths avoids checkout/cargo failures on deep dependency paths. + build-and-test-kernel-windows: + name: Test (kernel backend, Windows) + timeout-minutes: 40 + concurrency: + group: kernel-build-windows-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + runs-on: + group: databricks-protected-runner-group + labels: windows-server-latest + defaults: + run: + shell: bash + + steps: + # Deep cargo dependency paths can exceed the legacy 260-char Windows MAX_PATH; + # enable long paths before checkout so neither the clone nor the Rust build + # trips on it. + - name: Enable Git long paths + run: git config --system core.longpaths true + + - name: Check out code into the Go module directory + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + # Go module proxy (GOPROXY + ~/.netrc) via JFrog OIDC. Also exports + # JFROG_ACCESS_TOKEN, which the cargo step below reuses. Kept for Windows + # because the protected runner blocks direct crates.io / public proxy egress. + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + + # Point cargo at the JFrog crates proxy (protected runner blocks crates.io), + # reusing setup-jfrog's JFROG_ACCESS_TOKEN — same as the linux kernel job. The + # ~/.cargo paths resolve under git-bash on the Windows runner (HOME is set). + - name: Configure cargo to use JFrog + run: | + set -euo pipefail + mkdir -p ~/.cargo + cat > ~/.cargo/config.toml << 'EOF' + [source.crates-io] + replace-with = "jfrog" + + [source.jfrog] + registry = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" + + [registries.jfrog] + index = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" + credential-provider = ["cargo:token"] + EOF + cat > ~/.cargo/credentials.toml << EOF + [registries.jfrog] + token = "Bearer ${JFROG_ACCESS_TOKEN}" + EOF + chmod 600 ~/.cargo/credentials.toml + + - name: Set up Go Toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: '1.25.x' + cache: false + + # Install 1.96.1 (lockstep with rust-toolchain.toml) WITH the windows-gnu + # target: the default host triple is -msvc, but cgo needs a -gnu archive. + - name: Set up Rust Toolchain (windows-gnu target) + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 + with: + toolchain: 1.96.1 + target: x86_64-pc-windows-gnu + + # mingw-w64 supplies the gcc that links the -gnu Rust archive AND is cgo's own + # C compiler on Windows. choco is preinstalled on the runner image. + - name: Install mingw-w64 (gcc for cgo + gnu link) + run: choco install mingw --no-progress -y + + - name: Generate GitHub App token for databricks-sql-kernel + id: kernel-token + uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 + with: + app-id: ${{ secrets.INTEGRATION_TEST_APP_ID }} + private-key: ${{ secrets.INTEGRATION_TEST_PRIVATE_KEY }} + owner: databricks + repositories: databricks-sql-kernel + + - name: Rewrite kernel repo URL to authenticated HTTPS + env: + TOKEN: ${{ steps.kernel-token.outputs.token }} + run: | + git config --global \ + url."https://x-access-token:${TOKEN}@github.com/databricks/".insteadOf \ + "https://github.com/databricks/" + + - name: Cache kernel static lib + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + internal/backend/kernel/lib + internal/backend/kernel/include + key: ${{ runner.os }}-kernellib-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + + - name: Cache cargo + kernel build tree + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + build/kernel-src/target + key: ${{ runner.os }}-kernel-cargo-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + restore-keys: | + ${{ runner.os }}-kernel-cargo- + + # make test-kernel builds the kernel lib (make kernel-lib) — cargo build with + # --target x86_64-pc-windows-gnu (set by the Makefile for GOOS=windows), reading + # the .a from target//release/ — then runs the tagged unit tests, + # linking via cgo_windows.go. No warehouse creds, so live e2e/parity self-skip. + - name: Build kernel lib + run tagged tests + run: make test-kernel From 8991f5cc2307ba81cb37f7b05e0c089178880683 Mon Sep 17 00:00:00 2001 From: Mani Kaustubh Mathur Date: Wed, 22 Jul 2026 22:02:12 +0000 Subject: [PATCH 08/10] fix(kernel): add bcrypt and ntdll to windows cgo link flags The first windows-server-latest CI run got through the windows-gnu kernel build and the cgo link, then failed with two undefined references: BCryptGenRandom (Rust getrandom crate) -> bcrypt.dll NtCreateNamedPipeFile (Rust std child_pipe, NT API) -> ntdll.dll mingw does not auto-link these transitively, so add the two system libs to cgo_windows.go's LDFLAGS alongside the existing Winsock / Restart Manager libs. The rest of the Windows path (App token on the allow-listed runner, JFrog cargo proxy, the windows-gnu targeted build producing the .a, and the cgo link invocation itself) worked on the first run; this was the only gap. Co-authored-by: Isaac Signed-off-by: Mani Kaustubh Mathur --- internal/backend/kernel/cgo_windows.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/internal/backend/kernel/cgo_windows.go b/internal/backend/kernel/cgo_windows.go index ae650b9b..a9f19fb0 100644 --- a/internal/backend/kernel/cgo_windows.go +++ b/internal/backend/kernel/cgo_windows.go @@ -2,19 +2,24 @@ package kernel -// Link flags for windows/amd64. NOTE: this platform is not yet exercised in CI -// (M0 is linux/amd64); the flags below are the intended shape but must be -// validated on windows before it is enabled. +// Link flags for windows/amd64, validated on a windows-server-latest CI runner +// (see the build-and-test-kernel-windows job). // // cgo on windows uses the mingw/gcc toolchain, which links GNU archives (.a) — // NOT the MSVC .lib that `cargo build --target x86_64-pc-windows-msvc` emits. // So the kernel must be built for the windows-gnu target // (`--target x86_64-pc-windows-gnu`) to produce a mingw-compatible .a; that -// target selection is the build step's responsibility. -lws2_32/-lwsock32 are -// the kernel's Winsock deps and -lrstrtmgr is Restart Manager (pulled in by the -// Rust std/dep graph on windows). +// target selection is the build step's responsibility. +// +// System libs the kernel's Rust std/dep graph pulls in on windows (mingw does +// NOT auto-link these transitively, so each must be named explicitly or the +// final cgo link fails with `undefined reference`): +// -lws2_32 / -lwsock32 Winsock (sockets) +// -lrstrtmgr Restart Manager +// -lbcrypt BCryptGenRandom — Rust's getrandom crate (RNG) +// -lntdll NtCreateNamedPipeFile — Rust std child_pipe (native NT API) /* -#cgo LDFLAGS: -L${SRCDIR}/lib/windows_amd64 -l:libdatabricks_sql_kernel.a -lws2_32 -lwsock32 -lrstrtmgr +#cgo LDFLAGS: -L${SRCDIR}/lib/windows_amd64 -l:libdatabricks_sql_kernel.a -lws2_32 -lwsock32 -lrstrtmgr -lbcrypt -lntdll */ import "C" From 607bfbdb4dadfa96258b68f55a6d1222a0087a9a Mon Sep 17 00:00:00 2001 From: Mani Kaustubh Mathur Date: Thu, 23 Jul 2026 04:41:32 +0000 Subject: [PATCH 09/10] ci: add on-demand (label-triggered) E2E across all reachable OSes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EXPERIMENTAL testing scaffold for this PR: adds on-demand-e2e.yml, triggered by adding the `run-e2e` label to the PR, running the live-warehouse E2E suites on every OS we can actually reach: Thrift E2E (pure-Go, creds only): linux + macOS + Windows (hosted) Kernel E2E (needs private kernel): linux + Windows (protected runners) macOS kernel E2E is intentionally absent — a hosted macOS runner can't mint the GitHub App token for the private kernel repo (org IP allow list) and no allow-listed macOS runner exists in the org. pull_request (not schedule/dispatch) so the PR branch's workflow + code run, and #421 is a same-repo branch so it receives the DATABRICKS_PECOTESTING_* secrets. Each job hard-fails loudly if a credential is missing rather than skipping green. Testing scaffold, expected to be removed or folded into nightly-e2e before merge. Co-authored-by: Isaac Signed-off-by: Mani Kaustubh Mathur --- .github/workflows/on-demand-e2e.yml | 333 ++++++++++++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 .github/workflows/on-demand-e2e.yml diff --git a/.github/workflows/on-demand-e2e.yml b/.github/workflows/on-demand-e2e.yml new file mode 100644 index 00000000..c57d40ad --- /dev/null +++ b/.github/workflows/on-demand-e2e.yml @@ -0,0 +1,333 @@ +name: On-Demand E2E (all reachable OSes) + +# EXPERIMENTAL / testing aid for PR #421. Runs the live-warehouse E2E suites +# across every OS we can actually reach, ON DEMAND — triggered by adding the +# `run-e2e` label to a PR (re-add to re-run). Unlike schedule/workflow_dispatch +# (which run the file from the default branch), a `pull_request` trigger runs the +# file from the PR BRANCH, so this exercises the branch's code + workflow. #421 is +# a same-repo (non-fork) branch, so it receives the DATABRICKS_PECOTESTING_* +# secrets a fork PR would not. +# +# Reachable live matrix (see the per-job comments for why each is/ isn't here): +# Thrift E2E (pure-Go, needs only creds): linux + macOS + Windows (hosted) +# Kernel E2E (needs the PRIVATE kernel repo): linux + Windows (protected runners) +# macOS kernel E2E is intentionally ABSENT: a hosted macOS runner cannot mint +# the GitHub App token for the private kernel repo (org IP allow list), and no +# allow-listed macOS runner exists in the org — the same block that gates the +# macOS kernel unit job in go.yml. +# +# This is a testing scaffold, not a required gate; it is expected to be removed (or +# folded into nightly-e2e) before #421 is considered done. + +on: + pull_request: + types: [labeled] + +permissions: + contents: read + id-token: write + +jobs: + # ── Thrift (pure-Go) E2E on every OS ──────────────────────────────────────── + # Pure-Go (CGO_ENABLED=0), so no private kernel, no JFrog, no Rust/C toolchain — + # it just needs the warehouse creds. That makes all three hosted OSes reachable. + thrift-e2e: + if: github.event.label.name == 'run-e2e' + name: Thrift E2E (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + defaults: + run: + shell: bash + steps: + - name: Check out the PR branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Go Toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: '1.25.x' + cache: true + cache-dependency-path: go.sum + + # Fail loud if a credential is missing rather than letting the tests silently + # t.Skip() into a misleading green (same guard as nightly-e2e). Accept _TOKEN + # or _TOKEN_PERSONAL. + - name: Verify warehouse credentials are present + env: + HOST: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }} + HTTP_PATH: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }} + TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} + TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} + run: | + if [ -z "$HOST" ] || [ -z "$HTTP_PATH" ] || { [ -z "$TOKEN" ] && [ -z "$TOKEN_PERSONAL" ]; }; then + echo "::error::E2E warehouse secrets not reachable on this run (need DATABRICKS_PECOTESTING_SERVER_HOSTNAME, _HTTP_PATH2, and _TOKEN or _TOKEN_PERSONAL). If this is a PR-context restriction, in-repo live E2E is not possible here." >&2 + exit 1 + fi + + - name: Run Thrift E2E + env: + CGO_ENABLED: 0 + DATABRICKS_PECOTESTING_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }} + DATABRICKS_PECOTESTING_HTTP_PATH2: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }} + DATABRICKS_PECOTESTING_TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} + DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} + run: | + go test -count=1 -v -timeout 15m \ + -run 'TestE2EArrowBatchesSurviveQueryContextCancellation|TestE2ECloudFetchExactRowCount' . + + # ── Kernel (SEA) E2E on linux ─────────────────────────────────────────────── + # Protected runner (allow-listed): App token for the private kernel works, and + # crates.io is blocked so cargo uses the JFrog proxy. Mirrors nightly-e2e's + # kernel-e2e job. + kernel-e2e-linux: + if: github.event.label.name == 'run-e2e' + name: Kernel E2E (linux) + timeout-minutes: 65 + concurrency: + group: ondemand-kernel-e2e-linux-${{ github.ref }} + cancel-in-progress: true + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest + steps: + - name: Check out the PR branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + + - name: Configure cargo to use JFrog + shell: bash + run: | + set -euo pipefail + mkdir -p ~/.cargo + cat > ~/.cargo/config.toml << 'EOF' + [source.crates-io] + replace-with = "jfrog" + + [source.jfrog] + registry = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" + + [registries.jfrog] + index = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" + credential-provider = ["cargo:token"] + EOF + cat > ~/.cargo/credentials.toml << EOF + [registries.jfrog] + token = "Bearer ${JFROG_ACCESS_TOKEN}" + EOF + chmod 600 ~/.cargo/credentials.toml + + - name: Set up Go Toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: '1.25.x' + cache: false + + - name: Set up Rust Toolchain + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 + with: + toolchain: 1.96.1 + + - name: Generate GitHub App token for databricks-sql-kernel + id: kernel-token + uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 + with: + app-id: ${{ secrets.INTEGRATION_TEST_APP_ID }} + private-key: ${{ secrets.INTEGRATION_TEST_PRIVATE_KEY }} + owner: databricks + repositories: databricks-sql-kernel + + - name: Rewrite kernel repo URL to authenticated HTTPS + env: + TOKEN: ${{ steps.kernel-token.outputs.token }} + run: | + git config --global \ + url."https://x-access-token:${TOKEN}@github.com/databricks/".insteadOf \ + "https://github.com/databricks/" + + - name: Cache kernel static lib + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + internal/backend/kernel/lib + internal/backend/kernel/include + key: ${{ runner.os }}-kernellib-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + + - name: Cache cargo + kernel build tree + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + build/kernel-src/target + key: ${{ runner.os }}-kernel-cargo-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + restore-keys: | + ${{ runner.os }}-kernel-cargo- + + - name: Install build prerequisites (make, C compiler) + run: | + if ! command -v make &> /dev/null ; then + apt-get update && apt-get install -y make + fi + if ! command -v cc &> /dev/null ; then + apt-get update && apt-get install -y build-essential + fi + + - name: Verify warehouse credentials are present + env: + HOST: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }} + HTTP_PATH: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }} + TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} + TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} + run: | + if [ -z "$HOST" ] || [ -z "$HTTP_PATH" ] || { [ -z "$TOKEN" ] && [ -z "$TOKEN_PERSONAL" ]; }; then + echo "::error::E2E warehouse secrets not reachable on this run." >&2 + exit 1 + fi + + - name: Build kernel lib + timeout-minutes: 25 + run: make kernel-lib + + - name: Run kernel E2E + env: + DATABRICKS_PECOTESTING_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }} + DATABRICKS_PECOTESTING_HTTP_PATH2: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }} + DATABRICKS_PECOTESTING_TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} + DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} + run: | + CGO_ENABLED=1 go test -tags databricks_kernel -count=1 -v -timeout 30m \ + -run 'TestKernelE2E|TestKernelThriftParity|TestKernelParamsVsThrift' \ + -skip 'TestKernelE2EM2M' . + + # ── Kernel (SEA) E2E on Windows ───────────────────────────────────────────── + # Protected windows-server-latest (allow-listed → App token works). Same + # windows-gnu build plumbing as go.yml's build-and-test-kernel-windows, plus the + # warehouse creds + the kernel E2E filter. + kernel-e2e-windows: + if: github.event.label.name == 'run-e2e' + name: Kernel E2E (Windows) + timeout-minutes: 65 + concurrency: + group: ondemand-kernel-e2e-windows-${{ github.ref }} + cancel-in-progress: true + runs-on: + group: databricks-protected-runner-group + labels: windows-server-latest + defaults: + run: + shell: bash + steps: + - name: Enable Git long paths + run: git config --system core.longpaths true + + - name: Check out the PR branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + + - name: Configure cargo to use JFrog + run: | + set -euo pipefail + mkdir -p ~/.cargo + cat > ~/.cargo/config.toml << 'EOF' + [source.crates-io] + replace-with = "jfrog" + + [source.jfrog] + registry = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" + + [registries.jfrog] + index = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" + credential-provider = ["cargo:token"] + EOF + cat > ~/.cargo/credentials.toml << EOF + [registries.jfrog] + token = "Bearer ${JFROG_ACCESS_TOKEN}" + EOF + chmod 600 ~/.cargo/credentials.toml + + - name: Set up Go Toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: '1.25.x' + cache: false + + - name: Set up Rust Toolchain (windows-gnu target) + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 + with: + toolchain: 1.96.1 + target: x86_64-pc-windows-gnu + + - name: Install mingw-w64 (gcc for cgo + gnu link) + run: choco install mingw --no-progress -y + + - name: Generate GitHub App token for databricks-sql-kernel + id: kernel-token + uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 + with: + app-id: ${{ secrets.INTEGRATION_TEST_APP_ID }} + private-key: ${{ secrets.INTEGRATION_TEST_PRIVATE_KEY }} + owner: databricks + repositories: databricks-sql-kernel + + - name: Rewrite kernel repo URL to authenticated HTTPS + env: + TOKEN: ${{ steps.kernel-token.outputs.token }} + run: | + git config --global \ + url."https://x-access-token:${TOKEN}@github.com/databricks/".insteadOf \ + "https://github.com/databricks/" + + - name: Cache kernel static lib + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + internal/backend/kernel/lib + internal/backend/kernel/include + key: ${{ runner.os }}-kernellib-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + + - name: Cache cargo + kernel build tree + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + build/kernel-src/target + key: ${{ runner.os }}-kernel-cargo-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + restore-keys: | + ${{ runner.os }}-kernel-cargo- + + - name: Verify warehouse credentials are present + env: + HOST: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }} + HTTP_PATH: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }} + TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} + TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} + run: | + if [ -z "$HOST" ] || [ -z "$HTTP_PATH" ] || { [ -z "$TOKEN" ] && [ -z "$TOKEN_PERSONAL" ]; }; then + echo "::error::E2E warehouse secrets not reachable on this run." >&2 + exit 1 + fi + + - name: Build kernel lib + timeout-minutes: 25 + run: make kernel-lib + + - name: Run kernel E2E + env: + DATABRICKS_PECOTESTING_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_PECOTESTING_SERVER_HOSTNAME }} + DATABRICKS_PECOTESTING_HTTP_PATH2: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }} + DATABRICKS_PECOTESTING_TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} + DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} + run: | + CGO_ENABLED=1 go test -tags databricks_kernel -count=1 -v -timeout 30m \ + -run 'TestKernelE2E|TestKernelThriftParity|TestKernelParamsVsThrift' \ + -skip 'TestKernelE2EM2M' . From e3412cc2802507e2af3cdd1b047eac82ec6c7fc0 Mon Sep 17 00:00:00 2001 From: Mani Kaustubh Mathur Date: Thu, 23 Jul 2026 04:44:47 +0000 Subject: [PATCH 10/10] fix(ci): stop git-bash mangling the HTTP-path secret in Windows E2E MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Thrift E2E on windows-latest failed: the DATABRICKS_PECOTESTING_HTTP_PATH2 secret starts with "/" (e.g. /sql/1.0/warehouses/...), and git-bash's MSYS layer auto-converted that leading-slash value into a Windows path ("C:/Program Files/Git/sql/..."), so the driver POSTed to a mangled path and got 404. Secrets were reachable and the driver was correct — only the shell mangled the value. linux/macOS are unaffected. Set MSYS2_ARG_CONV_EXCL='*' and MSYS_NO_PATHCONV=1 on the E2E test steps so the path reaches Go verbatim. Both are MSYS-only, so they are no-ops on ubuntu/macOS and on the linux kernel job. Co-authored-by: Isaac Signed-off-by: Mani Kaustubh Mathur --- .github/workflows/on-demand-e2e.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/on-demand-e2e.yml b/.github/workflows/on-demand-e2e.yml index c57d40ad..63f17cc1 100644 --- a/.github/workflows/on-demand-e2e.yml +++ b/.github/workflows/on-demand-e2e.yml @@ -76,6 +76,14 @@ jobs: DATABRICKS_PECOTESTING_HTTP_PATH2: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }} DATABRICKS_PECOTESTING_TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} + # The HTTP-path secret starts with "/" (e.g. /sql/1.0/warehouses/...). + # Under git-bash on Windows, MSYS auto-converts leading-slash values into + # a Windows path ("/sql/..." -> "C:/Program Files/Git/sql/..."), so the + # driver POSTed to a mangled path and got 404 (linux/macOS are unaffected). + # Disable MSYS arg/path conversion so the path reaches Go verbatim. These + # vars are MSYS-only, so they are harmless no-ops on ubuntu/macOS. + MSYS2_ARG_CONV_EXCL: '*' + MSYS_NO_PATHCONV: '1' run: | go test -count=1 -v -timeout 15m \ -run 'TestE2EArrowBatchesSurviveQueryContextCancellation|TestE2ECloudFetchExactRowCount' . @@ -201,6 +209,11 @@ jobs: DATABRICKS_PECOTESTING_HTTP_PATH2: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }} DATABRICKS_PECOTESTING_TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} + # Same MSYS leading-slash path-mangling guard as the Thrift E2E job (the + # HTTP-path secret starts with "/", which git-bash would rewrite to a + # Windows path). MSYS-only vars, so a harmless no-op on the linux job. + MSYS2_ARG_CONV_EXCL: '*' + MSYS_NO_PATHCONV: '1' run: | CGO_ENABLED=1 go test -tags databricks_kernel -count=1 -v -timeout 30m \ -run 'TestKernelE2E|TestKernelThriftParity|TestKernelParamsVsThrift' \ @@ -327,6 +340,11 @@ jobs: DATABRICKS_PECOTESTING_HTTP_PATH2: ${{ secrets.DATABRICKS_PECOTESTING_HTTP_PATH2 }} DATABRICKS_PECOTESTING_TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} + # Same MSYS leading-slash path-mangling guard as the Thrift E2E job (the + # HTTP-path secret starts with "/", which git-bash would rewrite to a + # Windows path). MSYS-only vars, so a harmless no-op on the linux job. + MSYS2_ARG_CONV_EXCL: '*' + MSYS_NO_PATHCONV: '1' run: | CGO_ENABLED=1 go test -tags databricks_kernel -count=1 -v -timeout 30m \ -run 'TestKernelE2E|TestKernelThriftParity|TestKernelParamsVsThrift' \