Andrew Kent (realark) · Oct 8, 2026 · Oct 7, 2026
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -6,9 +6,18 @@ on:
     branches: [ main, master ]
   pull_request:
     branches: [ main, master ]
+  schedule:
+    # 21:13 PST / 22:13 PDT, matching the CodeQL nightly schedule.
+    - cron: '13 5 * * *'
+
+concurrency:
+  # Only newer revisions of the same PR supersede an existing run.
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
-  build:
+  build-checks:
+    if: github.event_name != 'schedule'
     # we want to run ubuntu-latest but we'll pin to a specific version so workflow is reproducable
     runs-on: ubuntu-24.04
 
@@ -21,22 +30,128 @@ jobs:
         java-version: 17
         distribution: 'temurin'
 
+    - name: Restore Gradle state
+      id: gradle-cache
+      uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: |
+          ~/.gradle/caches
+          ~/.gradle/wrapper
+        # Successful PR and default-branch jobs can add dependencies and task results.
+        key: gradle-ci-v1-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }}-${{ github.run_attempt }}
+        restore-keys: |
+          gradle-ci-v1-${{ runner.os }}-${{ runner.arch }}-
+
+    - name: Setup Gradle
+      uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0
+      with:
+        cache-disabled: true
+
     - name: Grant execute permission for gradlew
       run: chmod +x gradlew
 
     - name: Run all checks and tests
       run: ./gradlew compileTestJava publishToMavenLocal check
 
+    - name: Save Gradle state
+      if: success() && (github.event_name == 'pull_request' || github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) && steps.gradle-cache.outputs.cache-hit != 'true'
+      uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: |
+          ~/.gradle/caches
+          ~/.gradle/wrapper
+        key: ${{ steps.gradle-cache.outputs.cache-primary-key }}
+
+    - name: Show workspace changes after failure
+      if: failure()
+      run: |
+        git status --porcelain
+        git diff --no-ext-diff
+        git diff --no-ext-diff --cached
+
     - name: Upload test results
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
       if: always()
       with:
         name: test-results
-        path: build/test-results/test/
+        path: '**/build/test-results/test/'
+
+  build-logic:
+    name: Build logic tests
+    if: github.event_name != 'schedule'
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+
+    steps:
+    - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+      with:
+        persist-credentials: false
+
+    - name: Set up JDK 17
+      uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0
+      with:
+        java-version: 17
+        distribution: 'temurin'
+
+    - name: Restore build-logic Gradle state
+      id: gradle-cache
+      uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: |
+          ~/.gradle/caches
+          ~/.gradle/wrapper
+        # Separate ownership: parallel jobs must not compete to save the same key.
+        key: gradle-build-logic-v1-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }}-${{ github.run_attempt }}
+        restore-keys: |
+          gradle-build-logic-v1-${{ runner.os }}-${{ runner.arch }}-
+
+    - name: Setup Gradle
+      uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0
+      with:
+        cache-disabled: true
+
+    - name: Test build logic
+      # Standalone buildSrc does not inherit the root project's caching setting.
+      run: ./gradlew -p buildSrc test --build-cache --no-configuration-cache
+
+    - name: Save build-logic Gradle state
+      if: success() && (github.event_name == 'pull_request' || github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) && steps.gradle-cache.outputs.cache-hit != 'true'
+      uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: |
+          ~/.gradle/caches
+          ~/.gradle/wrapper
+        key: ${{ steps.gradle-cache.outputs.cache-primary-key }}
+
+    - name: Upload build-logic test results
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      if: always()
+      with:
+        name: build-logic-test-results
+        path: 'buildSrc/build/test-results/test/'
+
+  # Preserve the existing required status check while the expensive jobs overlap.
+  build:
+    needs: [ build-checks, build-logic ]
+    # A skipped required check can permit merging; cancelled dependencies must fail this gate.
+    if: always() && github.event_name != 'schedule'
+    runs-on: ubuntu-24.04
+    permissions: {}
+    steps:
+    - name: Require both build jobs to pass
+      env:
+        CHECKS_RESULT: ${{ needs.build-checks.result }}
+        LOGIC_RESULT: ${{ needs.build-logic.result }}
+      run: |
+        if [ "$CHECKS_RESULT" != 'success' ] || [ "$LOGIC_RESULT" != 'success' ]; then
+          echo "::error::Build checks: $CHECKS_RESULT; build-logic tests: $LOGIC_RESULT"
+          exit 1
+        fi
 
   dependency-check:
     name: Dependency advisories
-    if: github.event_name == 'pull_request'
+    if: github.event_name == 'pull_request' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-24.04
     permissions:
       contents: read
@@ -52,23 +167,70 @@ jobs:
         java-version: 17
         distribution: 'temurin'
 
+    - name: Restore advisory Gradle downloads
+      id: gradle-downloads
+      # Only this job's downloads; never restore the full build's muzzle/task outputs.
+      uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: |
+          ~/.gradle/caches/modules-2
+          ~/.gradle/caches/*/generated-gradle-jars
+          ~/.gradle/wrapper
+        key: gradle-advisory-downloads-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.gradle', '**/gradle.properties', 'gradle/wrapper/gradle-wrapper.properties', 'buildSrc/src/main/**', '.github/dependency-graph.json') }}
+        restore-keys: |
+          gradle-advisory-downloads-v1-${{ runner.os }}-${{ runner.arch }}-
+
+    - name: Setup Gradle
+      uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0
+      with:
+        cache-disabled: true
+
     - name: Check shipped dependencies
       id: dependencies
-      # Check the full shipped graph, including advisories that also affect main.
+      # Fresh batched requests: GITHUB_TOKEN changes each job, invalidating prior ETags.
       env:
         GH_TOKEN: ${{ github.token }}
-      run: ./gradlew checkDependencies --console=plain
+      run: |
+        rm -f build/dependency-advisories/complete
+        ./gradlew checkDependencies --refresh-dependencies --console=plain --no-configuration-cache
+
+    - name: Identify complete advisory scan
+      id: advisory-scan
+      if: always() && (steps.dependencies.outcome == 'success' || steps.dependencies.outcome == 'failure')
+      run: |
+        if [ -f build/dependency-advisories/complete ] && [ -f build/dependency-advisories/cache.json ]; then
+          echo 'scan-complete=true' >> "$GITHUB_OUTPUT"
+        fi
+
+    - name: Save advisory Gradle downloads
+      # A completed scan with findings may save downloads, but still fails the job.
+      if: always() && steps.advisory-scan.outputs.scan-complete == 'true' && steps.gradle-downloads.outputs.cache-hit != 'true'
+      uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: |
+          ~/.gradle/caches/modules-2
+          ~/.gradle/caches/*/generated-gradle-jars
+          ~/.gradle/wrapper
+        key: ${{ steps.gradle-downloads.outputs.cache-primary-key }}
 
     - name: Report dependency scan outcome
       if: always()
       env:
         SCAN_OUTCOME: ${{ steps.dependencies.outcome }}
+        SCAN_COMPLETE: ${{ steps.advisory-scan.outputs.scan-complete }}
       run: |
         echo '## Dependency advisories' >> "$GITHUB_STEP_SUMMARY"
-        if [ "$SCAN_OUTCOME" = 'success' ]; then
-          echo 'No matching GitHub-reviewed advisories were found in shipped dependencies.' >> "$GITHUB_STEP_SUMMARY"
+        if [ "$SCAN_OUTCOME" = 'success' ] && [ "$SCAN_COMPLETE" = 'true' ]; then
+          echo 'The live scan completed; no matching GitHub-reviewed advisories were found in shipped dependencies.' >> "$GITHUB_STEP_SUMMARY"
+        elif [ "$SCAN_COMPLETE" = 'true' ]; then
+          echo '::error::Dependency scan found advisories. Review the Check shipped dependencies step logs.'
+          echo 'The live scan completed and found advisories. Review the **Check shipped dependencies** step logs; this is not a clean scan.' >> "$GITHUB_STEP_SUMMARY"
         else
-          echo '::error::Dependency scan found advisories or could not complete. Review the Check shipped dependencies step logs.'
-          echo 'The scan found advisories or could not complete. Review the **Check shipped dependencies** step logs; this is not a clean scan.' >> "$GITHUB_STEP_SUMMARY"
+          echo '::error::Dependency scan could not complete. Review the job logs.'
+          echo 'The scan could not complete. No new Gradle download snapshot was saved; this is not a clean scan. Review the job logs.' >> "$GITHUB_STEP_SUMMARY"
         fi
         echo 'This check covers the full shipped dependency graph, not just dependencies changed by this PR. Findings or scan errors fail the check.' >> "$GITHUB_STEP_SUMMARY"
+        echo 'Every CI run makes fresh batched advisory requests; responses are not cached across jobs because GitHub scopes ETags to the per-job token.' >> "$GITHUB_STEP_SUMMARY"
+        if [ "$SCAN_OUTCOME" != 'success' ] || [ "$SCAN_COMPLETE" != 'true' ]; then
+          exit 1
+        fi
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -7,7 +7,12 @@ on:
   pull_request:
     branches: [ main ]
   schedule:
-    - cron: '43 20 * * 0' # Sundays at 20:43 UTC.
+    - cron: '13 5 * * *' # Daily at 21:13 PST (22:13 PDT), 05:13 UTC.
+
+concurrency:
+  # Keep main, nightly, and manual runs independent; supersede only PR revisions.
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read
@@ -29,17 +34,71 @@ jobs:
         java-version: 17
         distribution: 'temurin'
 
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1
+    - name: Restore CodeQL Gradle downloads
+      id: gradle-downloads
+      # No fallback to the full build's muzzle/task-output snapshot.
+      uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: |
+          ~/.gradle/caches/modules-2
+          ~/.gradle/caches/*/generated-gradle-jars
+          ~/.gradle/wrapper
+        key: gradle-codeql-downloads-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.gradle', '**/gradle.properties', 'gradle/wrapper/gradle-wrapper.properties', 'buildSrc/src/main/**') }}
+        restore-keys: |
+          gradle-codeql-downloads-v1-${{ runner.os }}-${{ runner.arch }}-
+
+    - name: Set up CodeQL
+      id: codeql
+      uses: github/codeql-action/setup-codeql@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1
       with:
         languages: java-kotlin
-        build-mode: manual
 
-    - name: Build Java for analysis
-      # CodeQL must observe compilation, not reuse cached outputs or an existing daemon.
-      run: ./gradlew compileJava --no-daemon --no-build-cache --rerun-tasks
+    - name: Fingerprint analysis inputs
+      id: inputs
+      env:
+        CODEQL_PATH: ${{ steps.codeql.outputs.codeql-path }}
+      run: |
+        rm -f build/codeql/cache-key.txt build/codeql/cache-inputs.json
+        ./gradlew codeqlCacheKey --refresh-dependencies --no-configuration-cache --no-daemon --no-build-cache --rerun-tasks
+        printf 'key=%s\n' "$(cat build/codeql/cache-key.txt)" >> "$GITHUB_OUTPUT"
+
+    - name: Save CodeQL Gradle downloads
+      # Snapshot resolved inputs before tracing; never cache compilation/task outputs.
+      if: success() && steps.gradle-downloads.outputs.cache-hit != 'true'
+      uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: |
+          ~/.gradle/caches/modules-2
+          ~/.gradle/caches/*/generated-gradle-jars
+          ~/.gradle/wrapper
+        key: ${{ steps.gradle-downloads.outputs.cache-primary-key }}
+
+    - name: Restore CodeQL results
+      id: results
+      uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: build/codeql/scan/results.sarif
+        # Exact matches only. The digest covers the tree, scanner, queries and build inputs.
+        key: codeql-sarif-v1-${{ runner.os }}-${{ runner.arch }}-${{ steps.inputs.outputs.key }}
+
+    - name: Scan Java
+      if: steps.results.outputs.cache-hit != 'true'
+      env:
+        CODEQL_PATH: ${{ steps.codeql.outputs.codeql-path }}
+      # Consume the fingerprinted dependencies/spec and trace a fresh full compilation.
+      run: bash scripts/scan-codeql.sh build/codeql/scan --offline
 
-    - name: Analyze
-      uses: github/codeql-action/analyze@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1
+    - name: Upload CodeQL results
+      uses: github/codeql-action/upload-sarif@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1
       with:
+        sarif_file: build/codeql/scan/results.sarif
         category: '/language:java-kotlin'
+        wait-for-processing: true
+
+    - name: Save CodeQL results
+      if: success() && steps.results.outputs.cache-hit != 'true'
+      uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: build/codeql/scan/results.sarif
+        # PR writes stay scoped to that PR's merge ref; main supplies shared results.
+        key: ${{ steps.results.outputs.cache-primary-key }}
diff --git a/.gitignore b/.gitignore
@@ -27,6 +27,8 @@ Thumbs.db
 *.ear
 hs_err_pid*
 replay_pid*
+# HotSpot creates these transient attach markers in the test JVM's working directory.
+/.attach_pid[0-9]*
 
 # Logs
 *.log

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -20,3 +20,30 @@ Because the SDK is new and under active development, third-party contribution be
 ## Development
 
 See [AGENTS.md](./AGENTS.md) for best practices developing, testing, and releasing the SDK.
+
+### CodeQL result caching
+
+CI installs the recommended CodeQL bundle before looking up cached SARIF results.
+The exact-match key covers the Git source tree (not commit history), build/workflow
+configuration, actual CodeQL version and bundled queries, resolved external Gradle
+artifacts, Java toolchains, runner environment, and fetched OpenAPI spec. Identical
+trees can reuse results after squashing; changes to any fingerprinted input cause
+a fresh scan. There is no time-based expiry in the key.
+
+On a miss, CI runs the same full-repository `compileJava` analysis as the local
+scanner, not a PR-diff-filtered analysis. Compilation runs offline with task-output
+caching disabled, using the dependencies and spec resolved before cache lookup.
+Successful reports are uploaded for the current commit/ref on both hits and
+misses, including reports with findings. Failed scans are never saved. PR cache
+writes are scoped to that PR; successful default-branch reports can be shared.
+
+`codeqlCacheKey` requires `CODEQL_PATH` and `--refresh-dependencies`; CI also passes
+`--no-configuration-cache --no-daemon --no-build-cache --rerun-tasks`. It writes
+`build/codeql/cache-inputs.json` for diagnosing misses and
+`build/codeql/spec-input/openapi/spec.yaml` for the offline scan. Input-resolution
+failures fail the job rather than fall back to stale results. Cached scans require
+the bundled query packs and reproducible dependency versions.
+
+`./gradlew checkCodeQL` still performs a fresh local scan and fails on findings.
+Run cache-key regressions with
+`./gradlew -p buildSrc test --tests dev.braintrust.gradle.CodeqlCacheKeyTest`.
diff --git a/braintrust-java-agent/build.gradle b/braintrust-java-agent/build.gradle
@@ -62,10 +62,13 @@ subprojects { subproject ->
       dependencies.add('muzzleGenerator', instrumentationApi)
       dependencies.add('muzzleGenerator', "net.bytebuddy:byte-buddy:${byteBuddyVersion}")
 
+      def muzzleOutputDir = layout.buildDirectory.dir('generated/muzzle/classes')
       task generateMuzzle(type: JavaExec) {
         dependsOn compileJava, instrumentationApi.tasks.named('compileJava')
         description = 'Generates $Muzzle side-classes for InstrumentationModule subclasses'
         group = 'build'
+        outputs.dir(muzzleOutputDir)
+        doFirst { delete muzzleOutputDir }
 
         // Classpath: compiled instrumentation classes + instrumenter + bytebuddy + deps
         classpath = files(
@@ -74,8 +77,9 @@ subprojects { subproject ->
           configurations.compileClasspath
         )
         mainClass = 'dev.braintrust.instrumentation.muzzle.MuzzleGenerator'
-        args = [sourceSets.main.java.classesDirectory.get().asFile.absolutePath]
+        args = [muzzleOutputDir.get().asFile.absolutePath]
       }
+      sourceSets.main.output.dir(muzzleOutputDir, builtBy: generateMuzzle)
 
       // Run muzzle generation after compileJava, before jar
       tasks.named('classes').configure { dependsOn generateMuzzle }