diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f644f312..499c1215 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,9 +6,18 @@ on: branches: [ main, master ] pull_request: branches: [ main, master ] + schedule: + # 21:13 PST / 22:13 PDT, matching the CodeQL nightly schedule. + - cron: '13 5 * * *' + +concurrency: + # Only newer revisions of the same PR supersede an existing run. + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - build: + build-checks: + if: github.event_name != 'schedule' # we want to run ubuntu-latest but we'll pin to a specific version so workflow is reproducable runs-on: ubuntu-24.04 @@ -21,22 +30,128 @@ jobs: java-version: 17 distribution: 'temurin' + - name: Restore Gradle state + id: gradle-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + # Successful PR and default-branch jobs can add dependencies and task results. + key: gradle-ci-v1-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + gradle-ci-v1-${{ runner.os }}-${{ runner.arch }}- + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0 + with: + cache-disabled: true + - name: Grant execute permission for gradlew run: chmod +x gradlew - name: Run all checks and tests run: ./gradlew compileTestJava publishToMavenLocal check + - name: Save Gradle state + if: success() && (github.event_name == 'pull_request' || github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) && steps.gradle-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ steps.gradle-cache.outputs.cache-primary-key }} + + - name: Show workspace changes after failure + if: failure() + run: | + git status --porcelain + git diff --no-ext-diff + git diff --no-ext-diff --cached + - name: Upload test results uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: always() with: name: test-results - path: build/test-results/test/ + path: '**/build/test-results/test/' + + build-logic: + name: Build logic tests + if: github.event_name != 'schedule' + runs-on: ubuntu-24.04 + permissions: + contents: read + + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + - name: Set up JDK 17 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 + with: + java-version: 17 + distribution: 'temurin' + + - name: Restore build-logic Gradle state + id: gradle-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + # Separate ownership: parallel jobs must not compete to save the same key. + key: gradle-build-logic-v1-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + gradle-build-logic-v1-${{ runner.os }}-${{ runner.arch }}- + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0 + with: + cache-disabled: true + + - name: Test build logic + # Standalone buildSrc does not inherit the root project's caching setting. + run: ./gradlew -p buildSrc test --build-cache --no-configuration-cache + + - name: Save build-logic Gradle state + if: success() && (github.event_name == 'pull_request' || github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) && steps.gradle-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ steps.gradle-cache.outputs.cache-primary-key }} + + - name: Upload build-logic test results + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: build-logic-test-results + path: 'buildSrc/build/test-results/test/' + + # Preserve the existing required status check while the expensive jobs overlap. + build: + needs: [ build-checks, build-logic ] + # A skipped required check can permit merging; cancelled dependencies must fail this gate. + if: always() && github.event_name != 'schedule' + runs-on: ubuntu-24.04 + permissions: {} + steps: + - name: Require both build jobs to pass + env: + CHECKS_RESULT: ${{ needs.build-checks.result }} + LOGIC_RESULT: ${{ needs.build-logic.result }} + run: | + if [ "$CHECKS_RESULT" != 'success' ] || [ "$LOGIC_RESULT" != 'success' ]; then + echo "::error::Build checks: $CHECKS_RESULT; build-logic tests: $LOGIC_RESULT" + exit 1 + fi dependency-check: name: Dependency advisories - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-24.04 permissions: contents: read @@ -52,23 +167,70 @@ jobs: java-version: 17 distribution: 'temurin' + - name: Restore advisory Gradle downloads + id: gradle-downloads + # Only this job's downloads; never restore the full build's muzzle/task outputs. + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.gradle/caches/modules-2 + ~/.gradle/caches/*/generated-gradle-jars + ~/.gradle/wrapper + key: gradle-advisory-downloads-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.gradle', '**/gradle.properties', 'gradle/wrapper/gradle-wrapper.properties', 'buildSrc/src/main/**', '.github/dependency-graph.json') }} + restore-keys: | + gradle-advisory-downloads-v1-${{ runner.os }}-${{ runner.arch }}- + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0 + with: + cache-disabled: true + - name: Check shipped dependencies id: dependencies - # Check the full shipped graph, including advisories that also affect main. + # Fresh batched requests: GITHUB_TOKEN changes each job, invalidating prior ETags. env: GH_TOKEN: ${{ github.token }} - run: ./gradlew checkDependencies --console=plain + run: | + rm -f build/dependency-advisories/complete + ./gradlew checkDependencies --refresh-dependencies --console=plain --no-configuration-cache + + - name: Identify complete advisory scan + id: advisory-scan + if: always() && (steps.dependencies.outcome == 'success' || steps.dependencies.outcome == 'failure') + run: | + if [ -f build/dependency-advisories/complete ] && [ -f build/dependency-advisories/cache.json ]; then + echo 'scan-complete=true' >> "$GITHUB_OUTPUT" + fi + + - name: Save advisory Gradle downloads + # A completed scan with findings may save downloads, but still fails the job. + if: always() && steps.advisory-scan.outputs.scan-complete == 'true' && steps.gradle-downloads.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.gradle/caches/modules-2 + ~/.gradle/caches/*/generated-gradle-jars + ~/.gradle/wrapper + key: ${{ steps.gradle-downloads.outputs.cache-primary-key }} - name: Report dependency scan outcome if: always() env: SCAN_OUTCOME: ${{ steps.dependencies.outcome }} + SCAN_COMPLETE: ${{ steps.advisory-scan.outputs.scan-complete }} run: | echo '## Dependency advisories' >> "$GITHUB_STEP_SUMMARY" - if [ "$SCAN_OUTCOME" = 'success' ]; then - echo 'No matching GitHub-reviewed advisories were found in shipped dependencies.' >> "$GITHUB_STEP_SUMMARY" + if [ "$SCAN_OUTCOME" = 'success' ] && [ "$SCAN_COMPLETE" = 'true' ]; then + echo 'The live scan completed; no matching GitHub-reviewed advisories were found in shipped dependencies.' >> "$GITHUB_STEP_SUMMARY" + elif [ "$SCAN_COMPLETE" = 'true' ]; then + echo '::error::Dependency scan found advisories. Review the Check shipped dependencies step logs.' + echo 'The live scan completed and found advisories. Review the **Check shipped dependencies** step logs; this is not a clean scan.' >> "$GITHUB_STEP_SUMMARY" else - echo '::error::Dependency scan found advisories or could not complete. Review the Check shipped dependencies step logs.' - echo 'The scan found advisories or could not complete. Review the **Check shipped dependencies** step logs; this is not a clean scan.' >> "$GITHUB_STEP_SUMMARY" + echo '::error::Dependency scan could not complete. Review the job logs.' + echo 'The scan could not complete. No new Gradle download snapshot was saved; this is not a clean scan. Review the job logs.' >> "$GITHUB_STEP_SUMMARY" fi echo 'This check covers the full shipped dependency graph, not just dependencies changed by this PR. Findings or scan errors fail the check.' >> "$GITHUB_STEP_SUMMARY" + echo 'Every CI run makes fresh batched advisory requests; responses are not cached across jobs because GitHub scopes ETags to the per-job token.' >> "$GITHUB_STEP_SUMMARY" + if [ "$SCAN_OUTCOME" != 'success' ] || [ "$SCAN_COMPLETE" != 'true' ]; then + exit 1 + fi diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 8ada2ae6..aa2ece25 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -7,7 +7,12 @@ on: pull_request: branches: [ main ] schedule: - - cron: '43 20 * * 0' # Sundays at 20:43 UTC. + - cron: '13 5 * * *' # Daily at 21:13 PST (22:13 PDT), 05:13 UTC. + +concurrency: + # Keep main, nightly, and manual runs independent; supersede only PR revisions. + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read @@ -29,17 +34,71 @@ jobs: java-version: 17 distribution: 'temurin' - - name: Initialize CodeQL - uses: github/codeql-action/init@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 + - name: Restore CodeQL Gradle downloads + id: gradle-downloads + # No fallback to the full build's muzzle/task-output snapshot. + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.gradle/caches/modules-2 + ~/.gradle/caches/*/generated-gradle-jars + ~/.gradle/wrapper + key: gradle-codeql-downloads-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.gradle', '**/gradle.properties', 'gradle/wrapper/gradle-wrapper.properties', 'buildSrc/src/main/**') }} + restore-keys: | + gradle-codeql-downloads-v1-${{ runner.os }}-${{ runner.arch }}- + + - name: Set up CodeQL + id: codeql + uses: github/codeql-action/setup-codeql@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 with: languages: java-kotlin - build-mode: manual - - name: Build Java for analysis - # CodeQL must observe compilation, not reuse cached outputs or an existing daemon. - run: ./gradlew compileJava --no-daemon --no-build-cache --rerun-tasks + - name: Fingerprint analysis inputs + id: inputs + env: + CODEQL_PATH: ${{ steps.codeql.outputs.codeql-path }} + run: | + rm -f build/codeql/cache-key.txt build/codeql/cache-inputs.json + ./gradlew codeqlCacheKey --refresh-dependencies --no-configuration-cache --no-daemon --no-build-cache --rerun-tasks + printf 'key=%s\n' "$(cat build/codeql/cache-key.txt)" >> "$GITHUB_OUTPUT" + + - name: Save CodeQL Gradle downloads + # Snapshot resolved inputs before tracing; never cache compilation/task outputs. + if: success() && steps.gradle-downloads.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.gradle/caches/modules-2 + ~/.gradle/caches/*/generated-gradle-jars + ~/.gradle/wrapper + key: ${{ steps.gradle-downloads.outputs.cache-primary-key }} + + - name: Restore CodeQL results + id: results + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: build/codeql/scan/results.sarif + # Exact matches only. The digest covers the tree, scanner, queries and build inputs. + key: codeql-sarif-v1-${{ runner.os }}-${{ runner.arch }}-${{ steps.inputs.outputs.key }} + + - name: Scan Java + if: steps.results.outputs.cache-hit != 'true' + env: + CODEQL_PATH: ${{ steps.codeql.outputs.codeql-path }} + # Consume the fingerprinted dependencies/spec and trace a fresh full compilation. + run: bash scripts/scan-codeql.sh build/codeql/scan --offline - - name: Analyze - uses: github/codeql-action/analyze@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 + - name: Upload CodeQL results + uses: github/codeql-action/upload-sarif@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 with: + sarif_file: build/codeql/scan/results.sarif category: '/language:java-kotlin' + wait-for-processing: true + + - name: Save CodeQL results + if: success() && steps.results.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: build/codeql/scan/results.sarif + # PR writes stay scoped to that PR's merge ref; main supplies shared results. + key: ${{ steps.results.outputs.cache-primary-key }} diff --git a/.gitignore b/.gitignore index 4ca2595a..30f3b6a9 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,8 @@ Thumbs.db *.ear hs_err_pid* replay_pid* +# HotSpot creates these transient attach markers in the test JVM's working directory. +/.attach_pid[0-9]* # Logs *.log diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 44aa7649..7d622627 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,3 +20,30 @@ Because the SDK is new and under active development, third-party contribution be ## Development See [AGENTS.md](./AGENTS.md) for best practices developing, testing, and releasing the SDK. + +### CodeQL result caching + +CI installs the recommended CodeQL bundle before looking up cached SARIF results. +The exact-match key covers the Git source tree (not commit history), build/workflow +configuration, actual CodeQL version and bundled queries, resolved external Gradle +artifacts, Java toolchains, runner environment, and fetched OpenAPI spec. Identical +trees can reuse results after squashing; changes to any fingerprinted input cause +a fresh scan. There is no time-based expiry in the key. + +On a miss, CI runs the same full-repository `compileJava` analysis as the local +scanner, not a PR-diff-filtered analysis. Compilation runs offline with task-output +caching disabled, using the dependencies and spec resolved before cache lookup. +Successful reports are uploaded for the current commit/ref on both hits and +misses, including reports with findings. Failed scans are never saved. PR cache +writes are scoped to that PR; successful default-branch reports can be shared. + +`codeqlCacheKey` requires `CODEQL_PATH` and `--refresh-dependencies`; CI also passes +`--no-configuration-cache --no-daemon --no-build-cache --rerun-tasks`. It writes +`build/codeql/cache-inputs.json` for diagnosing misses and +`build/codeql/spec-input/openapi/spec.yaml` for the offline scan. Input-resolution +failures fail the job rather than fall back to stale results. Cached scans require +the bundled query packs and reproducible dependency versions. + +`./gradlew checkCodeQL` still performs a fresh local scan and fails on findings. +Run cache-key regressions with +`./gradlew -p buildSrc test --tests dev.braintrust.gradle.CodeqlCacheKeyTest`. diff --git a/braintrust-java-agent/build.gradle b/braintrust-java-agent/build.gradle index 726bf3b9..d667d699 100644 --- a/braintrust-java-agent/build.gradle +++ b/braintrust-java-agent/build.gradle @@ -62,10 +62,13 @@ subprojects { subproject -> dependencies.add('muzzleGenerator', instrumentationApi) dependencies.add('muzzleGenerator', "net.bytebuddy:byte-buddy:${byteBuddyVersion}") + def muzzleOutputDir = layout.buildDirectory.dir('generated/muzzle/classes') task generateMuzzle(type: JavaExec) { dependsOn compileJava, instrumentationApi.tasks.named('compileJava') description = 'Generates $Muzzle side-classes for InstrumentationModule subclasses' group = 'build' + outputs.dir(muzzleOutputDir) + doFirst { delete muzzleOutputDir } // Classpath: compiled instrumentation classes + instrumenter + bytebuddy + deps classpath = files( @@ -74,8 +77,9 @@ subprojects { subproject -> configurations.compileClasspath ) mainClass = 'dev.braintrust.instrumentation.muzzle.MuzzleGenerator' - args = [sourceSets.main.java.classesDirectory.get().asFile.absolutePath] + args = [muzzleOutputDir.get().asFile.absolutePath] } + sourceSets.main.output.dir(muzzleOutputDir, builtBy: generateMuzzle) // Run muzzle generation after compileJava, before jar tasks.named('classes').configure { dependsOn generateMuzzle } diff --git a/braintrust-sdk/build.gradle b/braintrust-sdk/build.gradle index a2d78c47..1d2c947a 100644 --- a/braintrust-sdk/build.gradle +++ b/braintrust-sdk/build.gradle @@ -68,10 +68,13 @@ subprojects { subproject -> dependencies.add('muzzleGenerator', instrumentationApi) dependencies.add('muzzleGenerator', "net.bytebuddy:byte-buddy:${byteBuddyVersion}") + def muzzleOutputDir = layout.buildDirectory.dir('generated/muzzle/classes') task generateMuzzle(type: JavaExec) { dependsOn compileJava, instrumentationApi.tasks.named('compileJava') description = 'Generates $Muzzle side-classes for InstrumentationModule subclasses' group = 'build' + outputs.dir(muzzleOutputDir) + doFirst { delete muzzleOutputDir } // Classpath: compiled instrumentation classes + instrumenter + bytebuddy + deps classpath = files( @@ -80,8 +83,9 @@ subprojects { subproject -> configurations.compileClasspath ) mainClass = 'dev.braintrust.instrumentation.muzzle.MuzzleGenerator' - args = [sourceSets.main.java.classesDirectory.get().asFile.absolutePath] + args = [muzzleOutputDir.get().asFile.absolutePath] } + sourceSets.main.output.dir(muzzleOutputDir, builtBy: generateMuzzle) // Run muzzle generation after compileJava, before jar tasks.named('classes').configure { dependsOn generateMuzzle } diff --git a/build.gradle b/build.gradle index 8ffc4888..866ffe6b 100644 --- a/build.gradle +++ b/build.gradle @@ -160,3 +160,30 @@ tasks.register('checkCodeQL', Exec) { workingDir rootDir commandLine 'mise', 'exec', '--', './scripts/check-codeql.sh' } + +// Resolve and fingerprint inputs before deciding whether a full CodeQL scan is needed. +// --refresh-dependencies is mandatory; the subsequent scan consumes this lookup offline. +def resetCodeqlCacheKey = tasks.register('resetCodeqlCacheKey', Delete) { + delete layout.buildDirectory.file('codeql/cache-key.txt') + delete layout.buildDirectory.file('codeql/cache-inputs.json') + delete layout.buildDirectory.dir('codeql/spec-input') +} +def codeqlProjects = allprojects.collect { owner -> + owner.tasks.register('codeqlProjectInputs', dev.braintrust.gradle.CodeqlProjectInputsTask) { + dependsOn resetCodeqlCacheKey + outputFile = owner.layout.buildDirectory.file('codeql/project-inputs.json') + } +} +tasks.register('codeqlCacheKey', dev.braintrust.gradle.CodeqlCacheKeyTask) { + group = 'verification' + description = 'Fingerprints exact inputs for a full CodeQL scan without compiling application code' + dependsOn ':braintrust-api:fetchOpenApiSpec', resetCodeqlCacheKey + projectInputs.from(codeqlProjects) + openApiSpec = project(':braintrust-api').layout.buildDirectory.file('openapi/spec.yaml') + specSnapshot = layout.buildDirectory.file('codeql/spec-input/openapi/spec.yaml') + keyFile = layout.buildDirectory.file('codeql/cache-key.txt') + manifestFile = layout.buildDirectory.file('codeql/cache-inputs.json') +} +project(':braintrust-api').tasks.matching { it.name == 'fetchOpenApiSpec' }.configureEach { + mustRunAfter resetCodeqlCacheKey +} diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index df6d0fff..c27fbdea 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -10,4 +10,14 @@ repositories { dependencies { implementation gradleApi() implementation localGroovy() + testImplementation gradleTestKit() + testImplementation 'org.junit.jupiter:junit-jupiter:5.11.4' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' +} + +test { + useJUnitPlatform() + systemProperty 'muzzle.test.classpath', sourceSets.main.runtimeClasspath.asPath + systemProperty 'codeql.test.classpath', sourceSets.main.runtimeClasspath.asPath + systemProperty 'advisories.test.classpath', sourceSets.main.runtimeClasspath.asPath } diff --git a/buildSrc/src/main/groovy/dev/braintrust/gradle/CheckDependenciesTask.groovy b/buildSrc/src/main/groovy/dev/braintrust/gradle/CheckDependenciesTask.groovy index dadab0a1..8ec79fc5 100644 --- a/buildSrc/src/main/groovy/dev/braintrust/gradle/CheckDependenciesTask.groovy +++ b/buildSrc/src/main/groovy/dev/braintrust/gradle/CheckDependenciesTask.groovy @@ -1,62 +1,60 @@ package dev.braintrust.gradle -import groovy.json.JsonSlurper import org.gradle.api.DefaultTask import org.gradle.api.GradleException import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.DirectoryProperty import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.LocalState import org.gradle.api.tasks.TaskAction -import java.util.concurrent.Callable -import java.util.concurrent.Executors - abstract class CheckDependenciesTask extends DefaultTask { /** Coordinate lists written by each shipped project's {@link ShippedDependenciesTask}. */ @InputFiles abstract ConfigurableFileCollection getShippedDependencies() + /** Sources and shipped-graph policy defining the compatibility of cached API responses. */ + @InputFiles + abstract ConfigurableFileCollection getScannerSourceFiles() + + @LocalState + abstract DirectoryProperty getAdvisoryCacheDirectory() + CheckDependenciesTask() { group = 'verification' description = 'Checks shipped dependencies against GitHub-reviewed advisories (requires authenticated gh)' // Advisories change independently of the inputs, so never treat a previous scan as up to date. outputs.upToDateWhen { false } + outputs.cacheIf { false } + advisoryCacheDirectory.convention(project.rootProject.layout.buildDirectory.dir('dependency-advisories')) + scannerSourceFiles.from( + project.rootProject.file('buildSrc/src/main/groovy/dev/braintrust/gradle/CheckDependenciesTask.groovy'), + project.rootProject.file('buildSrc/src/main/groovy/dev/braintrust/gradle/DependencyAdvisories.groovy'), + project.rootProject.file('.github/dependency-graph.json')) } @TaskAction void checkDependencies() { + def directory = advisoryCacheDirectory.get().asFile + DependencyAdvisories.clearCompletion(directory) logger.lifecycle('Resolving shipped dependencies from the current working tree (including uncommitted changes)...') def packages = shippedPackages() - logger.lifecycle("Checking ${packages.size()} shipped dependency versions against GitHub-reviewed advisories...") - - def executor = Executors.newFixedThreadPool(6) - List results - try { - def futures = packages.collect { coordinate -> - executor.submit({ -> CheckDependenciesTask.advisories(coordinate) } as Callable) - } - // An API failure must never be reported as a clean or complete scan. - results = futures.collect { it.get() } - } catch (Exception e) { - throw new GradleException("Dependency scan incomplete: ${e.cause?.message ?: e.message}", e) - } finally { - executor.shutdownNow() - } - - def findings = results.findAll { !it.advisories.isEmpty() } - if (findings.isEmpty()) { + def result = DependencyAdvisories.scan(packages, + DependencyAdvisories.identity(scannerSourceFiles.files), directory, + { url, etag -> requestAdvisories(url, etag) }, + { message -> logger.lifecycle(message.toString()) }) + if (result.findings.isEmpty()) { logger.lifecycle("PASS: No matching GitHub-reviewed advisories in ${packages.size()} shipped dependency versions.") return } - findings.each { finding -> - logger.lifecycle("\n${finding.coordinate}") - finding.advisories.each { advisory -> - logger.lifecycle(" ${advisory.severity.toUpperCase(Locale.ROOT)} ${advisory.cve_id ?: advisory.ghsa_id}: ${advisory.summary}") - logger.lifecycle(" ${advisory.html_url}") - } + result.findings.each { finding -> + def advisory = finding.advisory + logger.lifecycle("\n${advisory.severity.toUpperCase(Locale.ROOT)} ${advisory.ghsa_id}: ${advisory.summary}") + logger.lifecycle(" Relevant packages: ${finding.packages.join(', ')}") + logger.lifecycle(" ${advisory.html_url}") } - int count = findings.sum { it.advisories.size() } - throw new GradleException("${count} advisory matches across ${findings.size()} shipped dependency versions.") + throw new GradleException("${result.findings.size()} active GitHub-reviewed advisories match the shipped dependency graph.") } private SortedSet shippedPackages() { @@ -70,36 +68,8 @@ abstract class CheckDependenciesTask extends DefaultTask { return packages } - private static Map advisories(String coordinate) { - def stdout = new StringBuilder() - def stderr = new StringBuilder() - def process = new ProcessBuilder([ - 'gh', 'api', '--hostname', 'github.com', '--method', 'GET', - '--paginate', '--slurp', '/advisories', - '-f', 'type=reviewed', '-f', 'ecosystem=maven', - '-f', "affects=${coordinate}".toString(), '-f', 'per_page=100' - ]).start() - try { - process.waitForProcessOutput(stdout, stderr) - if (process.exitValue() != 0) { - throw new GradleException("gh exited with status ${process.exitValue()} for ${coordinate}:\n${stdout}\n${stderr}") - } - } finally { - process.destroy() - } - def pages = new JsonSlurper().parseText(stdout.toString()) - if (!(pages instanceof List) || pages.any { !(it instanceof List) }) { - throw new GradleException("Invalid GitHub advisory response for ${coordinate}") - } - def active = [:] - pages.flatten().each { advisory -> - if (!(advisory instanceof Map) || !advisory.ghsa_id) { - throw new GradleException("Invalid GitHub advisory for ${coordinate}") - } - if (!advisory.withdrawn_at) { - active[advisory.ghsa_id] = advisory - } - } - return [coordinate: coordinate, advisories: active.values().sort { it.ghsa_id }] + /** Kept injectable for task integration tests; production transport remains authenticated gh. */ + protected Map requestAdvisories(String url, String etag) { + DependencyAdvisories.request(url, etag) } } diff --git a/buildSrc/src/main/groovy/dev/braintrust/gradle/CodeqlCacheKeyTask.groovy b/buildSrc/src/main/groovy/dev/braintrust/gradle/CodeqlCacheKeyTask.groovy new file mode 100644 index 00000000..c8b671e8 --- /dev/null +++ b/buildSrc/src/main/groovy/dev/braintrust/gradle/CodeqlCacheKeyTask.groovy @@ -0,0 +1,220 @@ +package dev.braintrust.gradle + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.TaskAction + +import java.nio.file.FileVisitOption +import java.nio.file.Files +import java.nio.file.StandardCopyOption +import java.security.MessageDigest + +/** Exact, history-independent inputs for the standalone, full-repository CodeQL scan. */ +abstract class CodeqlCacheKeyTask extends DefaultTask { + @InputFiles + abstract ConfigurableFileCollection getProjectInputs() + + // The fetch task owns this file. Do not fingerprint it before that task runs. + @Internal + abstract RegularFileProperty getOpenApiSpec() + + @OutputFile + abstract RegularFileProperty getKeyFile() + + @OutputFile + abstract RegularFileProperty getManifestFile() + + @OutputFile + abstract RegularFileProperty getSpecSnapshot() + + private final Map digests = [:] + private final byte[] digestBuffer = new byte[65536] + + CodeqlCacheKeyTask() { + outputs.upToDateWhen { false } + outputs.cacheIf { false } + } + + @TaskAction + void writeKey() { + Files.deleteIfExists(keyFile.get().asFile.toPath()) + Files.deleteIfExists(manifestFile.get().asFile.toPath()) + digests.clear() + if (!project.gradle.startParameter.refreshDependencies || project.gradle.startParameter.offline) { + throw new GradleException('codeqlCacheKey requires --refresh-dependencies and an online dependency lookup') + } + def projects = projectInputs.files.collect { input -> + def data = new JsonSlurper().parse(input, 'UTF-8') + data.configurations.values().each { configuration -> + configuration.artifacts.each { artifact -> + if (artifact.file) artifact.sha256 = digestPath(new File(artifact.remove('file').toString())) + } + } + data.compilers.values().each { compiler -> + compiler.releaseSha256 = digestFile(new File(compiler.remove('releaseFile').toString())) + } + data + }.sort { it.project } + + def snapshot = specSnapshot.get().asFile + snapshot.parentFile.mkdirs() + Files.copy(openApiSpec.get().asFile.toPath(), snapshot.toPath(), StandardCopyOption.REPLACE_EXISTING) + def manifest = [schema: 1, source: sourceInputs(project.rootDir), projects: projects, + openApiSpec: digestFile(snapshot), codeql: codeqlInputs(), + runtime: runtimeInputs()] + def canonical = canonicalJson(manifest) + def output = manifestFile.get().asFile + output.parentFile.mkdirs() + output.setText(JsonOutput.prettyPrint(canonical) + '\n', 'UTF-8') + // Publish the key last: failures never produce a usable cache lookup key. + keyFile.get().asFile.setText(sha256(canonical.getBytes('UTF-8')) + '\n', 'UTF-8') + } + + Map sourceInputs(File root) { + def tree = command(root, ['git', 'rev-parse', 'HEAD^{tree}']).trim() + // Binary diff includes both staged and unstaged content relative to HEAD, not its commit ID. + def diff = commandBytes(root, ['git', '-c', 'core.quotePath=false', 'diff', '--no-ext-diff', + '--no-textconv', '--binary', '--ignore-submodules=none', 'HEAD', '--']) + def tracked = command(root, ['git', 'ls-files', '--stage', '-z']) + if (tracked.split('\u0000').any { it.startsWith('160000 ') }) { + throw new GradleException('CodeQL cache fingerprint does not support submodule worktrees') + } + def untracked = new TreeMap() + command(root, ['git', 'ls-files', '--others', '--exclude-standard', '-z']).split('\u0000') + .findAll { !it.isEmpty() }.each { name -> untracked[name] = digestPath(new File(root, name)) } + [tree: tree, trackedDiff: sha256(diff), untracked: untracked] + } + + Map codeqlInputs() { + def executable = System.getenv('CODEQL_PATH') + if (!executable || !new File(executable).isFile()) { + throw new GradleException('CODEQL_PATH must name the installed CodeQL executable') + } + def version = new JsonSlurper().parseText(command(project.rootDir, [executable, 'version', '--format=json'])) + if (!version.version || !version.sha || !version.unpackedLocation) { + throw new GradleException('CodeQL version output is missing version, sha, or unpackedLocation') + } + if (version.configFileFound) { + throw new GradleException('Custom CodeQL CLI configuration is unsupported for cached scans') + } + def installation = new File(version.unpackedLocation.toString()).canonicalFile + def packs = new JsonSlurper().parseText(command(project.rootDir, + [executable, 'resolve', 'qlpacks', '--format=json'])) + if (!(packs instanceof Map) || !packs['codeql/java-queries'] || !packs['codeql/java-all']) { + throw new GradleException('The installed CodeQL bundle must contain Java query and library packs') + } + def packInputs = new TreeMap() + packs.each { name, locations -> + if (!(locations instanceof List) || locations.isEmpty()) { + throw new GradleException("No resolved CodeQL pack locations for ${name}") + } + packInputs[name.toString()] = locations.collect { location -> + def directory = new File(location.toString()).canonicalFile + if (!directory.toPath().startsWith(installation.toPath())) { + throw new GradleException("Non-bundled CodeQL pack is unsupported: ${name}") + } + def metadata = new TreeMap() + ['qlpack.yml', 'codeql-pack.yml', 'codeql-pack.lock.yml'].each { filename -> + def file = new File(directory, filename) + if (file.isFile()) metadata[filename] = digestFile(file) + } + // legacy-upgrades is not a versioned query pack. + if (metadata.isEmpty() && name != 'legacy-upgrades') { + throw new GradleException("Missing metadata for CodeQL pack ${name}") + } + [metadata: metadata, + content: name == 'codeql/java-queries' || name == 'codeql/java-all' ? digestPath(directory) : null] + }.unique { canonicalJson(it) }.sort { canonicalJson(it) } + } + // Java query-pack content includes its bundled transitive library packs under .codeql/libraries. + [version: version.version, sha: version.sha, executable: digestFile(new File(executable)), + suite: 'java-code-scanning.qls', packs: packInputs] + } + + Map runtimeInputs() { + def properties = new TreeMap() + ['java.version', 'java.vendor', 'java.runtime.version', 'java.vm.name', 'java.vm.version', + 'java.vm.vendor', 'os.name', 'os.version', 'os.arch', 'file.encoding', 'user.language', + 'user.country', 'user.timezone'].each { name -> properties[name] = System.getProperty(name, '') } + def environment = new TreeMap() + System.getenv().each { name, value -> + if (name.startsWith('CODEQL_EXTRACTOR_') || name.startsWith('LGTM_') || + name in ['JAVA_TOOL_OPTIONS', 'JDK_JAVA_OPTIONS', '_JAVA_OPTIONS', 'GRADLE_OPTS', + 'LANG', 'LC_ALL', 'TZ', 'ImageOS', 'ImageVersion']) { + // Hash values: diagnostics must not disclose options containing credentials. + environment[name] = sha256(value.getBytes('UTF-8')) + } + } + [gradle: project.gradle.gradleVersion, properties: properties, environment: environment, + projectProperties: project.gradle.startParameter.projectProperties.collectEntries { name, value -> + [(name): sha256(value.getBytes('UTF-8'))] + }] + } + + String digestPath(File file) { + if (file.isFile()) return digestFile(file) + if (!file.isDirectory()) throw new GradleException("Missing fingerprint input: ${file}") + def entries = new TreeMap() + Files.walk(file.toPath(), FileVisitOption.FOLLOW_LINKS).withCloseable { paths -> + paths.filter { !Files.isDirectory(it) }.forEach { path -> + entries[file.toPath().relativize(path).toString().replace(File.separator, '/')] = digestFile(path.toFile()) + } + } + sha256(canonicalJson(entries).getBytes('UTF-8')) + } + + String digestFile(File file) { + def path = file.canonicalPath + if (!digests.containsKey(path)) { + if (!file.isFile()) throw new GradleException("Missing fingerprint input: ${file}") + def digest = MessageDigest.getInstance('SHA-256') + byte[] buffer = digestBuffer + file.withInputStream { input -> + int count + while ((count = input.read(buffer)) != -1) digest.update(buffer, 0, count) + } + digests[path] = digest.digest().encodeHex().toString() + } + digests[path] + } + + static String canonicalJson(Object input) { + JsonOutput.toJson(canonicalize(input)) + } + + private static Object canonicalize(Object input) { + if (input instanceof Map) { + def sorted = new TreeMap() + input.each { key, value -> sorted[key.toString()] = canonicalize(value) } + return sorted + } + if (input instanceof Collection) return input.collect { canonicalize(it) } + input + } + + static String sha256(byte[] input) { + MessageDigest.getInstance('SHA-256').digest(input).encodeHex().toString() + } + + static String command(File directory, List arguments) { + new String(commandBytes(directory, arguments), 'UTF-8') + } + + private static byte[] commandBytes(File directory, List arguments) { + def process = new ProcessBuilder(arguments).directory(directory).start() + def stdout = new ByteArrayOutputStream() + def stderr = new ByteArrayOutputStream() + process.waitForProcessOutput(stdout, stderr) + if (process.exitValue() != 0) { + throw new GradleException("Cannot fingerprint input (${arguments.join(' ')}): ${stderr.toString('UTF-8')}") + } + stdout.toByteArray() + } +} diff --git a/buildSrc/src/main/groovy/dev/braintrust/gradle/CodeqlProjectInputsTask.groovy b/buildSrc/src/main/groovy/dev/braintrust/gradle/CodeqlProjectInputsTask.groovy new file mode 100644 index 00000000..9c83876c --- /dev/null +++ b/buildSrc/src/main/groovy/dev/braintrust/gradle/CodeqlProjectInputsTask.groovy @@ -0,0 +1,88 @@ +package dev.braintrust.gradle + +import groovy.json.JsonOutput +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.artifacts.component.ModuleComponentIdentifier +import org.gradle.api.artifacts.component.ProjectComponentIdentifier +import org.gradle.api.artifacts.result.UnresolvedDependencyResult +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.TaskAction +import org.gradle.api.tasks.compile.JavaCompile + +/** Resolves inputs in the owning project, without requesting any project artifact builds. */ +abstract class CodeqlProjectInputsTask extends DefaultTask { + @OutputFile + abstract RegularFileProperty getOutputFile() + + CodeqlProjectInputsTask() { + outputs.upToDateWhen { false } + outputs.cacheIf { false } + } + + @TaskAction + void collectInputs() { + def configurations = new TreeMap() + def buildDirectories = project.rootProject.allprojects.collectEntries { + [(it.path): it.layout.buildDirectory.get().asFile.toPath()] + } + [project: project.configurations, buildscript: project.buildscript.configurations].each { scope, container -> + container.findAll { it.canBeResolved }.sort { it.name }.each { configuration -> + // Fixed dependencies plus a fresh lookup are required before the offline scan. + if (configuration.state == org.gradle.api.artifacts.Configuration.State.UNRESOLVED) { + configuration.resolutionStrategy.failOnNonReproducibleResolution() + } + def graph = configuration.incoming.resolutionResult + graph.allDependencies.each { dependency -> + if (dependency instanceof UnresolvedDependencyResult) { + throw new GradleException("Cannot fingerprint ${project.path}:${configuration.name}", dependency.failure) + } + } + def modules = graph.allComponents.findAll { it.id instanceof ModuleComponentIdentifier } + .collect { it.id.displayName }.sort() + def artifacts = configuration.incoming.artifactView { + componentFilter { !(it instanceof ProjectComponentIdentifier) } + }.artifacts.artifacts.collect { artifact -> + def id = artifact.id.componentIdentifier + def input = [component: id instanceof ModuleComponentIdentifier ? id.displayName : 'file', + name: artifact.file.name, + attributes: CodeqlProjectInputsTask.attributes(artifact.variant.attributes)] + def generated = buildDirectories.find { owner, directory -> artifact.file.toPath().startsWith(directory) } + if (generated) { + // Source/configuration and producer dependencies are fingerprinted instead: + // these outputs may not exist until the cache-miss compilation runs. + input.attributes.remove('artifactType') + input.generated = "${generated.key}:${generated.value.relativize(artifact.file.toPath())}".toString() + } else { + input.file = artifact.file.absolutePath + } + input + } + configurations["${scope}:${configuration.name}".toString()] = [ + attributes: CodeqlProjectInputsTask.attributes(configuration.attributes), modules: modules, artifacts: artifacts] + } + } + def compilers = new TreeMap() + project.tasks.withType(JavaCompile).each { compile -> + def metadata = compile.javaCompiler.get().metadata + compilers[compile.name] = [ + languageVersion: metadata.languageVersion.asInt(), vendor: metadata.vendor, + javaRuntimeVersion: metadata.javaRuntimeVersion, jvmVersion: metadata.jvmVersion, + releaseFile: new File(metadata.installationPath.asFile, 'release').absolutePath, + sourceCompatibility: compile.sourceCompatibility, targetCompatibility: compile.targetCompatibility, + release: compile.options.release.orNull, encoding: compile.options.encoding, + compilerArgs: compile.options.compilerArgs, + forkJvmArgs: compile.options.forkOptions.jvmArgs] + } + def output = outputFile.get().asFile + output.parentFile.mkdirs() + output.setText(JsonOutput.toJson([project: project.path, configurations: configurations, compilers: compilers]), 'UTF-8') + } + + private static Map attributes(container) { + def result = new TreeMap() + container.keySet().each { key -> result[key.name] = container.getAttribute(key).toString() } + result + } +} diff --git a/buildSrc/src/main/groovy/dev/braintrust/gradle/DependencyAdvisories.groovy b/buildSrc/src/main/groovy/dev/braintrust/gradle/DependencyAdvisories.groovy new file mode 100644 index 00000000..562baa1b --- /dev/null +++ b/buildSrc/src/main/groovy/dev/braintrust/gradle/DependencyAdvisories.groovy @@ -0,0 +1,378 @@ +package dev.braintrust.gradle + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import org.gradle.api.GradleException + +import java.nio.file.AtomicMoveNotSupportedException +import java.nio.file.Files +import java.nio.file.StandardCopyOption +import java.security.MessageDigest +import java.time.OffsetDateTime + +/** Live, page-by-page revalidation; cached responses are candidates, never offline answers. */ +class DependencyAdvisories { + static final String API_VERSION = '2022-11-28' + static final int MAX_URL_LENGTH = 5800 + static final int MAX_PACKAGES = 1000 + static final Map POLICY = [format: 1, apiVersion: API_VERSION, type: 'reviewed', ecosystem: 'maven', perPage: 100] + private static final String PREFIX = 'https://api.github.com/advisories?type=reviewed&ecosystem=maven&per_page=100&affects=' + + static String identity(Collection inputs) { + if (inputs.isEmpty()) { + throw new GradleException('Scanner implementation/policy inputs are missing.') + } + def digest = MessageDigest.getInstance('SHA-256') + inputs.sort { it.name }.each { file -> + if (!file.isFile()) { + throw new GradleException("Scanner implementation/policy input is missing: ${file}") + } + digest.update(file.name.getBytes('UTF-8')) + digest.update((byte) 0) + digest.update(file.bytes) + digest.update((byte) 0) + } + digest.digest().encodeHex().toString() + } + + static List batches(Collection coordinates) { + def sorted = new TreeSet(coordinates) + if (sorted.isEmpty()) { + throw new GradleException('The shipped dependency graph is empty; refusing to report a clean scan.') + } + def result = [] + def current = [] + def url = new StringBuilder(PREFIX) + sorted.each { coordinate -> + if (!(coordinate ==~ /[^\s,:@]+:[^\s,:@]+@[^\s,@]+/)) { + throw new GradleException("Invalid shipped dependency coordinate: ${coordinate}") + } + String encoded = URLEncoder.encode(coordinate, 'UTF-8') + if (PREFIX.length() + encoded.length() > MAX_URL_LENGTH) { + throw new GradleException("Dependency coordinate exceeds the advisory URL limit: ${coordinate}") + } + int separatorLength = current.isEmpty() ? 0 : 3 // Percent-encoded comma. + if (current.size() == MAX_PACKAGES || url.length() + separatorLength + encoded.length() > MAX_URL_LENGTH) { + result.add([coordinates: current, url: url.toString()]) + current = [] + url.setLength(PREFIX.length()) + } + if (!current.isEmpty()) { + url.append('%2C') + } + current.add(coordinate) + url.append(encoded) + } + if (!current.isEmpty()) { + result.add([coordinates: current, url: url.toString()]) + } + result + } + + static void clearCompletion(File directory) { + Files.deleteIfExists(new File(directory, 'complete').toPath()) + } + + /** Transport returns [exitCode, stdout, stderr] from gh api --include (including its HTTP status). */ + static Map scan(Collection coordinates, String implementationIdentity, File directory, + Closure transport, Closure log) { + clearCompletion(directory) + int pages = 0 + int hits = 0 + int completedBatches = 0 + def chunks = batches(coordinates) + def cacheFile = new File(directory, 'cache.json') + def previous = loadCache(cacheFile, implementationIdentity) + def nextPages = new TreeMap() + def advisories = new TreeMap() + log.call("Checking ${coordinates.toSet().size()} shipped dependency versions in ${chunks.size()} advisory batches; ${previous.size()} candidate cached pages.") + try { + chunks.eachWithIndex { batch, index -> + String url = batch.url + def seen = new HashSet() + int batchPages = 0 + int batchHits = 0 + while (url != null) { + validateUrl(url, batch.url) + if (!seen.add(url)) { + throw new GradleException('Pagination loop in GitHub advisory response.') + } + def candidate = previous[url] + if (!validCandidate(candidate, batch.coordinates, batch.url) || + (candidate?.next == null && candidate?.body?.size() == POLICY.perPage)) { + // A full terminal page can gain a next Link without changing its body/ETag. + // Fetch it unconditionally so an omitted Link on 304 cannot hide a new page. + candidate = null + } + def response = parseResponse(transport.call(url, candidate?.etag)) + Map page + if (response.status == 304) { + // gh reports a genuine 304 as exit 1. Other failures must not become hits. + if (!candidate || !candidate.etag || !(response.exitCode in [0, 1]) || + (response.exitCode == 1 && response.stderr.trim() != 'gh: HTTP 304') || + !response.body.trim().isEmpty()) { + throw new GradleException('Unusable HTTP 304: no matching validated cache response or gh transport failed.') + } + page = [coordinates: candidate.coordinates, etag: candidate.etag, + next: candidate.next, body: candidate.body] + if (response.headers.containsKey('link')) { + page.next = nextLink(response.headers.link) + if (page.next != null) { + validateUrl(page.next, batch.url) + } + } + hits++ + batchHits++ + } else if (response.status == 200 && response.exitCode == 0) { + def body = new JsonSlurper().parseText(response.body) + validateBody(body) + def next = nextLink(response.headers.link) + if (next != null) { + validateUrl(next, batch.url) + } + String etag = response.headers.etag + if (etag != null && !(etag ==~ /(?:W\/)?"[^"\r\n]*"/)) { + throw new GradleException('Malformed ETag in GitHub advisory response.') + } + page = [coordinates: batch.coordinates, etag: etag, next: next, body: body] + } else { + throw new GradleException("GitHub advisory request failed (HTTP ${response.status}, gh exit ${response.exitCode}); check gh authentication, permissions and API availability.") + } + nextPages[url] = page + page.body.each { advisory -> + def existing = advisories[advisory.ghsa_id] + // Concurrent API updates can expose conflicting duplicates across pages/batches. + // A later withdrawn copy must not erase an active match already observed. + if (existing == null || existing.withdrawn_at != null || advisory.withdrawn_at == null) { + advisories[advisory.ghsa_id] = advisory + } + } + pages++ + batchPages++ + url = page.next + } + completedBatches++ + log.call("Advisory batch ${index + 1}/${chunks.size()}: ${batch.coordinates.size()} versions, ${batchPages} pages, ${batchHits} revalidated cache hits.") + } + def names = coordinates.collect { it.substring(0, it.lastIndexOf('@')) }.toSet() + def findings = advisories.values().findAll { it.withdrawn_at == null }.collect { advisory -> + def relevant = advisory.vulnerabilities.findAll { + it.package.ecosystem == 'maven' && names.contains(it.package.name) + }.collect { it.package.name }.toSorted().unique() + if (relevant.isEmpty()) { + throw new GradleException("GitHub advisory ${advisory.ghsa_id} has no relevant Maven package in its response.") + } + [advisory: advisory, packages: relevant] + } + Files.createDirectories(directory.toPath()) + def temporary = Files.createTempFile(directory.toPath(), 'cache-', '.tmp') + try { + Files.writeString(temporary, JsonOutput.toJson(canonical([policy: POLICY, identity: implementationIdentity, pages: nextPages])) + '\n') + try { + Files.move(temporary, cacheFile.toPath(), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING) + } catch (AtomicMoveNotSupportedException ignored) { + Files.move(temporary, cacheFile.toPath(), StandardCopyOption.REPLACE_EXISTING) + } + } finally { + Files.deleteIfExists(temporary) + } + Files.writeString(new File(directory, 'complete').toPath(), 'complete\n') + log.call("Advisory scan complete: ${completedBatches} batches, ${pages} pages, ${hits} revalidated cache hits, ${findings.size()} active GHSAs.") + return [findings: findings, batches: chunks.size(), pages: pages, hits: hits] + } catch (Exception e) { + clearCompletion(directory) + // Do not echo gh output: it may contain credentials or other sensitive transport details. + String detail = e instanceof GradleException ? e.message : + "${e.class.simpleName} while fetching, validating or saving advisory responses; check API availability and cache directory permissions." + throw new GradleException("Dependency scan incomplete after ${completedBatches}/${chunks.size()} batches, ${pages} pages, ${hits} cache hits: ${detail}") + } + } + + private static Object canonical(def value) { + if (value instanceof Map) { + def sorted = new TreeMap() + value.each { key, item -> sorted[key] = canonical(item) } + return sorted + } + if (value instanceof List) { + return value.collect { canonical(it) } + } + value + } + + private static Map loadCache(File file, String identity) { + if (!file.isFile()) { + return [:] + } + try { + def value = new JsonSlurper().parse(file, 'UTF-8') + if (value instanceof Map && value.policy == POLICY && value.identity == identity && value.pages instanceof Map) { + return value.pages + } + } catch (Exception ignored) { + // A corrupt cache is only a miss. A fresh, complete live scan is still required. + } + [:] + } + + private static boolean validCandidate(def candidate, List coordinates, String batchUrl) { + try { + if (!(candidate instanceof Map) || candidate.coordinates != coordinates || + !(candidate.etag instanceof String) || !(candidate.etag ==~ /(?:W\/)?"[^"\r\n]*"/) || + !candidate.containsKey('next')) { + return false + } + validateBody(candidate.body) + if (candidate.next != null) { + validateUrl(candidate.next, batchUrl) + } + return true + } catch (Exception ignored) { + return false + } + } + + private static Map parseResponse(def result) { + if (!(result instanceof Map) || !(result.exitCode instanceof Number) || + !(result.stdout instanceof String) || !(result.stderr instanceof String)) { + throw new GradleException('Invalid gh transport response.') + } + String text = result.stdout.replace('\r\n', '\n') + int separator = text.indexOf('\n\n') + if (separator < 0) { + throw new GradleException("Missing HTTP headers from gh (exit ${result.exitCode}); check authentication and connectivity.") + } + def lines = text.substring(0, separator).split('\n') + def status = lines[0] =~ /^HTTP\/\d+(?:\.\d+)?\s+(\d{3})(?:\s+.*)?$/ + if (!status.matches()) { + throw new GradleException('Malformed HTTP status from gh.') + } + def headers = [:] + lines.drop(1).each { line -> + int colon = line.indexOf(':') + if (colon <= 0) { + throw new GradleException('Malformed HTTP header from gh.') + } + String name = line.substring(0, colon).toLowerCase(Locale.ROOT) + String value = line.substring(colon + 1).trim() + if (headers.containsKey(name)) { + if (name == 'etag') { + throw new GradleException('Duplicate ETag header from gh.') + } + if (name == 'link') { + value = headers[name] + ', ' + value + } + } + headers[name] = value + } + [status: Integer.parseInt(status[0][1]), headers: headers, + body: text.substring(separator + 2), exitCode: result.exitCode, stderr: result.stderr] + } + + private static String nextLink(String link) { + if (link == null) { + return null + } + String next = null + // Commas within angle brackets belong to URLs, not to Link separators. + def links = link.split(/,(?=\s*<)/) + links.each { part -> + def match = part.trim() =~ /^<([^<>]+)>;\s*rel="([a-z ]+)"$/ + if (!match.matches()) { + throw new GradleException('Malformed GitHub advisory pagination Link header.') + } + if (match[0][2].split(' ').contains('next')) { + if (next != null) { + throw new GradleException('Multiple next pages in GitHub advisory response.') + } + next = match[0][1] + } + } + next + } + + private static void validateUrl(String url, String batchUrl) { + def uri = new URI(url) + if (uri.scheme != 'https' || uri.rawAuthority != 'api.github.com' || + uri.rawPath != '/advisories' || uri.rawFragment != null || uri.rawQuery == null) { + throw new GradleException('Unsafe GitHub advisory pagination URL; expected https://api.github.com/advisories.') + } + def parameters = queryParameters(uri) + def expected = queryParameters(new URI(batchUrl)) + if (expected.any { key, value -> parameters[key] != value } || + parameters.keySet().any { !expected.containsKey(it) && !(it in ['after', 'before']) } || + (parameters.containsKey('after') && parameters.containsKey('before')) || + ['after', 'before'].any { parameters.containsKey(it) && parameters[it].isBlank() }) { + throw new GradleException('GitHub advisory pagination changed the dependency query or filters.') + } + } + + private static Map queryParameters(URI uri) { + def parameters = [:] + uri.rawQuery.split('&').each { part -> + def pair = part.split('=', 2) + if (pair.length != 2) { + throw new GradleException('Malformed advisory pagination query.') + } + String name = URLDecoder.decode(pair[0], 'UTF-8') + if (parameters.containsKey(name)) { + throw new GradleException('Duplicate advisory pagination query parameter.') + } + parameters[name] = URLDecoder.decode(pair[1], 'UTF-8') + } + parameters + } + + private static void validateBody(def body) { + if (!(body instanceof List) || body.size() > POLICY.perPage) { + throw new GradleException('GitHub advisory response must be an array of at most 100 entries.') + } + body.each { advisory -> + if (!(advisory instanceof Map) || !(advisory.ghsa_id instanceof String) || + !(advisory.ghsa_id ==~ /GHSA-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}/) || + !(advisory.summary instanceof String) || advisory.summary.isBlank() || + !(advisory.severity in ['unknown', 'low', 'medium', 'high', 'critical']) || + advisory.html_url != "https://github.com/advisories/${advisory.ghsa_id}".toString() || + !advisory.containsKey('withdrawn_at') || + !(advisory.vulnerabilities instanceof List) || advisory.vulnerabilities.isEmpty()) { + throw new GradleException('Malformed or incomplete GitHub advisory schema.') + } + if (advisory.withdrawn_at != null) { + if (!(advisory.withdrawn_at instanceof String)) { + throw new GradleException('Malformed advisory withdrawal timestamp.') + } + OffsetDateTime.parse(advisory.withdrawn_at) + } + advisory.vulnerabilities.each { vulnerability -> + if (!(vulnerability instanceof Map) || !(vulnerability.package instanceof Map) || + !(vulnerability.package.ecosystem instanceof String) || vulnerability.package.ecosystem.isBlank() || + !(vulnerability.package.name instanceof String) || vulnerability.package.name.isBlank()) { + throw new GradleException('Malformed or incomplete GitHub advisory vulnerability schema.') + } + } + } + } + + static Map request(String url, String etag) { + def command = ['gh', 'api', '--hostname', 'github.com', '--method', 'GET', '--include', + '-H', 'Accept: application/vnd.github+json', '-H', "X-GitHub-Api-Version: ${API_VERSION}".toString()] + if (etag != null) { + command.addAll(['-H', "If-None-Match: ${etag}".toString()]) + } + command.add(url) + def stdout = new StringBuilder() + def stderr = new StringBuilder() + Process process + try { + process = new ProcessBuilder(command).start() + } catch (IOException ignored) { + throw new GradleException('Cannot start gh; install the GitHub CLI and authenticate it before scanning dependencies.') + } + try { + process.waitForProcessOutput(stdout, stderr) + [exitCode: process.exitValue(), stdout: stdout.toString(), stderr: stderr.toString()] + } finally { + process.destroy() + } + } +} diff --git a/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzlePlugin.groovy b/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzlePlugin.groovy index fecb0a94..09807ab0 100644 --- a/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzlePlugin.groovy +++ b/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzlePlugin.groovy @@ -32,13 +32,30 @@ class MuzzlePlugin implements Plugin { @Override void apply(Project project) { // Register the DSL extension - project.extensions.create('muzzle', MuzzleExtension) + def extension = project.extensions.create('muzzle', MuzzleExtension) - // Register the muzzle task (not part of check/build — run explicitly). - // The task depends on 'classes' which already includes generateMuzzle - // for instrumentation subprojects (wired in the parent build.gradle). - project.tasks.register('muzzle', MuzzleTask) { - dependsOn 'classes' + // Instrumentation projects wire this into check. Their classes task already + // includes generateMuzzle, so generated references exist before input snapshots. + def muzzle = project.tasks.register('muzzle', MuzzleTask) + project.afterEvaluate { + // The plugin is also applied to bootstrap and other non-instrumentation + // projects. Do not infer unrelated producers (or a bootstrap self-cycle). + if (!extension.directives.isEmpty()) { + muzzle.configure { task -> + task.dependsOn(project.tasks.named('classes'), + ':braintrust-java-agent:bootstrap:classes') + task.instrumentationClasspath.from(project.sourceSets.main.output, + project.configurations.compileClasspath) + task.bootstrapClasspath.from(project.provider { + def bootstrap = project.project(':braintrust-java-agent:bootstrap') + def agent = project.project(':braintrust-java-agent') + // Raw files deliberately avoid attaching root-project task + // dependencies through its configuration/source-set model. + bootstrap.sourceSets.main.output.files.toList() + + agent.configurations.bootstrapLibs.resolve().toList() + }) + } + } } } } diff --git a/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleTask.groovy b/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleTask.groovy index 6dc68a0e..19c4c6ac 100644 --- a/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleTask.groovy +++ b/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleTask.groovy @@ -3,7 +3,12 @@ package dev.braintrust.gradle.muzzle import org.gradle.api.DefaultTask import org.gradle.api.GradleException import org.gradle.api.artifacts.Configuration -import org.gradle.api.tasks.InputFiles +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.tasks.CacheableTask +import org.gradle.api.tasks.Classpath +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.OutputFile import org.gradle.api.tasks.TaskAction /** @@ -13,89 +18,153 @@ import org.gradle.api.tasks.TaskAction * creates an isolated classloader for each version, and verifies that the instrumentation's * muzzle references (and helper classes) either match or don't match as expected. */ -class MuzzleTask extends DefaultTask { +@CacheableTask +abstract class MuzzleTask extends DefaultTask { + + @Classpath + abstract ConfigurableFileCollection getInstrumentationClasspath() + + @Classpath + abstract ConfigurableFileCollection getBootstrapClasspath() + + @OutputFile + abstract RegularFileProperty getReportFile() + + private List resolvedTargets + + @Input + List getDirectiveInputs() { + directives().collect { directive -> + [ + group: directive.group, + module: directive.module, + versions: directive.versions ?: '', + assertPass: directive.assertPass, + pinnedVersions: directive.pinnedVersions.toList(), + skipVersions: directive.skipVersions.toList().sort(), + additionalDependencies: directive.additionalDependencies.toList(), + excludedDependencies: directive.excludedDependencies.toList(), + ignoredInstrumentation: directive.ignoredInstrumentation.toList() + ] + } + } + + @Input + Map getCheckerRuntime() { + // The checker runs inside the Gradle JVM, not the Java compilation toolchain. + ['java.version', 'java.runtime.version', 'java.vendor', 'java.vm.name', + 'java.vm.version', 'java.vm.vendor', 'os.name', 'os.arch'].collectEntries { + [(it): System.getProperty(it, '')] + } + } + + @Input + List getTargetInputs() { + targetPlan().collect { target -> + [directive: target.directive, coordinate: target.coordinate, artifacts: target.artifacts] + } + } + + @Classpath + List getTargetClasspath() { + // Keep the file order as loaded. targetInputs also records per-target boundaries, + // coordinates and artifact order, which a flattened classpath alone cannot express. + targetPlan().collectMany { it.files } + } + + private List directives() { + project.extensions.getByType(MuzzleExtension).directives + } + + private List targetPlan() { + if (resolvedTargets != null) { + return resolvedTargets + } + List targets = [] + def configuredDirectives = directives() + for (int index = 0; index < configuredDirectives.size(); index++) { + MuzzleDirective directive = configuredDirectives[index] + List versions + try { + versions = directive.pinnedVersions + ? directive.pinnedVersions.toList() : discoverVersions(directive) + } catch (Exception e) { + throw new GradleException("[muzzle] Failed to resolve versions for ${directive}: ${e.message}", e) + } + if (versions.isEmpty()) { + logger.warn("[muzzle] No versions found for ${directive.group}:${directive.module} in range ${directive.versions}") + } + for (String version : versions) { + String coordinate = "${directive.group}:${directive.module}:${version}" + try { + def library = resolveLibraryVersion(directive, version) + targets.add([directive: index, coordinate: coordinate, version: version, + files: library.files, artifacts: library.artifacts]) + } catch (Exception e) { + throw new GradleException("[muzzle] Failed to resolve ${coordinate}; refusing an incomplete check", e) + } + } + } + // Only memoize within this task instance. In particular, metadata discovery must + // run again on the next invocation, even with an otherwise warm build cache. + resolvedTargets = targets + return resolvedTargets + } + + protected List discoverVersions(MuzzleDirective directive) { + MavenVersions.resolve(directive.group, directive.module, directive.versions, directive.skipVersions) + } MuzzleTask() { group = 'verification' description = 'Checks instrumentation muzzle references against library versions' + reportFile.convention(project.layout.buildDirectory.file('reports/muzzle/result.txt')) + // Discovery uses live project configurations and must not be replayed from a + // configuration-cache snapshot. Build-cache result reuse is independent of this. + notCompatibleWithConfigurationCache('Muzzle discovers current published targets before build-cache lookup') } @TaskAction void run() { - // Create a bootstrap classloader for this task run. - def bootstrapUrls = collectBootstrapClasspath() - def bootstrapCL = new URLClassLoader(bootstrapUrls as URL[], (ClassLoader) null) - - // Initialize the instrumentation classloader once per task run — it doesn't - // change between library versions. - def instrumentationClasspath = collectInstrumentationClasspath() - def instrumentationUrls = instrumentationClasspath.collect { it.toURI().toURL() } as URL[] + // Resolve once before checking, using exactly the inputs Gradle fingerprinted. + def targets = targetPlan() + def report = reportFile.get().asFile + java.nio.file.Files.deleteIfExists(report.toPath()) + def bootstrapUrls = bootstrapClasspath.files.collect { it.toURI().toURL() } as URL[] + def bootstrapCL = new URLClassLoader(bootstrapUrls, (ClassLoader) null) + def instrumentationUrls = instrumentationClasspath.files.collect { it.toURI().toURL() } as URL[] def instrumentationCL = new URLClassLoader(instrumentationUrls, ClassLoader.systemClassLoader) try { - def extension = project.extensions.findByType(MuzzleExtension) - if (!extension || extension.directives.isEmpty()) { - logger.lifecycle('[muzzle] No muzzle directives configured — skipping') - return - } + def configuredDirectives = directives() int totalVersions = 0 int totalFailures = 0 List failureMessages = [] - for (MuzzleDirective directive : extension.directives) { - logger.lifecycle("[muzzle] Checking: ${directive}") - - List versions - if (directive.pinnedVersions) { - versions = directive.pinnedVersions - logger.lifecycle("[muzzle] Using ${versions.size()} pinned version(s)") + for (def target : targets) { + MuzzleDirective directive = configuredDirectives[target.directive] + String version = target.version + logger.lifecycle("[muzzle] Checking: ${directive.group}:${directive.module}:${version}") + totalVersions++ + def result = checkVersion(target.files, directive, version, bootstrapCL, instrumentationCL, directive.ignoredInstrumentation as Set) + + if (result.passed && directive.assertPass) { + logger.lifecycle("[muzzle] ${version} PASS") + } else if (!result.passed && !directive.assertPass) { + logger.lifecycle("[muzzle] ${version} PASS (expected failure, correctly failed)") + } else if (!result.passed && directive.assertPass) { + totalFailures++ + def msg = "[muzzle] ${version} FAIL — expected to pass but got mismatches:" + logger.error(msg) + result.messages.each { logger.error("[muzzle] ${it}") } + failureMessages.add("${directive.group}:${directive.module}:${version} — ${result.messages.join('; ')}") } else { - try { - versions = MavenVersions.resolve( - directive.group, directive.module, directive.versions, directive.skipVersions) - } catch (Exception e) { - throw new GradleException("[muzzle] Failed to resolve versions for ${directive}: ${e.message}", e) - } - - if (versions.isEmpty()) { - logger.warn("[muzzle] No versions found for ${directive.group}:${directive.module} in range ${directive.versions}") - continue - } - } - - logger.lifecycle("[muzzle] Found ${versions.size()} version(s) to check") - - for (String version : versions) { - totalVersions++ - - List libraryJars - try { - libraryJars = resolveLibraryVersion(directive, version) - } catch (Exception e) { - logger.warn("[muzzle] Failed to resolve ${directive.group}:${directive.module}:${version}: ${e.message}") - continue - } - - def result = checkVersion(libraryJars, directive, version, bootstrapCL, instrumentationCL, directive.ignoredInstrumentation as Set) - - if (result.passed && directive.assertPass) { - logger.lifecycle("[muzzle] ${version} PASS") - } else if (!result.passed && !directive.assertPass) { - logger.lifecycle("[muzzle] ${version} PASS (expected failure, correctly failed)") - } else if (!result.passed && directive.assertPass) { - totalFailures++ - def msg = "[muzzle] ${version} FAIL — expected to pass but got mismatches:" - logger.error(msg) - result.messages.each { logger.error("[muzzle] ${it}") } - failureMessages.add("${directive.group}:${directive.module}:${version} — ${result.messages.join('; ')}") - } else { - // passed but assertPass=false - totalFailures++ - def msg = "[muzzle] ${version} FAIL — expected to fail but muzzle passed" - logger.error(msg) - failureMessages.add("${directive.group}:${directive.module}:${version} — unexpectedly passed") - } + // passed but assertPass=false + totalFailures++ + def msg = "[muzzle] ${version} FAIL — expected to fail but muzzle passed" + logger.error(msg) + failureMessages.add("${directive.group}:${directive.module}:${version} — unexpectedly passed") } } @@ -105,53 +174,24 @@ class MuzzleTask extends DefaultTask { throw new GradleException( "[muzzle] ${totalFailures} version(s) failed:\n " + failureMessages.join('\n ')) } + // No timestamps, absolute paths or partial results: only completed checks + // produce the output that Gradle may restore from its native build cache. + report.parentFile.mkdirs() + report.setText("Checked ${totalVersions} version(s), 0 failures\n" + + targets.collect { target -> + "${target.coordinate} ${configuredDirectives[target.directive].assertPass ? 'pass' : 'fail'}" + }.join('\n') + '\n', 'UTF-8') } finally { instrumentationCL?.close() bootstrapCL.close() } } - /** - * Collects the classpath needed to load InstrumentationModule classes from the current project. - * Includes the project's own compiled classes, instrumenter, and ByteBuddy. - */ - private List collectInstrumentationClasspath() { - def files = [] - - // The project's own compiled classes - project.sourceSets.main.output.classesDirs.each { files.add(it) } - - // Add compile classpath (includes instrumenter, bytebuddy, etc.) - project.configurations.compileClasspath.resolve().each { files.add(it) } - - return files - } - - /** - * Collects the bootstrap classpath: the bootstrap module's compiled classes + bootstrapLibs - * (OTel API and its transitive deps). This simulates what ends up on the real JVM bootstrap - * classpath when the agent runs. - */ - private List collectBootstrapClasspath() { - def agentProject = project.project(':braintrust-java-agent') - def urls = [] - - // Bootstrap module compiled classes - def bootstrapProject = project.project(':braintrust-java-agent:bootstrap') - bootstrapProject.sourceSets.main.output.classesDirs.each { urls.add(it.toURI().toURL()) } - - // bootstrapLibs configuration (OTel API + transitive deps like context, common) - agentProject.configurations.bootstrapLibs.resolve().each { urls.add(it.toURI().toURL()) } - - return urls - } - /** * Resolves a specific version of the library, returning the JAR files. * Uses Gradle's dependency resolution for transitive deps. */ - private List resolveLibraryVersion(MuzzleDirective directive, String version) { - def configName = "muzzleCheck_${directive.group}_${directive.module}_${version}".replace('.', '_').replace('-', '_') + private Map resolveLibraryVersion(MuzzleDirective directive, String version) { // Create a detached configuration for this specific version def deps = [] @@ -181,7 +221,14 @@ class MuzzleTask extends DefaultTask { config.transitive = true config.resolutionStrategy.failOnNonReproducibleResolution() - return config.resolve().toList() + def files = config.resolve().toList() + def artifactsByFile = config.resolvedConfiguration.resolvedArtifacts.collectEntries { artifact -> + [(artifact.file): artifact.id.displayName] + } + return [files: files, artifacts: files.collect { file -> + // Artifact identity is separate from @Classpath's content normalization. + [id: artifactsByFile[file], name: file.name] + }] } /** diff --git a/buildSrc/src/test/groovy/dev/braintrust/gradle/CheckDependenciesTaskTest.groovy b/buildSrc/src/test/groovy/dev/braintrust/gradle/CheckDependenciesTaskTest.groovy new file mode 100644 index 00000000..86f23e3e --- /dev/null +++ b/buildSrc/src/test/groovy/dev/braintrust/gradle/CheckDependenciesTaskTest.groovy @@ -0,0 +1,448 @@ +package dev.braintrust.gradle + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import org.gradle.api.GradleException +import org.gradle.testkit.runner.GradleRunner +import org.gradle.testkit.runner.TaskOutcome +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import static org.junit.jupiter.api.Assertions.* + +class CheckDependenciesTaskTest { + @TempDir + File directory + File cacheDirectory + List requests + List responses + static final List COORDINATES = ['example:library@1.0'] + + @BeforeEach + void fixture() { + cacheDirectory = new File(directory, 'cache') + requests = [] + responses = [] + } + + private static Map advisory(String id = 'GHSA-aaaa-bbbb-cccc', String withdrawn = null) { + [ghsa_id: id, cve_id: null, summary: 'Fixture vulnerability', severity: 'high', + html_url: "https://github.com/advisories/${id}".toString(), withdrawn_at: withdrawn, + vulnerabilities: [[package: [ecosystem: 'maven', name: 'example:library'], + vulnerable_version_range: '< 2.0']]] + } + + private static Map ok(List body = [], String etag = '"first"', String link = null) { + def headers = "HTTP/2.0 200 OK\r\nContent-Type: application/json\r\n" + if (etag != null) headers += "ETag: ${etag}\r\n" + if (link != null) headers += "Link: ${link}\r\n" + [exitCode: 0, stdout: headers + '\r\n' + JsonOutput.toJson(body), stderr: ''] + } + + private static Map unchanged() { + [exitCode: 1, stdout: 'HTTP/2.0 304 Not Modified\r\nETag: "first"\r\n\r\n', stderr: 'gh: HTTP 304\n'] + } + + private Map scan(List coordinates = COORDINATES, String identity = 'implementation-one') { + DependencyAdvisories.scan(coordinates, identity, cacheDirectory, { url, etag -> + requests.add([url: url, etag: etag]) + assertFalse(responses.isEmpty(), 'Unexpected API request') + def response = responses.remove(0) + if (response instanceof Exception) throw response + response + }, { message -> }) + } + + private File cache() { new File(cacheDirectory, 'cache.json') } + private File complete() { new File(cacheDirectory, 'complete') } + private String firstUrl(List coordinates = COORDINATES) { DependencyAdvisories.batches(coordinates)[0].url } + + @Test + void coldAndWarmCleanAlwaysRevalidateAndKeepStableSnapshot() { + responses.add(ok()) + def cold = scan() + assertEquals(0, cold.hits) + assertEquals([], cold.findings) + assertEquals([[url: firstUrl(), etag: null]], requests) + assertTrue(complete().isFile()) + def bytes = cache().bytes + responses.add(unchanged()) + def warm = scan() + assertEquals(1, warm.hits) + assertEquals([], warm.findings) + assertEquals('"first"', requests.last().etag) + assertArrayEquals(bytes, cache().bytes) + } + + @Test + void coldAndWarmFindingsRetainUniqueAdvisoriesAndRelevantPackageNames() { + responses.add(ok([advisory(), advisory()])) + def cold = scan(['example:library@1.0', 'example:library@3.0', 'other:unaffected@1.0']) + assertEquals(1, cold.findings.size()) + assertEquals(['example:library'], cold.findings[0].packages) + def snapshot = cache().bytes + responses.add(unchanged()) + def warm = scan(['example:library@1.0', 'example:library@3.0', 'other:unaffected@1.0']) + assertEquals(cold.findings, warm.findings) + assertEquals(1, warm.hits) + assertArrayEquals(snapshot, cache().bytes) + assertTrue(complete().exists()) + } + + @Test + void conflictingDuplicatePagesCannotHideAnActiveFinding() { + def second = firstUrl() + '&after=next%2Fpage%3D%3D' + responses.addAll([ + ok([advisory()], '"first"', "<${second}>; rel=\"next\""), + ok([advisory('GHSA-aaaa-bbbb-cccc', '2026-01-01T00:00:00Z')], '"second"') + ]) + assertEquals(['GHSA-aaaa-bbbb-cccc'], scan().findings.collect { it.advisory.ghsa_id }) + responses.addAll([unchanged(), unchanged()]) + assertEquals(['GHSA-aaaa-bbbb-cccc'], scan().findings.collect { it.advisory.ghsa_id }) + } + + @Test + void serverMatchedAdvisoryWithNullableRangeRemainsAFinding() { + def finding = advisory() + finding.vulnerabilities[0].vulnerable_version_range = null + responses.add(ok([finding])) + assertEquals(['GHSA-aaaa-bbbb-cccc'], scan().findings.collect { it.advisory.ghsa_id }) + responses.add(unchanged()) + assertEquals(['GHSA-aaaa-bbbb-cccc'], scan().findings.collect { it.advisory.ghsa_id }) + } + + @Test + void changedDependenciesAndImplementationRequireFreshResponses() { + responses.add(ok()) + scan() + responses.add(ok()) + def changed = ['example:library@2.0'] + scan(changed) + assertNull(requests.last().etag) + assertEquals(firstUrl(changed), requests.last().url) + responses.add(ok()) + scan(changed, 'implementation-two') + assertNull(requests.last().etag) + responses.add(unchanged()) + assertEquals(1, scan(changed, 'implementation-two').hits) + } + + @Test + void sourceAndPolicyEditsChangeIdentityAndPolicyMismatchIsNeverReused() { + def source = new File(directory, 'Scanner.groovy') + def policy = new File(directory, 'policy.json') + source.text = 'scanner-one' + policy.text = 'policy-one' + def initial = DependencyAdvisories.identity([source, policy]) + assertEquals(initial, DependencyAdvisories.identity([policy, source])) + responses.add(ok()) + scan(COORDINATES, initial) + source.text = 'scanner-two' + def changedSource = DependencyAdvisories.identity([source, policy]) + assertNotEquals(initial, changedSource) + responses.add(ok()) + scan(COORDINATES, changedSource) + assertNull(requests.last().etag) + policy.text = 'policy-two' + def changedPolicy = DependencyAdvisories.identity([source, policy]) + assertNotEquals(changedSource, changedPolicy) + responses.add(ok()) + scan(COORDINATES, changedPolicy) + assertNull(requests.last().etag) + def stored = new JsonSlurper().parse(cache()) + stored.policy.apiVersion = 'unsupported' + cache().text = JsonOutput.toJson(stored) + responses.add(ok()) + scan(COORDINATES, changedPolicy) + assertNull(requests.last().etag) + assertThrows(GradleException) { DependencyAdvisories.identity([new File(directory, 'missing')]) } + } + + @Test + void newAndWithdrawnAdvisoriesReplaceCachedBodiesOn200() { + responses.add(ok()) + assertTrue(scan().findings.isEmpty()) + responses.add(ok([advisory()], '"new"')) + assertEquals(1, scan().findings.size()) + assertEquals('"first"', requests.last().etag) + responses.add(ok([advisory('GHSA-aaaa-bbbb-cccc', '2026-01-01T00:00:00Z')], '"withdrawn"')) + assertTrue(scan().findings.isEmpty()) + assertEquals('"new"', requests.last().etag) + responses.add(unchanged()) + assertTrue(scan().findings.isEmpty()) + assertEquals('"withdrawn"', requests.last().etag) + } + + @Test + void unchangedFirstPageStillRevalidatesChangedLaterPage() { + def second = firstUrl() + '&after=next%2Fpage%3D%3D' + responses.addAll([ok([], '"first"', "<${second}>; rel=\"next\""), ok([], '"second"')]) + assertEquals(2, scan().pages) + requests.clear() + responses.addAll([unchanged(), ok([advisory()], '"changed-second"')]) + def warm = scan() + assertEquals(1, warm.findings.size()) + assertEquals(1, warm.hits) + assertEquals([[url: firstUrl(), etag: '"first"'], [url: second, etag: '"second"']], requests) + responses.addAll([unchanged(), unchanged()]) + assertEquals(2, scan().hits) + } + + @Test + void fullTerminalPageRefreshesLinkEvenWhenItsBodyIsUnchanged() { + def full = (0..<100).collect { advisory(String.format('GHSA-aaaa-bbbb-%04d', it)) } + responses.add(ok(full)) + assertEquals(100, scan().findings.size()) + def second = firstUrl() + '&after=next%2Fpage%3D%3D' + responses.addAll([ok(full, '"first"', "<${second}>; rel=\"next\""), ok([advisory()], '"second"')]) + requests.clear() + def result = scan() + assertNull(requests[0].etag, 'A full terminal page must refresh pagination metadata unconditionally') + assertEquals(second, requests[1].url) + assertEquals(101, result.findings.size()) + responses.addAll([unchanged(), unchanged()]) + assertEquals(2, scan().hits) + } + + @Test + void paginationMetadataProvidedOn304ReplacesCachedLinks() { + def second = firstUrl() + '&after=next%2Fpage%3D%3D' + responses.addAll([ok([], '"first"', "<${second}>; rel=\"next\""), ok([], '"second"')]) + scan() + requests.clear() + responses.add(unchanged() + [ + stdout: "HTTP/2.0 304 Not Modified\nLink: <${firstUrl()}>; rel=\"first\"\n\n".toString()]) + assertEquals(1, scan().pages) + assertEquals(1, requests.size()) + responses.add(unchanged() + [ + stdout: 'HTTP/2.0 304 Not Modified\nLink: ; rel="next"\n\n']) + assertThrows(GradleException) { scan() } + assertFalse(complete().exists()) + } + + @Test + void unrelatedProcessFailureCannotMasqueradeAs304WithValidCandidate() { + [unchanged() + [stderr: 'connection error'], unchanged() + [exitCode: 2]].each { response -> + responses.add(ok()) + scan() + def snapshot = cache().bytes + responses.add(response) + assertThrows(GradleException) { scan() } + assertEquals('"first"', requests.last().etag) + assertFalse(complete().exists()) + assertArrayEquals(snapshot, cache().bytes) + } + } + + @Test + void failuresRemoveCompletionAndNeverPersistPartialScans() { + def failures = [ + new IOException('transport unavailable'), + [exitCode: 1, stdout: '', stderr: 'secret token must not be logged'], + [exitCode: 1, stdout: 'HTTP/2.0 403 Forbidden\n\n{}', stderr: 'gh: forbidden'], + [exitCode: 1, stdout: 'HTTP/2.0 200 OK\n\n[]', stderr: 'other failure'], + [exitCode: 0, stdout: 'HTTP/2.0 200 OK\n\nnot json', stderr: ''], + [exitCode: 0, stdout: 'HTTP/2.0 200 OK\n\n{}', stderr: ''], + ok([[ghsa_id: 'GHSA-aaaa-bbbb-cccc']]), + ok([advisory() + [withdrawn_at: false]]), + ok([advisory() + [withdrawn_at: 'yesterday']]), + ok([advisory() + [vulnerabilities: []]]), + unchanged() + [stderr: 'gh: HTTP 304\nconnection failed'], + unchanged() + [exitCode: 2], + unchanged() + [stdout: 'HTTP/2.0 304 Not Modified\n\n[]'] + ] + def second = firstUrl() + '&after=next%2Fpage%3D%3D' + failures.each { failure -> + responses.add(ok()) + scan() + def snapshot = cache().bytes + responses.addAll([ok([], '"changed"', "<${second}>; rel=\"next\""), failure]) + def exception = assertThrows(GradleException) { scan() } + assertTrue(exception.message.contains('Dependency scan incomplete'), exception.message) + assertFalse(exception.message.contains('secret token')) + assertFalse(complete().exists()) + assertArrayEquals(snapshot, cache().bytes) + } + } + + @Test + void unexpected304AndCorruptCandidatesCannotBecomeCleanResults() { + responses.add(unchanged()) + assertThrows(GradleException) { scan() } + assertFalse(cache().exists()) + assertFalse(complete().exists()) + responses.add(ok()) + scan() + def saved = cache().text + def corruptions = [ + { map -> map.pages[firstUrl()].body = [:] }, + { map -> map.pages[firstUrl()].coordinates = ['unrelated:package@1.0'] }, + { map -> map.pages[firstUrl()].next = 'https://evil.example/advisories' }, + { map -> map.pages[firstUrl()].etag = '"bad\r\nAuthorization: secret"' } + ] + corruptions.each { corrupt -> + def stored = new JsonSlurper().parseText(saved) + corrupt(stored) + cache().text = JsonOutput.toJson(stored) + responses.add(unchanged()) + assertThrows(GradleException) { scan() } + assertNull(requests.last().etag) + assertFalse(complete().exists()) + } + cache().text = 'not json' + responses.add(ok()) + assertTrue(scan().findings.isEmpty()) + assertNull(requests.last().etag) + } + + @Test + void unsafeMalformedAndLoopingPaginationFailsBeforeForwardingCredentials() { + def badUrls = [ + 'https://evil.example/advisories', 'http://api.github.com/advisories', + 'https://user@api.github.com/advisories', 'https://api.github.com:444/advisories', + 'https://api.github.com/other', firstUrl() + '#fragment', + firstUrl().replace('type=reviewed', 'type=unreviewed'), + firstUrl().replace('example', 'unrelated'), firstUrl() + '&after=', + firstUrl() + '&after=next&before=previous', + firstUrl() + '&affects=another%3Apackage%401' + ] + badUrls.each { url -> + requests.clear() + responses.add(ok([], '"first"', "<${url}>; rel=\"next\"")) + assertThrows(GradleException) { scan() } + assertEquals(1, requests.size()) + assertFalse(complete().exists()) + } + ['not a link', "<${firstUrl()}>; rel=\"next\", <${firstUrl()}>; rel=\"next\""].each { link -> + responses.add(ok([], '"first"', link)) + assertThrows(GradleException) { scan() } + } + requests.clear() + responses.add(ok([], '"first"', "<${firstUrl()}>; rel=\"next\"")) + assertThrows(GradleException) { scan() } + assertEquals(1, requests.size()) + assertFalse(cache().exists()) + } + + @Test + void absentEtagsAlwaysRequireFresh200AndEmptyGraphsRemoveCompletion() { + responses.add(ok([], null)) + scan() + responses.add(ok([], null)) + scan() + assertNull(requests.last().etag) + assertThrows(GradleException) { scan([]) } + assertFalse(complete().exists()) + assertThrows(GradleException) { scan(['unresolved']) } + } + + @Test + void cacheWriteFailureCannotPublishCompletion() { + assertTrue(cache().mkdirs()) + complete().text = 'previous complete\n' + responses.add(ok()) + assertThrows(GradleException) { scan() } + assertFalse(complete().exists()) + } + + @Test + void failedLaterBatchCannotSaveEarlierSuccessfulBatch() { + def coordinates = (0..<300).collect { "example:long-library-name-${it}@1.0".toString() } + assertTrue(DependencyAdvisories.batches(coordinates).size() > 1) + responses.addAll([ok(), new IOException('second batch failed')]) + assertThrows(GradleException) { scan(coordinates) } + assertEquals(2, requests.size()) + assertFalse(cache().exists()) + assertFalse(complete().exists()) + } + + @Test + void batchingHonorsEncodedLengthCountAndSortedUniqueCoordinates() { + def coordinates = (0..<2200).collect { "example:library-${it}@1.0+metadata".toString() } + def chunks = DependencyAdvisories.batches(coordinates.reverse() + coordinates[0]) + assertTrue(chunks.size() > 1) + assertEquals(new TreeSet(coordinates).toList(), chunks.collectMany { it.coordinates }) + chunks.eachWithIndex { chunk, index -> + assertTrue(chunk.url.length() <= DependencyAdvisories.MAX_URL_LENGTH) + assertTrue(chunk.coordinates.size() <= DependencyAdvisories.MAX_PACKAGES) + assertTrue(chunk.url.contains('%2Bmetadata')) + def affects = chunk.url.substring(chunk.url.indexOf('&affects=') + '&affects='.length()) + assertEquals(chunk.coordinates.join(','), URLDecoder.decode(affects, 'UTF-8')) + if (index + 1 < chunks.size()) { + def expanded = chunk.coordinates + chunks[index + 1].coordinates[0] + assertTrue(DependencyAdvisories.batches(expanded).size() > 1) + } + } + def prefixLength = firstUrl(['a:b@c']).length() - URLEncoder.encode('a:b@c', 'UTF-8').length() + def coordinate = 'a:b@' + 'v' * (DependencyAdvisories.MAX_URL_LENGTH - prefixLength - URLEncoder.encode('a:b@', 'UTF-8').length()) + assertEquals(DependencyAdvisories.MAX_URL_LENGTH, firstUrl([coordinate]).length()) + assertThrows(GradleException) { DependencyAdvisories.batches([coordinate + 'v']) } + responses.addAll(chunks.collect { ok() }) + def result = scan(coordinates) + assertEquals(chunks.size(), result.batches) + assertEquals(chunks.size(), requests.size()) + requests.clear() + responses.addAll(chunks.collect { unchanged() }) + assertEquals(chunks.size(), scan(coordinates).hits) + } + + @Test + void realGradleTaskAlwaysRunsCachesFindingsAndFailsClosedOnEmptyGraph() { + def project = new File(directory, 'project') + project.mkdirs() + new File(project, 'settings.gradle').text = "rootProject.name = 'advisory-fixture'\n" + new File(project, 'gradle.properties').text = 'org.gradle.workers.max=2\n' + def classpath = System.getProperty('advisories.test.classpath').split(File.pathSeparator) + .findAll { new File(it).exists() } + .collect { "'${it.replace('\\', '\\\\').replace("'", "\\'")}'" }.join(',') + new File(project, 'build.gradle').text = "buildscript { dependencies { classpath files(${classpath}) } }\n" + ''' +import dev.braintrust.gradle.CheckDependenciesTask +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +abstract class FixtureCheck extends CheckDependenciesTask { + @Override + protected Map requestAdvisories(String url, String etag) { + project.file('requests.jsonl').append(JsonOutput.toJson([url: url, etag: etag]) + '\\n', 'UTF-8') + new JsonSlurper().parse(project.file('response.json')) as Map + } +} +tasks.register('checkDependencies', FixtureCheck) { + shippedDependencies.from(files('manifest.txt')) + scannerSourceFiles.setFrom(files('scanner-input.txt')) +} +''' + new File(project, 'scanner-input.txt').text = 'fixture implementation and policy' + def manifest = new File(project, 'manifest.txt') + manifest.text = COORDINATES.join('\n') + '\n' + def response = new File(project, 'response.json') + def runner = { + GradleRunner.create().withProjectDir(project).withTestKitDir(new File(directory, 'testkit')) + .withArguments('checkDependencies', '--build-cache', '--no-configuration-cache', '--stacktrace') + } + def taskCache = new File(project, 'build/dependency-advisories/cache.json') + def marker = new File(project, 'build/dependency-advisories/complete') + response.text = JsonOutput.toJson(ok([advisory()])) + def cold = runner().buildAndFail() + assertTrue(cold.output.contains('example:library'), cold.output) + assertTrue(cold.output.contains('GHSA-aaaa-bbbb-cccc'), cold.output) + assertTrue(marker.exists()) + assertTrue(taskCache.exists()) + def snapshot = taskCache.bytes + response.text = JsonOutput.toJson(unchanged()) + def warm = runner().buildAndFail() + assertTrue(warm.output.contains('GHSA-aaaa-bbbb-cccc'), warm.output) + assertTrue(marker.exists()) + assertArrayEquals(snapshot, taskCache.bytes) + response.text = JsonOutput.toJson(ok([advisory('GHSA-aaaa-bbbb-cccc', '2026-01-01T00:00:00Z')])) + assertEquals(TaskOutcome.SUCCESS, runner().build().task(':checkDependencies').outcome) + response.text = JsonOutput.toJson(unchanged()) + assertEquals(TaskOutcome.SUCCESS, runner().build().task(':checkDependencies').outcome) + def calls = new File(project, 'requests.jsonl').readLines().collect { new JsonSlurper().parseText(it) } + assertEquals(4, calls.size()) + assertEquals([null, '"first"', '"first"', '"first"'], calls.collect { it.etag }) + manifest.text = '' + runner().buildAndFail() + assertFalse(marker.exists()) + } +} diff --git a/buildSrc/src/test/groovy/dev/braintrust/gradle/CodeqlCacheKeyTest.groovy b/buildSrc/src/test/groovy/dev/braintrust/gradle/CodeqlCacheKeyTest.groovy new file mode 100644 index 00000000..a9c82f02 --- /dev/null +++ b/buildSrc/src/test/groovy/dev/braintrust/gradle/CodeqlCacheKeyTest.groovy @@ -0,0 +1,199 @@ +package dev.braintrust.gradle + +import groovy.json.JsonOutput +import org.gradle.testkit.runner.GradleRunner +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import java.util.jar.JarEntry +import java.util.jar.JarOutputStream + +import static org.junit.jupiter.api.Assertions.* + +/** Real key generation and dependency resolution, using only local fixture inputs. */ +class CodeqlCacheKeyTest { + @TempDir + File directory + File projectDir + File bundle + + private static void put(File root, String path, String text) { + def file = new File(root, path) + file.parentFile.mkdirs() + file.setText(text, 'UTF-8') + } + + private String git(String... arguments) { + CodeqlCacheKeyTask.command(projectDir, ['git'] + arguments.toList()).trim() + } + + private void artifact(String content) { + def base = 'repo/fixture/external/1.0/external-1.0' + put(projectDir, "${base}.pom", '''4.0.0 +fixtureexternal1.0''') + new JarOutputStream(new FileOutputStream(new File(projectDir, "${base}.jar"))).withCloseable { jar -> + def entry = new JarEntry('input.txt') + entry.time = 0 + jar.putNextEntry(entry) + jar.write(content.getBytes('UTF-8')) + jar.closeEntry() + } + } + + @BeforeEach + void fixture() { + projectDir = new File(directory, 'project') + bundle = new File(directory, 'bundle') + projectDir.mkdirs() + bundle.mkdirs() + put(projectDir, 'settings.gradle', "rootProject.name = 'codeql-fixture'\ninclude ':lib'\n") + put(projectDir, 'gradle.properties', 'org.gradle.workers.max=2\n') + put(projectDir, '.gitignore', '.gradle/\nbuild/\nrepo/\ntestkit/\nspec.yaml\n') + put(projectDir, 'spec.yaml', 'openapi: 3.0.0\n') + def classpath = System.getProperty('codeql.test.classpath').split(File.pathSeparator) + .findAll { new File(it).exists() } + .collect { "'${it.replace('\\', '\\\\').replace("'", "\\'")}'" }.join(',') + put(projectDir, 'build.gradle', "buildscript { dependencies { classpath files(${classpath}) } }\n" + ''' +import dev.braintrust.gradle.CodeqlCacheKeyTask +import dev.braintrust.gradle.CodeqlProjectInputsTask +allprojects { + apply plugin: 'java' + version = 'git rev-parse HEAD'.execute(null, rootDir).text.trim() + repositories { maven { url = rootProject.uri('repo') } } + tasks.withType(JavaCompile).configureEach { + doFirst { throw new GradleException('Application compilation must not run during lookup') } + } +} +dependencies { + implementation 'fixture:external:1.0' + implementation project(':lib') + compileOnly files(layout.buildDirectory.dir('generated/classes')) +} +def resetKey = tasks.register('resetKey', Delete) { + delete layout.buildDirectory.file('codeql/cache-key.txt') +} +def collectors = allprojects.collect { owner -> + owner.tasks.register('codeqlProjectInputs', CodeqlProjectInputsTask) { + dependsOn resetKey + outputFile = owner.layout.buildDirectory.file('codeql/project-inputs.json') + } +} +tasks.register('codeqlCacheKey', CodeqlCacheKeyTask) { + projectInputs.from(collectors) + openApiSpec = layout.projectDirectory.file('spec.yaml') + specSnapshot = layout.buildDirectory.file('codeql/spec-input/openapi/spec.yaml') + keyFile = layout.buildDirectory.file('codeql/cache-key.txt') + manifestFile = layout.buildDirectory.file('codeql/cache-inputs.json') +} +''') + // These sources are intentionally not compilable. Resolution must not request project JARs. + put(projectDir, 'src/main/java/Broken.java', 'not compilable\n') + put(projectDir, 'lib/src/main/java/Broken.java', 'also not compilable\n') + artifact('first') + put(bundle, 'java-queries/qlpack.yml', 'name: codeql/java-queries\nversion: 1.0.0\n') + put(bundle, 'java-queries/query.ql', 'select 1\n') + put(bundle, 'java-all/qlpack.yml', 'name: codeql/java-all\nversion: 1.0.0\n') + put(bundle, 'version.json', JsonOutput.toJson([version: '1.0.0', sha: 'fixture-cli-sha', + unpackedLocation: bundle.absolutePath, configFileFound: false])) + put(bundle, 'packs.json', JsonOutput.toJson([ + 'codeql/java-queries': [new File(bundle, 'java-queries').absolutePath], + 'codeql/java-all': [new File(bundle, 'java-all').absolutePath]])) + put(bundle, 'codeql', '''#!/bin/sh +set -eu +bundle=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +case "$1" in + version) cat "$bundle/version.json" ;; + resolve) cat "$bundle/packs.json" ;; + *) exit 2 ;; +esac +''') + assertTrue(new File(bundle, 'codeql').setExecutable(true)) + git('init') + git('config', 'user.email', 'fixture@example.invalid') + git('config', 'user.name', 'Fixture') + git('add', '.') + git('commit', '-m', 'Initial content') + } + + private GradleRunner runner() { + def environment = new HashMap(System.getenv()) + environment.CODEQL_PATH = new File(bundle, 'codeql').absolutePath + GradleRunner.create().withProjectDir(projectDir) + .withTestKitDir(new File(projectDir, 'testkit')) + .withEnvironment(environment) + .withArguments('codeqlCacheKey', '--refresh-dependencies', '--no-configuration-cache', + '--no-build-cache', '--rerun-tasks', '--stacktrace') + } + + private String key() { + runner().build() + def value = new File(projectDir, 'build/codeql/cache-key.txt').getText('UTF-8') + assertTrue(value ==~ /[0-9a-f]{64}\n/) + value + } + + @Test + void historyOnlyCommitsAndInstallLocationsDoNotChangeKey() { + def before = key() + def commit = git('rev-parse', 'HEAD') + git('commit', '--allow-empty', '-m', 'Different history, same source tree') + assertNotEquals(commit, git('rev-parse', 'HEAD')) + assertEquals(before, key()) + put(projectDir, 'build/generated/classes/Compiled.class', 'generated output') + assertEquals(before, key()) + def moved = new File(directory, 'relocated-bundle') + assertTrue(bundle.renameTo(moved)) + bundle = moved + put(bundle, 'version.json', JsonOutput.toJson([version: '1.0.0', sha: 'fixture-cli-sha', + unpackedLocation: bundle.absolutePath, configFileFound: false])) + put(bundle, 'packs.json', JsonOutput.toJson([ + 'codeql/java-queries': [new File(bundle, 'java-queries').absolutePath], + 'codeql/java-all': [new File(bundle, 'java-all').absolutePath]])) + assertEquals(before, key()) + } + + @Test + void trackedAndUntrackedContentInvalidateKey() { + def initial = key() + put(projectDir, 'src/main/java/Broken.java', 'different source\n') + def edited = key() + assertNotEquals(initial, edited) + put(projectDir, 'src/main/java/New.java', 'new source\n') + assertNotEquals(edited, key()) + } + + @Test + void artifactSpecAndQueryContentInvalidateWithoutCoordinateChanges() { + def initial = key() + artifact('replacement at identical coordinates') + def changedArtifact = key() + assertNotEquals(initial, changedArtifact) + put(projectDir, 'spec.yaml', 'openapi: 3.1.0\n') + def changedSpec = key() + assertNotEquals(changedArtifact, changedSpec) + put(bundle, 'java-queries/query.ql', 'select 2\n') + def changedQuery = key() + assertNotEquals(changedSpec, changedQuery) + put(bundle, 'version.json', JsonOutput.toJson([version: '1.0.1', sha: 'updated-cli-sha', + unpackedLocation: bundle.absolutePath, configFileFound: false])) + assertNotEquals(changedQuery, key()) + } + + @Test + void unresolvedInputsRemovePreviousKey() { + key() + new File(projectDir, 'build.gradle').append("\ndependencies { implementation 'fixture:missing:1.0' }\n", 'UTF-8') + def result = runner().buildAndFail() + assertTrue(result.output.contains('fixture:missing:1.0'), result.output) + assertFalse(new File(projectDir, 'build/codeql/cache-key.txt').exists()) + } + + @Test + void dynamicVersionsFailBeforeLookup() { + key() + new File(projectDir, 'build.gradle').append("\ndependencies { implementation 'fixture:external:+' }\n", 'UTF-8') + runner().buildAndFail() + assertFalse(new File(projectDir, 'build/codeql/cache-key.txt').exists()) + } +} diff --git a/buildSrc/src/test/groovy/dev/braintrust/gradle/muzzle/MuzzleTaskCacheTest.groovy b/buildSrc/src/test/groovy/dev/braintrust/gradle/muzzle/MuzzleTaskCacheTest.groovy new file mode 100644 index 00000000..3b1c8c13 --- /dev/null +++ b/buildSrc/src/test/groovy/dev/braintrust/gradle/muzzle/MuzzleTaskCacheTest.groovy @@ -0,0 +1,292 @@ +package dev.braintrust.gradle.muzzle + +import org.gradle.testkit.runner.GradleRunner +import org.gradle.testkit.runner.TaskOutcome +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource + +import java.util.jar.JarEntry +import java.util.jar.JarOutputStream + +import static org.junit.jupiter.api.Assertions.* + +/** Real Gradle build-cache round trips, with only local Maven artifacts and checker fixtures. */ +class MuzzleTaskCacheTest { + @TempDir + File directory + + private File report(String task = 'muzzle') { + new File(directory, "build/reports/muzzle/${task == 'muzzle' ? 'result' : task}.txt") + } + + private void put(String path, String text) { + def file = new File(directory, path) + file.parentFile.mkdirs() + file.setText(text, 'UTF-8') + } + + private void artifact(String module, String version, Map entries, String dependencies = '') { + def base = "repo/fixture/${module}/${version}/${module}-${version}" + put("${base}.pom", """4.0.0fixture +${module}${version}${dependencies}""") + def file = new File(directory, "${base}.jar") + new JarOutputStream(new FileOutputStream(file)).withCloseable { jar -> + entries.each { name, value -> + def entry = new JarEntry(name) + entry.time = 0 + jar.putNextEntry(entry) + jar.write(value.getBytes('UTF-8')) + jar.closeEntry() + } + } + } + + @BeforeEach + void fixture() { + put('settings.gradle', ''' +rootProject.name = 'muzzle-cache-fixture' +include ':braintrust-java-agent:bootstrap' +buildCache { local { directory = new File(settingsDir, 'cache') } } +''') + put('gradle.properties', 'org.gradle.caching=true\norg.gradle.workers.max=2\n') + put('versions.txt', '1.0\n') + put('directive.properties', 'range=[1.0,)\npins=1.0\nassertPass=true\n') + put('build.gradle', ''' +import dev.braintrust.gradle.muzzle.MuzzleTask +import dev.braintrust.gradle.muzzle.MuzzleDirective +import dev.braintrust.gradle.muzzle.MavenVersions +import org.gradle.api.tasks.CacheableTask + +plugins { + id 'java' + id 'dev.braintrust.muzzle' +} + +repositories { maven { url = uri('repo') } } +project(':braintrust-java-agent') { configurations { bootstrapLibs } } +project(':braintrust-java-agent:bootstrap') { + apply plugin: 'java' + apply plugin: 'dev.braintrust.muzzle' +} + +def settings = new Properties() +file('directive.properties').withInputStream { settings.load(it) } +def check = new MuzzleDirective( + group: 'fixture', module: 'target', versions: settings.getProperty('range'), + assertPass: settings.getProperty('assertPass', 'true').toBoolean(), + pinnedVersions: settings.getProperty('pins', '').tokenize(','), + skipVersions: settings.getProperty('skip', '').tokenize(',') as Set, + additionalDependencies: settings.getProperty('extra', '').tokenize(','), + excludedDependencies: settings.getProperty('excluded', '').tokenize(','), + ignoredInstrumentation: settings.getProperty('ignored', '').tokenize(',')) +muzzle.directives.add(check) +dependencies { compileOnly files('dependency.jar') } +project(':braintrust-java-agent').dependencies.add('bootstrapLibs', files('bootstrap.jar')) + +// Keep generated references separate from compiler-owned outputs. +def generatedMuzzleDir = layout.buildDirectory.dir('generated/muzzle') +tasks.register('generateMuzzle') { + dependsOn tasks.named('compileJava') + outputs.dir(generatedMuzzleDir) + doLast { + def marker = new File(generatedMuzzleDir.get().asFile, 'generated.txt') + marker.parentFile.mkdirs() + marker.text = 'generated' + } +} +sourceSets.main.output.dir(generatedMuzzleDir, builtBy: 'generateMuzzle') +tasks.named('classes') { dependsOn tasks.named('generateMuzzle') } + +@CacheableTask +abstract class DiscoveredMuzzleTask extends MuzzleTask { + @Override + protected List discoverVersions(MuzzleDirective directive) { + logger.lifecycle('Reading fresh fixture metadata') + def range = MavenVersions.parseRange(directive.versions) + project.file('versions.txt').readLines().findAll { + it && range.contains(it) && !directive.skipVersions.contains(it) + } + } +} +tasks.register('discoveredMuzzle', DiscoveredMuzzleTask) { + dependsOn tasks.named('classes'), ':braintrust-java-agent:bootstrap:classes' + instrumentationClasspath.from(sourceSets.main.output, configurations.compileClasspath) + bootstrapClasspath.from(provider { + project(':braintrust-java-agent:bootstrap').sourceSets.main.output.files.toList() + }) + reportFile = layout.buildDirectory.file('reports/muzzle/discoveredMuzzle.txt') +} +''') + put('src/main/java/dev/braintrust/instrumentation/InstrumentationModule.java', ''' +package dev.braintrust.instrumentation; +public abstract class InstrumentationModule { + public abstract String name(); + public abstract Matcher classLoaderMatcher(); + public String[] getHelperClassNames() { return new String[0]; } + public String[] getMuzzleIgnoredClassNames() { return new String[0]; } + public static class Matcher { + public boolean matches(ClassLoader loader) { return loader.getResource("compatible.txt") != null; } + public String[] getMismatchedReferenceSources(ClassLoader loader) { return new String[] {"fixture mismatch"}; } + } +} +''') + put('src/main/java/fixture/TestModule.java', ''' +package fixture; +import dev.braintrust.instrumentation.InstrumentationModule; +public class TestModule extends InstrumentationModule { + public String name() { return "fixture"; } + public Matcher classLoaderMatcher() { return new Matcher(); } + public static class Muzzle { + public static Matcher create() { return new Matcher(); } + } +} +''') + put('src/main/resources/META-INF/services/dev.braintrust.instrumentation.InstrumentationModule', 'fixture.TestModule\n') + put('braintrust-java-agent/bootstrap/src/main/java/fixture/Bootstrap.java', + 'package fixture; public class Bootstrap { public static int value() { return 1; } }\n') + artifact('dependency', '1.0', ['dependency.txt': 'one']) + new File(directory, 'dependency.jar').bytes = new File(directory, 'repo/fixture/dependency/1.0/dependency-1.0.jar').bytes + artifact('bootstrap-library', '1.0', ['bootstrap-library.txt': 'one']) + new File(directory, 'bootstrap.jar').bytes = new File(directory, 'repo/fixture/bootstrap-library/1.0/bootstrap-library-1.0.jar').bytes + artifact('transitive', '1.0', ['transitive.txt': 'one']) + artifact('target', '1.0', ['compatible.txt': 'one'], ''' +fixturetransitive1.0''') + artifact('target', '2.0', ['compatible.txt': 'two']) + artifact('target', '3.0', ['incompatible.txt': 'three']) + } + + private GradleRunner runner(String task) { + GradleRunner.create() + .withProjectDir(directory) + .withTestKitDir(new File(directory, 'testkit')) + .withPluginClasspath(System.getProperty('muzzle.test.classpath').split(File.pathSeparator).collect { new File(it) }) + .withArguments(task, '--build-cache', '--offline', '--stacktrace', '--no-configuration-cache') + } + + private def initialSuccess(String task = 'muzzle') { + def result = runner(task).build() + assertEquals(TaskOutcome.SUCCESS, result.task(":${task}").outcome, result.output) + assertTrue(report(task).isFile()) + return result + } + + @Test + void restoresCompletedResultAfterOutputDeletion() { + def initial = initialSuccess() + def bytes = report().bytes + assertTrue(report().delete()) + def result = runner('muzzle').build() + assertEquals(TaskOutcome.FROM_CACHE, result.task(':muzzle').outcome, initial.output + result.output) + assertArrayEquals(bytes, report().bytes) + } + + @ParameterizedTest + @ValueSource(strings = ['code', 'resource', 'dependency', 'bootstrap', 'bootstrapLibrary', 'generated', 'transitive']) + void changingClasspathInputsRunsChecksAgain(String input) { + initialSuccess() + switch (input) { + case 'code': + def source = new File(directory, 'src/main/java/fixture/TestModule.java') + source.text = source.text.replace('return "fixture"', 'return "changed"') + break + case 'resource': + put('src/main/resources/instrumentation.txt', 'changed') + break + case 'dependency': + artifact('dependency', '2.0', ['dependency.txt': 'two']) + new File(directory, 'dependency.jar').bytes = new File(directory, 'repo/fixture/dependency/2.0/dependency-2.0.jar').bytes + break + case 'bootstrap': + put('braintrust-java-agent/bootstrap/src/main/java/fixture/Bootstrap.java', + 'package fixture; public class Bootstrap { public static int value() { return 2; } }\n') + break + case 'bootstrapLibrary': + artifact('bootstrap-library', '1.0', ['bootstrap-library.txt': 'changed']) + new File(directory, 'bootstrap.jar').bytes = new File(directory, 'repo/fixture/bootstrap-library/1.0/bootstrap-library-1.0.jar').bytes + break + case 'generated': + // Keep the task implementation unchanged; mutate an additional generated classpath resource. + put('build/generated/muzzle/extra-generated.txt', 'changed') + break + case 'transitive': + artifact('transitive', '1.0', ['transitive.txt': 'changed']) + break + } + assertTrue(report().delete()) + assertEquals(TaskOutcome.SUCCESS, runner('muzzle').build().task(':muzzle').outcome) + } + + @ParameterizedTest + @ValueSource(strings = ['range=[0.0,)\npins=1.0', 'range=[1.0,)\npins=1.0\nskip=9.0', + 'range=[1.0,)\npins=1.0\nextra=fixture:dependency:1.0', + 'range=[1.0,)\npins=1.0\nexcluded=fixture:transitive', + 'range=[1.0,)\npins=1.0\nignored=fixture.NotPresent', + 'range=[1.0,)\npins=2.0']) + void changingDirectiveInputsRunsChecksAgain(String properties) { + initialSuccess() + put('directive.properties', properties + '\nassertPass=true\n') + assertTrue(report().delete()) + assertEquals(TaskOutcome.SUCCESS, runner('muzzle').build().task(':muzzle').outcome) + } + + @Test + void discoversNewTargetsBeforeCacheLookup() { + put('directive.properties', 'range=[1.0,)\nassertPass=true\n') + initialSuccess('discoveredMuzzle') + assertTrue(report('discoveredMuzzle').delete()) + def hit = runner('discoveredMuzzle').build() + assertEquals(TaskOutcome.FROM_CACHE, hit.task(':discoveredMuzzle').outcome) + put('versions.txt', '1.0\n2.0\n') + assertTrue(report('discoveredMuzzle').delete()) + def changed = runner('discoveredMuzzle').build() + assertEquals(TaskOutcome.SUCCESS, changed.task(':discoveredMuzzle').outcome) + assertTrue(report('discoveredMuzzle').text.contains('fixture:target:2.0 pass')) + } + + @Test + void emptyDiscoveredSetRemainsValidButDoesNotHideNewTargets() { + put('directive.properties', 'range=[1.0,)\nassertPass=true\n') + put('versions.txt', '') + initialSuccess('discoveredMuzzle') + put('versions.txt', '1.0\n') + assertTrue(report('discoveredMuzzle').delete()) + assertEquals(TaskOutcome.SUCCESS, runner('discoveredMuzzle').build().task(':discoveredMuzzle').outcome) + assertTrue(report('discoveredMuzzle').text.contains('Checked 1 version(s)')) + } + + @Test + void mismatchNeverBecomesCachedSuccess() { + initialSuccess() + put('directive.properties', 'range=[1.0,)\npins=3.0\nassertPass=true\n') + assertEquals(TaskOutcome.FAILED, runner('muzzle').buildAndFail().task(':muzzle').outcome) + assertFalse(report().exists()) + assertEquals(TaskOutcome.FAILED, runner('muzzle').buildAndFail().task(':muzzle').outcome) + assertFalse(report().exists()) + put('directive.properties', 'range=[1.0,)\npins=3.0\nassertPass=false\n') + initialSuccess() + assertTrue(report().delete()) + assertEquals(TaskOutcome.FROM_CACHE, runner('muzzle').build().task(':muzzle').outcome) + } + + @Test + void changedPassFailExpectationCannotReuseSuccess() { + initialSuccess() + put('directive.properties', 'range=[1.0,)\npins=1.0\nassertPass=false\n') + assertEquals(TaskOutcome.FAILED, runner('muzzle').buildAndFail().task(':muzzle').outcome) + assertFalse(report().exists()) + } + + @Test + void unresolvedConcreteTargetFailsBeforeCacheReuse() { + initialSuccess() + assertTrue(report().delete()) + put('directive.properties', 'range=[1.0,)\npins=1.0,9.0\nassertPass=true\n') + def result = runner('muzzle').buildAndFail() + assertTrue(result.output.contains('refusing an incomplete check')) + assertFalse(report().exists()) + assertTrue(runner('muzzle').buildAndFail().output.contains('refusing an incomplete check')) + } +} diff --git a/scripts/check-codeql.sh b/scripts/check-codeql.sh index 7ded68af..4dfb834e 100755 --- a/scripts/check-codeql.sh +++ b/scripts/check-codeql.sh @@ -4,7 +4,8 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" -if ! command -v codeql >/dev/null 2>&1; then +CODEQL="${CODEQL_PATH:-codeql}" +if ! command -v "$CODEQL" >/dev/null 2>&1; then echo "CodeQL is required. Run 'mise install', then 'mise exec -- ./scripts/check-codeql.sh'." >&2 exit 2 fi @@ -13,23 +14,11 @@ mkdir -p "$ROOT/build/codeql" SCAN_DIR="$(mktemp -d "$ROOT/build/codeql/scan.XXXXXX")" echo "CodeQL database and results: $SCAN_DIR" -# Match CI: capture fresh compilation, including generated sources. -# Invoke the wrapper's Java entry point directly: macOS system shells can break tracing. -codeql database create "$SCAN_DIR/java-db" \ - --language=java \ - --source-root="$ROOT" \ - --command='java -classpath gradle/wrapper/gradle-wrapper.jar org.gradle.wrapper.GradleWrapperMain compileJava --no-daemon --no-build-cache --rerun-tasks' - -codeql database analyze "$SCAN_DIR/java-db" \ - java-code-scanning.qls \ - --format=sarif-latest \ - --output="$SCAN_DIR/results.sarif" \ - --sarif-category='/language:java-kotlin' \ - --threads=0 +bash "$ROOT/scripts/scan-codeql.sh" "$SCAN_DIR" # Reuse the computed results; CSV has one record per alert and no header. # This lets us fail on findings without another JSON parser dependency. -codeql database interpret-results "$SCAN_DIR/java-db" \ +"$CODEQL" database interpret-results "$SCAN_DIR/java-db" \ java-code-scanning.qls \ --format=csv \ --output="$SCAN_DIR/results.csv" diff --git a/scripts/scan-codeql.sh b/scripts/scan-codeql.sh new file mode 100755 index 00000000..b9743754 --- /dev/null +++ b/scripts/scan-codeql.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ] || { [ "$#" -eq 2 ] && [ "$2" != '--offline' ]; }; then + echo "Usage: scan-codeql.sh [--offline]" >&2 + exit 2 +fi + +CODEQL="${CODEQL_PATH:-codeql}" +if ! command -v "$CODEQL" >/dev/null 2>&1; then + echo "CodeQL is required. Run 'mise install', then 'mise exec -- ./scripts/check-codeql.sh'." >&2 + exit 2 +fi + +SCAN_DIR="$1" +mkdir -p "$SCAN_DIR" + +# Full-repository analysis: do not filter queries/results using a PR's diff. +# macOS system shells can break tracing, so invoke the wrapper's Java entry point. +BUILD_COMMAND='java -classpath gradle/wrapper/gradle-wrapper.jar org.gradle.wrapper.GradleWrapperMain compileJava --no-daemon --no-build-cache --rerun-tasks' +if [ "${2:-}" = '--offline' ]; then + # Use the spec frozen by codeqlCacheKey, not a second network fetch during extraction. + mkdir -p "$ROOT/braintrust-api/build/openapi" + cp "$ROOT/build/codeql/spec-input/openapi/spec.yaml" "$ROOT/braintrust-api/build/openapi/spec.yaml" + BUILD_COMMAND="$BUILD_COMMAND --offline -x :braintrust-api:fetchOpenApiSpec" +fi + +"$CODEQL" database create "$SCAN_DIR/java-db" \ + --language=java \ + --source-root="$ROOT" \ + --command="$BUILD_COMMAND" + +# A successful scan can contain findings. CI uploads those results just like a +# fresh scan; the local check-codeql.sh command separately fails on findings. +"$CODEQL" database analyze "$SCAN_DIR/java-db" \ + java-code-scanning.qls \ + --format=sarif-latest \ + --output="$SCAN_DIR/results.sarif" \ + --sarif-category='/language:java-kotlin' \ + --threads=0