unified: Always run the corpus tests

The corpus tests skipped themselves when the `swift-syntax-parse` binary
could not be launched. That was meant to keep the suite usable without a
Swift toolchain, but it hid far more than it helped: a skip still
reports `test result: ok`, and the message explaining why only appears
under `cargo test -- --nocapture`. `scripts/update-corpus.sh` never set
the variable that locates the parser, so the documented way to
regenerate the corpus silently exercised nothing at all.

Drop the guard, so a missing parser fails loudly, and make the parser
easy to find so that failing is rare:

- Resolve the parser one directory above the running executable as well
  as beside it. Test binaries live in `target/<profile>/deps/`, so the
  existing sibling lookup could never find
  `target/<profile>/swift-syntax-parse` and `cargo test` always fell
  through to `PATH`.
- Report a missing binary with the command that builds it, rather than a
  bare `No such file or directory`.
- Build the parser in `scripts/update-corpus.sh`, so regenerating the
  corpus works from a clean checkout.

`cargo test` in `extractor` still needs the parser built first, since
the extractor deliberately does not depend on the crate that provides it
-- that is what keeps the Swift toolchain off the build path for the
other languages. `AGENTS.md` now says so.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: tausbn

unified/AGENTS.md

@@ -17,7 +17,7 @@ This is a CodeQL extractor based on tree-sitter.
 
 - The mapping from the parse tree to the target AST is found in `extractor/src/languages/swift/swift.rs`
 
-- To run tests for the parser and mapping, run `cargo test` in the `extractor` directory.
+- To run tests for the parser and mapping, run `cargo test` in the `extractor` directory. The corpus tests shell out to the `swift-syntax-parse` binary, which lives in a separate crate that `cargo test` does not build, so build it first with `cargo build -p swift-syntax-rs --bin swift-syntax-parse` (this needs a Swift toolchain; `scripts/update-corpus.sh` does it for you).
 
 - Extractor test cases are located at `extractor/tests/corpus/swift/*/*.swift`.
 

unified/extractor/src/languages/swift/parse.rs

@@ -43,46 +43,43 @@ pub fn parse(source: &[u8]) -> Result<ParsedTree, String> {
 ///    extractor pack lays it out (`tools/<platform>/{extractor,
 ///    swift-syntax-parse}`), so a packaged extractor is self-contained with no
 ///    environment setup;
-/// 3. a bare `swift-syntax-parse`, looked up on `PATH`.
+/// 3. a copy one directory further up, which is where `cargo` leaves it when
+///    the running executable is a test binary: those live in
+///    `target/<profile>/deps/`, one level below `target/<profile>/`;
+/// 4. a bare `swift-syntax-parse`, looked up on `PATH`.
 fn parse_bin() -> String {
     if let Ok(bin) = std::env::var(PARSE_BIN_ENV) {
         if !bin.is_empty() {
             return bin;
         }
     }
     if let Ok(exe) = std::env::current_exe() {
-        if let Some(sibling) = exe.parent().map(|dir| dir.join(PARSE_BIN_NAME)) {
-            if sibling.is_file() {
-                return sibling.to_string_lossy().into_owned();
+        let exe_dir = exe.parent();
+        let candidates = [exe_dir, exe_dir.and_then(|dir| dir.parent())];
+        for dir in candidates.into_iter().flatten() {
+            let candidate = dir.join(PARSE_BIN_NAME);
+            if candidate.is_file() {
+                return candidate.to_string_lossy().into_owned();
             }
         }
     }
     PARSE_BIN_NAME.to_string()
 }
 
-/// Whether the `swift-syntax-parse` executable can be launched at all.
-///
-/// This reports availability of the *executable*, deliberately not whether
-/// parsing succeeds: a binary that launches but then crashes or emits invalid
-/// JSON is still "available", so callers run and surface the failure rather
-/// than silently skipping. Only a genuinely missing/unlaunchable binary (e.g.
-/// no Swift toolchain is installed) reports `false`.
-pub fn binary_available() -> bool {
-    match Command::new(parse_bin())
-        .stdin(Stdio::null())
-        .stdout(Stdio::null())
-        .stderr(Stdio::null())
-        .spawn()
-    {
-        Ok(mut child) => {
-            let _ = child.wait();
-            true
-        }
-        Err(e) if e.kind() == std::io::ErrorKind::NotFound => false,
-        // Any other spawn failure (e.g. a permissions problem) is a genuine
-        // issue worth surfacing, so treat the parser as available and let the
-        // caller fail rather than masking it as "unavailable".
-        Err(_) => true,
+/// Explain a failure to launch the parser. A missing binary is by far the most
+/// common way this fails — the Swift half of the build is separate, so it is
+/// easy to have never built it — so that case carries the remedy rather than a
+/// bare OS error.
+fn spawn_error(bin: &str, error: std::io::Error) -> String {
+    if error.kind() == std::io::ErrorKind::NotFound {
+        format!(
+            "could not find the Swift parser `{bin}`. Build it with \
+             `cargo build -p swift-syntax-rs --bin swift-syntax-parse` (this needs a Swift \
+             toolchain — see `unified/swift-syntax-rs/.swift-version` for the pinned version), \
+             or point `{PARSE_BIN_ENV}` at an existing copy."
+        )
+    } else {
+        format!("failed to spawn Swift parser `{bin}`: {error}")
     }
 }
 
@@ -95,7 +92,7 @@ fn run_parser(source: &str) -> Result<String, String> {
         .stdout(Stdio::piped())
         .stderr(Stdio::piped())
         .spawn()
-        .map_err(|e| format!("failed to spawn Swift parser `{bin}`: {e}"))?;
+        .map_err(|e| spawn_error(&bin, e))?;
 
     // The parser reads all of stdin before writing any stdout, so writing the
     // whole source and then closing stdin (by dropping it) cannot deadlock.

unified/extractor/tests/corpus_tests.rs

@@ -20,20 +20,6 @@ fn update_mode_enabled() -> bool {
         .unwrap_or(false)
 }
 
-/// Whether the external swift-syntax parser is available. When the parser
-/// binary genuinely cannot be found/launched (e.g. no Swift toolchain, and
-/// neither `CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE` nor a `swift-syntax-parse`
-/// on `PATH`), the corpus test is skipped rather than failed — it cannot run
-/// without the Swift-backed parser.
-///
-/// Crucially this checks only that the executable *launches*: a parser that is
-/// present but crashes, emits invalid JSON, or otherwise regresses is
-/// considered available, so the suite runs and fails (rather than silently
-/// skipping the very failures CI needs to catch).
-fn parser_available() -> bool {
-    languages::swift_parse::binary_available()
-}
-
 /// Parse a corpus `.output` file. The file holds a single test case made of
 /// three sections separated by `---` delimiter lines:
 ///
@@ -112,14 +98,6 @@ fn collect_corpus_stems(dir: &Path, out: &mut Vec<std::path::PathBuf>) {
 
 #[test]
 fn test_corpus() {
-    if !parser_available() {
-        eprintln!(
-            "skipping test_corpus: the swift-syntax parser is unavailable \
-             (set CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE or put \
-             `swift-syntax-parse` on PATH)"
-        );
-        return;
-    }
     let update_mode = update_mode_enabled();
     let all_languages = languages::all_language_specs();
     let corpus_dir = Path::new("tests/corpus");

unified/scripts/update-corpus.sh

@@ -4,5 +4,12 @@ IFS=$'\n\t'
 
 cd "$(dirname "$0")/.."
 
+# The corpus is produced by the external swift-syntax parser. That parser is a
+# separate crate which `cargo test` does not build (the extractor deliberately
+# does not depend on it, so working on other languages needs no Swift
+# toolchain), so build it up front — otherwise the tests below fail on a
+# missing binary.
+cargo build -p swift-syntax-rs --bin swift-syntax-parse
+
 cd extractor
 UNIFIED_UPDATE_CORPUS=1 cargo test