diff --git a/apps/heft-native/.gitignore b/apps/heft-native/.gitignore new file mode 100644 index 0000000000..75aaec4cd8 --- /dev/null +++ b/apps/heft-native/.gitignore @@ -0,0 +1,2 @@ +target/ +!Cargo.lock diff --git a/apps/heft-native/Cargo.lock b/apps/heft-native/Cargo.lock new file mode 100644 index 0000000000..2e4c76736e --- /dev/null +++ b/apps/heft-native/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "heft-native" +version = "1.3.1" diff --git a/apps/heft-native/Cargo.toml b/apps/heft-native/Cargo.toml new file mode 100644 index 0000000000..14c682b14e --- /dev/null +++ b/apps/heft-native/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "heft-native" +version = "1.3.1" +edition = "2021" +publish = false + +[[bin]] +name = "heft" +path = "src/main.rs" + +[dependencies] + +[profile.release] +opt-level = "s" +lto = true +codegen-units = 1 +panic = "abort" +strip = true diff --git a/apps/heft-native/README.md b/apps/heft-native/README.md new file mode 100644 index 0000000000..7366050229 --- /dev/null +++ b/apps/heft-native/README.md @@ -0,0 +1,108 @@ +# heft-native + +A self-contained, std-only Rust implementation of the `heft` command line. It is an experiment: it is +**opt-in**, it is not a Rush project (there is no `package.json` here), and nothing else in the repository +depends on it. `rush install` and `rush build` never look at this folder and work without Cargo. + +## Building + +```bash +cd apps/heft-native +cargo build --release +``` + +The binary is written to `apps/heft-native/target/release/heft` (`target/` is ignored by Git). The crate has +no dependencies; `Cargo.lock` only lists `heft-native` itself. + +## Using it + +Run the binary instead of `heft` from a project folder: + +```bash +cd build-tests/heft-node-everything-test +../../apps/heft-native/target/release/heft build --help +``` + +The binary implements `@rushstack/heft` of the version in `Cargo.toml` (kept equal to `apps/heft/package.json` by +a unit test). It first makes the same decision as Heft's JavaScript version selector +(`lib-commonjs/startWithVersionSelector.js`): with `--unmanaged`, outside of any project, or when the project's +`package.json` has no `@rushstack/heft` dependency, it runs itself; when the project's +`node_modules/@rushstack/heft` is the binary's own JavaScript companion (the same folder, for example through a +workspace or pnpm link) with the `lib-commonjs` layout, it runs itself too. In every other case (an older or newer +local Heft, a copy of the same version in another folder such as a patched or linked package, the legacy `lib/` +layout, a missing entry point, a `package.json` that `JSON.parse()` rejects, a companion of another version) it +prints nothing and delegates the whole invocation to its JavaScript companion, whose version selector then behaves +exactly like before. + +Whatever the binary does not handle natively is delegated the same way: `node /bin/heft `. +The companion is the `@rushstack/heft` package that belongs to the binary, found in this order: + +1. `HEFT_NATIVE_JS_BIN`: the path of a Heft `bin/heft` script +2. next to the executable: `apps/heft-native/target//heft` uses `apps/heft/bin/heft` of the same + checkout, and a future `@rushstack//bin/heft` uses `@rushstack/heft/bin/heft` next to it +3. `/node_modules/@rushstack/heft/bin/heft` + +## What runs without Node.js + +- Help, usage and command line errors (`--help`, ` --help`, `--version`, unknown actions or parameters, + unknown phases, "No phases were selected") are printed by the binary itself. +- `heft clean` runs natively for any project: it deletes `temp//` and the `cleanFiles` of the + selected phases and prints Heft's summary. +- A build runs natively when every task of the project is a `copy-files-plugin`, `delete-files-plugin` or + `set-environment-variables-plugin` task of `@rushstack/heft`, there are no lifecycle plugins, and the selected + phases and tasks form a single chain (so the output order does not depend on timing). The binary produces the + same output, exit code, copied files and `file-copy.json` incremental state as the JavaScript Heft. +- It does so only when standard input is the null device (as for builds started by Rush) and without `--watch`, + `--verbose` or `--debug`, and only for glob patterns of the forms `**/*`, `*`, `**/*.ext`, `**/*.{a,b}`, + `name.*` and literal paths without `excludeGlobs`. Symbolic links, unusual file names, files outside those + patterns or a `file-copy.json` that is not plain JSON make it hand the invocation to the JavaScript Heft before + anything is printed. +- If standard output is closed while it writes (`heft build | head`), it stops where Heft would stop and lets + `node` report the error exactly like Heft does. + +## Plugin host + +Invocations that need JavaScript plugins run in the plugin host of the selected Heft package +(`node /lib-commonjs/host/HostEntry.js --heft-plan-fd=`). The binary writes the plan to an unnamed +temporary file (`O_TMPFILE` in `$XDG_RUNTIME_DIR`, `$TMPDIR`, `/dev/shm` or `/tmp`), which `node` inherits and +the host reads and closes before any plugin is loaded; nothing is added to the environment or left on disk. On +Unix the binary replaces itself with `node` (`exec`), so signals, exit codes and memory use are the ones of the +JavaScript Heft; on other platforms it runs `node` and waits for it (only Linux is tested). `node` is looked up +in `PATH`, like `#!/usr/bin/env node` does. + +## Warm plugin host (opt-in) + +With `HEFT_WARM_HOST=1`, plugin-host runs whose standard input is the null device and whose standard output and +error are not terminals (for example builds started by Rush) are sent to a pre-warmed Node.js host over a Unix +domain socket in `$XDG_RUNTIME_DIR/heft-host-` (or `/tmp/heft-host-`). The folder must be a real +directory owned by the user with mode `0700`; otherwise the warm host is not used. The host's output is streamed +back and the binary exits with the host's exit code; Ctrl+C and `SIGTERM` are forwarded. + +Each host serves one run and then starts its successor; a host exits after `HEFT_WARM_HOST_IDLE_MS` milliseconds +without a run. Any mismatch (Heft or Node.js version, environment, umask, changed Heft files, Rush reporter or IPC +file descriptors) makes the host refuse and the binary run cold. When no host accepts the run, the binary runs +cold as usual and starts a host (`lib-commonjs/host/WarmHostEntry.js`) in the background for the next run, unless +four hosts are already running for the user. + +## Static build (optional) + +`cargo build --release --target x86_64-unknown-linux-musl` (after `rustup target add x86_64-unknown-linux-musl`) +produces a statically linked binary in `target/x86_64-unknown-linux-musl/release/heft`. It is about as fast as the +default build and uses less memory (about 1 MB instead of 2.8 MB peak for a native build), but it is about 90 KB +larger. + +## Layout + +Each folder under `src/` is a module with a single owner: + +| Module | Responsibility | +| --- | --- | +| `json`, `schema`, `regex` | JSONC parsing, JSON schema validation, the regular expression subset used by schemas | +| `cli` | command line model, parsing, help and error rendering | +| `config` | `heft.json`, rigs, `heft-plugin.json`, plugin options, package resolution | +| `graph`, `run`, `builtin`, `terminal` | phase and task graph, execution, native built-in plugins, terminal output | +| `sys` | the only place with `unsafe` code (minimal operating system calls) | +| `process`, `version`, `host_link` | running Node.js, selecting the Heft version, the connection to the JavaScript plugin host | + +The code follows these rules: only the Rust standard library, `#![deny(unsafe_code)]` outside `src/sys`, no +comments in the code, source files of at most 200 lines, and a stripped release binary of at most 1 MB. diff --git a/apps/heft-native/src/builtin/base64.rs b/apps/heft-native/src/builtin/base64.rs new file mode 100644 index 0000000000..d716d2549e --- /dev/null +++ b/apps/heft-native/src/builtin/base64.rs @@ -0,0 +1,44 @@ +const STANDARD_ALPHABET: &[u8; 64] = + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +pub fn append_standard_base64(bytes: &[u8], output: &mut String) { + output.reserve(bytes.len().div_ceil(3) * 4); + let (chunks, remainder) = bytes.as_chunks::<3>(); + for chunk in chunks { + let combined_bits = + ((chunk[0] as u32) << 16) | ((chunk[1] as u32) << 8) | (chunk[2] as u32); + output.push(STANDARD_ALPHABET[((combined_bits >> 18) & 0x3f) as usize] as char); + output.push(STANDARD_ALPHABET[((combined_bits >> 12) & 0x3f) as usize] as char); + output.push(STANDARD_ALPHABET[((combined_bits >> 6) & 0x3f) as usize] as char); + output.push(STANDARD_ALPHABET[(combined_bits & 0x3f) as usize] as char); + } + if remainder.len() == 1 { + let combined_bits = (remainder[0] as u32) << 16; + output.push(STANDARD_ALPHABET[((combined_bits >> 18) & 0x3f) as usize] as char); + output.push(STANDARD_ALPHABET[((combined_bits >> 12) & 0x3f) as usize] as char); + output.push('='); + output.push('='); + } else if remainder.len() == 2 { + let combined_bits = ((remainder[0] as u32) << 16) | ((remainder[1] as u32) << 8); + output.push(STANDARD_ALPHABET[((combined_bits >> 18) & 0x3f) as usize] as char); + output.push(STANDARD_ALPHABET[((combined_bits >> 12) & 0x3f) as usize] as char); + output.push(STANDARD_ALPHABET[((combined_bits >> 6) & 0x3f) as usize] as char); + output.push('='); + } +} + +pub fn sha256_digest_as_base64(digest: &[u8; 32]) -> [u8; 44] { + let mut encoded = [b'='; 44]; + let (chunks, remainder) = digest.as_chunks::<3>(); + for (chunk_index, chunk) in chunks.iter().enumerate() { + let combined_bits = ((chunk[0] as u32) << 16) | ((chunk[1] as u32) << 8) | (chunk[2] as u32); + for (offset, shift) in [18, 12, 6, 0].into_iter().enumerate() { + encoded[chunk_index * 4 + offset] = STANDARD_ALPHABET[((combined_bits >> shift) & 0x3f) as usize]; + } + } + let combined_bits = ((remainder[0] as u32) << 16) | ((remainder[1] as u32) << 8); + for (offset, shift) in [18, 12, 6].into_iter().enumerate() { + encoded[40 + offset] = STANDARD_ALPHABET[((combined_bits >> shift) & 0x3f) as usize]; + } + encoded +} diff --git a/apps/heft-native/src/builtin/build_info.rs b/apps/heft-native/src/builtin/build_info.rs new file mode 100644 index 0000000000..a6ec3d44be --- /dev/null +++ b/apps/heft-native/src/builtin/build_info.rs @@ -0,0 +1,116 @@ +use std::collections::HashMap; +use std::fs::{self, File}; +use std::io::{BufWriter, ErrorKind, Write}; + +use super::build_info_json::{is_array_index_key, parse_build_info_json}; +use super::javascript_json::append_json_string; +use super::node_file_system_error::{is_node_not_exist_error, NodeFileSystemError}; +use super::posix_path::{directory_name, relative_path, resolve_path}; + +pub struct IncrementalBuildInfo { + pub configuration_hash: String, + pub input_file_versions: Vec<(String, String)>, +} + +pub enum BuildInfoReadResult { + Missing, + Found(IncrementalBuildInfo), + NeedsJavaScript, +} + +pub fn try_read_build_info(build_info_path: &str) -> BuildInfoReadResult { + let bytes = match fs::read(build_info_path) { + Ok(bytes) => bytes, + Err(error) if is_node_not_exist_error(&error) => return BuildInfoReadResult::Missing, + Err(_) => return BuildInfoReadResult::NeedsJavaScript, + }; + let Some(parsed) = std::str::from_utf8(&bytes).ok().and_then(parse_build_info_json) else { + return BuildInfoReadResult::NeedsJavaScript; + }; + drop(bytes); + let base_folder_path = directory_name(build_info_path); + let mut input_file_versions: Vec<(String, String)> = parsed + .input_file_versions + .into_iter() + .map(|(relative_file_path, version)| (resolve_path(base_folder_path, &relative_file_path), version)) + .collect(); + let mut duplicates: Vec<(usize, usize)> = Vec::new(); + { + let mut first_index_by_path: HashMap<&str, usize> = HashMap::with_capacity(input_file_versions.len()); + for (index, (absolute_file_path, _)) in input_file_versions.iter().enumerate() { + if let Some(&first_index) = first_index_by_path.get(absolute_file_path.as_str()) { + duplicates.push((first_index, index)); + } else { + first_index_by_path.insert(absolute_file_path, index); + } + } + } + if !duplicates.is_empty() { + for &(first_index, duplicate_index) in &duplicates { + input_file_versions[first_index].1 = std::mem::take(&mut input_file_versions[duplicate_index].1); + } + let mut is_duplicate = vec![false; input_file_versions.len()]; + for &(_, duplicate_index) in &duplicates { + is_duplicate[duplicate_index] = true; + } + let mut flags = is_duplicate.into_iter(); + input_file_versions.retain(|_| !flags.next().unwrap_or(false)); + } + BuildInfoReadResult::Found(IncrementalBuildInfo { + configuration_hash: parsed.configuration_hash, + input_file_versions, + }) +} + +pub fn write_build_info<'entries>( + configuration_hash: &str, + input_file_versions: impl Iterator + Clone, + build_info_path: &str, +) -> Result<(), NodeFileSystemError> { + let base_folder_path = directory_name(build_info_path); + let relative_entries = input_file_versions.map(|(absolute_file_path, version)| (relative_path(base_folder_path, absolute_file_path), version)); + let mut array_index_entries: Vec<(String, &str)> = + relative_entries.clone().filter(|(key, _)| is_array_index_key(key)).collect(); + array_index_entries.sort_by_key(|(key, _)| key.parse::().unwrap_or(0)); + let named_entries = relative_entries.filter(|(key, _)| !is_array_index_key(key)); + let file = create_file_ensuring_folder_exists(build_info_path)?; + let mut writer = BufWriter::with_capacity(16384, file); + let mut chunk = String::with_capacity(256); + chunk.push_str("{\"configHash\":"); + append_json_string(configuration_hash, &mut chunk); + chunk.push_str(",\"inputFileVersions\":{"); + for (index, (key, version)) in array_index_entries.into_iter().chain(named_entries).enumerate() { + if index > 0 { + chunk.push(','); + } + append_json_string(&key, &mut chunk); + chunk.push(':'); + append_json_string(version, &mut chunk); + write_chunk(&mut writer, &mut chunk, build_info_path)?; + } + chunk.push_str("}}"); + write_chunk(&mut writer, &mut chunk, build_info_path)?; + writer + .flush() + .map_err(|error| NodeFileSystemError::new(error, "write", build_info_path, None)) +} + +fn write_chunk(writer: &mut BufWriter, chunk: &mut String, file_path: &str) -> Result<(), NodeFileSystemError> { + writer + .write_all(chunk.as_bytes()) + .map_err(|error| NodeFileSystemError::new(error, "write", file_path, None))?; + chunk.clear(); + Ok(()) +} + +fn create_file_ensuring_folder_exists(file_path: &str) -> Result { + match File::create(file_path) { + Ok(file) => Ok(file), + Err(error) if error.kind() == ErrorKind::NotFound => { + fs::create_dir_all(directory_name(file_path)) + .map_err(|error| NodeFileSystemError::new(error, "mkdir", directory_name(file_path), None))?; + File::create(file_path).map_err(|error| NodeFileSystemError::new(error, "open", file_path, None)) + } + Err(error) => Err(NodeFileSystemError::new(error, "open", file_path, None)), + } +} diff --git a/apps/heft-native/src/builtin/build_info_json.rs b/apps/heft-native/src/builtin/build_info_json.rs new file mode 100644 index 0000000000..6ccb087723 --- /dev/null +++ b/apps/heft-native/src/builtin/build_info_json.rs @@ -0,0 +1,152 @@ +pub struct BuildInfoJson { + pub configuration_hash: String, + pub input_file_versions: Vec<(String, String)>, +} + +pub fn parse_build_info_json(text: &str) -> Option { + let mut reader = StrictJsonReader { bytes: text.as_bytes(), position: 0 }; + let mut configuration_hash = None; + let mut input_file_versions = None; + reader.expect_byte(b'{')?; + loop { + let key = reader.read_string()?; + reader.expect_byte(b':')?; + match key.as_str() { + "configHash" if configuration_hash.is_none() => configuration_hash = Some(reader.read_string()?), + "inputFileVersions" if input_file_versions.is_none() => input_file_versions = Some(reader.read_string_map()?), + _ => return None, + } + if !reader.read_separator_or_end(b'}')? { + break; + } + } + reader.skip_whitespace(); + if reader.position != reader.bytes.len() { + return None; + } + Some(BuildInfoJson { + configuration_hash: configuration_hash?, + input_file_versions: input_file_versions?, + }) +} + +pub fn is_array_index_key(key: &str) -> bool { + !key.is_empty() + && key.bytes().all(|byte| byte.is_ascii_digit()) + && (key == "0" || !key.starts_with('0')) + && key.parse::().is_ok_and(|index| index < u32::MAX as u64) +} + +struct StrictJsonReader<'text> { + bytes: &'text [u8], + position: usize, +} + +impl StrictJsonReader<'_> { + fn skip_whitespace(&mut self) { + while let Some(b' ' | b'\t' | b'\n' | b'\r') = self.bytes.get(self.position) { + self.position += 1; + } + } + + fn expect_byte(&mut self, expected: u8) -> Option<()> { + self.skip_whitespace(); + (self.bytes.get(self.position) == Some(&expected)).then(|| self.position += 1) + } + + fn read_separator_or_end(&mut self, end: u8) -> Option { + self.skip_whitespace(); + let byte = *self.bytes.get(self.position)?; + self.position += 1; + match byte { + b',' => Some(true), + _ if byte == end => Some(false), + _ => None, + } + } + + fn read_string_map(&mut self) -> Option> { + self.expect_byte(b'{')?; + let mut entries: Vec<(String, String)> = Vec::new(); + let mut seen_keys: std::collections::HashSet = std::collections::HashSet::new(); + self.skip_whitespace(); + if self.bytes.get(self.position) == Some(&b'}') { + self.position += 1; + return Some(entries); + } + loop { + let key = self.read_string()?; + self.expect_byte(b':')?; + let value = self.read_string()?; + if is_array_index_key(&key) || !seen_keys.insert(key.clone()) { + return None; + } + entries.push((key, value)); + if !self.read_separator_or_end(b'}')? { + return Some(entries); + } + } + } + + fn read_string(&mut self) -> Option { + self.expect_byte(b'"')?; + let mut value = String::new(); + loop { + let start = self.position; + while let Some(&byte) = self.bytes.get(self.position) { + if byte == b'"' || byte == b'\\' || byte < 0x20 { + break; + } + self.position += 1; + } + value.push_str(std::str::from_utf8(&self.bytes[start..self.position]).ok()?); + let byte = *self.bytes.get(self.position)?; + self.position += 1; + match byte { + b'"' => return Some(value), + b'\\' => value.push(self.read_escape()?), + _ => return None, + } + } + } + + fn read_escape(&mut self) -> Option { + let byte = *self.bytes.get(self.position)?; + self.position += 1; + match byte { + b'"' => Some('"'), + b'\\' => Some('\\'), + b'/' => Some('/'), + b'b' => Some('\u{8}'), + b'f' => Some('\u{c}'), + b'n' => Some('\n'), + b'r' => Some('\r'), + b't' => Some('\t'), + b'u' => { + let first = self.read_hex_code_unit()?; + if !(0xd800..0xdc00).contains(&first) { + return char::from_u32(first); + } + if self.bytes.get(self.position..self.position + 2) != Some(b"\\u") { + return None; + } + self.position += 2; + let second = self.read_hex_code_unit()?; + if !(0xdc00..0xe000).contains(&second) { + return None; + } + char::from_u32(0x10000 + ((first - 0xd800) << 10) + (second - 0xdc00)) + } + _ => None, + } + } + + fn read_hex_code_unit(&mut self) -> Option { + let digits = std::str::from_utf8(self.bytes.get(self.position..self.position + 4)?).ok()?; + self.position += 4; + if !digits.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return None; + } + u32::from_str_radix(digits, 16).ok() + } +} diff --git a/apps/heft-native/src/builtin/build_info_json_tests.rs b/apps/heft-native/src/builtin/build_info_json_tests.rs new file mode 100644 index 0000000000..dbf0efa6c6 --- /dev/null +++ b/apps/heft-native/src/builtin/build_info_json_tests.rs @@ -0,0 +1,51 @@ +use super::build_info_json::{is_array_index_key, parse_build_info_json}; + +fn entries(text: &str) -> Option<(String, Vec<(String, String)>)> { + parse_build_info_json(text).map(|parsed| (parsed.configuration_hash, parsed.input_file_versions)) +} + +fn owned(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs.iter().map(|(key, value)| ((*key).to_owned(), (*value).to_owned())).collect() +} + +#[test] +fn reads_the_state_json_parse_would_read() { + assert_eq!( + entries(r#"{"configHash":"h","inputFileVersions":{"../a.txt":"v1","../b\u00e9.txt":"v2"}}"#), + Some(("h".to_owned(), owned(&[("../a.txt", "v1"), ("../bé.txt", "v2")]))) + ); + assert_eq!(entries(" { \"configHash\" : \"h\" , \"inputFileVersions\" : { } } "), Some(("h".to_owned(), Vec::new()))); + assert_eq!( + entries(r#"{"inputFileVersions":{"x":"1"},"configHash":"\"q\\"}"#), + Some(("\"q\\".to_owned(), owned(&[("x", "1")]))) + ); +} + +#[test] +fn refuses_everything_outside_the_shape_heft_writes() { + for text in [ + "", + "{}", + r#"{"configHash":"h"}"#, + r#"{"configHash":1,"inputFileVersions":{}}"#, + r#"{"configHash":"h","inputFileVersions":{"a":1}}"#, + r#"{"configHash":"h","inputFileVersions":{"a":"1","a":"2"}}"#, + r#"{"configHash":"h","inputFileVersions":{"7":"1"}}"#, + r#"{"configHash":"h","inputFileVersions":{},"fileDependencies":{}}"#, + r#"{"configHash":"h","configHash":"h","inputFileVersions":{}}"#, + r#"{"configHash":"h","inputFileVersions":{}} x"#, + r#"{"configHash":"h","inputFileVersions":{},}"#, + "{\"configHash\":\"\u{1}\",\"inputFileVersions\":{}}", + r#"{"configHash":"\ud800","inputFileVersions":{}}"#, + "\u{feff}{\"configHash\":\"h\",\"inputFileVersions\":{}}", + ] { + assert!(parse_build_info_json(text).is_none(), "{text:?}"); + } +} + +#[test] +fn array_index_keys_follow_ecmascript() { + assert!(is_array_index_key("0") && is_array_index_key("42") && is_array_index_key("4294967294")); + assert!(!is_array_index_key("") && !is_array_index_key("01") && !is_array_index_key("4294967295")); + assert!(!is_array_index_key("-1") && !is_array_index_key("1.5") && !is_array_index_key("../1")); +} diff --git a/apps/heft-native/src/builtin/builtin_task.rs b/apps/heft-native/src/builtin/builtin_task.rs new file mode 100644 index 0000000000..b1cf88dbb3 --- /dev/null +++ b/apps/heft-native/src/builtin/builtin_task.rs @@ -0,0 +1,109 @@ +use super::copy_files::{preflight_copy_files, run_copy_files_task, CopyFilesTaskPlan}; +use super::copy_operation::{plan_copy_operations, CopyOperation}; +use super::delete_files::run_delete_operations; +use super::deletion_permissions::selections_are_deletable_without_permission_errors; +use super::file_selection::{AbsoluteFileSelection, FileSelectionSpecifier}; +use super::node_file_system_error::NodeFileSystemError; +use super::set_environment_variables::{order_like_javascript_object_entries, run_set_environment_variables}; +use crate::terminal::ScopedLoggerOutput; + +pub enum BuiltinTaskOptions { + CopyFiles(Vec), + DeleteFiles(Vec), + SetEnvironmentVariables(Vec<(String, String)>), +} + +pub enum PlannedBuiltinTask { + CopyFiles(CopyFilesTaskPlan), + DeleteFiles(Vec), + SetEnvironmentVariables(Vec<(String, String)>), +} + +pub fn plan_builtin_task( + options: &BuiltinTaskOptions, + build_folder_path: &str, + task_temp_folder_path: &str, +) -> Option { + match options { + BuiltinTaskOptions::CopyFiles(copy_operations) => { + let (operations, configuration_hash) = plan_copy_operations(build_folder_path, copy_operations)?; + Some(PlannedBuiltinTask::CopyFiles(CopyFilesTaskPlan { + operations, + configuration_hash, + build_info_path: format!("{task_temp_folder_path}/file-copy.json"), + preflight: None, + })) + } + BuiltinTaskOptions::DeleteFiles(delete_operations) => delete_operations + .iter() + .map(|operation| operation.to_absolute_selection(build_folder_path)) + .collect::>>() + .map(PlannedBuiltinTask::DeleteFiles), + BuiltinTaskOptions::SetEnvironmentVariables(variables) => Some(PlannedBuiltinTask::SetEnvironmentVariables( + order_like_javascript_object_entries(variables.clone()), + )), + } +} + +pub fn plan_phase_clean( + clean_files: &[FileSelectionSpecifier], + build_folder_path: &str, + temp_folder_path: &str, + phase_name: &str, +) -> Option> { + let temp_folder_selection = FileSelectionSpecifier { + source_path: Some(temp_folder_path.to_owned()), + include_globs: Some(vec![phase_name.to_owned(), format!("{phase_name}.*")]), + ..FileSelectionSpecifier::default() + }; + clean_files + .iter() + .chain(std::iter::once(&temp_folder_selection)) + .map(|specifier| specifier.to_absolute_selection(build_folder_path)) + .collect() +} + +pub fn builtin_task_touches_files(planned_task: &PlannedBuiltinTask) -> bool { + !matches!(planned_task, PlannedBuiltinTask::SetEnvironmentVariables(_)) +} + +pub fn builtin_task_passes_preflight( + planned_task: &mut PlannedBuiltinTask, + temp_folder_path: &str, + keep_preflight_results: bool, +) -> bool { + match planned_task { + PlannedBuiltinTask::CopyFiles(plan) => { + let temp_folder_prefix = format!("{temp_folder_path}/"); + let destinations_are_outside_temp = plan.operations.iter().all(|operation| { + operation + .destination_folder_paths + .iter() + .all(|folder| folder != temp_folder_path && !folder.starts_with(&temp_folder_prefix)) + }); + let Some(preflight) = preflight_copy_files(plan).filter(|_| destinations_are_outside_temp) else { + return false; + }; + if keep_preflight_results { + plan.preflight = Some(preflight); + } + true + } + PlannedBuiltinTask::DeleteFiles(selections) => selections_are_deletable_without_permission_errors(selections), + PlannedBuiltinTask::SetEnvironmentVariables(_) => true, + } +} + +pub fn run_planned_builtin_task( + planned_task: PlannedBuiltinTask, + output: &ScopedLoggerOutput<'_>, +) -> Result<(), NodeFileSystemError> { + match planned_task { + PlannedBuiltinTask::CopyFiles(plan) => run_copy_files_task(plan, output), + PlannedBuiltinTask::DeleteFiles(selections) => run_delete_operations(&selections, output), + PlannedBuiltinTask::SetEnvironmentVariables(variables) => { + run_set_environment_variables(&variables, output); + Ok(()) + } + } +} diff --git a/apps/heft-native/src/builtin/copy_files.rs b/apps/heft-native/src/builtin/copy_files.rs new file mode 100644 index 0000000000..9f80b4464b --- /dev/null +++ b/apps/heft-native/src/builtin/copy_files.rs @@ -0,0 +1,175 @@ +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::fs::File; +use std::io::Read; + +use super::base64::sha256_digest_as_base64; +use super::build_info::{try_read_build_info, write_build_info, BuildInfoReadResult}; +use super::copy_operation::AbsoluteCopyOperation; +use super::delete_files::glob_changed_during_run; +use super::file_operations::{copy_file_overwriting, hard_link_overwriting}; +use super::node_file_system_error::NodeFileSystemError; +use super::posix_path::{base_name, relative_path}; +use super::sha256::Sha256; +use super::simple_glob::GlobbedEntry; +use crate::terminal::ScopedLoggerOutput; + +pub struct CopyFilesTaskPlan { + pub operations: Vec, + pub configuration_hash: String, + pub build_info_path: String, + pub preflight: Option, +} + +pub struct CopyFilesPreflight { + pub source_files: Vec>, + pub build_info: BuildInfoReadResult, +} + +struct CopyDescriptor { + operation_index: usize, + source_index: usize, + destination_path: String, +} + +pub fn preflight_copy_files(plan: &CopyFilesTaskPlan) -> Option { + let source_files = plan.operations.iter().map(|operation| operation.selection.select(false)).collect::>>()?; + let build_info = try_read_build_info(&plan.build_info_path); + (!matches!(build_info, BuildInfoReadResult::NeedsJavaScript)).then_some(CopyFilesPreflight { source_files, build_info }) +} + +pub fn run_copy_files_task(plan: CopyFilesTaskPlan, output: &ScopedLoggerOutput<'_>) -> Result<(), NodeFileSystemError> { + let CopyFilesPreflight { source_files, build_info } = match plan.preflight { + Some(preflight) => preflight, + None => preflight_copy_files(&plan).ok_or_else(glob_changed_during_run)?, + }; + let copy_descriptors = collect_copy_descriptors(&plan.operations, &source_files)?; + if copy_descriptors.is_empty() { + return Ok(()); + } + let source_path_of = |descriptor: &CopyDescriptor| source_files[descriptor.operation_index][descriptor.source_index].absolute_path.as_str(); + let old_entries: Vec<(String, String)> = match build_info { + BuildInfoReadResult::Found(old) if old.configuration_hash == plan.configuration_hash => old.input_file_versions, + _ => Vec::new(), + }; + let old_versions: HashMap<&str, &str> = old_entries.iter().map(|(path, version)| (path.as_str(), version.as_str())).collect(); + let mut new_versions: HashMap<&str, [u8; 44]> = HashMap::with_capacity(copy_descriptors.len()); + let mut added_input_files: Vec<&str> = Vec::new(); + for descriptor in ©_descriptors { + let source_path = source_path_of(descriptor); + if let Entry::Vacant(vacant) = new_versions.entry(source_path) { + vacant.insert(hash_file_contents(source_path)?); + if !old_versions.contains_key(source_path) { + added_input_files.push(source_path); + } + } + } + let version_text = |path: &str| new_versions.get(path).and_then(|version| std::str::from_utf8(version).ok()); + let mut copied_file_count = 0; + let mut linked_file_count = 0; + let mut last_existing_folder: Option = None; + for descriptor in ©_descriptors { + let source_path = source_path_of(descriptor); + if old_versions.get(source_path).copied() == version_text(source_path) { + continue; + } + if plan.operations[descriptor.operation_index].hardlink { + linked_file_count += 1; + hard_link_overwriting(source_path, &descriptor.destination_path)?; + } else { + copied_file_count += 1; + copy_file_overwriting(source_path, &descriptor.destination_path, &mut last_existing_folder)?; + } + } + if copied_file_count == 0 && linked_file_count == 0 { + output.write_line("All requested file copy operations are up to date. Nothing to do."); + return Ok(()); + } + output.write_line(&format!( + "Copied {copied_file_count} file{} and linked {linked_file_count} file{}", + if copied_file_count == 1 { "" } else { "s" }, + if linked_file_count == 1 { "" } else { "s" } + )); + if output.output_is_closed() { + return Ok(()); + } + let input_file_versions = old_entries + .iter() + .map(move |(path, version)| (path.as_str(), version_text(path).unwrap_or(version.as_str()))) + .chain(added_input_files.iter().map(move |path| (*path, version_text(path).unwrap_or_default()))); + write_build_info(&plan.configuration_hash, input_file_versions, &plan.build_info_path) +} + +fn path_relative_to_source_folder<'path>(source_folder_path: &str, file_path: &'path str) -> std::borrow::Cow<'path, str> { + let prefix_length = if source_folder_path.ends_with('/') { source_folder_path.len() } else { source_folder_path.len() + 1 }; + match file_path.get(prefix_length..) { + Some(relative) if file_path.starts_with(source_folder_path) && file_path.as_bytes()[prefix_length - 1] == b'/' => { + std::borrow::Cow::Borrowed(relative) + } + _ => std::borrow::Cow::Owned(relative_path(source_folder_path, file_path)), + } +} + +fn collect_copy_descriptors( + operations: &[AbsoluteCopyOperation], + source_files: &[Vec], +) -> Result, NodeFileSystemError> { + let mut candidates: Vec = Vec::new(); + for (operation_index, operation) in operations.iter().enumerate() { + for destination_folder_path in &operation.destination_folder_paths { + for (source_index, source_file) in source_files[operation_index].iter().enumerate() { + let destination_relative_path = if operation.flatten { + std::borrow::Cow::Borrowed(base_name(&source_file.absolute_path)) + } else { + path_relative_to_source_folder(&operation.selection.source_folder_path, &source_file.absolute_path) + }; + let mut destination_path = String::with_capacity(destination_folder_path.len() + destination_relative_path.len() + 1); + destination_path.push_str(destination_folder_path); + if !destination_folder_path.ends_with('/') { + destination_path.push('/'); + } + destination_path.push_str(&destination_relative_path); + candidates.push(CopyDescriptor { operation_index, source_index, destination_path }); + } + } + } + let mut keep = vec![true; candidates.len()]; + let mut first_index_by_destination: HashMap<&str, usize> = HashMap::with_capacity(candidates.len()); + for (index, candidate) in candidates.iter().enumerate() { + match first_index_by_destination.entry(candidate.destination_path.as_str()) { + Entry::Occupied(occupied) => { + let existing = &candidates[*occupied.get()]; + let same_source = source_files[existing.operation_index][existing.source_index].absolute_path + == source_files[candidate.operation_index][candidate.source_index].absolute_path; + if !same_source || operations[existing.operation_index].hardlink != operations[candidate.operation_index].hardlink { + return Err(NodeFileSystemError::from_message(&format!( + "Cannot copy multiple files to the same destination \"{}\".", + candidate.destination_path + ))); + } + keep[index] = false; + } + Entry::Vacant(vacant) => { + vacant.insert(index); + } + } + } + drop(first_index_by_destination); + let mut keep_flags = keep.into_iter(); + candidates.retain(|_| keep_flags.next().unwrap_or(false)); + Ok(candidates) +} + +fn hash_file_contents(file_path: &str) -> Result<[u8; 44], NodeFileSystemError> { + let mut file = File::open(file_path).map_err(|error| NodeFileSystemError::new(error, "open", file_path, None))?; + let mut hasher = Sha256::new(); + let mut buffer = [0u8; 16384]; + loop { + let read_length = file.read(&mut buffer).map_err(|error| NodeFileSystemError::new(error, "read", file_path, None))?; + if read_length == 0 { + break; + } + hasher.update(&buffer[..read_length]); + } + Ok(sha256_digest_as_base64(&hasher.finalize())) +} diff --git a/apps/heft-native/src/builtin/copy_operation.rs b/apps/heft-native/src/builtin/copy_operation.rs new file mode 100644 index 0000000000..c5a0b88dab --- /dev/null +++ b/apps/heft-native/src/builtin/copy_operation.rs @@ -0,0 +1,173 @@ +use super::base64::append_standard_base64; +use super::file_selection::{AbsoluteFileSelection, FileSelectionSpecifier}; +use super::javascript_json::{append_json_string, append_json_string_array}; +use super::posix_path::{relative_path, resolve_path}; +use super::sha256::Sha256; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CopyOperationField { + SourcePath, + DestinationFolders, + FileExtensions, + ExcludeGlobs, + IncludeGlobs, + Flatten, + Hardlink, +} + +#[derive(Clone, Debug)] +pub struct CopyOperation { + pub selection: FileSelectionSpecifier, + pub destination_folders: Vec, + pub flatten: Option, + pub hardlink: Option, + pub field_order: Vec, +} + +pub struct AbsoluteCopyOperation { + pub selection: AbsoluteFileSelection, + pub destination_folder_paths: Vec, + pub flatten: bool, + pub hardlink: bool, +} + +pub fn plan_copy_operations( + root_folder_path: &str, + copy_operations: &[CopyOperation], +) -> Option<(Vec, String)> { + let mut hasher = Sha256::new(); + let mut absolute_operations = Vec::with_capacity(copy_operations.len()); + for copy_operation in copy_operations { + let selection = copy_operation.selection.to_absolute_selection(root_folder_path)?; + let destination_folder_paths: Vec = copy_operation + .destination_folders + .iter() + .map(|folder| resolve_path(root_folder_path, folder)) + .collect(); + if destination_folder_paths.iter().any(|path| path.contains('\\')) { + return None; + } + let portable_json = portable_operation_json(root_folder_path, copy_operation, &selection, &destination_folder_paths); + hasher.update(portable_json.as_bytes()); + absolute_operations.push(AbsoluteCopyOperation { + selection, + destination_folder_paths, + flatten: copy_operation.flatten.unwrap_or(false), + hardlink: copy_operation.hardlink.unwrap_or(false), + }); + } + let mut configuration_hash = String::with_capacity(44); + append_standard_base64(&hasher.finalize(), &mut configuration_hash); + Some((absolute_operations, configuration_hash)) +} + +fn portable_operation_json( + root_folder_path: &str, + copy_operation: &CopyOperation, + selection: &AbsoluteFileSelection, + destination_folder_paths: &[String], +) -> String { + let mut field_order = copy_operation.field_order.clone(); + for appended_field in [CopyOperationField::SourcePath, CopyOperationField::IncludeGlobs] { + if !field_order.contains(&appended_field) { + field_order.push(appended_field); + } + } + let mut json = String::from("{"); + for field in field_order { + let before_field = json.len(); + if json.len() > 1 { + json.push(','); + } + match field { + CopyOperationField::SourcePath => { + json.push_str("\"sourcePath\":"); + append_json_string(&relative_path(root_folder_path, &selection.source_folder_path), &mut json); + } + CopyOperationField::DestinationFolders => { + json.push_str("\"destinationFolders\":"); + let relative_folders: Vec = destination_folder_paths + .iter() + .map(|folder| relative_path(root_folder_path, folder)) + .collect(); + append_json_string_array(relative_folders.iter().map(String::as_str), &mut json); + } + CopyOperationField::FileExtensions => json.truncate(before_field), + CopyOperationField::ExcludeGlobs => match ©_operation.selection.exclude_globs { + Some(exclude_globs) => { + json.push_str("\"excludeGlobs\":"); + append_json_string_array(exclude_globs.iter().map(String::as_str), &mut json); + } + None => json.truncate(before_field), + }, + CopyOperationField::IncludeGlobs => { + json.push_str("\"includeGlobs\":"); + append_json_string_array(selection.include_globs.iter().map(String::as_str), &mut json); + } + CopyOperationField::Flatten => append_boolean_field(&mut json, "flatten", copy_operation.flatten, before_field), + CopyOperationField::Hardlink => append_boolean_field(&mut json, "hardlink", copy_operation.hardlink, before_field), + } + } + json.push('}'); + json +} + +fn append_boolean_field(json: &mut String, name: &str, value: Option, before_field: usize) { + match value { + Some(value) => { + json.push('"'); + json.push_str(name); + json.push_str("\":"); + json.push_str(if value { "true" } else { "false" }); + } + None => json.truncate(before_field), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use CopyOperationField::*; + + fn strings(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() + } + + fn copy_operation(selection: FileSelectionSpecifier, destinations: &[&str], field_order: Vec) -> CopyOperation { + CopyOperation { selection, destination_folders: strings(destinations), flatten: None, hardlink: None, field_order } + } + + #[test] + fn configuration_hashes_match_heft_for_the_same_options() { + let extensions = FileSelectionSpecifier { + source_path: Some("src".into()), + file_extensions: Some(strings(&[".txt", ".json"])), + ..Default::default() + }; + let operation = copy_operation(extensions, &["lib"], vec![SourcePath, DestinationFolders, FileExtensions]); + let (_, hash) = plan_copy_operations("/p", &[operation]).unwrap(); + assert_eq!(hash, "Gq3fRWEm4dEAs+3oVto6GIrpNKnSLQ/XBYVSpRegiZo="); + let everything = FileSelectionSpecifier { source_path: Some("src/a".into()), include_globs: Some(strings(&["**/*"])), ..Default::default() }; + let operation = copy_operation(everything, &["lib-a"], vec![SourcePath, DestinationFolders, IncludeGlobs]); + assert_eq!(plan_copy_operations("/p", &[operation]).unwrap().1, "9+DnO5MwqlLC5GeUyIJuG6xrulClDZhgS+BkuxHPFKA="); + } + + #[test] + fn configuration_hash_follows_key_order_and_appends_computed_keys() { + let first_selection = FileSelectionSpecifier { source_path: Some(".".into()), include_globs: Some(strings(&["a.txt", "**/*"])), ..Default::default() }; + let mut first = copy_operation(first_selection, &["out"], vec![DestinationFolders, SourcePath, Flatten, Hardlink, IncludeGlobs]); + first.flatten = Some(true); + first.hardlink = Some(false); + let second_selection = FileSelectionSpecifier { + source_path: Some("src".into()), + exclude_globs: Some(Vec::new()), + include_globs: Some(strings(&["**/*.md"])), + ..Default::default() + }; + let second = copy_operation(second_selection, &["x", "../y"], vec![SourcePath, DestinationFolders, ExcludeGlobs, IncludeGlobs]); + let (operations, hash) = plan_copy_operations("/p", &[first, second]).unwrap(); + assert_eq!(hash, "8UB3GDscKljuhypd+V2Z1Vswt+/+JjoZWSnXRJFyqdM="); + assert_eq!(operations[1].destination_folder_paths, strings(&["/p/x", "/y"])); + assert!(operations[0].flatten && !operations[0].hardlink); + } +} diff --git a/apps/heft-native/src/builtin/delete_files.rs b/apps/heft-native/src/builtin/delete_files.rs new file mode 100644 index 0000000000..f9786bba4c --- /dev/null +++ b/apps/heft-native/src/builtin/delete_files.rs @@ -0,0 +1,53 @@ +use std::collections::HashSet; + +use super::file_operations::{delete_file_if_it_exists, delete_folder_recursively}; +use super::file_selection::AbsoluteFileSelection; +use super::node_file_system_error::NodeFileSystemError; +use crate::terminal::ScopedLoggerOutput; + +pub fn run_delete_operations( + selections: &[AbsoluteFileSelection], + output: &ScopedLoggerOutput<'_>, +) -> Result<(), NodeFileSystemError> { + let mut files_to_delete: Vec = Vec::new(); + let mut folders_to_delete: Vec = Vec::new(); + let mut seen_files: HashSet = HashSet::new(); + let mut seen_folders: HashSet = HashSet::new(); + for selection in selections { + let entries = selection.select(true).ok_or_else(glob_changed_during_run)?; + for entry in entries { + let (paths, seen) = if entry.is_directory { + (&mut folders_to_delete, &mut seen_folders) + } else { + (&mut files_to_delete, &mut seen_files) + }; + if seen.insert(entry.absolute_path.clone()) { + paths.push(entry.absolute_path); + } + } + } + let mut deleted_file_count = 0; + for file_path in &files_to_delete { + if delete_file_if_it_exists(file_path)? { + deleted_file_count += 1; + } + } + let mut deleted_folder_count = 0; + for folder_path in folders_to_delete.iter().rev() { + if delete_folder_recursively(folder_path)? { + deleted_folder_count += 1; + } + } + if deleted_file_count > 0 || deleted_folder_count > 0 { + output.write_line(&format!( + "Deleted {deleted_file_count} file{} and {deleted_folder_count} folder{}", + if deleted_file_count != 1 { "s" } else { "" }, + if deleted_folder_count != 1 { "s" } else { "" } + )); + } + Ok(()) +} + +pub fn glob_changed_during_run() -> NodeFileSystemError { + NodeFileSystemError::from_message("The files selected by a glob changed while heft was running.") +} diff --git a/apps/heft-native/src/builtin/deletion_permissions.rs b/apps/heft-native/src/builtin/deletion_permissions.rs new file mode 100644 index 0000000000..6948945418 --- /dev/null +++ b/apps/heft-native/src/builtin/deletion_permissions.rs @@ -0,0 +1,56 @@ +use super::file_selection::AbsoluteFileSelection; + +#[cfg(unix)] +pub fn selections_are_deletable_without_permission_errors(selections: &[AbsoluteFileSelection]) -> bool { + use std::fs; + use std::os::unix::fs::MetadataExt; + + use super::posix_path::directory_name; + + let user_id = crate::sys::effective_user_id(); + let folder_is_modifiable = |path: &str| { + user_id == 0 + || fs::symlink_metadata(path).is_ok_and(|metadata| { + metadata.is_dir() && metadata.uid() == user_id && metadata.mode() & 0o1300 == 0o300 + }) + }; + for selection in selections { + let Some(entries) = selection.select(true) else { + return false; + }; + for entry in entries { + if !folder_is_modifiable(directory_name(&entry.absolute_path)) { + return false; + } + if !entry.is_directory { + continue; + } + let mut folders_to_check = vec![entry.absolute_path]; + while let Some(folder) = folders_to_check.pop() { + if !folder_is_modifiable(&folder) { + return false; + } + let Ok(reader) = fs::read_dir(&folder) else { + return false; + }; + for child in reader { + let Ok(child) = child else { + return false; + }; + if child.file_type().is_ok_and(|file_type| file_type.is_dir()) { + let Ok(child_path) = child.path().into_os_string().into_string() else { + return false; + }; + folders_to_check.push(child_path); + } + } + } + } + } + true +} + +#[cfg(not(unix))] +pub fn selections_are_deletable_without_permission_errors(_selections: &[AbsoluteFileSelection]) -> bool { + false +} diff --git a/apps/heft-native/src/builtin/file_operations.rs b/apps/heft-native/src/builtin/file_operations.rs new file mode 100644 index 0000000000..a8842ab9e3 --- /dev/null +++ b/apps/heft-native/src/builtin/file_operations.rs @@ -0,0 +1,107 @@ +use std::fs; +use std::io::ErrorKind; +use std::path::Path; + +use super::node_file_system_error::{is_node_not_exist_error, NodeFileSystemError}; +use super::node_rimraf::remove_like_node_rimraf; +use super::posix_path::directory_name; + +const EEXIST: i32 = 17; + +pub fn copy_file_overwriting( + source_path: &str, + destination_path: &str, + last_existing_folder: &mut Option, +) -> Result<(), NodeFileSystemError> { + let source_metadata = + fs::symlink_metadata(source_path).map_err(|error| NodeFileSystemError::new(error, "lstat", source_path, None))?; + let destination_metadata = match fs::symlink_metadata(destination_path) { + Ok(metadata) => Some(metadata), + Err(error) if error.kind() == ErrorKind::NotFound => None, + Err(error) => return Err(NodeFileSystemError::new(error, "lstat", destination_path, None)), + }; + if let Some(destination_metadata) = &destination_metadata { + if are_the_same_file(&source_metadata, destination_metadata) { + return Err(NodeFileSystemError::from_message("Source and destination must not be the same.")); + } + if !source_metadata.is_dir() && destination_metadata.is_dir() { + return Err(NodeFileSystemError::from_message(&format!( + "Cannot overwrite directory '{destination_path}' with non-directory '{source_path}'." + ))); + } + } + let destination_folder = directory_name(destination_path); + if destination_metadata.is_none() && last_existing_folder.as_deref() != Some(destination_folder) { + ensure_folder_exists(destination_folder)?; + *last_existing_folder = Some(destination_folder.to_owned()); + } + if destination_metadata.is_some() { + fs::remove_file(destination_path) + .map_err(|error| NodeFileSystemError::new(error, "unlink", destination_path, None))?; + } + fs::copy(source_path, destination_path) + .map(|_| ()) + .map_err(|error| NodeFileSystemError::new(error, "copyfile", source_path, Some(destination_path))) +} + +pub fn hard_link_overwriting(link_target_path: &str, new_link_path: &str) -> Result<(), NodeFileSystemError> { + let link = || { + fs::hard_link(link_target_path, new_link_path) + .map_err(|error| NodeFileSystemError::new(error, "link", link_target_path, Some(new_link_path))) + }; + match fs::hard_link(link_target_path, new_link_path) { + Ok(()) => Ok(()), + Err(error) if error.raw_os_error() == Some(EEXIST) => { + delete_file_if_it_exists(new_link_path).map_err(NodeFileSystemError::wrapped_again)?; + link() + } + Err(error) if is_node_not_exist_error(&error) && Path::new(link_target_path).exists() => { + ensure_folder_exists(directory_name(new_link_path)).map_err(NodeFileSystemError::wrapped_again)?; + link() + } + Err(error) => Err(NodeFileSystemError::new(error, "link", link_target_path, Some(new_link_path))), + } +} + +pub fn delete_file_if_it_exists(file_path: &str) -> Result { + match fs::remove_file(file_path) { + Ok(()) => Ok(true), + Err(error) if is_node_not_exist_error(&error) => Ok(false), + Err(error) => Err(NodeFileSystemError::new(error, "unlink", file_path, None)), + } +} + +pub fn delete_folder_recursively(folder_path: &str) -> Result { + match fs::symlink_metadata(folder_path) { + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(true), + Err(error) if is_node_not_exist_error(&error) => return Ok(false), + Err(error) => return Err(NodeFileSystemError::new(error, "lstat", folder_path, None)), + Ok(_) => {} + } + match remove_like_node_rimraf(Path::new(folder_path)) { + Ok(()) => Ok(true), + Err(failure) if is_node_not_exist_error(&failure.error) => Ok(false), + Err(failure) => Err(NodeFileSystemError::new(failure.error, failure.syscall, &failure.path, None)), + } +} + +#[cfg(unix)] +fn are_the_same_file(source_metadata: &fs::Metadata, destination_metadata: &fs::Metadata) -> bool { + use std::os::unix::fs::MetadataExt; + destination_metadata.ino() != 0 + && destination_metadata.dev() != 0 + && destination_metadata.ino() == source_metadata.ino() + && destination_metadata.dev() == source_metadata.dev() +} + +#[cfg(not(unix))] +fn are_the_same_file(_source_metadata: &fs::Metadata, _destination_metadata: &fs::Metadata) -> bool { + false +} + +fn ensure_folder_exists(folder_path: &str) -> Result<(), NodeFileSystemError> { + if Path::new(folder_path).exists() { + return Ok(()); + } + fs::create_dir_all(folder_path).map_err(|error| NodeFileSystemError::new(error, "mkdir", folder_path, None)) +} diff --git a/apps/heft-native/src/builtin/file_selection.rs b/apps/heft-native/src/builtin/file_selection.rs new file mode 100644 index 0000000000..bffee4753e --- /dev/null +++ b/apps/heft-native/src/builtin/file_selection.rs @@ -0,0 +1,97 @@ +use super::posix_path::resolve_path; +use super::simple_glob::{patterns_are_simple, try_simple_glob, GlobbedEntry}; +use super::simple_glob_pattern::is_extension; + +#[derive(Clone, Debug, Default)] +pub struct FileSelectionSpecifier { + pub source_path: Option, + pub file_extensions: Option>, + pub exclude_globs: Option>, + pub include_globs: Option>, +} + +#[derive(Clone, Debug)] +pub struct AbsoluteFileSelection { + pub source_folder_path: String, + pub include_globs: Vec, +} + +impl FileSelectionSpecifier { + pub fn to_absolute_selection(&self, root_folder_path: &str) -> Option { + if self.exclude_globs.as_ref().is_some_and(|globs| !globs.is_empty()) { + return None; + } + let source_folder_path = match self.source_path.as_deref() { + Some(source_path) if !source_path.is_empty() => resolve_path(root_folder_path, source_path), + _ => root_folder_path.to_owned(), + }; + if source_folder_path.contains('\\') { + return None; + } + let include_globs = self.included_glob_patterns()?; + patterns_are_simple(&include_globs).then_some(AbsoluteFileSelection { + source_folder_path, + include_globs, + }) + } + + fn included_glob_patterns(&self) -> Option> { + let mut escaped_file_extensions: Vec<&str> = Vec::new(); + for file_extension in self.file_extensions.iter().flatten() { + let escaped_file_extension = file_extension.strip_prefix('.').unwrap_or(file_extension); + if !is_extension(escaped_file_extension) { + return None; + } + if !escaped_file_extensions.contains(&escaped_file_extension) { + escaped_file_extensions.push(escaped_file_extension); + } + } + let mut patterns_to_glob: Vec = Vec::new(); + match escaped_file_extensions.as_slice() { + [] => {} + [single_extension] => patterns_to_glob.push(format!("**/*.{single_extension}")), + many_extensions => patterns_to_glob.push(format!("**/*.{{{}}}", many_extensions.join(","))), + } + for include_glob in self.include_globs.iter().flatten() { + if !patterns_to_glob.contains(include_glob) { + patterns_to_glob.push(include_glob.clone()); + } + } + if patterns_to_glob.is_empty() { + patterns_to_glob.push(String::from("**/*")); + } + Some(patterns_to_glob) + } +} + +impl AbsoluteFileSelection { + pub fn select(&self, include_folders: bool) -> Option> { + try_simple_glob(&self.include_globs, &self.source_folder_path, !include_folders) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn globs(values: &[&str]) -> Option> { + Some(values.iter().map(|value| (*value).to_owned()).collect()) + } + + #[test] + fn include_globs_are_computed_like_heft() { + let specifier = FileSelectionSpecifier { + source_path: Some("src/assets".into()), + file_extensions: globs(&[".txt", ".json", ".txt"]), + include_globs: globs(&["**/*.{txt,json}", "lib"]), + exclude_globs: None, + }; + let absolute = specifier.to_absolute_selection("/p").unwrap(); + assert_eq!(absolute.source_folder_path, "/p/src/assets"); + assert_eq!(absolute.include_globs, vec!["**/*.{txt,json}".to_owned(), "lib".to_owned()]); + let default_selection = FileSelectionSpecifier::default().to_absolute_selection("/p").unwrap(); + assert_eq!(default_selection.include_globs, vec!["**/*".to_owned()]); + let excluded = FileSelectionSpecifier { exclude_globs: globs(&["x"]), ..Default::default() }; + assert!(excluded.to_absolute_selection("/p").is_none()); + } +} diff --git a/apps/heft-native/src/builtin/javascript_json.rs b/apps/heft-native/src/builtin/javascript_json.rs new file mode 100644 index 0000000000..424c9aa418 --- /dev/null +++ b/apps/heft-native/src/builtin/javascript_json.rs @@ -0,0 +1,45 @@ +pub fn append_json_string(value: &str, output: &mut String) { + output.push('"'); + for character in value.chars() { + match character { + '"' => output.push_str("\\\""), + '\\' => output.push_str("\\\\"), + '\u{8}' => output.push_str("\\b"), + '\u{c}' => output.push_str("\\f"), + '\n' => output.push_str("\\n"), + '\r' => output.push_str("\\r"), + '\t' => output.push_str("\\t"), + control if (control as u32) < 0x20 => { + output.push_str(&format!("\\u{:04x}", control as u32)); + } + other => output.push(other), + } + } + output.push('"'); +} + +pub fn append_json_string_array<'a>(values: impl Iterator, output: &mut String) { + output.push('['); + for (index, value) in values.enumerate() { + if index > 0 { + output.push(','); + } + append_json_string(value, output); + } + output.push(']'); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strings_serialize_like_json_stringify() { + let mut output = String::new(); + append_json_string("a\"b\\c\n\u{1}\u{7f}é\u{2028}", &mut output); + assert_eq!(output, "\"a\\\"b\\\\c\\n\\u0001\u{7f}é\u{2028}\""); + let mut array = String::new(); + append_json_string_array(["x", "y"].into_iter(), &mut array); + assert_eq!(array, "[\"x\",\"y\"]"); + } +} diff --git a/apps/heft-native/src/builtin/mod.rs b/apps/heft-native/src/builtin/mod.rs new file mode 100644 index 0000000000..50adf41e77 --- /dev/null +++ b/apps/heft-native/src/builtin/mod.rs @@ -0,0 +1,31 @@ +mod base64; +mod build_info; +mod build_info_json; +#[cfg(test)] +mod build_info_json_tests; +mod builtin_task; +mod copy_files; +mod copy_operation; +mod delete_files; +mod deletion_permissions; +mod file_operations; +mod file_selection; +mod javascript_json; +mod node_file_system_error; +mod node_rimraf; +mod posix_path; +mod set_environment_variables; +mod sha256; +#[cfg(test)] +mod sha256_tests; +mod simple_glob; +mod simple_glob_pattern; + +pub use builtin_task::{ + builtin_task_passes_preflight, builtin_task_touches_files, plan_builtin_task, plan_phase_clean, + run_planned_builtin_task, BuiltinTaskOptions, PlannedBuiltinTask, +}; +pub use copy_operation::{CopyOperation, CopyOperationField}; +pub use deletion_permissions::selections_are_deletable_without_permission_errors; +pub use delete_files::run_delete_operations; +pub use file_selection::{AbsoluteFileSelection, FileSelectionSpecifier}; diff --git a/apps/heft-native/src/builtin/node_file_system_error.rs b/apps/heft-native/src/builtin/node_file_system_error.rs new file mode 100644 index 0000000000..8122223e33 --- /dev/null +++ b/apps/heft-native/src/builtin/node_file_system_error.rs @@ -0,0 +1,112 @@ +use std::io; + +const LIBUV_ERROR_NAMES_AND_DESCRIPTIONS: &[(i32, &str, &str)] = &[ + (1, "EPERM", "operation not permitted"), + (2, "ENOENT", "no such file or directory"), + (5, "EIO", "i/o error"), + (9, "EBADF", "bad file descriptor"), + (12, "ENOMEM", "not enough memory"), + (13, "EACCES", "permission denied"), + (16, "EBUSY", "resource busy or locked"), + (17, "EEXIST", "file already exists"), + (18, "EXDEV", "cross-device link not permitted"), + (20, "ENOTDIR", "not a directory"), + (21, "EISDIR", "illegal operation on a directory"), + (22, "EINVAL", "invalid argument"), + (23, "ENFILE", "file table overflow"), + (24, "EMFILE", "too many open files"), + (26, "ETXTBSY", "text file is busy"), + (27, "EFBIG", "file too large"), + (28, "ENOSPC", "no space left on device"), + (30, "EROFS", "read-only file system"), + (31, "EMLINK", "too many links"), + (36, "ENAMETOOLONG", "name too long"), + (39, "ENOTEMPTY", "directory not empty"), + (40, "ELOOP", "too many symbolic links encountered"), +]; + +const ENOENT: i32 = 2; +const ENOTDIR: i32 = 20; + +#[derive(Debug)] +pub struct NodeFileSystemError { + pub message: String, + node_core_library_prefix: Option, +} + +impl NodeFileSystemError { + pub fn new(error: io::Error, syscall: &str, path: &str, destination: Option<&str>) -> NodeFileSystemError { + let Some(errno) = error.raw_os_error() else { + return NodeFileSystemError::from_message(&error.to_string()); + }; + let (code, description) = LIBUV_ERROR_NAMES_AND_DESCRIPTIONS + .iter() + .find(|(known_errno, _, _)| *known_errno == errno) + .map(|(_, code, description)| (*code, *description)) + .unwrap_or(("UNKNOWN", "unknown error")); + let mut node_message = format!("{code}: {description}, {syscall} '{path}'"); + if let Some(destination) = destination { + node_message.push_str(&format!(" -> '{destination}'")); + } + let node_core_library_prefix = match code { + "ENOENT" => Some(format!("File does not exist: {path}\n")), + "ENOTDIR" => Some(format!("Folder does not exist: {path}\n")), + "EEXIST" => Some(format!("File or folder already exists: {}\n", destination.unwrap_or("undefined"))), + "EPERM" if syscall == "unlink" => Some(format!("File or folder could not be deleted: {path}\n")), + "EISDIR" => Some(format!("Target is a folder, not a file: {path}\n")), + _ => None, + }; + let message = format!("{}{node_message}", node_core_library_prefix.as_deref().unwrap_or_default()); + NodeFileSystemError { message, node_core_library_prefix } + } + + pub fn from_message(message: &str) -> NodeFileSystemError { + NodeFileSystemError { message: message.to_owned(), node_core_library_prefix: None } + } + + pub fn wrapped_again(mut self) -> NodeFileSystemError { + if let Some(prefix) = &self.node_core_library_prefix { + self.message.insert_str(0, prefix); + } + self + } +} + +pub fn is_node_not_exist_error(error: &io::Error) -> bool { + matches!(error.raw_os_error(), Some(ENOENT | ENOTDIR)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn errno(code: i32) -> io::Error { + io::Error::from_raw_os_error(code) + } + + #[test] + fn messages_match_node_and_node_core_library() { + assert_eq!( + NodeFileSystemError::new(errno(13), "open", "/p/a.txt", None).message, + "EACCES: permission denied, open '/p/a.txt'" + ); + assert_eq!( + NodeFileSystemError::new(errno(2), "open", "/p/a.txt", None).message, + "File does not exist: /p/a.txt\nENOENT: no such file or directory, open '/p/a.txt'" + ); + assert_eq!( + NodeFileSystemError::new(errno(13), "copyfile", "/p/a", Some("/p/b")).message, + "EACCES: permission denied, copyfile '/p/a' -> '/p/b'" + ); + assert_eq!( + NodeFileSystemError::new(errno(1), "unlink", "/p/x", None).message, + "File or folder could not be deleted: /p/x\nEPERM: operation not permitted, unlink '/p/x'" + ); + let twice = NodeFileSystemError::new(errno(21), "unlink", "/p/d", None).wrapped_again(); + assert_eq!( + twice.message, + "Target is a folder, not a file: /p/d\nTarget is a folder, not a file: /p/d\nEISDIR: illegal operation on a directory, unlink '/p/d'" + ); + assert_eq!(NodeFileSystemError::from_message("plain").wrapped_again().message, "plain"); + } +} diff --git a/apps/heft-native/src/builtin/node_rimraf.rs b/apps/heft-native/src/builtin/node_rimraf.rs new file mode 100644 index 0000000000..a7e084a3a0 --- /dev/null +++ b/apps/heft-native/src/builtin/node_rimraf.rs @@ -0,0 +1,71 @@ +use std::fs; +use std::io::{self, ErrorKind}; +use std::path::{Path, PathBuf}; + +const EPERM: i32 = 1; +const EEXIST: i32 = 17; +const ENOTDIR: i32 = 20; +const EISDIR: i32 = 21; +const ENOTEMPTY: i32 = 39; + +pub struct RimrafFailure { + pub error: io::Error, + pub syscall: &'static str, + pub path: String, +} + +fn failure(error: io::Error, syscall: &'static str, path: &Path) -> RimrafFailure { + RimrafFailure { error, syscall, path: path.to_string_lossy().into_owned() } +} + +pub fn remove_like_node_rimraf(path: &Path) -> Result<(), RimrafFailure> { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() => return remove_folder_like_node_rimraf(path, None), + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(()), + _ => {} + } + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(()), + Err(error) if matches!(error.raw_os_error(), Some(EISDIR | EPERM)) => { + remove_folder_like_node_rimraf(path, Some(failure(error, "unlink", path))) + } + Err(error) => Err(failure(error, "unlink", path)), + } +} + +fn remove_folder_like_node_rimraf(path: &Path, original_failure: Option) -> Result<(), RimrafFailure> { + match fs::remove_dir(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(()), + Err(error) if matches!(error.raw_os_error(), Some(ENOTEMPTY | EEXIST | EPERM)) => remove_children_then_folder(path), + Err(error) if error.raw_os_error() == Some(ENOTDIR) => match original_failure { + Some(original_failure) => Err(original_failure), + None => Err(failure(error, "rmdir", path)), + }, + Err(error) => Err(failure(error, "rmdir", path)), + } +} + +fn remove_children_then_folder(path: &Path) -> Result<(), RimrafFailure> { + let reader = fs::read_dir(path).map_err(|error| failure(error, "scandir", path))?; + let mut child_paths: Vec = Vec::new(); + for child in reader { + child_paths.push(child.map_err(|error| failure(error, "scandir", path))?.path()); + } + child_paths.sort(); + let mut first_failure: Option = None; + for child_path in child_paths { + if let Err(child_failure) = remove_like_node_rimraf(&child_path) { + first_failure.get_or_insert(child_failure); + } + } + if let Some(first_failure) = first_failure { + return Err(first_failure); + } + match fs::remove_dir(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(()), + Err(error) => Err(failure(error, "rmdir", path)), + } +} diff --git a/apps/heft-native/src/builtin/posix_path.rs b/apps/heft-native/src/builtin/posix_path.rs new file mode 100644 index 0000000000..c584509fc9 --- /dev/null +++ b/apps/heft-native/src/builtin/posix_path.rs @@ -0,0 +1,93 @@ +pub fn resolve_path(base_folder: &str, path: &str) -> String { + if path.starts_with('/') { + normalize_absolute_path(path) + } else if path.is_empty() { + normalize_absolute_path(base_folder) + } else { + normalize_absolute_path(&format!("{base_folder}/{path}")) + } +} + +pub fn normalize_absolute_path(path: &str) -> String { + let mut segments: Vec<&str> = Vec::new(); + for segment in path.split('/') { + match segment { + "" | "." => {} + ".." => { + segments.pop(); + } + _ => segments.push(segment), + } + } + let mut normalized = String::with_capacity(path.len() + 1); + for segment in &segments { + normalized.push('/'); + normalized.push_str(segment); + } + if normalized.is_empty() { + normalized.push('/'); + } + normalized +} + +pub fn relative_path(from_folder: &str, to_path: &str) -> String { + let from_segments: Vec<&str> = from_folder.split('/').filter(|s| !s.is_empty()).collect(); + let to_segments: Vec<&str> = to_path.split('/').filter(|s| !s.is_empty()).collect(); + let common_length = from_segments + .iter() + .zip(to_segments.iter()) + .take_while(|(from_segment, to_segment)| from_segment == to_segment) + .count(); + let mut relative = String::new(); + for _ in common_length..from_segments.len() { + if !relative.is_empty() { + relative.push('/'); + } + relative.push_str(".."); + } + for segment in &to_segments[common_length..] { + if !relative.is_empty() { + relative.push('/'); + } + relative.push_str(segment); + } + relative +} + +pub fn base_name(path: &str) -> &str { + let trimmed = path.trim_end_matches('/'); + match trimmed.rfind('/') { + Some(separator_index) => &trimmed[separator_index + 1..], + None => trimmed, + } +} + +pub fn directory_name(path: &str) -> &str { + match path.rfind('/') { + Some(0) => "/", + Some(separator_index) => &path[..separator_index], + None => ".", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn paths_resolve_and_relativize_like_node_posix_path() { + assert_eq!(resolve_path("/p", "src"), "/p/src"); + assert_eq!(resolve_path("/p", "./a//b/../c/"), "/p/a/c"); + assert_eq!(resolve_path("/p", "/abs/x"), "/abs/x"); + assert_eq!(resolve_path("/p", "../../.."), "/"); + assert_eq!(resolve_path("/p/q", ""), "/p/q"); + assert_eq!(relative_path("/p", "/p"), ""); + assert_eq!(relative_path("/p", "/p/lib/a"), "lib/a"); + assert_eq!(relative_path("/p/temp/build/copy", "/p/src/x.txt"), "../../../src/x.txt"); + assert_eq!(relative_path("/foo/bar", "/foo/barbaz"), "../barbaz"); + assert_eq!(relative_path("/", "/foo"), "foo"); + assert_eq!(relative_path("/foo/bar", "/"), "../.."); + assert_eq!(base_name("/p/src/x.txt"), "x.txt"); + assert_eq!(directory_name("/p/temp/file-copy.json"), "/p/temp"); + } +} diff --git a/apps/heft-native/src/builtin/set_environment_variables.rs b/apps/heft-native/src/builtin/set_environment_variables.rs new file mode 100644 index 0000000000..d056ba1a9d --- /dev/null +++ b/apps/heft-native/src/builtin/set_environment_variables.rs @@ -0,0 +1,16 @@ +use super::build_info_json::is_array_index_key; +use crate::terminal::ScopedLoggerOutput; + +pub fn order_like_javascript_object_entries(entries: Vec<(String, String)>) -> Vec<(String, String)> { + let (mut array_index_entries, named_entries): (Vec<_>, Vec<_>) = + entries.into_iter().partition(|(key, _)| is_array_index_key(key)); + array_index_entries.sort_by_key(|(key, _)| key.parse::().unwrap_or(0)); + array_index_entries.extend(named_entries); + array_index_entries +} + +pub fn run_set_environment_variables(variables: &[(String, String)], output: &ScopedLoggerOutput<'_>) { + for (name, value) in variables { + output.write_line(&format!("Setting environment variable {name}={value}")); + } +} diff --git a/apps/heft-native/src/builtin/sha256.rs b/apps/heft-native/src/builtin/sha256.rs new file mode 100644 index 0000000000..c5fc5e073b --- /dev/null +++ b/apps/heft-native/src/builtin/sha256.rs @@ -0,0 +1,152 @@ +const INITIAL_STATE: [u32; 8] = [ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, +]; + +const ROUND_CONSTANTS: [u32; 64] = [ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +]; + +pub struct Sha256 { + state: [u32; 8], + block_buffer: [u8; 64], + block_buffer_len: usize, + length_bytes: u64, +} + +impl Sha256 { + pub fn new() -> Sha256 { + Sha256 { + state: INITIAL_STATE, + block_buffer: [0; 64], + block_buffer_len: 0, + length_bytes: 0, + } + } + + pub fn update(&mut self, mut data: &[u8]) { + self.length_bytes = self.length_bytes.wrapping_add(data.len() as u64); + if self.block_buffer_len != 0 { + let remaining_space = 64 - self.block_buffer_len; + let copied_len = remaining_space.min(data.len()); + self.block_buffer[self.block_buffer_len..self.block_buffer_len + copied_len] + .copy_from_slice(&data[..copied_len]); + self.block_buffer_len += copied_len; + data = &data[copied_len..]; + if self.block_buffer_len == 64 { + let full_block = self.block_buffer; + self.process_block(&full_block); + self.block_buffer_len = 0; + } + } + while data.len() >= 64 { + let mut full_block = [0u8; 64]; + full_block.copy_from_slice(&data[..64]); + self.process_block(&full_block); + data = &data[64..]; + } + if !data.is_empty() { + self.block_buffer[..data.len()].copy_from_slice(data); + self.block_buffer_len = data.len(); + } + } + + pub fn finalize(mut self) -> [u8; 32] { + let length_bits = self.length_bytes.wrapping_mul(8); + self.block_buffer[self.block_buffer_len] = 0x80; + self.block_buffer_len += 1; + if self.block_buffer_len > 56 { + for byte in &mut self.block_buffer[self.block_buffer_len..] { + *byte = 0; + } + let full_block = self.block_buffer; + self.process_block(&full_block); + self.block_buffer = [0; 64]; + self.block_buffer_len = 0; + } + for byte in &mut self.block_buffer[self.block_buffer_len..56] { + *byte = 0; + } + self.block_buffer[56..64].copy_from_slice(&length_bits.to_be_bytes()); + let final_block = self.block_buffer; + self.process_block(&final_block); + let mut digest = [0u8; 32]; + for (word_index, state_word) in self.state.iter().enumerate() { + digest[word_index * 4..word_index * 4 + 4].copy_from_slice(&state_word.to_be_bytes()); + } + digest + } + + fn process_block(&mut self, block: &[u8; 64]) { + let mut message_schedule = [0u32; 64]; + for (word_index, message_word) in message_schedule.iter_mut().enumerate().take(16) { + let byte_index = word_index * 4; + *message_word = u32::from_be_bytes([ + block[byte_index], + block[byte_index + 1], + block[byte_index + 2], + block[byte_index + 3], + ]); + } + for word_index in 16..64 { + let small_sigma_zero = message_schedule[word_index - 15].rotate_right(7) + ^ message_schedule[word_index - 15].rotate_right(18) + ^ (message_schedule[word_index - 15] >> 3); + let small_sigma_one = message_schedule[word_index - 2].rotate_right(17) + ^ message_schedule[word_index - 2].rotate_right(19) + ^ (message_schedule[word_index - 2] >> 10); + message_schedule[word_index] = message_schedule[word_index - 16] + .wrapping_add(small_sigma_zero) + .wrapping_add(message_schedule[word_index - 7]) + .wrapping_add(small_sigma_one); + } + let mut a = self.state[0]; + let mut b = self.state[1]; + let mut c = self.state[2]; + let mut d = self.state[3]; + let mut e = self.state[4]; + let mut f = self.state[5]; + let mut g = self.state[6]; + let mut h = self.state[7]; + for round_index in 0..64 { + let big_sigma_one = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25); + let choice = (e & f) ^ (!e & g); + let temporary_one = h + .wrapping_add(big_sigma_one) + .wrapping_add(choice) + .wrapping_add(ROUND_CONSTANTS[round_index]) + .wrapping_add(message_schedule[round_index]); + let big_sigma_zero = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22); + let majority = (a & b) ^ (a & c) ^ (b & c); + let temporary_two = big_sigma_zero.wrapping_add(majority); + h = g; + g = f; + f = e; + e = d.wrapping_add(temporary_one); + d = c; + c = b; + b = a; + a = temporary_one.wrapping_add(temporary_two); + } + self.state[0] = self.state[0].wrapping_add(a); + self.state[1] = self.state[1].wrapping_add(b); + self.state[2] = self.state[2].wrapping_add(c); + self.state[3] = self.state[3].wrapping_add(d); + self.state[4] = self.state[4].wrapping_add(e); + self.state[5] = self.state[5].wrapping_add(f); + self.state[6] = self.state[6].wrapping_add(g); + self.state[7] = self.state[7].wrapping_add(h); + } +} + +impl Default for Sha256 { + fn default() -> Self { + Self::new() + } +} diff --git a/apps/heft-native/src/builtin/sha256_tests.rs b/apps/heft-native/src/builtin/sha256_tests.rs new file mode 100644 index 0000000000..8b6917ec0d --- /dev/null +++ b/apps/heft-native/src/builtin/sha256_tests.rs @@ -0,0 +1,125 @@ +use super::base64::append_standard_base64; +use super::sha256::Sha256; + +fn digest_hex_for_chunks(bytes: &[u8], chunks: &[usize]) -> String { + let mut hasher = Sha256::new(); + let mut start = 0; + for chunk_len in chunks { + if start >= bytes.len() { + break; + } + let end = (start + *chunk_len).min(bytes.len()); + hasher.update(&bytes[start..end]); + start = end; + } + if start < bytes.len() { + hasher.update(&bytes[start..]); + } + bytes_to_lower_hex(&hasher.finalize()) +} + +fn bytes_to_lower_hex(bytes: &[u8]) -> String { + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + use std::fmt::Write as _; + write!(&mut output, "{byte:02x}").unwrap(); + } + output +} + +fn next_xorshift64(value: &mut u64) -> u64 { + *value ^= *value << 13; + *value ^= *value >> 7; + *value ^= *value << 17; + *value +} + +fn deterministic_buffer(len: usize, seed: &mut u64) -> Vec { + let mut buffer = Vec::with_capacity(len); + while buffer.len() < len { + let random_word = next_xorshift64(seed).to_le_bytes(); + let remaining_len = len - buffer.len(); + buffer.extend_from_slice(&random_word[..remaining_len.min(random_word.len())]); + } + buffer +} + +fn hash_bytes_with_random_chunks(bytes: &[u8], seed: &mut u64) -> [u8; 32] { + let mut hasher = Sha256::new(); + let mut offset = 0; + while offset < bytes.len() { + let chunk_len = (next_xorshift64(seed) as usize % 211) + 1; + let end = (offset + chunk_len).min(bytes.len()); + hasher.update(&bytes[offset..end]); + offset = end; + } + hasher.finalize() +} + +#[test] +fn sha256_nist_vectors_match_expected_hex() { + assert_eq!( + digest_hex_for_chunks(b"", &[1, 2, 3]), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert_eq!( + digest_hex_for_chunks(b"abc", &[1, 1, 1]), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + let long_vector = b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; + assert_eq!( + digest_hex_for_chunks(long_vector, &[7, 13, 3, 29, 11]), + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" + ); + let million_a = vec![b'a'; 1_000_000]; + assert_eq!( + digest_hex_for_chunks(&million_a, &[17, 1024, 3, 65537, 91]), + "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0" + ); +} + +#[test] +fn base64_rfc_4648_section_10_cases_match() { + let cases = [ + (b"".as_slice(), ""), + (b"f".as_slice(), "Zg=="), + (b"fo".as_slice(), "Zm8="), + (b"foo".as_slice(), "Zm9v"), + (b"foob".as_slice(), "Zm9vYg=="), + (b"fooba".as_slice(), "Zm9vYmE="), + (b"foobar".as_slice(), "Zm9vYmFy"), + ]; + for (bytes, expected) in cases { + let mut encoded = String::from("prefix"); + append_standard_base64(bytes, &mut encoded); + assert_eq!(&encoded[6..], expected); + } +} + + +#[test] +fn random_chunking_matches_single_update() { + let mut seed = 0x9e37_79b9_7f4a_7c15; + for length in [0, 1, 55, 56, 63, 64, 65, 127, 128, 1000, 4099] { + let bytes = deterministic_buffer(length, &mut seed); + let mut single_update = Sha256::new(); + single_update.update(&bytes); + assert_eq!(hash_bytes_with_random_chunks(&bytes, &mut seed), single_update.finalize()); + } +} + +#[test] +fn base64_digests_match_node_crypto() { + let cases = [ + ("hello\n", "WJG1tSLV3whtD/CxEPvZ0hu0/HFjrzTQgoai6Eb2vgM="), + ("native fixture asset 1\n", "YugSLTOiQ6bFB9QpUFAMbncAUYN9ON2l3qYqwGllgBA="), + ("{\"fixture\":\"native\"}", "MiIsKqCR7Ps4OlPdpBh2XdRWhTIHPEclpVUVVA6w81g="), + ]; + for (text, expected) in cases { + let mut hasher = Sha256::new(); + hasher.update(text.as_bytes()); + let mut encoded = String::new(); + append_standard_base64(&hasher.finalize(), &mut encoded); + assert_eq!(encoded, expected); + } +} diff --git a/apps/heft-native/src/builtin/simple_glob.rs b/apps/heft-native/src/builtin/simple_glob.rs new file mode 100644 index 0000000000..492d4d06d3 --- /dev/null +++ b/apps/heft-native/src/builtin/simple_glob.rs @@ -0,0 +1,159 @@ +use std::collections::HashSet; +use std::fs; +use std::io::ErrorKind; + +use super::posix_path::resolve_path; +use super::simple_glob_pattern::{parse_simple_glob_pattern, SimpleGlobPattern}; + +pub struct GlobbedEntry { + pub absolute_path: String, + pub is_directory: bool, +} + +struct FolderToRead { + folder_path: String, + relative_folder_path: String, +} + +#[derive(Default)] +struct PatternSummary { + literal_paths: Vec, + match_any_recursive: bool, + match_any_top_level: bool, + suffixes: Vec, + prefixes: Vec, +} + +pub fn patterns_are_simple(patterns: &[String]) -> bool { + summarize_patterns(patterns).is_some() +} + +fn summarize_patterns(patterns: &[String]) -> Option { + if patterns.is_empty() { + return None; + } + let mut summary = PatternSummary::default(); + for pattern in patterns { + for simple_pattern in parse_simple_glob_pattern(pattern)? { + match simple_pattern { + SimpleGlobPattern::Literal(path) => summary.literal_paths.push(path), + SimpleGlobPattern::AnyRecursive => summary.match_any_recursive = true, + SimpleGlobPattern::AnyTopLevel => summary.match_any_top_level = true, + SimpleGlobPattern::Suffix(suffix) => summary.suffixes.push(suffix), + SimpleGlobPattern::TopLevelPrefix(prefix) => summary.prefixes.push(prefix), + } + } + } + Some(summary) +} + +pub fn try_simple_glob(patterns: &[String], cwd: &str, only_files: bool) -> Option> { + let summary = summarize_patterns(patterns)?; + if cwd.is_empty() { + return None; + } + let is_recursive = summary.match_any_recursive || !summary.suffixes.is_empty(); + let has_dynamic_patterns = is_recursive || summary.match_any_top_level || !summary.prefixes.is_empty(); + let mut entries: Vec = Vec::new(); + let mut literal_relative_paths: HashSet<&str> = HashSet::new(); + for literal_path in &summary.literal_paths { + let absolute_path = resolve_path(cwd, literal_path); + match fs::symlink_metadata(&absolute_path) { + Ok(metadata) if metadata.file_type().is_symlink() => return None, + Ok(metadata) => { + if (!only_files || metadata.is_file()) && literal_relative_paths.insert(literal_path) { + entries.push(GlobbedEntry { absolute_path, is_directory: metadata.is_dir() }); + } + } + Err(error) if error.kind() == ErrorKind::NotFound => {} + Err(_) => return None, + } + } + if has_dynamic_patterns { + let mut folders_to_read = vec![FolderToRead { + folder_path: resolve_path(cwd, ""), + relative_folder_path: String::new(), + }]; + let mut is_top_level = true; + while !folders_to_read.is_empty() { + let mut next_folders_to_read = Vec::new(); + for folder in &folders_to_read { + let children = match read_sorted_folder(&folder.folder_path) { + FolderReadResult::Entries(children) => children, + FolderReadResult::Missing if is_top_level => continue, + FolderReadResult::Missing | FolderReadResult::Unsupported => return None, + }; + for (name, is_directory, is_file) in children { + let is_match = summary.match_any_recursive + || (is_top_level && summary.match_any_top_level) + || summary.suffixes.iter().any(|suffix| name.ends_with(suffix.as_str())) + || (is_top_level && summary.prefixes.iter().any(|prefix| name.starts_with(prefix.as_str()))); + if is_match && (!only_files || is_file) { + let is_literal_duplicate = !literal_relative_paths.is_empty() + && literal_relative_paths.contains(join_path_segments(&folder.relative_folder_path, &name).as_str()); + if !is_literal_duplicate { + entries.push(GlobbedEntry { + absolute_path: join_path_segments(&folder.folder_path, &name), + is_directory, + }); + } + } + if is_recursive && is_directory { + next_folders_to_read.push(FolderToRead { + folder_path: join_path_segments(&folder.folder_path, &name), + relative_folder_path: join_path_segments(&folder.relative_folder_path, &name), + }); + } + } + } + folders_to_read = next_folders_to_read; + is_top_level = false; + } + } + Some(entries) +} + +enum FolderReadResult { + Entries(Vec<(String, bool, bool)>), + Missing, + Unsupported, +} + +fn read_sorted_folder(folder_path: &str) -> FolderReadResult { + let reader = match fs::read_dir(folder_path) { + Ok(reader) => reader, + Err(error) if error.kind() == ErrorKind::NotFound => return FolderReadResult::Missing, + Err(_) => return FolderReadResult::Unsupported, + }; + let mut children: Vec<(String, bool, bool)> = Vec::new(); + for child in reader { + let Ok(child) = child else { + return FolderReadResult::Unsupported; + }; + let Ok(file_type) = child.file_type() else { + return FolderReadResult::Unsupported; + }; + if file_type.is_symlink() { + return FolderReadResult::Unsupported; + } + let Ok(name) = child.file_name().into_string() else { + return FolderReadResult::Unsupported; + }; + if name.contains(['\\', '\n', '\r', '\u{2028}', '\u{2029}']) { + return FolderReadResult::Unsupported; + } + children.push((name, file_type.is_dir(), file_type.is_file())); + } + children.sort_by(|left, right| left.0.as_bytes().cmp(right.0.as_bytes())); + FolderReadResult::Entries(children) +} + +fn join_path_segments(left: &str, right: &str) -> String { + let mut joined = String::with_capacity(left.len() + right.len() + 1); + joined.push_str(left); + if !left.is_empty() && !left.ends_with('/') { + joined.push('/'); + } + joined.push_str(right); + joined +} diff --git a/apps/heft-native/src/builtin/simple_glob_pattern.rs b/apps/heft-native/src/builtin/simple_glob_pattern.rs new file mode 100644 index 0000000000..340203cc2a --- /dev/null +++ b/apps/heft-native/src/builtin/simple_glob_pattern.rs @@ -0,0 +1,96 @@ +#[derive(Debug, PartialEq, Eq)] +pub enum SimpleGlobPattern { + Literal(String), + AnyRecursive, + AnyTopLevel, + Suffix(String), + TopLevelPrefix(String), +} + +pub fn parse_simple_glob_pattern(pattern: &str) -> Option> { + if pattern == "**/*" { + return Some(vec![SimpleGlobPattern::AnyRecursive]); + } + if pattern == "*" { + return Some(vec![SimpleGlobPattern::AnyTopLevel]); + } + if let Some(extension_part) = pattern.strip_prefix("**/*.") { + if let Some(extension_list) = extension_part.strip_prefix('{') { + let extension_list = extension_list.strip_suffix('}')?; + if extension_list.contains(['/', '{', '}']) || extension_list.is_empty() { + return None; + } + let extensions: Vec<&str> = extension_list.split(',').collect(); + if extensions.len() < 2 || !extensions.iter().all(|extension| is_extension(extension)) { + return None; + } + return Some( + extensions + .into_iter() + .map(|extension| SimpleGlobPattern::Suffix(format!(".{extension}"))) + .collect(), + ); + } + if extension_part.is_empty() || extension_part.contains(['/', '{', '}']) { + return None; + } + return is_extension(extension_part) + .then(|| vec![SimpleGlobPattern::Suffix(format!(".{extension_part}"))]); + } + if let Some(prefix) = pattern.strip_suffix('*') { + if prefix.ends_with('.') && is_extension(&prefix[..prefix.len() - 1]) { + return Some(vec![SimpleGlobPattern::TopLevelPrefix(prefix.to_owned())]); + } + } + let every_segment_is_literal = pattern.split('/').all(|segment| { + !segment.is_empty() + && segment.bytes().all(is_literal_segment_byte) + && !segment.bytes().all(|byte| byte == b'.') + }); + every_segment_is_literal.then(|| vec![SimpleGlobPattern::Literal(pattern.to_owned())]) +} + +fn is_literal_segment_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'.' || byte == b'-' +} + +fn is_extension_word_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-' +} + +pub fn is_extension(text: &str) -> bool { + !text.is_empty() + && text + .split('.') + .all(|word| !word.is_empty() && word.bytes().all(is_extension_word_byte)) +} + +#[cfg(test)] +mod tests { + use super::*; + use SimpleGlobPattern::*; + + #[test] + fn patterns_parse_like_the_heft_simple_glob() { + assert_eq!(parse_simple_glob_pattern("**/*"), Some(vec![AnyRecursive])); + assert_eq!(parse_simple_glob_pattern("*"), Some(vec![AnyTopLevel])); + assert_eq!(parse_simple_glob_pattern("**/*.txt"), Some(vec![Suffix(".txt".into())])); + assert_eq!(parse_simple_glob_pattern("**/*.d.ts"), Some(vec![Suffix(".d.ts".into())])); + assert_eq!( + parse_simple_glob_pattern("**/*.{txt,json}"), + Some(vec![Suffix(".txt".into()), Suffix(".json".into())]) + ); + assert_eq!(parse_simple_glob_pattern("**/*.{txt}"), None); + assert_eq!(parse_simple_glob_pattern("**/*.{txt,}"), None); + assert_eq!(parse_simple_glob_pattern("**/*.t?t"), None); + assert_eq!(parse_simple_glob_pattern("build.*"), Some(vec![TopLevelPrefix("build.".into())])); + assert_eq!(parse_simple_glob_pattern("lib"), Some(vec![Literal("lib".into())])); + assert_eq!(parse_simple_glob_pattern("a/b.txt"), Some(vec![Literal("a/b.txt".into())])); + assert_eq!(parse_simple_glob_pattern("a/../b"), None); + assert_eq!(parse_simple_glob_pattern("./a"), None); + assert_eq!(parse_simple_glob_pattern("a//b"), None); + assert_eq!(parse_simple_glob_pattern("src/**/*.txt"), None); + assert_eq!(parse_simple_glob_pattern(".*"), None); + assert_eq!(parse_simple_glob_pattern(""), None); + } +} diff --git a/apps/heft-native/src/cli/action_invocation.rs b/apps/heft-native/src/cli/action_invocation.rs new file mode 100644 index 0000000000..cb400132f2 --- /dev/null +++ b/apps/heft-native/src/cli/action_invocation.rs @@ -0,0 +1,131 @@ +use std::borrow::Cow; + +use super::action_text::{action_documentation, alias_documentation, alias_expansion_message}; +use super::actions::{selected_phases_from, ActionKind, ActionTable, AliasEntry}; +use super::defined_parameter::DefinedParameter; +use super::help_builders::{action_help_parser, ActionHelpText}; +use super::model::CliModel; +use super::outcome::{CliOutcome, ParsedCommand, ParsedParameters}; +use super::parameters::{push_builtin_parameters, push_plugin_parameters, push_scoping_parameters, verbose_parameter, ROOT_PARAMETER_NAMES}; +use super::parse::{parse_arguments, ArgumentError, ParameterValue, ParseOutcome}; +use super::phase_selection::{select_phases, PhaseSelection}; +use super::registration::{try_register_parameters, Registration}; +use super::render::{ambiguity_error_output, argument_error_message, help_output, usage_error_output}; +use super::run_invocation::invoke_run; + +pub struct ActionRequest<'a, 't> { + pub model: &'a CliModel<'a>, + pub table: &'t ActionTable<'a>, + pub command_name: &'a str, + pub alias: Option<&'t AliasEntry<'a>>, + pub action_index: usize, + pub tool_args: &'a [&'a str], + pub rest: &'a [&'a str], + pub width: Option, + pub supports_color: Option<&'t dyn Fn() -> bool>, +} + +pub fn phase_action_parameters<'a>(model: &'a CliModel<'a>, table: &ActionTable<'a>, phase_index: usize, watch: bool) -> Option>> { + let mut parameters: Vec> = Vec::new(); + push_builtin_parameters(&mut parameters, watch); + push_plugin_parameters(&mut parameters, model, &selected_phases_from(table, [phase_index]))?; + Some(parameters) +} + +pub fn invoke_action<'a>(request: ActionRequest<'a, '_>) -> Option> { + let mut action_args: Vec<&'a str> = Vec::with_capacity(request.rest.len() + 4); + if let Some(alias) = request.alias { + action_args.extend(alias.default_parameters.iter().copied()); + } + action_args.extend(request.rest.iter().copied()); + let action = &request.table.actions[request.action_index]; + match action.kind { + ActionKind::Run => invoke_run(&request, &action_args), + ActionKind::Clean => { + let mut parameters: Vec> = Vec::new(); + push_scoping_parameters(&mut parameters, "clean"); + parameters.push(verbose_parameter()); + invoke_simple(&request, parameters, &action_args) + } + ActionKind::Phase(phase_index) => { + let parameters = phase_action_parameters(request.model, request.table, phase_index, action.watch)?; + invoke_simple(&request, parameters, &action_args) + } + } +} + +pub fn help_text<'a>(request: &ActionRequest<'a, '_>, epilog: Option) -> ActionHelpText<'a> { + let description: Cow<'a, str> = match request.alias { + Some(alias) => Cow::Owned(alias_documentation(request.table, alias)), + None => action_documentation(request.model, &request.table.actions[request.action_index]), + }; + ActionHelpText { prog: Cow::Owned(format!("heft {}", request.command_name)), description, epilog } +} + +pub fn print_help<'a>(request: &ActionRequest<'a, '_>, registration: &Registration, parameters: &[DefinedParameter<'_>], has_remainder: bool) -> Option> { + let parser = action_help_parser(registration, parameters, help_text(request, None), has_remainder)?; + Some(CliOutcome::Print(help_output(&parser, request.width, 1)?)) +} + +pub fn print_error<'a>(request: &ActionRequest<'a, '_>, registration: &Registration, parameters: &[DefinedParameter<'_>], has_remainder: bool, error: &ArgumentError<'_>) -> Option> { + let message: String = argument_error_message(error, registration, parameters)?; + let mut text: ActionHelpText<'a> = help_text(request, None); + let error_prog: String = text.prog.to_string(); + if matches!(error, ArgumentError::Ambiguous(_)) { + text.prog = Cow::Owned(format!("heft {}", request.table.actions[request.action_index].name)); + } + let parser = action_help_parser(registration, parameters, text, has_remainder)?; + let output = match error { + ArgumentError::Ambiguous(_) => ambiguity_error_output(&parser, request.width, &error_prog, &message)?, + _ => usage_error_output(&parser, request.width, &message)?, + }; + Some(CliOutcome::Print(output)) +} + +pub fn execute<'a>(request: &ActionRequest<'a, '_>, selected_phases: Vec, parameters: ParsedParameters<'a>, scoped: Option<(ParsedParameters<'a>, Vec<&'a str>)>) -> Option> { + let (scoped_parameters, remainder) = match scoped { + Some((scoped_parameters, remainder)) => (Some(scoped_parameters), remainder), + None => (None, Vec::new()), + }; + let action = &request.table.actions[request.action_index]; + let defaults: &[&str] = request.alias.map_or(&[], |alias| alias.default_parameters); + let uses_explicit_values: bool = defaults.iter().chain(request.rest).any(|arg| arg.starts_with('-') && arg.contains('=')); + Some(CliOutcome::Execute(Box::new(ParsedCommand { + command_name: request.command_name, + unaliased_command_name: action.name.to_string(), + action_kind: action.kind, + watch: action.watch, + debug: request.tool_args.contains(&"--debug"), + alias_expansion_message: request.alias.map(|alias| alias_expansion_message(request.table, alias)), + phase_name: match action.kind { + ActionKind::Phase(phase_index) => Some(request.model.phases[phase_index].name), + _ => None, + }, + selected_phases, + parameters, + scoped_parameters, + remainder, + parsed_like_v2_lean_parser: !uses_explicit_values, + }))) +} + +fn invoke_simple<'a>(request: &ActionRequest<'a, '_>, definitions: Vec>, action_args: &[&'a str]) -> Option> { + let registration: Registration = try_register_parameters(&definitions, &ROOT_PARAMETER_NAMES)?; + let values: Vec> = match parse_arguments(®istration, &definitions, action_args, false) { + ParseOutcome::Help => return print_help(request, ®istration, &definitions, false), + ParseOutcome::Failed(error) => return print_error(request, ®istration, &definitions, false, &error), + ParseOutcome::Delegate => return None, + ParseOutcome::Parsed { values, .. } => values, + }; + let selected_phases: Vec = match request.table.actions[request.action_index].kind { + ActionKind::Phase(phase_index) => selected_phases_from(request.table, [phase_index]), + _ if values.iter().all(|value| *value == ParameterValue::Absent || *value == ParameterValue::Flag) => { + (0..request.model.phases.len()).collect() + } + _ => match select_phases(request, &values) { + PhaseSelection::Selected(selected) => selected, + _ => return None, + }, + }; + execute(request, selected_phases, ParsedParameters { definitions, values }, None) +} diff --git a/apps/heft-native/src/cli/action_text.rs b/apps/heft-native/src/cli/action_text.rs new file mode 100644 index 0000000000..90ad3ed920 --- /dev/null +++ b/apps/heft-native/src/cli/action_text.rs @@ -0,0 +1,68 @@ +use std::borrow::Cow; + +use super::actions::{ActionEntry, ActionKind, AliasEntry, ActionTable}; +use super::model::CliModel; + +pub const CLEAN_ACTION_DOCUMENTATION: &str = "Clean the project, removing temporary task folders and specified clean paths."; + +pub fn run_documentation(watch: bool) -> &'static str { + if watch { + "Run a provided selection of Heft phases in watch mode.." + } else { + "Run a provided selection of Heft phases." + } +} + +fn phase_summary(phase_name: &str, watch: bool) -> String { + let ending: &str = if watch { ", in watch mode." } else { "." }; + format!("Runs to the {phase_name} phase, including all transitive dependencies{ending}") +} + +pub fn action_summary<'x>(model: &'x CliModel<'x>, action: &ActionEntry<'_>) -> Cow<'x, str> { + match action.kind { + ActionKind::Clean => Cow::Borrowed(CLEAN_ACTION_DOCUMENTATION), + ActionKind::Run => Cow::Borrowed(run_documentation(action.watch)), + ActionKind::Phase(phase_index) => Cow::Owned(phase_summary(model.phases[phase_index].name, action.watch)), + } +} + +pub fn action_documentation<'x>(model: &'x CliModel<'x>, action: &ActionEntry<'_>) -> Cow<'x, str> { + match action.kind { + ActionKind::Phase(phase_index) => { + let phase = &model.phases[phase_index]; + let mut documentation: String = phase_summary(phase.name, action.watch); + if let Some(description) = phase.description.filter(|description| !description.is_empty()) { + documentation.push_str(" "); + documentation.push_str(description); + } + Cow::Owned(documentation) + } + _ => action_summary(model, action), + } +} + +pub fn alias_expanded_command(table: &ActionTable<'_>, alias: &AliasEntry<'_>) -> String { + let mut expanded: String = format!("heft {}", table.actions[alias.target_index].name); + let defaults: String = alias.default_parameters.join(" "); + if !defaults.is_empty() { + expanded.push(' '); + expanded.push_str(&defaults); + } + expanded +} + +pub fn alias_summary(table: &ActionTable<'_>, alias: &AliasEntry<'_>) -> String { + format!("An alias for \"{}\".", alias_expanded_command(table, alias)) +} + +pub fn alias_documentation(table: &ActionTable<'_>, alias: &AliasEntry<'_>) -> String { + format!( + "{} For more information on the aliased command, use \"heft {} --help\".", + alias_summary(table, alias), + table.actions[alias.target_index].name + ) +} + +pub fn alias_expansion_message(table: &ActionTable<'_>, alias: &AliasEntry<'_>) -> String { + format!("The \"heft {}\" alias was expanded to \"{}\".", alias.name, alias_expanded_command(table, alias)) +} diff --git a/apps/heft-native/src/cli/actions.rs b/apps/heft-native/src/cli/actions.rs new file mode 100644 index 0000000000..1cb1b39114 --- /dev/null +++ b/apps/heft-native/src/cli/actions.rs @@ -0,0 +1,120 @@ +use std::borrow::Cow; + +use super::model::CliModel; +use super::validate::is_valid_action_name; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum ActionKind { + Clean, + Run, + Phase(usize), +} + +#[derive(Debug)] +pub struct ActionEntry<'a> { + pub name: Cow<'a, str>, + pub kind: ActionKind, + pub watch: bool, +} + +#[derive(Debug)] +pub struct AliasEntry<'a> { + pub name: &'a str, + pub target_index: usize, + pub default_parameters: &'a [&'a str], +} + +#[derive(Debug)] +pub struct ActionTable<'a> { + pub actions: Vec>, + pub aliases: Vec>, + pub phase_dependencies: Vec>, +} + +impl<'a> ActionTable<'a> { + pub fn find_action(&self, name: &str) -> Option { + self.actions.iter().position(|action| action.name == name) + } + + pub fn find_alias(&self, name: &str) -> Option<&AliasEntry<'a>> { + self.aliases.iter().find(|alias| alias.name == name) + } + + pub fn command_names(&self) -> impl Iterator { + self.actions.iter().map(|action| action.name.as_ref()).chain(self.aliases.iter().map(|alias| alias.name)) + } + + fn try_add(&mut self, name: Cow<'a, str>, kind: ActionKind, watch: bool) -> bool { + if !is_valid_action_name(&name) || self.find_action(&name).is_some() { + return false; + } + self.actions.push(ActionEntry { name, kind, watch }); + true + } +} + +fn resolve_phase_dependencies(model: &CliModel<'_>) -> Option>> { + let mut all_dependencies: Vec> = Vec::with_capacity(model.phases.len()); + for phase in &model.phases { + let mut dependencies: Vec = Vec::with_capacity(phase.dependency_names.len()); + for dependency_name in &phase.dependency_names { + let dependency_index: usize = model.find_phase_index(dependency_name)?; + if !dependencies.contains(&dependency_index) { + dependencies.push(dependency_index); + } + } + all_dependencies.push(dependencies); + } + Some(all_dependencies) +} + +pub fn build_action_table<'a>(model: &'a CliModel<'a>) -> Option> { + let mut table: ActionTable<'a> = ActionTable { + actions: Vec::with_capacity(3 + 2 * model.phases.len()), + aliases: Vec::with_capacity(model.aliases.len()), + phase_dependencies: resolve_phase_dependencies(model)?, + }; + table.try_add(Cow::Borrowed("clean"), ActionKind::Clean, false); + table.try_add(Cow::Borrowed("run"), ActionKind::Run, false); + for (phase_index, phase) in model.phases.iter().enumerate() { + if !table.try_add(Cow::Borrowed(phase.name), ActionKind::Phase(phase_index), false) { + return None; + } + } + if !table.try_add(Cow::Borrowed("run-watch"), ActionKind::Run, true) { + return None; + } + for (phase_index, phase) in model.phases.iter().enumerate() { + if !table.try_add(Cow::Owned(format!("{}-watch", phase.name)), ActionKind::Phase(phase_index), true) { + return None; + } + } + for alias in &model.aliases { + let target_index: usize = table.find_action(alias.action_name)?; + let is_duplicate: bool = table.find_action(alias.name).is_some() || table.find_alias(alias.name).is_some(); + if is_duplicate || !is_valid_action_name(alias.name) { + return None; + } + table.aliases.push(AliasEntry { name: alias.name, target_index, default_parameters: &alias.default_parameters }); + } + Some(table) +} + +pub fn selected_phases_from(table: &ActionTable<'_>, seeds: impl IntoIterator) -> Vec { + let mut selected: Vec = Vec::new(); + for seed in seeds { + if !selected.contains(&seed) { + selected.push(seed); + } + } + let mut cursor: usize = 0; + while cursor < selected.len() { + for dependency in &table.phase_dependencies[selected[cursor]] { + if !selected.contains(dependency) { + selected.push(*dependency); + } + } + cursor += 1; + } + selected +} diff --git a/apps/heft-native/src/cli/defined_parameter.rs b/apps/heft-native/src/cli/defined_parameter.rs new file mode 100644 index 0000000000..46be96e9d1 --- /dev/null +++ b/apps/heft-native/src/cli/defined_parameter.rs @@ -0,0 +1,94 @@ +use std::borrow::Cow; + +use super::model::{DefaultValue, ParameterKind}; +use super::text::{format_javascript_integer, is_javascript_whitespace, push_json_string}; + +#[derive(Clone, Debug)] +pub struct DefinedParameter<'a> { + pub kind: ParameterKind, + pub long_name: &'a str, + pub short_name: Option<&'a str>, + pub scope: Option<&'a str>, + pub scoping_group: bool, + pub required: bool, + pub argument_name: Option<&'a str>, + pub alternatives: Vec<&'a str>, + pub default_value: Option>, + pub description: Cow<'a, str>, +} + +impl<'a> DefinedParameter<'a> { + pub fn flag(long_name: &'a str, short_name: Option<&'a str>, description: Cow<'a, str>) -> Self { + DefinedParameter { + kind: ParameterKind::Flag, + long_name, + short_name, + scope: None, + scoping_group: false, + required: false, + argument_name: None, + alternatives: Vec::new(), + default_value: None, + description, + } + } + + pub fn string_list(long_name: &'a str, argument_name: &'a str, description: Cow<'a, str>, scoping: bool) -> Self { + DefinedParameter { + kind: ParameterKind::StringList, + argument_name: Some(argument_name), + scoping_group: scoping, + ..DefinedParameter::flag(long_name, None, description) + } + } + + pub fn scoped_long_name(&self) -> Option { + self.scope.map(|scope| format!("--{}:{}", scope, &self.long_name[2..])) + } + + pub fn help_text(&self) -> Option> { + let mut notes: Vec = Vec::new(); + match (self.kind, self.default_value) { + (ParameterKind::Choice, Some(DefaultValue::Text(value))) => { + notes.push(format!("The default value is \"{}\".", value)); + } + (ParameterKind::Integer, Some(DefaultValue::Number(value))) => { + notes.push(format!("The default value is {}.", format_javascript_integer(value)?)); + } + (ParameterKind::String, Some(DefaultValue::Text(value))) => { + if value.len() < 160 { + let mut note: String = String::from("The default value is "); + if !push_json_string(&mut note, value) { + return None; + } + note.push('.'); + notes.push(note); + } + } + (_, None) => {} + _ => return None, + } + if notes.is_empty() { + return Some(self.description.clone()); + } + let mut final_description: String = self.description.to_string(); + if ends_with_word_character_before_whitespace(&final_description) { + let trimmed_length: usize = final_description.trim_end_matches(is_trimmed_character).len(); + final_description.truncate(trimmed_length); + final_description.push('.'); + } + final_description.push(' '); + final_description.push_str(¬es.join(" ")); + Some(Cow::Owned(final_description)) + } +} + +fn is_trimmed_character(character: char) -> bool { + character.is_ascii() && is_javascript_whitespace(character as u8) +} + +fn ends_with_word_character_before_whitespace(text: &str) -> bool { + let trimmed: &str = text.trim_end_matches(is_trimmed_character); + let without_quote: &str = trimmed.strip_suffix('"').unwrap_or(trimmed); + without_quote.bytes().last().is_some_and(|byte| byte.is_ascii_alphanumeric()) +} diff --git a/apps/heft-native/src/cli/entry.rs b/apps/heft-native/src/cli/entry.rs new file mode 100644 index 0000000000..12d08e705b --- /dev/null +++ b/apps/heft-native/src/cli/entry.rs @@ -0,0 +1,41 @@ +use std::env; +use std::ffi::OsString; +use std::io::Write; + +use super::invocation::interpret_with_output; +use super::model::CliModel; +use super::outcome::{CliOutcome, PrintedOutput}; +use super::width::read_help_width; + +pub fn command_line_strings(command_line_arguments: &[OsString]) -> Option> { + command_line_arguments.iter().map(|argument| argument.to_str()).collect() +} + +fn is_native_output_allowed() -> bool { + env::var_os("_RUSH_REPORTER_CHILD_FD").is_none() && env::var_os("_RUSH_REPORTER_CHILD_ACK_FD").is_none() +} + +pub fn interpret_command_line_with_color<'a>( + args: &'a [&'a str], + model: &'a CliModel<'a>, + supports_color: &dyn Fn() -> bool, +) -> CliOutcome<'a> { + if !is_native_output_allowed() { + return CliOutcome::Delegate; + } + interpret_with_output(args, model, read_help_width(), Some(supports_color)) +} + +pub fn write_printed_output(output: &PrintedOutput) -> i32 { + if !output.stdout.is_empty() { + let mut stdout = std::io::stdout().lock(); + let _ = stdout.write_all(output.stdout.as_bytes()); + let _ = stdout.flush(); + } + if !output.stderr.is_empty() { + let mut stderr = std::io::stderr().lock(); + let _ = stderr.write_all(output.stderr.as_bytes()); + let _ = stderr.flush(); + } + output.exit_code +} diff --git a/apps/heft-native/src/cli/help_args.rs b/apps/heft-native/src/cli/help_args.rs new file mode 100644 index 0000000000..6f520f815f --- /dev/null +++ b/apps/heft-native/src/cli/help_args.rs @@ -0,0 +1,171 @@ +use super::help_model::{HelpAction, HelpNargs}; + +pub fn build_metavar(action: &HelpAction<'_>, default_metavar: &str) -> String { + if let Some(metavar) = action.metavar { + return metavar.to_string(); + } + if let Some(choices) = &action.choices { + let mut result: String = String::from("{"); + for (choice_index, choice) in choices.iter().enumerate() { + if choice_index > 0 { + result.push(','); + } + result.push_str(choice); + } + result.push('}'); + return result; + } + default_metavar.to_string() +} + +pub fn format_args(action: &HelpAction<'_>, default_metavar: &str) -> String { + let metavar: String = build_metavar(action, default_metavar); + match action.nargs { + HelpNargs::Single | HelpNargs::Zero => metavar, + HelpNargs::ZeroOrMore => format!("[{} [{} ...]]", metavar, metavar), + HelpNargs::Remainder => String::from("..."), + HelpNargs::Parser => format!("{} ...", metavar), + } +} + +pub fn format_action_invocation(action: &HelpAction<'_>) -> String { + if !action.is_optional() { + return build_metavar(action, &action.dest); + } + let mut result: String = String::new(); + if action.nargs == HelpNargs::Zero { + for (index, option_string) in action.option_strings.iter().enumerate() { + if index > 0 { + result.push_str(", "); + } + result.push_str(option_string); + } + return result; + } + let args_string: String = format_args(action, &action.dest.to_ascii_uppercase()); + for (index, option_string) in action.option_strings.iter().enumerate() { + if index > 0 { + result.push_str(", "); + } + result.push_str(option_string); + result.push(' '); + result.push_str(&args_string); + } + result +} + +pub fn format_actions_usage(actions: &[&HelpAction<'_>]) -> String { + let mut text: String = String::new(); + for action in actions { + if action.help.is_suppressed() { + continue; + } + let part: String = if !action.is_optional() { + format_args(action, &action.dest) + } else { + let option_string: &str = &action.option_strings[0]; + let unbracketed: String = if action.nargs == HelpNargs::Zero { + option_string.to_string() + } else { + format!("{} {}", option_string, format_args(action, &action.dest.to_ascii_uppercase())) + }; + if action.required { + unbracketed + } else { + format!("[{}]", unbracketed) + } + }; + if part.is_empty() { + continue; + } + if !text.is_empty() { + text.push(' '); + } + text.push_str(&part); + } + clean_usage_separators(&text) +} + +fn clean_usage_separators(text: &str) -> String { + let without_space_after_open: String = remove_space_after_open_bracket(text); + let without_space_before_close: String = remove_space_before_close_bracket(&without_space_after_open); + let without_empty_brackets: String = remove_empty_pair(&without_space_before_close, b'[', b']'); + let without_empty_parens: String = remove_empty_pair(&without_empty_brackets, b'(', b')'); + let unwrapped: String = unwrap_single_parenthesized_groups(&without_empty_parens); + super::text::trim_javascript_whitespace(&unwrapped).to_string() +} + +fn remove_space_after_open_bracket(text: &str) -> String { + let bytes: &[u8] = text.as_bytes(); + let mut output: Vec = Vec::with_capacity(bytes.len()); + let mut index: usize = 0; + while index < bytes.len() { + output.push(bytes[index]); + if (bytes[index] == b'[' || bytes[index] == b'(') && bytes.get(index + 1) == Some(&b' ') { + index += 2; + } else { + index += 1; + } + } + String::from_utf8(output).unwrap_or_default() +} + +fn remove_space_before_close_bracket(text: &str) -> String { + let bytes: &[u8] = text.as_bytes(); + let mut output: Vec = Vec::with_capacity(bytes.len()); + let mut index: usize = 0; + while index < bytes.len() { + if bytes[index] == b' ' && matches!(bytes.get(index + 1), Some(b']') | Some(b')')) { + output.push(bytes[index + 1]); + index += 2; + } else { + output.push(bytes[index]); + index += 1; + } + } + String::from_utf8(output).unwrap_or_default() +} + +fn remove_empty_pair(text: &str, open: u8, close: u8) -> String { + let bytes: &[u8] = text.as_bytes(); + let mut output: Vec = Vec::with_capacity(bytes.len()); + let mut index: usize = 0; + while index < bytes.len() { + if bytes[index] == open { + let mut lookahead: usize = index + 1; + while lookahead < bytes.len() && bytes[lookahead] == b' ' { + lookahead += 1; + } + if lookahead < bytes.len() && bytes[lookahead] == close { + index = lookahead + 1; + continue; + } + } + output.push(bytes[index]); + index += 1; + } + String::from_utf8(output).unwrap_or_default() +} + +fn unwrap_single_parenthesized_groups(text: &str) -> String { + let bytes: &[u8] = text.as_bytes(); + let mut output: Vec = Vec::with_capacity(bytes.len()); + let mut index: usize = 0; + while index < bytes.len() { + if bytes[index] == b'(' { + let group_end: usize = bytes[index + 1..] + .iter() + .position(|byte| *byte == b'|') + .map_or(bytes.len(), |offset| index + 1 + offset); + if let Some(offset) = bytes[index + 1..group_end].iter().rposition(|byte| *byte == b')') { + let close_index: usize = index + 1 + offset; + output.extend_from_slice(&bytes[index + 1..close_index]); + index = close_index + 1; + continue; + } + } + output.push(bytes[index]); + index += 1; + } + String::from_utf8(output).unwrap_or_default() +} diff --git a/apps/heft-native/src/cli/help_builders.rs b/apps/heft-native/src/cli/help_builders.rs new file mode 100644 index 0000000000..aefdc9b3da --- /dev/null +++ b/apps/heft-native/src/cli/help_builders.rs @@ -0,0 +1,125 @@ +use std::borrow::Cow; + +use super::defined_parameter::DefinedParameter; +use super::help_model::{bold, HelpAction, HelpGroup, HelpNargs, HelpParser, HelpText}; +use super::model::ParameterKind; +use super::registration::{Registration, RegistrationStep}; + +pub const HEFT_TOOL_DESCRIPTION: &str = "Heft is a pluggable build system designed for web projects."; +const DEBUG_DESCRIPTION: &str = "Show the full call stack if an error occurs while executing the tool"; +const UNMANAGED_DESCRIPTION: &str = "Disables the Heft version selector: When Heft is invoked via the shell path, normally it will examine the project's package.json dependencies and try to use the locally installed version of Heft. Specify \"--unmanaged\" to force the invoked version of Heft to be used. This is useful for example if you want to test a different version of Heft."; +pub const REMAINDER_DESCRIPTION: &str = "Scoped parameters. Must be prefixed with \"--\", ex. \"-- --scopedParameter foo --scopedFlag\". For more information on available scoped parameters, use \"-- --help\"."; + +pub fn root_help_parser<'a>(summaries: Vec<(Cow<'a, str>, Cow<'a, str>)>) -> HelpParser<'a> { + let subactions: Vec> = summaries + .into_iter() + .map(|(name, summary)| HelpAction { + option_strings: Vec::new(), + dest: name, + nargs: HelpNargs::Single, + metavar: None, + help: HelpText::Text(summary), + choices: None, + required: false, + subactions: Vec::new(), + }) + .collect(); + let subparsers: HelpAction<'a> = HelpAction { + option_strings: Vec::new(), + dest: Cow::Borrowed("action"), + nargs: HelpNargs::Parser, + metavar: Some(""), + help: HelpText::Absent, + choices: None, + required: false, + subactions, + }; + HelpParser { + prog: Cow::Borrowed("heft"), + description: Some(Cow::Borrowed(HEFT_TOOL_DESCRIPTION)), + epilog: Some(Cow::Owned(bold("For detailed help about a specific command, use: heft -h"))), + actions: vec![ + HelpAction::help_option(), + subparsers, + HelpAction::flag_option("--debug", DEBUG_DESCRIPTION), + HelpAction::flag_option("--unmanaged", UNMANAGED_DESCRIPTION), + ], + groups: vec![ + HelpGroup { title: Cow::Borrowed("Positional arguments"), action_indices: vec![1] }, + HelpGroup { title: Cow::Borrowed("Optional arguments"), action_indices: vec![0, 2, 3] }, + ], + } +} + +fn parameter_action<'a>(parameter: &'a DefinedParameter<'a>, option_strings: &[Cow<'a, str>]) -> Option> { + let help_text: Cow<'a, str> = parameter.help_text()?; + Some(HelpAction { + option_strings: option_strings.to_vec(), + dest: Cow::Borrowed(parameter.long_name), + nargs: if parameter.kind == ParameterKind::Flag { HelpNargs::Zero } else { HelpNargs::Single }, + metavar: parameter.argument_name, + help: HelpText::Text(help_text), + choices: if parameter.kind.has_alternatives() { Some(parameter.alternatives.clone()) } else { None }, + required: parameter.required, + subactions: Vec::new(), + }) +} + +pub struct ActionHelpText<'a> { + pub prog: Cow<'a, str>, + pub description: Cow<'a, str>, + pub epilog: Option, +} + +pub fn action_help_parser<'a>( + registration: &Registration<'a>, + parameters: &'a [DefinedParameter<'a>], + text: ActionHelpText<'a>, + has_remainder: bool, +) -> Option> { + let mut actions: Vec> = vec![HelpAction::help_option()]; + let mut positionals: Vec = Vec::new(); + let mut optionals: Vec = vec![0]; + let mut scoping: Vec = Vec::new(); + for step in ®istration.steps { + let action_index: usize = actions.len(); + match step { + RegistrationStep::Ambiguous(name) => { + actions.push(HelpAction::hidden_option(name.clone())); + optionals.push(action_index); + } + RegistrationStep::Parameter { parameter_index, option_strings } => { + let parameter: &'a DefinedParameter<'a> = ¶meters[*parameter_index]; + actions.push(parameter_action(parameter, option_strings)?); + if parameter.scoping_group { scoping.push(action_index) } else { optionals.push(action_index) } + } + } + } + if has_remainder { + positionals.push(actions.len()); + actions.push(HelpAction { + option_strings: Vec::new(), + dest: Cow::Borrowed("..."), + nargs: HelpNargs::Remainder, + metavar: Some("\"...\""), + help: HelpText::Text(Cow::Borrowed(REMAINDER_DESCRIPTION)), + choices: None, + required: true, + subactions: Vec::new(), + }); + } + let mut groups: Vec> = vec![ + HelpGroup { title: Cow::Borrowed("Positional arguments"), action_indices: positionals }, + HelpGroup { title: Cow::Borrowed("Optional arguments"), action_indices: optionals }, + ]; + if !scoping.is_empty() { + groups.push(HelpGroup { title: Cow::Borrowed("Optional scoping arguments"), action_indices: scoping }); + } + Some(HelpParser { + prog: text.prog, + description: Some(text.description), + epilog: text.epilog.map(Cow::Owned), + actions, + groups, + }) +} diff --git a/apps/heft-native/src/cli/help_format.rs b/apps/heft-native/src/cli/help_format.rs new file mode 100644 index 0000000000..2d6e3756bd --- /dev/null +++ b/apps/heft-native/src/cli/help_format.rs @@ -0,0 +1,150 @@ +use super::help_args::format_action_invocation; +use super::help_lines::{fill_help_text, for_each_help_line}; +use super::help_model::{HelpAction, HelpParser}; +use super::help_usage::format_usage; +use super::text::push_spaces; + +const MAX_HELP_POSITION: f64 = 24.0; + +fn action_invocations(action: &HelpAction<'_>) -> Vec { + if action.help.is_suppressed() { + return Vec::new(); + } + let mut invocations: Vec = Vec::with_capacity(1 + action.subactions.len()); + invocations.push(format_action_invocation(action)); + invocations.extend(action.subactions.iter().map(format_action_invocation)); + invocations +} + +fn compute_action_max_length(invocations: &[Vec]) -> f64 { + let longest: Option = invocations.iter().flatten().map(String::len).max(); + longest.map_or(0.0, |length| length as f64 + 2.0) +} + +struct ActionLayout { + width: f64, + action_max_length: f64, +} + +fn format_action(output: &mut String, action: &HelpAction<'_>, invocations: &[String], current_indent: f64, layout: &ActionLayout) { + let help_position: f64 = (layout.action_max_length + 2.0).min(MAX_HELP_POSITION); + let help_width: f64 = layout.width - help_position; + let action_width: f64 = help_position - current_indent - 2.0; + let header: &str = &invocations[0]; + let help_text: Option<&str> = action.help.visible_text(); + let mut indent_first: f64 = 0.0; + push_spaces(output, current_indent); + output.push_str(header); + match help_text { + None => output.push('\n'), + Some(_) if header.len() as f64 <= action_width => { + output.push_str(" "); + push_spaces(output, action_width - header.len() as f64); + } + Some(_) => { + output.push('\n'); + indent_first = help_position; + } + } + if let Some(text) = help_text { + for_each_help_line(text, help_width, |line_index, line| { + push_spaces(output, if line_index == 0 { indent_first } else { help_position }); + output.push_str(line); + output.push('\n'); + }); + } + for (subaction_index, subaction) in action.subactions.iter().enumerate() { + format_action(output, subaction, &invocations[1 + subaction_index..], current_indent + 2.0, layout); + } +} + +fn format_text_block(output: &mut String, text: &str, width: f64) { + fill_help_text(output, text, width, ""); + output.push_str("\n\n"); +} + +fn is_usage_text_ascii(action: &HelpAction<'_>) -> bool { + action.option_strings.iter().all(|option_string| option_string.is_ascii()) + && action.dest.is_ascii() + && action.metavar.is_none_or(str::is_ascii) + && action.choices.as_ref().is_none_or(|choices| choices.iter().all(|choice| choice.is_ascii())) +} + +fn is_action_text_ascii(action: &HelpAction<'_>) -> bool { + is_usage_text_ascii(action) + && action.help.visible_text().is_none_or(str::is_ascii) + && action.subactions.iter().all(is_action_text_ascii) +} + +fn is_parser_text_ascii(parser: &HelpParser<'_>) -> bool { + parser.prog.is_ascii() + && parser.description.as_deref().is_none_or(str::is_ascii) + && parser.epilog.as_deref().is_none_or(str::is_ascii) + && parser.actions.iter().all(is_action_text_ascii) +} + +pub fn format_help(parser: &HelpParser<'_>, width: f64) -> Option { + if !is_parser_text_ascii(parser) { + return None; + } + let invocations: Vec> = parser.actions.iter().map(action_invocations).collect(); + let layout: ActionLayout = ActionLayout { width, action_max_length: compute_action_max_length(&invocations) }; + let mut help: String = format_usage(&parser.prog, &parser.actions, width)?; + if let Some(description) = parser.description.as_deref().filter(|text| !text.is_empty()) { + format_text_block(&mut help, description, width); + } + for group in &parser.groups { + let mut section: String = String::new(); + for action_index in &group.action_indices { + let action: &HelpAction<'_> = &parser.actions[*action_index]; + if !action.help.is_suppressed() { + format_action(&mut section, action, &invocations[*action_index], 2.0, &layout); + } + } + if !section.is_empty() { + help.push('\n'); + help.push_str(&group.title); + help.push_str(":\n"); + help.push_str(§ion); + help.push('\n'); + } + } + if let Some(epilog) = parser.epilog.as_deref().filter(|text| !text.is_empty()) { + format_text_block(&mut help, epilog, width); + } + Some(finish_help(help)) +} + +fn finish_help(help: String) -> String { + let mut collapsed: String = String::with_capacity(help.len()); + let mut newline_run: usize = 0; + for character in help.chars() { + if character == '\n' { + newline_run += 1; + continue; + } + push_newlines(&mut collapsed, newline_run); + newline_run = 0; + collapsed.push(character); + } + push_newlines(&mut collapsed, newline_run); + let trimmed: &str = collapsed.trim_matches('\n'); + let mut result: String = String::with_capacity(trimmed.len() + 1); + result.push_str(trimmed); + result.push('\n'); + result +} + +fn push_newlines(output: &mut String, count: usize) { + for _ in 0..count.min(2) { + output.push('\n'); + } +} + +pub fn format_usage_only(parser: &HelpParser<'_>, width: f64) -> Option { + if !parser.prog.is_ascii() || !parser.actions.iter().all(is_usage_text_ascii) { + return None; + } + let usage: String = format_usage(&parser.prog, &parser.actions, width)?; + Some(finish_help(usage)) +} diff --git a/apps/heft-native/src/cli/help_lines.rs b/apps/heft-native/src/cli/help_lines.rs new file mode 100644 index 0000000000..b99af5703c --- /dev/null +++ b/apps/heft-native/src/cli/help_lines.rs @@ -0,0 +1,65 @@ +use super::text::{is_javascript_whitespace, javascript_substring}; + +fn is_wrap_delimiter(byte: u8) -> bool { + matches!(byte, b' ' | b'.' | b',' | b'!' | b'?') +} + +fn find_last_delimiter_index(segment: &str) -> f64 { + match segment.bytes().rposition(is_wrap_delimiter) { + Some(index) => index as f64, + None => f64::NAN, + } +} + +fn normalize_help_whitespace(text: &str) -> String { + let mut normalized: String = String::with_capacity(text.len()); + let mut pending_space: bool = false; + for character in text.chars() { + let is_whitespace: bool = matches!(character, '|') || (character.is_ascii() && is_javascript_whitespace(character as u8)); + if is_whitespace { + pending_space = !normalized.is_empty(); + } else { + if pending_space { + normalized.push(' '); + pending_space = false; + } + normalized.push(character); + } + } + normalized +} + +pub fn for_each_help_line(text: &str, width: f64, mut emit: impl FnMut(usize, &str)) { + let line: String = normalize_help_whitespace(text); + let length: f64 = line.len() as f64; + if width >= length { + emit(0, &line); + return; + } + let mut line_index: usize = 0; + let mut wrap_start: f64 = 0.0; + let mut wrap_end: f64 = width; + while wrap_end <= length { + if wrap_end != length { + let segment: &str = javascript_substring(&line, wrap_start, wrap_end); + wrap_end = wrap_start + find_last_delimiter_index(segment) + 1.0; + } + emit(line_index, javascript_substring(&line, wrap_start, wrap_end)); + line_index += 1; + wrap_start = wrap_end; + wrap_end += width; + } + if wrap_start < length { + emit(line_index, javascript_substring(&line, wrap_start, wrap_end)); + } +} + +pub fn fill_help_text(output: &mut String, text: &str, width: f64, indent: &str) { + for_each_help_line(text, width, |line_index, line| { + if line_index > 0 { + output.push('\n'); + } + output.push_str(indent); + output.push_str(line); + }); +} diff --git a/apps/heft-native/src/cli/help_model.rs b/apps/heft-native/src/cli/help_model.rs new file mode 100644 index 0000000000..52dc8aa3e3 --- /dev/null +++ b/apps/heft-native/src/cli/help_model.rs @@ -0,0 +1,111 @@ +use std::borrow::Cow; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum HelpNargs { + Single, + Zero, + ZeroOrMore, + Parser, + Remainder, +} + +#[derive(Clone, Debug)] +pub enum HelpText<'a> { + Absent, + Suppressed, + Text(Cow<'a, str>), +} + +impl<'a> HelpText<'a> { + pub fn is_suppressed(&self) -> bool { + matches!(self, HelpText::Suppressed) + } + + pub fn visible_text(&self) -> Option<&str> { + match self { + HelpText::Text(text) if !text.is_empty() => Some(text.as_ref()), + _ => None, + } + } +} + +#[derive(Clone, Debug)] +pub struct HelpAction<'a> { + pub option_strings: Vec>, + pub dest: Cow<'a, str>, + pub nargs: HelpNargs, + pub metavar: Option<&'a str>, + pub help: HelpText<'a>, + pub choices: Option>, + pub required: bool, + pub subactions: Vec>, +} + +impl<'a> HelpAction<'a> { + pub fn is_optional(&self) -> bool { + !self.option_strings.is_empty() + } + + pub fn help_option() -> HelpAction<'static> { + HelpAction { + option_strings: vec![Cow::Borrowed("-h"), Cow::Borrowed("--help")], + dest: Cow::Borrowed("==SUPPRESS=="), + nargs: HelpNargs::Zero, + metavar: None, + help: HelpText::Text(Cow::Borrowed("Show this help message and exit.")), + choices: None, + required: false, + subactions: Vec::new(), + } + } + + pub fn flag_option(option_string: &'a str, help: &'a str) -> HelpAction<'a> { + HelpAction { + option_strings: vec![Cow::Borrowed(option_string)], + dest: Cow::Borrowed(option_string), + nargs: HelpNargs::Zero, + metavar: None, + help: HelpText::Text(Cow::Borrowed(help)), + choices: None, + required: false, + subactions: Vec::new(), + } + } + + pub fn hidden_option(option_string: Cow<'a, str>) -> HelpAction<'a> { + HelpAction { + dest: option_string.clone(), + option_strings: vec![option_string], + nargs: HelpNargs::ZeroOrMore, + metavar: None, + help: HelpText::Suppressed, + choices: None, + required: false, + subactions: Vec::new(), + } + } +} + +pub struct HelpGroup<'a> { + pub title: Cow<'a, str>, + pub action_indices: Vec, +} + +pub struct HelpParser<'a> { + pub prog: Cow<'a, str>, + pub description: Option>, + pub epilog: Option>, + pub actions: Vec>, + pub groups: Vec>, +} + +pub const BOLD_START: &str = "\u{1b}[1m"; +pub const BOLD_END: &str = "\u{1b}[22m"; + +pub fn bold(text: &str) -> String { + let mut output: String = String::with_capacity(text.len() + BOLD_START.len() + BOLD_END.len()); + output.push_str(BOLD_START); + output.push_str(text); + output.push_str(BOLD_END); + output +} diff --git a/apps/heft-native/src/cli/help_usage.rs b/apps/heft-native/src/cli/help_usage.rs new file mode 100644 index 0000000000..1af232ee37 --- /dev/null +++ b/apps/heft-native/src/cli/help_usage.rs @@ -0,0 +1,110 @@ +use super::help_args::format_actions_usage; +use super::help_model::HelpAction; + +fn find_closing(bytes: &[u8], start: usize, close: u8) -> Option { + let mut index: usize = start + 1; + while index < bytes.len() { + if bytes[index] == close { + let mut end: usize = index + 1; + while end < bytes.len() && bytes[end] == close { + end += 1; + } + return Some(end); + } + index += 1; + } + None +} + +pub fn split_usage_parts(text: &str) -> Vec<&str> { + let bytes: &[u8] = text.as_bytes(); + let mut parts: Vec<&str> = Vec::new(); + let mut index: usize = 0; + while index < bytes.len() { + if super::text::is_javascript_whitespace(bytes[index]) { + index += 1; + continue; + } + let grouped_end: Option = match bytes[index] { + b'(' => find_closing(bytes, index, b')'), + b'[' => find_closing(bytes, index, b']'), + _ => None, + }; + let end: usize = grouped_end.unwrap_or_else(|| { + let mut end: usize = index; + while end < bytes.len() && !super::text::is_javascript_whitespace(bytes[end]) { + end += 1; + } + end + }); + parts.push(&text[index..end]); + index = end; + } + parts +} + +fn collect_wrapped_lines(parts: &[&str], indent: &str, prefix: Option<&str>, text_width: f64) -> Vec { + let mut lines: Vec = Vec::new(); + let mut line: Vec<&str> = Vec::new(); + let mut line_length: f64 = prefix.map_or(indent.len(), str::len) as f64 - 1.0; + for part in parts { + if line_length + 1.0 + part.len() as f64 > text_width { + lines.push(format!("{}{}", indent, line.join(" "))); + line.clear(); + line_length = indent.len() as f64 - 1.0; + } + line.push(part); + line_length += part.len() as f64 + 1.0; + } + lines.push(format!("{}{}", indent, line.join(" "))); + if prefix.is_some() { + let first: String = lines[0].get(indent.len()..).unwrap_or("").to_string(); + lines[0] = first; + } + lines +} + +pub fn format_usage(prog: &str, actions: &[HelpAction<'_>], width: f64) -> Option { + let prefix: &str = "usage: "; + let optionals: Vec<&HelpAction<'_>> = actions.iter().filter(|action| action.is_optional()).collect(); + let positionals: Vec<&HelpAction<'_>> = actions.iter().filter(|action| !action.is_optional()).collect(); + let mut ordered: Vec<&HelpAction<'_>> = Vec::with_capacity(actions.len()); + ordered.extend(optionals.iter().copied()); + ordered.extend(positionals.iter().copied()); + let action_usage: String = format_actions_usage(&ordered); + let mut usage: String = if actions.is_empty() { prog.to_string() } else { format!("{} {}", prog, action_usage) }; + let text_width: f64 = width; + if !actions.is_empty() && (prefix.len() + usage.len()) as f64 > text_width { + let optional_usage: String = format_actions_usage(&optionals); + let positional_usage: String = format_actions_usage(&positionals); + let optional_parts: Vec<&str> = split_usage_parts(&optional_usage); + let positional_parts: Vec<&str> = split_usage_parts(&positional_usage); + if optional_parts.is_empty() || optional_parts.join(" ") != optional_usage || positional_parts.join(" ") != positional_usage { + return None; + } + let lines: Vec = if (prefix.len() + prog.len()) as f64 <= 0.75 * text_width { + let indent: String = " ".repeat(prefix.len() + prog.len() + 1); + let mut first_parts: Vec<&str> = Vec::with_capacity(optional_parts.len() + 1); + first_parts.push(prog); + first_parts.extend(optional_parts.iter().copied()); + let mut lines: Vec = collect_wrapped_lines(&first_parts, &indent, Some(prefix), text_width); + lines.extend(collect_wrapped_lines(&positional_parts, &indent, None, text_width)); + lines + } else { + let indent: String = " ".repeat(prefix.len()); + let mut all_parts: Vec<&str> = optional_parts.clone(); + all_parts.extend(positional_parts.iter().copied()); + let mut lines: Vec = collect_wrapped_lines(&all_parts, &indent, None, text_width); + if lines.len() > 1 { + lines = collect_wrapped_lines(&optional_parts, &indent, None, text_width); + lines.extend(collect_wrapped_lines(&positional_parts, &indent, None, text_width)); + } + let mut with_prog: Vec = Vec::with_capacity(lines.len() + 1); + with_prog.push(prog.to_string()); + with_prog.extend(lines); + with_prog + }; + usage = lines.join("\n"); + } + Some(format!("{}{}\n\n", prefix, usage)) +} diff --git a/apps/heft-native/src/cli/invocation.rs b/apps/heft-native/src/cli/invocation.rs new file mode 100644 index 0000000000..8d8b100999 --- /dev/null +++ b/apps/heft-native/src/cli/invocation.rs @@ -0,0 +1,113 @@ +use std::borrow::Cow; + +use super::action_invocation::{invoke_action, phase_action_parameters, ActionRequest}; +use super::action_text::{action_summary, alias_summary}; +use super::actions::{build_action_table, ActionTable, AliasEntry}; +use super::help_builders::root_help_parser; +use super::model::CliModel; +use super::outcome::CliOutcome; +use super::parameters::ROOT_PARAMETER_NAMES; +use super::registration::try_register_parameters; +use super::render::{help_output, invalid_command_message, usage_error_output}; +use super::validate::is_valid_long_name; + +const ROOT_OPTION_STRINGS: [&str; 4] = ["-h", "--help", "--debug", "--unmanaged"]; +const TOO_FEW_ARGUMENTS: &str = "too few arguments"; + +#[cfg(test)] +pub fn interpret_with_width<'a>(args: &'a [&'a str], model: &'a CliModel<'a>, width: Option) -> CliOutcome<'a> { + interpret_with_output(args, model, width, None) +} + +pub fn interpret_with_output<'a>( + args: &'a [&'a str], + model: &'a CliModel<'a>, + width: Option, + supports_color: Option<&dyn Fn() -> bool>, +) -> CliOutcome<'a> { + match interpret(args, model, width, supports_color) { + Some(CliOutcome::Print(mut output)) if is_debug_enabled(args) && !model.debug_messages.is_empty() => { + let mut stdout: String = String::with_capacity(output.stdout.len() + 256); + for message in &model.debug_messages { + stdout.push_str(message); + stdout.push('\n'); + } + stdout.push_str(&output.stdout); + output.stdout = stdout; + CliOutcome::Print(output) + } + Some(outcome) => outcome, + None => CliOutcome::Delegate, + } +} + +fn is_debug_enabled(args: &[&str]) -> bool { + args.iter().take_while(|arg| arg.starts_with('-')).any(|arg| *arg == "--debug") +} + +fn root_summaries<'x>(model: &'x CliModel<'x>, table: &'x ActionTable<'x>) -> Vec<(Cow<'x, str>, Cow<'x, str>)> { + let mut summaries: Vec<(Cow<'x, str>, Cow<'x, str>)> = Vec::with_capacity(table.actions.len() + table.aliases.len()); + for action in &table.actions { + summaries.push((Cow::Borrowed(action.name.as_ref()), action_summary(model, action))); + } + for alias in &table.aliases { + summaries.push((Cow::Borrowed(alias.name), Cow::Owned(alias_summary(table, alias)))); + } + summaries +} + +fn print_root_help<'a>(model: &CliModel<'_>, table: &ActionTable<'_>, width: Option) -> Option> { + let parser = root_help_parser(root_summaries(model, table)); + Some(CliOutcome::Print(help_output(&parser, width, 1)?)) +} + +fn root_usage_error<'a>(message: &str, width: Option) -> Option> { + let parser = root_help_parser(Vec::new()); + Some(CliOutcome::Print(usage_error_output(&parser, width, message)?)) +} + +fn is_unknown_tool_option(arg: &str) -> bool { + is_valid_long_name(arg) && !ROOT_OPTION_STRINGS.iter().any(|option_string| option_string.starts_with(arg)) +} + +fn interpret<'a>(args: &'a [&'a str], model: &'a CliModel<'a>, width: Option, supports_color: Option<&dyn Fn() -> bool>) -> Option> { + let table: ActionTable<'a> = build_action_table(model)?; + for phase_index in 0..model.phases.len() { + let parameters = phase_action_parameters(model, &table, phase_index, false)?; + try_register_parameters(¶meters, &ROOT_PARAMETER_NAMES)?; + } + let action_position: Option = args.iter().position(|arg| !arg.starts_with('-')); + let mut has_unknown_tool_option: bool = false; + for arg in &args[..action_position.unwrap_or(args.len())] { + match *arg { + "-h" | "--help" => return print_root_help(model, &table, width), + "--debug" | "--unmanaged" => {} + "--" => return root_usage_error(&invalid_command_message("--", table.command_names()), width), + _ if is_unknown_tool_option(arg) => has_unknown_tool_option = true, + _ => return None, + } + } + let Some(action_position) = action_position else { + return if args.is_empty() { print_root_help(model, &table, width) } else { root_usage_error(TOO_FEW_ARGUMENTS, width) }; + }; + let command_name: &'a str = args[action_position]; + if has_unknown_tool_option || command_name.is_empty() { + return None; + } + let alias: Option<&AliasEntry<'a>> = table.find_alias(command_name); + let action_index: usize = match alias.map(|alias| alias.target_index).or_else(|| table.find_action(command_name)) { + Some(action_index) => action_index, + None => return root_usage_error(&invalid_command_message(command_name, table.command_names()), width), + }; + invoke_action(ActionRequest { + model, + table: &table, + command_name, + alias, + action_index, + tool_args: &args[..action_position], + rest: &args[action_position + 1..], + width, + supports_color, + }) +} diff --git a/apps/heft-native/src/cli/mod.rs b/apps/heft-native/src/cli/mod.rs new file mode 100644 index 0000000000..07cc4fb0de --- /dev/null +++ b/apps/heft-native/src/cli/mod.rs @@ -0,0 +1,34 @@ +mod action_invocation; +mod action_text; +pub mod actions; +pub mod defined_parameter; +pub mod entry; +mod help_args; +mod help_builders; +mod help_format; +mod help_lines; +mod help_model; +mod help_usage; +mod invocation; +pub mod model; +pub mod outcome; +mod parameters; +mod plan_command; +pub mod parse; +mod phase_selection; +mod registration; +mod render; +mod run_invocation; +mod text; +mod validate; +mod width; + +#[allow(unused_imports)] +pub use self::{actions::ActionKind, defined_parameter::DefinedParameter, parse::ParameterValue}; + +#[cfg(test)] +mod test_harness; +#[cfg(test)] +mod tests_cli; +#[cfg(test)] +mod test_model; diff --git a/apps/heft-native/src/cli/model.rs b/apps/heft-native/src/cli/model.rs new file mode 100644 index 0000000000..5fee4ac01e --- /dev/null +++ b/apps/heft-native/src/cli/model.rs @@ -0,0 +1,89 @@ +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum ParameterKind { + Flag, + String, + StringList, + Integer, + IntegerList, + Choice, + ChoiceList, +} + +impl ParameterKind { + pub fn takes_argument(self) -> bool { + !matches!(self, ParameterKind::Flag) + } + + pub fn is_list(self) -> bool { + matches!( + self, + ParameterKind::StringList | ParameterKind::IntegerList | ParameterKind::ChoiceList + ) + } + + pub fn has_alternatives(self) -> bool { + matches!(self, ParameterKind::Choice | ParameterKind::ChoiceList) + } + + pub fn is_integer(self) -> bool { + matches!(self, ParameterKind::Integer | ParameterKind::IntegerList) + } +} + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum DefaultValue<'a> { + Text(&'a str), + Number(f64), +} + +#[derive(Clone, Debug)] +pub struct PluginParameterDefinition<'a> { + pub kind: ParameterKind, + pub long_name: &'a str, + pub short_name: Option<&'a str>, + pub description: &'a str, + pub required: bool, + pub argument_name: Option<&'a str>, + pub alternatives: Vec<&'a str>, + pub default_value: Option>, +} + +#[derive(Clone, Debug)] +pub struct PluginModel<'a> { + #[allow(dead_code)] + pub plugin_name: &'a str, + #[allow(dead_code)] + pub package_name: &'a str, + pub parameter_scope: &'a str, + pub parameters: Vec>, +} + +#[derive(Clone, Debug)] +pub struct PhaseModel<'a> { + pub name: &'a str, + pub description: Option<&'a str>, + pub dependency_names: Vec<&'a str>, + pub task_plugin_indices: Vec, +} + +#[derive(Clone, Debug)] +pub struct AliasModel<'a> { + pub name: &'a str, + pub action_name: &'a str, + pub default_parameters: Vec<&'a str>, +} + +#[derive(Clone, Debug, Default)] +pub struct CliModel<'a> { + pub phases: Vec>, + pub aliases: Vec>, + pub lifecycle_plugin_indices: Vec, + pub plugins: Vec>, + pub debug_messages: Vec<&'a str>, +} + +impl<'a> CliModel<'a> { + pub fn find_phase_index(&self, phase_name: &str) -> Option { + self.phases.iter().position(|phase| phase.name == phase_name) + } +} diff --git a/apps/heft-native/src/cli/outcome.rs b/apps/heft-native/src/cli/outcome.rs new file mode 100644 index 0000000000..ed4202a13f --- /dev/null +++ b/apps/heft-native/src/cli/outcome.rs @@ -0,0 +1,39 @@ +use super::actions::ActionKind; +use super::defined_parameter::DefinedParameter; +use super::parse::ParameterValue; + +#[derive(Debug, PartialEq, Eq)] +pub struct PrintedOutput { + pub stdout: String, + pub stderr: String, + pub exit_code: i32, +} + +#[derive(Debug)] +pub struct ParsedParameters<'a> { + pub definitions: Vec>, + pub values: Vec>, +} + +#[derive(Debug)] +pub struct ParsedCommand<'a> { + pub command_name: &'a str, + pub unaliased_command_name: String, + pub action_kind: ActionKind, + pub watch: bool, + pub debug: bool, + pub alias_expansion_message: Option, + pub phase_name: Option<&'a str>, + pub selected_phases: Vec, + pub parameters: ParsedParameters<'a>, + pub scoped_parameters: Option>, + pub remainder: Vec<&'a str>, + pub parsed_like_v2_lean_parser: bool, +} + +#[derive(Debug)] +pub enum CliOutcome<'a> { + Print(PrintedOutput), + Execute(Box>), + Delegate, +} diff --git a/apps/heft-native/src/cli/parameters.rs b/apps/heft-native/src/cli/parameters.rs new file mode 100644 index 0000000000..986317a7b2 --- /dev/null +++ b/apps/heft-native/src/cli/parameters.rs @@ -0,0 +1,103 @@ +use std::borrow::Cow; + +use super::defined_parameter::DefinedParameter; +use super::model::{CliModel, ParameterKind, PluginParameterDefinition}; +use super::validate::is_valid_definition; + +pub const ROOT_PARAMETER_NAMES: [Cow<'static, str>; 2] = [Cow::Borrowed("--debug"), Cow::Borrowed("--unmanaged")]; + +const VERBOSE_DESCRIPTION: &str = "If specified, log information useful for debugging."; +const PRODUCTION_DESCRIPTION: &str = "If specified, run Heft in production mode."; +const LOCALES_DESCRIPTION: &str = "Use the specified locale for this run, if applicable."; +const CLEAN_DESCRIPTION: &str = + "If specified, clean the outputs at the beginning of the lifecycle and before running each phase."; +const CLEAN_WATCH_DESCRIPTION: &str = "If specified, clean the outputs at the beginning of the lifecycle and before running each phase. Cleaning will only be performed once for the lifecycle and each phase, and further incremental runs will not be cleaned for the duration of execution."; + +pub fn verbose_parameter() -> DefinedParameter<'static> { + DefinedParameter::flag("--verbose", Some("-v"), Cow::Borrowed(VERBOSE_DESCRIPTION)) +} + +pub fn push_builtin_parameters(parameters: &mut Vec>, watch: bool) { + parameters.push(verbose_parameter()); + parameters.push(DefinedParameter::flag("--production", None, Cow::Borrowed(PRODUCTION_DESCRIPTION))); + parameters.push(DefinedParameter::string_list("--locales", "LOCALE", Cow::Borrowed(LOCALES_DESCRIPTION), false)); + let clean_description: &'static str = if watch { CLEAN_WATCH_DESCRIPTION } else { CLEAN_DESCRIPTION }; + parameters.push(DefinedParameter::flag("--clean", None, Cow::Borrowed(clean_description))); +} + +pub fn push_scoping_parameters(parameters: &mut Vec>, action_name: &str) { + let to_description: String = format!("The phase to {action_name} to, including all transitive dependencies."); + let to_except_description: String = + format!("The phase to {action_name} to (but not include), including all transitive dependencies."); + let only_description: String = format!("The phase to {action_name}."); + parameters.push(DefinedParameter::string_list("--to", "PHASE", Cow::Owned(to_description), true)); + parameters.push(DefinedParameter::string_list("--to-except", "PHASE", Cow::Owned(to_except_description), true)); + parameters.push(DefinedParameter::string_list("--only", "PHASE", Cow::Owned(only_description), true)); +} + +fn push_unique(indices: &mut Vec, index: usize) { + if !indices.contains(&index) { + indices.push(index); + } +} + +fn unique_alternatives<'a>(alternatives: &[&'a str]) -> Vec<&'a str> { + let mut unique: Vec<&'a str> = Vec::with_capacity(alternatives.len()); + for alternative in alternatives { + if !unique.contains(alternative) { + unique.push(alternative); + } + } + unique +} + +fn define_plugin_parameter<'a>(definition: &'a PluginParameterDefinition<'a>, scope: &'a str) -> DefinedParameter<'a> { + let kind: ParameterKind = definition.kind; + let takes_argument_name: bool = kind.takes_argument() && !kind.has_alternatives(); + let takes_default: bool = matches!(kind, ParameterKind::Choice | ParameterKind::Integer | ParameterKind::String); + DefinedParameter { + kind, + long_name: definition.long_name, + short_name: definition.short_name.filter(|short_name| !short_name.is_empty()), + scope: Some(scope), + scoping_group: false, + required: definition.required, + argument_name: if takes_argument_name { definition.argument_name } else { None }, + alternatives: if kind.has_alternatives() { unique_alternatives(&definition.alternatives) } else { Vec::new() }, + default_value: if takes_default { definition.default_value } else { None }, + description: Cow::Borrowed(definition.description), + } +} + +pub fn push_plugin_parameters<'a>( + parameters: &mut Vec>, + model: &'a CliModel<'a>, + selected_phases: &[usize], +) -> Option<()> { + let mut plugin_indices: Vec = Vec::new(); + for plugin_index in &model.lifecycle_plugin_indices { + push_unique(&mut plugin_indices, *plugin_index); + } + for phase_index in selected_phases { + for plugin_index in &model.phases[*phase_index].task_plugin_indices { + push_unique(&mut plugin_indices, *plugin_index); + } + } + let mut plugins_by_scope: Vec<(&str, usize)> = Vec::with_capacity(plugin_indices.len()); + for plugin_index in plugin_indices { + let plugin = model.plugins.get(plugin_index)?; + match plugins_by_scope.iter().find(|(scope, _)| *scope == plugin.parameter_scope) { + Some((_, existing_index)) if *existing_index != plugin_index => return None, + Some(_) => {} + None => plugins_by_scope.push((plugin.parameter_scope, plugin_index)), + } + for definition in &plugin.parameters { + let parameter: DefinedParameter<'a> = define_plugin_parameter(definition, plugin.parameter_scope); + if !is_valid_definition(¶meter) { + return None; + } + parameters.push(parameter); + } + } + Some(()) +} diff --git a/apps/heft-native/src/cli/parse.rs b/apps/heft-native/src/cli/parse.rs new file mode 100644 index 0000000000..8e0092e36e --- /dev/null +++ b/apps/heft-native/src/cli/parse.rs @@ -0,0 +1,173 @@ +use super::defined_parameter::DefinedParameter; +use super::model::ParameterKind; +use super::registration::{OptionTarget, Registration}; +use super::text::is_javascript_whitespace; +use super::validate::is_valid_long_name; + +#[derive(Clone, Debug, PartialEq)] +pub enum ParameterValue<'s> { + Absent, + Flag, + Text(&'s str), + Integer(i64), + TextList(Vec<&'s str>), + IntegerList(Vec), +} + +#[derive(Debug, PartialEq)] +pub enum ArgumentError<'s> { + ExpectedOneArgument(usize), + InvalidChoice(usize, &'s str), + InvalidInteger(usize, &'s str), + Required(usize), + Unrecognized(Vec<&'s str>), + Ambiguous(usize), +} + +#[derive(Debug, PartialEq)] +pub enum ParseOutcome<'s> { + Parsed { values: Vec>, remainder_start: Option }, + Help, + Failed(ArgumentError<'s>), + Delegate, +} + +enum IntegerCheck { + Valid(i64), + Invalid, + Unknown, +} + +fn check_integer(text: &str) -> IntegerCheck { + let bytes: &[u8] = text.as_bytes(); + if !bytes.is_empty() && bytes.len() <= 15 && bytes.iter().all(u8::is_ascii_digit) { + return text.parse::().map_or(IntegerCheck::Unknown, IntegerCheck::Valid); + } + match bytes.first() { + Some(first) if first.is_ascii() && !first.is_ascii_digit() && !is_javascript_whitespace(*first) && *first != b'+' && *first != b'-' => { + IntegerCheck::Invalid + } + _ => IntegerCheck::Unknown, + } +} + +enum Consumed<'s> { + Continue, + Stop(ParseOutcome<'s>), +} + +fn store_value<'s>(parameter_index: usize, parameter: &DefinedParameter<'_>, raw: &'s str, values: &mut [ParameterValue<'s>]) -> Consumed<'s> { + let slot: &mut ParameterValue<'s> = &mut values[parameter_index]; + let integer: Option = if parameter.kind.is_integer() { + match check_integer(raw) { + IntegerCheck::Valid(value) => Some(value), + IntegerCheck::Invalid => return Consumed::Stop(ParseOutcome::Failed(ArgumentError::InvalidInteger(parameter_index, raw))), + IntegerCheck::Unknown => return Consumed::Stop(ParseOutcome::Delegate), + } + } else { + None + }; + if parameter.kind.has_alternatives() && !parameter.alternatives.contains(&raw) { + return Consumed::Stop(ParseOutcome::Failed(ArgumentError::InvalidChoice(parameter_index, raw))); + } + match (parameter.kind.is_list(), &mut *slot, integer) { + (false, ParameterValue::Absent, Some(value)) => *slot = ParameterValue::Integer(value), + (false, ParameterValue::Absent, None) => *slot = ParameterValue::Text(raw), + (true, ParameterValue::Absent, Some(value)) => *slot = ParameterValue::IntegerList(vec![value]), + (true, ParameterValue::Absent, None) => *slot = ParameterValue::TextList(vec![raw]), + (true, ParameterValue::IntegerList(list), Some(value)) => list.push(value), + (true, ParameterValue::TextList(list), None) => list.push(raw), + _ => return Consumed::Stop(ParseOutcome::Delegate), + } + Consumed::Continue +} + +pub fn parse_arguments<'s>( + registration: &Registration, + parameters: &[DefinedParameter<'_>], + args: &[&'s str], + allow_remainder: bool, +) -> ParseOutcome<'s> { + let mut values: Vec> = vec![ParameterValue::Absent; parameters.len()]; + let mut extras: Vec<&'s str> = Vec::new(); + let mut remainder_start: Option = None; + let mut first_ambiguous_step: Option = None; + let mut index: usize = 0; + while index < args.len() { + let token: &'s str = args[index]; + if allow_remainder && token == "--" { + remainder_start = Some(index); + break; + } + index += 1; + match registration.find_target(token) { + Some(OptionTarget::Help) => return ParseOutcome::Help, + Some(OptionTarget::Ambiguous(step_index)) => { + first_ambiguous_step = Some(first_ambiguous_step.map_or(step_index, |first| first.min(step_index))); + while index < args.len() && !args[index].starts_with('-') { + if args[index].is_empty() { + return ParseOutcome::Delegate; + } + index += 1; + } + } + Some(OptionTarget::Parameter(parameter_index)) => { + if registration.poisoned_parameters.contains(¶meter_index) { + return ParseOutcome::Delegate; + } + let parameter: &DefinedParameter<'_> = ¶meters[parameter_index]; + if parameter.kind == ParameterKind::Flag { + values[parameter_index] = ParameterValue::Flag; + continue; + } + let raw: &'s str = match args.get(index) { + None => return ParseOutcome::Failed(ArgumentError::ExpectedOneArgument(parameter_index)), + Some(raw) if raw.starts_with('-') || raw.is_empty() => return ParseOutcome::Delegate, + Some(raw) => raw, + }; + index += 1; + if let Consumed::Stop(outcome) = store_value(parameter_index, parameter, raw, &mut values) { + return outcome; + } + } + None if token.starts_with('-') && token.contains('=') => { + let Some((name, value)) = token.split_once('=') else { + return ParseOutcome::Delegate; + }; + let Some(OptionTarget::Parameter(parameter_index)) = registration.find_target(name) else { + return ParseOutcome::Delegate; + }; + let parameter: &DefinedParameter<'_> = ¶meters[parameter_index]; + let is_poisoned: bool = registration.poisoned_parameters.contains(¶meter_index); + if is_poisoned || parameter.kind == ParameterKind::Flag || value.is_empty() { + return ParseOutcome::Delegate; + } + if let Consumed::Stop(outcome) = store_value(parameter_index, parameter, value, &mut values) { + return outcome; + } + } + None if token.starts_with('-') => { + if !is_valid_long_name(token) || registration.is_prefix_of_any_option(token) { + return ParseOutcome::Delegate; + } + extras.push(token); + } + None if token.is_empty() => return ParseOutcome::Delegate, + None if allow_remainder => { + remainder_start = Some(index - 1); + break; + } + None => extras.push(token), + } + } + if let Some(missing_index) = (0..parameters.len()).find(|index| parameters[*index].required && values[*index] == ParameterValue::Absent) { + return ParseOutcome::Failed(ArgumentError::Required(missing_index)); + } + if !extras.is_empty() { + return ParseOutcome::Failed(ArgumentError::Unrecognized(extras)); + } + if let Some(step_index) = first_ambiguous_step { + return ParseOutcome::Failed(ArgumentError::Ambiguous(step_index)); + } + ParseOutcome::Parsed { values, remainder_start } +} diff --git a/apps/heft-native/src/cli/phase_selection.rs b/apps/heft-native/src/cli/phase_selection.rs new file mode 100644 index 0000000000..d2c6cd503d --- /dev/null +++ b/apps/heft-native/src/cli/phase_selection.rs @@ -0,0 +1,61 @@ +use super::action_invocation::ActionRequest; +use super::actions::selected_phases_from; +use super::defined_parameter::DefinedParameter; +use super::parse::ParameterValue; + +pub fn phase_names<'v, 'a>(values: &'v [ParameterValue<'a>], index: usize) -> &'v [&'a str] { + match &values[index] { + ParameterValue::TextList(names) => names, + _ => &[], + } +} + +pub enum PhaseSelection { + UnknownPhase, + Empty, + Selected(Vec), +} + +pub const NO_PHASES_SELECTED_ERROR: &str = "\n\u{1b}[31mError: No phases were selected. Provide at least one phase to the \"--to\", \"--to-except\", or \"--only\" parameters.\u{1b}[39m\n"; + +pub fn select_phases(request: &ActionRequest<'_, '_>, values: &[ParameterValue<'_>]) -> PhaseSelection { + let resolve = |index: usize| -> Option> { + phase_names(values, index).iter().map(|name| request.model.find_phase_index(name)).collect() + }; + let (Some(to), Some(to_except), Some(only)) = (resolve(0), resolve(1), resolve(2)) else { + return PhaseSelection::UnknownPhase; + }; + let mut selected: Vec = selected_phases_from(request.table, to); + let mut except_dependencies: Vec = Vec::new(); + for phase_index in to_except { + except_dependencies.extend(request.table.phase_dependencies[phase_index].iter().copied()); + } + for phase_index in selected_phases_from(request.table, except_dependencies).into_iter().chain(only) { + if !selected.contains(&phase_index) { + selected.push(phase_index); + } + } + if selected.is_empty() { PhaseSelection::Empty } else { PhaseSelection::Selected(selected) } +} + +fn is_json_stringify_verbatim(text: &str) -> bool { + !text.chars().any(|character| character < ' ' || matches!(character, '"' | '\\' | '\u{2028}' | '\u{2029}')) +} + +pub fn unknown_phase_error(request: &ActionRequest<'_, '_>, definitions: &[DefinedParameter<'_>], values: &[ParameterValue<'_>]) -> Option { + let supports_color = request.supports_color?; + for index in [2, 0, 1] { + for name in phase_names(values, index) { + if request.model.find_phase_index(name).is_some() { + continue; + } + if !is_json_stringify_verbatim(name) { + return None; + } + let long_name: &str = definitions[index].long_name; + let message: String = format!("The phase name \"{name}\" passed to \"{long_name}\" does not exist in heft.json."); + return Some(if supports_color() { format!("\u{1b}[31m{message}\u{1b}[39m\n") } else { format!("{message}\n") }); + } + } + None +} diff --git a/apps/heft-native/src/cli/plan_command.rs b/apps/heft-native/src/cli/plan_command.rs new file mode 100644 index 0000000000..7b54bac023 --- /dev/null +++ b/apps/heft-native/src/cli/plan_command.rs @@ -0,0 +1,98 @@ +use std::fmt::Write; + +use crate::json::write_json_string_for_javascript; + +use super::actions::ActionKind; +use super::outcome::{ParsedCommand, ParsedParameters}; +use super::parse::ParameterValue; + +fn write_string(output: &mut String, text: &str) { + let _ = write_json_string_for_javascript(text, output); +} + +fn write_value(output: &mut String, value: &ParameterValue<'_>) { + match value { + ParameterValue::Absent => output.push_str("null"), + ParameterValue::Flag => output.push_str("true"), + ParameterValue::Text(text) => write_string(output, text), + ParameterValue::Integer(number) => { + let _ = write!(output, "{number}"); + } + ParameterValue::TextList(items) => { + output.push('['); + for (index, item) in items.iter().enumerate() { + if index > 0 { + output.push(','); + } + write_string(output, item); + } + output.push(']'); + } + ParameterValue::IntegerList(numbers) => { + output.push('['); + for (index, number) in numbers.iter().enumerate() { + if index > 0 { + output.push(','); + } + let _ = write!(output, "{number}"); + } + output.push(']'); + } + } +} + +fn write_values(output: &mut String, parameters: &ParsedParameters<'_>) { + output.push('['); + for (index, (definition, value)) in parameters.definitions.iter().zip(¶meters.values).enumerate() { + if index > 0 { + output.push(','); + } + output.push('['); + match definition.scoped_long_name() { + Some(scoped_long_name) => write_string(output, &scoped_long_name), + None => write_string(output, definition.long_name), + } + output.push(','); + write_value(output, value); + output.push(']'); + } + output.push(']'); +} + +impl ParsedCommand<'_> { + pub fn write_plan_command(&self, output: &mut String) -> bool { + if !self.parsed_like_v2_lean_parser { + return false; + } + let scoped_parameters: Option<&ParsedParameters<'_>> = match (self.action_kind, self.phase_name) { + (ActionKind::Phase(_), Some(_)) => None, + (ActionKind::Run, _) if self.scoped_parameters.is_some() => self.scoped_parameters.as_ref(), + _ => return false, + }; + output.push_str("{\"commandName\":"); + write_string(output, self.command_name); + output.push_str(",\"unaliasedCommandName\":"); + write_string(output, &self.unaliased_command_name); + match self.phase_name { + Some(phase_name) if scoped_parameters.is_none() => { + output.push_str(",\"actionKind\":\"phase\",\"phaseName\":"); + write_string(output, phase_name); + } + _ => output.push_str(",\"actionKind\":\"run\""), + } + output.push_str(if self.watch { ",\"watch\":true,\"values\":" } else { ",\"watch\":false,\"values\":" }); + write_values(output, &self.parameters); + if let Some(scoped_parameters) = scoped_parameters { + output.push_str(",\"remainder\":"); + write_value(output, &ParameterValue::TextList(self.remainder.clone())); + output.push_str(",\"scopedValues\":"); + write_values(output, scoped_parameters); + } + if let Some(message) = &self.alias_expansion_message { + output.push_str(",\"aliasExpansionMessage\":"); + write_string(output, message); + } + output.push('}'); + true + } +} diff --git a/apps/heft-native/src/cli/registration.rs b/apps/heft-native/src/cli/registration.rs new file mode 100644 index 0000000000..bafe2a4255 --- /dev/null +++ b/apps/heft-native/src/cli/registration.rs @@ -0,0 +1,141 @@ +use std::borrow::Cow; + +use super::defined_parameter::DefinedParameter; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum OptionTarget { + Help, + Ambiguous(usize), + Parameter(usize), +} + +#[derive(Debug)] +pub enum RegistrationStep<'a> { + Parameter { parameter_index: usize, option_strings: Vec> }, + Ambiguous(Cow<'a, str>), +} + +#[derive(Debug, Default)] +pub struct Registration<'a> { + pub poisoned_parameters: Vec, + pub steps: Vec>, +} + +const HELP_OPTION_STRINGS: [&str; 2] = ["-h", "--help"]; + +impl<'a> Registration<'a> { + fn find_target_matching(&self, matches: impl Fn(&str) -> bool) -> Option { + if HELP_OPTION_STRINGS.iter().any(|option_string| matches(option_string)) { + return Some(OptionTarget::Help); + } + for (step_index, step) in self.steps.iter().enumerate() { + match step { + RegistrationStep::Parameter { parameter_index, option_strings } => { + if option_strings.iter().any(|option_string| matches(option_string)) { + return Some(OptionTarget::Parameter(*parameter_index)); + } + } + RegistrationStep::Ambiguous(name) if matches(name) => return Some(OptionTarget::Ambiguous(step_index)), + RegistrationStep::Ambiguous(_) => {} + } + } + None + } + + pub fn find_target(&self, option_string: &str) -> Option { + self.find_target_matching(|registered| registered == option_string) + } + + pub fn is_prefix_of_any_option(&self, text: &str) -> bool { + self.find_target_matching(|registered| registered.starts_with(text)).is_some() + } + + pub fn registered_names(&self) -> impl Iterator> { + self.steps.iter().flat_map(|step| match step { + RegistrationStep::Parameter { option_strings, .. } => option_strings.as_slice(), + RegistrationStep::Ambiguous(_) => &[], + }) + } + + pub fn option_strings_of(&self, parameter_index: usize) -> Option<&[Cow<'a, str>]> { + self.steps.iter().find_map(|step| match step { + RegistrationStep::Parameter { parameter_index: index, option_strings } if *index == parameter_index => { + Some(option_strings.as_slice()) + } + _ => None, + }) + } + + fn parameter_registered_as(&self, name: &str) -> Option { + match self.find_target(name) { + Some(OptionTarget::Parameter(parameter_index)) => Some(parameter_index), + _ => None, + } + } +} + +fn push_unique<'a>(names: &mut Vec>, name: Cow<'a, str>) { + if !names.contains(&name) { + names.push(name); + } +} + +fn count_short_name(parameters: &[DefinedParameter<'_>], short_name: &str) -> usize { + parameters.iter().filter(|parameter| parameter.short_name == Some(short_name)).count() +} + +fn count_long_name(parameters: &[DefinedParameter<'_>], long_name: &str) -> usize { + parameters.iter().filter(|parameter| parameter.long_name == long_name).count() +} + +fn long_name_order<'p>(parameters: &'p [DefinedParameter<'_>]) -> impl Iterator + 'p { + let first_index_of = |long_name: &str| parameters.iter().position(|parameter| parameter.long_name == long_name); + (0..parameters.len()) + .filter(move |index| first_index_of(parameters[*index].long_name) == Some(*index)) + .flat_map(move |first| (first..parameters.len()).filter(move |index| parameters[*index].long_name == parameters[first].long_name)) +} + +pub fn try_register_parameters<'a>(parameters: &[DefinedParameter<'a>], parent_names: &[Cow<'a, str>]) -> Option> { + let mut registration: Registration<'a> = Registration::default(); + let mut ambiguous_names: Vec> = Vec::new(); + for parameter in parameters { + if let Some(short_name) = parameter.short_name.filter(|short_name| count_short_name(parameters, short_name) > 1) { + push_unique(&mut ambiguous_names, Cow::Borrowed(short_name)); + } + } + for parameter_index in long_name_order(parameters) { + let parameter: &DefinedParameter<'a> = ¶meters[parameter_index]; + let use_scoped_long_name: bool = count_long_name(parameters, parameter.long_name) > 1; + if use_scoped_long_name { + parameter.scope?; + push_unique(&mut ambiguous_names, Cow::Borrowed(parameter.long_name)); + } + let mut names: Vec> = Vec::with_capacity(3); + if let Some(short_name) = parameter.short_name.filter(|short_name| count_short_name(parameters, short_name) == 1) { + names.push(Cow::Borrowed(short_name)); + } + if !use_scoped_long_name { + names.push(Cow::Borrowed(parameter.long_name)); + } + if let Some(scoped_long_name) = parameter.scoped_long_name() { + names.push(Cow::Owned(scoped_long_name)); + } + if names.iter().any(|name| registration.find_target(name).is_some()) { + return None; + } + registration.steps.push(RegistrationStep::Parameter { parameter_index, option_strings: names }); + } + for parent_name in parent_names { + push_unique(&mut ambiguous_names, parent_name.clone()); + } + for ambiguous_name in ambiguous_names { + if let Some(parameter_index) = registration.parameter_registered_as(&ambiguous_name) { + registration.poisoned_parameters.push(parameter_index); + } else if registration.find_target(&ambiguous_name).is_some() { + return None; + } else { + registration.steps.push(RegistrationStep::Ambiguous(ambiguous_name)); + } + } + Some(registration) +} diff --git a/apps/heft-native/src/cli/render.rs b/apps/heft-native/src/cli/render.rs new file mode 100644 index 0000000000..0727aae3f2 --- /dev/null +++ b/apps/heft-native/src/cli/render.rs @@ -0,0 +1,94 @@ +use super::defined_parameter::DefinedParameter; +use super::help_format::{format_help, format_usage_only}; +use super::help_model::HelpParser; +use super::outcome::PrintedOutput; +use super::parse::ArgumentError; +use super::registration::{Registration, RegistrationStep}; + +pub fn help_output(parser: &HelpParser<'_>, width: Option, exit_code: i32) -> Option { + Some(PrintedOutput { stdout: format_help(parser, width?)?, stderr: String::new(), exit_code }) +} + +pub fn usage_error_output(parser: &HelpParser<'_>, width: Option, message: &str) -> Option { + if message.contains('%') { + return None; + } + let stdout: String = format_usage_only(parser, width?)?; + let stderr: String = format!("{}: error: {}\n\n", parser.prog, message); + Some(PrintedOutput { stdout, stderr, exit_code: 1 }) +} + +pub fn invalid_command_message<'x>(value: &str, command_names: impl Iterator) -> String { + let choices: Vec<&str> = command_names.collect(); + format!("argument \"\": Invalid choice: {} (choose from [{}])", value, choices.join(", ")) +} + +pub fn argument_error_message(error: &ArgumentError<'_>, registration: &Registration, parameters: &[DefinedParameter<'_>]) -> Option { + let name_of = |parameter_index: usize| -> Option { + Some(registration.option_strings_of(parameter_index)?.join("/")) + }; + Some(match error { + ArgumentError::ExpectedOneArgument(index) => format!("argument \"{}\": Expected one argument. null", name_of(*index)?), + ArgumentError::InvalidChoice(index, value) => format!( + "argument \"{}\": Invalid choice: {} (choose from [{}])", + name_of(*index)?, + value, + parameters[*index].alternatives.join(", ") + ), + ArgumentError::InvalidInteger(index, value) => format!("argument \"{}\": Invalid int value: {}", name_of(*index)?, value), + ArgumentError::Required(index) => format!("Argument \"{}\" is required", name_of(*index)?), + ArgumentError::Unrecognized(extras) => format!("Unrecognized arguments: {}.", extras.join(" ")), + ArgumentError::Ambiguous(step_index) => ambiguity_message(registration, parameters, *step_index)?, + }) +} + +fn ambiguity_message(registration: &Registration<'_>, parameters: &[DefinedParameter<'_>], step_index: usize) -> Option { + let RegistrationStep::Ambiguous(name) = registration.steps.get(step_index)? else { + return None; + }; + let count_long_name = |long_name: &str| parameters.iter().filter(|parameter| parameter.long_name == long_name).count(); + let mut candidates: Vec = Vec::new(); + if parameters.iter().any(|parameter| parameter.short_name == Some(name.as_ref())) { + for parameter in parameters.iter().filter(|parameter| parameter.short_name == Some(name.as_ref())) { + candidates.push(if count_long_name(parameter.long_name) > 1 { parameter.scoped_long_name()? } else { parameter.long_name.to_string() }); + } + } else { + for parameter in parameters.iter().filter(|parameter| parameter.long_name == name.as_ref()) { + candidates.push(parameter.scoped_long_name()?); + } + } + Some(if candidates.is_empty() { + format!("Ambiguous option: \"{name}\".") + } else { + format!("Ambiguous option: \"{name}\" could match {}.", candidates.join(", ")) + }) +} + +pub fn ambiguity_error_output(usage_parser: &HelpParser<'_>, width: Option, error_prog: &str, message: &str) -> Option { + let mut stdout: String = format_usage_only(usage_parser, width?)?; + stdout.push('\n'); + let stderr: String = format!("Error: {}: error: {}\n\n", error_prog, message.trim()); + Some(PrintedOutput { stdout, stderr, exit_code: 1 }) +} + +pub fn terminal_error_output(parser: &HelpParser<'_>, width: Option, message: &str, supports_color: bool) -> Option { + if message.contains('%') || message.contains('\u{1b}') { + return None; + } + let stdout: String = format_usage_only(parser, width?)?; + let stderr: String = terminal_error_line(&format!("Error: {}: error: {}\n", parser.prog, message), supports_color); + Some(PrintedOutput { stdout, stderr, exit_code: 1 }) +} + +pub fn terminal_error_line(text: &str, supports_color: bool) -> String { + if supports_color { format!("\u{1b}[31m{text}\u{1b}[39m\n") } else { format!("{text}\n") } +} + +pub fn remainder_error_output(parser: &HelpParser<'_>, width: Option, action_name: &str, first_argument: &str, supports_color: bool) -> Option { + if first_argument.contains('%') || first_argument.contains('\u{1b}') { + return None; + } + let usage: String = format_usage_only(parser, width?)?; + let text: String = format!("Error: {usage}\nheft {action_name}: error: Unrecognized arguments: {first_argument}.\n"); + Some(PrintedOutput { stdout: String::new(), stderr: terminal_error_line(&text, supports_color), exit_code: 1 }) +} diff --git a/apps/heft-native/src/cli/run_invocation.rs b/apps/heft-native/src/cli/run_invocation.rs new file mode 100644 index 0000000000..84e6e5ac9d --- /dev/null +++ b/apps/heft-native/src/cli/run_invocation.rs @@ -0,0 +1,107 @@ +use std::borrow::Cow; + +use super::action_invocation::{execute, help_text, print_error, print_help, ActionRequest}; +use super::action_text::{alias_documentation, alias_expansion_message, run_documentation}; +use super::defined_parameter::DefinedParameter; +use super::help_builders::{action_help_parser, ActionHelpText}; +use super::help_model::bold; +use super::outcome::{CliOutcome, ParsedParameters, PrintedOutput}; +use super::parameters::{push_builtin_parameters, push_plugin_parameters, push_scoping_parameters, ROOT_PARAMETER_NAMES}; +use super::parse::{parse_arguments, ArgumentError, ParameterValue, ParseOutcome}; +use super::phase_selection::{phase_names, select_phases, unknown_phase_error, PhaseSelection, NO_PHASES_SELECTED_ERROR}; +use super::registration::{try_register_parameters, Registration}; +use super::render::{argument_error_message, help_output, remainder_error_output, terminal_error_output}; + +pub fn invoke_run<'a>(request: &ActionRequest<'a, '_>, action_args: &[&'a str]) -> Option> { + let action = &request.table.actions[request.action_index]; + let mut definitions: Vec> = Vec::new(); + push_scoping_parameters(&mut definitions, &action.name); + let registration: Registration = try_register_parameters(&definitions, &ROOT_PARAMETER_NAMES)?; + let (values, remainder_start) = match parse_arguments(®istration, &definitions, action_args, true) { + ParseOutcome::Help => return print_help(request, ®istration, &definitions, true), + ParseOutcome::Failed(error) => return print_error(request, ®istration, &definitions, true, &error), + ParseOutcome::Delegate => return None, + ParseOutcome::Parsed { values, remainder_start } => (values, remainder_start), + }; + let selected_phases: Vec = match select_phases(request, &values) { + PhaseSelection::Selected(selected) => selected, + PhaseSelection::Empty if request.alias.is_none() => { + let stderr: String = NO_PHASES_SELECTED_ERROR.to_string(); + return Some(CliOutcome::Print(PrintedOutput { stdout: String::new(), stderr, exit_code: 1 })); + } + PhaseSelection::UnknownPhase => { + let stderr: String = unknown_phase_error(request, &definitions, &values)?; + return Some(CliOutcome::Print(PrintedOutput { stdout: String::new(), stderr, exit_code: 1 })); + } + _ => return None, + }; + if let Some(start) = remainder_start.filter(|start| action_args[*start] != "--") { + if request.alias.is_some() || request.tool_args.contains(&"--debug") { + return None; + } + let supports_color: bool = (request.supports_color?)(); + let parser = action_help_parser(®istration, &definitions, help_text(request, None), true)?; + let output = remainder_error_output(&parser, request.width, &action.name, action_args[start], supports_color)?; + return Some(CliOutcome::Print(output)); + } + let mut scoped: Vec> = Vec::new(); + push_builtin_parameters(&mut scoped, action.watch); + push_plugin_parameters(&mut scoped, request.model, &selected_phases)?; + let mut parent_names: Vec> = ROOT_PARAMETER_NAMES.to_vec(); + parent_names.extend(registration.registered_names().cloned()); + let scoped_registration: Registration = try_register_parameters(&scoped, &parent_names)?; + let scoped_args: &[&'a str] = remainder_start.map_or(&[], |start| &action_args[start + 1..]); + match parse_arguments(&scoped_registration, &scoped, scoped_args, false) { + ParseOutcome::Help => { + let (text, banner) = scoped_help_text(request, &definitions, &values); + let parser = action_help_parser(&scoped_registration, &scoped, text, false)?; + let mut output = help_output(&parser, request.width, 0)?; + output.stdout.insert_str(0, &banner); + Some(CliOutcome::Print(output)) + } + ParseOutcome::Failed(error) if !request.tool_args.contains(&"--debug") => { + let supports_color = request.supports_color?; + if matches!(error, ArgumentError::Ambiguous(_)) { + return None; + } + let message: String = argument_error_message(&error, &scoped_registration, &scoped)?; + let (text, banner) = scoped_help_text(request, &definitions, &values); + let parser = action_help_parser(&scoped_registration, &scoped, text, false)?; + let mut output = terminal_error_output(&parser, request.width, &message, supports_color())?; + output.stdout.insert_str(0, &banner); + Some(CliOutcome::Print(output)) + } + ParseOutcome::Parsed { values: scoped_values, .. } => execute( + request, + selected_phases, + ParsedParameters { definitions, values }, + Some((ParsedParameters { definitions: scoped, values: scoped_values }, remainder_start.map_or(Vec::new(), |start| action_args[start..].to_vec()))), + ), + _ => None, + } +} + +fn scoped_help_text<'a>(request: &ActionRequest<'a, '_>, definitions: &[DefinedParameter<'a>], values: &[ParameterValue<'a>]) -> (ActionHelpText<'a>, String) { + let action = &request.table.actions[request.action_index]; + let mut scope: Vec<&str> = Vec::new(); + for (index, definition) in definitions.iter().enumerate() { + for name in phase_names(values, index) { + scope.push(definition.long_name); + scope.push(name); + } + } + let epilog: String = bold(&format!("For more information on available unscoped parameters, use \"heft {} --help\"", action.name)); + let (prog, description, banner) = match request.alias { + Some(alias) => ( + format!("heft {}", request.command_name), + Cow::Owned(alias_documentation(request.table, alias)), + format!("{}\n", alias_expansion_message(request.table, alias)), + ), + None => ( + format!("heft {} {} --", request.command_name, scope.join(" ")), + Cow::Borrowed(run_documentation(action.watch)), + String::new(), + ), + }; + (ActionHelpText { prog: Cow::Owned(prog), description, epilog: Some(epilog) }, banner) +} diff --git a/apps/heft-native/src/cli/test_harness.rs b/apps/heft-native/src/cli/test_harness.rs new file mode 100644 index 0000000000..2ec0943deb --- /dev/null +++ b/apps/heft-native/src/cli/test_harness.rs @@ -0,0 +1,64 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use super::invocation::interpret_with_output; +use super::model::CliModel; +use super::outcome::CliOutcome; +use super::test_model::{list_field, load_model, optional_field}; +use super::width::help_width_from_columns; + +fn version_selector_banner(args: &[&str]) -> &'static str { + let tool_args = || args.iter().take_while(|arg| arg.starts_with('-')); + if tool_args().any(|arg| *arg == "--unmanaged") { + "Bypassing the Heft version selector because \"--unmanaged\" was specified.\n\n" + } else if tool_args().any(|arg| *arg == "--debug") { + "Searching for a locally installed version of Heft. Use the \"--unmanaged\" flag if you want to avoid this.\n" + } else { + "" + } +} + +fn run_case(model: &'static CliModel<'static>, args: &'static [&'static str], env: (Option<&str>, Option<&str>), output_folder: &Path) { + let (columns, force_color) = env; + let color_decision = move || force_color == Some("1"); + let supports_color: Option<&dyn Fn() -> bool> = if force_color.is_none_or(|value| value == "1") { Some(&color_decision) } else { None }; + fs::create_dir_all(output_folder.parent().expect("parent")).expect("mkdir"); + let outcome: CliOutcome<'_> = interpret_with_output(args, model, help_width_from_columns(columns), supports_color); + let result: String = match outcome { + CliOutcome::Print(printed) => { + fs::write(output_folder.with_extension("stdout"), format!("{}{}", version_selector_banner(args), printed.stdout)).expect("write"); + fs::write(output_folder.with_extension("stderr"), &printed.stderr).expect("write"); + format!("print {}", printed.exit_code) + } + CliOutcome::Execute(command) => { + let mut plan_command: String = String::new(); + if command.write_plan_command(&mut plan_command) { + fs::write(output_folder.with_extension("plan"), plan_command).expect("write"); + } + format!("execute {}", command.unaliased_command_name) + } + CliOutcome::Delegate => "delegate".to_string(), + }; + fs::write(output_folder.with_extension("result"), result).expect("write"); +} + +#[test] +fn snapshot_harness() { + let Some(harness_folder) = std::env::var_os("HEFT_NATIVE_CLI_HARNESS_DIR").map(PathBuf::from) else { + return; + }; + let cases: String = fs::read_to_string(harness_folder.join("cases.txt")).expect("cases"); + let mut models: Vec<(String, &'static CliModel<'static>)> = Vec::new(); + for line in cases.lines() { + let fields: Vec<&str> = line.split('\t').collect(); + let project: &str = fields[0]; + if !models.iter().any(|(name, _)| name == project) { + let text: String = fs::read_to_string(harness_folder.join(format!("{project}.model"))).expect("model"); + models.push((project.to_string(), Box::leak(Box::new(load_model(&text))))); + } + let model: &'static CliModel<'static> = models.iter().find(|(name, _)| name == project).expect("model").1; + let args: &'static [&'static str] = Box::leak(list_field(fields[2]).into_boxed_slice()); + let output_folder: PathBuf = harness_folder.join("out").join(project).join(fields[1]); + run_case(model, args, (optional_field(fields[3]), fields.get(4).and_then(|field| optional_field(field))), &output_folder); + } +} diff --git a/apps/heft-native/src/cli/test_model.rs b/apps/heft-native/src/cli/test_model.rs new file mode 100644 index 0000000000..ea720acd4e --- /dev/null +++ b/apps/heft-native/src/cli/test_model.rs @@ -0,0 +1,98 @@ +use super::model::{AliasModel, CliModel, DefaultValue, ParameterKind, PhaseModel, PluginModel, PluginParameterDefinition}; + +pub fn leak(text: String) -> &'static str { + Box::leak(text.into_boxed_str()) +} + +pub fn percent_decode(text: &str) -> &'static str { + let bytes: &[u8] = text.as_bytes(); + let mut decoded: Vec = Vec::with_capacity(bytes.len()); + let mut index: usize = 0; + while index < bytes.len() { + if bytes[index] == b'%' { + let hex: &str = &text[index + 1..index + 3]; + decoded.push(u8::from_str_radix(hex, 16).expect("hex")); + index += 3; + } else { + decoded.push(bytes[index]); + index += 1; + } + } + leak(String::from_utf8(decoded).expect("utf8")) +} + +pub fn optional_field(field: &str) -> Option<&'static str> { + field.strip_prefix('=').map(percent_decode) +} + +pub fn list_field(field: &str) -> Vec<&'static str> { + if field.is_empty() { Vec::new() } else { field.split(',').map(percent_decode).collect() } +} + +fn index_list(field: &str) -> Vec { + if field.is_empty() { Vec::new() } else { field.split(',').map(|item| item.parse().expect("index")).collect() } +} + +fn parameter_kind(text: &str) -> ParameterKind { + match text { + "flag" => ParameterKind::Flag, + "string" => ParameterKind::String, + "stringList" => ParameterKind::StringList, + "integer" => ParameterKind::Integer, + "integerList" => ParameterKind::IntegerList, + "choice" => ParameterKind::Choice, + "choiceList" => ParameterKind::ChoiceList, + other => panic!("unknown kind {other}"), + } +} + +fn default_value(field: &str) -> Option> { + match field.split_at_checked(1) { + Some(("T", rest)) => optional_field(rest).map(DefaultValue::Text), + Some(("N", rest)) => optional_field(rest).map(|number| DefaultValue::Number(number.parse().expect("number"))), + _ => None, + } +} + +pub fn load_model(text: &str) -> CliModel<'static> { + let mut model: CliModel<'static> = CliModel::default(); + for line in text.lines() { + let fields: Vec<&str> = line.split('\t').collect(); + match fields[0] { + "L" => model.lifecycle_plugin_indices = index_list(fields[1]), + "G" => model.plugins.push(PluginModel { + plugin_name: optional_field(fields[2]).expect("name"), + package_name: optional_field(fields[3]).expect("package"), + parameter_scope: optional_field(fields[4]).expect("scope"), + parameters: Vec::new(), + }), + "Q" => { + let plugin_index: usize = fields[1].parse().expect("plugin index"); + model.plugins[plugin_index].parameters.push(PluginParameterDefinition { + kind: parameter_kind(fields[2]), + long_name: optional_field(fields[3]).expect("long name"), + short_name: optional_field(fields[4]), + description: optional_field(fields[5]).unwrap_or(""), + required: fields[6] == "1", + argument_name: optional_field(fields[7]), + alternatives: list_field(fields[8]), + default_value: default_value(fields[9]), + }); + } + "F" => model.phases.push(PhaseModel { + name: optional_field(fields[1]).expect("phase"), + description: optional_field(fields[2]), + dependency_names: list_field(fields[3]), + task_plugin_indices: index_list(fields[4]), + }), + "A" => model.aliases.push(AliasModel { + name: optional_field(fields[1]).expect("alias"), + action_name: optional_field(fields[2]).expect("action"), + default_parameters: list_field(fields[3]), + }), + "D" => model.debug_messages.push(optional_field(fields[1]).expect("debug message")), + other => panic!("unknown line {other}"), + } + } + model +} diff --git a/apps/heft-native/src/cli/tests_cli.rs b/apps/heft-native/src/cli/tests_cli.rs new file mode 100644 index 0000000000..837da538b7 --- /dev/null +++ b/apps/heft-native/src/cli/tests_cli.rs @@ -0,0 +1,180 @@ +use super::invocation::{interpret_with_output, interpret_with_width}; +use super::model::{CliModel, ParameterKind, PhaseModel, PluginModel, PluginParameterDefinition}; +use super::outcome::{CliOutcome, PrintedOutput}; +use super::width::help_width_from_columns; + +const FIX_DESCRIPTION: &str = "Fix all encountered rule violations where the violated rule provides a fixer. When running in production mode, fixes will be disabled regardless of this parameter."; +const ROOT_USAGE: &str = "usage: heft [-h] [--debug] [--unmanaged] ...\n"; +const UNKNOWN_ACTION_ERROR: &str = "heft: error: argument \"\": Invalid choice: nosuch-action (choose from [clean, run, build, test, run-watch, build-watch, test-watch])\n\n"; +const LOCALES_ERROR: &str = "heft build: error: argument \"--locales\": Expected one argument. null\n\n"; +const NO_PHASES_ERROR: &str = "\n\u{1b}[31mError: No phases were selected. Provide at least one phase to the \"--to\", \"--to-except\", or \"--only\" parameters.\u{1b}[39m\n"; +const ROOT_HELP_120: &str = concat!( + "usage: heft [-h] [--debug] [--unmanaged] ...\n", + "\n", + "Heft is a pluggable build system designed for web projects.\n", + "\n", + "Positional arguments:\n", + " \n", + " clean Clean the project, removing temporary task folders and specified clean paths.\n", + " run Run a provided selection of Heft phases.\n", + " build Runs to the build phase, including all transitive dependencies.\n", + " test Runs to the test phase, including all transitive dependencies.\n", + " run-watch Run a provided selection of Heft phases in watch mode..\n", + " build-watch\n", + " Runs to the build phase, including all transitive dependencies, in watch mode.\n", + " test-watch\n", + " Runs to the test phase, including all transitive dependencies, in watch mode.\n", + "\n", + "Optional arguments:\n", + " -h, --help Show this help message and exit.\n", + " --debug Show the full call stack if an error occurs while executing the tool\n", + " --unmanaged Disables the Heft version selector: When Heft is invoked via the shell path, normally it will examine \n", + " the project's package.json dependencies and try to use the locally installed version of Heft. Specify \n", + " \"--unmanaged\" to force the invoked version of Heft to be used. This is useful for example if you want \n", + " to test a different version of Heft.\n", + "\n", + "\u{1b}[1mFor detailed help about a specific command, use: heft -h\u{1b}[22m\n", +); +const BUILD_HELP: &str = concat!( + "usage: heft build [-h] [-v] [--production] [--locales LOCALE] [--clean]\n", + " [--fix]\n", + " \n", + "\n", + "Runs to the build phase, including all transitive dependencies.\n", + "\n", + "Optional arguments:\n", + " -h, --help Show this help message and exit.\n", + " -v, --verbose If specified, log information useful for debugging.\n", + " --production If specified, run Heft in production mode.\n", + " --locales LOCALE Use the specified locale for this run, if applicable.\n", + " --clean If specified, clean the outputs at the beginning of the \n", + " lifecycle and before running each phase.\n", + " --fix, --lint:fix Fix all encountered rule violations where the violated \n", + " rule provides a fixer. When running in production mode, \n", + " fixes will be disabled regardless of this parameter.\n", +); +const SCOPED_HELP: &str = concat!( + "usage: heft run --only build -- [-h] [-v] [--production] [--locales LOCALE]\n", + " [--clean] [--fix]\n", + " \n", + "\n", + "Run a provided selection of Heft phases.\n", + "\n", + "Optional arguments:\n", + " -h, --help Show this help message and exit.\n", + " -v, --verbose If specified, log information useful for debugging.\n", + " --production If specified, run Heft in production mode.\n", + " --locales LOCALE Use the specified locale for this run, if applicable.\n", + " --clean If specified, clean the outputs at the beginning of the \n", + " lifecycle and before running each phase.\n", + " --fix, --lint:fix Fix all encountered rule violations where the violated \n", + " rule provides a fixer. When running in production mode, \n", + " fixes will be disabled regardless of this parameter.\n", + "\n", + "\u{1b}[1mFor more information on available unscoped parameters, use \"heft run \n", + "--help\"\u{1b}[22m\n", +); +const BUILD_USAGE: &str = concat!( + "usage: heft build [-h] [-v] [--production] [--locales LOCALE] [--clean]\n", + " [--fix]\n", + " \n", +); + +fn model() -> CliModel<'static> { + let fix: PluginParameterDefinition<'static> = PluginParameterDefinition { + kind: ParameterKind::Flag, + long_name: "--fix", + short_name: None, + description: FIX_DESCRIPTION, + required: false, + argument_name: None, + alternatives: Vec::new(), + default_value: None, + }; + let lint: PluginModel<'static> = PluginModel { plugin_name: "lint-plugin", package_name: "@rushstack/heft-lint-plugin", parameter_scope: "lint", parameters: vec![fix] }; + let build: PhaseModel<'static> = PhaseModel { name: "build", description: None, dependency_names: Vec::new(), task_plugin_indices: vec![0] }; + let test: PhaseModel<'static> = PhaseModel { name: "test", description: None, dependency_names: vec!["build"], task_plugin_indices: Vec::new() }; + CliModel { phases: vec![build, test], plugins: vec![lint], ..CliModel::default() } +} + +fn printed(args: &[&str], columns: Option<&str>) -> PrintedOutput { + let model: CliModel<'static> = model(); + match interpret_with_width(args, &model, help_width_from_columns(columns)) { + CliOutcome::Print(output) => output, + other => panic!("expected printed output for {args:?}, got {other:?}"), + } +} + +fn assert_printed(args: &[&str], columns: Option<&str>, stdout: &str, stderr: &str, exit_code: i32) { + let output: PrintedOutput = printed(args, columns); + assert_eq!((output.stdout.as_str(), output.stderr.as_str(), output.exit_code), (stdout, stderr, exit_code), "{args:?}"); +} + +#[test] +fn renders_help_like_argparse() { + assert_printed(&["--help"], Some("120"), ROOT_HELP_120, "", 1); + assert_printed(&["build", "--help"], None, BUILD_HELP, "", 1); + assert_printed(&["build", "--nosuch-flag", "-h"], None, BUILD_HELP, "", 1); + assert_printed(&["run", "--only", "build", "--", "--help"], None, SCOPED_HELP, "", 0); +} + +#[test] +fn renders_errors_like_argparse_and_ts_command_line() { + assert_printed(&["nosuch-action"], None, ROOT_USAGE, UNKNOWN_ACTION_ERROR, 1); + assert_printed(&["--version"], None, ROOT_USAGE, "heft: error: too few arguments\n\n", 1); + let dash_dash_error: &str = "heft: error: argument \"\": Invalid choice: -- (choose from [clean, run, build, test, run-watch, build-watch, test-watch])\n\n"; + assert_printed(&["--debug", "--"], None, ROOT_USAGE, dash_dash_error, 1); + assert_printed(&["build", "--locales"], None, BUILD_USAGE, LOCALES_ERROR, 1); + assert_printed(&["build", "extra"], None, BUILD_USAGE, "heft build: error: Unrecognized arguments: extra.\n\n", 1); + assert_printed(&["run", "--", "--help"], None, "", NO_PHASES_ERROR, 1); + let ambiguity: String = format!("{BUILD_USAGE}\n"); + assert_printed(&["build", "--debug", "x"], None, &ambiguity, "Error: heft build: error: Ambiguous option: \"--debug\".\n\n", 1); +} + +#[test] +fn reports_unknown_phases_like_the_heft_terminal() { + let model: CliModel<'static> = model(); + let args: &[&str] = &["run", "--to-except", "build", "--only", "nosuch", "--to", "other"]; + let message: &str = "The phase name \"nosuch\" passed to \"--only\" does not exist in heft.json."; + for (color, expected) in [(false, format!("{message}\n")), (true, format!("\u{1b}[31m{message}\u{1b}[39m\n"))] { + let CliOutcome::Print(output) = interpret_with_output(args, &model, None, Some(&move || color)) else { + panic!("expected printed output"); + }; + assert_eq!((output.stdout.as_str(), output.stderr.as_str(), output.exit_code), ("", expected.as_str(), 1)); + } +} + +#[test] +fn delegates_what_it_cannot_prove() { + let model: CliModel<'static> = model(); + for args in [&["build", "-vq"][..], &["build", "--fi"], &["build", "--clean=x"], &["run", "--to", "nosuch"], &["--deb", "build"], &["run", "--only", "build", "x"]] { + assert!(matches!(interpret_with_width(args, &model, Some(78.0)), CliOutcome::Delegate), "{args:?}"); + } +} + +#[test] +fn hands_parsed_commands_to_the_run_module() { + let model: CliModel<'static> = model(); + let CliOutcome::Execute(command) = interpret_with_width(&["test", "--fix", "--locales", "en-us"], &model, None) else { + panic!("expected execute"); + }; + assert_eq!(command.selected_phases, vec![1, 0]); + let mut plan: String = String::new(); + assert!(command.write_plan_command(&mut plan)); + let CliOutcome::Execute(explicit) = interpret_with_width(&["test", "--locales=en-us"], &model, None) else { + panic!("expected execute"); + }; + assert!(!explicit.write_plan_command(&mut String::new())); + assert_eq!(plan, "{\"commandName\":\"test\",\"unaliasedCommandName\":\"test\",\"actionKind\":\"phase\",\"phaseName\":\"test\",\"watch\":false,\"values\":[[\"--verbose\",null],[\"--production\",null],[\"--locales\",[\"en-us\"]],[\"--clean\",null],[\"--lint:fix\",true]]}"); +} + +#[test] +fn reads_columns_like_javascript() { + assert_eq!(help_width_from_columns(None), Some(78.0)); + assert_eq!(help_width_from_columns(Some("")), Some(78.0)); + assert_eq!(help_width_from_columns(Some("120")), Some(118.0)); + assert_eq!(help_width_from_columns(Some("-1")), Some(-3.0)); + assert!(help_width_from_columns(Some("abc")).is_some_and(f64::is_nan)); + assert_eq!(help_width_from_columns(Some("Infinity")), None); + assert_eq!(help_width_from_columns(Some("1e2")), None); +} diff --git a/apps/heft-native/src/cli/text.rs b/apps/heft-native/src/cli/text.rs new file mode 100644 index 0000000000..3574c128bc --- /dev/null +++ b/apps/heft-native/src/cli/text.rs @@ -0,0 +1,77 @@ +pub fn push_spaces(output: &mut String, count: f64) { + let mut remaining: f64 = count; + while remaining > 0.0 { + output.push(' '); + remaining -= 1.0; + } +} + +pub fn is_javascript_whitespace(byte: u8) -> bool { + matches!(byte, b' ' | b'\t' | b'\n' | 0x0b | 0x0c | b'\r') +} + +pub fn trim_javascript_whitespace(text: &str) -> &str { + let bytes: &[u8] = text.as_bytes(); + let mut start: usize = 0; + let mut end: usize = bytes.len(); + while start < end && is_javascript_whitespace(bytes[start]) { + start += 1; + } + while end > start && is_javascript_whitespace(bytes[end - 1]) { + end -= 1; + } + &text[start..end] +} + +pub fn javascript_substring(text: &str, start: f64, end: f64) -> &str { + let length: f64 = text.len() as f64; + let clamp = |value: f64| -> f64 { + if value.is_nan() || value < 0.0 { + 0.0 + } else if value > length { + length + } else { + value.floor() + } + }; + let (mut from, mut to) = (clamp(start), clamp(end)); + if from > to { + std::mem::swap(&mut from, &mut to); + } + &text[from as usize..to as usize] +} + +pub fn push_json_string(output: &mut String, value: &str) -> bool { + output.push('"'); + for byte in value.bytes() { + match byte { + b'"' => output.push_str("\\\""), + b'\\' => output.push_str("\\\\"), + 0x08 => output.push_str("\\b"), + 0x0c => output.push_str("\\f"), + b'\n' => output.push_str("\\n"), + b'\r' => output.push_str("\\r"), + b'\t' => output.push_str("\\t"), + 0x00..=0x1f => { + const HEX: &[u8; 16] = b"0123456789abcdef"; + output.push_str("\\u00"); + output.push(HEX[(byte >> 4) as usize] as char); + output.push(HEX[(byte & 0x0f) as usize] as char); + } + 0x80..=0xff => return false, + _ => output.push(byte as char), + } + } + output.push('"'); + true +} + +pub fn format_javascript_integer(value: f64) -> Option { + if value.fract() != 0.0 || value.abs() >= 1e15 || value.is_nan() { + return None; + } + if value == 0.0 { + return Some("0".to_string()); + } + Some(format!("{}", value as i64)) +} diff --git a/apps/heft-native/src/cli/validate.rs b/apps/heft-native/src/cli/validate.rs new file mode 100644 index 0000000000..fbc5d8410a --- /dev/null +++ b/apps/heft-native/src/cli/validate.rs @@ -0,0 +1,107 @@ +use super::defined_parameter::DefinedParameter; +use super::model::{DefaultValue, ParameterKind}; + +fn is_lower_alphanumeric(byte: u8) -> bool { + byte.is_ascii_lowercase() || byte.is_ascii_digit() +} + +fn is_lower_alphanumeric_word(word: &str) -> bool { + !word.is_empty() && word.bytes().all(is_lower_alphanumeric) +} + +pub fn is_valid_long_name(name: &str) -> bool { + match name.strip_prefix('-') { + Some(rest) => rest.starts_with('-') && rest[1..].split('-').all(is_lower_alphanumeric_word), + None => false, + } +} + +pub fn is_valid_short_name(name: &str) -> bool { + let bytes: &[u8] = name.as_bytes(); + bytes.len() == 2 && bytes[0] == b'-' && bytes[1].is_ascii_alphabetic() +} + +pub fn is_valid_scope(scope: &str) -> bool { + scope.split('-').all(is_lower_alphanumeric_word) +} + +pub fn is_valid_argument_name(name: &str) -> bool { + !name.is_empty() && name.bytes().all(|byte| byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'_') +} + +pub fn is_valid_action_name(name: &str) -> bool { + let bytes: &[u8] = name.as_bytes(); + if bytes.is_empty() || !bytes[0].is_ascii_lowercase() { + return false; + } + let mut previous_was_separator: bool = false; + for byte in &bytes[1..] { + if *byte == b'-' || *byte == b':' { + if previous_was_separator { + return false; + } + previous_was_separator = true; + } else if is_lower_alphanumeric(*byte) { + previous_was_separator = false; + } else { + return false; + } + } + !previous_was_separator +} + +fn is_truthy_default(default_value: Option>) -> bool { + match default_value { + Some(DefaultValue::Text(text)) => !text.is_empty(), + Some(DefaultValue::Number(number)) => number != 0.0 && !number.is_nan(), + None => false, + } +} + +pub fn is_valid_definition(parameter: &DefinedParameter<'_>) -> bool { + if !is_valid_long_name(parameter.long_name) { + return false; + } + if parameter.short_name.is_some_and(|short_name| !is_valid_short_name(short_name)) { + return false; + } + if parameter.scope.is_some_and(|scope| !is_valid_scope(scope)) { + return false; + } + let needs_argument_name: bool = matches!( + parameter.kind, + ParameterKind::String | ParameterKind::StringList | ParameterKind::Integer | ParameterKind::IntegerList + ); + match parameter.argument_name { + Some(argument_name) if needs_argument_name => { + if !is_valid_argument_name(argument_name) { + return false; + } + } + None if needs_argument_name => return false, + Some(_) => return false, + None => {} + } + if parameter.kind.has_alternatives() { + if parameter.alternatives.is_empty() { + return false; + } + if let Some(DefaultValue::Text(default_text)) = parameter.default_value { + if !default_text.is_empty() && !parameter.alternatives.contains(&default_text) { + return false; + } + } + } else if !parameter.alternatives.is_empty() { + return false; + } + let default_matches_kind: bool = matches!( + (parameter.kind, parameter.default_value), + (_, None) + | (ParameterKind::Choice | ParameterKind::String, Some(DefaultValue::Text(_))) + | (ParameterKind::Integer, Some(DefaultValue::Number(_))) + ); + if !default_matches_kind { + return false; + } + !(parameter.required && is_truthy_default(parameter.default_value)) +} diff --git a/apps/heft-native/src/cli/width.rs b/apps/heft-native/src/cli/width.rs new file mode 100644 index 0000000000..03f903182f --- /dev/null +++ b/apps/heft-native/src/cli/width.rs @@ -0,0 +1,23 @@ +pub fn help_width_from_columns(columns: Option<&str>) -> Option { + let value: &str = match columns { + None | Some("") => return Some(78.0), + Some(value) => value, + }; + let digits: &str = value.strip_prefix('-').unwrap_or(value); + if !digits.is_empty() && digits.len() <= 9 && digits.bytes().all(|byte| byte.is_ascii_digit()) { + let magnitude: f64 = digits.parse::().ok()? as f64; + let columns_number: f64 = if digits.len() == value.len() { magnitude } else { -magnitude }; + return Some(columns_number - 2.0); + } + if value != "Infinity" && value.bytes().all(|byte| byte.is_ascii_alphabetic()) { + return Some(f64::NAN); + } + None +} + +pub fn read_help_width() -> Option { + match std::env::var_os("COLUMNS") { + None => Some(78.0), + Some(raw_value) => help_width_from_columns(Some(raw_value.to_str()?)), + } +} diff --git a/apps/heft-native/src/config/cli_model_builder.rs b/apps/heft-native/src/config/cli_model_builder.rs new file mode 100644 index 0000000000..bceda1dce5 --- /dev/null +++ b/apps/heft-native/src/config/cli_model_builder.rs @@ -0,0 +1,118 @@ +use super::fallback::{fallback, ConfigResult}; +use super::model::HeftConfigurationModel; +use crate::cli::model::{ + AliasModel, CliModel, DefaultValue, ParameterKind, PhaseModel, PluginModel, + PluginParameterDefinition, +}; +use crate::json::JsonValue; + +fn parameter_kind(name: &str) -> ConfigResult { + match name { + "flag" => Ok(ParameterKind::Flag), + "string" => Ok(ParameterKind::String), + "stringList" => Ok(ParameterKind::StringList), + "integer" => Ok(ParameterKind::Integer), + "integerList" => Ok(ParameterKind::IntegerList), + "choice" => Ok(ParameterKind::Choice), + "choiceList" => Ok(ParameterKind::ChoiceList), + _ => fallback("a plugin parameter has an unknown parameterKind"), + } +} + +fn optional_text<'a>(parameter: &'a JsonValue<'a>, key: &str) -> ConfigResult> { + match parameter.get(key) { + None => Ok(None), + Some(JsonValue::String(text)) => Ok(Some(text)), + Some(_) => fallback("a plugin parameter field is not a string"), + } +} + +fn required_text<'a>(parameter: &'a JsonValue<'a>, key: &str) -> ConfigResult<&'a str> { + match optional_text(parameter, key)? { + Some(text) => Ok(text), + None => fallback("a plugin parameter has no required text field"), + } +} + +fn alternative_names<'a>(parameter: &'a JsonValue<'a>) -> ConfigResult> { + match parameter.get("alternatives") { + None => Ok(Vec::new()), + Some(JsonValue::Array(alternatives)) => alternatives + .iter() + .map(|alternative| required_text(alternative, "name")) + .collect(), + Some(_) => fallback("plugin parameter alternatives are not an array"), + } +} + +fn parameter_definition<'a>( + parameter: &'a JsonValue<'a>, +) -> ConfigResult> { + let required: bool = match parameter.get("required") { + None => false, + Some(JsonValue::Boolean(required)) => *required, + Some(_) => return fallback("a plugin parameter's required field is not a boolean"), + }; + let default_value: Option> = match parameter.get("defaultValue") { + None => None, + Some(JsonValue::String(text)) => Some(DefaultValue::Text(text)), + Some(JsonValue::Number(number)) => Some(DefaultValue::Number(number.value)), + Some(_) => return fallback("a plugin parameter's defaultValue has an unexpected type"), + }; + Ok(PluginParameterDefinition { + kind: parameter_kind(required_text(parameter, "parameterKind")?)?, + long_name: required_text(parameter, "longName")?, + short_name: optional_text(parameter, "shortName")? + .filter(|short_name| !short_name.is_empty()), + description: required_text(parameter, "description")?, + required, + argument_name: optional_text(parameter, "argumentName")?, + alternatives: alternative_names(parameter)?, + default_value, + }) +} + +pub fn build_cli_model<'a>(model: &'a HeftConfigurationModel<'a>) -> ConfigResult> { + let mut plugins: Vec> = Vec::with_capacity(model.plugins.len()); + for plugin in &model.plugins { + let parameters = plugin + .parameters + .iter() + .map(parameter_definition) + .collect::>>()?; + plugins.push(PluginModel { + plugin_name: plugin.plugin_name, + package_name: plugin.package_name, + parameter_scope: plugin.parameter_scope, + parameters, + }); + } + Ok(CliModel { + phases: model + .phases + .iter() + .map(|phase| PhaseModel { + name: phase.name, + description: phase.description, + dependency_names: phase.dependency_names.clone(), + task_plugin_indices: phase.tasks.iter().map(|task| task.plugin).collect(), + }) + .collect(), + aliases: model + .aliases + .iter() + .map(|alias| AliasModel { + name: alias.name, + action_name: alias.action_name, + default_parameters: alias.default_parameters.clone(), + }) + .collect(), + lifecycle_plugin_indices: model + .lifecycle_plugins + .iter() + .map(|lifecycle_plugin| lifecycle_plugin.plugin) + .collect(), + plugins, + debug_messages: model.debug_messages.iter().map(String::as_str).collect(), + }) +} diff --git a/apps/heft-native/src/config/embedded_schemas.rs b/apps/heft-native/src/config/embedded_schemas.rs new file mode 100644 index 0000000000..0ec5fa541a --- /dev/null +++ b/apps/heft-native/src/config/embedded_schemas.rs @@ -0,0 +1,13 @@ +use super::fallback::{fallback, ConfigResult}; +use crate::json::{parse_json_with_comments_exactly_like_jju, JsonValue}; + +pub const HEFT_JSON_SCHEMA_TEXT: &str = include_str!("../../../heft/src/schemas/heft.schema.json"); +pub const HEFT_PLUGIN_JSON_SCHEMA_TEXT: &str = + include_str!("../../../heft/src/schemas/heft-plugin.schema.json"); + +pub fn parse_embedded_schema(schema_text: &'static str) -> ConfigResult> { + match parse_json_with_comments_exactly_like_jju(schema_text) { + Ok(schema_document) => Ok(schema_document), + Err(_) => fallback("an embedded schema can't be parsed"), + } +} diff --git a/apps/heft-native/src/config/fallback.rs b/apps/heft-native/src/config/fallback.rs new file mode 100644 index 0000000000..f1b35861cf --- /dev/null +++ b/apps/heft-native/src/config/fallback.rs @@ -0,0 +1,10 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Fallback { + pub reason: &'static str, +} + +pub type ConfigResult = Result; + +pub fn fallback(reason: &'static str) -> ConfigResult { + Err(Fallback { reason }) +} diff --git a/apps/heft-native/src/config/fs_probe.rs b/apps/heft-native/src/config/fs_probe.rs new file mode 100644 index 0000000000..319776ca90 --- /dev/null +++ b/apps/heft-native/src/config/fs_probe.rs @@ -0,0 +1,113 @@ +use std::collections::HashMap; +use std::fs; +use std::io; + +use super::fallback::{fallback, ConfigResult}; +use super::real_path_resolver::RealPathResolver; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum EntryKind { + File, + Directory, + Other, +} + +fn is_missing_entry_error(error: &io::Error) -> bool { + matches!( + error.kind(), + io::ErrorKind::NotFound | io::ErrorKind::NotADirectory + ) || matches!(error.raw_os_error(), Some(2) | Some(20)) +} + +pub fn stat_entry_kind(path: &str) -> ConfigResult> { + match fs::metadata(path) { + Ok(metadata) => { + let file_type: fs::FileType = metadata.file_type(); + if file_type.is_dir() { + Ok(Some(EntryKind::Directory)) + } else if file_type.is_file() || is_fifo(&file_type) { + Ok(Some(EntryKind::File)) + } else { + Ok(Some(EntryKind::Other)) + } + } + Err(error) if is_missing_entry_error(&error) => Ok(None), + Err(_) => fallback("stat failed with an unexpected error"), + } +} + +#[cfg(unix)] +fn is_fifo(file_type: &fs::FileType) -> bool { + use std::os::unix::fs::FileTypeExt; + file_type.is_fifo() +} + +#[cfg(not(unix))] +fn is_fifo(_file_type: &fs::FileType) -> bool { + false +} + +pub fn is_file_like_resolve(path: &str) -> ConfigResult { + Ok(stat_entry_kind(path)? == Some(EntryKind::File)) +} + +pub fn is_directory_like_resolve(path: &str) -> ConfigResult { + Ok(stat_entry_kind(path)? == Some(EntryKind::Directory)) +} + +pub fn exists_like_exists_sync(path: &str) -> bool { + fs::metadata(path).is_ok() +} + +pub fn read_text_or_missing(path: &str) -> ConfigResult> { + match fs::read(path) { + Ok(bytes) => match String::from_utf8(bytes) { + Ok(text) => Ok(Some(text)), + Err(_) => fallback("a file is not valid UTF-8"), + }, + Err(error) if is_missing_entry_error(&error) => Ok(None), + Err(_) => fallback("reading a file failed with an unexpected error"), + } +} + +#[derive(Default)] +pub struct FileSystemProbeCache { + real_paths: HashMap>, + entry_kinds: HashMap>, + resolver: RealPathResolver, +} + +impl FileSystemProbeCache { + pub fn real_path_or_missing(&mut self, path: &str) -> ConfigResult> { + if let Some(cached) = self.real_paths.get(path) { + return Ok(cached.clone()); + } + let real_path: Option = self.resolver.resolve_real_path(path)?; + self.real_paths.insert(path.to_string(), real_path.clone()); + Ok(real_path) + } + + pub fn real_path(&mut self, path: &str) -> ConfigResult { + match self.real_path_or_missing(path)? { + Some(real_path) => Ok(real_path), + None => fallback("realpath of a missing path"), + } + } + + fn entry_kind(&mut self, path: &str) -> ConfigResult> { + if let Some(cached) = self.entry_kinds.get(path) { + return Ok(*cached); + } + let entry_kind: Option = stat_entry_kind(path)?; + self.entry_kinds.insert(path.to_string(), entry_kind); + Ok(entry_kind) + } + + pub fn is_file_like_resolve(&mut self, path: &str) -> ConfigResult { + Ok(self.entry_kind(path)? == Some(EntryKind::File)) + } + + pub fn is_directory_like_resolve(&mut self, path: &str) -> ConfigResult { + Ok(self.entry_kind(path)? == Some(EntryKind::Directory)) + } +} diff --git a/apps/heft-native/src/config/heft_json_chain.rs b/apps/heft-native/src/config/heft_json_chain.rs new file mode 100644 index 0000000000..ae1f5ff6eb --- /dev/null +++ b/apps/heft-native/src/config/heft_json_chain.rs @@ -0,0 +1,106 @@ +use super::fallback::{fallback, ConfigResult}; +use super::fs_probe::{read_text_or_missing, FileSystemProbeCache}; +use super::node_path::{dirname, resolve}; +use super::node_resolve::resolve_module; +use super::package_json::PackageJsonLookup; +use super::rig::{resolve_rig_profile_folder, RigConfigData}; +use crate::json::{parse_json_with_comments_exactly_like_jju, JsonValue}; + +pub const HEFT_JSON_PROJECT_RELATIVE_PATH: &str = "config/heft.json"; + +pub struct ConfigurationFileText { + pub path: String, + pub text: String, + pub parent: Option, +} + +#[derive(Default)] +pub struct HeftJsonChain { + pub files: Vec, + pub debug_messages: Vec, + pub configuration_file_paths: Vec, + pub entry: usize, +} + +fn read_extends_property(text: &str) -> ConfigResult> { + let parsed: JsonValue = match parse_json_with_comments_exactly_like_jju(text) { + Ok(value @ JsonValue::Object(_)) => value, + _ => return fallback("heft.json can't be parsed exactly"), + }; + match parsed.get("extends") { + None | Some(JsonValue::Null) | Some(JsonValue::Boolean(false)) => Ok(None), + Some(JsonValue::Number(number)) if number.value == 0.0 => Ok(None), + Some(JsonValue::String(extends_path)) if extends_path.is_empty() => Ok(None), + Some(JsonValue::String(extends_path)) => Ok(Some(extends_path.to_string())), + Some(_) => fallback("extends is not a string"), + } +} + +pub fn discover_heft_json_chain( + lookup: &mut PackageJsonLookup, + project_path: &str, + rig: &RigConfigData, +) -> ConfigResult { + lookup.try_get_package_folder_for(project_path)?; + let project_file_path: String = resolve(project_path, HEFT_JSON_PROJECT_RELATIVE_PATH); + let mut chain: HeftJsonChain = HeftJsonChain::default(); + let mut visited: Vec = Vec::with_capacity(4); + if let Some(entry) = + chain.load_file(&mut lookup.file_system, &project_file_path, &mut visited)? + { + chain.entry = entry; + return Ok(chain); + } + if !rig.rig_found { + return fallback("heft.json does not exist and there is no rig"); + } + let profile_folder: String = resolve_rig_profile_folder(&mut lookup.file_system, rig)?; + chain.debug_messages.push(format!( + "Configuration file \"{project_file_path}\" does not exist. Attempting to load via rig (\"{profile_folder}\")." + )); + let rig_file_path: String = resolve(&profile_folder, HEFT_JSON_PROJECT_RELATIVE_PATH); + match chain.load_file(&mut lookup.file_system, &rig_file_path, &mut visited)? { + Some(entry) => { + chain.entry = entry; + Ok(chain) + } + None => fallback("heft.json does not exist in the project or in the rig"), + } +} + +impl HeftJsonChain { + fn load_file( + &mut self, + file_system: &mut FileSystemProbeCache, + path: &str, + visited: &mut Vec, + ) -> ConfigResult> { + if visited.iter().any(|visited_path| visited_path == path) { + return fallback("a loop in the extends chain"); + } + visited.push(path.to_string()); + let text: String = match read_text_or_missing(path)? { + Some(text) => text, + None => return Ok(None), + }; + let extends_path: Option = read_extends_property(&text)?; + self.configuration_file_paths.push(path.to_string()); + let parent: Option = match extends_path { + None => None, + Some(extends_path) => { + let parent_path: String = + resolve_module(file_system, &extends_path, dirname(path))?; + match self.load_file(file_system, &parent_path, visited)? { + Some(parent) => Some(parent), + None => return fallback("a file referenced by extends does not exist"), + } + } + }; + self.files.push(ConfigurationFileText { + path: path.to_string(), + text, + parent, + }); + Ok(Some(self.files.len() - 1)) + } +} diff --git a/apps/heft-native/src/config/heft_json_merge.rs b/apps/heft-native/src/config/heft_json_merge.rs new file mode 100644 index 0000000000..d83e76f004 --- /dev/null +++ b/apps/heft-native/src/config/heft_json_merge.rs @@ -0,0 +1,161 @@ +use std::borrow::Cow; + +use super::fallback::{fallback, ConfigResult}; +use super::heft_json_chain::{ConfigurationFileText, HeftJsonChain}; +use super::merge::{merge_objects, InheritanceType, MergeOptions}; +use super::node_path::{dirname, join}; +use super::node_resolve::resolve_package; +use super::package_json::PackageJsonLookup; +use super::tree::{ConfigTree, NodeId, NodeValue, Slot}; +use crate::json::{parse_json_with_comments_exactly_like_jju, JsonValue}; + +const HEFT_MODULE_THAT_RESOLVES_ITS_OWN_PACKAGE: &str = "CoreConfigFiles.js"; + +const PLUGIN_PACKAGE_JSON_PATHS: [&[&str]; 2] = [ + &["heftPlugins", "*", "pluginPackage"], + &[ + "phasesByName", + "*", + "tasksByName", + "*", + "taskPlugin", + "pluginPackage", + ], +]; + +pub struct PluginPackageResolver<'lookup> { + pub lookup: &'lookup mut PackageJsonLookup, + pub heft_module_folder: &'lookup str, + pub heft_package_folder: Option, +} + +impl PluginPackageResolver<'_> { + pub fn resolve_plugin_package( + &mut self, + package_name: &str, + configuration_file_path: &str, + ) -> ConfigResult { + if package_name != "@rushstack/heft" { + return resolve_package( + self.lookup, + package_name, + dirname(configuration_file_path), + true, + ); + } + if self.heft_package_folder.is_none() { + let module_path: String = join( + self.heft_module_folder, + HEFT_MODULE_THAT_RESOLVES_ITS_OWN_PACKAGE, + ); + let real_module_path: String = self.lookup.file_system.real_path(&module_path)?; + self.heft_package_folder = self + .lookup + .try_get_package_folder_for(dirname(&real_module_path))?; + } + match &self.heft_package_folder { + Some(folder) => Ok(folder.clone()), + None => fallback("the @rushstack/heft package folder was not found"), + } + } +} + +pub fn merge_heft_json_chain<'text>( + tree: &mut ConfigTree<'text>, + chain: &'text HeftJsonChain, + index: usize, + resolver: &mut PluginPackageResolver, +) -> ConfigResult { + let file: &'text ConfigurationFileText = &chain.files[index]; + let parent: Option = match file.parent { + Some(parent) => Some(merge_heft_json_chain(tree, chain, parent, resolver)?), + None => None, + }; + let configuration_file: u32 = tree.add_configuration_file_path(&file.path); + let current: NodeId = contextualize(tree, file, configuration_file, resolver)?; + let options: MergeOptions = MergeOptions { + configuration_file, + default_array_inheritance: InheritanceType::Append, + default_object_inheritance: InheritanceType::Merge, + ignored_property_names: &["extends", "$schema"], + }; + let result: NodeId = merge_objects(tree, parent, current, options)?; + let schema_original_value: Slot = tree + .get(current, "$schema") + .map_or(Slot::Undefined, Slot::Node); + if let Some(annotation) = tree.node(result).annotation { + tree.annotations[annotation as usize].schema_property_original_value = + Some(schema_original_value); + } + Ok(result) +} + +fn contextualize<'text>( + tree: &mut ConfigTree<'text>, + file: &'text ConfigurationFileText, + configuration_file: u32, + resolver: &mut PluginPackageResolver, +) -> ConfigResult { + let parsed: JsonValue<'text> = match parse_json_with_comments_exactly_like_jju(&file.text) { + Ok(parsed) => parsed, + Err(_) => return fallback("heft.json can't be parsed exactly"), + }; + let root: NodeId = tree.import_json(parsed)?; + tree.annotate_properties(root, configuration_file); + for json_path in PLUGIN_PACKAGE_JSON_PATHS { + let mut matches: Vec<(NodeId, usize)> = Vec::new(); + collect_json_path_matches(tree, root, json_path, &mut matches); + for (parent, entry_index) in matches { + let value_node: NodeId = tree.object_entries(parent)[entry_index].1; + let package_name: String = match tree.string_value(value_node) { + Some(package_name) => package_name.to_string(), + None => return fallback("a pluginPackage is not a string"), + }; + let resolved: String = resolver.resolve_plugin_package(&package_name, &file.path)?; + let resolved_node: NodeId = + tree.add_node(NodeValue::String(Cow::Owned(resolved)), None); + if let NodeValue::Object(entries) = &mut tree.nodes[parent as usize].value { + entries[entry_index].1 = resolved_node; + } + } + } + Ok(root) +} + +fn collect_json_path_matches( + tree: &ConfigTree, + node: NodeId, + path: &[&str], + matches: &mut Vec<(NodeId, usize)>, +) { + let (segment, rest) = match path.split_first() { + Some(split) => split, + None => return, + }; + if *segment != "*" { + if let Some(index) = tree + .object_entries(node) + .iter() + .position(|(key, _)| key == segment) + { + match rest.is_empty() { + true => matches.push((node, index)), + false => collect_json_path_matches( + tree, + tree.object_entries(node)[index].1, + rest, + matches, + ), + } + } + return; + } + let children: Vec = match &tree.node(node).value { + NodeValue::Object(entries) => entries.iter().map(|(_, child)| *child).collect(), + NodeValue::Array(items) => items.clone(), + _ => Vec::new(), + }; + for child in children { + collect_json_path_matches(tree, child, rest, matches); + } +} diff --git a/apps/heft-native/src/config/javascript_order.rs b/apps/heft-native/src/config/javascript_order.rs new file mode 100644 index 0000000000..5337d93ea6 --- /dev/null +++ b/apps/heft-native/src/config/javascript_order.rs @@ -0,0 +1,46 @@ +use std::borrow::Cow; + +const OBJECT_PROTOTYPE_PROPERTY_NAMES: [&str; 12] = [ + "__proto__", + "__defineGetter__", + "__defineSetter__", + "__lookupGetter__", + "__lookupSetter__", + "constructor", + "hasOwnProperty", + "isPrototypeOf", + "propertyIsEnumerable", + "toLocaleString", + "toString", + "valueOf", +]; + +pub fn is_object_prototype_property_name(key: &str) -> bool { + OBJECT_PROTOTYPE_PROPERTY_NAMES.contains(&key) +} + +pub fn array_index_of_key(key: &str) -> Option { + let bytes: &[u8] = key.as_bytes(); + if bytes.is_empty() || bytes.len() > 10 || !bytes.iter().all(u8::is_ascii_digit) { + return None; + } + if bytes.len() > 1 && bytes[0] == b'0' { + return None; + } + match key.parse::() { + Ok(index) if index < 4_294_967_295 => Some(index as u32), + _ => None, + } +} + +pub fn order_entries_like_javascript(entries: &mut [(Cow<'_, str>, T)]) { + if entries + .iter() + .any(|(key, _)| array_index_of_key(key).is_some()) + { + entries.sort_by_key(|(key, _)| match array_index_of_key(key) { + Some(index) => (0u8, index), + None => (1u8, 0), + }); + } +} diff --git a/apps/heft-native/src/config/loader.rs b/apps/heft-native/src/config/loader.rs new file mode 100644 index 0000000000..dedef80b77 --- /dev/null +++ b/apps/heft-native/src/config/loader.rs @@ -0,0 +1,140 @@ +use super::embedded_schemas::{ + parse_embedded_schema, HEFT_JSON_SCHEMA_TEXT, HEFT_PLUGIN_JSON_SCHEMA_TEXT, +}; +use super::fallback::{fallback, ConfigResult}; +use super::heft_json_chain::{discover_heft_json_chain, HeftJsonChain}; +use super::heft_json_merge::{merge_heft_json_chain, PluginPackageResolver}; +use super::normalize::normalize_heft_configuration; +use super::package_json::PackageJsonLookup; +use super::plugin_manifest::{ + load_plugin_definitions, parse_plugin_package_manifest, read_plugin_package_manifest, +}; +use super::plugin_manifest::{PluginDefinition, PluginPackageManifest}; +use super::plugin_options::validate_plugin_options; +use super::plugin_references::{collect_plugin_references, PluginReferences}; +use super::plugin_selection::select_plugin_definitions; +use super::rig::{load_rig_config_data, RigConfigData}; +use super::tree::{ConfigTree, NodeId}; +use super::tree_json::tree_to_json_value; +use crate::json::JsonValue; +use crate::schema::{compile_json_schema_for_fast_validation, CompiledJsonSchema}; + +pub struct HeftConfigurationRequest<'request> { + pub build_folder_path: &'request str, + pub heft_module_folder: &'request str, +} + +pub struct LoadedHeftConfiguration<'loaded> { + pub build_folder_path: &'loaded str, + pub rig: &'loaded RigConfigData, + pub heft_json_chain: &'loaded HeftJsonChain, + pub tree: &'loaded ConfigTree<'loaded>, + pub heft_json: NodeId, + pub references: &'loaded PluginReferences<'loaded>, + pub manifests: &'loaded [PluginPackageManifest], + pub parsed_manifests: &'loaded [JsonValue<'loaded>], + pub definitions: &'loaded [PluginDefinition<'loaded>], + pub selected_definitions: &'loaded [usize], +} + +fn compile_embedded_schema<'schema>( + schema_document: &'schema JsonValue<'schema>, +) -> ConfigResult> { + match compile_json_schema_for_fast_validation(schema_document) { + Some(compiled_schema) => Ok(compiled_schema), + None => fallback("an embedded schema is outside the fast validation subset"), + } +} + +fn validate_merged_heft_json(tree: &ConfigTree, merged: NodeId) -> ConfigResult<()> { + let schema_document: JsonValue<'static> = parse_embedded_schema(HEFT_JSON_SCHEMA_TEXT)?; + let compiled_schema: CompiledJsonSchema = compile_embedded_schema(&schema_document)?; + if compiled_schema.is_definitely_valid(&tree_to_json_value(tree, merged)) { + Ok(()) + } else { + fallback("heft.json is not definitely valid") + } +} + +fn read_plugin_package_manifests( + references: &PluginReferences, +) -> ConfigResult> { + let mut manifests: Vec = Vec::new(); + for reference in references.all_plugin_references() { + if !manifests + .iter() + .any(|manifest| manifest.package_root == reference.package_root) + { + manifests.push(read_plugin_package_manifest( + reference.package_root, + reference.package_name, + )?); + } + } + Ok(manifests) +} + +fn parse_plugin_package_manifests( + manifests: &[PluginPackageManifest], +) -> ConfigResult>> { + let schema_document: JsonValue<'static> = parse_embedded_schema(HEFT_PLUGIN_JSON_SCHEMA_TEXT)?; + let compiled_schema: CompiledJsonSchema = compile_embedded_schema(&schema_document)?; + manifests + .iter() + .map(|manifest| parse_plugin_package_manifest(manifest, &compiled_schema)) + .collect() +} + +pub fn load_heft_configuration_and_then( + request: &HeftConfigurationRequest, + lookup: &mut PackageJsonLookup, + consume: impl FnOnce(&LoadedHeftConfiguration) -> Output, +) -> ConfigResult { + if cfg!(not(unix)) { + return fallback("native configuration loading implements POSIX paths only"); + } + let rig: RigConfigData = load_rig_config_data(request.build_folder_path)?; + let heft_json_chain: HeftJsonChain = + discover_heft_json_chain(lookup, request.build_folder_path, &rig)?; + let mut tree: ConfigTree = ConfigTree::default(); + let mut resolver: PluginPackageResolver = PluginPackageResolver { + lookup, + heft_module_folder: request.heft_module_folder, + heft_package_folder: None, + }; + let merged: NodeId = merge_heft_json_chain( + &mut tree, + &heft_json_chain, + heft_json_chain.entry, + &mut resolver, + )?; + validate_merged_heft_json(&tree, merged)?; + let heft_json: NodeId = normalize_heft_configuration(&mut tree, merged)?; + let tree: ConfigTree = tree; + let references: PluginReferences = collect_plugin_references(&tree, heft_json)?; + let manifests: Vec = read_plugin_package_manifests(&references)?; + let parsed_manifests: Vec = parse_plugin_package_manifests(&manifests)?; + let mut definitions: Vec = Vec::new(); + for (package, (manifest, parsed)) in manifests.iter().zip(&parsed_manifests).enumerate() { + load_plugin_definitions(package, manifest, parsed, &mut definitions)?; + } + let package_roots: Vec<&str> = manifests + .iter() + .map(|manifest| manifest.package_root.as_str()) + .collect(); + let selected_definitions: Vec = + select_plugin_definitions(&references, &package_roots, &definitions)?; + validate_plugin_options(&tree, &references, &selected_definitions, &definitions)?; + Ok(consume(&LoadedHeftConfiguration { + build_folder_path: request.build_folder_path, + rig: &rig, + heft_json_chain: &heft_json_chain, + tree: &tree, + heft_json, + references: &references, + manifests: &manifests, + parsed_manifests: &parsed_manifests, + definitions: &definitions, + selected_definitions: &selected_definitions, + })) +} diff --git a/apps/heft-native/src/config/merge.rs b/apps/heft-native/src/config/merge.rs new file mode 100644 index 0000000000..422c3777dc --- /dev/null +++ b/apps/heft-native/src/config/merge.rs @@ -0,0 +1,196 @@ +use std::borrow::Cow; + +use super::fallback::{fallback, ConfigResult}; +use super::merge_arrays::append_arrays; +use super::tree::{Annotation, ConfigTree, Entries, NodeId, Slot}; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum InheritanceType { + Append, + Merge, + Replace, +} + +#[derive(Clone, Copy)] +pub struct MergeOptions<'options> { + pub configuration_file: u32, + pub default_array_inheritance: InheritanceType, + pub default_object_inheritance: InheritanceType, + pub ignored_property_names: &'options [&'options str], +} + +fn inheritance_annotation_target(property_name: &str) -> Option<&str> { + let target: &str = property_name + .strip_prefix('$')? + .strip_suffix(".inheritanceType")?; + if target.is_empty() || target.contains('.') { + None + } else { + Some(target) + } +} + +fn find_entry(entries: &[(Cow<'_, str>, NodeId)], key: &str) -> Option { + entries + .iter() + .find(|(entry_key, _)| entry_key == key) + .map(|(_, child)| *child) +} + +fn parse_inheritance_type(raw: &str) -> Option { + match raw.to_ascii_lowercase().as_str() { + "append" => Some(InheritanceType::Append), + "merge" => Some(InheritanceType::Merge), + "replace" => Some(InheritanceType::Replace), + _ => None, + } +} + +fn collect_merged_names_and_inheritance_types<'text>( + tree: &ConfigTree<'text>, + current_entries: &[(Cow<'text, str>, NodeId)], + ignored_property_names: &[&str], + merged_names: &mut Vec>, +) -> ConfigResult> { + let mut inheritance_types: Vec<(String, InheritanceType)> = Vec::new(); + for (property_name, value) in current_entries { + if ignored_property_names.contains(&property_name.as_ref()) { + continue; + } + let target: &str = match inheritance_annotation_target(property_name) { + Some(target) => target, + None => { + if !merged_names.iter().any(|name| name == property_name) { + merged_names.push(property_name.clone()); + } + continue; + } + }; + let target_value: NodeId = match find_entry(current_entries, target) { + Some(target_value) => target_value, + None => return fallback("an inheritance type has no matching property"), + }; + if !tree.is_object_or_array(target_value) && !tree.is_null(target_value) { + return fallback("an inheritance type is set on a primitive"); + } + let inheritance_type: InheritanceType = + match tree.string_value(*value).and_then(parse_inheritance_type) { + Some(inheritance_type) => inheritance_type, + None => return fallback("an unsupported inheritance type"), + }; + inheritance_types.retain(|(name, _)| name != target); + inheritance_types.push((target.to_string(), inheritance_type)); + } + Ok(inheritance_types) +} + +pub fn merge_objects<'text>( + tree: &mut ConfigTree<'text>, + parent: Option, + current: NodeId, + options: MergeOptions, +) -> ConfigResult { + let current_entries: Entries<'text> = tree.object_entries(current).to_vec(); + let parent_entries: Entries<'text> = + parent.map_or_else(Vec::new, |id| tree.object_entries(id).to_vec()); + let mut merged_names: Vec> = parent_entries + .iter() + .map(|(name, _)| name.clone()) + .collect(); + let inheritance_types: Vec<(String, InheritanceType)> = + collect_merged_names_and_inheritance_types( + tree, + ¤t_entries, + options.ignored_property_names, + &mut merged_names, + )?; + let mut result_entries: Entries<'text> = Vec::with_capacity(merged_names.len()); + let mut original_values: Vec<(Cow<'text, str>, Slot)> = Vec::with_capacity(merged_names.len()); + for property_name in merged_names { + let current_value: Option = find_entry(¤t_entries, &property_name); + let parent_value: Option = find_entry(&parent_entries, &property_name); + let parent_original = |tree: &ConfigTree<'text>| match parent { + Some(parent) => tree.original_value(parent, &property_name), + None => Slot::Undefined, + }; + match (current_value, parent_value) { + (Some(value), parent_value) if tree.is_null(value) => { + if parent_value.is_some() { + original_values.push((property_name.clone(), parent_original(tree))); + } + } + (Some(value), None) => { + original_values.push(( + property_name.clone(), + tree.original_value(current, &property_name), + )); + result_entries.push((property_name, value)); + } + (None, Some(parent_value)) => { + original_values.push((property_name.clone(), parent_original(tree))); + result_entries.push((property_name, parent_value)); + } + (Some(value), Some(parent_value)) => { + let configured: Option = inheritance_types + .iter() + .find(|(name, _)| *name == property_name) + .map(|(_, kind)| *kind); + let inheritance_type: InheritanceType = configured.unwrap_or_else(|| { + default_inheritance_type(tree, value, parent_value, &options) + }); + let new_value: NodeId = match inheritance_type { + InheritanceType::Replace => { + original_values.push(( + property_name.clone(), + tree.original_value(current, &property_name), + )); + value + } + InheritanceType::Append => append_arrays(tree, parent_value, value)?, + InheritanceType::Merge => { + merge_nested_objects(tree, parent_value, value, &options)? + } + }; + result_entries.push((property_name, new_value)); + } + (None, None) => {} + } + } + let annotation = tree.add_annotation(Annotation { + configuration_file: Some(options.configuration_file), + original_values, + schema_property_original_value: None, + }); + Ok(tree.add_object(result_entries, Some(annotation))) +} + +fn default_inheritance_type( + tree: &ConfigTree, + value: NodeId, + parent_value: NodeId, + options: &MergeOptions, +) -> InheritanceType { + if tree.is_array(value) && tree.is_array(parent_value) { + options.default_array_inheritance + } else if tree.is_object_or_array(value) && tree.is_object_or_array(parent_value) { + options.default_object_inheritance + } else { + InheritanceType::Replace + } +} + +fn merge_nested_objects( + tree: &mut ConfigTree, + parent_value: NodeId, + value: NodeId, + options: &MergeOptions, +) -> ConfigResult { + if !tree.is_object(value) || !tree.is_object(parent_value) { + return fallback("merge on a non-object"); + } + let nested_options: MergeOptions = MergeOptions { + ignored_property_names: &[], + ..*options + }; + merge_objects(tree, Some(parent_value), value, nested_options) +} diff --git a/apps/heft-native/src/config/merge_arrays.rs b/apps/heft-native/src/config/merge_arrays.rs new file mode 100644 index 0000000000..5cc4395dd1 --- /dev/null +++ b/apps/heft-native/src/config/merge_arrays.rs @@ -0,0 +1,50 @@ +use std::borrow::Cow; + +use super::fallback::{fallback, ConfigResult}; +use super::tree::{Annotation, AnnotationId, ConfigTree, NodeId, NodeValue, Slot}; + +fn array_items(tree: &ConfigTree, id: NodeId) -> ConfigResult> { + match &tree.node(id).value { + NodeValue::Array(items) => Ok(items.clone()), + _ => fallback("append on a non-array"), + } +} + +fn annotation_of(tree: &ConfigTree, id: NodeId) -> ConfigResult { + match tree.node(id).annotation { + Some(annotation) => Ok(annotation), + None => fallback("an appended array has no annotation"), + } +} + +pub fn append_arrays( + tree: &mut ConfigTree, + parent_array: NodeId, + current_array: NodeId, +) -> ConfigResult { + let mut items: Vec = array_items(tree, parent_array)?; + items.extend(array_items(tree, current_array)?); + let parent_annotation: AnnotationId = annotation_of(tree, parent_array)?; + let current_annotation: AnnotationId = annotation_of(tree, current_array)?; + let mut original_values: Vec<(Cow, Slot)> = tree.annotations[parent_annotation as usize] + .original_values + .clone(); + for (key, slot) in tree.annotations[current_annotation as usize] + .original_values + .clone() + { + match original_values + .iter_mut() + .find(|(existing_key, _)| *existing_key == key) + { + Some(existing) => existing.1 = slot, + None => original_values.push((key, slot)), + } + } + let annotation: AnnotationId = tree.add_annotation(Annotation { + configuration_file: None, + original_values, + schema_property_original_value: None, + }); + Ok(tree.add_node(NodeValue::Array(items), Some(annotation))) +} diff --git a/apps/heft-native/src/config/mod.rs b/apps/heft-native/src/config/mod.rs new file mode 100644 index 0000000000..44c3e860f5 --- /dev/null +++ b/apps/heft-native/src/config/mod.rs @@ -0,0 +1,41 @@ +#![allow(dead_code, unused_imports)] + +pub mod cli_model_builder; +pub mod embedded_schemas; +pub mod fallback; +pub mod fs_probe; +pub mod heft_json_chain; +pub mod heft_json_merge; +pub mod javascript_order; +pub mod loader; +pub mod merge; +pub mod merge_arrays; +pub mod model; +pub mod node_builtins; +pub mod node_path; +pub mod node_resolve; +pub mod normalize; +pub mod package_json; +pub mod plan_graph_numbering; +pub mod plan_graph_writer; +pub mod plan_members; +pub mod plugin_manifest; +pub mod plugin_options; +pub mod plugin_references; +pub mod plugin_selection; +pub mod real_path_resolver; +pub mod rig; +#[cfg(test)] +mod tests_merge_semantics; +#[cfg(test)] +mod tests_node_paths; +#[cfg(all(test, unix))] +mod tests_real_path_resolver; +#[cfg(all(test, unix))] +mod tests_synthetic_projects; +#[cfg(all(test, unix))] +mod tests_synthetic_workspace; +pub mod tree; +pub mod tree_json; +pub mod tree_properties; +pub mod tree_queries; diff --git a/apps/heft-native/src/config/model.rs b/apps/heft-native/src/config/model.rs new file mode 100644 index 0000000000..19839a6224 --- /dev/null +++ b/apps/heft-native/src/config/model.rs @@ -0,0 +1,164 @@ +use super::fallback::{fallback, ConfigResult}; +use super::loader::LoadedHeftConfiguration; +use super::plugin_manifest::PluginKind; +use super::plugin_references::PluginReference; +use super::tree::ConfigTree; +use super::tree_json::tree_to_json_value; +use super::tree_properties::{json_property, object_members, optional_string, string_list}; +use crate::json::JsonValue; + +pub struct TaskModel<'a> { + pub name: &'a str, + pub task_dependency_names: Vec<&'a str>, + pub plugin: usize, + pub options: Option>, +} + +pub struct PhaseModel<'a> { + pub name: &'a str, + pub description: Option<&'a str>, + pub dependency_names: Vec<&'a str>, + pub clean_files: Option>, + pub tasks: Vec>, +} + +pub struct AliasModel<'a> { + pub name: &'a str, + pub action_name: &'a str, + pub default_parameters: Vec<&'a str>, +} + +pub struct LifecyclePluginModel<'a> { + pub plugin: usize, + pub options: Option>, +} + +pub struct PluginModel<'a> { + pub kind: PluginKind, + pub package_root: &'a str, + pub package_name: &'a str, + pub plugin_name: &'a str, + pub entry_point: &'a str, + pub parameter_scope: &'a str, + pub parameters: &'a [JsonValue<'a>], +} + +pub struct HeftConfigurationModel<'a> { + pub build_folder_path: &'a str, + pub phases: Vec>, + pub aliases: Vec>, + pub lifecycle_plugins: Vec>, + pub plugins: Vec>, + pub debug_messages: &'a [String], +} + +struct PluginIndexer<'a> { + definitions_in_order: Vec, + loaded: &'a LoadedHeftConfiguration<'a>, +} + +impl PluginIndexer<'_> { + fn plugin_index_for_reference(&mut self, reference_index: usize) -> usize { + let definition: usize = self.loaded.selected_definitions[reference_index]; + match self + .definitions_in_order + .iter() + .position(|known| *known == definition) + { + Some(index) => index, + None => { + self.definitions_in_order.push(definition); + self.definitions_in_order.len() - 1 + } + } + } +} + +fn plugin_options<'a>(tree: &ConfigTree<'a>, reference: &PluginReference) -> Option> { + reference + .options + .map(|options| tree_to_json_value(tree, options)) +} + +pub fn build_heft_configuration_model<'a>( + loaded: &'a LoadedHeftConfiguration<'a>, +) -> ConfigResult> { + let tree: &'a ConfigTree<'a> = loaded.tree; + let mut indexer: PluginIndexer = PluginIndexer { + definitions_in_order: Vec::new(), + loaded, + }; + let mut lifecycle_plugins: Vec> = Vec::new(); + for (index, reference) in loaded.references.lifecycle_plugins.iter().enumerate() { + let plugin: usize = indexer.plugin_index_for_reference(index); + lifecycle_plugins.push(LifecyclePluginModel { + plugin, + options: plugin_options(tree, reference), + }); + } + let first_task_reference: usize = loaded.references.lifecycle_plugins.len(); + let mut task_references = loaded.references.tasks.iter().enumerate(); + let mut phases: Vec> = Vec::new(); + for (phase_name, phase) in object_members(tree, loaded.heft_json, "phasesByName") { + let mut tasks: Vec> = Vec::new(); + for (task_name, task) in object_members(tree, phase, "tasksByName") { + let (index, reference) = match task_references.next() { + Some((index, reference)) + if reference.task == task && reference.phase_name == phase_name => + { + (index, reference) + } + _ => return fallback("the task references do not match the normalized heft.json"), + }; + tasks.push(TaskModel { + name: task_name, + task_dependency_names: string_list(tree, task, "taskDependencies")?, + plugin: indexer.plugin_index_for_reference(first_task_reference + index), + options: plugin_options(tree, &reference.plugin), + }); + } + phases.push(PhaseModel { + name: phase_name, + description: optional_string(tree, phase, "phaseDescription")?, + dependency_names: string_list(tree, phase, "phaseDependencies")?, + clean_files: json_property(tree, phase, "cleanFiles"), + tasks, + }); + } + let mut aliases: Vec> = Vec::new(); + for (alias_name, alias) in object_members(tree, loaded.heft_json, "aliasesByName") { + let action_name: &'a str = match optional_string(tree, alias, "actionName")? { + Some(action_name) => action_name, + None => return fallback("an alias has no actionName"), + }; + aliases.push(AliasModel { + name: alias_name, + action_name, + default_parameters: string_list(tree, alias, "defaultParameters")?, + }); + } + let plugins: Vec> = indexer + .definitions_in_order + .iter() + .map(|definition| { + let definition = &loaded.definitions[*definition]; + PluginModel { + kind: definition.kind, + package_root: &loaded.manifests[definition.package].package_root, + package_name: &loaded.manifests[definition.package].package_name, + plugin_name: definition.plugin_name, + entry_point: &definition.entry_point, + parameter_scope: definition.parameter_scope, + parameters: definition.parameters, + } + }) + .collect(); + Ok(HeftConfigurationModel { + build_folder_path: loaded.build_folder_path, + phases, + aliases, + lifecycle_plugins, + plugins, + debug_messages: &loaded.heft_json_chain.debug_messages, + }) +} diff --git a/apps/heft-native/src/config/node_builtins.rs b/apps/heft-native/src/config/node_builtins.rs new file mode 100644 index 0000000000..e07407e5ef --- /dev/null +++ b/apps/heft-native/src/config/node_builtins.rs @@ -0,0 +1,65 @@ +const NODE_BUILTIN_MODULE_NAMES: &[&str] = &[ + "_http_agent", + "_http_client", + "_http_common", + "_http_incoming", + "_http_outgoing", + "_http_server", + "_stream_duplex", + "_stream_passthrough", + "_stream_readable", + "_stream_transform", + "_stream_wrap", + "_stream_writable", + "_tls_common", + "_tls_wrap", + "assert", + "async_hooks", + "buffer", + "child_process", + "cluster", + "console", + "constants", + "crypto", + "dgram", + "diagnostics_channel", + "dns", + "domain", + "events", + "fs", + "http", + "http2", + "https", + "inspector", + "module", + "net", + "os", + "path", + "perf_hooks", + "process", + "punycode", + "querystring", + "readline", + "repl", + "sea", + "sqlite", + "stream", + "string_decoder", + "sys", + "test", + "timers", + "tls", + "trace_events", + "tty", + "url", + "util", + "v8", + "vm", + "wasi", + "worker_threads", + "zlib", +]; + +pub fn is_node_builtin_module_name(module_name: &str) -> bool { + NODE_BUILTIN_MODULE_NAMES.contains(&module_name) || module_name.starts_with("node:") +} diff --git a/apps/heft-native/src/config/node_path.rs b/apps/heft-native/src/config/node_path.rs new file mode 100644 index 0000000000..66caa44ade --- /dev/null +++ b/apps/heft-native/src/config/node_path.rs @@ -0,0 +1,125 @@ +pub fn is_absolute(path: &str) -> bool { + path.starts_with('/') +} + +fn normalize_segments(path: &str, allow_above_root: bool) -> String { + let mut segments: Vec<&str> = Vec::with_capacity(16); + let mut leading_parent_count: usize = 0; + for segment in path.split('/') { + match segment { + "" | "." => {} + ".." => { + if segments.pop().is_none() && allow_above_root { + leading_parent_count += 1; + } + } + _ => segments.push(segment), + } + } + let mut result: String = String::with_capacity(path.len()); + for _ in 0..leading_parent_count { + if !result.is_empty() { + result.push('/'); + } + result.push_str(".."); + } + for segment in segments { + if !result.is_empty() { + result.push('/'); + } + result.push_str(segment); + } + result +} + +pub fn normalize(path: &str) -> String { + if path.is_empty() { + return ".".to_string(); + } + let absolute: bool = is_absolute(path); + let trailing_separator: bool = path.ends_with('/'); + let mut normalized: String = normalize_segments(path, !absolute); + if normalized.is_empty() { + if absolute { + return "/".to_string(); + } + return if trailing_separator { + "./".to_string() + } else { + ".".to_string() + }; + } + if trailing_separator { + normalized.push('/'); + } + if absolute { + normalized.insert(0, '/'); + } + normalized +} + +pub fn resolve(base: &str, relative: &str) -> String { + if is_absolute(relative) { + return resolve_absolute(relative); + } + let mut combined: String = String::with_capacity(base.len() + relative.len() + 1); + combined.push_str(base); + combined.push('/'); + combined.push_str(relative); + resolve_absolute(&combined) +} + +pub fn resolve_absolute(absolute_path: &str) -> String { + let normalized: String = normalize_segments(absolute_path, false); + let mut result: String = String::with_capacity(normalized.len() + 1); + result.push('/'); + result.push_str(&normalized); + result +} + +pub fn join(base: &str, relative: &str) -> String { + if relative.is_empty() { + return normalize(base); + } + if base.is_empty() { + return normalize(relative); + } + let mut combined: String = String::with_capacity(base.len() + relative.len() + 1); + combined.push_str(base); + combined.push('/'); + combined.push_str(relative); + normalize(&combined) +} + +pub fn dirname(path: &str) -> &str { + let bytes: &[u8] = path.as_bytes(); + if bytes.is_empty() { + return "."; + } + let has_root: bool = bytes[0] == b'/'; + let mut end: Option = None; + let mut matched_slash: bool = true; + let mut index: usize = bytes.len() - 1; + while index >= 1 { + if bytes[index] == b'/' { + if !matched_slash { + end = Some(index); + break; + } + } else { + matched_slash = false; + } + index -= 1; + } + match end { + None => { + if has_root { + "/" + } else { + "." + } + } + Some(1) if has_root => "//", + Some(end_index) => &path[..end_index], + } +} diff --git a/apps/heft-native/src/config/node_resolve.rs b/apps/heft-native/src/config/node_resolve.rs new file mode 100644 index 0000000000..369ec169bb --- /dev/null +++ b/apps/heft-native/src/config/node_resolve.rs @@ -0,0 +1,137 @@ +use super::fallback::{fallback, ConfigResult}; +use super::fs_probe::FileSystemProbeCache; +use super::node_builtins::is_node_builtin_module_name; +use super::node_path::{dirname, is_absolute, join, resolve, resolve_absolute}; +use super::package_json::PackageJsonLookup; + +pub fn is_definitely_valid_package_name(package_name: &str) -> bool { + if package_name.is_empty() || package_name.len() > 214 { + return false; + } + let (scope, unscoped_name) = match package_name.strip_prefix('@') { + Some(rest) => match rest.split_once('/') { + Some((scope, name)) => (Some(scope), name), + None => return false, + }, + None => (None, package_name), + }; + let is_scope_char = + |c: char| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '-' | '_' | '.'); + let is_name_char = |c: char| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'); + if let Some(scope) = scope { + if scope.is_empty() || !scope.chars().all(is_scope_char) || scope.chars().all(|c| c == '.') + { + return false; + } + } + match unscoped_name.chars().next() { + Some(first) if first.is_ascii_alphanumeric() || first == '-' => {} + _ => return false, + } + unscoped_name.chars().all(is_name_char) && unscoped_name != "." && unscoped_name != ".." +} + +pub fn node_modules_folders(start: &str) -> Vec { + let absolute_start: String = resolve_absolute(start); + let mut folders: Vec = Vec::with_capacity(16); + let mut current: String = absolute_start; + loop { + folders.push(resolve(¤t, "node_modules")); + let parent: &str = dirname(¤t); + if parent == current { + break; + } + current = parent.to_string(); + } + folders +} + +fn realpath_like_resolve( + file_system: &mut FileSystemProbeCache, + path: String, +) -> ConfigResult { + Ok(file_system.real_path_or_missing(&path)?.unwrap_or(path)) +} + +pub fn resolve_node_modules_file( + file_system: &mut FileSystemProbeCache, + request: &str, + base_folder: &str, + preserve_symlinks: bool, +) -> ConfigResult { + for node_modules_folder in node_modules_folders(base_folder) { + let candidate: String = join(&node_modules_folder, request); + if file_system.is_directory_like_resolve(dirname(&candidate))? { + if file_system.is_file_like_resolve(&candidate)? { + return if preserve_symlinks { + Ok(candidate) + } else { + realpath_like_resolve(file_system, candidate) + }; + } + let mut with_extension: String = String::with_capacity(candidate.len() + 3); + with_extension.push_str(&candidate); + with_extension.push_str(".js"); + if file_system.is_file_like_resolve(&with_extension)? + || file_system.is_directory_like_resolve(&candidate)? + { + return fallback("resolve would probe extensions or a folder"); + } + } + } + fallback("a module was not found in node_modules") +} + +pub fn resolve_package( + lookup: &mut PackageJsonLookup, + package_name: &str, + base_folder_path: &str, + allow_self_reference: bool, +) -> ConfigResult { + let normalized_root_path: String = lookup.file_system.real_path(base_folder_path)?; + if allow_self_reference { + if let Some(own_package_folder) = + lookup.try_get_package_folder_for(&normalized_root_path)? + { + let (own_name, _) = lookup.load_identity_for_folder(&own_package_folder)?; + if own_name == package_name { + return Ok(dirname(&join(&own_package_folder, "package.json")).to_string()); + } + } + } + if !is_definitely_valid_package_name(package_name) { + return fallback("a package name might be invalid"); + } + let mut request: String = String::with_capacity(package_name.len() + 13); + request.push_str(package_name); + request.push_str("/package.json"); + let package_json_path: String = resolve_node_modules_file( + &mut lookup.file_system, + &request, + &normalized_root_path, + false, + )?; + Ok(dirname(&package_json_path).to_string()) +} + +pub fn resolve_module( + file_system: &mut FileSystemProbeCache, + module_path: &str, + base_folder_path: &str, +) -> ConfigResult { + if is_absolute(module_path) { + return Ok(module_path.to_string()); + } + let normalized_root_path: String = file_system.real_path(base_folder_path)?; + if module_path.starts_with('.') { + return Ok(resolve(&normalized_root_path, module_path)); + } + let module_name: &str = module_path.split('/').next().unwrap_or(module_path); + if is_node_builtin_module_name(module_name) + || module_path.contains('\\') + || module_path.contains(':') + { + return fallback("a module path might refer to a builtin module"); + } + resolve_node_modules_file(file_system, module_path, &normalized_root_path, false) +} diff --git a/apps/heft-native/src/config/normalize.rs b/apps/heft-native/src/config/normalize.rs new file mode 100644 index 0000000000..0fa4507d48 --- /dev/null +++ b/apps/heft-native/src/config/normalize.rs @@ -0,0 +1,118 @@ +use std::borrow::Cow; + +use super::fallback::{fallback, ConfigResult}; +use super::tree::{ConfigTree, Entries, NodeId, NodeValue, Slot}; + +fn spread_object_with<'text>( + tree: &mut ConfigTree<'text>, + source: NodeId, + overrides: &[(&'static str, NodeId)], +) -> NodeId { + let annotation = tree.node(source).annotation; + let mut entries: Entries<'text> = tree.object_entries(source).to_vec(); + for (key, value) in overrides { + match entries.iter_mut().find(|(entry_key, _)| entry_key == key) { + Some(entry) => entry.1 = *value, + None => entries.push((Cow::Borrowed(*key), *value)), + } + } + tree.add_object(entries, annotation) +} + +fn normalize_plugin_specifier( + tree: &mut ConfigTree, + raw_specifier: NodeId, +) -> ConfigResult { + if !tree.is_object(raw_specifier) { + return fallback("a plugin specifier is not an object"); + } + let package_root: NodeId = match tree.get(raw_specifier, "pluginPackage") { + Some(package_root) => package_root, + None => return fallback("a plugin specifier has no pluginPackage"), + }; + let original_package_name: NodeId = match tree.original_value(raw_specifier, "pluginPackage") { + Slot::Node(original) => original, + Slot::Undefined => return fallback("a plugin specifier has no original pluginPackage"), + }; + let overrides: [(&'static str, NodeId); 2] = [ + ("pluginPackageRoot", package_root), + ("pluginPackage", original_package_name), + ]; + Ok(spread_object_with(tree, raw_specifier, &overrides)) +} + +fn object_entries_or_empty<'text>( + tree: &ConfigTree<'text>, + parent: NodeId, + key: &str, +) -> ConfigResult> { + match tree.get(parent, key) { + None => Ok(Vec::new()), + Some(id) if tree.is_falsy(id) => Ok(Vec::new()), + Some(id) if tree.is_object(id) => Ok(tree.object_entries(id).to_vec()), + Some(_) => fallback("an object property has an unexpected type"), + } +} + +fn normalize_task(tree: &mut ConfigTree, task: NodeId) -> ConfigResult { + if !tree.is_object(task) { + return fallback("a task is not an object"); + } + match tree.get(task, "taskPlugin") { + Some(task_plugin) if !tree.is_falsy(task_plugin) => { + let normalized_plugin: NodeId = normalize_plugin_specifier(tree, task_plugin)?; + Ok(spread_object_with( + tree, + task, + &[("taskPlugin", normalized_plugin)], + )) + } + _ => Ok(task), + } +} + +fn normalize_heft_plugins(tree: &mut ConfigTree, configuration: NodeId) -> ConfigResult { + let items: Vec = match tree + .get(configuration, "heftPlugins") + .map(|list| &tree.node(list).value) + { + None | Some(NodeValue::Null) => Vec::new(), + Some(NodeValue::Array(items)) => items.clone(), + Some(_) => return fallback("heftPlugins is not an array"), + }; + let mut normalized: Vec = Vec::with_capacity(items.len()); + for item in items { + normalized.push(normalize_plugin_specifier(tree, item)?); + } + Ok(tree.add_node(NodeValue::Array(normalized), None)) +} + +pub fn normalize_heft_configuration( + tree: &mut ConfigTree, + configuration: NodeId, +) -> ConfigResult { + let heft_plugins: NodeId = normalize_heft_plugins(tree, configuration)?; + let phases_by_name = object_entries_or_empty(tree, configuration, "phasesByName")?; + let mut normalized_phases: Entries = Vec::with_capacity(phases_by_name.len()); + for (phase_name, phase) in phases_by_name { + if !tree.is_object(phase) { + return fallback("a phase is not an object"); + } + let tasks_by_name = object_entries_or_empty(tree, phase, "tasksByName")?; + let mut normalized_tasks: Entries = Vec::with_capacity(tasks_by_name.len()); + for (task_name, task) in tasks_by_name { + let normalized_task: NodeId = normalize_task(tree, task)?; + normalized_tasks.push((task_name, normalized_task)); + } + let tasks_node: NodeId = tree.add_object(normalized_tasks, None); + let normalized_phase: NodeId = + spread_object_with(tree, phase, &[("tasksByName", tasks_node)]); + normalized_phases.push((phase_name, normalized_phase)); + } + let phases_node: NodeId = tree.add_object(normalized_phases, None); + Ok(spread_object_with( + tree, + configuration, + &[("heftPlugins", heft_plugins), ("phasesByName", phases_node)], + )) +} diff --git a/apps/heft-native/src/config/package_json.rs b/apps/heft-native/src/config/package_json.rs new file mode 100644 index 0000000000..01802cd0e0 --- /dev/null +++ b/apps/heft-native/src/config/package_json.rs @@ -0,0 +1,101 @@ +use std::collections::HashMap; + +use super::fallback::{fallback, ConfigResult}; +use super::fs_probe::{read_text_or_missing, FileSystemProbeCache}; +use super::node_path::{dirname, join, resolve_absolute}; +use crate::json::{parse_json_with_comments_exactly_like_jju, JsonValue}; + +#[derive(Clone)] +pub struct PackageJsonIdentity { + pub name: Option, + pub version: Option, +} + +#[derive(Default)] +pub struct PackageJsonLookup { + package_folder_by_path: HashMap>, + identity_by_real_path: HashMap, + pub file_system: FileSystemProbeCache, +} + +fn optional_string_field(value: &JsonValue, key: &str) -> ConfigResult> { + match value.get(key) { + None | Some(JsonValue::Null) | Some(JsonValue::Boolean(false)) => Ok(None), + Some(JsonValue::String(text)) if text.is_empty() => Ok(None), + Some(JsonValue::String(text)) => Ok(Some(text.to_string())), + Some(JsonValue::Number(number)) if number.value == 0.0 => Ok(None), + Some(_) => fallback("a package.json identity field is not a string"), + } +} + +impl PackageJsonLookup { + fn try_load_identity( + &mut self, + package_json_path: &str, + ) -> ConfigResult> { + let real_path: String = match self.file_system.real_path_or_missing(package_json_path)? { + Some(real_path) => real_path, + None => return Ok(None), + }; + if let Some(identity) = self.identity_by_real_path.get(&real_path) { + return Ok(Some(identity.clone())); + } + let text: String = match read_text_or_missing(&real_path)? { + Some(text) => text, + None => return fallback("a package.json disappeared while it was read"), + }; + let parsed: JsonValue = match parse_json_with_comments_exactly_like_jju(&text) { + Ok(value @ JsonValue::Object(_)) => value, + _ => return fallback("a package.json can't be parsed exactly"), + }; + let identity: PackageJsonIdentity = PackageJsonIdentity { + name: optional_string_field(&parsed, "name")?, + version: optional_string_field(&parsed, "version")?, + }; + self.identity_by_real_path + .insert(real_path, identity.clone()); + Ok(Some(identity)) + } + + pub fn try_get_package_folder_for( + &mut self, + file_or_folder_path: &str, + ) -> ConfigResult> { + let resolved_path: String = resolve_absolute(file_or_folder_path); + if let Some(cached) = self.package_folder_by_path.get(&resolved_path) { + return Ok(cached.clone()); + } + let mut package_json_path: String = String::with_capacity(resolved_path.len() + 13); + package_json_path.push_str(&resolved_path); + package_json_path.push_str("/package.json"); + let identity: Option = self.try_load_identity(&package_json_path)?; + let result: Option = if identity.is_some_and(|identity| identity.name.is_some()) { + Some(resolved_path.clone()) + } else { + let parent_folder: &str = dirname(&resolved_path); + if parent_folder.is_empty() || parent_folder == resolved_path { + None + } else { + let parent_folder: String = parent_folder.to_string(); + self.try_get_package_folder_for(&parent_folder)? + } + }; + self.package_folder_by_path + .insert(resolved_path, result.clone()); + Ok(result) + } + + pub fn load_identity_for_folder( + &mut self, + package_folder: &str, + ) -> ConfigResult<(String, String)> { + let package_json_path: String = join(package_folder, "package.json"); + match self.try_load_identity(&package_json_path)? { + Some(PackageJsonIdentity { + name: Some(name), + version: Some(version), + }) => Ok((name, version)), + _ => fallback("a package.json is missing its name or version"), + } + } +} diff --git a/apps/heft-native/src/config/plan_graph_numbering.rs b/apps/heft-native/src/config/plan_graph_numbering.rs new file mode 100644 index 0000000000..3083bb9da4 --- /dev/null +++ b/apps/heft-native/src/config/plan_graph_numbering.rs @@ -0,0 +1,59 @@ +use super::tree::{AnnotationId, ConfigTree, NodeId, NodeValue, Slot}; + +const UNNUMBERED: u32 = u32::MAX; + +pub struct CanonicalGraphNumbering { + pub node_numbers: Vec, + pub node_order: Vec, + annotation_visited: Vec, +} + +impl CanonicalGraphNumbering { + pub fn number_graph_from_root(tree: &ConfigTree, root: NodeId) -> CanonicalGraphNumbering { + let mut numbering: CanonicalGraphNumbering = CanonicalGraphNumbering { + node_numbers: vec![UNNUMBERED; tree.nodes.len()], + node_order: Vec::new(), + annotation_visited: vec![false; tree.annotations.len()], + }; + numbering.visit_node(tree, root); + numbering + } + + fn visit_slot(&mut self, tree: &ConfigTree, slot: Slot) { + if let Slot::Node(id) = slot { + self.visit_node(tree, id); + } + } + + fn visit_node(&mut self, tree: &ConfigTree, id: NodeId) { + if !tree.is_object_or_array(id) || self.node_numbers[id as usize] != UNNUMBERED { + return; + } + self.node_numbers[id as usize] = self.node_order.len() as u32; + self.node_order.push(id); + match &tree.node(id).value { + NodeValue::Object(entries) => entries + .iter() + .for_each(|(_, child)| self.visit_node(tree, *child)), + NodeValue::Array(items) => items.iter().for_each(|child| self.visit_node(tree, *child)), + _ => {} + } + if let Some(annotation) = tree.node(id).annotation { + self.visit_annotation(tree, annotation); + } + } + + fn visit_annotation(&mut self, tree: &ConfigTree, annotation: AnnotationId) { + if self.annotation_visited[annotation as usize] { + return; + } + self.annotation_visited[annotation as usize] = true; + let annotation = &tree.annotations[annotation as usize]; + for (_, slot) in &annotation.original_values { + self.visit_slot(tree, *slot); + } + if let Some(slot) = annotation.schema_property_original_value { + self.visit_slot(tree, slot); + } + } +} diff --git a/apps/heft-native/src/config/plan_graph_writer.rs b/apps/heft-native/src/config/plan_graph_writer.rs new file mode 100644 index 0000000000..63ebda01d7 --- /dev/null +++ b/apps/heft-native/src/config/plan_graph_writer.rs @@ -0,0 +1,121 @@ +use std::borrow::Cow; +use std::fmt::{self, Write}; + +use super::plan_graph_numbering::CanonicalGraphNumbering; +use super::tree::{Annotation, ConfigTree, NodeId, NodeValue, Slot}; +use crate::json::{write_json_for_javascript, write_json_string_for_javascript, JsonValue}; + +struct GraphWriter<'graph, 'text> { + tree: &'graph ConfigTree<'text>, + numbering: &'graph CanonicalGraphNumbering, +} + +impl GraphWriter<'_, '_> { + fn write_value_reference(&self, slot: Slot, out: &mut String) -> fmt::Result { + let id: NodeId = match slot { + Slot::Undefined => return out.write_str("{\"undefined\":true}"), + Slot::Node(id) => id, + }; + let primitive: JsonValue = match &self.tree.node(id).value { + NodeValue::Object(_) | NodeValue::Array(_) => { + return write!( + out, + "{{\"node\":{}}}", + self.numbering.node_numbers[id as usize] + ); + } + NodeValue::Null => JsonValue::Null, + NodeValue::Boolean(flag) => JsonValue::Boolean(*flag), + NodeValue::Number(number) => JsonValue::Number(*number), + NodeValue::String(text) => JsonValue::String(Cow::Borrowed(text)), + }; + out.write_str("{\"json\":")?; + write_json_for_javascript(&primitive, out)?; + out.write_char('}') + } + + fn write_keyed_references<'key>( + &self, + entries: impl Iterator, + out: &mut String, + ) -> fmt::Result { + out.write_char('[')?; + for (index, (key, slot)) in entries.enumerate() { + out.write_str(if index == 0 { "[" } else { ",[" })?; + write_json_string_for_javascript(key, out)?; + out.write_char(',')?; + self.write_value_reference(slot, out)?; + out.write_char(']')?; + } + out.write_char(']') + } + + fn write_annotation(&self, annotation: &Annotation, out: &mut String) -> fmt::Result { + out.write_str(",\"annotation\":{\"originalValues\":")?; + let original_values = annotation.original_values.iter(); + self.write_keyed_references( + original_values.map(|(key, slot)| (key.as_ref(), *slot)), + out, + )?; + if let Some(file) = annotation.configuration_file { + out.write_str(",\"configurationFilePath\":")?; + write_json_string_for_javascript( + &self.tree.configuration_file_paths[file as usize], + out, + )?; + } + if let Some(slot) = annotation.schema_property_original_value { + out.write_str(",\"schemaPropertyOriginalValue\":")?; + self.write_value_reference(slot, out)?; + } + out.write_char('}') + } + + fn write_node(&self, id: NodeId, out: &mut String) -> fmt::Result { + out.write_str("{\"entries\":")?; + match &self.tree.node(id).value { + NodeValue::Object(entries) => { + let references = entries + .iter() + .map(|(key, child)| (key.as_ref(), Slot::Node(*child))); + self.write_keyed_references(references, out)?; + } + NodeValue::Array(items) => { + let index_keys: Vec = + (0..items.len()).map(|index| index.to_string()).collect(); + let references = index_keys + .iter() + .zip(items) + .map(|(key, child)| (key.as_str(), Slot::Node(*child))); + self.write_keyed_references(references, out)?; + out.write_str(",\"isArray\":true")?; + } + _ => {} + } + if let Some(annotation) = self.tree.node(id).annotation { + self.write_annotation(&self.tree.annotations[annotation as usize], out)?; + } + out.write_char('}') + } +} + +pub fn write_heft_json_graph(tree: &ConfigTree, root: NodeId, out: &mut String) -> fmt::Result { + let numbering: CanonicalGraphNumbering = + CanonicalGraphNumbering::number_graph_from_root(tree, root); + let writer: GraphWriter = GraphWriter { + tree, + numbering: &numbering, + }; + write!( + out, + "{{\"rootNode\":{},\"nodes\":[", + numbering.node_numbers[root as usize] + )?; + for (index, id) in numbering.node_order.iter().enumerate() { + if index > 0 { + out.write_char(',')?; + } + writer.write_node(*id, out)?; + } + out.write_str("]}") +} diff --git a/apps/heft-native/src/config/plan_members.rs b/apps/heft-native/src/config/plan_members.rs new file mode 100644 index 0000000000..83f5a4f498 --- /dev/null +++ b/apps/heft-native/src/config/plan_members.rs @@ -0,0 +1,79 @@ +use std::fmt::{self, Write}; + +use super::fallback::{fallback, ConfigResult}; +use super::loader::LoadedHeftConfiguration; +use super::plan_graph_writer::write_heft_json_graph; +use crate::json::{write_json_for_javascript, write_json_string_for_javascript}; + +fn write_string_list(values: &[String], out: &mut String) -> fmt::Result { + out.write_char('[')?; + for (index, value) in values.iter().enumerate() { + if index > 0 { + out.write_char(',')?; + } + write_json_string_for_javascript(value, out)?; + } + out.write_char(']') +} + +fn write_config_value(loaded: &LoadedHeftConfiguration, out: &mut String) -> fmt::Result { + out.write_str("{\"buildFolderPath\":")?; + write_json_string_for_javascript(loaded.build_folder_path, out)?; + out.write_str(",\"heftJson\":")?; + write_heft_json_graph(loaded.tree, loaded.heft_json, out)?; + out.write_str(",\"debugMessages\":")?; + write_string_list(&loaded.heft_json_chain.debug_messages, out)?; + out.write_char('}') +} + +fn write_plugins_value(loaded: &LoadedHeftConfiguration, out: &mut String) -> fmt::Result { + out.write_char('[')?; + let manifests = loaded.manifests.iter().zip(loaded.parsed_manifests); + for (index, (manifest, parsed_manifest)) in manifests.enumerate() { + out.write_str(if index == 0 { + "{\"packageRoot\":" + } else { + ",{\"packageRoot\":" + })?; + write_json_string_for_javascript(&manifest.package_root, out)?; + out.write_str(",\"packageName\":")?; + write_json_string_for_javascript(&manifest.package_name, out)?; + out.write_str(",\"manifest\":")?; + write_json_for_javascript(parsed_manifest, out)?; + out.write_char('}')?; + } + out.write_char(']') +} + +fn exactly_written(written: fmt::Result) -> ConfigResult<()> { + match written { + Ok(()) => Ok(()), + Err(_) => fallback("the plan configuration members can't be written exactly"), + } +} + +pub fn write_plan_config_section_value( + loaded: &LoadedHeftConfiguration, + out: &mut String, +) -> ConfigResult<()> { + exactly_written(write_config_value(loaded, out)) +} + +pub fn write_plan_plugins_section_value( + loaded: &LoadedHeftConfiguration, + out: &mut String, +) -> ConfigResult<()> { + exactly_written(write_plugins_value(loaded, out)) +} + +pub fn write_plan_configuration_members( + loaded: &LoadedHeftConfiguration, + out: &mut String, +) -> ConfigResult<()> { + out.push_str("\"config\":"); + write_plan_config_section_value(loaded, out)?; + out.push_str(",\"plugins\":"); + write_plan_plugins_section_value(loaded, out)?; + out.push_str(",\"optionsValidated\":true"); + Ok(()) +} diff --git a/apps/heft-native/src/config/plugin_manifest.rs b/apps/heft-native/src/config/plugin_manifest.rs new file mode 100644 index 0000000000..182b0b5be9 --- /dev/null +++ b/apps/heft-native/src/config/plugin_manifest.rs @@ -0,0 +1,155 @@ +use super::fallback::{fallback, ConfigResult}; +use super::fs_probe::{exists_like_exists_sync, read_text_or_missing}; +use super::node_path::resolve; +use crate::json::{parse_json_with_comments_exactly_like_jju, JsonValue}; +use crate::schema::CompiledJsonSchema; + +pub struct PluginPackageManifest { + pub package_root: String, + pub package_name: String, + pub manifest_file_path: String, + pub text: String, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum PluginKind { + Lifecycle, + Task, +} + +pub struct PluginDefinition<'manifest> { + pub kind: PluginKind, + pub package: usize, + pub plugin_name: &'manifest str, + pub entry_point: String, + pub options_schema_path: Option, + pub parameter_scope: &'manifest str, + pub parameters: &'manifest [JsonValue<'manifest>], +} + +pub fn read_plugin_package_manifest( + package_root: &str, + package_name: &str, +) -> ConfigResult { + let mut manifest_file_path: String = String::with_capacity(package_root.len() + 17); + manifest_file_path.push_str(package_root); + manifest_file_path.push_str("/heft-plugin.json"); + match read_text_or_missing(&manifest_file_path)? { + Some(text) => Ok(PluginPackageManifest { + package_root: package_root.to_string(), + package_name: package_name.to_string(), + manifest_file_path, + text, + }), + None => fallback("a heft-plugin.json file does not exist"), + } +} + +pub fn parse_plugin_package_manifest<'manifest>( + manifest: &'manifest PluginPackageManifest, + schema: &CompiledJsonSchema, +) -> ConfigResult> { + match parse_json_with_comments_exactly_like_jju(&manifest.text) { + Ok(parsed) if schema.is_definitely_valid(&parsed) => Ok(parsed), + _ => fallback("a heft-plugin.json file is not definitely valid"), + } +} + +fn plugin_list<'manifest>( + manifest: &'manifest JsonValue<'manifest>, + key: &str, +) -> ConfigResult<&'manifest [JsonValue<'manifest>]> { + match manifest.get(key) { + None => Ok(&[]), + Some(JsonValue::Array(items)) => Ok(items), + Some(_) => fallback("a plugin list is not an array"), + } +} + +fn truthy_string<'manifest>( + definition: &'manifest JsonValue<'manifest>, + key: &str, +) -> ConfigResult> { + match definition.get(key) { + None => Ok(None), + Some(JsonValue::String(text)) if text.is_empty() => Ok(None), + Some(JsonValue::String(text)) => Ok(Some(text)), + Some(_) => fallback("a plugin definition field is not a string"), + } +} + +fn load_plugin_definition<'manifest>( + kind: PluginKind, + package: usize, + manifest: &PluginPackageManifest, + definition: &'manifest JsonValue<'manifest>, +) -> ConfigResult> { + let (plugin_name, entry_point) = + match (definition.get("pluginName"), definition.get("entryPoint")) { + (Some(JsonValue::String(plugin_name)), Some(JsonValue::String(entry_point))) => { + (plugin_name, entry_point) + } + _ => return fallback("a plugin definition has no pluginName or entryPoint"), + }; + let parameters: &'manifest [JsonValue<'manifest>] = plugin_list(definition, "parameters")?; + for (index, parameter) in parameters.iter().enumerate() { + let long_name: Option<&str> = parameter.get("longName").and_then(JsonValue::as_str); + if long_name.is_none() + || parameters[..index] + .iter() + .any(|other| other.get("longName").and_then(JsonValue::as_str) == long_name) + { + return fallback("a plugin defines a parameter more than once"); + } + } + let options_schema_path: Option = match truthy_string(definition, "optionsSchema")? { + Some(options_schema) => { + let resolved_schema_path: String = resolve(&manifest.package_root, options_schema); + if !exists_like_exists_sync(&resolved_schema_path) { + return fallback("a plugin options schema file does not exist"); + } + Some(resolved_schema_path) + } + None => None, + }; + Ok(PluginDefinition { + kind, + package, + plugin_name, + entry_point: resolve(&manifest.package_root, entry_point), + options_schema_path, + parameter_scope: truthy_string(definition, "parameterScope")?.unwrap_or(plugin_name), + parameters, + }) +} + +pub fn load_plugin_definitions<'manifest>( + package: usize, + manifest: &PluginPackageManifest, + parsed: &'manifest JsonValue<'manifest>, + definitions: &mut Vec>, +) -> ConfigResult<()> { + let lifecycle_plugins: &[JsonValue] = plugin_list(parsed, "lifecyclePlugins")?; + let task_plugins: &[JsonValue] = plugin_list(parsed, "taskPlugins")?; + if lifecycle_plugins.is_empty() && task_plugins.is_empty() { + return fallback("a plugin package does not contain any plugins"); + } + let first_definition: usize = definitions.len(); + for (kind, list) in [ + (PluginKind::Lifecycle, lifecycle_plugins), + (PluginKind::Task, task_plugins), + ] { + for definition in list { + let loaded: PluginDefinition = + load_plugin_definition(kind, package, manifest, definition)?; + if definitions[first_definition..] + .iter() + .any(|other| other.plugin_name == loaded.plugin_name) + { + return fallback("a plugin package contains duplicate plugin names"); + } + definitions.push(loaded); + } + } + Ok(()) +} diff --git a/apps/heft-native/src/config/plugin_options.rs b/apps/heft-native/src/config/plugin_options.rs new file mode 100644 index 0000000000..4cb9a924cc --- /dev/null +++ b/apps/heft-native/src/config/plugin_options.rs @@ -0,0 +1,54 @@ +use super::fallback::{fallback, ConfigResult}; +use super::fs_probe::read_text_or_missing; +use super::plugin_manifest::PluginDefinition; +use super::plugin_references::PluginReferences; +use super::tree::{ConfigTree, NodeId}; +use super::tree_json::{empty_json_object, tree_to_json_value}; +use crate::json::{parse_json_with_comments_exactly_like_jju, JsonValue}; +use crate::schema::{compile_json_schema_for_fast_validation, CompiledJsonSchema}; + +fn options_are_definitely_valid(schema_path: &str, options: &JsonValue) -> ConfigResult { + let schema_text: String = match read_text_or_missing(schema_path)? { + Some(schema_text) => schema_text, + None => return fallback("a plugin options schema file disappeared"), + }; + let schema_document: JsonValue = match parse_json_with_comments_exactly_like_jju(&schema_text) { + Ok(schema_document) => schema_document, + Err(_) => return fallback("a plugin options schema can't be parsed exactly"), + }; + let compiled_schema: CompiledJsonSchema = + match compile_json_schema_for_fast_validation(&schema_document) { + Some(compiled_schema) => compiled_schema, + None => { + return fallback("a plugin options schema is outside the fast validation subset") + } + }; + Ok(compiled_schema.is_definitely_valid(options)) +} + +fn options_or_empty_object<'text>( + tree: &ConfigTree<'text>, + options: Option, +) -> JsonValue<'text> { + match options { + Some(options) if !tree.is_falsy(options) => tree_to_json_value(tree, options), + _ => empty_json_object(), + } +} + +pub fn validate_plugin_options( + tree: &ConfigTree, + references: &PluginReferences, + selected: &[usize], + definitions: &[PluginDefinition], +) -> ConfigResult<()> { + for (reference, definition) in references.all_plugin_references().zip(selected) { + if let Some(schema_path) = &definitions[*definition].options_schema_path { + let options: JsonValue = options_or_empty_object(tree, reference.options); + if !options_are_definitely_valid(schema_path, &options)? { + return fallback("plugin options are not definitely valid"); + } + } + } + Ok(()) +} diff --git a/apps/heft-native/src/config/plugin_references.rs b/apps/heft-native/src/config/plugin_references.rs new file mode 100644 index 0000000000..dfb8bdadf7 --- /dev/null +++ b/apps/heft-native/src/config/plugin_references.rs @@ -0,0 +1,132 @@ +use super::fallback::{fallback, ConfigResult}; +use super::plugin_manifest::PluginKind; +use super::tree::{ConfigTree, NodeId, NodeValue}; + +pub struct PluginReference<'tree> { + pub kind: PluginKind, + pub specifier: NodeId, + pub package_root: &'tree str, + pub package_name: &'tree str, + pub plugin_name: Option<&'tree str>, + pub options: Option, +} + +pub struct TaskReference<'tree> { + pub phase_name: &'tree str, + pub task_name: &'tree str, + pub task: NodeId, + pub plugin: PluginReference<'tree>, +} + +pub struct PluginReferences<'tree> { + pub lifecycle_plugins: Vec>, + pub tasks: Vec>, +} + +fn required_string<'tree>( + tree: &'tree ConfigTree, + object: NodeId, + key: &str, +) -> ConfigResult<&'tree str> { + match tree + .get(object, key) + .and_then(|value| tree.string_value(value)) + { + Some(text) => Ok(text), + None => fallback("a plugin specifier field is not a string"), + } +} + +fn plugin_reference<'tree>( + tree: &'tree ConfigTree, + kind: PluginKind, + specifier: NodeId, +) -> ConfigResult> { + let plugin_name: Option<&'tree str> = match tree.get(specifier, "pluginName") { + None => None, + Some(name) if tree.is_falsy(name) => None, + Some(name) => match tree.string_value(name) { + Some(name) => Some(name), + None => return fallback("a pluginName is not a string"), + }, + }; + Ok(PluginReference { + kind, + specifier, + package_root: required_string(tree, specifier, "pluginPackageRoot")?, + package_name: required_string(tree, specifier, "pluginPackage")?, + plugin_name, + options: tree.get(specifier, "options"), + }) +} + +fn object_members<'tree>( + tree: &'tree ConfigTree, + object: NodeId, + key: &str, +) -> ConfigResult> { + match tree.get(object, key) { + Some(members) if tree.is_object(members) => Ok(tree + .object_entries(members) + .iter() + .map(|(name, child)| (name.as_ref(), *child)) + .collect()), + _ => fallback("a normalized heft.json member is not an object"), + } +} + +pub fn collect_plugin_references<'tree>( + tree: &'tree ConfigTree, + configuration: NodeId, +) -> ConfigResult> { + let lifecycle_specifiers: Vec = match tree + .get(configuration, "heftPlugins") + .map(|list| &tree.node(list).value) + { + Some(NodeValue::Array(items)) => items.clone(), + _ => return fallback("normalized heftPlugins is not an array"), + }; + let mut references: PluginReferences = PluginReferences { + lifecycle_plugins: Vec::new(), + tasks: Vec::new(), + }; + for specifier in lifecycle_specifiers { + references.lifecycle_plugins.push(plugin_reference( + tree, + PluginKind::Lifecycle, + specifier, + )?); + } + for (phase_name, phase) in object_members(tree, configuration, "phasesByName")? { + if phase_name == "lifecycle" { + return fallback("the phase name lifecycle is reserved"); + } + for (task_name, task) in object_members(tree, phase, "tasksByName")? { + if task_name == "clean" { + return fallback("the task name clean is reserved"); + } + let task_plugin: NodeId = match tree.get(task, "taskPlugin") { + Some(task_plugin) if !tree.is_falsy(task_plugin) && tree.is_object(task_plugin) => { + task_plugin + } + _ => return fallback("a task has no task plugin"), + }; + let plugin: PluginReference = plugin_reference(tree, PluginKind::Task, task_plugin)?; + references.tasks.push(TaskReference { + phase_name, + task_name, + task, + plugin, + }); + } + } + Ok(references) +} + +impl<'tree> PluginReferences<'tree> { + pub fn all_plugin_references(&self) -> impl Iterator> { + self.lifecycle_plugins + .iter() + .chain(self.tasks.iter().map(|task| &task.plugin)) + } +} diff --git a/apps/heft-native/src/config/plugin_selection.rs b/apps/heft-native/src/config/plugin_selection.rs new file mode 100644 index 0000000000..ab4a5c709c --- /dev/null +++ b/apps/heft-native/src/config/plugin_selection.rs @@ -0,0 +1,69 @@ +use super::fallback::{fallback, ConfigResult}; +use super::plugin_manifest::{PluginDefinition, PluginKind}; +use super::plugin_references::{PluginReference, PluginReferences}; + +fn find_plugin_definition( + definitions: &[PluginDefinition], + package: usize, + reference: &PluginReference, +) -> ConfigResult { + let mut candidates = definitions + .iter() + .enumerate() + .filter(|(_, definition)| definition.package == package); + let found: Option = match reference.plugin_name { + None => { + let first: Option = candidates.next().map(|(index, _)| index); + if candidates.next().is_some() { + return fallback( + "a plugin package contains multiple plugins and no plugin name was specified", + ); + } + first + } + Some(plugin_name) => candidates + .find(|(_, definition)| definition.plugin_name == plugin_name) + .map(|(index, _)| index), + }; + match found { + Some(index) if definitions[index].kind == reference.kind => Ok(index), + Some(_) => fallback("a plugin is not of the kind that its specifier requires"), + None => fallback("a plugin package does not contain the specified plugin"), + } +} + +pub fn select_plugin_definitions( + references: &PluginReferences, + package_roots: &[&str], + definitions: &[PluginDefinition], +) -> ConfigResult> { + let mut selected: Vec = + Vec::with_capacity(references.lifecycle_plugins.len() + references.tasks.len()); + for reference in references.all_plugin_references() { + let package: usize = match package_roots + .iter() + .position(|root| *root == reference.package_root) + { + Some(package) => package, + None => return fallback("a plugin package was not loaded"), + }; + let definition: usize = find_plugin_definition(definitions, package, reference)?; + if reference.kind == PluginKind::Lifecycle && selected.contains(&definition) { + return fallback("a lifecycle plugin is specified more than once"); + } + selected.push(definition); + } + for (index, definition) in selected.iter().enumerate() { + let current: &PluginDefinition = &definitions[*definition]; + let has_other_entry_point = selected[..index].iter().any(|other| { + let other: &PluginDefinition = &definitions[*other]; + other.plugin_name == current.plugin_name && other.entry_point != current.entry_point + }); + if has_other_entry_point { + return fallback( + "multiple plugins with the same name were loaded from different paths", + ); + } + } + Ok(selected) +} diff --git a/apps/heft-native/src/config/real_path_resolver.rs b/apps/heft-native/src/config/real_path_resolver.rs new file mode 100644 index 0000000000..aba09cb27c --- /dev/null +++ b/apps/heft-native/src/config/real_path_resolver.rs @@ -0,0 +1,104 @@ +use std::collections::HashMap; +use std::fs; +use std::io; + +use super::fallback::{fallback, ConfigResult}; + +const MAXIMUM_FOLLOWED_SYMBOLIC_LINKS: u32 = 40; + +#[derive(Clone)] +enum PathComponentKind { + NotASymbolicLink, + SymbolicLink(String), +} + +#[derive(Default)] +pub struct RealPathResolver { + component_kinds: HashMap>, +} + +fn is_missing_entry_error(error: &io::Error) -> bool { + matches!( + error.kind(), + io::ErrorKind::NotFound | io::ErrorKind::NotADirectory + ) || matches!(error.raw_os_error(), Some(2) | Some(20)) +} + +fn classify_path_component(path: &str) -> ConfigResult> { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() => match fs::read_link(path) { + Ok(target) => match target.into_os_string().into_string() { + Ok(target) => Ok(Some(PathComponentKind::SymbolicLink(target))), + Err(_) => fallback("a symbolic link target is not valid UTF-8"), + }, + Err(error) if is_missing_entry_error(&error) => Ok(None), + Err(_) => fallback("readlink failed with an unexpected error"), + }, + Ok(_) => Ok(Some(PathComponentKind::NotASymbolicLink)), + Err(error) if is_missing_entry_error(&error) => Ok(None), + Err(_) => fallback("lstat failed with an unexpected error"), + } +} + +fn push_components_in_reverse(pending: &mut Vec, path: &str) { + pending.extend( + path.split('/') + .filter(|component| !component.is_empty()) + .rev() + .map(str::to_string), + ); +} + +impl RealPathResolver { + fn component_kind(&mut self, path: &str) -> ConfigResult> { + if let Some(kind) = self.component_kinds.get(path) { + return Ok(kind.clone()); + } + let kind: Option = classify_path_component(path)?; + self.component_kinds.insert(path.to_string(), kind.clone()); + Ok(kind) + } + + pub fn resolve_real_path(&mut self, path: &str) -> ConfigResult> { + if !path.starts_with('/') { + return fallback("realpath of a relative path"); + } + let mut pending: Vec = Vec::new(); + push_components_in_reverse(&mut pending, path); + let mut resolved: String = String::with_capacity(path.len()); + let mut followed_symbolic_links: u32 = 0; + while let Some(component) = pending.pop() { + if component == "." { + continue; + } + if component == ".." { + let parent_length: usize = resolved.rfind('/').unwrap_or(0); + resolved.truncate(parent_length); + continue; + } + let parent_length: usize = resolved.len(); + resolved.push('/'); + resolved.push_str(&component); + match self.component_kind(&resolved)? { + None => return Ok(None), + Some(PathComponentKind::NotASymbolicLink) => {} + Some(PathComponentKind::SymbolicLink(target)) => { + followed_symbolic_links += 1; + if followed_symbolic_links > MAXIMUM_FOLLOWED_SYMBOLIC_LINKS { + return fallback("too many levels of symbolic links"); + } + resolved.truncate(if target.starts_with('/') { + 0 + } else { + parent_length + }); + push_components_in_reverse(&mut pending, &target); + } + } + } + if resolved.is_empty() { + resolved.push('/'); + } + Ok(Some(resolved)) + } +} diff --git a/apps/heft-native/src/config/rig.rs b/apps/heft-native/src/config/rig.rs new file mode 100644 index 0000000000..4ca009f00d --- /dev/null +++ b/apps/heft-native/src/config/rig.rs @@ -0,0 +1,114 @@ +use super::fallback::{fallback, ConfigResult}; +use super::fs_probe::{exists_like_exists_sync, read_text_or_missing, FileSystemProbeCache}; +use super::node_path::{dirname, join, resolve_absolute}; +use super::node_resolve::resolve_node_modules_file; +use crate::json::{parse_json_with_comments_exactly_like_jju, JsonObject, JsonValue}; + +#[derive(Clone, Debug, PartialEq)] +pub struct RigConfigData { + pub project_folder_original_path: String, + pub project_folder_path: String, + pub rig_found: bool, + pub file_path: String, + pub rig_package_name: String, + pub rig_profile: String, + pub relative_profile_folder_path: String, +} + +fn is_rig_package_name(name: &str) -> bool { + let is_name_char = |c: char| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'); + let unscoped_name: &str = match name.strip_prefix('@') { + Some(rest) => match rest.split_once('/') { + Some((scope, unscoped)) if !scope.is_empty() && scope.chars().all(is_name_char) => { + unscoped + } + _ => return false, + }, + None => name, + }; + !unscoped_name.is_empty() + && unscoped_name.chars().all(is_name_char) + && (name.ends_with("-rig") || name.ends_with("-rig-test")) +} + +fn is_rig_profile_name(profile: &str) -> bool { + let is_word_char = + |c: char| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '_' | '.'); + !profile.is_empty() + && profile + .split('-') + .all(|word| !word.is_empty() && word.chars().all(is_word_char)) +} + +pub fn load_rig_config_data(project_folder_path: &str) -> ConfigResult { + let rig_config_file_path: String = join(project_folder_path, "config/rig.json"); + let not_found: RigConfigData = RigConfigData { + project_folder_original_path: project_folder_path.to_string(), + project_folder_path: resolve_absolute(project_folder_path), + rig_found: false, + file_path: String::new(), + rig_package_name: String::new(), + rig_profile: String::new(), + relative_profile_folder_path: String::new(), + }; + let text: String = match read_text_or_missing(&rig_config_file_path)? { + Some(text) => text, + None => return Ok(not_found), + }; + let object: JsonObject = match parse_json_with_comments_exactly_like_jju(&text) { + Ok(JsonValue::Object(object)) => object, + _ => return fallback("rig.json can't be parsed exactly"), + }; + let mut rig_package_name: Option<&str> = None; + let mut rig_profile: Option<&str> = None; + for (key, value) in object.entries() { + match (key.as_ref(), value) { + ("$schema", _) => {} + ("rigPackageName", JsonValue::String(name)) => rig_package_name = Some(name), + ("rigProfile", JsonValue::String(profile)) => rig_profile = Some(profile), + _ => return fallback("rig.json has a field that RigConfig might treat differently"), + } + } + let rig_package_name: &str = match rig_package_name { + Some(name) if is_rig_package_name(name) => name, + _ => return fallback("rig.json has an invalid rigPackageName"), + }; + let rig_profile: &str = match rig_profile { + None => "default", + Some(profile) if is_rig_profile_name(profile) => profile, + Some(_) => return fallback("rig.json has an invalid rigProfile"), + }; + let mut relative_profile_folder_path: String = String::with_capacity(9 + rig_profile.len()); + relative_profile_folder_path.push_str("profiles/"); + relative_profile_folder_path.push_str(rig_profile); + Ok(RigConfigData { + rig_found: true, + file_path: rig_config_file_path, + rig_package_name: rig_package_name.to_string(), + rig_profile: rig_profile.to_string(), + relative_profile_folder_path, + ..not_found + }) +} + +pub fn resolve_rig_profile_folder( + file_system: &mut FileSystemProbeCache, + rig: &RigConfigData, +) -> ConfigResult { + if !rig.rig_found { + return fallback("there is no rig"); + } + let mut request: String = String::with_capacity(rig.rig_package_name.len() + 13); + request.push_str(&rig.rig_package_name); + request.push_str("/package.json"); + let rig_package_json_path: String = + resolve_node_modules_file(file_system, &request, &rig.project_folder_path, true)?; + let profile_folder: String = join( + dirname(&rig_package_json_path), + &rig.relative_profile_folder_path, + ); + if !exists_like_exists_sync(&profile_folder) { + return fallback("the rig profile folder does not exist"); + } + Ok(profile_folder) +} diff --git a/apps/heft-native/src/config/tests_merge_semantics.rs b/apps/heft-native/src/config/tests_merge_semantics.rs new file mode 100644 index 0000000000..7732d89b38 --- /dev/null +++ b/apps/heft-native/src/config/tests_merge_semantics.rs @@ -0,0 +1,116 @@ +use std::borrow::Cow; + +use super::javascript_order::order_entries_like_javascript; +use super::merge::{merge_objects, InheritanceType, MergeOptions}; +use super::tree::{ConfigTree, NodeId, NodeValue, Slot}; +use super::tree_json::tree_to_json_value; +use crate::json::{parse_json_with_comments_exactly_like_jju, write_json_for_javascript}; + +fn import_annotated<'text>( + tree: &mut ConfigTree<'text>, + text: &'text str, + file: &str, +) -> Result { + let file_index: u32 = tree.add_configuration_file_path(file); + let parsed = parse_json_with_comments_exactly_like_jju(text).unwrap(); + let root: NodeId = tree + .import_json(parsed) + .map_err(|fallback| fallback.reason)?; + tree.annotate_properties(root, file_index); + Ok(root) +} + +fn merge_texts(parent_text: &str, current_text: &str) -> Result { + let mut tree: ConfigTree = ConfigTree::default(); + let parent: NodeId = import_annotated(&mut tree, parent_text, "/parent.json")?; + let current: NodeId = import_annotated(&mut tree, current_text, "/current.json")?; + let options: MergeOptions = MergeOptions { + configuration_file: 1, + default_array_inheritance: InheritanceType::Append, + default_object_inheritance: InheritanceType::Merge, + ignored_property_names: &["extends", "$schema"], + }; + let merged: NodeId = merge_objects(&mut tree, Some(parent), current, options) + .map_err(|fallback| fallback.reason)?; + let mut out: String = String::new(); + write_json_for_javascript(&tree_to_json_value(&tree, merged), &mut out).unwrap(); + let annotation = &tree.annotations[tree.node(merged).annotation.unwrap() as usize]; + for (key, slot) in &annotation.original_values { + out.push_str(if *slot == Slot::Undefined { " -" } else { " +" }); + out.push_str(key); + } + Ok(out) +} + +#[test] +fn object_keys_are_ordered_like_javascript() { + let mut entries: Vec<(Cow, u8)> = + ["b", "10", "a", "2", "01", "4294967295", "4294967294", "-1"] + .iter() + .map(|key| (Cow::Borrowed(*key), 0)) + .collect(); + order_entries_like_javascript(&mut entries); + let keys: Vec<&str> = entries.iter().map(|(key, _)| key.as_ref()).collect(); + assert_eq!( + keys, + ["2", "10", "4294967294", "b", "a", "01", "4294967295", "-1"] + ); +} + +#[test] +fn default_inheritance_appends_arrays_and_merges_objects() { + let merged = merge_texts( + r#"{"list":[1],"map":{"a":1,"b":{"x":1}},"text":"p","onlyParent":true}"#, + r#"{"list":[2],"map":{"b":{"y":2},"c":3},"text":"c","extends":"x","$schema":"s"}"#, + ); + assert_eq!( + merged.unwrap(), + r#"{"list":[1,2],"map":{"a":1,"b":{"x":1,"y":2},"c":3},"text":"c","onlyParent":true} +text +onlyParent"# + ); +} + +#[test] +fn inheritance_type_annotations_and_null_deletion() { + let merged = merge_texts( + r#"{"list":[1],"map":{"a":1},"gone":{"z":1},"keep":[0]}"#, + r#"{"$list.inheritanceType":"REPLACE","list":[2],"$map.inheritanceType":"replace","map":{"b":2},"gone":null}"#, + ); + assert_eq!( + merged.unwrap(), + r#"{"list":[2],"map":{"b":2},"keep":[0]} +list +map +gone +keep"# + ); +} + +#[test] +fn original_values_of_merged_parent_properties_are_undefined() { + let mut tree: ConfigTree = ConfigTree::default(); + let grandparent: NodeId = + import_annotated(&mut tree, r#"{"map":{"a":1}}"#, "/grandparent.json").unwrap(); + let parent: NodeId = import_annotated(&mut tree, r#"{"map":{"b":2}}"#, "/parent.json").unwrap(); + let current: NodeId = import_annotated(&mut tree, r#"{"other":1}"#, "/current.json").unwrap(); + let options: MergeOptions = MergeOptions { + configuration_file: 2, + default_array_inheritance: InheritanceType::Append, + default_object_inheritance: InheritanceType::Merge, + ignored_property_names: &["extends", "$schema"], + }; + let first: NodeId = merge_objects(&mut tree, Some(grandparent), parent, options).unwrap(); + let second: NodeId = merge_objects(&mut tree, Some(first), current, options).unwrap(); + assert_eq!(tree.original_value(second, "map"), Slot::Undefined); + let annotation = &tree.annotations[tree.node(second).annotation.unwrap() as usize]; + assert_eq!(annotation.original_values.len(), 2); + assert!(matches!( + tree.node(tree.get(second, "map").unwrap()).value, + NodeValue::Object(_) + )); +} + +#[test] +fn unsupported_merges_fall_back() { + assert!(merge_texts(r#"{}"#, r#"{"$a.inheritanceType":"append"}"#).is_err()); + assert!(merge_texts(r#"{}"#, r#"{"$a.inheritanceType":"custom","a":[]}"#).is_err()); + assert!(merge_texts(r#"{}"#, r#"{"$a.inheritanceType":"merge","a":"text"}"#).is_err()); + assert!(merge_texts(r#"{"a":[1]}"#, r#"{"$a.inheritanceType":"merge","a":[2]}"#).is_err()); + assert!(merge_texts(r#"{"a":{}}"#, r#"{"$a.inheritanceType":"append","a":{}}"#).is_err()); + assert!(merge_texts(r#"{}"#, r#"{"nested":{"constructor":1}}"#).is_err()); +} diff --git a/apps/heft-native/src/config/tests_node_paths.rs b/apps/heft-native/src/config/tests_node_paths.rs new file mode 100644 index 0000000000..5ccf710233 --- /dev/null +++ b/apps/heft-native/src/config/tests_node_paths.rs @@ -0,0 +1,147 @@ +use super::node_path::{dirname, is_absolute, join, normalize, resolve}; +use super::node_resolve::{is_definitely_valid_package_name, node_modules_folders}; + +const RESOLVE_CASES: [(&str, &str, &str); 12] = [ + ("/a/b", "c", "/a/b/c"), + ("/a/b", "../c", "/a/c"), + ("/a/b", "/x/y", "/x/y"), + ("/a/b/", "./c/", "/a/b/c"), + ("/", "../..", "/"), + ("/a", "c/./d/../e", "/a/c/e"), + ("/a/b", ".", "/a/b"), + ("/a/b", "", "/a/b"), + ("//a", "b", "/a/b"), + ("/a/b", "c//d", "/a/b/c/d"), + ("/a/b", "...", "/a/b/..."), + ("/a/b", ".hidden/x", "/a/b/.hidden/x"), +]; + +const NORMALIZE_CASES: [(&str, &str); 11] = [ + ("/a/b/../c", "/a/c"), + ("/a/./b/", "/a/b/"), + ("//a//b", "/a/b"), + ("/..", "/"), + ("/a/b/..", "/a"), + ("a/../..", ".."), + ("./a", "a"), + ("", "."), + (".", "."), + ("/a/b/c/../../d", "/a/d"), + ("a//b/", "a/b/"), +]; + +const DIRNAME_CASES: [(&str, &str); 10] = [ + ("/a/b/c", "/a/b"), + ("/a/b/", "/a"), + ("/a", "/"), + ("/", "/"), + ("//a", "//"), + ("a", "."), + ("", "."), + ("a/b", "a"), + ("//", "/"), + ("///a/b", "///a"), +]; + +const JOIN_CASES: [(&str, &str, &str); 6] = [ + ("/a", "b", "/a/b"), + ("/a/", "/b", "/a/b"), + ("a", "", "a"), + ("/a", "../b", "/b"), + ("/a", "./b/c", "/a/b/c"), + ("", "b", "b"), +]; + +#[test] +fn posix_paths_match_node_path_posix() { + for (base, relative, expected) in RESOLVE_CASES { + assert_eq!( + resolve(base, relative), + expected, + "resolve({base}, {relative})" + ); + } + for (path, expected) in NORMALIZE_CASES { + assert_eq!(normalize(path), expected, "normalize({path})"); + } + for (path, expected) in DIRNAME_CASES { + assert_eq!(dirname(path), expected, "dirname({path})"); + } + for (base, relative, expected) in JOIN_CASES { + assert_eq!(join(base, relative), expected, "join({base}, {relative})"); + } + assert!(is_absolute("/a")); + assert!(!is_absolute("a")); + assert!(!is_absolute("")); + assert!(!is_absolute("./a")); + assert!(is_absolute("//")); +} + +#[test] +fn node_modules_folders_match_resolve_1_22() { + assert_eq!( + node_modules_folders("/a/b/c"), + [ + "/a/b/c/node_modules", + "/a/b/node_modules", + "/a/node_modules", + "/node_modules" + ] + ); + assert_eq!( + node_modules_folders("/a/node_modules/b"), + [ + "/a/node_modules/b/node_modules", + "/a/node_modules/node_modules", + "/a/node_modules", + "/node_modules" + ] + ); + assert_eq!(node_modules_folders("/"), ["/node_modules"]); + assert_eq!( + node_modules_folders("/a/b/node_modules/@s/p/node_modules/q"), + [ + "/a/b/node_modules/@s/p/node_modules/q/node_modules", + "/a/b/node_modules/@s/p/node_modules/node_modules", + "/a/b/node_modules/@s/p/node_modules", + "/a/b/node_modules/@s/node_modules", + "/a/b/node_modules/node_modules", + "/a/b/node_modules", + "/a/node_modules", + "/node_modules" + ] + ); +} + +#[test] +fn package_names_that_resolve_accepts_without_doubt() { + assert!( + is_definitely_valid_package_name("@rushstack/heft"), + "@rushstack/heft" + ); + assert!( + is_definitely_valid_package_name("heft-plugin"), + "heft-plugin" + ); + assert!(is_definitely_valid_package_name("a"), "a"); + assert!(is_definitely_valid_package_name("@s/a.b_c-d"), "@s/a.b_c-d"); + assert!(is_definitely_valid_package_name("UpperCase"), "UpperCase"); + assert!(is_definitely_valid_package_name("-dash"), "-dash"); + assert!(!is_definitely_valid_package_name(""), ""); + assert!( + !is_definitely_valid_package_name("@rushstack"), + "@rushstack" + ); + assert!(!is_definitely_valid_package_name("@/x"), "@/x"); + assert!(!is_definitely_valid_package_name("@Scope/x"), "@Scope/x"); + assert!(!is_definitely_valid_package_name("./x"), "./x"); + assert!(!is_definitely_valid_package_name("../x"), "../x"); + assert!(!is_definitely_valid_package_name("/abs"), "/abs"); + assert!(!is_definitely_valid_package_name("a/b"), "a/b"); + assert!(!is_definitely_valid_package_name(".hidden"), ".hidden"); + assert!(!is_definitely_valid_package_name("_under"), "_under"); + assert!(!is_definitely_valid_package_name("@s/.x"), "@s/.x"); + assert!(!is_definitely_valid_package_name("@../x"), "@../x"); + assert!(!is_definitely_valid_package_name("a b"), "a b"); + assert!(!is_definitely_valid_package_name("x:y"), "x:y"); +} diff --git a/apps/heft-native/src/config/tests_real_path_resolver.rs b/apps/heft-native/src/config/tests_real_path_resolver.rs new file mode 100644 index 0000000000..9f079484f6 --- /dev/null +++ b/apps/heft-native/src/config/tests_real_path_resolver.rs @@ -0,0 +1,72 @@ +use std::fs; +use std::os::unix::fs::symlink; +use std::path::PathBuf; + +use super::real_path_resolver::RealPathResolver; + +fn canonicalize_or_missing(path: &str) -> Option { + fs::canonicalize(path) + .ok() + .map(|real_path| real_path.to_str().unwrap().to_string()) +} + +#[test] +fn real_paths_match_the_operating_system_realpath() { + let root: PathBuf = + std::env::temp_dir().join(format!("heft-native-real-path-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(root.join("store/pkg/lib")).unwrap(); + fs::write(root.join("store/pkg/package.json"), "{}").unwrap(); + fs::create_dir_all(root.join("project/node_modules/@scope")).unwrap(); + symlink( + "../../../store/pkg", + root.join("project/node_modules/@scope/pkg"), + ) + .unwrap(); + symlink( + root.join("project/node_modules"), + root.join("absolute-link"), + ) + .unwrap(); + symlink("absolute-link/@scope/pkg/lib/..", root.join("chained-link")).unwrap(); + symlink("loop-b", root.join("loop-a")).unwrap(); + symlink("loop-a", root.join("loop-b")).unwrap(); + let root: String = fs::canonicalize(&root) + .unwrap() + .to_str() + .unwrap() + .to_string(); + let mut resolver: RealPathResolver = RealPathResolver::default(); + for relative_path in [ + "project/node_modules/@scope/pkg/package.json", + "project/node_modules/@scope/pkg", + "absolute-link/@scope/pkg/package.json", + "chained-link/package.json", + "chained-link/lib/../package.json", + "project/node_modules/@scope/pkg/missing.json", + "project/node_modules/@scope/pkg/package.json/child", + "store/./pkg/../pkg/lib", + "", + ] { + let path: String = format!("{root}/{relative_path}"); + assert_eq!( + resolver.resolve_real_path(&path).unwrap(), + canonicalize_or_missing(&path), + "{path}" + ); + assert_eq!( + resolver.resolve_real_path(&path).unwrap(), + canonicalize_or_missing(&path), + "{path}" + ); + } + assert!(resolver + .resolve_real_path(&format!("{root}/loop-a/x")) + .is_err()); + assert!(resolver.resolve_real_path("relative/path").is_err()); + assert_eq!( + resolver.resolve_real_path("/").unwrap().as_deref(), + Some("/") + ); + let _ = fs::remove_dir_all(&root); +} diff --git a/apps/heft-native/src/config/tests_synthetic_projects.rs b/apps/heft-native/src/config/tests_synthetic_projects.rs new file mode 100644 index 0000000000..fca5a02099 --- /dev/null +++ b/apps/heft-native/src/config/tests_synthetic_projects.rs @@ -0,0 +1,158 @@ +use super::tests_synthetic_workspace::SyntheticWorkspace; + +const BASE_HEFT_JSON: &str = r#"{ + "heftPlugins": [{ "pluginPackage": "plugin-a", "pluginName": "lifecycle-a" }], + "aliasesByName": { "b": { "actionName": "build" } }, + "phasesByName": { + "build": { + "phaseDescription": "Build", + "tasksByName": { + "compile": { "taskPlugin": { "pluginPackage": "plugin-a", "pluginName": "task-a", "options": { "level": 1 } } } + } + } + } +}"#; + +fn workspace_with_project_heft_json(project_heft_json: &str) -> SyntheticWorkspace { + let workspace: SyntheticWorkspace = SyntheticWorkspace::with_standard_layout(); + workspace.write("project/config/base-heft.json", BASE_HEFT_JSON); + workspace.write("project/config/heft.json", project_heft_json); + workspace +} + +#[test] +fn extends_chain_with_symlinked_plugin_package_and_heft_self_reference() { + let workspace = workspace_with_project_heft_json( + r#"{ + // comment + "$schema": "https://developer.microsoft.com/json-schemas/heft/v0/heft.schema.json", + "extends": "./base-heft.json", + "phasesByName": { + "build": { "tasksByName": { "compile": { "taskPlugin": { "options": { "level": 2 } } } } }, + "test": { + "phaseDependencies": ["build"], + "tasksByName": { "copy": { "taskPlugin": { "pluginPackage": "@rushstack/heft" } } }, + }, + }, + }"#, + ); + let members: String = workspace.load_plan_members().unwrap(); + let store: String = workspace.path("store/plugin-a"); + let heft: String = workspace.path("heft"); + assert!(members.starts_with(r#""config":{"buildFolderPath":"#)); + assert!(members.contains(&format!(r#"["pluginPackageRoot",{{"json":"{store}"}}]"#))); + assert!(members.contains(&format!(r#"["pluginPackageRoot",{{"json":"{heft}"}}]"#))); + assert!(members.contains(r#"["level",{"json":2}]"#)); + assert!(!members.contains(r#"["level",{"json":1}]"#) || members.contains(r#"{"json":1}"#)); + assert!(members.contains(&format!( + r#"{{"packageRoot":"{store}","packageName":"plugin-a","manifest":"# + ))); + assert!(members.contains(&format!( + r#"{{"packageRoot":"{heft}","packageName":"@rushstack/heft","manifest":"# + ))); + assert!(members.contains(r#""debugMessages":[]}"#)); + assert!(members.ends_with(r#","optionsValidated":true"#)); +} + +#[test] +fn heft_json_from_the_rig_profile() { + let workspace = SyntheticWorkspace::with_standard_layout(); + workspace.write( + "project/config/rig.json", + r#"{ "rigPackageName": "my-rig" }"#, + ); + workspace.write( + "project/node_modules/my-rig/package.json", + r#"{"name":"my-rig","version":"1.0.0"}"#, + ); + workspace.write( + "project/node_modules/my-rig/profiles/default/config/heft.json", + BASE_HEFT_JSON, + ); + let members: String = workspace.load_plan_members().unwrap(); + let expected_message: String = format!( + r#""debugMessages":["Configuration file \"{}\" does not exist. Attempting to load via rig (\"{}\")."]"#, + workspace.path("project/config/heft.json"), + workspace.path("project/node_modules/my-rig/profiles/default") + ); + assert!(members.contains(&expected_message), "{members}"); +} + +fn assert_falls_back(project_heft_json: &str) { + let workspace = workspace_with_project_heft_json(project_heft_json); + assert!( + workspace.load_plan_members().is_err(), + "{project_heft_json}" + ); +} + +#[test] +fn every_error_condition_falls_back() { + assert_falls_back(r#"{ "extends": "./heft.json" }"#); + assert_falls_back(r#"{ "extends": "./missing.json" }"#); + assert_falls_back(r#"{ "extends": 1 }"#); + assert_falls_back(r#"{ "unknownProperty": true }"#); + assert_falls_back( + r#"{ "extends": "./base-heft.json", "heftPlugins": [{ "pluginPackage": "plugin-missing" }] }"#, + ); + assert_falls_back( + r#"{ "extends": "./base-heft.json", "heftPlugins": [{ "pluginPackage": "plugin-a", "pluginName": "lifecycle-a" }] }"#, + ); + assert_falls_back(r#"{ "heftPlugins": [{ "pluginPackage": "plugin-a" }] }"#); + assert_falls_back( + r#"{ "heftPlugins": [{ "pluginPackage": "plugin-a", "pluginName": "task-a" }] }"#, + ); + assert_falls_back( + r#"{ "heftPlugins": [{ "pluginPackage": "plugin-a", "pluginName": "missing" }] }"#, + ); + assert_falls_back(r#"{ "phasesByName": { "lifecycle": {} } }"#); + assert_falls_back( + r#"{ "phasesByName": { "p": { "tasksByName": { "clean": { "taskPlugin": { "pluginPackage": "@rushstack/heft" } } } } } }"#, + ); + assert_falls_back(r#"{ "phasesByName": { "p": { "tasksByName": { "t": {} } } } }"#); + assert_falls_back( + r#"{ "extends": "./base-heft.json", "phasesByName": { "build": { "tasksByName": { "compile": { "taskPlugin": { "options": { "level": "x" } } } } } } }"#, + ); + assert_falls_back( + r#"{ "phasesByName": { "p": { "tasksByName": { "t": { "taskPlugin": { "pluginPackage": "plugin-a", "pluginName": "lifecycle-a" } } } } } }"#, + ); + assert_falls_back("{ 'json5': true }"); +} + +#[test] +fn a_missing_options_schema_file_falls_back() { + let workspace = workspace_with_project_heft_json(r#"{ "extends": "./base-heft.json" }"#); + assert!(workspace.load_plan_members().is_ok()); + std::fs::remove_file(workspace.root.join("store/plugin-a/schema.json")).unwrap(); + assert!(workspace.load_plan_members().is_err()); +} + +#[test] +fn the_heft_package_is_found_from_the_real_path_of_its_module_like_dirname() { + let workspace = workspace_with_project_heft_json( + r#"{ "phasesByName": { "p": { "tasksByName": { "copy": { "taskPlugin": { "pluginPackage": "@rushstack/heft" } } } } } }"#, + ); + workspace.write( + "heft-build/package.json", + r#"{"name":"heft-build","version":"1.0.0"}"#, + ); + workspace.write( + "heft-build/heft-plugin.json", + r#"{"taskPlugins":[{"pluginName":"copy-files-plugin","entryPoint":"./lib/CopyFilesPlugin"}]}"#, + ); + workspace.write("heft-build/lib-commonjs/utilities/CoreConfigFiles.js", ""); + std::fs::remove_dir_all(workspace.root.join("heft/lib-commonjs")).unwrap(); + std::os::unix::fs::symlink( + "../heft-build/lib-commonjs", + workspace.root.join("heft/lib-commonjs"), + ) + .unwrap(); + let members: String = workspace.load_plan_members().unwrap(); + let real_heft_package: String = workspace.path("heft-build"); + assert!( + members.contains(&format!( + r#"["pluginPackageRoot",{{"json":"{real_heft_package}"}}]"# + )), + "{members}" + ); +} diff --git a/apps/heft-native/src/config/tests_synthetic_workspace.rs b/apps/heft-native/src/config/tests_synthetic_workspace.rs new file mode 100644 index 0000000000..d70d2125f5 --- /dev/null +++ b/apps/heft-native/src/config/tests_synthetic_workspace.rs @@ -0,0 +1,98 @@ +use std::fs; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use super::loader::{load_heft_configuration_and_then, HeftConfigurationRequest}; +use super::package_json::PackageJsonLookup; +use super::plan_members::write_plan_configuration_members; + +static NEXT_WORKSPACE_NUMBER: AtomicUsize = AtomicUsize::new(0); + +pub struct SyntheticWorkspace { + pub root: PathBuf, +} + +impl SyntheticWorkspace { + pub fn with_standard_layout() -> SyntheticWorkspace { + let number: usize = NEXT_WORKSPACE_NUMBER.fetch_add(1, Ordering::SeqCst); + let name: String = format!("heft-native-config-tests-{}-{number}", std::process::id()); + let root: PathBuf = std::env::temp_dir().join(name); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).unwrap(); + let workspace: SyntheticWorkspace = SyntheticWorkspace { + root: fs::canonicalize(&root).unwrap(), + }; + workspace.write( + "heft/package.json", + r#"{"name":"@rushstack/heft","version":"1.3.1"}"#, + ); + workspace.write("heft/lib-commonjs/utilities/CoreConfigFiles.js", ""); + workspace.write( + "heft/heft-plugin.json", + r#"{"taskPlugins":[{"pluginName":"copy-files-plugin","entryPoint":"./lib/CopyFilesPlugin"}]}"#, + ); + workspace.write( + "store/plugin-a/package.json", + r#"{"name":"plugin-a","version":"1.0.0"}"#, + ); + workspace.write( + "store/plugin-a/heft-plugin.json", + r#"{ + "lifecyclePlugins": [{ "pluginName": "lifecycle-a", "entryPoint": "./lib/lifecycle" }], + "taskPlugins": [{ + "pluginName": "task-a", "entryPoint": "./lib/task", "optionsSchema": "./schema.json", + "parameters": [{ "longName": "--level", "parameterKind": "integer", "argumentName": "LEVEL", "description": "d" }] + }], + }"#, + ); + workspace.write( + "store/plugin-a/schema.json", + r#"{"$schema":"http://json-schema.org/draft-04/schema#","type":"object","properties":{"level":{"type":"integer"}},"additionalProperties":false}"#, + ); + workspace.write( + "project/package.json", + r#"{"name":"project","version":"1.0.0"}"#, + ); + fs::create_dir_all(workspace.root.join("project/node_modules")).unwrap(); + std::os::unix::fs::symlink( + "../../store/plugin-a", + workspace.root.join("project/node_modules/plugin-a"), + ) + .unwrap(); + workspace + } + + pub fn write(&self, relative_path: &str, text: &str) { + let path: PathBuf = self.root.join(relative_path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); + } + + pub fn path(&self, relative_path: &str) -> String { + self.root.join(relative_path).to_str().unwrap().to_string() + } + + pub fn load_plan_members(&self) -> Result { + let build_folder_path: String = self.path("project"); + let heft_module_folder: String = self.path("heft/lib-commonjs/utilities"); + let request = HeftConfigurationRequest { + build_folder_path: &build_folder_path, + heft_module_folder: &heft_module_folder, + }; + let mut lookup: PackageJsonLookup = PackageJsonLookup::default(); + let loaded = load_heft_configuration_and_then(&request, &mut lookup, |loaded| { + let mut out: String = String::new(); + write_plan_configuration_members(loaded, &mut out).map(|_| out) + }); + match loaded { + Ok(Ok(out)) => Ok(out), + Ok(Err(fallback)) | Err(fallback) => Err(fallback.reason), + } + } +} + +impl Drop for SyntheticWorkspace { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.root); + } +} diff --git a/apps/heft-native/src/config/tree.rs b/apps/heft-native/src/config/tree.rs new file mode 100644 index 0000000000..1e952415ea --- /dev/null +++ b/apps/heft-native/src/config/tree.rs @@ -0,0 +1,172 @@ +use std::borrow::Cow; + +use super::fallback::{fallback, ConfigResult}; +use super::javascript_order::{is_object_prototype_property_name, order_entries_like_javascript}; +use crate::json::{JsonNumber, JsonObject, JsonValue}; + +pub type NodeId = u32; +pub type AnnotationId = u32; +pub type Entries<'text> = Vec<(Cow<'text, str>, NodeId)>; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Slot { + Undefined, + Node(NodeId), +} + +pub enum NodeValue<'text> { + Null, + Boolean(bool), + Number(JsonNumber<'text>), + String(Cow<'text, str>), + Array(Vec), + Object(Entries<'text>), +} + +pub struct Annotation<'text> { + pub configuration_file: Option, + pub original_values: Vec<(Cow<'text, str>, Slot)>, + pub schema_property_original_value: Option, +} + +pub struct Node<'text> { + pub value: NodeValue<'text>, + pub annotation: Option, +} + +#[derive(Default)] +pub struct ConfigTree<'text> { + pub nodes: Vec>, + pub annotations: Vec>, + pub configuration_file_paths: Vec, +} + +impl<'text> ConfigTree<'text> { + pub fn add_node( + &mut self, + value: NodeValue<'text>, + annotation: Option, + ) -> NodeId { + self.nodes.push(Node { value, annotation }); + (self.nodes.len() - 1) as NodeId + } + + pub fn add_object( + &mut self, + mut entries: Entries<'text>, + annotation: Option, + ) -> NodeId { + order_entries_like_javascript(&mut entries); + self.add_node(NodeValue::Object(entries), annotation) + } + + pub fn add_annotation(&mut self, mut annotation: Annotation<'text>) -> AnnotationId { + order_entries_like_javascript(&mut annotation.original_values); + self.annotations.push(annotation); + (self.annotations.len() - 1) as AnnotationId + } + + pub fn add_configuration_file_path(&mut self, path: &str) -> u32 { + self.configuration_file_paths.push(path.to_string()); + (self.configuration_file_paths.len() - 1) as u32 + } + + pub fn node(&self, id: NodeId) -> &Node<'text> { + &self.nodes[id as usize] + } + + pub fn import_json(&mut self, value: JsonValue<'text>) -> ConfigResult { + let node_value: NodeValue<'text> = match value { + JsonValue::Null => NodeValue::Null, + JsonValue::Boolean(flag) => NodeValue::Boolean(flag), + JsonValue::Number(number) => NodeValue::Number(number), + JsonValue::String(text) => NodeValue::String(text), + JsonValue::Array(items) => { + let mut children: Vec = Vec::with_capacity(items.len()); + for item in items { + children.push(self.import_json(item)?); + } + NodeValue::Array(children) + } + JsonValue::Object(object) => return self.import_json_object(object), + }; + Ok(self.add_node(node_value, None)) + } + + fn import_json_object(&mut self, object: JsonObject<'text>) -> ConfigResult { + let mut children: Entries<'text> = Vec::with_capacity(object.len()); + for (key, item) in object.into_entries() { + if is_object_prototype_property_name(&key) { + return fallback("a property name is a member of Object.prototype"); + } + let child: NodeId = self.import_json(item)?; + children.push((key, child)); + } + Ok(self.add_object(children, None)) + } + + pub fn object_entries(&self, id: NodeId) -> &[(Cow<'text, str>, NodeId)] { + match &self.node(id).value { + NodeValue::Object(entries) => entries, + _ => &[], + } + } + + pub fn own_enumerable_entries(&self, id: NodeId) -> Vec<(Cow<'text, str>, Slot)> { + match &self.node(id).value { + NodeValue::Object(entries) => entries + .iter() + .map(|(key, child)| (key.clone(), Slot::Node(*child))) + .collect(), + NodeValue::Array(items) => items + .iter() + .enumerate() + .map(|(index, child)| (Cow::Owned(index.to_string()), Slot::Node(*child))) + .collect(), + _ => Vec::new(), + } + } + + pub fn annotate_properties(&mut self, root: NodeId, configuration_file: u32) { + let mut queue: Vec = vec![root]; + let mut index: usize = 0; + while index < queue.len() { + let id: NodeId = queue[index]; + index += 1; + if !self.is_object_or_array(id) { + continue; + } + let original_values: Vec<(Cow<'text, str>, Slot)> = self.own_enumerable_entries(id); + for (_, slot) in &original_values { + if let Slot::Node(child) = slot { + queue.push(*child); + } + } + let annotation: AnnotationId = self.add_annotation(Annotation { + configuration_file: Some(configuration_file), + original_values, + schema_property_original_value: None, + }); + self.nodes[id as usize].annotation = Some(annotation); + } + } + + pub fn get(&self, id: NodeId, key: &str) -> Option { + self.object_entries(id) + .iter() + .find(|(entry_key, _)| entry_key == key) + .map(|(_, child)| *child) + } + + pub fn original_value(&self, id: NodeId, key: &str) -> Slot { + let annotation: AnnotationId = match self.node(id).annotation { + Some(annotation) => annotation, + None => return Slot::Undefined, + }; + let original_values = &self.annotations[annotation as usize].original_values; + original_values + .iter() + .find(|(original_key, _)| original_key == key) + .map_or(Slot::Undefined, |(_, slot)| *slot) + } +} diff --git a/apps/heft-native/src/config/tree_json.rs b/apps/heft-native/src/config/tree_json.rs new file mode 100644 index 0000000000..7144003f1f --- /dev/null +++ b/apps/heft-native/src/config/tree_json.rs @@ -0,0 +1,28 @@ +use super::tree::{ConfigTree, NodeId, NodeValue}; +use crate::json::{JsonObject, JsonValue}; + +pub fn tree_to_json_value<'text>(tree: &ConfigTree<'text>, id: NodeId) -> JsonValue<'text> { + match &tree.node(id).value { + NodeValue::Null => JsonValue::Null, + NodeValue::Boolean(flag) => JsonValue::Boolean(*flag), + NodeValue::Number(number) => JsonValue::Number(*number), + NodeValue::String(text) => JsonValue::String(text.clone()), + NodeValue::Array(items) => JsonValue::Array( + items + .iter() + .map(|child| tree_to_json_value(tree, *child)) + .collect(), + ), + NodeValue::Object(entries) => { + let mut object: JsonObject<'text> = JsonObject::with_capacity(entries.len()); + for (key, child) in entries { + object.set_keeping_first_position(key.clone(), tree_to_json_value(tree, *child)); + } + JsonValue::Object(object) + } + } +} + +pub fn empty_json_object<'text>() -> JsonValue<'text> { + JsonValue::Object(JsonObject::default()) +} diff --git a/apps/heft-native/src/config/tree_properties.rs b/apps/heft-native/src/config/tree_properties.rs new file mode 100644 index 0000000000..907732b544 --- /dev/null +++ b/apps/heft-native/src/config/tree_properties.rs @@ -0,0 +1,62 @@ +use super::fallback::{fallback, ConfigResult}; +use super::tree::{ConfigTree, NodeId, NodeValue}; +use super::tree_json::tree_to_json_value; +use crate::json::JsonValue; + +pub fn optional_string<'a>( + tree: &'a ConfigTree<'a>, + object: NodeId, + key: &str, +) -> ConfigResult> { + match tree.get(object, key) { + None => Ok(None), + Some(value) => match tree.string_value(value) { + Some(text) => Ok(Some(text)), + None => fallback("a heft.json property is not a string"), + }, + } +} + +pub fn string_list<'a>( + tree: &'a ConfigTree<'a>, + object: NodeId, + key: &str, +) -> ConfigResult> { + let items: &[NodeId] = match tree.get(object, key).map(|list| &tree.node(list).value) { + None => return Ok(Vec::new()), + Some(NodeValue::Array(items)) => items, + Some(_) => return fallback("a heft.json property is not an array"), + }; + let mut strings: Vec<&'a str> = Vec::with_capacity(items.len()); + for item in items { + match tree.string_value(*item) { + Some(text) => strings.push(text), + None => return fallback("a heft.json array item is not a string"), + } + } + Ok(strings) +} + +pub fn json_property<'a>( + tree: &ConfigTree<'a>, + object: NodeId, + key: &str, +) -> Option> { + tree.get(object, key) + .map(|value| tree_to_json_value(tree, value)) +} + +pub fn object_members<'a>( + tree: &'a ConfigTree<'a>, + object: NodeId, + key: &str, +) -> Vec<(&'a str, NodeId)> { + match tree.get(object, key) { + Some(members) => tree + .object_entries(members) + .iter() + .map(|(name, child)| (name.as_ref(), *child)) + .collect(), + None => Vec::new(), + } +} diff --git a/apps/heft-native/src/config/tree_queries.rs b/apps/heft-native/src/config/tree_queries.rs new file mode 100644 index 0000000000..9ce6061954 --- /dev/null +++ b/apps/heft-native/src/config/tree_queries.rs @@ -0,0 +1,38 @@ +use super::tree::{ConfigTree, NodeId, NodeValue}; + +impl ConfigTree<'_> { + pub fn string_value(&self, id: NodeId) -> Option<&str> { + match &self.node(id).value { + NodeValue::String(text) => Some(text), + _ => None, + } + } + + pub fn is_object(&self, id: NodeId) -> bool { + matches!(self.node(id).value, NodeValue::Object(_)) + } + + pub fn is_array(&self, id: NodeId) -> bool { + matches!(self.node(id).value, NodeValue::Array(_)) + } + + pub fn is_object_or_array(&self, id: NodeId) -> bool { + matches!( + self.node(id).value, + NodeValue::Object(_) | NodeValue::Array(_) + ) + } + + pub fn is_null(&self, id: NodeId) -> bool { + matches!(self.node(id).value, NodeValue::Null) + } + + pub fn is_falsy(&self, id: NodeId) -> bool { + match &self.node(id).value { + NodeValue::Null | NodeValue::Boolean(false) => true, + NodeValue::Number(number) => number.value == 0.0, + NodeValue::String(text) => text.is_empty(), + _ => false, + } + } +} diff --git a/apps/heft-native/src/graph/mod.rs b/apps/heft-native/src/graph/mod.rs new file mode 100644 index 0000000000..daeaa7dd3e --- /dev/null +++ b/apps/heft-native/src/graph/mod.rs @@ -0,0 +1,3 @@ +mod operation_graph; + +pub use operation_graph::{plan_sequential_operations, HeftOperation, PhaseShape}; diff --git a/apps/heft-native/src/graph/operation_graph.rs b/apps/heft-native/src/graph/operation_graph.rs new file mode 100644 index 0000000000..af001efaca --- /dev/null +++ b/apps/heft-native/src/graph/operation_graph.rs @@ -0,0 +1,146 @@ +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum HeftOperation { + Phase { phase_index: usize }, + Task { phase_index: usize, task_index: usize }, +} + +pub struct PhaseShape<'model> { + pub dependency_phase_indices: &'model [usize], + pub task_dependency_indices: Vec<&'model [usize]>, +} + +pub struct SequentialOperationPlan { + pub operations_in_execution_order: Vec, + pub selection_is_missing_phase_dependencies: bool, +} + +pub fn plan_sequential_operations( + phases: &[PhaseShape<'_>], + selected_phase_indices: &[usize], +) -> Option { + let mut operations: Vec = Vec::new(); + let mut dependencies: Vec> = Vec::new(); + let operation_index = |operations: &mut Vec, dependencies: &mut Vec>, operation| { + match operations.iter().position(|existing| *existing == operation) { + Some(index) => index, + None => { + operations.push(operation); + dependencies.push(Vec::new()); + operations.len() - 1 + } + } + }; + let mut selection_is_missing_phase_dependencies = false; + for &phase_index in selected_phase_indices { + let phase = phases.get(phase_index)?; + if phase.dependency_phase_indices.iter().any(|dependency| !selected_phase_indices.contains(dependency)) { + selection_is_missing_phase_dependencies = true; + } + let phase_operation = operation_index(&mut operations, &mut dependencies, HeftOperation::Phase { phase_index }); + for (task_index, task_dependencies) in phase.task_dependency_indices.iter().enumerate() { + let task_operation = + operation_index(&mut operations, &mut dependencies, HeftOperation::Task { phase_index, task_index }); + dependencies[task_operation].push(phase_operation); + for &dependency_task_index in task_dependencies.iter() { + let dependency_operation = operation_index( + &mut operations, + &mut dependencies, + HeftOperation::Task { phase_index, task_index: dependency_task_index }, + ); + dependencies[task_operation].push(dependency_operation); + } + for (consuming_phase_index, consuming_phase) in phases.iter().enumerate() { + if consuming_phase.dependency_phase_indices.contains(&phase_index) + && selected_phase_indices.contains(&consuming_phase_index) + { + let consuming_operation = operation_index( + &mut operations, + &mut dependencies, + HeftOperation::Phase { phase_index: consuming_phase_index }, + ); + dependencies[consuming_operation].push(task_operation); + dependencies[consuming_operation].push(phase_operation); + } + } + } + } + let operations_in_execution_order = unique_topological_order(&operations, &dependencies)?; + Some(SequentialOperationPlan { + operations_in_execution_order, + selection_is_missing_phase_dependencies, + }) +} + +fn unique_topological_order(operations: &[HeftOperation], dependencies: &[Vec]) -> Option> { + let mut remaining_dependency_counts: Vec = dependencies + .iter() + .map(|operation_dependencies| { + let mut unique = operation_dependencies.clone(); + unique.sort_unstable(); + unique.dedup(); + unique.len() + }) + .collect(); + let mut completed = vec![false; operations.len()]; + let mut order = Vec::with_capacity(operations.len()); + for _ in 0..operations.len() { + let mut ready = (0..operations.len()).filter(|&index| !completed[index] && remaining_dependency_counts[index] == 0); + let next = ready.next()?; + if ready.next().is_some() { + return None; + } + completed[next] = true; + order.push(operations[next]); + for (index, operation_dependencies) in dependencies.iter().enumerate() { + if !completed[index] && operation_dependencies.contains(&next) { + remaining_dependency_counts[index] -= 1; + } + } + } + Some(order) +} + +#[cfg(test)] +mod tests { + use super::*; + use HeftOperation::*; + + #[test] + fn chained_tasks_and_phases_run_in_one_possible_order() { + let no_dependencies: &[usize] = &[]; + let build = PhaseShape { + dependency_phase_indices: no_dependencies, + task_dependency_indices: vec![&[], &[0], &[1]], + }; + let test = PhaseShape { dependency_phase_indices: &[0], task_dependency_indices: vec![&[]] }; + let plan = plan_sequential_operations(&[build, test], &[0, 1]).unwrap(); + assert!(!plan.selection_is_missing_phase_dependencies); + assert_eq!( + plan.operations_in_execution_order, + vec![ + Phase { phase_index: 0 }, + Task { phase_index: 0, task_index: 0 }, + Task { phase_index: 0, task_index: 1 }, + Task { phase_index: 0, task_index: 2 }, + Phase { phase_index: 1 }, + Task { phase_index: 1, task_index: 0 }, + ] + ); + } + + #[test] + fn parallel_work_or_cycles_are_not_sequential() { + let parallel = PhaseShape { dependency_phase_indices: &[], task_dependency_indices: vec![&[], &[]] }; + assert!(plan_sequential_operations(&[parallel], &[0]).is_none()); + let cycle = PhaseShape { dependency_phase_indices: &[], task_dependency_indices: vec![&[1], &[0]] }; + assert!(plan_sequential_operations(&[cycle], &[0]).is_none()); + let empty = PhaseShape { dependency_phase_indices: &[], task_dependency_indices: vec![] }; + let consumer = PhaseShape { dependency_phase_indices: &[0], task_dependency_indices: vec![&[]] }; + assert!(plan_sequential_operations(&[empty, consumer], &[0, 1]).is_none()); + let only_test = PhaseShape { dependency_phase_indices: &[0], task_dependency_indices: vec![&[]] }; + let build = PhaseShape { dependency_phase_indices: &[], task_dependency_indices: vec![&[]] }; + let plan = plan_sequential_operations(&[build, only_test], &[1]).unwrap(); + assert!(plan.selection_is_missing_phase_dependencies); + assert_eq!(plan.operations_in_execution_order.len(), 2); + } +} diff --git a/apps/heft-native/src/host_link/mod.rs b/apps/heft-native/src/host_link/mod.rs new file mode 100644 index 0000000000..5a2b848b64 --- /dev/null +++ b/apps/heft-native/src/host_link/mod.rs @@ -0,0 +1,75 @@ +mod plan_file; +mod plan_writer; +#[cfg(unix)] +mod warm_client; +#[cfg(unix)] +mod warm_frames; +#[cfg(unix)] +mod warm_run_payload; +#[cfg(unix)] +mod warm_socket; + +#[cfg(all(test, target_os = "linux"))] +mod tests_plan_file; +#[cfg(all(test, unix))] +mod tests_warm_protocol; + +use std::ffi::OsString; + +use crate::process::{ + exec_javascript_heft, exec_node_script, exit_because_node_could_not_be_started, +}; +use crate::version::{write_version_selector_banner, NativeHeftContext}; + +pub use plan_writer::NodeHostPlanWriter; + +pub const NODE_HOST_ENTRY_PATH_INSIDE_HEFT_PACKAGE: &str = "lib-commonjs/host/HostEntry.js"; +pub const HEFT_BIN_PATH_INSIDE_HEFT_PACKAGE: &str = "bin/heft"; +const PLAN_FILE_DESCRIPTOR_ARGUMENT_PREFIX: &str = "--heft-plan-fd="; + +pub struct NodeHostPlan { + pub plan_json: String, +} + +pub fn run_in_node_host( + native_heft_context: &NativeHeftContext, + node_host_plan: &NodeHostPlan, + command_line_arguments: &[OsString], +) -> ! { + #[cfg(unix)] + if warm_client::warm_node_host_was_requested() { + if let Some(exit_code) = + warm_client::try_run_in_warm_node_host(native_heft_context, node_host_plan) + { + let _ = std::io::Write::flush(&mut std::io::stdout()); + std::process::exit(exit_code); + } + } + exec_cold_node_host(native_heft_context, node_host_plan, command_line_arguments) +} + +pub fn exec_cold_node_host( + native_heft_context: &NativeHeftContext, + node_host_plan: &NodeHostPlan, + command_line_arguments: &[OsString], +) -> ! { + let Some(heft_package_folder) = &native_heft_context.companion_heft_package_folder else { + exec_javascript_heft(command_line_arguments) + }; + let node_host_entry = heft_package_folder.join(NODE_HOST_ENTRY_PATH_INSIDE_HEFT_PACKAGE); + if !node_host_entry.is_file() { + exec_javascript_heft(command_line_arguments) + } + let Some(plan_file) = + plan_file::write_plan_to_inheritable_anonymous_file(&node_host_plan.plan_json) + else { + exec_javascript_heft(command_line_arguments) + }; + write_version_selector_banner(native_heft_context.version_selector_banner); + let plan_file_argument = OsString::from(format!( + "{PLAN_FILE_DESCRIPTOR_ARGUMENT_PREFIX}{}", + plan_file.file_descriptor + )); + let node_host_start_error = exec_node_script(&node_host_entry, &[plan_file_argument]); + exit_because_node_could_not_be_started(&node_host_start_error) +} diff --git a/apps/heft-native/src/host_link/plan_file.rs b/apps/heft-native/src/host_link/plan_file.rs new file mode 100644 index 0000000000..15cd9290c2 --- /dev/null +++ b/apps/heft-native/src/host_link/plan_file.rs @@ -0,0 +1,60 @@ +use std::fs::File; +use std::path::{Path, PathBuf}; + +const FOLDERS_FOR_ANONYMOUS_FILES_ENVIRONMENT_VARIABLES: [&str; 2] = ["XDG_RUNTIME_DIR", "TMPDIR"]; +const FALLBACK_FOLDERS_FOR_ANONYMOUS_FILES: [&str; 2] = ["/dev/shm", "/tmp"]; + +pub struct InheritablePlanFile { + pub file_descriptor: i32, + _file: File, +} + +pub fn write_plan_to_inheritable_anonymous_file(plan_json: &str) -> Option { + folders_for_anonymous_files() + .find_map(|folder| write_plan_to_anonymous_file_in_folder(&folder, plan_json)) +} + +fn folders_for_anonymous_files() -> impl Iterator { + FOLDERS_FOR_ANONYMOUS_FILES_ENVIRONMENT_VARIABLES + .into_iter() + .filter_map(std::env::var_os) + .filter(|folder| !folder.is_empty()) + .map(PathBuf::from) + .chain( + FALLBACK_FOLDERS_FOR_ANONYMOUS_FILES + .into_iter() + .map(PathBuf::from), + ) +} + +#[cfg(target_os = "linux")] +fn write_plan_to_anonymous_file_in_folder( + folder: &Path, + plan_json: &str, +) -> Option { + use std::io::{Seek, SeekFrom, Write}; + use std::os::unix::fs::OpenOptionsExt; + const OPEN_UNNAMED_TEMPORARY_FILE: i32 = 0o20200000; + let mut file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .custom_flags(OPEN_UNNAMED_TEMPORARY_FILE) + .mode(0o600) + .open(folder) + .ok()?; + file.write_all(plan_json.as_bytes()).ok()?; + file.seek(SeekFrom::Start(0)).ok()?; + let file_descriptor = crate::sys::let_executed_program_inherit_file(&file).ok()?; + Some(InheritablePlanFile { + file_descriptor, + _file: file, + }) +} + +#[cfg(not(target_os = "linux"))] +fn write_plan_to_anonymous_file_in_folder( + _folder: &Path, + _plan_json: &str, +) -> Option { + None +} diff --git a/apps/heft-native/src/host_link/plan_writer.rs b/apps/heft-native/src/host_link/plan_writer.rs new file mode 100644 index 0000000000..001a50d632 --- /dev/null +++ b/apps/heft-native/src/host_link/plan_writer.rs @@ -0,0 +1,51 @@ +use std::ffi::OsString; + +use crate::json::write_json_string_for_javascript; +use crate::version::{NativeHeftContext, HEFT_VERSION_IMPLEMENTED_BY_THIS_BINARY}; + +use super::{NodeHostPlan, HEFT_BIN_PATH_INSIDE_HEFT_PACKAGE}; + +const NODE_HOST_PLAN_PROTOCOL_VERSION: u32 = 1; +const TYPICAL_NODE_HOST_PLAN_CAPACITY: usize = 32 * 1024; + +pub struct NodeHostPlanWriter { + plan_json: String, +} + +impl NodeHostPlanWriter { + pub fn start( + command_line_arguments: &[OsString], + native_heft_context: &NativeHeftContext, + ) -> Option { + let companion_heft_bin = native_heft_context + .companion_heft_package_folder + .as_ref()? + .join(HEFT_BIN_PATH_INSIDE_HEFT_PACKAGE); + let current_folder = std::env::current_dir().ok()?; + let mut plan_json = String::with_capacity(TYPICAL_NODE_HOST_PLAN_CAPACITY); + plan_json.push_str("{\"kind\":\"heft-plan\",\"protocolVersion\":"); + plan_json.push_str(&NODE_HOST_PLAN_PROTOCOL_VERSION.to_string()); + plan_json.push_str(",\"heftVersion\":"); + write_json_string_for_javascript(HEFT_VERSION_IMPLEMENTED_BY_THIS_BINARY, &mut plan_json) + .ok()?; + plan_json.push_str(",\"argv\":["); + for (argument_index, argument) in command_line_arguments.iter().enumerate() { + if argument_index > 0 { + plan_json.push(','); + } + write_json_string_for_javascript(argument.to_str()?, &mut plan_json).ok()?; + } + plan_json.push_str("],\"cwd\":"); + write_json_string_for_javascript(current_folder.to_str()?, &mut plan_json).ok()?; + plan_json.push_str(",\"heftBinPath\":"); + write_json_string_for_javascript(companion_heft_bin.to_str()?, &mut plan_json).ok()?; + Some(NodeHostPlanWriter { plan_json }) + } + + pub fn finish(mut self) -> NodeHostPlan { + self.plan_json.push('}'); + NodeHostPlan { + plan_json: self.plan_json, + } + } +} diff --git a/apps/heft-native/src/host_link/tests_plan_file.rs b/apps/heft-native/src/host_link/tests_plan_file.rs new file mode 100644 index 0000000000..60307e1365 --- /dev/null +++ b/apps/heft-native/src/host_link/tests_plan_file.rs @@ -0,0 +1,39 @@ +use super::plan_file::write_plan_to_inheritable_anonymous_file; + +const CLOSE_ON_EXEC_OPEN_FLAG: u32 = 0o2000000; + +fn file_descriptor_information_field(file_descriptor: i32, field_name: &str) -> String { + let file_descriptor_information = + std::fs::read_to_string(format!("/proc/self/fdinfo/{file_descriptor}")).unwrap(); + file_descriptor_information + .lines() + .find_map(|line| line.strip_prefix(field_name)) + .unwrap() + .trim() + .to_owned() +} + +#[test] +fn plan_is_written_to_an_unnamed_inheritable_file_positioned_at_its_start() { + let plan_json = "{\"kind\":\"heft-plan\",\"argv\":[\"build\"]}"; + let plan_file = write_plan_to_inheritable_anonymous_file(plan_json).unwrap(); + let file_descriptor = plan_file.file_descriptor; + let open_flags = u32::from_str_radix( + &file_descriptor_information_field(file_descriptor, "flags:"), + 8, + ) + .unwrap(); + assert_eq!(open_flags & CLOSE_ON_EXEC_OPEN_FLAG, 0); + assert_eq!( + file_descriptor_information_field(file_descriptor, "pos:"), + "0" + ); + let link_target = std::fs::read_link(format!("/proc/self/fd/{file_descriptor}")).unwrap(); + assert!( + link_target.to_string_lossy().ends_with("(deleted)"), + "{link_target:?}" + ); + let plan_text_read_back = + std::fs::read_to_string(format!("/proc/self/fd/{file_descriptor}")).unwrap(); + assert_eq!(plan_text_read_back, plan_json); +} diff --git a/apps/heft-native/src/host_link/tests_warm_protocol.rs b/apps/heft-native/src/host_link/tests_warm_protocol.rs new file mode 100644 index 0000000000..362656f0ec --- /dev/null +++ b/apps/heft-native/src/host_link/tests_warm_protocol.rs @@ -0,0 +1,106 @@ +use std::io::Cursor; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; + +use super::warm_frames::{ + exit_code_of_exit_frame, read_frame, write_frame, EXIT_FRAME, MAXIMUM_FRAME_PAYLOAD_BYTES, + STANDARD_OUTPUT_FRAME, +}; +use super::warm_socket::{ + count_live_warm_hosts_removing_stale_sockets, folder_is_private_to_this_user, + warm_host_socket_path, +}; + +#[test] +fn frames_round_trip_and_end_cleanly_only_at_frame_boundaries() { + let mut encoded_frames = Vec::new(); + write_frame(&mut encoded_frames, STANDARD_OUTPUT_FRAME, b"partial line").unwrap(); + write_frame(&mut encoded_frames, EXIT_FRAME, &(-3i32).to_le_bytes()).unwrap(); + assert_eq!(&encoded_frames[..5], &[12, 0, 0, 0, STANDARD_OUTPUT_FRAME]); + let mut input = Cursor::new(encoded_frames.clone()); + let mut payload = Vec::new(); + assert_eq!( + read_frame(&mut input, &mut payload).unwrap(), + Some(STANDARD_OUTPUT_FRAME) + ); + assert_eq!(payload, b"partial line"); + assert_eq!( + read_frame(&mut input, &mut payload).unwrap(), + Some(EXIT_FRAME) + ); + assert_eq!(exit_code_of_exit_frame(&payload), Some(-3)); + assert_eq!(read_frame(&mut input, &mut payload).unwrap(), None); + let mut truncated_input = Cursor::new(encoded_frames[..8].to_vec()); + assert!(read_frame(&mut truncated_input, &mut payload).is_err()); + assert_eq!(exit_code_of_exit_frame(&[1, 2, 3]), None); +} + +#[test] +fn oversized_frames_are_rejected_before_allocating() { + let oversized_length = (MAXIMUM_FRAME_PAYLOAD_BYTES as u32 + 1).to_le_bytes(); + let mut input = Cursor::new(vec![ + oversized_length[0], + oversized_length[1], + oversized_length[2], + oversized_length[3], + STANDARD_OUTPUT_FRAME, + ]); + let mut payload = Vec::new(); + assert!(read_frame(&mut input, &mut payload).is_err()); + assert_eq!(payload.capacity(), 0); +} + +#[test] +fn socket_names_are_fnv_1a_64_of_the_nul_joined_identity() { + let folder = Path::new("/run/user/1/heft-host-1"); + assert_eq!( + warm_host_socket_path(folder, &[]), + folder.join("h-cbf29ce484222325.sock") + ); + assert_eq!( + warm_host_socket_path(folder, &["a"]), + folder.join("h-af63dc4c8601ec8c.sock") + ); + assert_ne!( + warm_host_socket_path(folder, &["ab", "c"]), + warm_host_socket_path(folder, &["a", "bc"]) + ); +} + +#[test] +fn only_private_real_folders_of_this_user_are_accepted() { + let root = std::env::temp_dir().join(format!("heft-native-warm-{}", std::process::id())); + let private_folder = root.join("private"); + let shared_folder = root.join("shared"); + let linked_folder = root.join("linked"); + std::fs::create_dir_all(&private_folder).unwrap(); + std::fs::create_dir_all(&shared_folder).unwrap(); + std::fs::set_permissions(&private_folder, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::set_permissions(&shared_folder, std::fs::Permissions::from_mode(0o750)).unwrap(); + std::os::unix::fs::symlink(&private_folder, &linked_folder).unwrap(); + assert!(folder_is_private_to_this_user(&private_folder)); + assert!(!folder_is_private_to_this_user(&shared_folder)); + assert!(!folder_is_private_to_this_user(&linked_folder)); + assert!(!folder_is_private_to_this_user(&root.join("missing"))); + std::fs::remove_dir_all(&root).unwrap(); +} + +#[test] +fn live_warm_hosts_are_counted_and_stale_sockets_are_removed() { + let socket_folder = + std::env::temp_dir().join(format!("heft-native-warm-count-{}", std::process::id())); + std::fs::create_dir_all(&socket_folder).unwrap(); + let live_host = + std::os::unix::net::UnixListener::bind(socket_folder.join("h-live.sock")).unwrap(); + drop(std::os::unix::net::UnixListener::bind(socket_folder.join("h-stale.sock")).unwrap()); + std::fs::write(socket_folder.join("h-unrelated.txt"), "").unwrap(); + assert_eq!( + count_live_warm_hosts_removing_stale_sockets(&socket_folder), + 1 + ); + assert!(!socket_folder.join("h-stale.sock").exists()); + assert!(socket_folder.join("h-live.sock").exists()); + assert!(socket_folder.join("h-unrelated.txt").exists()); + drop(live_host); + std::fs::remove_dir_all(&socket_folder).unwrap(); +} diff --git a/apps/heft-native/src/host_link/warm_client.rs b/apps/heft-native/src/host_link/warm_client.rs new file mode 100644 index 0000000000..8e584e88e8 --- /dev/null +++ b/apps/heft-native/src/host_link/warm_client.rs @@ -0,0 +1,168 @@ +use std::ffi::OsString; +use std::io::{IsTerminal, Write}; +use std::os::unix::net::UnixStream; +use std::os::unix::process::CommandExt; +use std::path::PathBuf; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use crate::sys::{ + forward_interrupt_and_termination_signals_to_warm_host, signal_forwarded_to_warm_host, + terminate_by_signal, +}; +use crate::version::{ + find_package_json_path_governing_folder, write_version_selector_banner, NativeHeftContext, +}; + +use super::warm_frames::{ + exit_code_of_exit_frame, read_frame, write_frame, ACCEPT_FRAME, EXIT_FRAME, RUN_FRAME, + STANDARD_ERROR_FRAME, STANDARD_OUTPUT_FRAME, +}; +use super::warm_run_payload::write_run_payload; +use super::warm_socket::{ + count_live_warm_hosts_removing_stale_sockets, ensure_folder_exists_and_is_private_to_this_user, + locate_node_executable_on_path, standard_input_is_the_null_device, warm_host_socket_folder, + warm_host_socket_path, +}; +use super::NodeHostPlan; + +pub const WARM_HOST_OPT_IN_ENVIRONMENT_VARIABLE: &str = "HEFT_WARM_HOST"; +pub const WARM_HOST_ENTRY_PATH_INSIDE_HEFT_PACKAGE: &str = "lib-commonjs/host/WarmHostEntry.js"; +const WARM_HOST_SOCKET_ARGUMENT_PREFIX: &str = "--heft-warm-host-socket="; +const MAXIMUM_LIVE_WARM_HOSTS_PER_USER: usize = 4; +const WARM_HOST_ACCEPT_TIMEOUT: Duration = Duration::from_millis(2000); +const TYPICAL_OUTPUT_FRAME_CAPACITY: usize = 64 * 1024; + +struct WarmHostTarget { + heft_package_folder: PathBuf, + node_executable: PathBuf, + socket_path: PathBuf, +} + +pub fn warm_node_host_was_requested() -> bool { + std::env::var_os(WARM_HOST_OPT_IN_ENVIRONMENT_VARIABLE).is_some_and(|opt_in| opt_in == "1") +} + +pub fn try_run_in_warm_node_host( + native_heft_context: &NativeHeftContext, + node_host_plan: &NodeHostPlan, +) -> Option { + let warm_host_target = locate_warm_host_target(native_heft_context)?; + match connect_and_wait_for_acceptance(&warm_host_target, node_host_plan) { + Some(warm_host_socket) => Some(stream_warm_host_run(warm_host_socket, native_heft_context)), + None => { + start_warm_node_host_in_background(&warm_host_target); + None + } + } +} + +fn locate_warm_host_target(native_heft_context: &NativeHeftContext) -> Option { + if std::io::stdout().is_terminal() + || std::io::stderr().is_terminal() + || !standard_input_is_the_null_device() + { + return None; + } + let companion_folder = native_heft_context.companion_heft_package_folder.as_ref()?; + let heft_package_folder = std::fs::canonicalize(companion_folder).ok()?; + let node_executable = std::fs::canonicalize(locate_node_executable_on_path()?).ok()?; + let mut build_folder = find_package_json_path_governing_folder(&std::env::current_dir().ok()?)?; + build_folder.pop(); + let socket_folder = warm_host_socket_folder(); + if !ensure_folder_exists_and_is_private_to_this_user(&socket_folder) { + return None; + } + let identity_parts = [ + build_folder.to_str()?, + heft_package_folder.to_str()?, + node_executable.to_str()?, + ]; + let socket_path = warm_host_socket_path(&socket_folder, &identity_parts); + Some(WarmHostTarget { + heft_package_folder, + node_executable, + socket_path, + }) +} + +fn connect_and_wait_for_acceptance( + warm_host_target: &WarmHostTarget, + node_host_plan: &NodeHostPlan, +) -> Option { + let mut warm_host_socket = UnixStream::connect(&warm_host_target.socket_path).ok()?; + warm_host_socket + .set_read_timeout(Some(WARM_HOST_ACCEPT_TIMEOUT)) + .ok()?; + warm_host_socket + .set_write_timeout(Some(WARM_HOST_ACCEPT_TIMEOUT)) + .ok()?; + let run_payload = write_run_payload(node_host_plan, &warm_host_target.node_executable)?; + write_frame(&mut warm_host_socket, RUN_FRAME, run_payload.as_bytes()).ok()?; + drop(run_payload); + let mut first_frame_payload = Vec::new(); + match read_frame(&mut warm_host_socket, &mut first_frame_payload) { + Ok(Some(ACCEPT_FRAME)) => {} + _ => return None, + } + warm_host_socket.set_read_timeout(None).ok()?; + Some(warm_host_socket) +} + +fn stream_warm_host_run( + mut warm_host_socket: UnixStream, + native_heft_context: &NativeHeftContext, +) -> i32 { + write_version_selector_banner(native_heft_context.version_selector_banner); + forward_interrupt_and_termination_signals_to_warm_host(&warm_host_socket); + let mut frame_payload = Vec::with_capacity(TYPICAL_OUTPUT_FRAME_CAPACITY); + let mut standard_output = std::io::stdout().lock(); + let mut standard_error = std::io::stderr().lock(); + loop { + match read_frame(&mut warm_host_socket, &mut frame_payload) { + Ok(Some(STANDARD_OUTPUT_FRAME)) => { + let _ = standard_output.write_all(&frame_payload); + let _ = standard_output.flush(); + } + Ok(Some(STANDARD_ERROR_FRAME)) => { + let _ = standard_error.write_all(&frame_payload); + } + Ok(Some(EXIT_FRAME)) => return exit_code_of_exit_frame(&frame_payload).unwrap_or(1), + Ok(Some(_)) => {} + Ok(None) | Err(_) => { + let _ = standard_output.flush(); + if let Some(forwarded_signal) = signal_forwarded_to_warm_host() { + terminate_by_signal(forwarded_signal) + } + return 1; + } + } + } +} + +fn start_warm_node_host_in_background(warm_host_target: &WarmHostTarget) { + let warm_host_entry = warm_host_target + .heft_package_folder + .join(WARM_HOST_ENTRY_PATH_INSIDE_HEFT_PACKAGE); + if !warm_host_entry.is_file() { + return; + } + let Some(socket_folder) = warm_host_target.socket_path.parent() else { + return; + }; + if count_live_warm_hosts_removing_stale_sockets(socket_folder) + >= MAXIMUM_LIVE_WARM_HOSTS_PER_USER + { + return; + } + let mut socket_argument = OsString::from(WARM_HOST_SOCKET_ARGUMENT_PREFIX); + socket_argument.push(&warm_host_target.socket_path); + let _ = Command::new(&warm_host_target.node_executable) + .arg(warm_host_entry) + .arg(socket_argument) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .process_group(0) + .spawn(); +} diff --git a/apps/heft-native/src/host_link/warm_frames.rs b/apps/heft-native/src/host_link/warm_frames.rs new file mode 100644 index 0000000000..29bf036202 --- /dev/null +++ b/apps/heft-native/src/host_link/warm_frames.rs @@ -0,0 +1,51 @@ +use std::io::{self, ErrorKind, Read, Write}; + +pub const RUN_FRAME: u8 = 0x01; +pub const ACCEPT_FRAME: u8 = 0x10; +pub const STANDARD_OUTPUT_FRAME: u8 = 0x12; +pub const STANDARD_ERROR_FRAME: u8 = 0x13; +pub const EXIT_FRAME: u8 = 0x14; +pub const MAXIMUM_FRAME_PAYLOAD_BYTES: usize = 1024 * 1024; +const FRAME_HEADER_BYTES: usize = 5; + +pub fn write_frame(output: &mut impl Write, frame_type: u8, payload: &[u8]) -> io::Result<()> { + let payload_length = + u32::try_from(payload.len()).map_err(|_| io::Error::from(ErrorKind::InvalidInput))?; + let mut frame_header = [0u8; FRAME_HEADER_BYTES]; + frame_header[..4].copy_from_slice(&payload_length.to_le_bytes()); + frame_header[4] = frame_type; + output.write_all(&frame_header)?; + output.write_all(payload) +} + +pub fn read_frame(input: &mut impl Read, payload: &mut Vec) -> io::Result> { + let mut frame_header = [0u8; FRAME_HEADER_BYTES]; + let first_byte_count = loop { + match input.read(&mut frame_header) { + Err(read_error) if read_error.kind() == ErrorKind::Interrupted => continue, + read_result => break read_result?, + } + }; + if first_byte_count == 0 { + return Ok(None); + } + input.read_exact(&mut frame_header[first_byte_count..])?; + let payload_length = u32::from_le_bytes([ + frame_header[0], + frame_header[1], + frame_header[2], + frame_header[3], + ]) as usize; + if payload_length > MAXIMUM_FRAME_PAYLOAD_BYTES { + return Err(io::Error::from(ErrorKind::InvalidData)); + } + payload.clear(); + payload.resize(payload_length, 0); + input.read_exact(payload)?; + Ok(Some(frame_header[4])) +} + +pub fn exit_code_of_exit_frame(payload: &[u8]) -> Option { + let exit_code_bytes: [u8; 4] = payload.try_into().ok()?; + Some(i32::from_le_bytes(exit_code_bytes)) +} diff --git a/apps/heft-native/src/host_link/warm_run_payload.rs b/apps/heft-native/src/host_link/warm_run_payload.rs new file mode 100644 index 0000000000..41e10fb8dd --- /dev/null +++ b/apps/heft-native/src/host_link/warm_run_payload.rs @@ -0,0 +1,46 @@ +use std::path::Path; + +use crate::json::write_json_string_for_javascript; + +use super::NodeHostPlan; + +const WARM_HOST_PROTOCOL_VERSION: u32 = 1; +const TYPICAL_ENVIRONMENT_JSON_CAPACITY: usize = 16 * 1024; +const PROCESS_STATUS_UMASK_FIELD: &str = "Umask:"; + +pub fn write_run_payload(node_host_plan: &NodeHostPlan, node_executable: &Path) -> Option { + let current_folder = std::env::current_dir().ok()?; + let mut run_payload = + String::with_capacity(node_host_plan.plan_json.len() + TYPICAL_ENVIRONMENT_JSON_CAPACITY); + run_payload.push_str("{\"protocolVersion\":"); + run_payload.push_str(&WARM_HOST_PROTOCOL_VERSION.to_string()); + run_payload.push_str(",\"plan\":"); + run_payload.push_str(&node_host_plan.plan_json); + run_payload.push_str(",\"env\":{"); + for (variable_index, (variable_name, variable_value)) in std::env::vars_os().enumerate() { + if variable_index > 0 { + run_payload.push(','); + } + write_json_string_for_javascript(variable_name.to_str()?, &mut run_payload).ok()?; + run_payload.push(':'); + write_json_string_for_javascript(variable_value.to_str()?, &mut run_payload).ok()?; + } + run_payload.push_str("},\"cwd\":"); + write_json_string_for_javascript(current_folder.to_str()?, &mut run_payload).ok()?; + run_payload.push_str(",\"nodeExecPath\":"); + write_json_string_for_javascript(node_executable.to_str()?, &mut run_payload).ok()?; + if let Some(umask) = file_mode_creation_mask_of_this_process() { + run_payload.push_str(",\"umask\":"); + write_json_string_for_javascript(&umask, &mut run_payload).ok()?; + } + run_payload.push_str(",\"stdoutIsTTY\":false,\"stderrIsTTY\":false}"); + Some(run_payload) +} + +fn file_mode_creation_mask_of_this_process() -> Option { + let process_status = std::fs::read_to_string("/proc/self/status").ok()?; + process_status + .lines() + .find_map(|status_line| status_line.strip_prefix(PROCESS_STATUS_UMASK_FIELD)) + .map(|umask_text| umask_text.trim().to_owned()) +} diff --git a/apps/heft-native/src/host_link/warm_socket.rs b/apps/heft-native/src/host_link/warm_socket.rs new file mode 100644 index 0000000000..fa5918b5ce --- /dev/null +++ b/apps/heft-native/src/host_link/warm_socket.rs @@ -0,0 +1,96 @@ +use std::os::unix::fs::{DirBuilderExt, MetadataExt, PermissionsExt}; +use std::path::{Path, PathBuf}; + +use crate::sys::effective_user_id; + +const SOCKET_FOLDER_PARENT_ENVIRONMENT_VARIABLE: &str = "XDG_RUNTIME_DIR"; +const FALLBACK_SOCKET_FOLDER_PARENT: &str = "/tmp"; +const PERMISSION_BITS_FOR_GROUP_AND_OTHERS: u32 = 0o077; +const PRIVATE_FOLDER_PERMISSIONS: u32 = 0o700; +const FNV_1A_64_OFFSET_BASIS: u64 = 0xcbf29ce484222325; +const FNV_1A_64_PRIME: u64 = 0x100000001b3; + +pub fn warm_host_socket_folder() -> PathBuf { + let socket_folder_parent = std::env::var_os(SOCKET_FOLDER_PARENT_ENVIRONMENT_VARIABLE) + .filter(|folder| !folder.is_empty()) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(FALLBACK_SOCKET_FOLDER_PARENT)); + socket_folder_parent.join(format!("heft-host-{}", effective_user_id())) +} + +pub fn ensure_folder_exists_and_is_private_to_this_user(folder: &Path) -> bool { + let _ = std::fs::DirBuilder::new() + .mode(PRIVATE_FOLDER_PERMISSIONS) + .create(folder); + folder_is_private_to_this_user(folder) +} + +pub fn folder_is_private_to_this_user(folder: &Path) -> bool { + match std::fs::symlink_metadata(folder) { + Ok(folder_metadata) => { + folder_metadata.file_type().is_dir() + && folder_metadata.uid() == effective_user_id() + && folder_metadata.permissions().mode() & PERMISSION_BITS_FOR_GROUP_AND_OTHERS == 0 + } + Err(_) => false, + } +} + +pub fn warm_host_socket_path(socket_folder: &Path, identity_parts: &[&str]) -> PathBuf { + let mut identity_hash = FNV_1A_64_OFFSET_BASIS; + for (part_index, identity_part) in identity_parts.iter().enumerate() { + let separator: &[u8] = if part_index > 0 { &[0] } else { &[] }; + for &identity_byte in separator.iter().chain(identity_part.as_bytes()) { + identity_hash ^= u64::from(identity_byte); + identity_hash = identity_hash.wrapping_mul(FNV_1A_64_PRIME); + } + } + socket_folder.join(format!("h-{identity_hash:016x}.sock")) +} + +pub fn count_live_warm_hosts_removing_stale_sockets(socket_folder: &Path) -> usize { + let Ok(folder_entries) = std::fs::read_dir(socket_folder) else { + return 0; + }; + let mut live_warm_host_count = 0; + for folder_entry in folder_entries.flatten() { + let entry_name = folder_entry.file_name(); + let entry_name = entry_name.to_string_lossy(); + if !(entry_name.starts_with("h-") && entry_name.ends_with(".sock")) { + continue; + } + match std::os::unix::net::UnixStream::connect(folder_entry.path()) { + Ok(_) => live_warm_host_count += 1, + Err(connect_error) if connect_error.kind() == std::io::ErrorKind::ConnectionRefused => { + let _ = std::fs::remove_file(folder_entry.path()); + } + Err(_) => {} + } + } + live_warm_host_count +} + +pub fn locate_node_executable_on_path() -> Option { + let search_path = std::env::var_os("PATH")?; + std::env::split_paths(&search_path) + .map(|search_folder| search_folder.join("node")) + .find(|candidate| { + std::fs::metadata(candidate).is_ok_and(|candidate_metadata| { + candidate_metadata.is_file() && candidate_metadata.permissions().mode() & 0o111 != 0 + }) + }) +} + +pub fn standard_input_is_the_null_device() -> bool { + use std::os::unix::fs::FileTypeExt; + match ( + std::fs::metadata("/dev/stdin"), + std::fs::metadata("/dev/null"), + ) { + (Ok(standard_input), Ok(null_device)) => { + standard_input.file_type().is_char_device() + && standard_input.rdev() == null_device.rdev() + } + _ => false, + } +} diff --git a/apps/heft-native/src/json/cursor.rs b/apps/heft-native/src/json/cursor.rs new file mode 100644 index 0000000000..84ae0a4789 --- /dev/null +++ b/apps/heft-native/src/json/cursor.rs @@ -0,0 +1,77 @@ +use super::duplicate_key_index::JsonObjectEntry; +use super::value::JsonValue; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JsonTextNeedsJavaScriptParser; + +pub(super) type JsonScanResult = Result; + +pub(super) const MAXIMUM_NESTING_DEPTH: u32 = 512; + +pub(super) struct JsonCursor<'text> { + pub(super) text: &'text str, + pub(super) bytes: &'text [u8], + pub(super) position: usize, + pub(super) depth: u32, + pub(super) allows_comments_and_trailing_commas: bool, + pub(super) pending_array_items: Vec>, + pub(super) pending_object_entries: Vec>, +} + +impl<'text> JsonCursor<'text> { + pub(super) fn new(text: &'text str, allows_comments_and_trailing_commas: bool) -> Self { + JsonCursor { + text, + bytes: text.as_bytes(), + position: 0, + depth: 0, + allows_comments_and_trailing_commas, + pending_array_items: Vec::new(), + pending_object_entries: Vec::new(), + } + } + + pub(super) fn refuse_unless_comments_and_trailing_commas_are_allowed( + &self, + ) -> JsonScanResult<()> { + if self.allows_comments_and_trailing_commas { + Ok(()) + } else { + Err(JsonTextNeedsJavaScriptParser) + } + } + + pub(super) fn peek_byte(&self) -> Option { + self.bytes.get(self.position).copied() + } + + pub(super) fn peek_byte_after(&self, offset: usize) -> Option { + self.bytes.get(self.position + offset).copied() + } + + pub(super) fn next_byte_or_refuse(&self) -> JsonScanResult { + self.peek_byte().ok_or(JsonTextNeedsJavaScriptParser) + } + + pub(super) fn is_at_line_or_paragraph_separator(&self) -> bool { + self.peek_byte() == Some(0xe2) + && self.peek_byte_after(1) == Some(0x80) + && matches!(self.peek_byte_after(2), Some(0xa8 | 0xa9)) + } + + pub(super) fn is_at_end(&self) -> bool { + self.position >= self.bytes.len() + } + + pub(super) fn enter_nested_value(&mut self) -> JsonScanResult<()> { + self.depth += 1; + if self.depth > MAXIMUM_NESTING_DEPTH { + return Err(JsonTextNeedsJavaScriptParser); + } + Ok(()) + } + + pub(super) fn leave_nested_value(&mut self) { + self.depth -= 1; + } +} diff --git a/apps/heft-native/src/json/duplicate_key_index.rs b/apps/heft-native/src/json/duplicate_key_index.rs new file mode 100644 index 0000000000..1269754e61 --- /dev/null +++ b/apps/heft-native/src/json/duplicate_key_index.rs @@ -0,0 +1,100 @@ +use std::borrow::Cow; + +use super::value::JsonValue; + +const LARGEST_OBJECT_SCANNED_LINEARLY_FOR_DUPLICATE_KEYS: usize = 16; +const EMPTY_KEY_INDEX_SLOT: usize = usize::MAX; + +pub(super) type JsonObjectEntry<'text> = (Cow<'text, str>, JsonValue<'text>); + +pub(super) struct DuplicateKeyIndex { + entry_offset_by_key_hash_slot: Vec, +} + +fn mix_word_into_key_hash(key_hash: u64, word: u64) -> u64 { + (key_hash.rotate_left(5) ^ word).wrapping_mul(0x517c_c1b7_2722_0a95) +} + +fn hash_of_object_key(key: &str) -> u64 { + let mut key_hash = key.len() as u64; + let (whole_words, trailing_bytes) = key.as_bytes().as_chunks::<8>(); + for whole_word in whole_words { + key_hash = mix_word_into_key_hash(key_hash, u64::from_le_bytes(*whole_word)); + } + let mut trailing_word = 0u64; + for (byte_index, byte) in trailing_bytes.iter().enumerate() { + trailing_word |= u64::from(*byte) << (byte_index * 8); + } + mix_word_into_key_hash(key_hash, trailing_word) +} + +impl DuplicateKeyIndex { + pub(super) fn new() -> Self { + DuplicateKeyIndex { + entry_offset_by_key_hash_slot: Vec::new(), + } + } + + pub(super) fn insert_keeping_first_position<'text>( + &mut self, + entries: &mut Vec>, + first_object_entry: usize, + key: Cow<'text, str>, + value: JsonValue<'text>, + ) { + if self.entry_offset_by_key_hash_slot.is_empty() { + let object_entries = &mut entries[first_object_entry..]; + if object_entries.len() < LARGEST_OBJECT_SCANNED_LINEARLY_FOR_DUPLICATE_KEYS { + match object_entries + .iter_mut() + .find(|(existing_key, _)| *existing_key == key) + { + Some(existing_entry) => existing_entry.1 = value, + None => entries.push((key, value)), + } + return; + } + self.rebuild_for_object_entries(&entries[first_object_entry..]); + } + let object_entries = &mut entries[first_object_entry..]; + let slot = self.slot_holding_or_awaiting_key(object_entries, &key); + match self.entry_offset_by_key_hash_slot[slot] { + EMPTY_KEY_INDEX_SLOT => { + let object_entry_count = object_entries.len() + 1; + self.entry_offset_by_key_hash_slot[slot] = object_entries.len(); + entries.push((key, value)); + if object_entry_count * 2 > self.entry_offset_by_key_hash_slot.len() { + self.rebuild_for_object_entries(&entries[first_object_entry..]); + } + } + existing_entry_offset => object_entries[existing_entry_offset].1 = value, + } + } + + fn slot_holding_or_awaiting_key( + &self, + object_entries: &[JsonObjectEntry<'_>], + key: &str, + ) -> usize { + let slot_count = self.entry_offset_by_key_hash_slot.len(); + let mut slot = (hash_of_object_key(key) >> (64 - slot_count.trailing_zeros())) as usize; + loop { + let entry_offset = self.entry_offset_by_key_hash_slot[slot]; + if entry_offset == EMPTY_KEY_INDEX_SLOT || object_entries[entry_offset].0 == key { + return slot; + } + slot = (slot + 1) & (slot_count - 1); + } + } + + fn rebuild_for_object_entries(&mut self, object_entries: &[JsonObjectEntry<'_>]) { + let slot_count = (object_entries.len() * 4).next_power_of_two(); + self.entry_offset_by_key_hash_slot.clear(); + self.entry_offset_by_key_hash_slot + .resize(slot_count, EMPTY_KEY_INDEX_SLOT); + for (entry_offset, (key, _)) in object_entries.iter().enumerate() { + let slot = self.slot_holding_or_awaiting_key(object_entries, key); + self.entry_offset_by_key_hash_slot[slot] = entry_offset; + } + } +} diff --git a/apps/heft-native/src/json/lexer_number.rs b/apps/heft-native/src/json/lexer_number.rs new file mode 100644 index 0000000000..d92b070c32 --- /dev/null +++ b/apps/heft-native/src/json/lexer_number.rs @@ -0,0 +1,54 @@ +use super::cursor::{JsonCursor, JsonScanResult, JsonTextNeedsJavaScriptParser}; +use super::value::JsonNumber; + +impl<'text> JsonCursor<'text> { + pub(super) fn read_number_literal(&mut self) -> JsonScanResult> { + let start = self.position; + if self.peek_byte() == Some(b'-') { + self.position += 1; + } + match self.next_byte_or_refuse()? { + b'0' => self.position += 1, + b'1'..=b'9' => self.skip_decimal_digits(), + _ => return Err(JsonTextNeedsJavaScriptParser), + } + if self.peek_byte() == Some(b'.') { + self.position += 1; + self.require_at_least_one_decimal_digit()?; + } + if let Some(b'e' | b'E') = self.peek_byte() { + self.position += 1; + if let Some(b'+' | b'-') = self.peek_byte() { + self.position += 1; + } + self.require_at_least_one_decimal_digit()?; + } + if let Some(following) = self.peek_byte() { + if following.is_ascii_alphanumeric() + || matches!(following, b'.' | b'_' | b'$') + || following >= 0x80 + { + return Err(JsonTextNeedsJavaScriptParser); + } + } + let source_text = &self.text[start..self.position]; + let value: f64 = source_text + .parse() + .map_err(|_| JsonTextNeedsJavaScriptParser)?; + Ok(JsonNumber { value, source_text }) + } + + fn skip_decimal_digits(&mut self) { + while let Some(b'0'..=b'9') = self.peek_byte() { + self.position += 1; + } + } + + fn require_at_least_one_decimal_digit(&mut self) -> JsonScanResult<()> { + if !matches!(self.peek_byte(), Some(b'0'..=b'9')) { + return Err(JsonTextNeedsJavaScriptParser); + } + self.skip_decimal_digits(); + Ok(()) + } +} diff --git a/apps/heft-native/src/json/lexer_string.rs b/apps/heft-native/src/json/lexer_string.rs new file mode 100644 index 0000000000..e35ff1e474 --- /dev/null +++ b/apps/heft-native/src/json/lexer_string.rs @@ -0,0 +1,124 @@ +use std::borrow::Cow; + +use super::cursor::{JsonCursor, JsonScanResult, JsonTextNeedsJavaScriptParser}; + +const FIRST_HIGH_SURROGATE: u32 = 0xd800; +const FIRST_LOW_SURROGATE: u32 = 0xdc00; +const END_OF_SURROGATES: u32 = 0xe000; + +impl<'text> JsonCursor<'text> { + pub(super) fn read_string_literal(&mut self) -> JsonScanResult> { + self.position += 1; + let content_start = self.position; + loop { + match self.next_byte_or_refuse()? { + b'"' => { + let content = &self.text[content_start..self.position]; + self.position += 1; + return Ok(Cow::Borrowed(content)); + } + b'\\' => { + return self + .read_string_literal_with_escapes(content_start) + .map(Cow::Owned) + } + 0x00..=0x1f => return Err(JsonTextNeedsJavaScriptParser), + 0xe2 if self.is_at_line_or_paragraph_separator() => { + return Err(JsonTextNeedsJavaScriptParser) + } + _ => self.position += 1, + } + } + } + + fn read_string_literal_with_escapes(&mut self, content_start: usize) -> JsonScanResult { + let mut decoded = String::with_capacity(self.position - content_start + 16); + decoded.push_str(&self.text[content_start..self.position]); + loop { + match self.next_byte_or_refuse()? { + b'"' => { + self.position += 1; + return Ok(decoded); + } + b'\\' => { + self.position += 1; + self.decode_escape_sequence_into(&mut decoded)?; + } + 0x00..=0x1f => return Err(JsonTextNeedsJavaScriptParser), + 0xe2 if self.is_at_line_or_paragraph_separator() => { + return Err(JsonTextNeedsJavaScriptParser) + } + _ => self.copy_run_of_ordinary_string_bytes_into(&mut decoded), + } + } + } + + fn copy_run_of_ordinary_string_bytes_into(&mut self, decoded: &mut String) { + let run_start = self.position; + self.position += 1; + while let Some(byte) = self.peek_byte() { + if byte == b'"' || byte == b'\\' || byte < 0x20 || byte == 0xe2 { + break; + } + self.position += 1; + } + decoded.push_str(&self.text[run_start..self.position]); + } + + fn decode_escape_sequence_into(&mut self, decoded: &mut String) -> JsonScanResult<()> { + let escape = self.next_byte_or_refuse()?; + self.position += 1; + let character = match escape { + b'"' => '"', + b'\\' => '\\', + b'/' => '/', + b'b' => '\u{8}', + b'f' => '\u{c}', + b'n' => '\n', + b'r' => '\r', + b't' => '\t', + b'u' => self.decode_unicode_escape_after_u()?, + _ => return Err(JsonTextNeedsJavaScriptParser), + }; + decoded.push(character); + Ok(()) + } + + fn decode_unicode_escape_after_u(&mut self) -> JsonScanResult { + let code_unit = self.read_four_hex_digits()?; + let code_point = if (FIRST_HIGH_SURROGATE..FIRST_LOW_SURROGATE).contains(&code_unit) { + if self.peek_byte() != Some(b'\\') || self.peek_byte_after(1) != Some(b'u') { + return Err(JsonTextNeedsJavaScriptParser); + } + self.position += 2; + let low_code_unit = self.read_four_hex_digits()?; + if !(FIRST_LOW_SURROGATE..END_OF_SURROGATES).contains(&low_code_unit) { + return Err(JsonTextNeedsJavaScriptParser); + } + 0x10000 + + ((code_unit - FIRST_HIGH_SURROGATE) << 10) + + (low_code_unit - FIRST_LOW_SURROGATE) + } else if (FIRST_LOW_SURROGATE..END_OF_SURROGATES).contains(&code_unit) { + return Err(JsonTextNeedsJavaScriptParser); + } else { + code_unit + }; + char::from_u32(code_point).ok_or(JsonTextNeedsJavaScriptParser) + } + + fn read_four_hex_digits(&mut self) -> JsonScanResult { + let digits = self + .bytes + .get(self.position..self.position + 4) + .ok_or(JsonTextNeedsJavaScriptParser)?; + let mut code_unit: u32 = 0; + for &digit in digits { + let digit_value = (digit as char) + .to_digit(16) + .ok_or(JsonTextNeedsJavaScriptParser)?; + code_unit = code_unit * 16 + digit_value; + } + self.position += 4; + Ok(code_unit) + } +} diff --git a/apps/heft-native/src/json/lexer_trivia.rs b/apps/heft-native/src/json/lexer_trivia.rs new file mode 100644 index 0000000000..81622f9b1b --- /dev/null +++ b/apps/heft-native/src/json/lexer_trivia.rs @@ -0,0 +1,48 @@ +use super::cursor::{JsonCursor, JsonScanResult, JsonTextNeedsJavaScriptParser}; + +impl JsonCursor<'_> { + pub(super) fn skip_whitespace_and_comments(&mut self) -> JsonScanResult<()> { + loop { + match self.peek_byte() { + Some(b' ' | b'\t' | b'\n' | b'\r') => self.position += 1, + Some(b'/') => { + self.refuse_unless_comments_and_trailing_commas_are_allowed()?; + match self.peek_byte_after(1) { + Some(b'/') => self.skip_line_comment()?, + Some(b'*') => self.skip_block_comment()?, + _ => return Err(JsonTextNeedsJavaScriptParser), + } + } + _ => return Ok(()), + } + } + } + + fn skip_line_comment(&mut self) -> JsonScanResult<()> { + self.position += 2; + while let Some(byte) = self.peek_byte() { + if byte == b'\n' || byte == b'\r' { + return Ok(()); + } + if self.is_at_line_or_paragraph_separator() { + return Err(JsonTextNeedsJavaScriptParser); + } + self.position += 1; + } + Ok(()) + } + + fn skip_block_comment(&mut self) -> JsonScanResult<()> { + self.position += 2; + loop { + match self.peek_byte() { + None => return Err(JsonTextNeedsJavaScriptParser), + Some(b'*') if self.peek_byte_after(1) == Some(b'/') => { + self.position += 2; + return Ok(()); + } + Some(_) => self.position += 1, + } + } + } +} diff --git a/apps/heft-native/src/json/mod.rs b/apps/heft-native/src/json/mod.rs new file mode 100644 index 0000000000..6c9b2414ae --- /dev/null +++ b/apps/heft-native/src/json/mod.rs @@ -0,0 +1,21 @@ +mod cursor; +mod duplicate_key_index; +mod lexer_number; +mod lexer_string; +mod lexer_trivia; +mod parser; +mod value; +mod writer; + +pub use parser::{parse_json_exactly_like_json_parse, parse_json_with_comments_exactly_like_jju}; +pub use value::{JsonNumber, JsonObject, JsonValue}; +pub use writer::{write_json_for_javascript, write_json_string_for_javascript}; + +#[cfg(test)] +mod tests_allocation_counts; +#[cfg(test)] +mod tests_large_objects; +#[cfg(test)] +mod tests_parser_acceptance; +#[cfg(test)] +mod tests_zero_copy_and_writer; diff --git a/apps/heft-native/src/json/parser.rs b/apps/heft-native/src/json/parser.rs new file mode 100644 index 0000000000..9acab4dc01 --- /dev/null +++ b/apps/heft-native/src/json/parser.rs @@ -0,0 +1,155 @@ +use super::cursor::{JsonCursor, JsonScanResult, JsonTextNeedsJavaScriptParser}; +use super::duplicate_key_index::DuplicateKeyIndex; +use super::value::{JsonObject, JsonValue}; + +pub fn parse_json_with_comments_exactly_like_jju( + text: &str, +) -> Result, JsonTextNeedsJavaScriptParser> { + parse_json_with_cursor(JsonCursor::new(text, true)) +} + +pub fn parse_json_exactly_like_json_parse( + text: &str, +) -> Result, JsonTextNeedsJavaScriptParser> { + parse_json_with_cursor(JsonCursor::new(text, false)) +} + +fn parse_json_with_cursor( + cursor: JsonCursor<'_>, +) -> Result, JsonTextNeedsJavaScriptParser> { + let mut cursor = cursor; + cursor.skip_whitespace_and_comments()?; + let value = cursor.read_value()?; + cursor.skip_whitespace_and_comments()?; + if !cursor.is_at_end() { + return Err(JsonTextNeedsJavaScriptParser); + } + Ok(value) +} + +impl<'text> JsonCursor<'text> { + fn read_value(&mut self) -> JsonScanResult> { + match self.next_byte_or_refuse()? { + b'{' => self.read_object(), + b'[' => self.read_array(), + b'"' => Ok(JsonValue::String(self.read_string_literal()?)), + b't' => self.read_keyword(b"true", JsonValue::Boolean(true)), + b'f' => self.read_keyword(b"false", JsonValue::Boolean(false)), + b'n' => self.read_keyword(b"null", JsonValue::Null), + b'-' | b'0'..=b'9' => Ok(JsonValue::Number(self.read_number_literal()?)), + _ => Err(JsonTextNeedsJavaScriptParser), + } + } + + fn read_keyword( + &mut self, + keyword: &[u8], + value: JsonValue<'text>, + ) -> JsonScanResult> { + if !self.bytes[self.position..].starts_with(keyword) { + return Err(JsonTextNeedsJavaScriptParser); + } + self.position += keyword.len(); + if let Some(following) = self.peek_byte() { + if following.is_ascii_alphanumeric() + || matches!(following, b'_' | b'$' | b'\\') + || following >= 0x80 + { + return Err(JsonTextNeedsJavaScriptParser); + } + } + Ok(value) + } + + fn read_array(&mut self) -> JsonScanResult> { + self.enter_nested_value()?; + self.position += 1; + self.skip_whitespace_and_comments()?; + let first_pending_item = self.pending_array_items.len(); + let mut is_first_item = true; + loop { + self.skip_whitespace_and_comments()?; + if self.next_byte_or_refuse()? == b']' { + if !is_first_item { + self.refuse_unless_comments_and_trailing_commas_are_allowed()?; + } + self.position += 1; + break; + } + is_first_item = false; + let item = self.read_value()?; + self.pending_array_items.push(item); + self.skip_whitespace_and_comments()?; + match self.next_byte_or_refuse()? { + b',' => self.position += 1, + b']' => { + self.position += 1; + break; + } + _ => return Err(JsonTextNeedsJavaScriptParser), + } + self.skip_whitespace_and_comments()?; + if self.next_byte_or_refuse()? == b',' { + return Err(JsonTextNeedsJavaScriptParser); + } + } + self.leave_nested_value(); + let items: Vec> = self + .pending_array_items + .drain(first_pending_item..) + .collect(); + Ok(JsonValue::Array(items)) + } + + fn read_object(&mut self) -> JsonScanResult> { + self.enter_nested_value()?; + self.position += 1; + self.skip_whitespace_and_comments()?; + let first_pending_entry = self.pending_object_entries.len(); + let mut duplicate_key_index = DuplicateKeyIndex::new(); + let mut is_first_entry = true; + loop { + self.skip_whitespace_and_comments()?; + match self.next_byte_or_refuse()? { + b'}' => { + if !is_first_entry { + self.refuse_unless_comments_and_trailing_commas_are_allowed()?; + } + self.position += 1; + break; + } + b'"' => is_first_entry = false, + _ => return Err(JsonTextNeedsJavaScriptParser), + } + let key = self.read_string_literal()?; + self.skip_whitespace_and_comments()?; + if self.next_byte_or_refuse()? != b':' { + return Err(JsonTextNeedsJavaScriptParser); + } + self.position += 1; + self.skip_whitespace_and_comments()?; + let value = self.read_value()?; + duplicate_key_index.insert_keeping_first_position( + &mut self.pending_object_entries, + first_pending_entry, + key, + value, + ); + self.skip_whitespace_and_comments()?; + match self.next_byte_or_refuse()? { + b',' => self.position += 1, + b'}' => { + self.position += 1; + break; + } + _ => return Err(JsonTextNeedsJavaScriptParser), + } + } + self.leave_nested_value(); + let entries = self + .pending_object_entries + .drain(first_pending_entry..) + .collect(); + Ok(JsonValue::Object(JsonObject { entries })) + } +} diff --git a/apps/heft-native/src/json/tests_allocation_counts.rs b/apps/heft-native/src/json/tests_allocation_counts.rs new file mode 100644 index 0000000000..d722a6932d --- /dev/null +++ b/apps/heft-native/src/json/tests_allocation_counts.rs @@ -0,0 +1,63 @@ +use crate::sys::allocation_counter::count_allocations_on_this_thread_while_running; + +use super::{parse_json_with_comments_exactly_like_jju, write_json_for_javascript, JsonValue}; + +#[test] +fn parsing_allocates_one_exact_vec_per_non_empty_container_plus_two_pending_stacks() { + let text = "{\"phasesByName\": {\"build\": {\"tasksByName\": {}}}, \"heftPlugins\": [{\"pluginPackage\": \"a\"}], \"none\": []}"; + let (value, allocations) = count_allocations_on_this_thread_while_running(|| { + parse_json_with_comments_exactly_like_jju(text) + }); + assert!(value.is_ok()); + assert_eq!(allocations, 5 + 2); +} + +#[test] +fn escaped_strings_cost_exactly_one_allocation_each() { + let text = "[\"plain\", \"with \\\\ escape\", \"\\u0041\"]"; + let (value, allocations) = count_allocations_on_this_thread_while_running(|| { + parse_json_with_comments_exactly_like_jju(text) + }); + assert!(value.is_ok()); + assert_eq!(allocations, 1 + 2 + 1); +} + +#[test] +fn writing_into_a_preallocated_buffer_does_not_allocate() { + let value = + parse_json_with_comments_exactly_like_jju("{\"a\": [1, \"b\", null, {\"c\": true}]}") + .unwrap(); + let mut output = String::with_capacity(64); + let (result, allocations) = count_allocations_on_this_thread_while_running(|| { + write_json_for_javascript(&value, &mut output) + }); + assert!(result.is_ok()); + assert_eq!(allocations, 0); + assert_eq!(output, "{\"a\":[1,\"b\",null,{\"c\":true}]}"); +} + +fn every_container_has_exactly_the_capacity_it_needs(value: &JsonValue<'_>) -> bool { + match value { + JsonValue::Array(items) => { + items.capacity() == items.len() + && items + .iter() + .all(every_container_has_exactly_the_capacity_it_needs) + } + JsonValue::Object(object) => { + object.entries.capacity() == object.entries.len() + && object + .entries + .iter() + .all(|(_, item)| every_container_has_exactly_the_capacity_it_needs(item)) + } + _ => true, + } +} + +#[test] +fn parsed_containers_are_allocated_with_exactly_the_capacity_they_need() { + let text = "{\"a\": [1, 2, 3, {\"b\": [], \"c\": {}}, [[4]]], \"d\": {\"e\": 5, \"f\": [6, 7, 8, 9, 10], \"e\": 11}}"; + let value = parse_json_with_comments_exactly_like_jju(text).unwrap(); + assert!(every_container_has_exactly_the_capacity_it_needs(&value)); +} diff --git a/apps/heft-native/src/json/tests_large_objects.rs b/apps/heft-native/src/json/tests_large_objects.rs new file mode 100644 index 0000000000..44ba8bddd0 --- /dev/null +++ b/apps/heft-native/src/json/tests_large_objects.rs @@ -0,0 +1,80 @@ +use std::borrow::Cow; + +use crate::sys::allocation_counter::count_allocations_on_this_thread_while_running; + +use super::{parse_json_exactly_like_json_parse, JsonNumber, JsonObject, JsonValue}; + +fn number_value(value: f64) -> JsonValue<'static> { + JsonValue::Number(JsonNumber { + value, + source_text: "", + }) +} + +fn pseudo_random_key_sequence(seed: u64, entry_count: usize, key_pool_size: u64) -> Vec { + let mut state = seed; + let mut keys = Vec::with_capacity(entry_count); + for _ in 0..entry_count { + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + keys.push(format!("@scope/package-{}", (state >> 33) % key_pool_size)); + } + keys +} + +#[test] +fn large_objects_keep_first_position_and_last_value_exactly_like_linear_insertion() { + for (seed, entry_count, key_pool_size) in + [(1, 300, 400), (2, 300, 40), (3, 17, 16), (4, 900, 5000)] + { + let keys = pseudo_random_key_sequence(seed, entry_count, key_pool_size); + let mut linear_reference = JsonObject::default(); + for (position, key) in keys.iter().enumerate() { + linear_reference + .set_keeping_first_position(Cow::Borrowed(key), number_value(position as f64)); + } + let mut text = String::from("{"); + for (position, key) in keys.iter().enumerate() { + if position > 0 { + text.push(','); + } + text.push_str(&format!("\"{key}\":{position}")); + } + text.push('}'); + let parsed = parse_json_exactly_like_json_parse(&text).unwrap(); + assert_eq!( + parsed.as_object().unwrap().entries(), + linear_reference.entries() + ); + } +} + +#[test] +fn duplicate_keys_keep_the_first_position_and_the_last_value() { + let parsed = parse_json_exactly_like_json_parse("{\"a\":1,\"b\":2,\"a\":3}").unwrap(); + assert_eq!( + parsed.as_object().unwrap().entries(), + &[ + (Cow::Borrowed("a"), number_value(3.0)), + (Cow::Borrowed("b"), number_value(2.0)) + ] + ); +} + +#[test] +fn large_objects_allocate_one_exact_vec_the_pending_stack_growth_and_a_short_lived_key_index() { + let mut text = String::from("{"); + for position in 0..64 { + if position > 0 { + text.push(','); + } + text.push_str(&format!("\"key-{position}\":{position}")); + } + text.push('}'); + let (parsed, allocations) = count_allocations_on_this_thread_while_running(|| { + parse_json_exactly_like_json_parse(&text) + }); + assert_eq!(parsed.unwrap().as_object().unwrap().len(), 64); + assert_eq!(allocations, 1 + 5 + 2); +} diff --git a/apps/heft-native/src/json/tests_parser_acceptance.rs b/apps/heft-native/src/json/tests_parser_acceptance.rs new file mode 100644 index 0000000000..99ce882588 --- /dev/null +++ b/apps/heft-native/src/json/tests_parser_acceptance.rs @@ -0,0 +1,139 @@ +use super::cursor::JsonTextNeedsJavaScriptParser; +use super::{ + parse_json_exactly_like_json_parse, parse_json_with_comments_exactly_like_jju, JsonValue, +}; + +fn parses(text: &str) -> bool { + parse_json_with_comments_exactly_like_jju(text).is_ok() +} + +fn refuses(text: &str) -> bool { + parse_json_with_comments_exactly_like_jju(text) == Err(JsonTextNeedsJavaScriptParser) +} + +#[test] +fn accepts_strict_json_comments_and_single_trailing_commas() { + assert!(parses( + "{\"a\": [1, 2, 3], \"b\": {\"c\": null, \"d\": true, \"e\": false}}" + )); + assert!(parses("// leading\n{ /* inner */ \"a\": 1 }")); + assert!(parses("[1, 2,]")); + assert!(parses("{\"a\": 1,}")); + assert!(parses(" \t\r\n \"text\" \n")); + assert!(parses("-0.5e+10")); +} + +#[test] +fn refuses_everything_that_only_jju_json5_mode_would_accept_or_reject() { + for text in [ + "\u{feff}{}", + "{'a': 1}", + "{a: 1}", + "[1,,2]", + "[,]", + "{,}", + "0x10", + "+1", + ".5", + "5.", + "01", + "Infinity", + "NaN", + "\"\\v\"", + "\"\\x41\"", + "\"\\0\"", + "\"line\\\ncontinuation\"", + "\"tab\there\"", + "\"\u{2028}\"", + "// comment \u{2029} end\n1", + "\"\\ud83d\"", + "\"\\udc00\"", + "nullx", + "truee", + "1 2", + "/* unterminated", + "", + " ", + "\u{a0}1", + ] { + assert!(refuses(text), "expected a refusal for {:?}", text); + } +} + +#[test] +fn refuses_nesting_deeper_than_the_supported_depth() { + let deep_but_supported = format!("{}{}", "[".repeat(512), "]".repeat(512)); + assert!(parses(&deep_but_supported)); + let too_deep = format!("{}{}", "[".repeat(513), "]".repeat(513)); + assert!(refuses(&too_deep)); +} + +#[test] +fn duplicate_keys_keep_the_first_position_and_take_the_last_value() { + let value = + parse_json_with_comments_exactly_like_jju("{\"a\": 1, \"b\": 2, \"a\": 3}").unwrap(); + let object = value.as_object().unwrap(); + assert_eq!(object.len(), 2); + assert_eq!(object.entries()[0].0, "a"); + assert_eq!(object.get("a").and_then(JsonValue::as_f64), Some(3.0)); +} + +#[test] +fn decodes_escapes_and_surrogate_pairs_like_javascript() { + let value = + parse_json_with_comments_exactly_like_jju("\"\\u0041\\ud83d\\ude00\\n\\/\\\"\"").unwrap(); + assert_eq!(value.as_str(), Some("A\u{1F600}\n/\"")); +} + +#[test] +fn numbers_keep_their_source_text_and_javascript_values() { + let value = + parse_json_with_comments_exactly_like_jju("[1e400, -0, 9007199254740993, 5e-324]").unwrap(); + let items = value.as_array().unwrap(); + match (&items[0], &items[1], &items[2], &items[3]) { + ( + JsonValue::Number(huge), + JsonValue::Number(negative_zero), + JsonValue::Number(rounded), + JsonValue::Number(tiny), + ) => { + assert!(huge.value.is_infinite()); + assert_eq!(huge.source_text, "1e400"); + assert!(negative_zero.value == 0.0 && negative_zero.value.is_sign_negative()); + assert_eq!(rounded.value, 9007199254740992.0); + assert_eq!(tiny.value, 5e-324); + } + _ => panic!("expected four numbers"), + } +} + +#[test] +fn strict_variant_accepts_only_what_json_parse_accepts_with_identical_values() { + assert_eq!( + parse_json_exactly_like_json_parse( + "{\"name\": \"@rushstack/heft\", \"dependencies\": {}, \"list\": [1, 2]}" + ) + .ok(), + parse_json_with_comments_exactly_like_jju( + "{\"name\": \"@rushstack/heft\", \"dependencies\": {}, \"list\": [1, 2]}" + ) + .ok() + ); + assert!(parse_json_exactly_like_json_parse("[]").is_ok()); + assert!(parse_json_exactly_like_json_parse("{}").is_ok()); + for text in [ + "{\"a\": 1,}", + "[1,]", + "// comment\n{}", + "{} /* trailing */", + "{\"a\": /* inner */ 1}", + ] { + assert_eq!( + parse_json_exactly_like_json_parse(text), + Err(JsonTextNeedsJavaScriptParser), + "expected {:?} to be refused", + text + ); + assert!(parses(text), "the jju variant should accept {:?}", text); + } +} diff --git a/apps/heft-native/src/json/tests_zero_copy_and_writer.rs b/apps/heft-native/src/json/tests_zero_copy_and_writer.rs new file mode 100644 index 0000000000..b3e18d4b0e --- /dev/null +++ b/apps/heft-native/src/json/tests_zero_copy_and_writer.rs @@ -0,0 +1,46 @@ +use std::borrow::Cow; + +use super::{parse_json_with_comments_exactly_like_jju, write_json_for_javascript, JsonValue}; + +fn every_string_is_borrowed(value: &JsonValue<'_>) -> bool { + match value { + JsonValue::String(text) => matches!(text, Cow::Borrowed(_)), + JsonValue::Array(items) => items.iter().all(every_string_is_borrowed), + JsonValue::Object(object) => object + .entries() + .iter() + .all(|(key, item)| matches!(key, Cow::Borrowed(_)) && every_string_is_borrowed(item)), + _ => true, + } +} + +#[test] +fn strings_and_keys_without_escapes_borrow_from_the_input_buffer() { + let text = "{\"phasesByName\": {\"build\": {\"tasksByName\": {\"typescript\": {\"taskPlugin\": {\"pluginPackage\": \"@rushstack/heft-typescript-plugin\"}}}}}}"; + let value = parse_json_with_comments_exactly_like_jju(text).unwrap(); + assert!(every_string_is_borrowed(&value)); +} + +#[test] +fn strings_with_escapes_are_owned() { + let value = parse_json_with_comments_exactly_like_jju("{\"a\\u0062\": \"c\\\\d\"}").unwrap(); + let (key, item) = &value.as_object().unwrap().entries()[0]; + assert!(matches!(key, Cow::Owned(_))); + assert_eq!(key, "ab"); + assert!(matches!(item, JsonValue::String(Cow::Owned(_)))); + assert_eq!(item.as_str(), Some("c\\d")); +} + +#[test] +fn writer_output_round_trips_and_escapes_like_json_stringify() { + let text = "{\"a\": [1, -0, 1e400, 2.50], \"b\": \"q\\\"uote\\\\ \\u0001 \\u2028 \u{1F600}\", \"c\": {}, \"d\": null}"; + let value = parse_json_with_comments_exactly_like_jju(text).unwrap(); + let mut output = String::new(); + write_json_for_javascript(&value, &mut output).unwrap(); + assert_eq!( + output, + "{\"a\":[1,-0,1e400,2.50],\"b\":\"q\\\"uote\\\\ \\u0001 \\u2028 \u{1F600}\",\"c\":{},\"d\":null}" + ); + let reparsed = parse_json_with_comments_exactly_like_jju(&output).unwrap(); + assert_eq!(reparsed, value); +} diff --git a/apps/heft-native/src/json/value.rs b/apps/heft-native/src/json/value.rs new file mode 100644 index 0000000000..a04d2c8b47 --- /dev/null +++ b/apps/heft-native/src/json/value.rs @@ -0,0 +1,113 @@ +use std::borrow::Cow; + +#[derive(Debug, Clone, PartialEq)] +pub enum JsonValue<'text> { + Null, + Boolean(bool), + Number(JsonNumber<'text>), + String(Cow<'text, str>), + Array(Vec>), + Object(JsonObject<'text>), +} + +#[derive(Debug, Clone, Copy)] +pub struct JsonNumber<'text> { + pub value: f64, + pub source_text: &'text str, +} + +impl PartialEq for JsonNumber<'_> { + fn eq(&self, other: &Self) -> bool { + self.value == other.value + } +} + +#[derive(Debug, Clone, Default, PartialEq)] +pub struct JsonObject<'text> { + pub(super) entries: Vec<(Cow<'text, str>, JsonValue<'text>)>, +} + +impl<'text> JsonObject<'text> { + pub fn with_capacity(capacity: usize) -> Self { + JsonObject { + entries: Vec::with_capacity(capacity), + } + } + + pub fn get(&self, key: &str) -> Option<&JsonValue<'text>> { + self.entries + .iter() + .find(|(existing_key, _)| existing_key.as_ref() == key) + .map(|(_, value)| value) + } + + pub fn contains_key(&self, key: &str) -> bool { + self.entries + .iter() + .any(|(existing_key, _)| existing_key.as_ref() == key) + } + + pub fn entries(&self) -> &[(Cow<'text, str>, JsonValue<'text>)] { + &self.entries + } + + pub fn into_entries(self) -> Vec<(Cow<'text, str>, JsonValue<'text>)> { + self.entries + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn set_keeping_first_position(&mut self, key: Cow<'text, str>, value: JsonValue<'text>) { + match self + .entries + .iter_mut() + .find(|(existing_key, _)| existing_key.as_ref() == key.as_ref()) + { + Some(existing_entry) => existing_entry.1 = value, + None => self.entries.push((key, value)), + } + } +} + +impl<'text> JsonValue<'text> { + pub fn as_object(&self) -> Option<&JsonObject<'text>> { + match self { + JsonValue::Object(object) => Some(object), + _ => None, + } + } + + pub fn as_array(&self) -> Option<&[JsonValue<'text>]> { + match self { + JsonValue::Array(items) => Some(items), + _ => None, + } + } + + pub fn as_str(&self) -> Option<&str> { + match self { + JsonValue::String(text) => Some(text), + _ => None, + } + } + + pub fn as_f64(&self) -> Option { + match self { + JsonValue::Number(number) => Some(number.value), + _ => None, + } + } + + pub fn as_bool(&self) -> Option { + match self { + JsonValue::Boolean(flag) => Some(*flag), + _ => None, + } + } + + pub fn get(&self, key: &str) -> Option<&JsonValue<'text>> { + self.as_object().and_then(|object| object.get(key)) + } +} diff --git a/apps/heft-native/src/json/writer.rs b/apps/heft-native/src/json/writer.rs new file mode 100644 index 0000000000..ab998e94c6 --- /dev/null +++ b/apps/heft-native/src/json/writer.rs @@ -0,0 +1,74 @@ +use std::fmt::{self, Write}; + +use super::value::JsonValue; + +pub fn write_json_for_javascript( + value: &JsonValue<'_>, + output: &mut Output, +) -> fmt::Result { + match value { + JsonValue::Null => output.write_str("null"), + JsonValue::Boolean(true) => output.write_str("true"), + JsonValue::Boolean(false) => output.write_str("false"), + JsonValue::Number(number) if !number.source_text.is_empty() => { + output.write_str(number.source_text) + } + JsonValue::Number(number) => write!(output, "{}", number.value), + JsonValue::String(text) => write_json_string_for_javascript(text, output), + JsonValue::Array(items) => { + output.write_char('[')?; + for (index, item) in items.iter().enumerate() { + if index > 0 { + output.write_char(',')?; + } + write_json_for_javascript(item, output)?; + } + output.write_char(']') + } + JsonValue::Object(object) => { + output.write_char('{')?; + for (index, (key, item)) in object.entries().iter().enumerate() { + if index > 0 { + output.write_char(',')?; + } + write_json_string_for_javascript(key, output)?; + output.write_char(':')?; + write_json_for_javascript(item, output)?; + } + output.write_char('}') + } + } +} + +const HEXADECIMAL_DIGITS: &[u8; 16] = b"0123456789abcdef"; + +pub fn write_json_string_for_javascript( + text: &str, + output: &mut Output, +) -> fmt::Result { + output.write_char('"')?; + let mut unescaped_run_start = 0; + for (index, character) in text.char_indices() { + let escape: Option<&str> = match character { + '"' => Some("\\\""), + '\\' => Some("\\\\"), + '\u{2028}' => Some("\\u2028"), + '\u{2029}' => Some("\\u2029"), + control if (control as u32) < 0x20 => None, + _ => continue, + }; + output.write_str(&text[unescaped_run_start..index])?; + match escape { + Some(sequence) => output.write_str(sequence)?, + None => { + let byte = character as u8; + output.write_str("\\u00")?; + output.write_char(HEXADECIMAL_DIGITS[(byte >> 4) as usize] as char)?; + output.write_char(HEXADECIMAL_DIGITS[(byte & 0xf) as usize] as char)?; + } + } + unescaped_run_start = index + character.len_utf8(); + } + output.write_str(&text[unescaped_run_start..])?; + output.write_char('"') +} diff --git a/apps/heft-native/src/main.rs b/apps/heft-native/src/main.rs new file mode 100644 index 0000000000..2a762f7608 --- /dev/null +++ b/apps/heft-native/src/main.rs @@ -0,0 +1,46 @@ +#![deny(unsafe_code)] + +mod builtin; +mod cli; +mod config; +mod graph; +mod host_link; +mod json; +mod process; +mod regex; +mod run; +mod schema; +mod sys; +mod terminal; +mod version; + +use std::ffi::OsString; + +use run::HeftRunDecision; +use version::HeftImplementationSelection; + +fn main() { + #[cfg(unix)] + sys::reset_inherited_ignored_signals_like_node(); + let command_line_arguments: Vec = std::env::args_os().skip(1).collect(); + let native_heft_context = match version::select_heft_implementation(&command_line_arguments) { + HeftImplementationSelection::ThisBinary(native_heft_context) => native_heft_context, + HeftImplementationSelection::DelegateToJavaScriptHeft => { + process::exec_javascript_heft(&command_line_arguments) + } + }; + match run::decide_how_to_run_heft(&command_line_arguments, &native_heft_context) { + HeftRunDecision::DelegateToJavaScriptHeft => { + process::exec_javascript_heft(&command_line_arguments) + } + HeftRunDecision::RunNatively(native_heft_run) => { + version::write_version_selector_banner(native_heft_context.version_selector_banner); + std::process::exit(run::run_heft_natively(native_heft_run)) + } + HeftRunDecision::RunInNodeHost(node_host_plan) => host_link::run_in_node_host( + &native_heft_context, + &node_host_plan, + &command_line_arguments, + ), + } +} diff --git a/apps/heft-native/src/process/mod.rs b/apps/heft-native/src/process/mod.rs new file mode 100644 index 0000000000..19f494c108 --- /dev/null +++ b/apps/heft-native/src/process/mod.rs @@ -0,0 +1,58 @@ +use std::ffi::OsString; +use std::io::{ErrorKind, Write}; +use std::path::Path; +use std::process::Command; + +use crate::version::{ + locate_companion_javascript_heft_bin, COMPANION_JAVASCRIPT_HEFT_BIN_ENVIRONMENT_VARIABLE, +}; + +const EXIT_CODE_WHEN_NODE_IS_NOT_FOUND: i32 = 127; +const EXIT_CODE_WHEN_NODE_CANNOT_BE_EXECUTED: i32 = 126; + +pub fn exec_javascript_heft(command_line_arguments: &[OsString]) -> ! { + let current_folder = std::env::current_dir().unwrap_or_default(); + let Some(companion_heft_bin) = locate_companion_javascript_heft_bin(¤t_folder) else { + let _ = writeln!( + std::io::stderr(), + "heft: unable to find the @rushstack/heft JavaScript package of this binary; set {COMPANION_JAVASCRIPT_HEFT_BIN_ENVIRONMENT_VARIABLE} to the path of its bin/heft script" + ); + std::process::exit(1) + }; + let node_start_error = exec_node_script(&companion_heft_bin, command_line_arguments); + exit_because_node_could_not_be_started(&node_start_error) +} + +pub fn exec_node_script(script_path: &Path, command_line_arguments: &[OsString]) -> std::io::Error { + let _ = std::io::stdout().flush(); + let _ = std::io::stderr().flush(); + let mut node_command = Command::new("node"); + node_command.arg(script_path).args(command_line_arguments); + replace_current_process_or_wait(node_command) +} + +pub fn exit_because_node_could_not_be_started(node_start_error: &std::io::Error) -> ! { + let _ = writeln!( + std::io::stderr(), + "heft: unable to run node: {node_start_error}" + ); + std::process::exit(if node_start_error.kind() == ErrorKind::NotFound { + EXIT_CODE_WHEN_NODE_IS_NOT_FOUND + } else { + EXIT_CODE_WHEN_NODE_CANNOT_BE_EXECUTED + }) +} + +#[cfg(unix)] +fn replace_current_process_or_wait(mut node_command: Command) -> std::io::Error { + use std::os::unix::process::CommandExt; + node_command.exec() +} + +#[cfg(not(unix))] +fn replace_current_process_or_wait(mut node_command: Command) -> std::io::Error { + match node_command.status() { + Ok(node_exit_status) => std::process::exit(node_exit_status.code().unwrap_or(1)), + Err(node_spawn_error) => node_spawn_error, + } +} diff --git a/apps/heft-native/src/regex/bit_parallel_matcher.rs b/apps/heft-native/src/regex/bit_parallel_matcher.rs new file mode 100644 index 0000000000..07e1303bc0 --- /dev/null +++ b/apps/heft-native/src/regex/bit_parallel_matcher.rs @@ -0,0 +1,117 @@ +use super::pike_vm::character_matcher_accepts; +use super::program::Instruction; + +pub(super) const LARGEST_PROGRAM_FOR_BIT_PARALLEL_MATCHING: usize = 63; +const POSITION_CONTEXT_COUNT: usize = 4; + +pub(super) struct BitParallelMatcher { + epsilon_closure_masks: Vec, + instruction_count: usize, + match_instruction_mask: u64, +} + +fn position_context_index(at_start: bool, at_end: bool) -> usize { + usize::from(at_start) * 2 + usize::from(at_end) +} + +fn epsilon_closure_mask( + instructions: &[Instruction], + program_counter: usize, + at_start: bool, + at_end: bool, + visited_mask: &mut u64, +) -> u64 { + let program_counter_bit = 1u64 << program_counter; + if *visited_mask & program_counter_bit != 0 { + return 0; + } + *visited_mask |= program_counter_bit; + let follow = |target: usize, visited_mask: &mut u64| { + epsilon_closure_mask(instructions, target, at_start, at_end, visited_mask) + }; + match &instructions[program_counter] { + Instruction::Jump(target) => follow(*target, visited_mask), + Instruction::Split(first, second) => { + follow(*first, visited_mask) | follow(*second, visited_mask) + } + Instruction::AssertStart if at_start => follow(program_counter + 1, visited_mask), + Instruction::AssertEnd if at_end => follow(program_counter + 1, visited_mask), + Instruction::AssertStart | Instruction::AssertEnd => 0, + Instruction::Character(_) | Instruction::Match => program_counter_bit, + } +} + +impl BitParallelMatcher { + pub(super) fn for_small_program(instructions: &[Instruction]) -> Option { + let instruction_count = instructions.len(); + if instruction_count > LARGEST_PROGRAM_FOR_BIT_PARALLEL_MATCHING { + return None; + } + let mut epsilon_closure_masks = + Vec::with_capacity(POSITION_CONTEXT_COUNT * instruction_count); + for (at_start, at_end) in [(false, false), (false, true), (true, false), (true, true)] { + for program_counter in 0..instruction_count { + let mut visited_mask = 0u64; + epsilon_closure_masks.push(epsilon_closure_mask( + instructions, + program_counter, + at_start, + at_end, + &mut visited_mask, + )); + } + } + let match_instruction_mask = instructions + .iter() + .enumerate() + .filter(|(_, instruction)| matches!(instruction, Instruction::Match)) + .fold(0u64, |mask, (program_counter, _)| { + mask | 1u64 << program_counter + }); + Some(BitParallelMatcher { + epsilon_closure_masks, + instruction_count, + match_instruction_mask, + }) + } + + fn epsilon_closure_of_threads(&self, threads: u64, at_start: bool, at_end: bool) -> u64 { + let context_offset = position_context_index(at_start, at_end) * self.instruction_count; + let mut remaining_threads = threads; + let mut closure = 0u64; + while remaining_threads != 0 { + let program_counter = remaining_threads.trailing_zeros() as usize; + remaining_threads &= remaining_threads - 1; + closure |= self.epsilon_closure_masks[context_offset + program_counter]; + } + closure + } + + pub(super) fn matches_anywhere(&self, instructions: &[Instruction], text: &str) -> bool { + let mut code_points = text.chars().peekable(); + let mut advanced_threads = 0u64; + let mut at_start = true; + loop { + let at_end = code_points.peek().is_none(); + let threads = self.epsilon_closure_of_threads(advanced_threads | 1, at_start, at_end); + if threads & self.match_instruction_mask != 0 { + return true; + } + let Some(character) = code_points.next() else { + return false; + }; + at_start = false; + advanced_threads = 0; + let mut remaining_threads = threads; + while remaining_threads != 0 { + let program_counter = remaining_threads.trailing_zeros() as usize; + remaining_threads &= remaining_threads - 1; + if let Instruction::Character(matcher) = &instructions[program_counter] { + if character_matcher_accepts(matcher, character as u32) { + advanced_threads |= 1u64 << (program_counter + 1); + } + } + } + } + } +} diff --git a/apps/heft-native/src/regex/character_sets.rs b/apps/heft-native/src/regex/character_sets.rs new file mode 100644 index 0000000000..374577c1a9 --- /dev/null +++ b/apps/heft-native/src/regex/character_sets.rs @@ -0,0 +1,46 @@ +pub(super) type CodePointRange = (u32, u32); + +pub(super) const DIGIT_CODE_POINTS: &[CodePointRange] = &[(0x30, 0x39)]; +pub(super) const WORD_CODE_POINTS: &[CodePointRange] = + &[(0x30, 0x39), (0x41, 0x5a), (0x5f, 0x5f), (0x61, 0x7a)]; +pub(super) const WHITESPACE_AND_LINE_TERMINATOR_CODE_POINTS: &[CodePointRange] = &[ + (0x09, 0x0d), + (0x20, 0x20), + (0xa0, 0xa0), + (0x1680, 0x1680), + (0x2000, 0x200a), + (0x2028, 0x2029), + (0x202f, 0x202f), + (0x205f, 0x205f), + (0x3000, 0x3000), + (0xfeff, 0xfeff), +]; +pub(super) const LINE_TERMINATOR_CODE_POINTS: &[CodePointRange] = + &[(0x0a, 0x0a), (0x0d, 0x0d), (0x2028, 0x2029)]; + +const SYNTAX_CHARACTERS: &str = "^$\\.*+?()[]{}|/"; + +pub(super) fn character_for_control_escape(escape: char) -> Option { + Some(match escape { + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + 'f' => '\u{c}', + 'v' => '\u{b}', + _ => return None, + }) +} + +pub(super) fn character_for_identity_escape_of_syntax_character(escape: char) -> Option { + if SYNTAX_CHARACTERS.contains(escape) { + Some(escape) + } else { + None + } +} + +pub(super) fn ranges_contain_code_point(ranges: &[CodePointRange], code_point: u32) -> bool { + ranges + .iter() + .any(|&(first, last)| code_point >= first && code_point <= last) +} diff --git a/apps/heft-native/src/regex/class_parser.rs b/apps/heft-native/src/regex/class_parser.rs new file mode 100644 index 0000000000..61f729d1f5 --- /dev/null +++ b/apps/heft-native/src/regex/class_parser.rs @@ -0,0 +1,70 @@ +use super::character_sets::{ + character_for_control_escape, character_for_identity_escape_of_syntax_character, + CodePointRange, DIGIT_CODE_POINTS, WHITESPACE_AND_LINE_TERMINATOR_CODE_POINTS, + WORD_CODE_POINTS, +}; +use super::syntax_tree::{CharacterMatcher, PatternParser}; + +enum ClassAtom { + CodePoint(u32), + Set(&'static [CodePointRange]), +} + +const TYPICAL_CLASS_RANGE_COUNT: usize = 4; +const LAST_CODE_POINT_THAT_IS_ONE_UTF16_CODE_UNIT: u32 = 0xffff; + +impl PatternParser { + pub(super) fn parse_class_after_opening_bracket(&mut self) -> Option { + let negated = self.peek() == Some('^'); + if negated { + self.position += 1; + } + let mut ranges: Vec = Vec::with_capacity(TYPICAL_CLASS_RANGE_COUNT); + loop { + if self.peek()? == ']' { + self.position += 1; + return Some(CharacterMatcher::Ranges(ranges, negated)); + } + let first = self.parse_class_atom()?; + let is_range = + self.peek() == Some('-') && self.peek_after(1).is_some_and(|next| next != ']'); + match first { + ClassAtom::CodePoint(low) if is_range => { + self.position += 1; + match self.parse_class_atom()? { + ClassAtom::CodePoint(high) if high >= low => ranges.push((low, high)), + _ => return None, + } + } + ClassAtom::CodePoint(code_point) => ranges.push((code_point, code_point)), + ClassAtom::Set(_) if is_range => return None, + ClassAtom::Set(set) => ranges.extend_from_slice(set), + } + } + } + + fn parse_class_atom(&mut self) -> Option { + let next = self.peek()?; + self.position += 1; + if next != '\\' { + if next as u32 > LAST_CODE_POINT_THAT_IS_ONE_UTF16_CODE_UNIT { + return None; + } + return Some(ClassAtom::CodePoint(next as u32)); + } + let escape = self.peek()?; + self.position += 1; + Some(match escape { + 'd' => ClassAtom::Set(DIGIT_CODE_POINTS), + 'w' => ClassAtom::Set(WORD_CODE_POINTS), + 's' => ClassAtom::Set(WHITESPACE_AND_LINE_TERMINATOR_CODE_POINTS), + 'b' => ClassAtom::CodePoint(0x08), + '-' => ClassAtom::CodePoint('-' as u32), + other => ClassAtom::CodePoint( + character_for_control_escape(other) + .or_else(|| character_for_identity_escape_of_syntax_character(other))? + as u32, + ), + }) + } +} diff --git a/apps/heft-native/src/regex/mod.rs b/apps/heft-native/src/regex/mod.rs new file mode 100644 index 0000000000..06fceb5a79 --- /dev/null +++ b/apps/heft-native/src/regex/mod.rs @@ -0,0 +1,68 @@ +mod bit_parallel_matcher; +mod character_sets; +mod class_parser; +mod pike_vm; +mod program; +mod syntax_tree; + +use bit_parallel_matcher::BitParallelMatcher; +use pike_vm::PikeVirtualMachine; +use program::{Instruction, ProgramBuilder}; +use syntax_tree::PatternParser; + +pub struct CompiledUnicodeRegex { + instructions: Vec, + bit_parallel_matcher: Option, + pattern_is_within_basic_multilingual_plane: bool, +} + +const LAST_BASIC_MULTILINGUAL_PLANE_CODE_POINT: u32 = 0xffff; + +fn is_within_basic_multilingual_plane(text: &str) -> bool { + text.chars() + .all(|character| character as u32 <= LAST_BASIC_MULTILINGUAL_PLANE_CODE_POINT) +} + +pub fn compile_unicode_regex_subset(pattern: &str) -> Option { + let mut parser = PatternParser { + characters: pattern.chars().collect(), + position: 0, + }; + let syntax = parser.parse_disjunction()?; + if parser.position != parser.characters.len() { + return None; + } + let mut builder = ProgramBuilder { + instructions: Vec::with_capacity(parser.characters.len() + 1), + }; + builder.compile_node(&syntax)?; + builder.emit_final_match()?; + Some(CompiledUnicodeRegex { + bit_parallel_matcher: BitParallelMatcher::for_small_program(&builder.instructions), + instructions: builder.instructions, + pattern_is_within_basic_multilingual_plane: is_within_basic_multilingual_plane(pattern), + }) +} + +impl CompiledUnicodeRegex { + pub fn matches_anywhere(&self, text: &str) -> bool { + match &self.bit_parallel_matcher { + Some(matcher) => matcher.matches_anywhere(&self.instructions, text), + None => PikeVirtualMachine::new(&self.instructions).matches_anywhere(text), + } + } + + pub fn matches_anywhere_without_unicode_flag(&self, text: &str) -> Option { + if !self.pattern_is_within_basic_multilingual_plane + || !is_within_basic_multilingual_plane(text) + { + return None; + } + Some(self.matches_anywhere(text)) + } +} + +#[cfg(test)] +mod tests_bit_parallel_matcher; +#[cfg(test)] +mod tests_regex_semantics; diff --git a/apps/heft-native/src/regex/pike_vm.rs b/apps/heft-native/src/regex/pike_vm.rs new file mode 100644 index 0000000000..56d3bef5bb --- /dev/null +++ b/apps/heft-native/src/regex/pike_vm.rs @@ -0,0 +1,100 @@ +use super::character_sets::ranges_contain_code_point; +use super::program::Instruction; +use super::syntax_tree::CharacterMatcher; + +pub(super) struct PikeVirtualMachine<'program> { + instructions: &'program [Instruction], + visited_generation: Vec, + generation: u32, +} + +pub(super) fn character_matcher_accepts(matcher: &CharacterMatcher, code_point: u32) -> bool { + match matcher { + CharacterMatcher::Single(expected) => *expected == code_point, + CharacterMatcher::Predefined(ranges, negated) => { + ranges_contain_code_point(ranges, code_point) != *negated + } + CharacterMatcher::Ranges(ranges, negated) => { + ranges_contain_code_point(ranges, code_point) != *negated + } + } +} + +impl<'program> PikeVirtualMachine<'program> { + pub(super) fn new(instructions: &'program [Instruction]) -> Self { + PikeVirtualMachine { + instructions, + visited_generation: vec![0; instructions.len()], + generation: 0, + } + } + + fn add_thread_with_epsilon_closure( + &mut self, + threads: &mut Vec, + program_counter: usize, + at_start: bool, + at_end: bool, + ) { + if self.visited_generation[program_counter] == self.generation { + return; + } + self.visited_generation[program_counter] = self.generation; + match &self.instructions[program_counter] { + Instruction::Jump(target) => { + self.add_thread_with_epsilon_closure(threads, *target, at_start, at_end) + } + Instruction::Split(first, second) => { + let (first, second) = (*first, *second); + self.add_thread_with_epsilon_closure(threads, first, at_start, at_end); + self.add_thread_with_epsilon_closure(threads, second, at_start, at_end); + } + Instruction::AssertStart if at_start => { + self.add_thread_with_epsilon_closure(threads, program_counter + 1, at_start, at_end) + } + Instruction::AssertEnd if at_end => { + self.add_thread_with_epsilon_closure(threads, program_counter + 1, at_start, at_end) + } + Instruction::AssertStart | Instruction::AssertEnd => {} + Instruction::Character(_) | Instruction::Match => threads.push(program_counter), + } + } + + pub(super) fn matches_anywhere(&mut self, text: &str) -> bool { + let mut closure: Vec = Vec::with_capacity(self.instructions.len()); + let mut advanced_threads: Vec = Vec::with_capacity(self.instructions.len()); + let mut code_points = text.chars().peekable(); + let mut at_start = true; + loop { + self.generation += 1; + let at_end = code_points.peek().is_none(); + closure.clear(); + for &program_counter in &advanced_threads { + self.add_thread_with_epsilon_closure( + &mut closure, + program_counter, + at_start, + at_end, + ); + } + self.add_thread_with_epsilon_closure(&mut closure, 0, at_start, at_end); + if closure.iter().any(|&program_counter| { + matches!(self.instructions[program_counter], Instruction::Match) + }) { + return true; + } + let Some(character) = code_points.next() else { + return false; + }; + at_start = false; + advanced_threads.clear(); + for &program_counter in &closure { + if let Instruction::Character(matcher) = &self.instructions[program_counter] { + if character_matcher_accepts(matcher, character as u32) { + advanced_threads.push(program_counter + 1); + } + } + } + } + } +} diff --git a/apps/heft-native/src/regex/program.rs b/apps/heft-native/src/regex/program.rs new file mode 100644 index 0000000000..db81886b47 --- /dev/null +++ b/apps/heft-native/src/regex/program.rs @@ -0,0 +1,106 @@ +use super::syntax_tree::{CharacterMatcher, SyntaxNode}; + +pub(super) enum Instruction { + Character(CharacterMatcher), + Split(usize, usize), + Jump(usize), + AssertStart, + AssertEnd, + Match, +} + +const MAXIMUM_PROGRAM_LENGTH: usize = 4096; + +pub(super) struct ProgramBuilder { + pub(super) instructions: Vec, +} + +impl ProgramBuilder { + fn emit(&mut self, instruction: Instruction) -> Option { + if self.instructions.len() >= MAXIMUM_PROGRAM_LENGTH { + return None; + } + self.instructions.push(instruction); + Some(self.instructions.len() - 1) + } + + pub(super) fn emit_final_match(&mut self) -> Option<()> { + self.emit(Instruction::Match).map(|_| ()) + } + + pub(super) fn compile_node(&mut self, node: &SyntaxNode) -> Option<()> { + match node { + SyntaxNode::Character(matcher) => { + self.emit(Instruction::Character(matcher.clone()))?; + } + SyntaxNode::StartAnchor => { + self.emit(Instruction::AssertStart)?; + } + SyntaxNode::EndAnchor => { + self.emit(Instruction::AssertEnd)?; + } + SyntaxNode::Sequence(items) => { + for item in items { + self.compile_node(item)?; + } + } + SyntaxNode::Alternation(alternatives) => self.compile_alternation(alternatives)?, + SyntaxNode::Repetition(inner, minimum, maximum) => { + self.compile_repetition(inner, *minimum, *maximum)? + } + } + Some(()) + } + + fn compile_alternation(&mut self, alternatives: &[SyntaxNode]) -> Option<()> { + let mut jumps_to_end = Vec::with_capacity(alternatives.len()); + for (index, alternative) in alternatives.iter().enumerate() { + if index + 1 == alternatives.len() { + self.compile_node(alternative)?; + continue; + } + let split = self.emit(Instruction::Split(0, 0))?; + self.compile_node(alternative)?; + jumps_to_end.push(self.emit(Instruction::Jump(0))?); + let next_alternative = self.instructions.len(); + self.instructions[split] = Instruction::Split(split + 1, next_alternative); + } + let end = self.instructions.len(); + for jump in jumps_to_end { + self.instructions[jump] = Instruction::Jump(end); + } + Some(()) + } + + fn compile_repetition( + &mut self, + inner: &SyntaxNode, + minimum: u32, + maximum: Option, + ) -> Option<()> { + for _ in 0..minimum { + self.compile_node(inner)?; + } + match maximum { + None => { + let split = self.emit(Instruction::Split(0, 0))?; + self.compile_node(inner)?; + self.emit(Instruction::Jump(split))?; + let after_loop = self.instructions.len(); + self.instructions[split] = Instruction::Split(split + 1, after_loop); + } + Some(maximum) => { + let mut optional_splits = Vec::with_capacity((maximum - minimum) as usize); + for _ in minimum..maximum { + optional_splits.push(self.emit(Instruction::Split(0, 0))?); + self.compile_node(inner)?; + } + let after_optional_copies = self.instructions.len(); + for split in optional_splits { + self.instructions[split] = Instruction::Split(split + 1, after_optional_copies); + } + } + } + Some(()) + } +} diff --git a/apps/heft-native/src/regex/syntax_tree.rs b/apps/heft-native/src/regex/syntax_tree.rs new file mode 100644 index 0000000000..f194fee82d --- /dev/null +++ b/apps/heft-native/src/regex/syntax_tree.rs @@ -0,0 +1,177 @@ +use super::character_sets::{ + character_for_control_escape, character_for_identity_escape_of_syntax_character, + CodePointRange, DIGIT_CODE_POINTS, LINE_TERMINATOR_CODE_POINTS, + WHITESPACE_AND_LINE_TERMINATOR_CODE_POINTS, WORD_CODE_POINTS, +}; + +#[derive(Clone)] +pub(super) enum CharacterMatcher { + Single(u32), + Predefined(&'static [CodePointRange], bool), + Ranges(Vec, bool), +} + +pub(super) enum SyntaxNode { + Character(CharacterMatcher), + StartAnchor, + EndAnchor, + Sequence(Vec), + Alternation(Vec), + Repetition(Box, u32, Option), +} + +pub(super) const MAXIMUM_REPETITION_BOUND: u32 = 100; + +pub(super) struct PatternParser { + pub(super) characters: Vec, + pub(super) position: usize, +} + +impl PatternParser { + pub(super) fn peek(&self) -> Option { + self.characters.get(self.position).copied() + } + + pub(super) fn peek_after(&self, offset: usize) -> Option { + self.characters.get(self.position + offset).copied() + } + + pub(super) fn parse_disjunction(&mut self) -> Option { + let mut alternatives = vec![self.parse_alternative()?]; + while self.peek() == Some('|') { + self.position += 1; + alternatives.push(self.parse_alternative()?); + } + if alternatives.len() == 1 { + alternatives.pop() + } else { + Some(SyntaxNode::Alternation(alternatives)) + } + } + + fn parse_alternative(&mut self) -> Option { + let mut terms = Vec::new(); + while let Some(next) = self.peek() { + if next == '|' || next == ')' { + break; + } + terms.push(self.parse_term()?); + } + Some(SyntaxNode::Sequence(terms)) + } + + fn parse_term(&mut self) -> Option { + let next = self.peek()?; + self.position += 1; + let atom = match next { + '^' => return Some(SyntaxNode::StartAnchor), + '$' => return Some(SyntaxNode::EndAnchor), + '(' => self.parse_group_after_opening_parenthesis()?, + '.' => SyntaxNode::Character(CharacterMatcher::Predefined( + LINE_TERMINATOR_CODE_POINTS, + true, + )), + '[' => SyntaxNode::Character(self.parse_class_after_opening_bracket()?), + '\\' => SyntaxNode::Character(self.parse_atom_escape_after_backslash()?), + '*' | '+' | '?' | '{' | '}' | ']' | ')' | '|' => return None, + literal => SyntaxNode::Character(CharacterMatcher::Single(literal as u32)), + }; + self.parse_optional_quantifier(atom) + } + + fn parse_group_after_opening_parenthesis(&mut self) -> Option { + if self.peek() == Some('?') { + if self.peek_after(1) != Some(':') { + return None; + } + self.position += 2; + } + let inner = self.parse_disjunction()?; + if self.peek() != Some(')') { + return None; + } + self.position += 1; + Some(inner) + } + + fn parse_atom_escape_after_backslash(&mut self) -> Option { + let escape = self.peek()?; + self.position += 1; + Some(match escape { + 'd' => CharacterMatcher::Predefined(DIGIT_CODE_POINTS, false), + 'D' => CharacterMatcher::Predefined(DIGIT_CODE_POINTS, true), + 'w' => CharacterMatcher::Predefined(WORD_CODE_POINTS, false), + 'W' => CharacterMatcher::Predefined(WORD_CODE_POINTS, true), + 's' => CharacterMatcher::Predefined(WHITESPACE_AND_LINE_TERMINATOR_CODE_POINTS, false), + 'S' => CharacterMatcher::Predefined(WHITESPACE_AND_LINE_TERMINATOR_CODE_POINTS, true), + other => CharacterMatcher::Single( + character_for_control_escape(other) + .or_else(|| character_for_identity_escape_of_syntax_character(other))? + as u32, + ), + }) + } + + fn parse_decimal_bound(&mut self) -> Option { + let start = self.position; + while matches!(self.peek(), Some('0'..='9')) { + self.position += 1; + } + if self.position == start || self.position - start > 4 { + return None; + } + self.characters[start..self.position] + .iter() + .try_fold(0u32, |bound, digit| Some(bound * 10 + digit.to_digit(10)?)) + } + + fn parse_optional_quantifier(&mut self, atom: SyntaxNode) -> Option { + let (minimum, maximum) = match self.peek() { + Some('*') => { + self.position += 1; + (0, None) + } + Some('+') => { + self.position += 1; + (1, None) + } + Some('?') => { + self.position += 1; + (0, Some(1)) + } + Some('{') => { + self.position += 1; + self.parse_braced_quantifier_bounds()? + } + _ => return Some(atom), + }; + if self.peek() == Some('?') { + self.position += 1; + } + Some(SyntaxNode::Repetition(Box::new(atom), minimum, maximum)) + } + + fn parse_braced_quantifier_bounds(&mut self) -> Option<(u32, Option)> { + let minimum = self.parse_decimal_bound()?; + let maximum = if self.peek() == Some(',') { + self.position += 1; + if self.peek() == Some('}') { + None + } else { + Some(self.parse_decimal_bound()?) + } + } else { + Some(minimum) + }; + if self.peek() != Some('}') { + return None; + } + self.position += 1; + let bounds_are_supported = minimum <= MAXIMUM_REPETITION_BOUND + && maximum.is_none_or(|upper| upper >= minimum && upper <= MAXIMUM_REPETITION_BOUND); + if !bounds_are_supported { + return None; + } + Some((minimum, maximum)) + } +} diff --git a/apps/heft-native/src/regex/tests_bit_parallel_matcher.rs b/apps/heft-native/src/regex/tests_bit_parallel_matcher.rs new file mode 100644 index 0000000000..f9c6ae24ed --- /dev/null +++ b/apps/heft-native/src/regex/tests_bit_parallel_matcher.rs @@ -0,0 +1,69 @@ +use super::pike_vm::PikeVirtualMachine; +use super::{compile_unicode_regex_subset, CompiledUnicodeRegex}; + +const PATTERNS: [&str; 22] = [ + "^-(-[a-z0-9]+)+$", + "^-[a-zA-Z]$", + "^[a-z][a-z0-9]*([-][a-z0-9]+)*$", + "[^\\\\]", + "^\\.[A-z0-9-_.]*[A-z0-9-_]+$", + "a|b", + "^$", + "$", + "^", + "(a|ab)(c|bcd)(d*)", + "a{2,5}?b", + "(?:x*)*y", + "^(a+)+$", + ".", + "\\d+\\s\\w", + "é+$", + "😀", + "^(?:[a-c]|x{2})*$", + "a{70}", + "(?:ab|cd){20,30}", + "^[^a-z]+\\.json$", + "(^a|b$)", +]; + +const TEXTS: [&str; 18] = [ + "", + "a", + "ab", + "--production", + "-v", + "-", + "copy-files-plugin", + "Typescript-Plugin", + ".json", + "ABC.json", + "xxxxxxy", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab", + "abcd", + "1 x", + "\n", + "ééé", + "😀a", + "abababababababababababababababababababababab", +]; + +fn pike_virtual_machine_verdict(compiled: &CompiledUnicodeRegex, text: &str) -> bool { + PikeVirtualMachine::new(&compiled.instructions).matches_anywhere(text) +} + +#[test] +fn bit_parallel_matching_agrees_with_the_pike_virtual_machine() { + let mut bit_parallel_program_count = 0; + for pattern in PATTERNS { + let compiled = compile_unicode_regex_subset(pattern).unwrap(); + bit_parallel_program_count += usize::from(compiled.bit_parallel_matcher.is_some()); + for text in TEXTS { + assert_eq!( + compiled.matches_anywhere(text), + pike_virtual_machine_verdict(&compiled, text), + "{pattern} on {text}" + ); + } + } + assert_eq!(bit_parallel_program_count, PATTERNS.len() - 2); +} diff --git a/apps/heft-native/src/regex/tests_regex_semantics.rs b/apps/heft-native/src/regex/tests_regex_semantics.rs new file mode 100644 index 0000000000..20d7ac42eb --- /dev/null +++ b/apps/heft-native/src/regex/tests_regex_semantics.rs @@ -0,0 +1,109 @@ +use super::compile_unicode_regex_subset; + +fn matches(pattern: &str, text: &str) -> bool { + compile_unicode_regex_subset(pattern) + .unwrap_or_else(|| panic!("pattern {:?} should be supported", pattern)) + .matches_anywhere(text) +} + +#[test] +fn heft_schema_patterns_behave_like_ecmascript_unicode_regexps() { + let package_name_without_backslash = "[^\\\\]"; + assert!(matches(package_name_without_backslash, "@rushstack/heft")); + assert!(!matches(package_name_without_backslash, "\\")); + let kebab_case_name = "^[a-z][a-z0-9]*([-][a-z0-9]+)*$"; + assert!(matches(kebab_case_name, "copy-json-schemas")); + assert!(!matches(kebab_case_name, "Copy")); + assert!(!matches(kebab_case_name, "a--b")); + let file_extension = "^\\.[A-z0-9-_.]*[A-z0-9-_]+$"; + assert!(matches(file_extension, ".schema.json")); + assert!(matches(file_extension, ".d_ts")); + assert!(!matches(file_extension, ".json.")); + let long_parameter_name = "^-(-[a-z0-9]+)+$"; + assert!(matches(long_parameter_name, "--lint-fix")); + assert!(!matches(long_parameter_name, "-x")); +} + +#[test] +fn dot_matches_code_points_but_not_line_terminators() { + assert!(matches("^.$", "\u{1F600}")); + assert!(matches("^\u{1F600}+$", "\u{1F600}\u{1F600}")); + assert!(!matches("^.$", "\n")); + assert!(!matches("^.$", "\u{2028}")); + assert!(matches("^[^]$", "\n")); + assert!(!matches("^[]$", "a")); +} + +#[test] +fn quantifiers_alternation_and_classes() { + assert!(matches("^(ab|cd){2,3}$", "abcdab")); + assert!(!matches("^(ab|cd){2,3}$", "ab")); + assert!(matches("^a*?b+?c??$", "aabb")); + assert!(matches("\\s", "x\u{3000}y")); + assert!(matches("^\\w+$", "abc_123")); + assert!(!matches("^\\w+$", "é")); + assert!(matches("^\\D\\W\\S$", "a-b")); +} + +#[test] +fn refuses_syntax_outside_the_supported_subset_or_invalid_in_unicode_mode() { + for pattern in [ + "(?=a)", + "(?!a)", + "(?a)", + "\\1", + "\\b", + "\\B", + "\\p{L}", + "\\u0041", + "\\x41", + "\\0", + "\\Z", + "a\\Z", + "[\\w-_]", + "[z-a]", + "a{2,1}", + "a{101}", + "*", + "a**", + "{", + "a{", + "a{1,x}", + "}", + "]", + "(", + ")", + "\\-", + "[\\D]", + "^*", + ] { + assert!( + compile_unicode_regex_subset(pattern).is_none(), + "expected {:?} to be refused", + pattern + ); + } +} + +#[test] +fn non_unicode_matching_is_only_answered_for_basic_multilingual_plane_text() { + let regex = compile_unicode_regex_subset("^x-").unwrap(); + assert_eq!( + regex.matches_anywhere_without_unicode_flag("x-y"), + Some(true) + ); + assert_eq!( + regex.matches_anywhere_without_unicode_flag("y"), + Some(false) + ); + assert_eq!( + regex.matches_anywhere_without_unicode_flag("x-\u{1F600}"), + None + ); + let astral_pattern = compile_unicode_regex_subset("\u{1F600}").unwrap(); + assert_eq!( + astral_pattern.matches_anywhere_without_unicode_flag("a"), + None + ); + assert!(compile_unicode_regex_subset("[\u{1F600}-\u{1F602}]").is_none()); +} diff --git a/apps/heft-native/src/run/builtin_options.rs b/apps/heft-native/src/run/builtin_options.rs new file mode 100644 index 0000000000..151655bdcf --- /dev/null +++ b/apps/heft-native/src/run/builtin_options.rs @@ -0,0 +1,95 @@ +use crate::builtin::{BuiltinTaskOptions, CopyOperation, CopyOperationField, FileSelectionSpecifier}; +use crate::json::{JsonObject, JsonValue}; + +pub const HEFT_PACKAGE_NAME: &str = "@rushstack/heft"; + +pub fn builtin_task_options_from_json(plugin_name: &str, options: Option<&JsonValue<'_>>) -> Option { + let options = options?.as_object()?; + let (option_name, option_value) = single_entry(options)?; + match (plugin_name, option_name) { + ("copy-files-plugin", "copyOperations") => option_value + .as_array()? + .iter() + .map(copy_operation_from_json) + .collect::>>() + .map(BuiltinTaskOptions::CopyFiles), + ("delete-files-plugin", "deleteOperations") => option_value + .as_array()? + .iter() + .map(|operation| file_selection_from_json(operation.as_object()?, false)) + .collect::>>() + .map(BuiltinTaskOptions::DeleteFiles), + ("set-environment-variables-plugin", "environmentVariablesToSet") => option_value + .as_object()? + .entries() + .iter() + .map(|(name, value)| Some((name.to_string(), value.as_str()?.to_owned()))) + .collect::>>() + .map(BuiltinTaskOptions::SetEnvironmentVariables), + _ => None, + } +} + +fn single_entry<'object, 'text>(object: &'object JsonObject<'text>) -> Option<(&'object str, &'object JsonValue<'text>)> { + match object.entries() { + [(name, value)] => Some((name.as_ref(), value)), + _ => None, + } +} + +fn copy_operation_from_json(operation: &JsonValue<'_>) -> Option { + let operation = operation.as_object()?; + let selection = file_selection_from_json(operation, true)?; + let mut destination_folders = None; + let mut flatten = None; + let mut hardlink = None; + let mut field_order = Vec::with_capacity(operation.len()); + for (name, value) in operation.entries() { + let field = match name.as_ref() { + "sourcePath" => CopyOperationField::SourcePath, + "fileExtensions" => CopyOperationField::FileExtensions, + "excludeGlobs" => CopyOperationField::ExcludeGlobs, + "includeGlobs" => CopyOperationField::IncludeGlobs, + "destinationFolders" => { + destination_folders = Some(string_array(value)?); + CopyOperationField::DestinationFolders + } + "flatten" => { + flatten = Some(value.as_bool()?); + CopyOperationField::Flatten + } + "hardlink" => { + hardlink = Some(value.as_bool()?); + CopyOperationField::Hardlink + } + _ => return None, + }; + field_order.push(field); + } + Some(CopyOperation { + selection, + destination_folders: destination_folders?, + flatten, + hardlink, + field_order, + }) +} + +pub fn file_selection_from_json(operation: &JsonObject<'_>, allows_copy_fields: bool) -> Option { + let mut selection = FileSelectionSpecifier::default(); + for (name, value) in operation.entries() { + match name.as_ref() { + "sourcePath" => selection.source_path = Some(value.as_str()?.to_owned()), + "fileExtensions" => selection.file_extensions = Some(string_array(value)?), + "excludeGlobs" => selection.exclude_globs = Some(string_array(value)?), + "includeGlobs" => selection.include_globs = Some(string_array(value)?), + "destinationFolders" | "flatten" | "hardlink" if allows_copy_fields => {} + _ => return None, + } + } + Some(selection) +} + +fn string_array(value: &JsonValue<'_>) -> Option> { + value.as_array()?.iter().map(|item| item.as_str().map(str::to_owned)).collect() +} diff --git a/apps/heft-native/src/run/closed_output.rs b/apps/heft-native/src/run/closed_output.rs new file mode 100644 index 0000000000..285b0ec37e --- /dev/null +++ b/apps/heft-native/src/run/closed_output.rs @@ -0,0 +1,25 @@ +use std::ffi::OsString; +use std::path::Path; + +use crate::process::{exec_node_script, exit_because_node_could_not_be_started}; +use crate::terminal::{ClosedOutput, OutputSeverity}; + +const WRITE_THROUGH_HEFT_TERMINAL_SCRIPT: &str = concat!( + "const [heftPackageFolder, prefixed, severity] = process.argv.slice(1);", + "const terminalPackage = require(require.resolve('@rushstack/terminal', { paths: [heftPackageFolder] }));", + "const consoleProvider = new terminalPackage.ConsoleTerminalProvider();", + "const provider = prefixed === 'prefixed' ? new terminalPackage.PrefixProxyTerminalProvider({ terminalProvider: consoleProvider, prefix: '' }) : consoleProvider;", + "const terminal = new terminalPackage.Terminal(provider);", + "if (severity === 'error') { terminal.writeErrorLine(''); } else { terminal.writeLine(''); }" +); + +pub fn exit_like_node_after_closed_output(closed_output: ClosedOutput, heft_package_folder: &str) -> ! { + let arguments = [ + OsString::from(WRITE_THROUGH_HEFT_TERMINAL_SCRIPT), + OsString::from(heft_package_folder), + OsString::from(if closed_output.prefixed { "prefixed" } else { "unprefixed" }), + OsString::from(if closed_output.severity == OutputSeverity::Error { "error" } else { "log" }), + ]; + let node_start_error = exec_node_script(Path::new("-e"), &arguments); + exit_because_node_could_not_be_started(&node_start_error) +} diff --git a/apps/heft-native/src/run/command_options.rs b/apps/heft-native/src/run/command_options.rs new file mode 100644 index 0000000000..16b16cf198 --- /dev/null +++ b/apps/heft-native/src/run/command_options.rs @@ -0,0 +1,55 @@ +use crate::cli::outcome::{ParsedCommand, ParsedParameters}; +use crate::cli::{ActionKind, ParameterValue}; + +pub struct TierZeroCommandOptions { + pub clean: bool, + pub selected_phase_indices: Vec, + pub alias_expansion_message: Option, +} + +pub fn tier_zero_command_options(command: &ParsedCommand<'_>) -> Option { + if command.watch || command.debug || matches!(command.action_kind, ActionKind::Clean) { + return None; + } + let mut clean = false; + for parameters in std::iter::once(&command.parameters).chain(command.scoped_parameters.as_ref()) { + clean |= parameters_allow_tier_zero(parameters)?; + } + Some(TierZeroCommandOptions { + clean, + selected_phase_indices: command.selected_phases.clone(), + alias_expansion_message: command.alias_expansion_message.clone(), + }) +} + +pub fn tier_zero_clean_options(command: &ParsedCommand<'_>) -> Option { + if command.watch || command.debug || !matches!(command.action_kind, ActionKind::Clean) { + return None; + } + let only_scoping_values = command.parameters.definitions.iter().zip(&command.parameters.values).all(|(definition, value)| { + matches!(value, ParameterValue::Absent) || matches!(definition.long_name, "--to" | "--to-except" | "--only") + }); + only_scoping_values.then(|| TierZeroCommandOptions { + clean: false, + selected_phase_indices: command.selected_phases.clone(), + alias_expansion_message: command.alias_expansion_message.clone(), + }) +} + +fn parameters_allow_tier_zero(parameters: &ParsedParameters<'_>) -> Option { + let mut clean = false; + for (definition, value) in parameters.definitions.iter().zip(¶meters.values) { + if matches!(value, ParameterValue::Absent) { + continue; + } + if definition.scope.is_some() { + return None; + } + match (definition.long_name, value) { + ("--clean", ParameterValue::Flag) => clean = true, + ("--production" | "--locales" | "--to" | "--to-except" | "--only", _) => {} + _ => return None, + } + } + Some(clean) +} diff --git a/apps/heft-native/src/run/configured_project.rs b/apps/heft-native/src/run/configured_project.rs new file mode 100644 index 0000000000..ff96e35d4e --- /dev/null +++ b/apps/heft-native/src/run/configured_project.rs @@ -0,0 +1,92 @@ +use super::builtin_options::{builtin_task_options_from_json, file_selection_from_json, HEFT_PACKAGE_NAME}; +use super::command_options::TierZeroCommandOptions; +use super::tier_zero_plan::{NativeBuildRequest, NativePhaseDefinition, NativeTaskDefinition}; +use crate::builtin::{AbsoluteFileSelection, FileSelectionSpecifier}; +use crate::config::model::HeftConfigurationModel; +use crate::config::plugin_manifest::PluginKind; +use crate::json::JsonValue; + +fn indices_of(names: &[&str], candidates: &[&str]) -> Option> { + names.iter().map(|name| candidates.iter().position(|candidate| candidate == name)).collect() +} + +pub fn native_clean_selections( + model: &HeftConfigurationModel<'_>, + selected_phase_indices: &[usize], +) -> Option> { + let temp_folder_path = format!("{}/temp", model.build_folder_path); + let mut specifiers: Vec = Vec::new(); + for &phase_index in selected_phase_indices { + let phase = model.phases.get(phase_index)?; + for task in &phase.tasks { + specifiers.push(FileSelectionSpecifier { + source_path: Some(format!("{temp_folder_path}/{}/{}", phase.name, task.name)), + ..FileSelectionSpecifier::default() + }); + } + specifiers.extend(clean_file_specifiers(&phase.clean_files)?); + } + specifiers.iter().map(|specifier| specifier.to_absolute_selection(model.build_folder_path)).collect() +} + +fn clean_file_specifiers(clean_files: &Option>) -> Option> { + match clean_files { + Some(JsonValue::Array(items)) => items.iter().map(|item| file_selection_from_json(item.as_object()?, false)).collect(), + Some(_) => None, + None => Some(Vec::new()), + } +} + +pub fn native_build_request( + model: &HeftConfigurationModel<'_>, + heft_package_folder: &str, + command_options: TierZeroCommandOptions, +) -> Option { + if !model.lifecycle_plugins.is_empty() { + return None; + } + let phase_names: Vec<&str> = model.phases.iter().map(|phase| phase.name).collect(); + let mut native_phases = Vec::with_capacity(model.phases.len()); + for (phase_index, phase) in model.phases.iter().enumerate() { + let dependency_phase_indices = indices_of(&phase.dependency_names, &phase_names)?; + if !command_options.selected_phase_indices.contains(&phase_index) { + native_phases.push(NativePhaseDefinition { + phase_name: phase.name.to_owned(), + dependency_phase_indices, + clean_files: Vec::new(), + tasks: Vec::new(), + }); + continue; + } + let task_names: Vec<&str> = phase.tasks.iter().map(|task| task.name).collect(); + let mut tasks = Vec::with_capacity(phase.tasks.len()); + for task in &phase.tasks { + let plugin = model.plugins.get(task.plugin)?; + if !matches!(plugin.kind, PluginKind::Task) + || plugin.package_name != HEFT_PACKAGE_NAME + || plugin.package_root != heft_package_folder + { + return None; + } + tasks.push(NativeTaskDefinition { + task_name: task.name.to_owned(), + dependency_task_indices: indices_of(&task.task_dependency_names, &task_names)?, + options: builtin_task_options_from_json(plugin.plugin_name, task.options.as_ref())?, + }); + } + let clean_files = clean_file_specifiers(&phase.clean_files)?; + native_phases.push(NativePhaseDefinition { + phase_name: phase.name.to_owned(), + dependency_phase_indices, + clean_files, + tasks, + }); + } + Some(NativeBuildRequest { + build_folder_path: model.build_folder_path.to_owned(), + phases: native_phases, + selected_phase_indices: command_options.selected_phase_indices, + clean: command_options.clean, + alias_expansion_message: command_options.alias_expansion_message, + }) +} diff --git a/apps/heft-native/src/run/mod.rs b/apps/heft-native/src/run/mod.rs new file mode 100644 index 0000000000..0e76c1f66f --- /dev/null +++ b/apps/heft-native/src/run/mod.rs @@ -0,0 +1,169 @@ +mod builtin_options; +mod closed_output; +mod command_options; +mod configured_project; +mod node_host_plan; +mod process_environment; +mod tier_zero_execution; +mod tier_zero_plan; +#[cfg(test)] +mod tests_tier_zero; + +use std::ffi::OsString; + +use crate::builtin::{selections_are_deletable_without_permission_errors, AbsoluteFileSelection}; +use crate::cli::outcome::{CliOutcome, ParsedCommand, PrintedOutput}; +use crate::config::cli_model_builder::build_cli_model; +use crate::config::loader::{load_heft_configuration_and_then, HeftConfigurationRequest, LoadedHeftConfiguration}; +use crate::config::model::{build_heft_configuration_model, HeftConfigurationModel}; +use crate::config::package_json::PackageJsonLookup; +use crate::host_link::NodeHostPlan; +use crate::terminal::{console_supports_color_for_this_process, HeftConsole}; +use crate::version::NativeHeftContext; + +use tier_zero_execution::TierZeroExit; +use tier_zero_plan::TierZeroPlan; + +pub enum HeftRunDecision { + DelegateToJavaScriptHeft, + RunNatively(NativeHeftRun), + RunInNodeHost(NodeHostPlan), +} + +pub enum NativeHeftRun { + PrintCliOutput(PrintedOutput), + BuildWithBuiltinTasks { plan: Box, console: NativeConsoleSettings }, + CleanProject { + selections: Vec, + alias_expansion_message: Option, + console: NativeConsoleSettings, + }, +} + +pub struct NativeConsoleSettings { + supports_color: bool, + heft_package_folder: String, +} + +struct HeftInvocation<'invocation> { + command_line_arguments: &'invocation [OsString], + arguments: &'invocation [&'invocation str], + native_heft_context: &'invocation NativeHeftContext, + heft_package_folder: &'invocation str, +} + +pub fn decide_how_to_run_heft( + command_line_arguments: &[OsString], + native_heft_context: &NativeHeftContext, +) -> HeftRunDecision { + decide_with_project(command_line_arguments, native_heft_context).unwrap_or(HeftRunDecision::DelegateToJavaScriptHeft) +} + +fn decide_with_project(command_line_arguments: &[OsString], native_heft_context: &NativeHeftContext) -> Option { + if process_environment::rush_child_reporter_is_requested() { + return None; + } + let arguments: Vec<&str> = crate::cli::entry::command_line_strings(command_line_arguments)?; + let heft_package_folder = native_heft_context.companion_heft_package_folder.as_ref()?; + let heft_package_folder = std::fs::canonicalize(heft_package_folder).ok()?.to_str()?.to_owned(); + let heft_module_folder = format!("{heft_package_folder}/lib-commonjs/utilities"); + let current_folder = std::env::current_dir().ok()?.to_str()?.to_owned(); + let mut lookup = PackageJsonLookup::default(); + let build_folder_path = lookup.try_get_package_folder_for(¤t_folder).ok()??; + lookup.load_identity_for_folder(&build_folder_path).ok()?; + let request = HeftConfigurationRequest { + build_folder_path: &build_folder_path, + heft_module_folder: &heft_module_folder, + }; + let invocation = HeftInvocation { + command_line_arguments, + arguments: &arguments, + native_heft_context, + heft_package_folder: &heft_package_folder, + }; + load_heft_configuration_and_then(&request, &mut lookup, |loaded| decide_with_configuration(&invocation, loaded)).ok()? +} + +fn decide_with_configuration(invocation: &HeftInvocation<'_>, loaded: &LoadedHeftConfiguration<'_>) -> Option { + let model = build_heft_configuration_model(loaded).ok()?; + let cli_model = build_cli_model(&model).ok()?; + match crate::cli::entry::interpret_command_line_with_color(invocation.arguments, &cli_model, &|| supports_color_for(invocation)) { + CliOutcome::Print(printed_output) => Some(HeftRunDecision::RunNatively(NativeHeftRun::PrintCliOutput(printed_output))), + CliOutcome::Delegate => None, + CliOutcome::Execute(command) => match native_build_run(invocation, &model, &command) { + Some(native_heft_run) => Some(HeftRunDecision::RunNatively(native_heft_run)), + None if command.watch || command.debug => None, + None => node_host_plan::node_host_plan( + invocation.command_line_arguments, + invocation.native_heft_context, + loaded, + &command, + ) + .map(HeftRunDecision::RunInNodeHost), + }, + } +} + +fn native_build_run( + invocation: &HeftInvocation<'_>, + model: &HeftConfigurationModel<'_>, + command: &ParsedCommand<'_>, +) -> Option { + if let Some(clean_options) = command_options::tier_zero_clean_options(command) { + let selections = configured_project::native_clean_selections(model, &clean_options.selected_phase_indices)?; + let passes_preflight = process_environment::standard_input_is_the_null_device() + && selections_are_deletable_without_permission_errors(&selections); + return passes_preflight.then(|| NativeHeftRun::CleanProject { + selections, + alias_expansion_message: clean_options.alias_expansion_message, + console: console_settings_for(invocation), + }); + } + let command_options = command_options::tier_zero_command_options(command)?; + let request = configured_project::native_build_request(model, invocation.heft_package_folder, command_options)?; + if !process_environment::standard_input_is_the_null_device() { + return None; + } + let plan = tier_zero_plan::plan_tier_zero_build(&request)?; + Some(NativeHeftRun::BuildWithBuiltinTasks { + plan: Box::new(plan), + console: console_settings_for(invocation), + }) +} + +fn supports_color_for(invocation: &HeftInvocation<'_>) -> bool { + let tool_arguments: Vec = invocation.arguments.iter().map(|argument| (*argument).to_owned()).collect(); + console_supports_color_for_this_process(&tool_arguments) +} + +fn console_settings_for(invocation: &HeftInvocation<'_>) -> NativeConsoleSettings { + NativeConsoleSettings { + supports_color: supports_color_for(invocation), + heft_package_folder: invocation.heft_package_folder.to_owned(), + } +} + +pub fn run_heft_natively(native_heft_run: NativeHeftRun) -> i32 { + match native_heft_run { + NativeHeftRun::PrintCliOutput(printed_output) => crate::cli::entry::write_printed_output(&printed_output), + NativeHeftRun::BuildWithBuiltinTasks { plan, console } => exit_code_of( + tier_zero_execution::execute_tier_zero_plan(*plan, &HeftConsole::new(console.supports_color)), + &console, + ), + NativeHeftRun::CleanProject { selections, alias_expansion_message, console } => exit_code_of( + tier_zero_execution::execute_tier_zero_clean( + &selections, + alias_expansion_message.as_deref(), + &HeftConsole::new(console.supports_color), + ), + &console, + ), + } +} + +fn exit_code_of(tier_zero_exit: TierZeroExit, console: &NativeConsoleSettings) -> i32 { + match tier_zero_exit { + TierZeroExit::Code(exit_code) => exit_code, + TierZeroExit::OutputClosed(closed) => closed_output::exit_like_node_after_closed_output(closed, &console.heft_package_folder), + } +} diff --git a/apps/heft-native/src/run/node_host_plan.rs b/apps/heft-native/src/run/node_host_plan.rs new file mode 100644 index 0000000000..cd5d80b7b0 --- /dev/null +++ b/apps/heft-native/src/run/node_host_plan.rs @@ -0,0 +1,32 @@ +use std::ffi::OsString; + +use crate::cli::outcome::ParsedCommand; +use crate::config::loader::LoadedHeftConfiguration; +use crate::config::plan_members::write_plan_configuration_members; +use crate::host_link::{NodeHostPlan, NodeHostPlanWriter, NODE_HOST_ENTRY_PATH_INSIDE_HEFT_PACKAGE}; +use crate::version::NativeHeftContext; + +pub fn node_host_plan( + command_line_arguments: &[OsString], + native_heft_context: &NativeHeftContext, + loaded: &LoadedHeftConfiguration<'_>, + command: &ParsedCommand<'_>, +) -> Option { + let heft_package_folder = native_heft_context.companion_heft_package_folder.as_ref()?; + if !heft_package_folder.join(NODE_HOST_ENTRY_PATH_INSIDE_HEFT_PACKAGE).is_file() { + return None; + } + let mut plan = NodeHostPlanWriter::start(command_line_arguments, native_heft_context)?.finish(); + if plan.plan_json.pop() != Some('}') { + return None; + } + plan.plan_json.push(','); + write_plan_configuration_members(loaded, &mut plan.plan_json).ok()?; + let length_without_command = plan.plan_json.len(); + plan.plan_json.push_str(",\"command\":"); + if !command.write_plan_command(&mut plan.plan_json) { + plan.plan_json.truncate(length_without_command); + } + plan.plan_json.push('}'); + Some(plan) +} diff --git a/apps/heft-native/src/run/process_environment.rs b/apps/heft-native/src/run/process_environment.rs new file mode 100644 index 0000000000..377f28c33e --- /dev/null +++ b/apps/heft-native/src/run/process_environment.rs @@ -0,0 +1,26 @@ +use std::env; + +pub fn rush_child_reporter_is_requested() -> bool { + env::var_os("_RUSH_REPORTER_CHILD_FD").is_some() || env::var_os("_RUSH_REPORTER_CHILD_ACK_FD").is_some() +} + +#[cfg(unix)] +pub fn standard_input_is_the_null_device() -> bool { + use std::os::fd::AsFd; + use std::os::unix::fs::{FileTypeExt, MetadataExt}; + let Ok(standard_input) = std::io::stdin().as_fd().try_clone_to_owned() else { + return false; + }; + let Ok(standard_input_metadata) = std::fs::File::from(standard_input).metadata() else { + return false; + }; + let Ok(null_device_metadata) = std::fs::metadata("/dev/null") else { + return false; + }; + standard_input_metadata.file_type().is_char_device() && standard_input_metadata.rdev() == null_device_metadata.rdev() +} + +#[cfg(not(unix))] +pub fn standard_input_is_the_null_device() -> bool { + false +} diff --git a/apps/heft-native/src/run/tests_tier_zero.rs b/apps/heft-native/src/run/tests_tier_zero.rs new file mode 100644 index 0000000000..1a8dee1868 --- /dev/null +++ b/apps/heft-native/src/run/tests_tier_zero.rs @@ -0,0 +1,161 @@ +use std::fs; +use std::path::PathBuf; + +use super::tier_zero_execution::execute_tier_zero_plan; +use super::tier_zero_plan::{plan_tier_zero_build, NativeBuildRequest, NativePhaseDefinition, NativeTaskDefinition}; +use crate::builtin::{BuiltinTaskOptions, CopyOperation, CopyOperationField, FileSelectionSpecifier}; +use crate::terminal::HeftConsole; + +const EXPECTED_BUILD_INFO_WRITTEN_BY_HEFT: &str = concat!( + "{\"configHash\":\"3k4X73dp7qIsqE9JHQnhS5p9FfuPWCYqtDUUgPOuA0w=\",\"inputFileVersions\":{", + "\"../../../src/assets/asset-1.txt\":\"YugSLTOiQ6bFB9QpUFAMbncAUYN9ON2l3qYqwGllgBA=\",", + "\"../../../src/assets/asset-2.txt\":\"cZKdgt8PJENLFts7Tnu5h5dFXfycpLZhBUTEkgTAwFQ=\",", + "\"../../../src/assets/asset-3.txt\":\"QGq7YmiHiJWYf2ThKttmrjAiAMVRZBSzNqB7u3KmHpI=\",", + "\"../../../src/assets/asset-4.txt\":\"ATcVvJraFzIqG2ZqUiQi6OVbM0VUPnvKYeb/YRykH2A=\",", + "\"../../../src/assets/asset-5.txt\":\"5DJsWWyqMWDacO0jLvWOGXEtZQeLcJyihUEG2920rjQ=\",", + "\"../../../src/assets/data.json\":\"MiIsKqCR7Ps4OlPdpBh2XdRWhTIHPEclpVUVVA6w81g=\"}}" +); + +pub fn strings(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +pub fn create_native_fixture(name: &str) -> PathBuf { + let root = std::env::temp_dir().join(format!("heft-native-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(root.join("src/assets")).unwrap(); + for index in 1..6 { + fs::write(root.join(format!("src/assets/asset-{index}.txt")), format!("native fixture asset {index}\n")).unwrap(); + } + fs::write(root.join("src/assets/data.json"), "{\"fixture\":\"native\"}").unwrap(); + fs::write(root.join("package.json"), "{\"name\":\"native\",\"version\":\"1.0.0\"}").unwrap(); + root +} + +pub fn native_fixture_request(root: &str, clean: bool) -> NativeBuildRequest { + let copy_assets = CopyOperation { + selection: FileSelectionSpecifier { + source_path: Some("src/assets".into()), + file_extensions: Some(strings(&[".txt", ".json"])), + ..Default::default() + }, + destination_folders: strings(&["lib/assets"]), + flatten: None, + hardlink: None, + field_order: vec![CopyOperationField::SourcePath, CopyOperationField::DestinationFolders, CopyOperationField::FileExtensions], + }; + let delete_scratch = FileSelectionSpecifier { + source_path: Some("temp/scratch".into()), + include_globs: Some(strings(&["**/*"])), + ..Default::default() + }; + let tasks = vec![ + NativeTaskDefinition { + task_name: "set-env".into(), + dependency_task_indices: vec![], + options: BuiltinTaskOptions::SetEnvironmentVariables(vec![("BENCH_NATIVE".into(), "1".into())]), + }, + NativeTaskDefinition { + task_name: "copy-assets".into(), + dependency_task_indices: vec![0], + options: BuiltinTaskOptions::CopyFiles(vec![copy_assets]), + }, + NativeTaskDefinition { + task_name: "delete-scratch".into(), + dependency_task_indices: vec![1], + options: BuiltinTaskOptions::DeleteFiles(vec![delete_scratch]), + }, + ]; + NativeBuildRequest { + build_folder_path: root.to_owned(), + phases: vec![NativePhaseDefinition { + phase_name: "build".into(), + dependency_phase_indices: vec![], + clean_files: vec![FileSelectionSpecifier { include_globs: Some(strings(&["lib"])), ..Default::default() }], + tasks, + }], + selected_phase_indices: vec![0], + clean, + alias_expansion_message: None, + } +} + +pub fn normalize_durations(text: &str) -> String { + let mut normalized = String::with_capacity(text.len()); + let mut remaining = text; + while let Some(open) = remaining.find('(') { + normalized.push_str(&remaining[..=open]); + remaining = &remaining[open + 1..]; + let number_length = remaining.bytes().take_while(|byte| byte.is_ascii_digit() || *byte == b'.').count(); + if number_length > 0 && remaining[number_length..].starts_with("s)") { + normalized.push_str(""); + remaining = &remaining[number_length..]; + } + } + normalized.push_str(remaining); + normalized +} + +pub fn run_and_capture(request: &NativeBuildRequest, supports_color: bool) -> (i32, String, String) { + let plan = plan_tier_zero_build(request).expect("native fixture is Tier 0"); + let console = HeftConsole::capturing(supports_color); + let exit_code = match execute_tier_zero_plan(plan, &console) { + super::tier_zero_execution::TierZeroExit::Code(exit_code) => exit_code, + super::tier_zero_execution::TierZeroExit::OutputClosed(_) => -1, + }; + let mut standard_output = String::new(); + let mut standard_error = String::new(); + for (severity, text) in console.captured_output() { + match severity { + crate::terminal::OutputSeverity::Log => standard_output.push_str(&text), + _ => standard_error.push_str(&text), + } + } + (exit_code, normalize_durations(&standard_output), normalize_durations(&standard_error)) +} + +#[test] +fn native_fixture_builds_incrementally_and_cleans_like_heft() { + let root = create_native_fixture("incremental"); + let root_text = root.to_str().unwrap(); + fs::create_dir_all(root.join("temp/scratch/nested")).unwrap(); + fs::write(root.join("temp/scratch/nested/file.txt"), "x").unwrap(); + let (exit_code, standard_output, standard_error) = run_and_capture(&native_fixture_request(root_text, false), false); + assert_eq!(exit_code, 0); + assert_eq!(standard_error, ""); + assert_eq!( + standard_output, + " ---- build started ---- \n[build:set-env] Setting environment variable BENCH_NATIVE=1\n[build:copy-assets] Copied 6 files and linked 0 files\n[build:delete-scratch] Deleted 1 file and 1 folder\n ---- build finished (s) ---- \n-------------------- Finished (s) --------------------\n" + ); + assert_eq!(fs::read_to_string(root.join("lib/assets/asset-3.txt")).unwrap(), "native fixture asset 3\n"); + let build_info = fs::read_to_string(root.join("temp/build/copy-assets/file-copy.json")).unwrap(); + assert_eq!(build_info, EXPECTED_BUILD_INFO_WRITTEN_BY_HEFT); + let (_, second_output, _) = run_and_capture(&native_fixture_request(root_text, false), true); + assert!(second_output.contains("[build:copy-assets] All requested file copy operations are up to date. Nothing to do.\n")); + assert!(second_output.ends_with("\x1b[1m\x1b[32m-------------------- Finished (s) --------------------\x1b[39m\x1b[22m\n")); + let (_, clean_output, _) = run_and_capture(&native_fixture_request(root_text, true), false); + assert!(clean_output.contains("\n[build:clean] Deleted 0 files and 2 folders\n[build:set-env]")); + assert!(clean_output.contains("[build:copy-assets] Copied 6 files and linked 0 files\n")); + let _ = fs::remove_dir_all(&root); +} + +#[test] +fn an_independent_set_env_task_followed_by_one_file_task_runs_in_task_order() { + let root = create_native_fixture("independent-pair"); + let mut request = native_fixture_request(root.to_str().unwrap(), false); + request.phases[0].tasks.truncate(2); + request.phases[0].tasks[1].dependency_task_indices.clear(); + let plan = plan_tier_zero_build(&request).expect("set-env then copy is served natively"); + let logger_names: Vec<&str> = plan + .steps + .iter() + .filter_map(|planned_step| match &planned_step.step { + super::tier_zero_plan::TierZeroStep::RunTask { logger_name, .. } => Some(logger_name.as_str()), + _ => None, + }) + .collect(); + assert_eq!(logger_names, vec!["build:set-env", "build:copy-assets"]); + request.phases[0].tasks.swap(0, 1); + assert!(plan_tier_zero_build(&request).is_none()); + let _ = fs::remove_dir_all(&root); +} diff --git a/apps/heft-native/src/run/tier_zero_execution.rs b/apps/heft-native/src/run/tier_zero_execution.rs new file mode 100644 index 0000000000..ad311f8534 --- /dev/null +++ b/apps/heft-native/src/run/tier_zero_execution.rs @@ -0,0 +1,122 @@ +use std::time::Instant; + +use super::tier_zero_plan::{TierZeroPlan, TierZeroStep}; +use crate::builtin::{run_delete_operations, run_planned_builtin_task, AbsoluteFileSelection}; +use crate::terminal::{ + bold, format_rounded_milliseconds_as_seconds, format_seconds_with_three_fraction_digits, green, red, + ClosedOutput, HeftConsole, +}; + +const MISSING_PHASE_DEPENDENCIES_NOTICE: &str = + "The provided list of phases does not contain all phase dependencies. You may need to run the excluded phases manually."; + +pub enum TierZeroExit { + Code(i32), + OutputClosed(ClosedOutput), +} + +fn stop_if_output_closed(console: &HeftConsole) -> Result<(), TierZeroExit> { + match console.closed_output() { + Some(closed_output) => Err(TierZeroExit::OutputClosed(closed_output)), + None => Ok(()), + } +} + +pub fn execute_tier_zero_clean( + selections: &[AbsoluteFileSelection], + alias_expansion_message: Option<&str>, + console: &HeftConsole, +) -> TierZeroExit { + if let Some(alias_expansion_message) = alias_expansion_message { + console.write_line(alias_expansion_message); + if let Err(closed) = stop_if_output_closed(console) { + return closed; + } + } + let run_started_at = Instant::now(); + let result = run_delete_operations(selections, &console.unprefixed_output()); + if let Err(closed) = stop_if_output_closed(console) { + return closed; + } + write_summary(console, run_started_at, result.is_err()); + let exit_code = match result { + Ok(()) => 0, + Err(failure) => { + console.write_error_line(&format!("Error: {}", failure.message)); + 1 + } + }; + stop_if_output_closed(console).map_or_else(|closed| closed, |()| TierZeroExit::Code(exit_code)) +} + +pub fn execute_tier_zero_plan(plan: TierZeroPlan, console: &HeftConsole) -> TierZeroExit { + match execute_steps(plan, console) { + Ok(exit_code) => TierZeroExit::Code(exit_code), + Err(closed) => closed, + } +} + +fn execute_steps(plan: TierZeroPlan, console: &HeftConsole) -> Result { + if let Some(alias_expansion_message) = &plan.alias_expansion_message { + console.write_line(alias_expansion_message); + } + if plan.selection_is_missing_phase_dependencies { + console.write_line(&bold(MISSING_PHASE_DEPENDENCIES_NOTICE)); + } + stop_if_output_closed(console)?; + let run_started_at = Instant::now(); + let mut phase_started_at = run_started_at; + let mut encountered_error = false; + for planned_step in plan.steps { + let step_result = match planned_step.step { + TierZeroStep::StartPhase { phase_name, clean_selections } => { + phase_started_at = Instant::now(); + console.write_line(&format!(" ---- {phase_name} started ---- ")); + stop_if_output_closed(console)?; + match clean_selections { + Some(selections) => { + run_delete_operations(&selections, &console.scoped_logger_output(&format!("{phase_name}:clean"))) + } + None => Ok(()), + } + } + TierZeroStep::RunTask { logger_name, planned_task } => { + run_planned_builtin_task(planned_task, &console.scoped_logger_output(&logger_name)) + } + }; + stop_if_output_closed(console)?; + let phase_duration_in_seconds = phase_started_at.elapsed().as_secs_f64(); + if let Err(failure) = &step_result { + encountered_error = true; + if !failure.message.is_empty() { + console.write_error_line(&failure.message); + } + } + if planned_step.completes_phase { + let finished_logging_word = if encountered_error { "encountered an error" } else { "finished" }; + console.write_line(&format!( + " ---- {} {finished_logging_word} ({}s) ---- ", + planned_step.phase_name, + format_seconds_with_three_fraction_digits(phase_duration_in_seconds) + )); + } + stop_if_output_closed(console)?; + if encountered_error { + break; + } + } + write_summary(console, run_started_at, encountered_error); + stop_if_output_closed(console)?; + Ok(if encountered_error { 1 } else { 0 }) +} + +fn write_summary(console: &HeftConsole, run_started_at: Instant, encountered_error: bool) { + let run_duration_in_milliseconds = run_started_at.elapsed().as_secs_f64() * 1000.0; + let finished_logging_word = if encountered_error { "Failed" } else { "Finished" }; + let finished_logging_line = format!( + "-------------------- {finished_logging_word} ({}s) --------------------", + format_rounded_milliseconds_as_seconds(run_duration_in_milliseconds) + ); + let colorize = if encountered_error { red } else { green }; + console.write_line(&bold(&colorize(&finished_logging_line))); +} diff --git a/apps/heft-native/src/run/tier_zero_plan.rs b/apps/heft-native/src/run/tier_zero_plan.rs new file mode 100644 index 0000000000..1394ddaf43 --- /dev/null +++ b/apps/heft-native/src/run/tier_zero_plan.rs @@ -0,0 +1,124 @@ +use crate::builtin::{ + builtin_task_passes_preflight, builtin_task_touches_files, plan_builtin_task, plan_phase_clean, + selections_are_deletable_without_permission_errors, AbsoluteFileSelection, BuiltinTaskOptions, FileSelectionSpecifier, + PlannedBuiltinTask, +}; +use crate::graph::{plan_sequential_operations, HeftOperation, PhaseShape}; + +pub struct NativePhaseDefinition { + pub phase_name: String, + pub dependency_phase_indices: Vec, + pub clean_files: Vec, + pub tasks: Vec, +} + +pub struct NativeTaskDefinition { + pub task_name: String, + pub dependency_task_indices: Vec, + pub options: BuiltinTaskOptions, +} + +pub struct NativeBuildRequest { + pub build_folder_path: String, + pub phases: Vec, + pub selected_phase_indices: Vec, + pub clean: bool, + pub alias_expansion_message: Option, +} + +pub enum TierZeroStep { + StartPhase { phase_name: String, clean_selections: Option> }, + RunTask { logger_name: String, planned_task: PlannedBuiltinTask }, +} + +pub struct TierZeroPlannedStep { + pub step: TierZeroStep, + pub phase_name: String, + pub completes_phase: bool, +} + +pub struct TierZeroPlan { + pub alias_expansion_message: Option, + pub selection_is_missing_phase_dependencies: bool, + pub steps: Vec, +} + +const FIRST_TASK_INDEX: [usize; 1] = [0]; + +fn is_output_before_io_pair(phase: &NativePhaseDefinition) -> bool { + matches!( + phase.tasks.as_slice(), + [first, second] + if first.dependency_task_indices.is_empty() + && second.dependency_task_indices.is_empty() + && matches!(first.options, BuiltinTaskOptions::SetEnvironmentVariables(_)) + && !matches!(second.options, BuiltinTaskOptions::SetEnvironmentVariables(_)) + ) +} + +pub fn plan_tier_zero_build(request: &NativeBuildRequest) -> Option { + let phase_shapes: Vec> = request + .phases + .iter() + .map(|phase| { + let mut task_dependency_indices: Vec<&[usize]> = + phase.tasks.iter().map(|task| task.dependency_task_indices.as_slice()).collect(); + if is_output_before_io_pair(phase) { + task_dependency_indices[1] = &FIRST_TASK_INDEX; + } + PhaseShape { dependency_phase_indices: &phase.dependency_phase_indices, task_dependency_indices } + }) + .collect(); + let sequential_plan = plan_sequential_operations(&phase_shapes, &request.selected_phase_indices)?; + let temp_folder_path = format!("{}/temp", request.build_folder_path); + let mut steps: Vec<(usize, TierZeroStep)> = Vec::new(); + let mut files_may_have_changed = false; + for operation in &sequential_plan.operations_in_execution_order { + match *operation { + HeftOperation::Phase { phase_index } => { + let phase = &request.phases[phase_index]; + let clean_selections = if request.clean { + let selections = plan_phase_clean( + &phase.clean_files, + &request.build_folder_path, + &temp_folder_path, + &phase.phase_name, + )?; + selections_are_deletable_without_permission_errors(&selections).then_some(())?; + files_may_have_changed = true; + Some(selections) + } else { + None + }; + steps.push((phase_index, TierZeroStep::StartPhase { phase_name: phase.phase_name.clone(), clean_selections })); + } + HeftOperation::Task { phase_index, task_index } => { + let phase = &request.phases[phase_index]; + let task = &phase.tasks[task_index]; + let task_temp_folder_path = format!("{temp_folder_path}/{}/{}", phase.phase_name, task.task_name); + let mut planned_task = plan_builtin_task(&task.options, &request.build_folder_path, &task_temp_folder_path)?; + builtin_task_passes_preflight(&mut planned_task, &temp_folder_path, !files_may_have_changed).then_some(())?; + files_may_have_changed |= builtin_task_touches_files(&planned_task); + let logger_name = format!("{}:{}", phase.phase_name, task.task_name); + steps.push((phase_index, TierZeroStep::RunTask { logger_name, planned_task })); + } + } + } + let last_step_of_phase = |phase_index: usize| steps.iter().rposition(|(index, _)| *index == phase_index); + let completes_phase: Vec = (0..steps.len()) + .map(|position| last_step_of_phase(steps[position].0) == Some(position)) + .collect(); + Some(TierZeroPlan { + alias_expansion_message: request.alias_expansion_message.clone(), + selection_is_missing_phase_dependencies: sequential_plan.selection_is_missing_phase_dependencies, + steps: steps + .into_iter() + .zip(completes_phase) + .map(|((phase_index, step), completes_phase)| TierZeroPlannedStep { + step, + phase_name: request.phases[phase_index].phase_name.clone(), + completes_phase, + }) + .collect(), + }) +} diff --git a/apps/heft-native/src/schema/compile_applicator_keywords.rs b/apps/heft-native/src/schema/compile_applicator_keywords.rs new file mode 100644 index 0000000000..834dd25daa --- /dev/null +++ b/apps/heft-native/src/schema/compile_applicator_keywords.rs @@ -0,0 +1,101 @@ +use crate::json::JsonValue; + +use super::compiled_node::{NodeIdentifier, SchemaNodeUnderConstruction}; +use super::compiler::SchemaCompiler; +use super::schema_keywords::{Keyword, SchemaKeywordValues}; + +const DEFINITIONS_KEYWORDS: [(Keyword, &str); 2] = [ + (Keyword::Definitions, "definitions"), + (Keyword::DollarDefinitions, "$defs"), +]; + +fn local_reference_to_definition(definitions_keyword: &str, definition_name: &str) -> String { + let escaped_name = definition_name.replace('~', "~0").replace('/', "~1"); + let mut reference = String::with_capacity(3 + definitions_keyword.len() + escaped_name.len()); + reference.push_str("#/"); + reference.push_str(definitions_keyword); + reference.push('/'); + reference.push_str(&escaped_name); + reference +} + +impl<'schema> SchemaCompiler<'schema> { + fn move_inline_subschemas_before_references(&self, schemas: &mut [NodeIdentifier]) { + let mut inline_schema_count = 0; + for index in 0..schemas.len() { + let schema = schemas[index]; + if self.nodes[schema as usize].reference.is_none() { + schemas.copy_within(inline_schema_count..index, inline_schema_count + 1); + schemas[inline_schema_count] = schema; + inline_schema_count += 1; + } + } + } + + pub(super) fn compile_applicator_keywords( + &mut self, + keywords: &SchemaKeywordValues<'schema>, + data_types: &[&'schema str], + node: &mut SchemaNodeUnderConstruction<'schema>, + ) -> Option<()> { + if let Some(all_of) = keywords.value(Keyword::AllOf) { + let mut all_of_schemas = self.compile_schema_array(all_of, data_types)?; + self.move_inline_subschemas_before_references(&mut all_of_schemas); + node.all_of = Some(all_of_schemas); + } + if let Some(any_of) = keywords.value(Keyword::AnyOf) { + node.uncommon.any_of = Some(self.compile_schema_array(any_of, data_types)?); + } + if let Some(one_of) = keywords.value(Keyword::OneOf) { + node.uncommon.one_of = Some(self.compile_schema_array(one_of, data_types)?); + } + if let Some(not) = keywords.value(Keyword::Not) { + node.uncommon.not = Some(self.compile_node(not, data_types, false)?); + } + let if_value = keywords.value(Keyword::If); + let then_value = keywords.value(Keyword::Then); + let else_value = keywords.value(Keyword::Else); + if if_value.is_none() && then_value.is_none() && else_value.is_none() { + return Some(()); + } + let if_value = if_value?; + if then_value.is_none() && else_value.is_none() { + return None; + } + node.uncommon.if_schema = Some(self.compile_node(if_value, data_types, false)?); + if let Some(then_value) = then_value { + node.uncommon.then_schema = Some(self.compile_node(then_value, data_types, false)?); + } + if let Some(else_value) = else_value { + node.uncommon.else_schema = Some(self.compile_node(else_value, data_types, false)?); + } + Some(()) + } + + pub(super) fn compile_definitions_and_reference( + &mut self, + keywords: &SchemaKeywordValues<'schema>, + is_root: bool, + node: &mut SchemaNodeUnderConstruction<'schema>, + ) -> Option<()> { + for (definitions_keyword, definitions_keyword_name) in DEFINITIONS_KEYWORDS { + let Some(definitions) = keywords.value(definitions_keyword) else { + continue; + }; + let definitions = match definitions { + JsonValue::Object(definitions) if is_root => definitions, + _ => return None, + }; + for (definition_name, _) in definitions.entries() { + self.compile_reference(&local_reference_to_definition( + definitions_keyword_name, + definition_name, + ))?; + } + } + if let Some(reference) = keywords.value(Keyword::Reference) { + node.reference = Some(self.compile_reference(reference.as_str()?)?); + } + Some(()) + } +} diff --git a/apps/heft-native/src/schema/compile_node.rs b/apps/heft-native/src/schema/compile_node.rs new file mode 100644 index 0000000000..4e8544ea92 --- /dev/null +++ b/apps/heft-native/src/schema/compile_node.rs @@ -0,0 +1,161 @@ +use crate::json::{JsonObject, JsonValue}; + +use super::compiled_node::{NodeIdentifier, NodeKind, SchemaNodeUnderConstruction}; +use super::compiler::SchemaCompiler; +use super::keyword_tables::{ + data_type_that_type_specific_keyword_applies_to, type_bit_for_json_schema_type_name, + type_list_has_type_applicable_to_keyword, type_list_includes_type_like_ajv, +}; +use super::schema_keywords::{Keyword, SchemaKeywordValues}; + +fn keyword_is_absent_or( + keywords: &SchemaKeywordValues<'_>, + keyword: Keyword, + expected_shape: fn(&JsonValue<'_>) -> bool, +) -> bool { + keywords.value(keyword).is_none_or(expected_shape) +} + +fn is_string_value(value: &JsonValue<'_>) -> bool { + matches!(value, JsonValue::String(_)) +} + +fn is_boolean_value(value: &JsonValue<'_>) -> bool { + matches!(value, JsonValue::Boolean(_)) +} + +fn is_array_value(value: &JsonValue<'_>) -> bool { + matches!(value, JsonValue::Array(_)) +} + +fn annotation_keywords_have_metaschema_shapes( + keywords: &SchemaKeywordValues<'_>, + is_draft_04: bool, + is_root: bool, +) -> bool { + let common_annotations_are_valid = [Keyword::Title, Keyword::Description, Keyword::Comment] + .into_iter() + .all(|keyword| keyword_is_absent_or(keywords, keyword, is_string_value)); + if !common_annotations_are_valid || (!is_root && keywords.has(Keyword::Schema)) { + return false; + } + is_draft_04 + || (keyword_is_absent_or(keywords, Keyword::Examples, is_array_value) + && [Keyword::ReadOnly, Keyword::WriteOnly, Keyword::Deprecated] + .into_iter() + .all(|keyword| keyword_is_absent_or(keywords, keyword, is_boolean_value)) + && [Keyword::ContentMediaType, Keyword::ContentEncoding] + .into_iter() + .all(|keyword| keyword_is_absent_or(keywords, keyword, is_string_value))) +} + +impl<'schema> SchemaCompiler<'schema> { + pub(super) fn compile_node( + &mut self, + schema: &'schema JsonValue<'schema>, + context_types: &[&'schema str], + is_root: bool, + ) -> Option { + let object = match schema { + JsonValue::Boolean(accepts_everything) => { + if self.is_draft_04 { + return None; + } + let kind = if *accepts_everything { + NodeKind::AlwaysValid + } else { + NodeKind::AlwaysInvalid + }; + return Some(self.allocate_node(SchemaNodeUnderConstruction::of_kind(kind))); + } + JsonValue::Object(object) => object, + _ => return None, + }; + let keywords = SchemaKeywordValues::gather_known_to_ajv( + object, + self.is_draft_04, + is_root, + &self.vendor_keywords, + )?; + if !annotation_keywords_have_metaschema_shapes(&keywords, self.is_draft_04, is_root) { + return None; + } + let mut node = SchemaNodeUnderConstruction::of_kind(NodeKind::Regular); + let data_types = compile_type_keyword_like_ajv_strict_types( + keywords.value(Keyword::Type), + context_types, + &mut node, + )?; + verify_type_specific_keywords_have_applicable_data_types(object, &data_types)?; + self.compile_enum_and_const_keywords(&keywords, &mut node)?; + self.compile_number_keywords(&keywords, &mut node)?; + self.compile_string_and_length_keywords(&keywords, &mut node)?; + self.compile_array_keywords(&keywords, &mut node)?; + self.compile_object_keywords(&keywords, &data_types, &mut node)?; + self.compile_applicator_keywords(&keywords, &data_types, &mut node)?; + self.compile_definitions_and_reference(&keywords, is_root, &mut node)?; + Some(self.allocate_node(node)) + } +} + +fn compile_type_keyword_like_ajv_strict_types<'schema>( + type_keyword: Option<&'schema JsonValue<'schema>>, + context_types: &[&'schema str], + node: &mut SchemaNodeUnderConstruction<'schema>, +) -> Option> { + let mut schema_types: Vec<&'schema str> = Vec::new(); + if let Some(type_value) = type_keyword { + match type_value { + JsonValue::String(type_name) => schema_types.push(type_name.as_ref()), + JsonValue::Array(items) if !items.is_empty() => { + schema_types.reserve_exact(items.len()); + for item in items { + schema_types.push(item.as_str()?); + } + } + _ => return None, + } + for (index, type_name) in schema_types.iter().enumerate() { + let type_bit = type_bit_for_json_schema_type_name(type_name)?; + if schema_types[..index].contains(type_name) { + return None; + } + node.allowed_type_bits |= type_bit; + } + } + if schema_types.is_empty() { + return Some(context_types.to_vec()); + } + if context_types.is_empty() { + return Some(schema_types); + } + if !schema_types + .iter() + .all(|type_name| type_list_includes_type_like_ajv(context_types, type_name)) + { + return None; + } + let mut narrowed_types: Vec<&'schema str> = Vec::with_capacity(context_types.len()); + for context_type in context_types { + if type_list_includes_type_like_ajv(&schema_types, context_type) { + narrowed_types.push(context_type); + } else if *context_type == "number" && schema_types.contains(&"integer") { + narrowed_types.push("integer"); + } + } + Some(narrowed_types) +} + +fn verify_type_specific_keywords_have_applicable_data_types( + object: &JsonObject<'_>, + data_types: &[&str], +) -> Option<()> { + for (key, _) in object.entries() { + if let Some(keyword_type) = data_type_that_type_specific_keyword_applies_to(key) { + if !type_list_has_type_applicable_to_keyword(data_types, keyword_type) { + return None; + } + } + } + Some(()) +} diff --git a/apps/heft-native/src/schema/compile_object_keywords.rs b/apps/heft-native/src/schema/compile_object_keywords.rs new file mode 100644 index 0000000000..fc0439afde --- /dev/null +++ b/apps/heft-native/src/schema/compile_object_keywords.rs @@ -0,0 +1,139 @@ +use crate::json::JsonValue; + +use super::compile_value_keywords::optional_count_keyword; +use super::compiled_node::{ + DependencyRequirement, NodeIdentifier, NodeKind, SchemaNodeUnderConstruction, +}; +use super::compiler::SchemaCompiler; +use super::keyword_tables::is_object_prototype_member_name; +use super::schema_keywords::{Keyword, SchemaKeywordValues}; + +fn unique_property_name_list<'schema>( + value: &'schema JsonValue<'schema>, +) -> Option> { + let items = value.as_array()?; + let mut names: Vec<&'schema str> = Vec::with_capacity(items.len()); + for item in items { + let name = item.as_str()?; + if is_object_prototype_member_name(name) || names.contains(&name) { + return None; + } + names.push(name); + } + Some(names) +} + +impl<'schema> SchemaCompiler<'schema> { + pub(super) fn compile_object_keywords( + &mut self, + keywords: &SchemaKeywordValues<'schema>, + data_types: &[&'schema str], + node: &mut SchemaNodeUnderConstruction<'schema>, + ) -> Option<()> { + if let Some(required) = keywords.value(Keyword::Required) { + let names = unique_property_name_list(required)?; + if self.is_draft_04 && names.is_empty() { + return None; + } + node.required = Some(names); + } + node.uncommon.min_properties = + optional_count_keyword(keywords.value(Keyword::MinProperties))?; + node.uncommon.max_properties = + optional_count_keyword(keywords.value(Keyword::MaxProperties))?; + if let Some(properties) = keywords.value(Keyword::Properties) { + node.properties = Some(self.compile_properties_keyword(properties)?); + } + if let Some(pattern_properties) = keywords.value(Keyword::PatternProperties) { + let compiled = self.compile_pattern_properties_keyword( + pattern_properties, + node.properties.as_deref(), + )?; + node.uncommon.pattern_properties = Some(compiled); + } + if let Some(additional_properties) = keywords.value(Keyword::AdditionalProperties) { + node.additional_properties = Some(match additional_properties { + JsonValue::Boolean(true) => { + self.allocate_node(SchemaNodeUnderConstruction::of_kind(NodeKind::AlwaysValid)) + } + JsonValue::Boolean(false) => self.allocate_node( + SchemaNodeUnderConstruction::of_kind(NodeKind::AlwaysInvalid), + ), + _ => self.compile_node(additional_properties, &[], false)?, + }); + } + if let Some(dependencies) = keywords.value(Keyword::Dependencies) { + node.uncommon.dependencies = + Some(self.compile_dependencies_keyword(dependencies, data_types)?); + } + if let Some(property_names) = keywords.value(Keyword::PropertyNames) { + node.uncommon.property_names = + Some(self.compile_node(property_names, &["string"], false)?); + } + Some(()) + } + + fn compile_properties_keyword( + &mut self, + properties: &'schema JsonValue<'schema>, + ) -> Option> { + let properties = properties.as_object()?; + let mut compiled = Vec::with_capacity(properties.len()); + for (name, subschema) in properties.entries() { + if is_object_prototype_member_name(name) { + return None; + } + compiled.push((name.as_ref(), self.compile_node(subschema, &[], false)?)); + } + Some(compiled) + } + + fn compile_pattern_properties_keyword( + &mut self, + pattern_properties: &'schema JsonValue<'schema>, + sibling_properties: Option<&[(&'schema str, NodeIdentifier)]>, + ) -> Option> { + let pattern_properties = pattern_properties.as_object()?; + let mut compiled = Vec::with_capacity(pattern_properties.len()); + for (pattern, subschema) in pattern_properties.entries() { + if pattern == "__proto__" { + return None; + } + let regex_index = self.regex_index_for_pattern(pattern.as_ref())?; + for (property_name, _) in sibling_properties.unwrap_or_default() { + let property_matches_pattern = self.regexes[regex_index] + .matches_anywhere_without_unicode_flag(property_name)?; + if property_matches_pattern { + return None; + } + } + compiled.push((regex_index, self.compile_node(subschema, &[], false)?)); + } + Some(compiled) + } + + fn compile_dependencies_keyword( + &mut self, + dependencies: &'schema JsonValue<'schema>, + data_types: &[&'schema str], + ) -> Option)>> { + let dependencies = dependencies.as_object()?; + let mut compiled = Vec::with_capacity(dependencies.len()); + for (name, dependency) in dependencies.entries() { + if is_object_prototype_member_name(name) { + return None; + } + let requirement = if matches!(dependency, JsonValue::Array(_)) { + let names = unique_property_name_list(dependency)?; + if self.is_draft_04 && names.is_empty() { + return None; + } + DependencyRequirement::RequiredPropertyNames(names) + } else { + DependencyRequirement::Subschema(self.compile_node(dependency, data_types, false)?) + }; + compiled.push((name.as_ref(), requirement)); + } + Some(compiled) + } +} diff --git a/apps/heft-native/src/schema/compile_value_keywords.rs b/apps/heft-native/src/schema/compile_value_keywords.rs new file mode 100644 index 0000000000..187771c245 --- /dev/null +++ b/apps/heft-native/src/schema/compile_value_keywords.rs @@ -0,0 +1,141 @@ +use crate::json::JsonValue; + +use super::compiled_node::SchemaNodeUnderConstruction; +use super::compiler::SchemaCompiler; +use super::data_rules::{ + is_simple_json_data, json_values_are_deeply_equal, non_negative_safe_integer_value, +}; +use super::schema_keywords::{Keyword, SchemaKeywordValues}; + +fn optional_number_keyword(keyword_value: Option<&JsonValue<'_>>) -> Option> { + match keyword_value { + None => Some(None), + Some(JsonValue::Number(number)) => Some(Some(number.value)), + Some(_) => None, + } +} + +pub(super) fn optional_count_keyword(keyword_value: Option<&JsonValue<'_>>) -> Option> { + match keyword_value { + None => Some(None), + Some(value) => non_negative_safe_integer_value(value).map(Some), + } +} + +fn apply_draft_04_boolean_exclusive_bound( + exclusive_keyword: Option<&JsonValue<'_>>, + inclusive_bound: &mut Option, + exclusive_bound: &mut Option, +) -> Option<()> { + let Some(exclusive_keyword) = exclusive_keyword else { + return Some(()); + }; + let bound_is_exclusive = exclusive_keyword.as_bool()?; + inclusive_bound.as_ref()?; + if bound_is_exclusive { + *exclusive_bound = inclusive_bound.take(); + } + Some(()) +} + +fn has_duplicate_values(values: &[JsonValue<'_>]) -> bool { + (1..values.len()).any(|index| { + values[..index] + .iter() + .any(|earlier| json_values_are_deeply_equal(earlier, &values[index])) + }) +} + +impl<'schema> SchemaCompiler<'schema> { + pub(super) fn compile_enum_and_const_keywords( + &mut self, + keywords: &SchemaKeywordValues<'schema>, + node: &mut SchemaNodeUnderConstruction<'schema>, + ) -> Option<()> { + if let Some(enum_value) = keywords.value(Keyword::Enum) { + let allowed_values = enum_value.as_array()?; + if allowed_values.is_empty() + || !is_simple_json_data(enum_value, 0) + || has_duplicate_values(allowed_values) + { + return None; + } + node.enum_values = Some(allowed_values); + } + if let Some(const_value) = keywords.value(Keyword::Const) { + if !is_simple_json_data(const_value, 0) { + return None; + } + node.uncommon.const_value = Some(const_value); + } + Some(()) + } + + pub(super) fn compile_number_keywords( + &mut self, + keywords: &SchemaKeywordValues<'schema>, + node: &mut SchemaNodeUnderConstruction<'schema>, + ) -> Option<()> { + node.uncommon.minimum = optional_number_keyword(keywords.value(Keyword::Minimum))?; + node.uncommon.maximum = optional_number_keyword(keywords.value(Keyword::Maximum))?; + if self.is_draft_04 { + apply_draft_04_boolean_exclusive_bound( + keywords.value(Keyword::ExclusiveMinimum), + &mut node.uncommon.minimum, + &mut node.uncommon.exclusive_minimum, + )?; + apply_draft_04_boolean_exclusive_bound( + keywords.value(Keyword::ExclusiveMaximum), + &mut node.uncommon.maximum, + &mut node.uncommon.exclusive_maximum, + )?; + } else { + node.uncommon.exclusive_minimum = + optional_number_keyword(keywords.value(Keyword::ExclusiveMinimum))?; + node.uncommon.exclusive_maximum = + optional_number_keyword(keywords.value(Keyword::ExclusiveMaximum))?; + } + if let Some(divisor) = optional_number_keyword(keywords.value(Keyword::MultipleOf))? { + if divisor.is_nan() || divisor <= 0.0 { + return None; + } + node.uncommon.multiple_of = Some(divisor); + } + Some(()) + } + + pub(super) fn compile_string_and_length_keywords( + &mut self, + keywords: &SchemaKeywordValues<'schema>, + node: &mut SchemaNodeUnderConstruction<'schema>, + ) -> Option<()> { + node.uncommon.min_length = optional_count_keyword(keywords.value(Keyword::MinLength))?; + node.uncommon.max_length = optional_count_keyword(keywords.value(Keyword::MaxLength))?; + if let Some(pattern) = keywords.value(Keyword::Pattern) { + node.pattern_regex_index = Some(self.regex_index_for_pattern(pattern.as_str()?)?); + } + Some(()) + } + + pub(super) fn compile_array_keywords( + &mut self, + keywords: &SchemaKeywordValues<'schema>, + node: &mut SchemaNodeUnderConstruction<'schema>, + ) -> Option<()> { + node.uncommon.min_items = optional_count_keyword(keywords.value(Keyword::MinItems))?; + node.uncommon.max_items = optional_count_keyword(keywords.value(Keyword::MaxItems))?; + if let Some(unique_items) = keywords.value(Keyword::UniqueItems) { + node.uncommon.unique_items = unique_items.as_bool()?; + } + if let Some(items) = keywords.value(Keyword::Items) { + if matches!(items, JsonValue::Array(_)) { + return None; + } + node.items = Some(self.compile_node(items, &[], false)?); + } + if let Some(contains) = keywords.value(Keyword::Contains) { + node.uncommon.contains = Some(self.compile_node(contains, &[], false)?); + } + Some(()) + } +} diff --git a/apps/heft-native/src/schema/compiled_node.rs b/apps/heft-native/src/schema/compiled_node.rs new file mode 100644 index 0000000000..68dbf29bfa --- /dev/null +++ b/apps/heft-native/src/schema/compiled_node.rs @@ -0,0 +1,149 @@ +use crate::json::JsonValue; + +pub(super) type NodeIdentifier = u32; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum NodeKind { + Regular, + AlwaysValid, + AlwaysInvalid, +} + +pub(super) enum DependencyRequirement<'schema> { + RequiredPropertyNames(Vec<&'schema str>), + Subschema(NodeIdentifier), +} + +#[derive(Default)] +pub(super) struct UncommonKeywordConstraints<'schema> { + pub(super) const_value: Option<&'schema JsonValue<'schema>>, + pub(super) minimum: Option, + pub(super) maximum: Option, + pub(super) exclusive_minimum: Option, + pub(super) exclusive_maximum: Option, + pub(super) multiple_of: Option, + pub(super) min_length: Option, + pub(super) max_length: Option, + pub(super) min_items: Option, + pub(super) max_items: Option, + pub(super) unique_items: bool, + pub(super) contains: Option, + pub(super) min_properties: Option, + pub(super) max_properties: Option, + pub(super) pattern_properties: Option>, + pub(super) dependencies: Option)>>, + pub(super) property_names: Option, + pub(super) any_of: Option>, + pub(super) one_of: Option>, + pub(super) not: Option, + pub(super) if_schema: Option, + pub(super) then_schema: Option, + pub(super) else_schema: Option, +} + +impl UncommonKeywordConstraints<'_> { + fn constrains_anything(&self) -> bool { + self.const_value.is_some() + || [ + self.minimum, + self.maximum, + self.exclusive_minimum, + self.exclusive_maximum, + ] + .iter() + .any(Option::is_some) + || self.multiple_of.is_some() + || [ + self.min_length, + self.max_length, + self.min_items, + self.max_items, + ] + .iter() + .any(Option::is_some) + || self.unique_items + || [self.min_properties, self.max_properties] + .iter() + .any(Option::is_some) + || self.pattern_properties.is_some() + || self.dependencies.is_some() + || self.any_of.is_some() + || self.one_of.is_some() + || [ + self.contains, + self.property_names, + self.not, + self.if_schema, + self.then_schema, + self.else_schema, + ] + .iter() + .any(Option::is_some) + } +} + +pub(super) struct CompiledSchemaNode<'schema> { + pub(super) kind: NodeKind, + pub(super) allowed_type_bits: u8, + pub(super) items: Option, + pub(super) additional_properties: Option, + pub(super) reference: Option, + pub(super) pattern_regex_index: Option, + pub(super) enum_values: Option<&'schema [JsonValue<'schema>]>, + pub(super) required: Option>, + pub(super) properties: Option>, + pub(super) all_of: Option>, + pub(super) uncommon: Option>>, +} + +pub(super) struct SchemaNodeUnderConstruction<'schema> { + pub(super) kind: NodeKind, + pub(super) allowed_type_bits: u8, + pub(super) items: Option, + pub(super) additional_properties: Option, + pub(super) reference: Option, + pub(super) pattern_regex_index: Option, + pub(super) enum_values: Option<&'schema [JsonValue<'schema>]>, + pub(super) required: Option>, + pub(super) properties: Option>, + pub(super) all_of: Option>, + pub(super) uncommon: UncommonKeywordConstraints<'schema>, +} + +impl<'schema> SchemaNodeUnderConstruction<'schema> { + pub(super) fn of_kind(kind: NodeKind) -> Self { + SchemaNodeUnderConstruction { + kind, + allowed_type_bits: 0, + items: None, + additional_properties: None, + reference: None, + pattern_regex_index: None, + enum_values: None, + required: None, + properties: None, + all_of: None, + uncommon: UncommonKeywordConstraints::default(), + } + } + + pub(super) fn into_compiled_node(self) -> CompiledSchemaNode<'schema> { + CompiledSchemaNode { + kind: self.kind, + allowed_type_bits: self.allowed_type_bits, + items: self.items, + additional_properties: self.additional_properties, + reference: self.reference, + pattern_regex_index: self.pattern_regex_index, + enum_values: self.enum_values, + required: self.required, + properties: self.properties, + all_of: self.all_of, + uncommon: if self.uncommon.constrains_anything() { + Some(Box::new(self.uncommon)) + } else { + None + }, + } + } +} diff --git a/apps/heft-native/src/schema/compiler.rs b/apps/heft-native/src/schema/compiler.rs new file mode 100644 index 0000000000..6740ca87b0 --- /dev/null +++ b/apps/heft-native/src/schema/compiler.rs @@ -0,0 +1,185 @@ +use std::borrow::Cow; + +use crate::json::{JsonObject, JsonValue}; +use crate::regex::{compile_unicode_regex_subset, CompiledUnicodeRegex}; + +use super::compiled_node::{ + CompiledSchemaNode, NodeIdentifier, NodeKind, SchemaNodeUnderConstruction, +}; +use super::keyword_tables::{is_object_prototype_member_name, is_vendor_extension_keyword}; + +pub struct CompiledJsonSchema<'schema> { + pub(super) nodes: Vec>, + pub(super) regexes: Vec, + pub(super) root: NodeIdentifier, +} + +pub(super) struct SchemaCompiler<'schema> { + pub(super) document_root: &'schema JsonValue<'schema>, + pub(super) is_draft_04: bool, + pub(super) vendor_keywords: Vec<&'schema str>, + pub(super) nodes: Vec>, + pub(super) regex_patterns: Vec<&'schema str>, + pub(super) regexes: Vec, + pub(super) reference_nodes: Vec<(String, NodeIdentifier)>, +} + +fn count_of_values_that_may_be_subschemas(value: &JsonValue<'_>) -> usize { + match value { + JsonValue::Object(object) => { + 1 + object + .entries() + .iter() + .map(|(_, item)| count_of_values_that_may_be_subschemas(item)) + .sum::() + } + JsonValue::Array(items) => items + .iter() + .map(count_of_values_that_may_be_subschemas) + .sum(), + JsonValue::Boolean(_) => 1, + _ => 0, + } +} + +fn schema_dialect_is_draft_04(root: &JsonObject<'_>) -> Option { + match root.get("$schema") { + None => Some(false), + Some(JsonValue::String(uri)) => match uri.as_ref() { + "http://json-schema.org/draft-07/schema" + | "http://json-schema.org/draft-07/schema#" => Some(false), + "http://json-schema.org/draft-04/schema" + | "http://json-schema.org/draft-04/schema#" => Some(true), + _ => None, + }, + Some(_) => None, + } +} + +pub fn compile_json_schema_for_fast_validation<'schema>( + schema_document: &'schema JsonValue<'schema>, +) -> Option> { + let root_object = schema_document.as_object()?; + let is_draft_04 = schema_dialect_is_draft_04(root_object)?; + if root_object.contains_key("id") || root_object.contains_key("$id") { + return None; + } + let vendor_keywords = root_object + .entries() + .iter() + .map(|(key, _)| key.as_ref()) + .filter(|key| is_vendor_extension_keyword(key)) + .collect(); + let mut compiler = SchemaCompiler { + document_root: schema_document, + is_draft_04, + vendor_keywords, + nodes: Vec::with_capacity(count_of_values_that_may_be_subschemas(schema_document)), + regex_patterns: Vec::new(), + regexes: Vec::new(), + reference_nodes: Vec::new(), + }; + let root = compiler.compile_node(schema_document, &[], true)?; + Some(CompiledJsonSchema { + nodes: compiler.nodes, + regexes: compiler.regexes, + root, + }) +} + +fn decode_json_pointer_segment(raw_segment: &str) -> Cow<'_, str> { + if raw_segment.contains('~') { + Cow::Owned(raw_segment.replace("~1", "/").replace("~0", "~")) + } else { + Cow::Borrowed(raw_segment) + } +} + +fn is_canonical_array_index(segment: &str) -> bool { + !segment.is_empty() + && segment.bytes().all(|byte| byte.is_ascii_digit()) + && (segment == "0" || !segment.starts_with('0')) +} + +impl<'schema> SchemaCompiler<'schema> { + pub(super) fn allocate_node( + &mut self, + node: SchemaNodeUnderConstruction<'schema>, + ) -> NodeIdentifier { + self.nodes.push(node.into_compiled_node()); + (self.nodes.len() - 1) as NodeIdentifier + } + + pub(super) fn regex_index_for_pattern(&mut self, pattern: &'schema str) -> Option { + if let Some(index) = self + .regex_patterns + .iter() + .position(|existing| *existing == pattern) + { + return Some(index); + } + let regex = compile_unicode_regex_subset(pattern)?; + self.regex_patterns.push(pattern); + self.regexes.push(regex); + Some(self.regexes.len() - 1) + } + + pub(super) fn compile_schema_array( + &mut self, + value: &'schema JsonValue<'schema>, + data_types: &[&'schema str], + ) -> Option> { + let items = value.as_array()?; + if items.is_empty() { + return None; + } + let mut identifiers = Vec::with_capacity(items.len()); + for item in items { + identifiers.push(self.compile_node(item, data_types, false)?); + } + Some(identifiers) + } + + pub(super) fn compile_reference(&mut self, reference: &str) -> Option { + if let Some((_, identifier)) = self + .reference_nodes + .iter() + .find(|(existing, _)| existing == reference) + { + return Some(*identifier); + } + let target = self.resolve_local_json_pointer(reference)?; + let placeholder = + self.allocate_node(SchemaNodeUnderConstruction::of_kind(NodeKind::Regular)); + self.reference_nodes + .push((reference.to_owned(), placeholder)); + let compiled_target = self.compile_node(target, &[], false)?; + self.nodes[placeholder as usize].reference = Some(compiled_target); + Some(placeholder) + } + + fn resolve_local_json_pointer(&self, reference: &str) -> Option<&'schema JsonValue<'schema>> { + let pointer = reference.strip_prefix('#')?; + if reference.contains('%') || (!pointer.is_empty() && !pointer.starts_with('/')) { + return None; + } + let mut target: &'schema JsonValue<'schema> = self.document_root; + if pointer.is_empty() { + return Some(target); + } + for raw_segment in pointer[1..].split('/') { + let segment = decode_json_pointer_segment(raw_segment); + if is_object_prototype_member_name(&segment) { + return None; + } + target = match target { + JsonValue::Array(items) if is_canonical_array_index(&segment) => { + items.get(segment.parse::().ok()?)? + } + JsonValue::Object(object) => object.get(&segment)?, + _ => return None, + }; + } + Some(target) + } +} diff --git a/apps/heft-native/src/schema/data_rules.rs b/apps/heft-native/src/schema/data_rules.rs new file mode 100644 index 0000000000..397bf90b5a --- /dev/null +++ b/apps/heft-native/src/schema/data_rules.rs @@ -0,0 +1,109 @@ +use crate::json::JsonValue; + +use super::keyword_tables::{ + TYPE_BIT_ARRAY, TYPE_BIT_BOOLEAN, TYPE_BIT_INTEGER, TYPE_BIT_NULL, TYPE_BIT_NUMBER, + TYPE_BIT_OBJECT, TYPE_BIT_STRING, +}; + +const OWN_KEYS_THAT_CHANGE_AJV_OR_FAST_DEEP_EQUAL_BEHAVIOR: [&str; 4] = + ["__proto__", "constructor", "valueOf", "toString"]; +const MAXIMUM_SIMPLE_DATA_DEPTH: u32 = 256; +const LARGEST_SAFE_INTEGER: f64 = 9007199254740991.0; + +pub(super) fn is_simple_json_data(value: &JsonValue<'_>, depth: u32) -> bool { + if depth > MAXIMUM_SIMPLE_DATA_DEPTH { + return false; + } + match value { + JsonValue::Number(number) => number.value.is_finite(), + JsonValue::Array(items) => items + .iter() + .all(|item| is_simple_json_data(item, depth + 1)), + JsonValue::Object(object) => object.entries().iter().all(|(key, item)| { + !OWN_KEYS_THAT_CHANGE_AJV_OR_FAST_DEEP_EQUAL_BEHAVIOR.contains(&key.as_ref()) + && is_simple_json_data(item, depth + 1) + }), + _ => true, + } +} + +pub(super) fn json_values_are_deeply_equal(first: &JsonValue<'_>, second: &JsonValue<'_>) -> bool { + match (first, second) { + (JsonValue::Null, JsonValue::Null) => true, + (JsonValue::Boolean(left), JsonValue::Boolean(right)) => left == right, + (JsonValue::Number(left), JsonValue::Number(right)) => left.value == right.value, + (JsonValue::String(left), JsonValue::String(right)) => left == right, + (JsonValue::Array(left), JsonValue::Array(right)) => { + left.len() == right.len() + && left + .iter() + .zip(right.iter()) + .all(|(left_item, right_item)| { + json_values_are_deeply_equal(left_item, right_item) + }) + } + (JsonValue::Object(left), JsonValue::Object(right)) => { + left.len() == right.len() + && left.entries().iter().all(|(key, left_item)| { + right.get(key).is_some_and(|right_item| { + json_values_are_deeply_equal(left_item, right_item) + }) + }) + } + _ => false, + } +} + +pub(super) fn json_schema_type_bits_of_data(value: &JsonValue<'_>) -> u8 { + match value { + JsonValue::Null => TYPE_BIT_NULL, + JsonValue::Boolean(_) => TYPE_BIT_BOOLEAN, + JsonValue::String(_) => TYPE_BIT_STRING, + JsonValue::Array(_) => TYPE_BIT_ARRAY, + JsonValue::Object(_) => TYPE_BIT_OBJECT, + JsonValue::Number(number) if !number.value.is_finite() => 0, + JsonValue::Number(number) if number.value.fract() == 0.0 => { + TYPE_BIT_NUMBER | TYPE_BIT_INTEGER + } + JsonValue::Number(_) => TYPE_BIT_NUMBER, + } +} + +pub(super) fn non_negative_safe_integer_value(value: &JsonValue<'_>) -> Option { + let number = value.as_f64()?; + let is_non_negative_safe_integer = + number >= 0.0 && number.fract() == 0.0 && number <= LARGEST_SAFE_INTEGER; + if is_non_negative_safe_integer { + Some(number as u64) + } else { + None + } +} + +fn leading_decimal_digits_of_exponential_notation(magnitude: f64) -> f64 { + let exponential_notation = format!("{:e}", magnitude); + let digit_count = exponential_notation + .bytes() + .take_while(u8::is_ascii_digit) + .count(); + exponential_notation[..digit_count] + .parse() + .unwrap_or(f64::NAN) +} + +pub(super) fn javascript_parse_int_of_number_to_string(number: f64) -> f64 { + if !number.is_finite() { + return f64::NAN; + } + let magnitude = number.abs(); + let uses_exponential_notation = magnitude >= 1e21 || (magnitude != 0.0 && magnitude < 1e-6); + if uses_exponential_notation { + let digits = leading_decimal_digits_of_exponential_notation(magnitude); + return if number < 0.0 { -digits } else { digits }; + } + let truncated = number.trunc(); + if truncated == 0.0 && number < 0.0 { + return -0.0; + } + truncated +} diff --git a/apps/heft-native/src/schema/keyword_tables.rs b/apps/heft-native/src/schema/keyword_tables.rs new file mode 100644 index 0000000000..3d28423507 --- /dev/null +++ b/apps/heft-native/src/schema/keyword_tables.rs @@ -0,0 +1,84 @@ +pub(super) const TYPE_BIT_NULL: u8 = 1; +pub(super) const TYPE_BIT_BOOLEAN: u8 = 2; +pub(super) const TYPE_BIT_OBJECT: u8 = 4; +pub(super) const TYPE_BIT_ARRAY: u8 = 8; +pub(super) const TYPE_BIT_NUMBER: u8 = 16; +pub(super) const TYPE_BIT_INTEGER: u8 = 32; +pub(super) const TYPE_BIT_STRING: u8 = 64; + +pub(super) fn type_bit_for_json_schema_type_name(type_name: &str) -> Option { + Some(match type_name { + "null" => TYPE_BIT_NULL, + "boolean" => TYPE_BIT_BOOLEAN, + "object" => TYPE_BIT_OBJECT, + "array" => TYPE_BIT_ARRAY, + "number" => TYPE_BIT_NUMBER, + "integer" => TYPE_BIT_INTEGER, + "string" => TYPE_BIT_STRING, + _ => return None, + }) +} + +pub(super) fn data_type_that_type_specific_keyword_applies_to( + keyword: &str, +) -> Option<&'static str> { + Some(match keyword { + "maximum" | "minimum" | "exclusiveMaximum" | "exclusiveMinimum" | "multipleOf" => "number", + "maxLength" | "minLength" | "pattern" => "string", + "maxProperties" + | "minProperties" + | "required" + | "properties" + | "patternProperties" + | "additionalProperties" + | "dependencies" + | "propertyNames" => "object", + "maxItems" | "minItems" | "uniqueItems" | "items" | "contains" => "array", + _ => return None, + }) +} + +pub(super) fn is_vendor_extension_keyword(key: &str) -> bool { + let Some(rest) = key.strip_prefix("x-") else { + return false; + }; + let mut segment_count = 0; + for segment in rest.split('-') { + let segment_is_valid = !segment.is_empty() + && segment + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit()); + if !segment_is_valid { + return false; + } + segment_count += 1; + } + segment_count >= 2 +} + +const OBJECT_PROTOTYPE_MEMBER_NAMES: [&str; 12] = [ + "constructor", + "__defineGetter__", + "__defineSetter__", + "hasOwnProperty", + "__lookupGetter__", + "__lookupSetter__", + "isPrototypeOf", + "propertyIsEnumerable", + "toString", + "valueOf", + "__proto__", + "toLocaleString", +]; + +pub(super) fn is_object_prototype_member_name(name: &str) -> bool { + OBJECT_PROTOTYPE_MEMBER_NAMES.contains(&name) +} + +pub(super) fn type_list_includes_type_like_ajv(types: &[&str], type_name: &str) -> bool { + types.contains(&type_name) || (type_name == "integer" && types.contains(&"number")) +} + +pub(super) fn type_list_has_type_applicable_to_keyword(types: &[&str], keyword_type: &str) -> bool { + types.contains(&keyword_type) || (keyword_type == "number" && types.contains(&"integer")) +} diff --git a/apps/heft-native/src/schema/mod.rs b/apps/heft-native/src/schema/mod.rs new file mode 100644 index 0000000000..18e4853c11 --- /dev/null +++ b/apps/heft-native/src/schema/mod.rs @@ -0,0 +1,18 @@ +mod compile_applicator_keywords; +mod compile_node; +mod compile_object_keywords; +mod compile_value_keywords; +mod compiled_node; +mod compiler; +mod data_rules; +mod keyword_tables; +mod schema_keywords; +mod validator; +mod validator_objects; + +pub use compiler::{compile_json_schema_for_fast_validation, CompiledJsonSchema}; + +#[cfg(test)] +mod tests_allocation_counts; +#[cfg(test)] +mod tests_schema_soundness; diff --git a/apps/heft-native/src/schema/schema_keywords.rs b/apps/heft-native/src/schema/schema_keywords.rs new file mode 100644 index 0000000000..6762f7e9b7 --- /dev/null +++ b/apps/heft-native/src/schema/schema_keywords.rs @@ -0,0 +1,156 @@ +use crate::json::{JsonObject, JsonValue}; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum Keyword { + Type, + Enum, + Const, + Properties, + PatternProperties, + AdditionalProperties, + Required, + Items, + MinItems, + MaxItems, + UniqueItems, + Contains, + MinLength, + MaxLength, + Pattern, + Minimum, + Maximum, + ExclusiveMinimum, + ExclusiveMaximum, + MultipleOf, + MinProperties, + MaxProperties, + Dependencies, + PropertyNames, + AllOf, + AnyOf, + OneOf, + Not, + If, + Then, + Else, + Reference, + Definitions, + DollarDefinitions, + Comment, + Schema, + Title, + Description, + Default, + Examples, + ReadOnly, + WriteOnly, + Deprecated, + ContentMediaType, + ContentEncoding, + Identifier, + DollarIdentifier, +} + +const KEYWORD_COUNT: usize = Keyword::DollarIdentifier as usize + 1; + +fn keyword_for_key(key: &str) -> Option { + Some(match key { + "type" => Keyword::Type, + "enum" => Keyword::Enum, + "const" => Keyword::Const, + "properties" => Keyword::Properties, + "patternProperties" => Keyword::PatternProperties, + "additionalProperties" => Keyword::AdditionalProperties, + "required" => Keyword::Required, + "items" => Keyword::Items, + "minItems" => Keyword::MinItems, + "maxItems" => Keyword::MaxItems, + "uniqueItems" => Keyword::UniqueItems, + "contains" => Keyword::Contains, + "minLength" => Keyword::MinLength, + "maxLength" => Keyword::MaxLength, + "pattern" => Keyword::Pattern, + "minimum" => Keyword::Minimum, + "maximum" => Keyword::Maximum, + "exclusiveMinimum" => Keyword::ExclusiveMinimum, + "exclusiveMaximum" => Keyword::ExclusiveMaximum, + "multipleOf" => Keyword::MultipleOf, + "minProperties" => Keyword::MinProperties, + "maxProperties" => Keyword::MaxProperties, + "dependencies" => Keyword::Dependencies, + "propertyNames" => Keyword::PropertyNames, + "allOf" => Keyword::AllOf, + "anyOf" => Keyword::AnyOf, + "oneOf" => Keyword::OneOf, + "not" => Keyword::Not, + "if" => Keyword::If, + "then" => Keyword::Then, + "else" => Keyword::Else, + "$ref" => Keyword::Reference, + "definitions" => Keyword::Definitions, + "$defs" => Keyword::DollarDefinitions, + "$comment" => Keyword::Comment, + "$schema" => Keyword::Schema, + "title" => Keyword::Title, + "description" => Keyword::Description, + "default" => Keyword::Default, + "examples" => Keyword::Examples, + "readOnly" => Keyword::ReadOnly, + "writeOnly" => Keyword::WriteOnly, + "deprecated" => Keyword::Deprecated, + "contentMediaType" => Keyword::ContentMediaType, + "contentEncoding" => Keyword::ContentEncoding, + "id" => Keyword::Identifier, + "$id" => Keyword::DollarIdentifier, + _ => return None, + }) +} + +fn keyword_is_known_to_draft_04(keyword: Keyword) -> bool { + !matches!( + keyword, + Keyword::Examples + | Keyword::ReadOnly + | Keyword::WriteOnly + | Keyword::Deprecated + | Keyword::ContentMediaType + | Keyword::ContentEncoding + ) +} + +pub(super) struct SchemaKeywordValues<'schema> { + values: [Option<&'schema JsonValue<'schema>>; KEYWORD_COUNT], +} + +impl<'schema> SchemaKeywordValues<'schema> { + pub(super) fn gather_known_to_ajv( + object: &'schema JsonObject<'schema>, + is_draft_04: bool, + is_root: bool, + vendor_keywords: &[&str], + ) -> Option { + let mut gathered = SchemaKeywordValues { + values: [None; KEYWORD_COUNT], + }; + for (key, value) in object.entries() { + match keyword_for_key(key) { + Some(Keyword::Identifier | Keyword::DollarIdentifier) if !is_root => return None, + Some(keyword) if is_draft_04 && !keyword_is_known_to_draft_04(keyword) => { + return None + } + Some(keyword) => gathered.values[keyword as usize] = Some(value), + None if vendor_keywords.contains(&key.as_ref()) => {} + None => return None, + } + } + Some(gathered) + } + + pub(super) fn value(&self, keyword: Keyword) -> Option<&'schema JsonValue<'schema>> { + self.values[keyword as usize] + } + + pub(super) fn has(&self, keyword: Keyword) -> bool { + self.values[keyword as usize].is_some() + } +} diff --git a/apps/heft-native/src/schema/tests_allocation_counts.rs b/apps/heft-native/src/schema/tests_allocation_counts.rs new file mode 100644 index 0000000000..9b34e53c9b --- /dev/null +++ b/apps/heft-native/src/schema/tests_allocation_counts.rs @@ -0,0 +1,50 @@ +use crate::json::parse_json_with_comments_exactly_like_jju; +use crate::sys::allocation_counter::count_allocations_on_this_thread_while_running; + +use super::compile_json_schema_for_fast_validation; + +#[test] +fn validation_without_patterns_does_not_allocate() { + let schema_document = parse_json_with_comments_exactly_like_jju( + "{\"type\": \"object\", \"required\": [\"a\"], \"additionalProperties\": false, \"properties\": {\"a\": {\"type\": \"array\", \"items\": {\"enum\": [1, 2, \"three\"]}}, \"b\": {\"type\": \"string\", \"minLength\": 1}}}", + ) + .unwrap(); + let compiled_schema = compile_json_schema_for_fast_validation(&schema_document).unwrap(); + let data = + parse_json_with_comments_exactly_like_jju("{\"a\": [1, 2, \"three\"], \"b\": \"text\"}") + .unwrap(); + let (is_valid, allocations) = count_allocations_on_this_thread_while_running(|| { + compiled_schema.is_definitely_valid(&data) + }); + assert!(is_valid); + assert_eq!(allocations, 0); +} + +fn allocations_of_one_pattern_test(pattern_schema: &str, text: &str) -> usize { + let schema_document = parse_json_with_comments_exactly_like_jju(pattern_schema).unwrap(); + let compiled_schema = compile_json_schema_for_fast_validation(&schema_document).unwrap(); + let data = parse_json_with_comments_exactly_like_jju(text).unwrap(); + let (is_valid, allocations) = count_allocations_on_this_thread_while_running(|| { + compiled_schema.is_definitely_valid(&data) + }); + assert!(is_valid); + allocations +} + +#[test] +fn pattern_tests_of_small_programs_do_not_allocate() { + let allocations = allocations_of_one_pattern_test( + "{\"type\": \"string\", \"pattern\": \"^[a-z][a-z0-9]*([-][a-z0-9]+)*$\"}", + "\"copy-json-schemas\"", + ); + assert_eq!(allocations, 0); +} + +#[test] +fn pattern_tests_of_large_programs_cost_three_allocations() { + let allocations = allocations_of_one_pattern_test( + "{\"type\": \"string\", \"pattern\": \"^a{70}\"}", + "\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"", + ); + assert_eq!(allocations, 3); +} diff --git a/apps/heft-native/src/schema/tests_schema_soundness.rs b/apps/heft-native/src/schema/tests_schema_soundness.rs new file mode 100644 index 0000000000..2b4064f045 --- /dev/null +++ b/apps/heft-native/src/schema/tests_schema_soundness.rs @@ -0,0 +1,148 @@ +use crate::json::parse_json_with_comments_exactly_like_jju; + +use super::compile_json_schema_for_fast_validation; + +fn verdicts_for(schema_text: &str, data_texts: &[&str]) -> Option> { + let schema_document = parse_json_with_comments_exactly_like_jju(schema_text).unwrap(); + let compiled_schema = compile_json_schema_for_fast_validation(&schema_document)?; + let verdicts = data_texts + .iter() + .map(|data_text| { + let data = parse_json_with_comments_exactly_like_jju(data_text).unwrap(); + compiled_schema.is_definitely_valid(&data) + }) + .collect(); + Some(verdicts) +} + +const HEFT_LIKE_SCHEMA: &str = r##"{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": false, + "definitions": { + "plugin": { + "type": "object", + "required": ["pluginPackage"], + "additionalProperties": false, + "properties": { "pluginPackage": { "type": "string", "pattern": "[^\\\\]" }, "options": { "type": "object" } } + } + }, + "properties": { + "$schema": { "type": "string" }, + "heftPlugins": { "type": "array", "items": { "$ref": "#/definitions/plugin" } }, + "phasesByName": { + "type": "object", + "additionalProperties": false, + "patternProperties": { "^[a-z][a-z0-9]*([-][a-z0-9]+)*$": { "type": "object" } } + } + } +}"##; + +#[test] +fn heft_like_schema_accepts_valid_and_rejects_invalid_data() { + let verdicts = verdicts_for( + HEFT_LIKE_SCHEMA, + &[ + r#"{"heftPlugins": [{"pluginPackage": "a"}], "phasesByName": {"build": {}}}"#, + r#"{"heftPlugins": [{"pluginPackage": "a", "extra": 1}]}"#, + r#"{"heftPlugins": [{}]}"#, + r#"{"phasesByName": {"Build": {}}}"#, + r#"{"unknown": true}"#, + r#"{"heftPlugins": [{"pluginPackage": "\\"}]}"#, + ], + ); + assert_eq!( + verdicts, + Some(vec![true, false, false, false, false, false]) + ); +} + +#[test] +fn refuses_schemas_that_ajv_strict_mode_would_reject_or_warn_about() { + for schema_text in [ + r#"{"foo": 1}"#, + r#"{"properties": {}}"#, + r#"{"$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "required": []}"#, + r#"{"type": "string", "format": "uri"}"#, + r#"{"type": "array", "items": [{}]}"#, + r#"{"if": {}}"#, + r#"{"then": {}}"#, + r#"{"type": "object", "properties": {"a": {}}, "patternProperties": {"^a": {}}}"#, + r#"{"type": "string", "pattern": "("}"#, + r##"{"$ref": "#/definitions/missing"}"##, + r#"{"type": "object", "properties": {"constructor": {}}}"#, + r#"{"enum": [1, 1]}"#, + r#"{"$schema": "http://json-schema.org/draft-06/schema#"}"#, + r#"{"$id": "http://example.com/schema"}"#, + r#"{"$schema": "http://json-schema.org/draft-04/schema#", "examples": []}"#, + r#"{"$schema": "http://json-schema.org/draft-04/schema#", "type": "number", "exclusiveMinimum": true}"#, + r#"{"type": "string", "title": 5}"#, + r#"{"type": "string", "minLength": 1.5}"#, + ] { + assert!( + verdicts_for(schema_text, &[]).is_none(), + "expected {} to be refused", + schema_text + ); + } +} + +#[test] +fn context_types_flow_into_in_place_applicators_like_ajv() { + let verdicts = verdicts_for( + r#"{"type": "object", "anyOf": [{"required": ["a"]}, {"required": ["b"]}]}"#, + &[r#"{"a": 1}"#, r#"{"b": 1}"#, r#"{"c": 1}"#], + ); + assert_eq!(verdicts, Some(vec![true, true, false])); +} + +#[test] +fn data_that_ajv_would_see_differently_is_never_definitely_valid() { + let verdicts = verdicts_for( + r#"{"type": ["object", "array"]}"#, + &[r#"{"toString": 1}"#, r#"[1e400]"#, r#"{"a": [1]}"#], + ); + assert_eq!(verdicts, Some(vec![false, false, true])); +} + +#[test] +fn draft_07_numbers_strings_and_multiple_of_follow_javascript_semantics() { + let verdicts = verdicts_for( + r#"{"type": "number", "exclusiveMinimum": 0, "maximum": 10, "multipleOf": 0.5}"#, + &["0", "0.5", "10", "10.5", "0.3"], + ); + assert_eq!(verdicts, Some(vec![false, true, true, false, false])); + let lengths = verdicts_for( + r#"{"type": "string", "minLength": 2, "maxLength": 2}"#, + &[r#""\ud83d\ude00a""#, r#""a""#], + ); + assert_eq!(lengths, Some(vec![true, false])); +} + +#[test] +fn every_all_of_member_still_applies_after_inline_members_move_before_references() { + let schema = r##"{ + "type": "object", + "definitions": { + "hasName": { "type": "object", "required": ["name"] }, + "hasKind": { "type": "object", "required": ["kind"] } + }, + "allOf": [ + { "$ref": "#/definitions/hasName" }, + { "properties": { "kind": { "enum": ["flag", "string"] } } }, + { "$ref": "#/definitions/hasKind" }, + { "properties": { "name": { "type": "string", "pattern": "^-(-[a-z0-9]+)+$" } } } + ] + }"##; + let verdicts = verdicts_for( + schema, + &[ + r#"{"name": "--verbose", "kind": "flag"}"#, + r#"{"kind": "flag"}"#, + r#"{"name": "--verbose", "kind": "choice"}"#, + r#"{"name": "--verbose"}"#, + r#"{"name": "verbose", "kind": "flag"}"#, + ], + ); + assert_eq!(verdicts, Some(vec![true, false, false, false, false])); +} diff --git a/apps/heft-native/src/schema/validator.rs b/apps/heft-native/src/schema/validator.rs new file mode 100644 index 0000000000..d281a1ef55 --- /dev/null +++ b/apps/heft-native/src/schema/validator.rs @@ -0,0 +1,200 @@ +use crate::json::JsonValue; + +use super::compiled_node::{ + CompiledSchemaNode, NodeIdentifier, NodeKind, UncommonKeywordConstraints, +}; +use super::compiler::CompiledJsonSchema; +use super::data_rules::{ + is_simple_json_data, javascript_parse_int_of_number_to_string, json_schema_type_bits_of_data, + json_values_are_deeply_equal, +}; +use super::keyword_tables::TYPE_BIT_NUMBER; + +impl CompiledJsonSchema<'_> { + pub fn is_definitely_valid(&self, data: &JsonValue<'_>) -> bool { + is_simple_json_data(data, 0) && self.node_accepts(self.root, data) + } + + pub(super) fn node_accepts(&self, identifier: NodeIdentifier, data: &JsonValue<'_>) -> bool { + let node = &self.nodes[identifier as usize]; + match node.kind { + NodeKind::AlwaysValid => return true, + NodeKind::AlwaysInvalid => return false, + NodeKind::Regular => {} + } + let data_type_bits = json_schema_type_bits_of_data(data); + if node.allowed_type_bits != 0 && node.allowed_type_bits & data_type_bits == 0 { + return false; + } + let enum_accepts = node.enum_values.is_none_or(|allowed_values| { + allowed_values + .iter() + .any(|allowed_value| json_values_are_deeply_equal(allowed_value, data)) + }); + let uncommon = node.uncommon.as_deref(); + let const_accepts = uncommon + .and_then(|uncommon| uncommon.const_value) + .is_none_or(|expected_value| json_values_are_deeply_equal(expected_value, data)); + if !enum_accepts || !const_accepts { + return false; + } + let type_specific_keywords_accept = match data { + JsonValue::Number(number) if data_type_bits & TYPE_BIT_NUMBER != 0 => { + uncommon.is_none_or(|uncommon| number_keywords_accept(uncommon, number.value)) + } + JsonValue::String(text) => self.string_keywords_accept(node, uncommon, text), + JsonValue::Array(items) => self.array_keywords_accept(node, uncommon, items), + JsonValue::Object(object) => self.object_keywords_accept(node, uncommon, object, data), + _ => true, + }; + type_specific_keywords_accept && self.applicator_keywords_accept(node, uncommon, data) + } + + fn string_keywords_accept( + &self, + node: &CompiledSchemaNode<'_>, + uncommon: Option<&UncommonKeywordConstraints<'_>>, + text: &str, + ) -> bool { + if let Some(uncommon) = uncommon + .filter(|uncommon| uncommon.min_length.is_some() || uncommon.max_length.is_some()) + { + let code_point_count = text.chars().count() as u64; + let length_is_accepted = uncommon + .min_length + .is_none_or(|minimum| code_point_count >= minimum) + && uncommon + .max_length + .is_none_or(|maximum| code_point_count <= maximum); + if !length_is_accepted { + return false; + } + } + node.pattern_regex_index + .is_none_or(|regex_index| self.regexes[regex_index].matches_anywhere(text)) + } + + fn array_keywords_accept( + &self, + node: &CompiledSchemaNode<'_>, + uncommon: Option<&UncommonKeywordConstraints<'_>>, + items: &[JsonValue<'_>], + ) -> bool { + if uncommon.is_some_and(|uncommon| !array_count_and_uniqueness_accept(uncommon, items)) { + return false; + } + let every_item_is_accepted = node.items.is_none_or(|item_schema| { + items + .iter() + .all(|item| self.node_accepts(item_schema, item)) + }); + every_item_is_accepted + && uncommon + .and_then(|uncommon| uncommon.contains) + .is_none_or(|contained_schema| { + items + .iter() + .any(|item| self.node_accepts(contained_schema, item)) + }) + } + + fn applicator_keywords_accept( + &self, + node: &CompiledSchemaNode<'_>, + uncommon: Option<&UncommonKeywordConstraints<'_>>, + data: &JsonValue<'_>, + ) -> bool { + let all_of_accepts = node.all_of.as_ref().is_none_or(|schemas| { + schemas + .iter() + .all(|&schema| self.node_accepts(schema, data)) + }); + all_of_accepts + && uncommon.is_none_or(|uncommon| self.uncommon_applicators_accept(uncommon, data)) + && node + .reference + .is_none_or(|referenced_schema| self.node_accepts(referenced_schema, data)) + } + + fn uncommon_applicators_accept( + &self, + node: &UncommonKeywordConstraints<'_>, + data: &JsonValue<'_>, + ) -> bool { + let any_of_accepts = node.any_of.as_ref().is_none_or(|schemas| { + schemas + .iter() + .any(|&schema| self.node_accepts(schema, data)) + }); + if !any_of_accepts { + return false; + } + if let Some(schemas) = &node.one_of { + let mut accepting_schema_count = 0; + for &schema in schemas { + if self.node_accepts(schema, data) { + accepting_schema_count += 1; + if accepting_schema_count > 1 { + return false; + } + } + } + if accepting_schema_count != 1 { + return false; + } + } + if node + .not + .is_some_and(|negated_schema| self.node_accepts(negated_schema, data)) + { + return false; + } + if let Some(condition) = node.if_schema { + let branch = if self.node_accepts(condition, data) { + node.then_schema + } else { + node.else_schema + }; + if branch.is_some_and(|branch_schema| !self.node_accepts(branch_schema, data)) { + return false; + } + } + true + } +} + +fn array_count_and_uniqueness_accept( + uncommon: &UncommonKeywordConstraints<'_>, + items: &[JsonValue<'_>], +) -> bool { + let item_count = items.len() as u64; + let count_is_accepted = uncommon + .min_items + .is_none_or(|minimum| item_count >= minimum) + && uncommon + .max_items + .is_none_or(|maximum| item_count <= maximum); + count_is_accepted + && !(uncommon.unique_items + && (1..items.len()).any(|index| { + items[..index] + .iter() + .any(|earlier| json_values_are_deeply_equal(earlier, &items[index])) + })) +} + +fn number_keywords_accept(node: &UncommonKeywordConstraints<'_>, number: f64) -> bool { + let bounds_accept = node.minimum.is_none_or(|minimum| number >= minimum) + && node.maximum.is_none_or(|maximum| number <= maximum) + && node + .exclusive_minimum + .is_none_or(|minimum| number > minimum) + && node + .exclusive_maximum + .is_none_or(|maximum| number < maximum); + bounds_accept + && node.multiple_of.is_none_or(|divisor| { + let quotient = number / divisor; + quotient == javascript_parse_int_of_number_to_string(quotient) + }) +} diff --git a/apps/heft-native/src/schema/validator_objects.rs b/apps/heft-native/src/schema/validator_objects.rs new file mode 100644 index 0000000000..c66cb18b4a --- /dev/null +++ b/apps/heft-native/src/schema/validator_objects.rs @@ -0,0 +1,113 @@ +use std::borrow::Cow; + +use crate::json::{JsonObject, JsonValue}; + +use super::compiled_node::{CompiledSchemaNode, DependencyRequirement, UncommonKeywordConstraints}; +use super::compiler::CompiledJsonSchema; + +impl CompiledJsonSchema<'_> { + pub(super) fn object_keywords_accept( + &self, + node: &CompiledSchemaNode<'_>, + uncommon: Option<&UncommonKeywordConstraints<'_>>, + object: &JsonObject<'_>, + object_as_json: &JsonValue<'_>, + ) -> bool { + let required_are_present = node + .required + .as_ref() + .is_none_or(|names| names.iter().all(|name| object.contains_key(name))); + let property_count = object.len() as u64; + let count_is_accepted = uncommon.is_none_or(|uncommon| { + uncommon + .min_properties + .is_none_or(|minimum| property_count >= minimum) + && uncommon + .max_properties + .is_none_or(|maximum| property_count <= maximum) + }); + if !required_are_present || !count_is_accepted { + return false; + } + let declared_properties_accept = node.properties.as_ref().is_none_or(|properties| { + properties.iter().all(|&(name, schema)| { + object + .get(name) + .is_none_or(|property_value| self.node_accepts(schema, property_value)) + }) + }); + declared_properties_accept + && self.undeclared_and_named_properties_accept(node, uncommon, object) + && uncommon + .is_none_or(|uncommon| self.dependencies_accept(uncommon, object, object_as_json)) + } + + fn undeclared_and_named_properties_accept( + &self, + node: &CompiledSchemaNode<'_>, + uncommon: Option<&UncommonKeywordConstraints<'_>>, + object: &JsonObject<'_>, + ) -> bool { + let pattern_properties = + uncommon.and_then(|uncommon| uncommon.pattern_properties.as_deref()); + let property_names = uncommon.and_then(|uncommon| uncommon.property_names); + if pattern_properties.is_none() + && node.additional_properties.is_none() + && property_names.is_none() + { + return true; + } + for (key, property_value) in object.entries() { + if let Some(property_names_schema) = property_names { + let key_as_json = JsonValue::String(Cow::Borrowed(key.as_ref())); + if !self.node_accepts(property_names_schema, &key_as_json) { + return false; + } + } + let mut is_additional = node + .properties + .as_ref() + .is_none_or(|properties| !properties.iter().any(|&(name, _)| name == key.as_ref())); + for &(regex_index, schema) in pattern_properties.unwrap_or_default() { + if self.regexes[regex_index].matches_anywhere(key) { + is_additional = false; + if !self.node_accepts(schema, property_value) { + return false; + } + } + } + let additional_is_rejected = is_additional + && node.additional_properties.is_some_and(|additional_schema| { + !self.node_accepts(additional_schema, property_value) + }); + if additional_is_rejected { + return false; + } + } + true + } + + fn dependencies_accept( + &self, + node: &UncommonKeywordConstraints<'_>, + object: &JsonObject<'_>, + object_as_json: &JsonValue<'_>, + ) -> bool { + let Some(dependencies) = &node.dependencies else { + return true; + }; + dependencies.iter().all(|(name, requirement)| { + if !object.contains_key(name) { + return true; + } + match requirement { + DependencyRequirement::RequiredPropertyNames(names) => { + names.iter().all(|required| object.contains_key(required)) + } + DependencyRequirement::Subschema(schema) => { + self.node_accepts(*schema, object_as_json) + } + } + }) + } +} diff --git a/apps/heft-native/src/sys/allocation_counter.rs b/apps/heft-native/src/sys/allocation_counter.rs new file mode 100644 index 0000000000..4e650f2b91 --- /dev/null +++ b/apps/heft-native/src/sys/allocation_counter.rs @@ -0,0 +1,94 @@ +#![allow(unsafe_code)] + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::Cell; + +pub struct AllocationCountingSystemAllocator; + +thread_local! { + static ALLOCATIONS_ON_THIS_THREAD: Cell = const { Cell::new(0) }; + static ALLOCATED_BYTES_ON_THIS_THREAD: Cell = const { Cell::new(0) }; + static LIVE_BYTES_ON_THIS_THREAD: Cell = const { Cell::new(0) }; + static PEAK_LIVE_BYTES_ON_THIS_THREAD: Cell = const { Cell::new(0) }; +} + +fn record_one_allocation_on_this_thread(allocated_byte_count: usize) { + let _ = ALLOCATIONS_ON_THIS_THREAD.try_with(|count| count.set(count.get() + 1)); + let _ = ALLOCATED_BYTES_ON_THIS_THREAD.try_with(|bytes| bytes.set(bytes.get() + allocated_byte_count)); + change_live_bytes_on_this_thread(allocated_byte_count as isize); +} + +fn change_live_bytes_on_this_thread(byte_delta: isize) { + let live_bytes_now = LIVE_BYTES_ON_THIS_THREAD + .try_with(|live_bytes| { + live_bytes.set(live_bytes.get() + byte_delta); + live_bytes.get() + }) + .unwrap_or(0); + let _ = PEAK_LIVE_BYTES_ON_THIS_THREAD.try_with(|peak_live_bytes| { + if live_bytes_now > peak_live_bytes.get() { + peak_live_bytes.set(live_bytes_now); + } + }); +} + +unsafe impl GlobalAlloc for AllocationCountingSystemAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + record_one_allocation_on_this_thread(layout.size()); + System.alloc(layout) + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + record_one_allocation_on_this_thread(layout.size()); + System.alloc_zeroed(layout) + } + + unsafe fn realloc(&self, pointer: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + record_one_allocation_on_this_thread(new_size); + change_live_bytes_on_this_thread(-(layout.size() as isize)); + System.realloc(pointer, layout, new_size) + } + + unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) { + change_live_bytes_on_this_thread(-(layout.size() as isize)); + System.dealloc(pointer, layout) + } +} + +#[global_allocator] +static ALLOCATION_COUNTING_SYSTEM_ALLOCATOR: AllocationCountingSystemAllocator = + AllocationCountingSystemAllocator; + +pub fn count_allocations_on_this_thread_while_running( + work: impl FnOnce() -> Output, +) -> (Output, usize) { + let allocations_before = ALLOCATIONS_ON_THIS_THREAD.with(Cell::get); + let output = work(); + let allocations_after = ALLOCATIONS_ON_THIS_THREAD.with(Cell::get); + (output, allocations_after - allocations_before) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AllocationsMadeOnThisThread { + pub allocation_count: usize, + pub allocated_byte_count: usize, + pub peak_live_bytes_above_start: isize, + pub live_bytes_left_above_start: isize, +} + +pub fn measure_allocations_on_this_thread_while_running( + work: impl FnOnce() -> Output, +) -> (Output, AllocationsMadeOnThisThread) { + let allocations_before = ALLOCATIONS_ON_THIS_THREAD.with(Cell::get); + let allocated_bytes_before = ALLOCATED_BYTES_ON_THIS_THREAD.with(Cell::get); + let live_bytes_before = LIVE_BYTES_ON_THIS_THREAD.with(Cell::get); + PEAK_LIVE_BYTES_ON_THIS_THREAD.with(|peak_live_bytes| peak_live_bytes.set(live_bytes_before)); + let output = work(); + let allocations_made = AllocationsMadeOnThisThread { + allocation_count: ALLOCATIONS_ON_THIS_THREAD.with(Cell::get) - allocations_before, + allocated_byte_count: ALLOCATED_BYTES_ON_THIS_THREAD.with(Cell::get) - allocated_bytes_before, + peak_live_bytes_above_start: PEAK_LIVE_BYTES_ON_THIS_THREAD.with(Cell::get) - live_bytes_before, + live_bytes_left_above_start: LIVE_BYTES_ON_THIS_THREAD.with(Cell::get) - live_bytes_before, + }; + (output, allocations_made) +} diff --git a/apps/heft-native/src/sys/file_descriptor_flags.rs b/apps/heft-native/src/sys/file_descriptor_flags.rs new file mode 100644 index 0000000000..ab3c9a8c0c --- /dev/null +++ b/apps/heft-native/src/sys/file_descriptor_flags.rs @@ -0,0 +1,23 @@ +use std::ffi::c_int; +use std::fs::File; +use std::os::fd::AsRawFd; + +const F_GETFD: c_int = 1; +const F_SETFD: c_int = 2; +const FD_CLOEXEC: c_int = 1; + +extern "C" { + fn fcntl(file_descriptor: c_int, command: c_int, ...) -> c_int; +} + +pub fn let_executed_program_inherit_file(file: &File) -> std::io::Result { + let file_descriptor = file.as_raw_fd(); + let descriptor_flags = unsafe { fcntl(file_descriptor, F_GETFD) }; + if descriptor_flags < 0 { + return Err(std::io::Error::last_os_error()); + } + if unsafe { fcntl(file_descriptor, F_SETFD, descriptor_flags & !FD_CLOEXEC) } < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(file_descriptor) +} diff --git a/apps/heft-native/src/sys/mod.rs b/apps/heft-native/src/sys/mod.rs new file mode 100644 index 0000000000..d372bb908a --- /dev/null +++ b/apps/heft-native/src/sys/mod.rs @@ -0,0 +1,24 @@ +#[cfg(test)] +pub mod allocation_counter; +#[cfg(test)] +mod tests_allocation_regressions; + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +mod file_descriptor_flags; +#[cfg(unix)] +#[allow(unsafe_code)] +mod signal_dispositions; +#[cfg(unix)] +#[allow(unsafe_code)] +mod signals_and_identity; + +#[cfg(target_os = "linux")] +pub use file_descriptor_flags::let_executed_program_inherit_file; +#[cfg(unix)] +pub use signal_dispositions::reset_inherited_ignored_signals_like_node; +#[cfg(unix)] +pub use signals_and_identity::{ + effective_user_id, forward_interrupt_and_termination_signals_to_warm_host, + signal_forwarded_to_warm_host, terminate_by_signal, +}; diff --git a/apps/heft-native/src/sys/signal_dispositions.rs b/apps/heft-native/src/sys/signal_dispositions.rs new file mode 100644 index 0000000000..a6c2c128ff --- /dev/null +++ b/apps/heft-native/src/sys/signal_dispositions.rs @@ -0,0 +1,61 @@ +use std::ffi::c_int; + +const SIGNAL_BROKEN_PIPE: c_int = 13; +const SIGNAL_FILE_SIZE_LIMIT_EXCEEDED: c_int = 25; +const HIGHEST_STANDARD_SIGNAL: c_int = 31; +const DEFAULT_SIGNAL_DISPOSITION: usize = 0; +const IGNORED_SIGNAL_DISPOSITION: usize = 1; +const FAILED_SIGNAL_DISPOSITION_CHANGE: usize = usize::MAX; + +extern "C" { + fn signal(signal_number: c_int, handler: usize) -> usize; +} + +pub fn reset_inherited_ignored_signals_like_node() { + for signal_number in 1..=HIGHEST_STANDARD_SIGNAL { + if signal_number == SIGNAL_BROKEN_PIPE { + continue; + } + if signal_number == SIGNAL_FILE_SIZE_LIMIT_EXCEEDED { + unsafe { signal(signal_number, IGNORED_SIGNAL_DISPOSITION) }; + continue; + } + let previous_disposition = unsafe { signal(signal_number, DEFAULT_SIGNAL_DISPOSITION) }; + let previous_disposition_was_a_handler = previous_disposition != IGNORED_SIGNAL_DISPOSITION + && previous_disposition != DEFAULT_SIGNAL_DISPOSITION + && previous_disposition != FAILED_SIGNAL_DISPOSITION_CHANGE; + if previous_disposition_was_a_handler { + unsafe { signal(signal_number, previous_disposition) }; + } + } +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + + const SIGNAL_USER_DEFINED_2: c_int = 12; + + fn ignored_signal_mask_of_this_process() -> u64 { + let process_status = std::fs::read_to_string("/proc/self/status").unwrap(); + let ignored_mask_text = process_status + .lines() + .find_map(|status_line| status_line.strip_prefix("SigIgn:")) + .unwrap(); + u64::from_str_radix(ignored_mask_text.trim(), 16).unwrap() + } + + fn is_ignored(signal_number: c_int) -> bool { + ignored_signal_mask_of_this_process() & (1u64 << (signal_number - 1)) != 0 + } + + #[test] + fn inherited_ignored_signals_are_reset_except_broken_pipe_and_file_size_limit() { + unsafe { signal(SIGNAL_USER_DEFINED_2, IGNORED_SIGNAL_DISPOSITION) }; + assert!(is_ignored(SIGNAL_USER_DEFINED_2)); + reset_inherited_ignored_signals_like_node(); + assert!(!is_ignored(SIGNAL_USER_DEFINED_2)); + assert!(is_ignored(SIGNAL_BROKEN_PIPE)); + assert!(is_ignored(SIGNAL_FILE_SIZE_LIMIT_EXCEEDED)); + } +} diff --git a/apps/heft-native/src/sys/signals_and_identity.rs b/apps/heft-native/src/sys/signals_and_identity.rs new file mode 100644 index 0000000000..85ac516e5f --- /dev/null +++ b/apps/heft-native/src/sys/signals_and_identity.rs @@ -0,0 +1,62 @@ +use std::ffi::c_int; +use std::os::fd::AsRawFd; +use std::os::unix::net::UnixStream; +use std::sync::atomic::{AtomicI32, Ordering}; + +const SIGNAL_INTERRUPT: c_int = 2; +const SIGNAL_TERMINATE: c_int = 15; +const DEFAULT_SIGNAL_DISPOSITION: usize = 0; +const WARM_HOST_SIGNAL_FRAME_TYPE: u8 = 0x02; +const EXIT_CODE_AFTER_SIGNAL_OFFSET: i32 = 128; + +static WARM_HOST_SOCKET_FILE_DESCRIPTOR: AtomicI32 = AtomicI32::new(-1); +static SIGNAL_FORWARDED_TO_WARM_HOST: AtomicI32 = AtomicI32::new(0); + +extern "C" { + fn geteuid() -> u32; + fn signal(signal_number: c_int, handler: usize) -> usize; + fn raise(signal_number: c_int) -> c_int; + fn write(file_descriptor: c_int, buffer: *const u8, byte_count: usize) -> isize; +} + +pub fn effective_user_id() -> u32 { + unsafe { geteuid() } +} + +extern "C" fn forward_signal_to_warm_host(signal_number: c_int) { + SIGNAL_FORWARDED_TO_WARM_HOST.store(signal_number, Ordering::SeqCst); + let socket_file_descriptor = WARM_HOST_SOCKET_FILE_DESCRIPTOR.load(Ordering::SeqCst); + if socket_file_descriptor >= 0 { + let signal_frame: [u8; 6] = [1, 0, 0, 0, WARM_HOST_SIGNAL_FRAME_TYPE, signal_number as u8]; + unsafe { + write( + socket_file_descriptor, + signal_frame.as_ptr(), + signal_frame.len(), + ) + }; + } +} + +pub fn forward_interrupt_and_termination_signals_to_warm_host(warm_host_socket: &UnixStream) { + WARM_HOST_SOCKET_FILE_DESCRIPTOR.store(warm_host_socket.as_raw_fd(), Ordering::SeqCst); + let handler = forward_signal_to_warm_host as extern "C" fn(c_int) as usize; + for signal_number in [SIGNAL_INTERRUPT, SIGNAL_TERMINATE] { + unsafe { signal(signal_number, handler) }; + } +} + +pub fn signal_forwarded_to_warm_host() -> Option { + match SIGNAL_FORWARDED_TO_WARM_HOST.load(Ordering::SeqCst) { + 0 => None, + signal_number => Some(signal_number), + } +} + +pub fn terminate_by_signal(signal_number: i32) -> ! { + unsafe { + signal(signal_number, DEFAULT_SIGNAL_DISPOSITION); + raise(signal_number); + } + std::process::exit(EXIT_CODE_AFTER_SIGNAL_OFFSET + signal_number) +} diff --git a/apps/heft-native/src/sys/tests_allocation_regressions.rs b/apps/heft-native/src/sys/tests_allocation_regressions.rs new file mode 100644 index 0000000000..e4c287f981 --- /dev/null +++ b/apps/heft-native/src/sys/tests_allocation_regressions.rs @@ -0,0 +1,60 @@ +use crate::json::parse_json_with_comments_exactly_like_jju; +use crate::sys::allocation_counter::measure_allocations_on_this_thread_while_running; + +#[test] +fn measures_allocated_bytes_peak_and_bytes_left_alive() { + let (kept_bytes, allocations_made) = measure_allocations_on_this_thread_while_running(|| { + let temporary_buffer: Vec = std::hint::black_box(vec![1u8; 4096]); + let mut kept_bytes: Vec = Vec::with_capacity(64); + kept_bytes.extend_from_slice(&temporary_buffer[..64]); + kept_bytes + }); + assert_eq!(kept_bytes.len(), 64); + assert_eq!(allocations_made.allocation_count, 2); + assert_eq!(allocations_made.allocated_byte_count, 4096 + 64); + assert_eq!(allocations_made.peak_live_bytes_above_start, 4096 + 64); + assert_eq!(allocations_made.live_bytes_left_above_start, 64); +} + +#[test] +fn ignores_allocations_made_by_other_threads() { + let (_, allocations_made) = measure_allocations_on_this_thread_while_running(|| { + std::thread::scope(|scope| { + scope.spawn(|| std::hint::black_box(vec![1u8; 1 << 20]).len()).join().unwrap_or(0) + }) + }); + assert!(allocations_made.allocated_byte_count < 1 << 20); +} + +fn heft_json_with_escape_free_strings_of_length(string_length: usize) -> String { + let long_text: String = "x".repeat(string_length); + format!( + "{{\"$schema\": \"https://developer.microsoft.com/json-schemas/heft/v0/heft.schema.json\", \"phasesByName\": {{\"build\": {{\"phaseDescription\": \"{long_text}\", \"cleanFiles\": [{{\"includeGlobs\": [\"lib-{long_text}\"]}}], \"tasksByName\": {{\"copy\": {{\"taskPlugin\": {{\"pluginPackage\": \"@rushstack/heft\", \"pluginName\": \"copy-files-plugin\", \"options\": {{\"copyOperations\": [{{\"sourcePath\": \"src/{long_text}\", \"destinationFolders\": [\"lib\"]}}]}}}}}}}}}}}}}}" + ) +} + +#[test] +fn parsed_json_memory_does_not_grow_with_the_length_of_escape_free_strings() { + let short_document: String = heft_json_with_escape_free_strings_of_length(8); + let long_document: String = heft_json_with_escape_free_strings_of_length(8192); + let (_, short_allocations) = measure_allocations_on_this_thread_while_running(|| { + parse_json_with_comments_exactly_like_jju(&short_document).map(|value| std::hint::black_box(value).clone()) + }); + let (_, long_allocations) = measure_allocations_on_this_thread_while_running(|| { + parse_json_with_comments_exactly_like_jju(&long_document).map(|value| std::hint::black_box(value).clone()) + }); + assert_eq!(short_allocations.allocation_count, long_allocations.allocation_count); + assert_eq!(short_allocations.allocated_byte_count, long_allocations.allocated_byte_count); + assert_eq!(short_allocations.peak_live_bytes_above_start, long_allocations.peak_live_bytes_above_start); +} + +#[test] +fn dropping_a_parsed_document_releases_every_byte() { + let document: String = heft_json_with_escape_free_strings_of_length(32); + let (parse_succeeded, allocations_made) = measure_allocations_on_this_thread_while_running(|| { + parse_json_with_comments_exactly_like_jju(&document).is_ok() + }); + assert!(parse_succeeded); + assert!(allocations_made.allocation_count > 0); + assert_eq!(allocations_made.live_bytes_left_above_start, 0); +} diff --git a/apps/heft-native/src/terminal/ansi_escape_codes.rs b/apps/heft-native/src/terminal/ansi_escape_codes.rs new file mode 100644 index 0000000000..18936df469 --- /dev/null +++ b/apps/heft-native/src/terminal/ansi_escape_codes.rs @@ -0,0 +1,66 @@ +use std::borrow::Cow; + +pub fn bold(text: &str) -> String { + format!("\x1b[1m{text}\x1b[22m") +} + +pub fn red(text: &str) -> String { + format!("\x1b[31m{text}\x1b[39m") +} + +pub fn green(text: &str) -> String { + format!("\x1b[32m{text}\x1b[39m") +} + +pub fn remove_ansi_escape_codes(text: &str) -> Cow<'_, str> { + let bytes = text.as_bytes(); + if !bytes.contains(&0x1b) { + return Cow::Borrowed(text); + } + let mut text_without_codes = String::with_capacity(text.len()); + let mut copied_until = 0; + let mut index = 0; + while index < bytes.len() { + if bytes[index] == 0x1b && bytes.get(index + 1) == Some(&b'[') { + if let Some(sequence_end) = control_sequence_end(bytes, index + 2) { + text_without_codes.push_str(&text[copied_until..index]); + index = sequence_end; + copied_until = sequence_end; + continue; + } + } + index += 1; + } + text_without_codes.push_str(&text[copied_until..]); + Cow::Owned(text_without_codes) +} + +fn control_sequence_end(bytes: &[u8], start: usize) -> Option { + let mut position = start; + while position < bytes.len() && (0x30..=0x3f).contains(&bytes[position]) { + position += 1; + } + while position < bytes.len() && (0x20..=0x2f).contains(&bytes[position]) { + position += 1; + } + match bytes.get(position) { + Some(final_byte) if (0x40..=0x7e).contains(final_byte) => Some(position + 1), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn removes_control_sequences_like_the_terminal_package() { + assert_eq!(remove_ansi_escape_codes("plain"), "plain"); + assert_eq!(remove_ansi_escape_codes(&bold(&green("done"))), "done"); + assert_eq!(remove_ansi_escape_codes("a\x1b[1;31mb\x1b[0m"), "ab"); + assert_eq!(remove_ansi_escape_codes("\x1b[\x1b[31mm"), "\x1b[m"); + assert_eq!(remove_ansi_escape_codes("\x1b[12"), "\x1b[12"); + assert_eq!(remove_ansi_escape_codes("x\x1b[ é"), "x\x1b[ é"); + assert_eq!(remove_ansi_escape_codes("\x1b[?25l\x1b[2 q"), ""); + } +} diff --git a/apps/heft-native/src/terminal/color_support.rs b/apps/heft-native/src/terminal/color_support.rs new file mode 100644 index 0000000000..f90a3d1a78 --- /dev/null +++ b/apps/heft-native/src/terminal/color_support.rs @@ -0,0 +1,174 @@ +use std::io::IsTerminal; + +use super::force_color::{environment_force_color, flag_force_color}; +use super::has_flag::has_flag; +use super::parse_int::parse_int_base_ten_like_javascript; +use super::term_patterns::{ + ends_with_256_color_pattern, matches_basic_term_pattern, matches_teamcity_version_pattern, +}; + +pub struct ColorSupportInputs<'a> { + pub environment_variable: &'a dyn Fn(&str) -> Option, + pub command_line_arguments: &'a [String], + pub standard_output_is_terminal: bool, + pub standard_error_is_terminal: bool, +} + +#[derive(Clone, Copy, Debug)] +enum ColorLevel { + False, + Level, +} + +impl ColorLevel { + fn is_truthy(self) -> bool { + matches!(self, Self::Level) + } +} + +pub fn console_supports_color(inputs: &ColorSupportInputs<'_>) -> bool { + let mut force_color = flag_force_color(inputs.command_line_arguments); + let output = color_support_level_for_stream_with_state( + inputs, + inputs.standard_output_is_terminal, + &mut force_color, + ); + let error = color_support_level_for_stream_with_state( + inputs, + inputs.standard_error_is_terminal, + &mut force_color, + ); + output.is_truthy() && error.is_truthy() +} + +pub fn console_supports_color_for_this_process(tool_arguments: &[String]) -> bool { + let mut process_arguments = vec![String::from("node"), String::from("heft")]; + process_arguments.extend_from_slice(tool_arguments); + let environment_variable = + |name: &str| std::env::var_os(name).map(|value| value.to_string_lossy().into_owned()); + let inputs = ColorSupportInputs { + environment_variable: &environment_variable, + command_line_arguments: &process_arguments, + standard_output_is_terminal: std::io::stdout().is_terminal(), + standard_error_is_terminal: std::io::stderr().is_terminal(), + }; + console_supports_color(&inputs) +} + +fn color_support_level_for_stream_with_state( + inputs: &ColorSupportInputs<'_>, + stream_is_terminal: bool, + flag_force_color_state: &mut Option, +) -> ColorLevel { + if let Some(environment_force_color) = environment_force_color(inputs.environment_variable) { + *flag_force_color_state = Some(environment_force_color); + } + let force_color = *flag_force_color_state; + let level = support_level_number(inputs, stream_is_terminal, force_color); + translate_level(level) +} + +fn support_level_number( + inputs: &ColorSupportInputs<'_>, + stream_is_terminal: bool, + force_color: Option, +) -> f64 { + let environment_variable = inputs.environment_variable; + if force_color == Some(0.0) { + return 0.0; + } + if has_flag("color=16m", inputs.command_line_arguments) + || has_flag("color=full", inputs.command_line_arguments) + || has_flag("color=truecolor", inputs.command_line_arguments) + { + return 3.0; + } + if has_flag("color=256", inputs.command_line_arguments) { + return 2.0; + } + if !stream_is_terminal && force_color.is_none() { + return 0.0; + } + let minimum = javascript_or_zero(force_color); + if environment_variable("TERM").as_deref() == Some("dumb") { + return minimum; + } + if windows_color_level().is_some() { + return windows_color_level().unwrap(); + } + if environment_variable("CI").is_some() { + if [ + "TRAVIS", + "CIRCLECI", + "APPVEYOR", + "GITLAB_CI", + "GITHUB_ACTIONS", + "BUILDKITE", + "DRONE", + ] + .iter() + .any(|name| environment_variable(name).is_some()) + || environment_variable("CI_NAME").as_deref() == Some("codeship") + { + return 1.0; + } + return minimum; + } + if let Some(teamcity_version) = environment_variable("TEAMCITY_VERSION") { + return if matches_teamcity_version_pattern(&teamcity_version) { + 1.0 + } else { + 0.0 + }; + } + if environment_variable("COLORTERM").as_deref() == Some("truecolor") { + return 3.0; + } + if let Some(term_program) = environment_variable("TERM_PROGRAM") { + let version_value = environment_variable("TERM_PROGRAM_VERSION").unwrap_or_default(); + let first_version_part = version_value.split('.').next().unwrap_or_default(); + let version = parse_int_base_ten_like_javascript(first_version_part); + if term_program == "iTerm.app" { + return if version >= 3.0 { 3.0 } else { 2.0 }; + } + if term_program == "Apple_Terminal" { + return 2.0; + } + } + let term = environment_variable("TERM").unwrap_or_else(|| "undefined".to_owned()); + if ends_with_256_color_pattern(&term) { + return 2.0; + } + if matches_basic_term_pattern(&term) { + return 1.0; + } + if environment_variable("COLORTERM").is_some() { + return 1.0; + } + minimum +} + +fn translate_level(level: f64) -> ColorLevel { + if level == 0.0 { + ColorLevel::False + } else { + ColorLevel::Level + } +} + +fn javascript_or_zero(value: Option) -> f64 { + match value { + Some(number) if number != 0.0 && !number.is_nan() => number, + _ => 0.0, + } +} + +#[cfg(windows)] +fn windows_color_level() -> Option { + Some(1.0) +} + +#[cfg(not(windows))] +fn windows_color_level() -> Option { + None +} diff --git a/apps/heft-native/src/terminal/color_support_tests.rs b/apps/heft-native/src/terminal/color_support_tests.rs new file mode 100644 index 0000000000..0eeb9dec1f --- /dev/null +++ b/apps/heft-native/src/terminal/color_support_tests.rs @@ -0,0 +1,57 @@ +use super::color_support::{console_supports_color, ColorSupportInputs}; + +type ColorCase = (&'static [(&'static str, &'static str)], &'static [&'static str], bool, bool, bool); + +const CASES_VERIFIED_WITH_SUPPORTS_COLOR_8_1_1: &[ColorCase] = &[ + (&[], &[], true, true, false), + (&[], &[], false, true, false), + (&[], &[], true, false, false), + (&[("TERM", "xterm-256color")], &[], true, true, true), + (&[("TERM", "dumb")], &[], true, true, false), + (&[("TERM", "dumb"), ("FORCE_COLOR", "1")], &[], true, true, true), + (&[("FORCE_COLOR", "0"), ("TERM", "xterm")], &[], true, true, false), + (&[("FORCE_COLOR", "")], &[], false, false, true), + (&[("FORCE_COLOR", "true")], &[], false, false, true), + (&[("FORCE_COLOR", "false"), ("TERM", "xterm")], &[], true, true, false), + (&[("FORCE_COLOR", "abc")], &[], false, false, false), + (&[("FORCE_COLOR", "3")], &[], false, false, true), + (&[("NO_COLOR", "1"), ("TERM", "xterm")], &[], true, true, true), + (&[("CI", "1"), ("TERM", "xterm")], &[], true, true, false), + (&[("CI", "1"), ("GITHUB_ACTIONS", "true")], &[], true, true, true), + (&[("CI", "1"), ("GITHUB_ACTIONS", "true")], &[], false, true, false), + (&[("TEAMCITY_VERSION", "9.1"), ("TERM", "xterm")], &[], true, true, false), + (&[("TEAMCITY_VERSION", "2023.05"), ("TERM", "xterm")], &[], true, true, true), + (&[("COLORTERM", "truecolor")], &[], true, true, true), + (&[("TERM", "vt100")], &[], true, true, true), + (&[("TERM", "foo")], &[], true, true, false), + (&[("TERM", "xterm")], &["--no-color"], true, true, false), + (&[("TERM", "xterm")], &["--", "--no-color"], true, true, true), + (&[], &["--color"], false, false, true), + (&[], &["--color=256"], false, false, true), + (&[("TERM", "screen")], &["build", "--color=false"], true, true, false), + (&[("FORCE_COLOR", "1")], &["--no-color"], false, false, true), + (&[("TERM_PROGRAM", "Apple_Terminal")], &[], true, true, true), +]; + +#[test] +fn console_color_decision_matches_supports_color() { + for (index, (environment, arguments, output_is_terminal, error_is_terminal, expected)) in + CASES_VERIFIED_WITH_SUPPORTS_COLOR_8_1_1.iter().enumerate() + { + let environment_variable = |name: &str| { + environment + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| (*value).to_owned()) + }; + let mut command_line_arguments = vec![String::from("node"), String::from("heft")]; + command_line_arguments.extend(arguments.iter().map(|argument| (*argument).to_owned())); + let inputs = ColorSupportInputs { + environment_variable: &environment_variable, + command_line_arguments: &command_line_arguments, + standard_output_is_terminal: *output_is_terminal, + standard_error_is_terminal: *error_is_terminal, + }; + assert_eq!(console_supports_color(&inputs), *expected, "case {index}"); + } +} diff --git a/apps/heft-native/src/terminal/force_color.rs b/apps/heft-native/src/terminal/force_color.rs new file mode 100644 index 0000000000..94ea926de8 --- /dev/null +++ b/apps/heft-native/src/terminal/force_color.rs @@ -0,0 +1,46 @@ +use super::has_flag::has_flag; +use super::parse_int::parse_int_base_ten_like_javascript; + +pub fn flag_force_color(command_line_arguments: &[String]) -> Option { + if has_flag("no-color", command_line_arguments) + || has_flag("no-colors", command_line_arguments) + || has_flag("color=false", command_line_arguments) + || has_flag("color=never", command_line_arguments) + { + Some(0.0) + } else if has_flag("color", command_line_arguments) + || has_flag("colors", command_line_arguments) + || has_flag("color=true", command_line_arguments) + || has_flag("color=always", command_line_arguments) + { + Some(1.0) + } else { + None + } +} + +pub fn environment_force_color( + environment_variable: &dyn Fn(&str) -> Option, +) -> Option { + environment_variable("FORCE_COLOR").map(|value| { + if value == "true" { + 1.0 + } else if value == "false" { + 0.0 + } else if value.is_empty() { + 1.0 + } else { + javascript_math_min(parse_int_base_ten_like_javascript(&value), 3.0) + } + }) +} + +fn javascript_math_min(left: f64, right: f64) -> f64 { + if left.is_nan() || right.is_nan() { + f64::NAN + } else if left < right { + left + } else { + right + } +} diff --git a/apps/heft-native/src/terminal/has_flag.rs b/apps/heft-native/src/terminal/has_flag.rs new file mode 100644 index 0000000000..60828f96df --- /dev/null +++ b/apps/heft-native/src/terminal/has_flag.rs @@ -0,0 +1,21 @@ +pub fn has_flag(flag: &str, command_line_arguments: &[String]) -> bool { + let prefix = if flag.starts_with('-') { + "" + } else if flag.chars().count() == 1 { + "-" + } else { + "--" + }; + let wanted = format!("{prefix}{flag}"); + let position = command_line_arguments + .iter() + .position(|argument| argument == &wanted); + let terminator_position = command_line_arguments + .iter() + .position(|argument| argument == "--"); + match (position, terminator_position) { + (Some(found), Some(terminator)) => found < terminator, + (Some(_), None) => true, + (None, _) => false, + } +} diff --git a/apps/heft-native/src/terminal/heft_console.rs b/apps/heft-native/src/terminal/heft_console.rs new file mode 100644 index 0000000000..ae7d337a07 --- /dev/null +++ b/apps/heft-native/src/terminal/heft_console.rs @@ -0,0 +1,168 @@ +use std::cell::{Cell, RefCell}; +use std::io::{ErrorKind, Write}; + +use super::ansi_escape_codes::{red, remove_ansi_escape_codes}; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum OutputSeverity { + Log, + Error, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct ClosedOutput { + pub severity: OutputSeverity, + pub prefixed: bool, +} + +pub struct HeftConsole { + supports_color: bool, + captured_output: Option>>, + closed_output: Cell>, +} + +impl HeftConsole { + pub fn new(supports_color: bool) -> HeftConsole { + HeftConsole { supports_color, captured_output: None, closed_output: Cell::new(None) } + } + + #[cfg(test)] + pub fn capturing(supports_color: bool) -> HeftConsole { + HeftConsole { supports_color, captured_output: Some(RefCell::new(Vec::new())), closed_output: Cell::new(None) } + } + + pub fn closed_output(&self) -> Option { + self.closed_output.get() + } + + #[cfg(test)] + pub fn captured_output(&self) -> Vec<(OutputSeverity, String)> { + self.captured_output.as_ref().map(|captured| captured.borrow().clone()).unwrap_or_default() + } + + pub fn write_line(&self, text: &str) { + self.write(OutputSeverity::Log, &self.format_line(text, OutputSeverity::Log), false); + } + + pub fn write_error_line(&self, text: &str) { + self.write(OutputSeverity::Error, &self.format_line(text, OutputSeverity::Error), false); + } + + fn write(&self, severity: OutputSeverity, data: &str, prefixed: bool) { + if self.closed_output.get().is_some() { + return; + } + match &self.captured_output { + Some(captured) => captured.borrow_mut().push((severity, data.to_owned())), + None => { + if write_to_stream(severity, data).is_err_and(|error| error.kind() == ErrorKind::BrokenPipe) { + self.closed_output.set(Some(ClosedOutput { severity, prefixed })); + } + } + } + } + + pub fn format_line(&self, text: &str, severity: OutputSeverity) -> String { + let text_with_severity_color = match severity { + OutputSeverity::Log => text.to_owned(), + OutputSeverity::Error => red(&remove_ansi_escape_codes(text)), + }; + let mut line = if self.supports_color { + text_with_severity_color + } else { + remove_ansi_escape_codes(&text_with_severity_color).into_owned() + }; + line.push('\n'); + line + } + + pub fn unprefixed_output(&self) -> ScopedLoggerOutput<'_> { + ScopedLoggerOutput { + console: self, + prefix: String::new(), + is_on_new_line: Cell::new(true), + } + } + + pub fn scoped_logger_output(&self, logger_name: &str) -> ScopedLoggerOutput<'_> { + ScopedLoggerOutput { + console: self, + prefix: format!("[{logger_name}] "), + is_on_new_line: Cell::new(true), + } + } +} + +pub struct ScopedLoggerOutput<'console> { + console: &'console HeftConsole, + prefix: String, + is_on_new_line: Cell, +} + +impl ScopedLoggerOutput<'_> { + pub fn write_line(&self, text: &str) { + let line = self.console.format_line(text, OutputSeverity::Log); + self.console.write(OutputSeverity::Log, &self.prefix_lines(&line), !self.prefix.is_empty()); + } + + pub fn output_is_closed(&self) -> bool { + self.console.closed_output().is_some() + } + + pub fn prefix_lines(&self, data: &str) -> String { + let mut prefixed = String::with_capacity(data.len() + self.prefix.len()); + let mut current_index = 0; + for (newline_index, _) in data.match_indices('\n') { + if self.is_on_new_line.get() { + prefixed.push_str(&self.prefix); + } + prefixed.push_str(&data[current_index..=newline_index]); + current_index = newline_index + 1; + self.is_on_new_line.set(true); + } + let remaining_data = &data[current_index..]; + if !remaining_data.is_empty() { + if self.is_on_new_line.get() { + prefixed.push_str(&self.prefix); + } + prefixed.push_str(remaining_data); + self.is_on_new_line.set(false); + } + prefixed + } +} + +fn write_to_stream(severity: OutputSeverity, data: &str) -> std::io::Result<()> { + if severity == OutputSeverity::Log { + let mut standard_output = std::io::stdout().lock(); + standard_output.write_all(data.as_bytes())?; + standard_output.flush() + } else { + std::io::stderr().lock().write_all(data.as_bytes()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn severity_colors_match_the_terminal_package() { + let colored = HeftConsole::new(true); + let plain = HeftConsole::new(false); + assert_eq!(colored.format_line("a\x1b[1mb", OutputSeverity::Error), "\x1b[31mab\x1b[39m\n"); + assert_eq!(plain.format_line("a\x1b[1mb", OutputSeverity::Error), "ab\n"); + assert_eq!(colored.format_line("\x1b[1mx\x1b[22m", OutputSeverity::Log), "\x1b[1mx\x1b[22m\n"); + assert_eq!(plain.format_line("\x1b[1mx\x1b[22m", OutputSeverity::Log), "x\n"); + } + + #[test] + fn scoped_output_prefixes_every_line() { + let console = HeftConsole::new(false); + let output = console.scoped_logger_output("build:set-env"); + assert_eq!(output.prefix_lines("a\nb\n"), "[build:set-env] a\n[build:set-env] b\n"); + assert_eq!(output.prefix_lines("partial"), "[build:set-env] partial"); + assert_eq!(output.prefix_lines(" rest\n"), " rest\n"); + assert_eq!(output.prefix_lines("\n"), "[build:set-env] \n"); + } +} diff --git a/apps/heft-native/src/terminal/javascript_number_format.rs b/apps/heft-native/src/terminal/javascript_number_format.rs new file mode 100644 index 0000000000..70f71a1d9d --- /dev/null +++ b/apps/heft-native/src/terminal/javascript_number_format.rs @@ -0,0 +1,55 @@ +pub fn format_rounded_milliseconds_as_seconds(duration_in_milliseconds: f64) -> String { + let rounded_milliseconds = duration_in_milliseconds.round() as u64; + let whole_seconds = rounded_milliseconds / 1000; + let remaining_milliseconds = rounded_milliseconds % 1000; + if remaining_milliseconds == 0 { + return whole_seconds.to_string(); + } + let fraction_digits = format!("{remaining_milliseconds:03}"); + format!("{whole_seconds}.{}", fraction_digits.trim_end_matches('0')) +} + +pub fn format_seconds_with_three_fraction_digits(seconds: f64) -> String { + let sixteenths = seconds * 16.0; + if sixteenths.fract() == 0.0 && sixteenths % 2.0 == 1.0 && sixteenths < 1e15 { + let thousandths_rounded_up = (sixteenths as u64 * 125).div_ceil(2); + return format!( + "{}.{:03}", + thousandths_rounded_up / 1000, + thousandths_rounded_up % 1000 + ); + } + format!("{seconds:.3}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rounded_milliseconds_print_like_javascript_numbers() { + assert_eq!(format_rounded_milliseconds_as_seconds(0.0), "0"); + assert_eq!(format_rounded_milliseconds_as_seconds(0.49999999999999994), "0"); + assert_eq!(format_rounded_milliseconds_as_seconds(0.5), "0.001"); + assert_eq!(format_rounded_milliseconds_as_seconds(12.4), "0.012"); + assert_eq!(format_rounded_milliseconds_as_seconds(99.5), "0.1"); + assert_eq!(format_rounded_milliseconds_as_seconds(1000.2), "1"); + assert_eq!(format_rounded_milliseconds_as_seconds(1234567.0), "1234.567"); + assert_eq!(format_rounded_milliseconds_as_seconds(60010.0), "60.01"); + } + + #[test] + fn fixed_three_digits_round_exact_ties_up_like_javascript() { + assert_eq!(format_seconds_with_three_fraction_digits(0.0), "0.000"); + assert_eq!(format_seconds_with_three_fraction_digits(0.0625), "0.063"); + assert_eq!(format_seconds_with_three_fraction_digits(0.1875), "0.188"); + assert_eq!(format_seconds_with_three_fraction_digits(0.3125), "0.313"); + assert_eq!(format_seconds_with_three_fraction_digits(2.5625), "2.563"); + assert_eq!(format_seconds_with_three_fraction_digits(0.0005), "0.001"); + assert_eq!(format_seconds_with_three_fraction_digits(0.0015), "0.002"); + assert_eq!(format_seconds_with_three_fraction_digits(0.0025), "0.003"); + assert_eq!(format_seconds_with_three_fraction_digits(1.0005), "1.000"); + assert_eq!(format_seconds_with_three_fraction_digits(0.0123456), "0.012"); + assert_eq!(format_seconds_with_three_fraction_digits(123.9999), "124.000"); + } +} diff --git a/apps/heft-native/src/terminal/mod.rs b/apps/heft-native/src/terminal/mod.rs new file mode 100644 index 0000000000..1e8e0c2e56 --- /dev/null +++ b/apps/heft-native/src/terminal/mod.rs @@ -0,0 +1,17 @@ +mod ansi_escape_codes; +mod color_support; +#[cfg(test)] +mod color_support_tests; +mod force_color; +mod has_flag; +mod heft_console; +mod javascript_number_format; +mod parse_int; +mod term_patterns; + +pub use ansi_escape_codes::{bold, green, red}; +pub use color_support::console_supports_color_for_this_process; +pub use heft_console::{ClosedOutput, HeftConsole, OutputSeverity, ScopedLoggerOutput}; +pub use javascript_number_format::{ + format_rounded_milliseconds_as_seconds, format_seconds_with_three_fraction_digits, +}; diff --git a/apps/heft-native/src/terminal/parse_int.rs b/apps/heft-native/src/terminal/parse_int.rs new file mode 100644 index 0000000000..d273f85a59 --- /dev/null +++ b/apps/heft-native/src/terminal/parse_int.rs @@ -0,0 +1,29 @@ +pub fn parse_int_base_ten_like_javascript(value: &str) -> f64 { + let bytes = value.as_bytes(); + let mut index = 0; + while index < bytes.len() && is_javascript_whitespace(bytes[index]) { + index += 1; + } + let mut sign = 1.0; + if index < bytes.len() && bytes[index] == b'+' { + index += 1; + } else if index < bytes.len() && bytes[index] == b'-' { + sign = -1.0; + index += 1; + } + let start = index; + let mut number = 0.0; + while index < bytes.len() && bytes[index].is_ascii_digit() { + number = number * 10.0 + f64::from(bytes[index] - b'0'); + index += 1; + } + if index == start { + f64::NAN + } else { + sign * number + } +} + +fn is_javascript_whitespace(byte: u8) -> bool { + matches!(byte, b'\t' | b'\n' | b'\x0B' | b'\x0C' | b'\r' | b' ') +} diff --git a/apps/heft-native/src/terminal/term_patterns.rs b/apps/heft-native/src/terminal/term_patterns.rs new file mode 100644 index 0000000000..52619aa259 --- /dev/null +++ b/apps/heft-native/src/terminal/term_patterns.rs @@ -0,0 +1,42 @@ +pub fn ends_with_256_color_pattern(value: &str) -> bool { + let lower = value.to_ascii_lowercase(); + lower.ends_with("-256") || lower.ends_with("-256color") +} + +pub fn matches_basic_term_pattern(value: &str) -> bool { + let lower = value.to_ascii_lowercase(); + lower.starts_with("screen") + || lower.starts_with("xterm") + || lower.starts_with("vt100") + || lower.starts_with("vt220") + || lower.starts_with("rxvt") + || lower.contains("color") + || lower.contains("ansi") + || lower.contains("cygwin") + || lower.contains("linux") +} + +pub fn matches_teamcity_version_pattern(value: &str) -> bool { + matches_teamcity_nine_pattern(value) || matches_two_or_more_digit_major_pattern(value) +} + +fn matches_teamcity_nine_pattern(value: &str) -> bool { + let Some(rest) = value.strip_prefix("9.") else { + return false; + }; + let Some(dot_index) = rest.find('.') else { + return false; + }; + let middle = &rest[..dot_index]; + if middle.is_empty() || !middle.bytes().all(|byte| byte.is_ascii_digit()) { + return false; + } + middle.bytes().any(|byte| byte != b'0') +} + +fn matches_two_or_more_digit_major_pattern(value: &str) -> bool { + let Some(dot_index) = value.find('.') else { + return false; + }; + dot_index >= 2 && value[..dot_index].bytes().all(|byte| byte.is_ascii_digit()) +} diff --git a/apps/heft-native/src/version/banner.rs b/apps/heft-native/src/version/banner.rs new file mode 100644 index 0000000000..04f36bfa66 --- /dev/null +++ b/apps/heft-native/src/version/banner.rs @@ -0,0 +1,27 @@ +use std::io::Write; + +use super::VersionSelectorBanner; + +impl VersionSelectorBanner { + pub fn text_printed_by_javascript_version_selector(self) -> &'static str { + match self { + VersionSelectorBanner::Silent => "", + VersionSelectorBanner::BypassingTheSelectorBecauseUnmanagedWasSpecified => { + "Bypassing the Heft version selector because \"--unmanaged\" was specified.\n\n" + } + VersionSelectorBanner::SearchingForLocalHeftBecauseDebugWasSpecified => { + "Searching for a locally installed version of Heft. Use the \"--unmanaged\" flag if you want to avoid this.\n" + } + } + } +} + +pub fn write_version_selector_banner(banner: VersionSelectorBanner) { + let banner_text = banner.text_printed_by_javascript_version_selector(); + if banner_text.is_empty() { + return; + } + let mut standard_output = std::io::stdout().lock(); + let _ = standard_output.write_all(banner_text.as_bytes()); + let _ = standard_output.flush(); +} diff --git a/apps/heft-native/src/version/companion.rs b/apps/heft-native/src/version/companion.rs new file mode 100644 index 0000000000..829c6d1f4e --- /dev/null +++ b/apps/heft-native/src/version/companion.rs @@ -0,0 +1,42 @@ +use std::path::{Path, PathBuf}; + +use super::selector::find_package_json_path_governing_folder; + +pub const COMPANION_JAVASCRIPT_HEFT_BIN_ENVIRONMENT_VARIABLE: &str = "HEFT_NATIVE_JS_BIN"; +const HEFT_BIN_PATH_INSIDE_SCOPE_FOLDER: &str = "heft/bin/heft"; +const HEFT_BIN_PATH_INSIDE_PROJECT_FOLDER: &str = "node_modules/@rushstack/heft/bin/heft"; + +pub fn locate_companion_javascript_heft_bin(current_folder: &Path) -> Option { + if let Some(explicit_bin) = std::env::var_os(COMPANION_JAVASCRIPT_HEFT_BIN_ENVIRONMENT_VARIABLE) + { + if !explicit_bin.is_empty() { + return Some(PathBuf::from(explicit_bin)); + } + } + std::env::current_exe() + .ok() + .and_then(|executable_path| locate_heft_bin_next_to_executable(&executable_path)) + .or_else(|| locate_heft_bin_of_project_governing_folder(current_folder)) +} + +pub fn locate_companion_javascript_heft_package_folder(current_folder: &Path) -> Option { + let mut companion_heft_bin = locate_companion_javascript_heft_bin(current_folder)?; + (companion_heft_bin.pop() && companion_heft_bin.pop()).then_some(companion_heft_bin) +} + +pub fn locate_heft_bin_next_to_executable(executable_path: &Path) -> Option { + let executable_folder = executable_path.parent()?; + executable_folder + .ancestors() + .skip(2) + .take(2) + .map(|scope_or_apps_folder| scope_or_apps_folder.join(HEFT_BIN_PATH_INSIDE_SCOPE_FOLDER)) + .find(|candidate_bin| candidate_bin.is_file()) +} + +fn locate_heft_bin_of_project_governing_folder(current_folder: &Path) -> Option { + let mut project_heft_bin = find_package_json_path_governing_folder(current_folder)?; + project_heft_bin.pop(); + project_heft_bin.push(HEFT_BIN_PATH_INSIDE_PROJECT_FOLDER); + project_heft_bin.is_file().then_some(project_heft_bin) +} diff --git a/apps/heft-native/src/version/mod.rs b/apps/heft-native/src/version/mod.rs new file mode 100644 index 0000000000..bef60af996 --- /dev/null +++ b/apps/heft-native/src/version/mod.rs @@ -0,0 +1,42 @@ +mod banner; +mod companion; +mod package_json_probe; +mod selector; + +#[cfg(test)] +mod tests_fixture_folder; +#[cfg(test)] +mod tests_package_json_and_arguments; +#[cfg(test)] +mod tests_selector; + +use std::path::PathBuf; + +pub use banner::write_version_selector_banner; +pub use companion::{ + locate_companion_javascript_heft_bin, COMPANION_JAVASCRIPT_HEFT_BIN_ENVIRONMENT_VARIABLE, +}; +#[cfg(unix)] +pub use selector::find_package_json_path_governing_folder; +pub use selector::select_heft_implementation; + +pub const HEFT_VERSION_IMPLEMENTED_BY_THIS_BINARY: &str = env!("CARGO_PKG_VERSION"); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VersionSelectorBanner { + Silent, + BypassingTheSelectorBecauseUnmanagedWasSpecified, + SearchingForLocalHeftBecauseDebugWasSpecified, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct NativeHeftContext { + pub version_selector_banner: VersionSelectorBanner, + pub companion_heft_package_folder: Option, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum HeftImplementationSelection { + DelegateToJavaScriptHeft, + ThisBinary(NativeHeftContext), +} diff --git a/apps/heft-native/src/version/package_json_probe.rs b/apps/heft-native/src/version/package_json_probe.rs new file mode 100644 index 0000000000..189666a36d --- /dev/null +++ b/apps/heft-native/src/version/package_json_probe.rs @@ -0,0 +1,62 @@ +use std::path::Path; + +use crate::json::{parse_json_exactly_like_json_parse, JsonValue}; + +const HEFT_PACKAGE_NAME: &str = "@rushstack/heft"; + +#[derive(Debug, PartialEq, Eq)] +pub enum ProjectPackageJsonProbe { + MustBeProbedByJavaScript, + DeclaresNoHeftDependency, + DeclaresHeftDependency, +} + +pub fn probe_project_package_json(package_json_path: &Path) -> ProjectPackageJsonProbe { + let Ok(package_json_text) = std::fs::read_to_string(package_json_path) else { + return ProjectPackageJsonProbe::MustBeProbedByJavaScript; + }; + match parse_json_exactly_like_json_parse(&package_json_text) { + Ok(JsonValue::Object(package_json)) => { + if dependency_map_declares_heft(package_json.get("dependencies")) + || dependency_map_declares_heft(package_json.get("devDependencies")) + { + ProjectPackageJsonProbe::DeclaresHeftDependency + } else { + ProjectPackageJsonProbe::DeclaresNoHeftDependency + } + } + Ok(JsonValue::Null) | Err(_) => ProjectPackageJsonProbe::MustBeProbedByJavaScript, + Ok(_) => ProjectPackageJsonProbe::DeclaresNoHeftDependency, + } +} + +pub fn package_json_declares_version(package_json_path: &Path, expected_version: &str) -> bool { + let Ok(package_json_text) = std::fs::read_to_string(package_json_path) else { + return false; + }; + match parse_json_exactly_like_json_parse(&package_json_text) { + Ok(package_json) => { + package_json.get("version").and_then(JsonValue::as_str) == Some(expected_version) + } + Err(_) => false, + } +} + +fn dependency_map_declares_heft(dependency_map: Option<&JsonValue>) -> bool { + match dependency_map { + Some(JsonValue::Object(dependencies)) => dependencies + .get(HEFT_PACKAGE_NAME) + .is_some_and(is_truthy_in_javascript), + _ => false, + } +} + +fn is_truthy_in_javascript(value: &JsonValue) -> bool { + match value { + JsonValue::Null => false, + JsonValue::Boolean(flag) => *flag, + JsonValue::Number(number) => number.value != 0.0 && !number.value.is_nan(), + JsonValue::String(text) => !text.is_empty(), + JsonValue::Array(_) | JsonValue::Object(_) => true, + } +} diff --git a/apps/heft-native/src/version/selector.rs b/apps/heft-native/src/version/selector.rs new file mode 100644 index 0000000000..b4b2ea848a --- /dev/null +++ b/apps/heft-native/src/version/selector.rs @@ -0,0 +1,153 @@ +use std::ffi::OsString; +use std::path::{Path, PathBuf}; + +use super::companion::locate_companion_javascript_heft_package_folder; +use super::package_json_probe::{ + package_json_declares_version, probe_project_package_json, ProjectPackageJsonProbe, +}; +use super::{ + HeftImplementationSelection, NativeHeftContext, VersionSelectorBanner, + HEFT_VERSION_IMPLEMENTED_BY_THIS_BINARY, +}; + +const UNMANAGED_PARAMETER_LONG_NAME: &[u8] = b"--unmanaged"; +const DEBUG_PARAMETER_LONG_NAME: &[u8] = b"--debug"; + +pub fn select_heft_implementation( + command_line_arguments: &[OsString], +) -> HeftImplementationSelection { + let (unmanaged_was_specified, debug_was_specified) = + find_version_selector_tool_parameters(command_line_arguments); + let banner = if unmanaged_was_specified { + VersionSelectorBanner::BypassingTheSelectorBecauseUnmanagedWasSpecified + } else if debug_was_specified { + VersionSelectorBanner::SearchingForLocalHeftBecauseDebugWasSpecified + } else { + VersionSelectorBanner::Silent + }; + let Ok(current_folder) = std::env::current_dir() else { + return HeftImplementationSelection::DelegateToJavaScriptHeft; + }; + let companion_heft_package_folder = + locate_companion_javascript_heft_package_folder(¤t_folder); + select_heft_implementation_for_folder(¤t_folder, banner, companion_heft_package_folder) +} + +pub fn select_heft_implementation_for_folder( + current_folder: &Path, + banner: VersionSelectorBanner, + companion_heft_package_folder: Option, +) -> HeftImplementationSelection { + if let Some(companion_folder) = &companion_heft_package_folder { + if !heft_package_is_the_version_of_this_binary(companion_folder) { + return HeftImplementationSelection::DelegateToJavaScriptHeft; + } + } + let native_heft_context = NativeHeftContext { + version_selector_banner: banner, + companion_heft_package_folder, + }; + if banner == VersionSelectorBanner::BypassingTheSelectorBecauseUnmanagedWasSpecified { + return HeftImplementationSelection::ThisBinary(native_heft_context); + } + let Some(mut package_json_path) = find_package_json_path_governing_folder(current_folder) + else { + return HeftImplementationSelection::ThisBinary(native_heft_context); + }; + match probe_project_package_json(&package_json_path) { + ProjectPackageJsonProbe::MustBeProbedByJavaScript => { + HeftImplementationSelection::DelegateToJavaScriptHeft + } + ProjectPackageJsonProbe::DeclaresNoHeftDependency => { + HeftImplementationSelection::ThisBinary(native_heft_context) + } + ProjectPackageJsonProbe::DeclaresHeftDependency => { + package_json_path.pop(); + package_json_path.push("node_modules/@rushstack/heft"); + select_project_local_heft(&package_json_path, native_heft_context) + } + } +} + +pub fn find_package_json_path_governing_folder(current_folder: &Path) -> Option { + let mut package_json_path = current_folder.join("package.json"); + loop { + if package_json_path.exists() { + return Some(package_json_path); + } + package_json_path.pop(); + if !package_json_path.pop() { + return None; + } + package_json_path.push("package.json"); + } +} + +pub(super) fn find_version_selector_tool_parameters( + command_line_arguments: &[OsString], +) -> (bool, bool) { + let mut unmanaged_was_specified = false; + let mut debug_was_specified = false; + for tool_parameter in command_line_arguments + .iter() + .map(|argument| argument.as_encoded_bytes()) + .take_while(|argument| argument.first() == Some(&b'-')) + { + unmanaged_was_specified |= tool_parameter == UNMANAGED_PARAMETER_LONG_NAME; + debug_was_specified |= tool_parameter == DEBUG_PARAMETER_LONG_NAME; + } + (unmanaged_was_specified, debug_was_specified) +} + +fn select_project_local_heft( + local_heft_package_folder: &Path, + native_heft_context: NativeHeftContext, +) -> HeftImplementationSelection { + let local_heft_is_the_companion = local_heft_package_folder + .join("lib-commonjs/start.js") + .exists() + && native_heft_context + .companion_heft_package_folder + .as_deref() + .is_some_and(|companion_folder| { + folders_are_the_same_real_folder(local_heft_package_folder, companion_folder) + }); + if local_heft_is_the_companion { + HeftImplementationSelection::ThisBinary(native_heft_context) + } else { + HeftImplementationSelection::DelegateToJavaScriptHeft + } +} + +#[cfg(unix)] +fn folders_are_the_same_real_folder(first_folder: &Path, second_folder: &Path) -> bool { + use std::os::unix::fs::MetadataExt; + match ( + std::fs::metadata(first_folder), + std::fs::metadata(second_folder), + ) { + (Ok(first_metadata), Ok(second_metadata)) => { + first_metadata.dev() == second_metadata.dev() + && first_metadata.ino() == second_metadata.ino() + } + _ => false, + } +} + +#[cfg(not(unix))] +fn folders_are_the_same_real_folder(first_folder: &Path, second_folder: &Path) -> bool { + match ( + std::fs::canonicalize(first_folder), + std::fs::canonicalize(second_folder), + ) { + (Ok(first_real_folder), Ok(second_real_folder)) => first_real_folder == second_real_folder, + _ => false, + } +} + +fn heft_package_is_the_version_of_this_binary(heft_package_folder: &Path) -> bool { + package_json_declares_version( + &heft_package_folder.join("package.json"), + HEFT_VERSION_IMPLEMENTED_BY_THIS_BINARY, + ) +} diff --git a/apps/heft-native/src/version/tests_fixture_folder.rs b/apps/heft-native/src/version/tests_fixture_folder.rs new file mode 100644 index 0000000000..66f6d42462 --- /dev/null +++ b/apps/heft-native/src/version/tests_fixture_folder.rs @@ -0,0 +1,82 @@ +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use super::selector::select_heft_implementation_for_folder; +use super::{ + HeftImplementationSelection, NativeHeftContext, VersionSelectorBanner, + HEFT_VERSION_IMPLEMENTED_BY_THIS_BINARY, +}; + +static NEXT_FIXTURE_NUMBER: AtomicUsize = AtomicUsize::new(0); +pub const LOCAL_HEFT: &str = "node_modules/@rushstack/heft"; +pub const LOCAL_HEFT_START: &str = "node_modules/@rushstack/heft/lib-commonjs/start.js"; +pub const LOCAL_HEFT_PACKAGE_JSON: &str = "node_modules/@rushstack/heft/package.json"; +pub const HEFT_DEPENDENCY: &str = r#"{"dependencies":{"@rushstack/heft":"*"}}"#; +pub const DELEGATE: HeftImplementationSelection = + HeftImplementationSelection::DelegateToJavaScriptHeft; + +pub struct FixtureFolder(pub PathBuf); + +impl FixtureFolder { + pub fn with_files(files: &[(&str, &str)]) -> FixtureFolder { + let fixture_number = NEXT_FIXTURE_NUMBER.fetch_add(1, Ordering::Relaxed); + let root = std::env::temp_dir().join(format!( + "heft-native-version-{}-{fixture_number}", + std::process::id() + )); + std::fs::create_dir_all(&root).unwrap(); + for (relative_path, content) in files { + let file_path = root.join(relative_path); + std::fs::create_dir_all(file_path.parent().unwrap()).unwrap(); + std::fs::write(&file_path, content).unwrap(); + } + FixtureFolder(root) + } + + pub fn select_with_companion( + &self, + relative_current_folder: &str, + companion: Option<&str>, + ) -> HeftImplementationSelection { + let companion_folder = companion.map(|relative_companion| self.0.join(relative_companion)); + select_heft_implementation_for_folder( + &self.0.join(relative_current_folder), + VersionSelectorBanner::Silent, + companion_folder, + ) + } + + pub fn select(&self, relative_current_folder: &str) -> HeftImplementationSelection { + let local_heft_exists = self.0.join(LOCAL_HEFT_PACKAGE_JSON).exists(); + self.select_with_companion( + relative_current_folder, + local_heft_exists.then_some(LOCAL_HEFT), + ) + } + + pub fn this_binary(&self, relative_companion: &str) -> HeftImplementationSelection { + HeftImplementationSelection::ThisBinary(NativeHeftContext { + version_selector_banner: VersionSelectorBanner::Silent, + companion_heft_package_folder: Some(self.0.join(relative_companion)), + }) + } +} + +impl Drop for FixtureFolder { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +pub fn heft_package_json(version: &str) -> String { + format!("{{\"name\":\"@rushstack/heft\",\"version\":\"{version}\"}}") +} + +pub fn fixture_for_project_package_json(package_json: &str) -> FixtureFolder { + let local_heft_package_json = heft_package_json(HEFT_VERSION_IMPLEMENTED_BY_THIS_BINARY); + FixtureFolder::with_files(&[ + ("package.json", package_json), + (LOCAL_HEFT_START, ""), + (LOCAL_HEFT_PACKAGE_JSON, &local_heft_package_json), + ]) +} diff --git a/apps/heft-native/src/version/tests_package_json_and_arguments.rs b/apps/heft-native/src/version/tests_package_json_and_arguments.rs new file mode 100644 index 0000000000..8adef70f13 --- /dev/null +++ b/apps/heft-native/src/version/tests_package_json_and_arguments.rs @@ -0,0 +1,98 @@ +use std::ffi::OsString; +use std::path::Path; + +use super::selector::find_version_selector_tool_parameters; +use super::tests_fixture_folder::{ + fixture_for_project_package_json, FixtureFolder, DELEGATE, HEFT_DEPENDENCY, LOCAL_HEFT, + LOCAL_HEFT_START, +}; +use super::HEFT_VERSION_IMPLEMENTED_BY_THIS_BINARY; + +#[test] +fn package_json_without_a_truthy_heft_dependency_runs_the_invoked_heft() { + for package_json in [ + "{}", + "[]", + "\"text\"", + "5", + "true", + r#"{"dependencies":{"@rushstack/heft":""}}"#, + r#"{"dependencies":{"@rushstack/heft":0}}"#, + r#"{"dependencies":{"@rushstack/heft":-0.0}}"#, + r#"{"dependencies":{"@rushstack/heft":false}}"#, + r#"{"dependencies":{"@rushstack/heft":null}}"#, + r#"{"dependencies":"@rushstack/heft"}"#, + r#"{"dependencies":["@rushstack/heft"]}"#, + r#"{"dependencies":{"__proto__":{"@rushstack/heft":"*"}}}"#, + r#"{"dependencies":{"@rushstack/heft":"*"},"dependencies":{}}"#, + r#"{"peerDependencies":{"@rushstack/heft":"*"}}"#, + ] { + let fixture = fixture_for_project_package_json(package_json); + assert_eq!( + fixture.select(""), + fixture.this_binary(LOCAL_HEFT), + "{package_json}" + ); + } +} + +#[test] +fn truthy_heft_dependencies_select_the_local_heft() { + for package_json in [ + HEFT_DEPENDENCY, + r#"{"devDependencies":{"@rushstack/heft":"workspace:*"}}"#, + r#"{"dependencies":{"@rushstack/heft":{}}}"#, + r#"{"dependencies":{"@rushstack/heft":[]}}"#, + r#"{"dependencies":{"@rushstack/heft":true}}"#, + r#"{"dependencies":{"@rushstack/heft":1e-5}}"#, + r#"{"dependencies":{"@rushstack\/heft":"*"}}"#, + r#"{"dependencies":{},"dependencies":{"@rushstack/heft":"*"}}"#, + ] { + let fixture = fixture_for_project_package_json(package_json); + std::fs::remove_file(fixture.0.join(LOCAL_HEFT_START)).unwrap(); + assert_eq!(fixture.select(""), DELEGATE, "{package_json}"); + } +} + +#[test] +fn package_json_that_json_parse_rejects_or_that_throws_is_delegated() { + for package_json in ["{ nope", "null", "{\"a\":1,}", "// c\n{}", "\u{feff}{}", ""] { + assert_eq!( + fixture_for_project_package_json(package_json).select(""), + DELEGATE, + "{package_json:?}" + ); + } + let fixture = FixtureFolder::with_files(&[("package.json/placeholder", "")]); + assert_eq!(fixture.select(""), DELEGATE); +} + +#[test] +fn only_leading_dash_arguments_are_version_selector_tool_parameters() { + let parameters = |arguments: &[&str]| { + let arguments: Vec = arguments.iter().map(OsString::from).collect(); + find_version_selector_tool_parameters(&arguments) + }; + assert_eq!(parameters(&["--unmanaged", "build"]), (true, false)); + assert_eq!(parameters(&["--debug", "--unmanaged"]), (true, true)); + assert_eq!(parameters(&["--debug", "x"]), (false, true)); + assert_eq!( + parameters(&["build", "--unmanaged", "--debug"]), + (false, false) + ); + assert_eq!(parameters(&["", "--debug"]), (false, false)); + assert_eq!( + parameters(&["--unmanaged=true", "--debug-x"]), + (false, false) + ); +} + +#[test] +fn binary_version_matches_the_heft_package_it_implements() { + let heft_package_json = + std::fs::read_to_string(Path::new(env!("CARGO_MANIFEST_DIR")).join("../heft/package.json")) + .unwrap(); + assert!(heft_package_json.contains(&format!( + "\"version\": \"{HEFT_VERSION_IMPLEMENTED_BY_THIS_BINARY}\"" + ))); +} diff --git a/apps/heft-native/src/version/tests_selector.rs b/apps/heft-native/src/version/tests_selector.rs new file mode 100644 index 0000000000..bd6538c8e9 --- /dev/null +++ b/apps/heft-native/src/version/tests_selector.rs @@ -0,0 +1,121 @@ +use super::companion::locate_heft_bin_next_to_executable; +use super::tests_fixture_folder::{ + heft_package_json, FixtureFolder, DELEGATE, HEFT_DEPENDENCY, LOCAL_HEFT, + LOCAL_HEFT_PACKAGE_JSON, LOCAL_HEFT_START, +}; +use super::{ + HeftImplementationSelection, NativeHeftContext, VersionSelectorBanner, + HEFT_VERSION_IMPLEMENTED_BY_THIS_BINARY, +}; + +#[test] +fn local_heft_that_is_the_companion_is_served_by_this_binary() { + let this_version = heft_package_json(HEFT_VERSION_IMPLEMENTED_BY_THIS_BINARY); + let fixture = FixtureFolder::with_files(&[ + ("package.json", HEFT_DEPENDENCY), + (LOCAL_HEFT_START, ""), + (LOCAL_HEFT_PACKAGE_JSON, &this_version), + ("src/sub/file.ts", ""), + ]); + assert_eq!(fixture.select(""), fixture.this_binary(LOCAL_HEFT)); + assert_eq!(fixture.select("src/sub"), fixture.this_binary(LOCAL_HEFT)); +} + +#[test] +fn same_version_local_heft_in_another_real_folder_is_delegated_unless_it_links_to_the_companion() { + let this_version = heft_package_json(HEFT_VERSION_IMPLEMENTED_BY_THIS_BINARY); + let patched_copy = FixtureFolder::with_files(&[ + ("package.json", HEFT_DEPENDENCY), + (LOCAL_HEFT_START, ""), + (LOCAL_HEFT_PACKAGE_JSON, &this_version), + ("store/heft/lib-commonjs/start.js", ""), + ("store/heft/package.json", &this_version), + ]); + assert_eq!( + patched_copy.select_with_companion("", Some("store/heft")), + DELEGATE + ); + let linked = FixtureFolder::with_files(&[ + ("package.json", HEFT_DEPENDENCY), + ("store/heft/lib-commonjs/start.js", ""), + ("store/heft/package.json", &this_version), + ]); + std::fs::create_dir_all(linked.0.join("node_modules/@rushstack")).unwrap(); + std::os::unix::fs::symlink(linked.0.join("store/heft"), linked.0.join(LOCAL_HEFT)).unwrap(); + assert_eq!( + linked.select_with_companion("", Some("store/heft")), + linked.this_binary("store/heft") + ); +} + +#[test] +fn companion_of_another_version_is_always_delegated_to() { + let other_version = heft_package_json("9.9.9"); + let fixture = FixtureFolder::with_files(&[ + ("package.json", "{}"), + ("companion/package.json", &other_version), + ]); + assert_eq!( + fixture.select_with_companion("", Some("companion")), + DELEGATE + ); + assert_eq!( + fixture.select_with_companion("", None), + HeftImplementationSelection::ThisBinary(NativeHeftContext { + version_selector_banner: VersionSelectorBanner::Silent, + companion_heft_package_folder: None, + }) + ); +} + +#[test] +fn other_local_heft_versions_layouts_and_missing_entry_points_are_delegated() { + let other_version = heft_package_json("9.9.9"); + let this_version = heft_package_json(HEFT_VERSION_IMPLEMENTED_BY_THIS_BINARY); + let legacy_start = "node_modules/@rushstack/heft/lib/start.js"; + for files in [ + vec![ + (LOCAL_HEFT_START, ""), + (LOCAL_HEFT_PACKAGE_JSON, other_version.as_str()), + ], + vec![ + (legacy_start, ""), + (LOCAL_HEFT_PACKAGE_JSON, this_version.as_str()), + ], + vec![(LOCAL_HEFT_PACKAGE_JSON, this_version.as_str())], + vec![(LOCAL_HEFT_START, "")], + vec![ + (LOCAL_HEFT_START, ""), + (LOCAL_HEFT_PACKAGE_JSON, "{\"version\":\"1.3.1\",}"), + ], + vec![], + ] { + let mut fixture_files = files.clone(); + fixture_files.push(("package.json", HEFT_DEPENDENCY)); + assert_eq!( + FixtureFolder::with_files(&fixture_files).select(""), + DELEGATE, + "{files:?}" + ); + } +} + +#[test] +fn companion_is_found_in_the_repository_and_the_npm_scope_layouts() { + let fixture = + FixtureFolder::with_files(&[("apps/heft/bin/heft", ""), ("scope/heft/bin/heft", "")]); + let repository_binary = fixture.0.join("apps/heft-native/target/release/heft"); + let npm_binary = fixture.0.join("scope/heft-native-linux-x64/bin/heft"); + assert_eq!( + locate_heft_bin_next_to_executable(&repository_binary), + Some(fixture.0.join("apps/heft/bin/heft")) + ); + assert_eq!( + locate_heft_bin_next_to_executable(&npm_binary), + Some(fixture.0.join("scope/heft/bin/heft")) + ); + assert_eq!( + locate_heft_bin_next_to_executable(&fixture.0.join("elsewhere/heft")), + None + ); +} diff --git a/apps/heft/eslint.config.js b/apps/heft/eslint.config.js index e54effd122..cec9e432db 100644 --- a/apps/heft/eslint.config.js +++ b/apps/heft/eslint.config.js @@ -14,5 +14,11 @@ module.exports = [ tsconfigRootDir: __dirname } } + }, + { + files: ['src/host/**/*.ts'], + rules: { + 'headers/header-format': 'off' + } } ]; diff --git a/apps/heft/src/configuration/HeftConfiguration.ts b/apps/heft/src/configuration/HeftConfiguration.ts index fbbf62d0d8..6a0c248276 100644 --- a/apps/heft/src/configuration/HeftConfiguration.ts +++ b/apps/heft/src/configuration/HeftConfiguration.ts @@ -18,7 +18,7 @@ import { type IRigConfig, type RigConfig } from '@rushstack/rig-package'; import { Constants } from '../utilities/Constants'; import type { RigPackageResolver, IRigPackageResolver } from './RigPackageResolver'; import { getSharedLeanPackageJsonLookup, LeanBailError } from './lean/LeanResolution'; -import { tryLoadProjectConfigurationFileLean } from './lean/LeanConfigurationFileSpecification'; +import type { tryLoadProjectConfigurationFileLean } from './lean/LeanConfigurationFileSpecification'; import type { ILeanLoadResult } from './lean/LeanProjectConfigurationFile'; import { LeanRigConfig, tryLoadRigConfigDataLean, type ILeanRigConfigData } from './lean/LeanRigConfig'; @@ -59,16 +59,25 @@ function getRigPackageResolverClass(): typeof RigPackageResolver { * Equivalent to `PackageJsonLookup.instance.tryGetPackageJsonFilePathFor(folderPath)`, without loading * `@rushstack/node-core-library` in the common case. */ -function tryGetPackageJsonFilePathFor(folderPath: string): { packageJsonPath: string | undefined; lean: boolean } { +function tryGetPackageJsonFilePathFor(folderPath: string): { + packageJsonPath: string | undefined; + lean: boolean; +} { let packageFolder: string | undefined; try { packageFolder = getSharedLeanPackageJsonLookup().tryGetPackageFolderFor(folderPath); } catch { // The lean lookup can't guarantee an identical result; use the original implementation - return { packageJsonPath: getPackageJsonLookupInstance().tryGetPackageJsonFilePathFor(folderPath), lean: false }; + return { + packageJsonPath: getPackageJsonLookupInstance().tryGetPackageJsonFilePathFor(folderPath), + lean: false + }; } - return { packageJsonPath: packageFolder ? path.join(packageFolder, 'package.json') : undefined, lean: true }; + return { + packageJsonPath: packageFolder ? path.join(packageFolder, 'package.json') : undefined, + lean: true + }; } /** @@ -250,7 +259,9 @@ export class HeftConfiguration { */ public async _checkForRigAsync(): Promise { if (!this.#rigConfig && !this.#leanRigConfig) { - const leanRigConfigData: ILeanRigConfigData | undefined = tryLoadRigConfigDataLean(this.buildFolderPath); + const leanRigConfigData: ILeanRigConfigData | undefined = tryLoadRigConfigDataLean( + this.buildFolderPath + ); if (leanRigConfigData) { this.#leanRigConfig = new LeanRigConfig(leanRigConfigData, () => this.rigConfig); return; @@ -365,13 +376,18 @@ export class HeftConfiguration { const leanRigConfig: LeanRigConfig | undefined = this.#leanRigConfig; const rigConfig: IRigConfig | undefined = leanRigConfig ?? this.#rigConfig; - const leanResult: ILeanLoadResult | undefined = tryLoadProjectConfigurationFileLean( - options, - this.buildFolderPath, - rigConfig, - // The profile folder of a LeanRigConfig can be resolved without side effects - rigConfig === leanRigConfig - ); + const { tryLoadProjectConfigurationFileLean: tryLoadProjectConfigurationFileLeanFunction } = + require('./lean/LeanConfigurationFileSpecification') as { + tryLoadProjectConfigurationFileLean: typeof tryLoadProjectConfigurationFileLean; + }; + const leanResult: ILeanLoadResult | undefined = + tryLoadProjectConfigurationFileLeanFunction( + options, + this.buildFolderPath, + rigConfig, + // The profile folder of a LeanRigConfig can be resolved without side effects + rigConfig === leanRigConfig + ); if (leanResult) { for (const message of leanResult.debugMessages) { terminal.writeDebugLine(message); diff --git a/apps/heft/src/configuration/HeftPluginConfiguration.ts b/apps/heft/src/configuration/HeftPluginConfiguration.ts index e5fa4e1979..bc92fd71e7 100644 --- a/apps/heft/src/configuration/HeftPluginConfiguration.ts +++ b/apps/heft/src/configuration/HeftPluginConfiguration.ts @@ -13,9 +13,8 @@ import { type IHeftTaskPluginDefinitionJson } from './HeftPluginDefinition'; import type { IHeftConfigurationJsonPluginSpecifier } from '../utilities/CoreConfigFiles'; -import heftPluginSchema from '../schemas/heft-plugin.schema.json'; -import { tryParseJsonLean } from './lean/LeanJson'; -import { tryValidateSchemaObject } from './lean/SchemaFastPath'; +import type { tryParseJsonLean } from './lean/LeanJson'; +import type { tryValidateSchemaObject } from './lean/SchemaFastPath'; export interface IHeftPluginConfigurationJson { lifecyclePlugins?: IHeftLifecyclePluginDefinitionJson[]; @@ -25,7 +24,13 @@ export interface IHeftPluginConfigurationJson { const HEFT_PLUGIN_CONFIGURATION_FILENAME: 'heft-plugin.json' = 'heft-plugin.json'; let _jsonSchema: JsonSchema | undefined; + +function getHeftPluginSchema(): object { + return require('../schemas/heft-plugin.schema.json'); +} + const _pluginConfigurationPromises: Map> = new Map(); +const _seededHeftPluginConfigurationJsonByPackageRoot: Map = new Map(); /** * Loads and validates the heft-plugin.json file without loading ajv, if the result is guaranteed to be identical @@ -39,14 +44,23 @@ function _tryLoadHeftPluginConfigurationJsonLean(filePath: string): IHeftPluginC return undefined; } - const parsed: { value: unknown } | undefined = tryParseJsonLean(fileText); - if (parsed && tryValidateSchemaObject(heftPluginSchema, parsed.value)) { + const { tryParseJsonLean: tryParseJsonLeanFunction } = require('./lean/LeanJson') as { + tryParseJsonLean: typeof tryParseJsonLean; + }; + const parsed: { value: unknown } | undefined = tryParseJsonLeanFunction(fileText); + const { tryValidateSchemaObject: tryValidateSchemaObjectFunction } = require('./lean/SchemaFastPath') as { + tryValidateSchemaObject: typeof tryValidateSchemaObject; + }; + if (parsed && tryValidateSchemaObjectFunction(getHeftPluginSchema(), parsed.value)) { return parsed.value as IHeftPluginConfigurationJson; } } -async function _loadHeftPluginConfigurationJsonAsync(filePath: string): Promise { - const leanResult: IHeftPluginConfigurationJson | undefined = _tryLoadHeftPluginConfigurationJsonLean(filePath); +async function _loadHeftPluginConfigurationJsonAsync( + filePath: string +): Promise { + const leanResult: IHeftPluginConfigurationJson | undefined = + _tryLoadHeftPluginConfigurationJsonLean(filePath); if (leanResult) { return leanResult; } @@ -54,7 +68,7 @@ async function _loadHeftPluginConfigurationJsonAsync(filePath: string): Promise< // Use the original implementation, which produces the canonical errors const { JsonFile, JsonSchema: JsonSchemaClass } = await import('@rushstack/node-core-library'); if (!_jsonSchema) { - _jsonSchema = JsonSchemaClass.fromLoadedObject(heftPluginSchema); + _jsonSchema = JsonSchemaClass.fromLoadedObject(getHeftPluginSchema()); } return await JsonFile.loadAndValidateAsync(filePath, _jsonSchema); @@ -104,7 +118,8 @@ export class HeftPluginConfiguration { if (!heftPluginConfigurationPromise) { heftPluginConfigurationPromise = (async () => { const heftPluginConfigurationJson: IHeftPluginConfigurationJson = - await _loadHeftPluginConfigurationJsonAsync(resolvedHeftPluginConfigurationJsonFilename); + _seededHeftPluginConfigurationJsonByPackageRoot.get(packageRoot) ?? + (await _loadHeftPluginConfigurationJsonAsync(resolvedHeftPluginConfigurationJsonFilename)); return new HeftPluginConfiguration(heftPluginConfigurationJson, packageRoot, packageName); })(); _pluginConfigurationPromises.set(packageRoot, heftPluginConfigurationPromise); @@ -113,6 +128,13 @@ export class HeftPluginConfiguration { return await heftPluginConfigurationPromise; } + public static _seedHeftPluginConfigurationJson( + packageRoot: string, + heftPluginConfigurationJson: IHeftPluginConfigurationJson + ): void { + _seededHeftPluginConfigurationJsonByPackageRoot.set(packageRoot, heftPluginConfigurationJson); + } + /** * Returns a loaded plugin definition for the provided specifier. Specifiers are normally obtained from the * heft.json file. diff --git a/apps/heft/src/configuration/HeftPluginDefinition.ts b/apps/heft/src/configuration/HeftPluginDefinition.ts index 2e220903a6..34331695f3 100644 --- a/apps/heft/src/configuration/HeftPluginDefinition.ts +++ b/apps/heft/src/configuration/HeftPluginDefinition.ts @@ -10,7 +10,7 @@ import type { IHeftPlugin } from '../pluginFramework/IHeftPlugin'; import type { IScopedLogger } from '../pluginFramework/logging/ScopedLogger'; import type { HeftLifecycleSession } from '../pluginFramework/HeftLifecycleSession'; import type { HeftTaskSession } from '../pluginFramework/HeftTaskSession'; -import { tryValidateSchemaFile } from './lean/SchemaFastPath'; +import type { tryValidateSchemaFile } from './lean/SchemaFastPath'; /** * "baseParameter" from heft-plugin.schema.json @@ -335,7 +335,10 @@ export abstract class HeftPluginDefinitionBase { public validateOptions(options: unknown): void { const optionsSchemaPath: string | undefined = this.#optionsSchemaPath; if (optionsSchemaPath) { - if (tryValidateSchemaFile(optionsSchemaPath, options || {})) { + const { tryValidateSchemaFile: tryValidateSchemaFileFunction } = require('./lean/SchemaFastPath') as { + tryValidateSchemaFile: typeof tryValidateSchemaFile; + }; + if (tryValidateSchemaFileFunction(optionsSchemaPath, options || {})) { // Guaranteed to produce the same outcome as the ajv-based validation below return; } diff --git a/apps/heft/src/host/CapturedOutputWorker.ts b/apps/heft/src/host/CapturedOutputWorker.ts new file mode 100644 index 0000000000..2de18db93f --- /dev/null +++ b/apps/heft/src/host/CapturedOutputWorker.ts @@ -0,0 +1,51 @@ +import * as path from 'node:path'; +import { Worker } from 'node:worker_threads'; + +import { + CONTROL_SOCKET_FILE_DESCRIPTOR_INDEX, + CONTROL_STATE_INDEX, + WORKER_STATE_FINISH_REQUESTED, + WORKER_STATE_FORWARDING, + type ICapturedOutputWorkerData +} from './capturedOutputWorkerThread'; + +const FINISH_TIMEOUT_MILLISECONDS: number = 2000; +const CONTROL_SLOT_COUNT: number = 2; + +export class CapturedOutputWorker { + readonly #control: Int32Array = new Int32Array( + new SharedArrayBuffer(CONTROL_SLOT_COUNT * Int32Array.BYTES_PER_ELEMENT) + ); + + public constructor(standardOutputReadFileDescriptor: number, standardErrorReadFileDescriptor: number) { + const capturedOutputWorkerData: ICapturedOutputWorkerData = { + standardOutputReadFileDescriptor, + standardErrorReadFileDescriptor, + controlBuffer: this.#control.buffer as SharedArrayBuffer + }; + const worker: Worker = new Worker(path.join(__dirname, 'capturedOutputWorkerThread.js'), { + workerData: capturedOutputWorkerData, + stdout: true, + stderr: true + }); + worker.on('error', () => undefined); + worker.unref(); + } + + public startForwardingTo(socketFileDescriptor: number): void { + Atomics.store(this.#control, CONTROL_SOCKET_FILE_DESCRIPTOR_INDEX, socketFileDescriptor); + Atomics.store(this.#control, CONTROL_STATE_INDEX, WORKER_STATE_FORWARDING); + Atomics.notify(this.#control, CONTROL_STATE_INDEX); + } + + public finishForwarding(): void { + Atomics.store(this.#control, CONTROL_STATE_INDEX, WORKER_STATE_FINISH_REQUESTED); + Atomics.notify(this.#control, CONTROL_STATE_INDEX); + Atomics.wait( + this.#control, + CONTROL_STATE_INDEX, + WORKER_STATE_FINISH_REQUESTED, + FINISH_TIMEOUT_MILLISECONDS + ); + } +} diff --git a/apps/heft/src/host/HostCommandLine.ts b/apps/heft/src/host/HostCommandLine.ts new file mode 100644 index 0000000000..b2d0986f04 --- /dev/null +++ b/apps/heft/src/host/HostCommandLine.ts @@ -0,0 +1,192 @@ +import os from 'node:os'; + +import { AlreadyReportedError, InternalError } from '@rushstack/node-core-library'; +import { + ConsoleTerminalProvider, + Terminal, + type ITerminal, + type ITerminalProvider +} from '@rushstack/terminal'; + +import { HeftConfiguration } from '../configuration/HeftConfiguration'; +import type { IHeftActionOptions } from '../cli/actions/IHeftAction'; +import type { IHeftCommandLineParserState } from '../cli/HeftCommandLineParser'; +import { MetricsCollector } from '../metrics/MetricsCollector'; +import { + InternalHeftSession, + type IInternalHeftSessionPlanSeed +} from '../pluginFramework/InternalHeftSession'; +import { LoggingManager } from '../pluginFramework/logging/LoggingManager'; +import type { HeftChildReporter } from '../pluginFramework/logging/HeftChildReporter'; +import { Constants } from '../utilities/Constants'; +import type { IHostPlan } from './HostPlan'; +import { createPlanSeed } from './createPlanSeed'; +import { makeRequireStacksMatchTheHeftCommandLine } from './makeRequireStacksMatchTheHeftCommandLine'; +import { tryExecutePlannedCommandAsync } from './tryExecutePlannedCommandAsync'; + +function isDebugToolParameterPresent(processArguments: ReadonlyArray): boolean { + for (let argumentIndex: number = 2; argumentIndex < processArguments.length; argumentIndex++) { + const processArgument: string = processArguments[argumentIndex]; + if (!processArgument.startsWith('-')) { + return false; + } + if (processArgument === Constants.debugParameterLongName) { + return true; + } + } + return false; +} + +function tryInitializeChildReporter(): HeftChildReporter | undefined { + const { + _RUSH_REPORTER_CHILD_FD: childReporterFd, + _RUSH_REPORTER_CHILD_ACK_FD: childReporterAckFd + }: Record = process.env; + if (childReporterFd === undefined && childReporterAckFd === undefined) { + return undefined; + } + const { HeftChildReporter: HeftChildReporterClass } = + require('../pluginFramework/logging/HeftChildReporter') as { + HeftChildReporter: typeof HeftChildReporter; + }; + return HeftChildReporterClass.tryInitialize(); +} + +export class HostCommandLine { + public readonly globalTerminal: ITerminal; + + readonly #plan: IHostPlan; + readonly #planSeed: IInternalHeftSessionPlanSeed | undefined; + readonly #debug: boolean; + readonly #terminalProvider: ITerminalProvider; + readonly #childReporter: HeftChildReporter | undefined; + readonly #loggingManager: LoggingManager; + readonly #metricsCollector: MetricsCollector; + readonly #heftConfiguration: HeftConfiguration; + + public constructor(plan: IHostPlan) { + this.#plan = plan; + this.#debug = isDebugToolParameterPresent(process.argv); + this.#childReporter = tryInitializeChildReporter(); + this.#terminalProvider = + this.#childReporter ?? + new ConsoleTerminalProvider({ + debugEnabled: this.#debug, + verboseEnabled: this.#debug + }); + if (this.#debug && this.#childReporter) { + this.#childReporter.debugEnabled = true; + this.#childReporter.verboseEnabled = true; + } + this.globalTerminal = new Terminal(this.#terminalProvider); + this.#loggingManager = new LoggingManager({ + terminalProvider: this.#terminalProvider, + childReporter: this.#childReporter + }); + if (this.#debug) { + this.#loggingManager.enablePrintStacks(); + InternalError.breakInDebugger = true; + } + + const numberOfCores: number = os.availableParallelism?.() ?? os.cpus().length; + this.#heftConfiguration = HeftConfiguration.initialize({ + cwd: process.cwd(), + terminalProvider: this.#terminalProvider, + numberOfCores + }); + this.#planSeed = createPlanSeed(plan, this.#heftConfiguration.buildFolderPath); + + this.#metricsCollector = new MetricsCollector(); + } + + public async executeAsync(): Promise { + process.exitCode = 1; + + try { + makeRequireStacksMatchTheHeftCommandLine(); + this.#normalizeCwd(); + + const internalHeftSession: InternalHeftSession = await InternalHeftSession.initializeAsync({ + debug: this.#debug, + heftConfiguration: this.#heftConfiguration, + loggingManager: this.#loggingManager, + metricsCollector: this.#metricsCollector, + planSeed: this.#planSeed + }); + + const actionOptions: IHeftActionOptions = { + internalHeftSession: internalHeftSession, + terminal: this.globalTerminal, + loggingManager: this.#loggingManager, + metricsCollector: this.#metricsCollector, + heftConfiguration: this.#heftConfiguration + }; + + const state: IHeftCommandLineParserState = { + internalHeftSession, + childReporter: this.#childReporter, + reportErrorAndSetExitCodeAsync: (error: Error) => this.#reportErrorAndSetExitCodeAsync(error) + }; + + const { command } = this.#plan; + const plannedResult: boolean | undefined = command + ? await tryExecutePlannedCommandAsync(command, actionOptions, state) + : undefined; + if (plannedResult !== undefined) { + return plannedResult; + } + + const args: string[] = process.argv.slice(2); + const { tryExecuteLeanCommandLineAsync } = await import('../cli/LeanHeftCommandLine'); + makeRequireStacksMatchTheHeftCommandLine(); + const leanResult: boolean | undefined = await tryExecuteLeanCommandLineAsync( + args, + actionOptions, + state + ); + if (leanResult !== undefined) { + return leanResult; + } + + const { HeftFullCommandLineParser } = await import('../cli/HeftFullCommandLineParser'); + makeRequireStacksMatchTheHeftCommandLine(); + const fullParser: InstanceType = new HeftFullCommandLineParser(state); + return await fullParser.defineActionsAndExecuteAsync(actionOptions, undefined); + } catch (error) { + await this.#reportErrorAndSetExitCodeAsync(error as Error); + return false; + } + } + + #normalizeCwd(): void { + const buildFolder: string = this.#heftConfiguration.buildFolderPath; + const currentCwd: string = process.cwd(); + if (currentCwd !== buildFolder) { + this.globalTerminal.writeVerboseLine(`CWD is "${currentCwd}". Normalizing to "${buildFolder}".`); + process.chdir(__dirname); + process.chdir(buildFolder); + } + } + + async #reportErrorAndSetExitCodeAsync(error: Error): Promise { + if (!(error instanceof AlreadyReportedError)) { + if (this.#childReporter) { + this.#childReporter.emitDiagnostic(Constants.heftPackageName, error, 'error'); + } else { + this.globalTerminal.writeErrorLine(error.toString()); + } + } + + if (this.#debug) { + this.globalTerminal.writeLine(); + this.globalTerminal.writeErrorLine(error.stack!); + } + + const exitCode: string | number | undefined = process.exitCode; + if (!exitCode || typeof exitCode !== 'number' || exitCode > 0) { + process.exit(exitCode); + } else { + process.exit(1); + } + } +} diff --git a/apps/heft/src/host/HostEntry.ts b/apps/heft/src/host/HostEntry.ts new file mode 100644 index 0000000000..6536d73a9e --- /dev/null +++ b/apps/heft/src/host/HostEntry.ts @@ -0,0 +1,113 @@ +import '../bootstrap/enableStartupCaches'; + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import type { IHostPlan } from './HostPlan'; +import type { HostCommandLine } from './HostCommandLine'; +import { + makeRequireStacksMatchTheHeftCommandLine, + presentTheMainModuleAsTheHeftBin +} from './makeRequireStacksMatchTheHeftCommandLine'; + +const HOST_PLAN_KIND: 'heft-plan' = 'heft-plan'; +const HOST_PLAN_PROTOCOL_VERSION: number = 1; +const DEBUG_TOOL_PARAMETER: '--debug' = '--debug'; + +const HOST_COMMAND_LINE_MODULE_PATH: string = path.join(__dirname, 'HostCommandLine.js'); +const V2_START_MODULE_PATH: string = path.join(__dirname, '../start.js'); +const OWN_PACKAGE_JSON_PATH: string = path.join(__dirname, '../../package.json'); +const PLAN_FILE_DESCRIPTOR_ARGUMENT_PREFIX: string = '--heft-plan-fd='; +const PLAN_FILE_PATH_ARGUMENT_PREFIX: string = '--heft-plan-file='; + +function readAllTextFromFileDescriptorAndCloseIt(fileDescriptor: number): string { + try { + return fs.readFileSync(fileDescriptor, 'utf8'); + } finally { + fs.closeSync(fileDescriptor); + } +} + +function readAllTextFromFileAndDeleteIt(filePath: string): string { + try { + return fs.readFileSync(filePath, 'utf8'); + } finally { + fs.rmSync(filePath, { force: true }); + } +} + +function readHostPlanTextFromProcessArguments(processArguments: ReadonlyArray): string | undefined { + for (const processArgument of processArguments.slice(2)) { + if (processArgument.startsWith(PLAN_FILE_DESCRIPTOR_ARGUMENT_PREFIX)) { + const fileDescriptorText: string = processArgument.slice(PLAN_FILE_DESCRIPTOR_ARGUMENT_PREFIX.length); + return readAllTextFromFileDescriptorAndCloseIt(Number.parseInt(fileDescriptorText, 10)); + } + if (processArgument.startsWith(PLAN_FILE_PATH_ARGUMENT_PREFIX)) { + return readAllTextFromFileAndDeleteIt(processArgument.slice(PLAN_FILE_PATH_ARGUMENT_PREFIX.length)); + } + } + return undefined; +} + +function getOwnHeftVersion(): string { + return (require(OWN_PACKAGE_JSON_PATH) as { version: string }).version; +} + +function hasDebugToolParameter(heftArguments: ReadonlyArray): boolean { + for (const heftArgument of heftArguments) { + if (!heftArgument.startsWith('-')) { + return false; + } + if (heftArgument === DEBUG_TOOL_PARAMETER) { + return true; + } + } + return false; +} + +function isPlanForThisHost(plan: IHostPlan): boolean { + return ( + plan.kind === HOST_PLAN_KIND && + plan.protocolVersion === HOST_PLAN_PROTOCOL_VERSION && + plan.heftVersion === getOwnHeftVersion() && + !hasDebugToolParameter(plan.argv) + ); +} + +function runHostCommandLine(plan: IHostPlan): void { + const { HostCommandLine: HostCommandLineClass } = require(HOST_COMMAND_LINE_MODULE_PATH) as { + HostCommandLine: typeof HostCommandLine; + }; + const hostCommandLine: HostCommandLine = new HostCommandLineClass(plan); + hostCommandLine + .executeAsync() + .then(() => { + process.exit(process.exitCode === undefined ? 0 : process.exitCode); + }) + .catch((error) => { + hostCommandLine.globalTerminal.writeErrorLine(error.toString()); + process.exit(1); + }); +} + +function runHostFromPlanText(planText: string | undefined, hostEntryPath: string): void { + if (planText === undefined) { + process.stderr.write('The Heft plugin host requires --heft-plan-fd= or --heft-plan-file=.\n'); + process.exit(1); + } + const plan: IHostPlan = JSON.parse(planText); + const heftBinPath: string = plan.heftBinPath ?? path.resolve(hostEntryPath, '../../../bin/heft'); + process.argv = [process.argv[0], heftBinPath, ...plan.argv]; + presentTheMainModuleAsTheHeftBin(heftBinPath); + if (plan.cwd !== undefined && plan.cwd !== process.cwd()) { + process.chdir(plan.cwd); + } + if (isPlanForThisHost(plan)) { + runHostCommandLine(plan); + } else { + require(V2_START_MODULE_PATH); + makeRequireStacksMatchTheHeftCommandLine(); + } +} + +runHostFromPlanText(readHostPlanTextFromProcessArguments(process.argv), process.argv[1]); diff --git a/apps/heft/src/host/HostPlan.ts b/apps/heft/src/host/HostPlan.ts new file mode 100644 index 0000000000..69a4edaf1d --- /dev/null +++ b/apps/heft/src/host/HostPlan.ts @@ -0,0 +1,66 @@ +import type { JsonNull } from '@rushstack/node-core-library'; + +import type { IHeftPluginConfigurationJson } from '../configuration/HeftPluginConfiguration'; + +export type HostPlanGraphValue = + | { readonly node: number } + | { readonly json: string | number | boolean | JsonNull } + | { readonly undefined: true }; + +export type HostPlanGraphEntry = readonly [key: string, value: HostPlanGraphValue]; + +export interface IHostPlanGraphAnnotation { + readonly configurationFilePath?: string; + readonly originalValues: ReadonlyArray; + readonly schemaPropertyOriginalValue?: HostPlanGraphValue; +} + +export interface IHostPlanGraphNode { + readonly isArray?: boolean; + readonly entries: ReadonlyArray; + readonly annotation?: IHostPlanGraphAnnotation; +} + +export interface IHostPlanObjectGraph { + readonly rootNode: number; + readonly nodes: ReadonlyArray; +} + +export interface IHostPlanConfiguration { + readonly buildFolderPath?: string; + readonly heftJson?: IHostPlanObjectGraph; + readonly debugMessages?: ReadonlyArray; +} + +export interface IHostPlanPluginManifest { + readonly packageRoot: string; + readonly packageName: string; + readonly manifest: IHeftPluginConfigurationJson; +} + +export type HostPlanParameterValue = readonly [parameterName: string, data: unknown]; + +export interface IHostPlanCommand { + readonly commandName: string; + readonly unaliasedCommandName: string; + readonly actionKind: 'phase' | 'run' | 'clean'; + readonly phaseName?: string; + readonly watch: boolean; + readonly aliasExpansionMessage?: string; + readonly values: ReadonlyArray; + readonly remainder?: ReadonlyArray; + readonly scopedValues?: ReadonlyArray; +} + +export interface IHostPlan { + readonly kind: 'heft-plan'; + readonly protocolVersion: number; + readonly heftVersion: string; + readonly argv: ReadonlyArray; + readonly cwd?: string; + readonly heftBinPath?: string; + readonly config?: IHostPlanConfiguration; + readonly plugins?: ReadonlyArray; + readonly optionsValidated?: boolean; + readonly command?: IHostPlanCommand; +} diff --git a/apps/heft/src/host/WarmHostEntry.ts b/apps/heft/src/host/WarmHostEntry.ts new file mode 100644 index 0000000000..d62521a738 --- /dev/null +++ b/apps/heft/src/host/WarmHostEntry.ts @@ -0,0 +1,150 @@ +import '../bootstrap/enableStartupCaches'; + +import * as fs from 'node:fs'; +import * as net from 'node:net'; +import * as path from 'node:path'; + +import { CapturedOutputWorker } from './CapturedOutputWorker'; +import { handleWarmHostConnection, type IWarmHostContext } from './warmHostRun'; +import { + getCapturePipePaths, + isFolderPrivateToCurrentUser, + openCapturePipesForReading, + parseWarmHostArguments, + removeCapturePipes, + startWarmHostProcess, + type ICapturePipesOfThisHost, + type IWarmHostArguments +} from './warmHostProcess'; + +const DEFAULT_IDLE_TIMEOUT_MILLISECONDS: number = 900000; +const PRELOADED_MODULES_AND_THEIR_LAZY_EXPORTS: ReadonlyArray]> = [ + ['./HostCommandLine', []], + ['../cli/HeftActionRunner', []], + ['../cli/actions/PhaseScoping', []], + ['../cli/CliConstants', []], + ['../operations/generateOperations', []], + ['../operations/OperationExecutionManager', []], + ['../pluginFramework/HeftPhaseSession', []], + ['../pluginFramework/HeftTaskSession', []], + ['../pluginFramework/logging/ScopedLogger', []], + ['@rushstack/heft-config-file/lib/ConfigurationFileAnnotation', []], + ['@rushstack/node-core-library', ['AlreadyReportedError', 'Async', 'InternalError', 'Text']], + ['@rushstack/terminal', ['Colorize', 'ConsoleTerminalProvider', 'PrefixProxyTerminalProvider', 'Terminal']], + ['tapable/lib/AsyncParallelHook', []], + ['tapable/lib/AsyncSeriesWaterfallHook', []], + ['tapable/lib/SyncHook', []] +]; + +function preloadHostModules(): void { + for (const [moduleRequest, lazyExportNames] of PRELOADED_MODULES_AND_THEIR_LAZY_EXPORTS) { + const moduleExports: Record = require(moduleRequest); + for (const lazyExportName of lazyExportNames) { + void moduleExports[lazyExportName]; + } + } + void process.stdout; + void process.stderr; +} + +function getFileIdentity(filePath: string): string | undefined { + try { + const fileStats: fs.BigIntStats = fs.statSync(filePath, { bigint: true }); + return `${fileStats.dev}:${fileStats.ino}:${fileStats.size}:${fileStats.mtimeNs}:${fileStats.ctimeNs}`; + } catch { + return undefined; + } +} + +function snapshotLoadedFileIdentities(): ReadonlyMap { + const fileIdentities: Map = new Map(); + for (const filePath of [...Object.keys(require.cache), process.execPath]) { + fileIdentities.set(filePath, getFileIdentity(filePath)); + } + return fileIdentities; +} + +function getIdleTimeoutMilliseconds(): number { + const configuredIdleTimeout: number = Number(process.env.HEFT_WARM_HOST_IDLE_MS); + return Number.isFinite(configuredIdleTimeout) && configuredIdleTimeout > 0 + ? configuredIdleTimeout + : DEFAULT_IDLE_TIMEOUT_MILLISECONDS; +} + +function listenForOneRun( + socketPath: string, + context: IWarmHostContext, + mayReplaceStaleSocket: boolean +): void { + const server: net.Server = net.createServer(); + let idleTimer: NodeJS.Timeout | undefined; + const stopListening: () => void = () => { + clearTimeout(idleTimer); + fs.rmSync(socketPath, { force: true }); + server.close(); + }; + server.on('connection', (socket: net.Socket) => + handleWarmHostConnection(socket, { context, stopListening }) + ); + server.on('error', (error: NodeJS.ErrnoException) => { + if (error.code !== 'EADDRINUSE' || !mayReplaceStaleSocket) { + process.exit(0); + } + const probe: net.Socket = net.connect(socketPath); + probe.on('connect', () => process.exit(0)); + probe.on('error', () => { + fs.rmSync(socketPath, { force: true }); + listenForOneRun(socketPath, context, false); + }); + }); + server.listen(socketPath, () => { + idleTimer = setTimeout(() => { + fs.rmSync(socketPath, { force: true }); + removeCapturePipes(context.capturePipePaths); + process.exit(0); + }, getIdleTimeoutMilliseconds()); + }); +} + +function startPrewarmedHost(socketPath: string, capturePipesBasePath: string): void { + const capturePipesOfThisHost: ICapturePipesOfThisHost = openCapturePipesForReading( + getCapturePipePaths(capturePipesBasePath) + ); + const [standardOutputReadEnd, standardErrorReadEnd] = capturePipesOfThisHost.readEnds; + const outputWorker: CapturedOutputWorker = new CapturedOutputWorker( + standardOutputReadEnd, + standardErrorReadEnd + ); + const preloadEnvironment: NodeJS.ProcessEnv = { ...process.env }; + preloadHostModules(); + const loadedFileIdentities: ReadonlyMap = snapshotLoadedFileIdentities(); + const context: IWarmHostContext = { + warmHostEntryPath: __filename, + socketPath, + heftVersion: (require('../../package.json') as { version: string }).version, + preloadEnvironment, + outputWorker, + capturePipePaths: capturePipesOfThisHost.capturePipePaths, + hasStaleModuleFiles: () => { + for (const [filePath, fileIdentity] of loadedFileIdentities) { + if (getFileIdentity(filePath) !== fileIdentity) { + return true; + } + } + return false; + } + }; + listenForOneRun(socketPath, context, true); +} + +const parsedWarmHostArguments: IWarmHostArguments | undefined = parseWarmHostArguments(process.argv); +if ( + !parsedWarmHostArguments || + !isFolderPrivateToCurrentUser(path.dirname(parsedWarmHostArguments.socketPath)) +) { + process.exit(1); +} else if (parsedWarmHostArguments.capturePipesBasePath === undefined) { + startWarmHostProcess(__filename, parsedWarmHostArguments.socketPath, process.env); +} else { + startPrewarmedHost(parsedWarmHostArguments.socketPath, parsedWarmHostArguments.capturePipesBasePath); +} diff --git a/apps/heft/src/host/capturedOutputWorkerThread.ts b/apps/heft/src/host/capturedOutputWorkerThread.ts new file mode 100644 index 0000000000..f5cfcde74e --- /dev/null +++ b/apps/heft/src/host/capturedOutputWorkerThread.ts @@ -0,0 +1,80 @@ +import * as fs from 'node:fs'; +import { workerData } from 'node:worker_threads'; + +import { + STANDARD_ERROR_FRAME_TYPE, + STANDARD_OUTPUT_FRAME_TYPE, + writeFrameSynchronously +} from './warmHostFrames'; + +export interface ICapturedOutputWorkerData { + readonly standardOutputReadFileDescriptor: number; + readonly standardErrorReadFileDescriptor: number; + readonly controlBuffer: SharedArrayBuffer; +} + +export const CONTROL_STATE_INDEX: number = 0; +export const CONTROL_SOCKET_FILE_DESCRIPTOR_INDEX: number = 1; +export const WORKER_STATE_WAITING_FOR_A_RUN: number = 0; +export const WORKER_STATE_FORWARDING: number = 1; +export const WORKER_STATE_FINISH_REQUESTED: number = 2; +export const WORKER_STATE_FINISHED: number = 3; + +const DRAIN_BUFFER_BYTES: number = 64 * 1024; +const IDLE_POLL_MILLISECONDS: number = 1; + +function forwardCapturedOutput(data: ICapturedOutputWorkerData): void { + const control: Int32Array = new Int32Array(data.controlBuffer); + const drainBuffer: Buffer = Buffer.allocUnsafe(DRAIN_BUFFER_BYTES); + Atomics.wait(control, CONTROL_STATE_INDEX, WORKER_STATE_WAITING_FOR_A_RUN); + const socketFileDescriptor: number = Atomics.load(control, CONTROL_SOCKET_FILE_DESCRIPTOR_INDEX); + let forwardingHasFailed: boolean = false; + + function drain(readFileDescriptor: number, frameType: number): boolean { + let forwardedAnything: boolean = false; + for (;;) { + let readByteCount: number; + try { + readByteCount = fs.readSync(readFileDescriptor, drainBuffer, 0, drainBuffer.length, null); + } catch { + return forwardedAnything; + } + if (readByteCount === 0) { + return forwardedAnything; + } + forwardedAnything = true; + if (!forwardingHasFailed) { + try { + writeFrameSynchronously(socketFileDescriptor, frameType, drainBuffer.subarray(0, readByteCount)); + } catch { + forwardingHasFailed = true; + } + } + } + } + + for (;;) { + const state: number = Atomics.load(control, CONTROL_STATE_INDEX); + const forwardedStandardOutput: boolean = drain( + data.standardOutputReadFileDescriptor, + STANDARD_OUTPUT_FRAME_TYPE + ); + const forwardedStandardError: boolean = drain( + data.standardErrorReadFileDescriptor, + STANDARD_ERROR_FRAME_TYPE + ); + if (forwardedStandardOutput || forwardedStandardError) { + continue; + } + if (state === WORKER_STATE_FINISH_REQUESTED) { + Atomics.store(control, CONTROL_STATE_INDEX, WORKER_STATE_FINISHED); + Atomics.notify(control, CONTROL_STATE_INDEX); + return; + } + Atomics.wait(control, CONTROL_STATE_INDEX, state, IDLE_POLL_MILLISECONDS); + } +} + +if (workerData) { + forwardCapturedOutput(workerData as ICapturedOutputWorkerData); +} diff --git a/apps/heft/src/host/createPlanSeed.ts b/apps/heft/src/host/createPlanSeed.ts new file mode 100644 index 0000000000..fd7e8edac0 --- /dev/null +++ b/apps/heft/src/host/createPlanSeed.ts @@ -0,0 +1,114 @@ +import type { CONFIGURATION_FILE_FIELD_ANNOTATION } from '@rushstack/heft-config-file/lib/ConfigurationFileAnnotation'; + +import type { HeftPluginConfiguration } from '../configuration/HeftPluginConfiguration'; +import type { IInternalHeftSessionPlanSeed } from '../pluginFramework/InternalHeftSession'; +import type { IHeftConfigurationJson } from '../utilities/CoreConfigFiles'; +import type { + HostPlanGraphEntry, + HostPlanGraphValue, + IHostPlan, + IHostPlanConfiguration, + IHostPlanGraphAnnotation, + IHostPlanGraphNode, + IHostPlanObjectGraph +} from './HostPlan'; + +interface IRestoredAnnotation { + configurationFilePath: string | undefined; + originalValues: Record; + schemaPropertyOriginalValue?: unknown; +} + +type RestoredNode = Record; + +function createEmptyNode(node: IHostPlanGraphNode): RestoredNode { + return (node.isArray ? [] : {}) as RestoredNode; +} + +function restoreHostPlanObjectGraph(graph: IHostPlanObjectGraph, annotationSymbol: symbol): unknown { + const { nodes } = graph; + const restoredNodes: RestoredNode[] = nodes.map(createEmptyNode); + + function resolveValue(value: HostPlanGraphValue): unknown { + if ('node' in value) { + const restoredNode: RestoredNode | undefined = restoredNodes[value.node]; + if (restoredNode === undefined) { + throw new Error(`The heft plan references the unknown node ${value.node}.`); + } + return restoredNode; + } + return 'json' in value ? value.json : undefined; + } + + function assignEntries(target: Record, entries: ReadonlyArray): void { + for (const [key, value] of entries) { + target[key] = resolveValue(value); + } + } + + function restoreAnnotation(annotation: IHostPlanGraphAnnotation): IRestoredAnnotation { + const originalValues: Record = {}; + assignEntries(originalValues, annotation.originalValues); + const restoredAnnotation: IRestoredAnnotation = { + configurationFilePath: annotation.configurationFilePath ?? undefined, + originalValues + }; + if (annotation.schemaPropertyOriginalValue !== undefined) { + restoredAnnotation.schemaPropertyOriginalValue = resolveValue(annotation.schemaPropertyOriginalValue); + } + return restoredAnnotation; + } + + for (let nodeIndex: number = 0; nodeIndex < nodes.length; nodeIndex++) { + const node: IHostPlanGraphNode = nodes[nodeIndex]; + const restoredNode: RestoredNode = restoredNodes[nodeIndex]; + assignEntries(restoredNode as Record, node.entries); + if (node.annotation) { + restoredNode[annotationSymbol] = restoreAnnotation(node.annotation); + } + } + + return resolveValue({ node: graph.rootNode }); +} + +function getConfigurationFileFieldAnnotationSymbol(): typeof CONFIGURATION_FILE_FIELD_ANNOTATION { + return ( + require('@rushstack/heft-config-file/lib/ConfigurationFileAnnotation') as { + CONFIGURATION_FILE_FIELD_ANNOTATION: typeof CONFIGURATION_FILE_FIELD_ANNOTATION; + } + ).CONFIGURATION_FILE_FIELD_ANNOTATION; +} + +function seedPluginManifests(plan: IHostPlan): void { + const { plugins } = plan; + if (!plugins?.length) { + return; + } + const { HeftPluginConfiguration: HeftPluginConfigurationClass } = + require('../configuration/HeftPluginConfiguration') as { + HeftPluginConfiguration: typeof HeftPluginConfiguration; + }; + for (const { packageRoot, manifest } of plugins) { + HeftPluginConfigurationClass._seedHeftPluginConfigurationJson(packageRoot, manifest); + } +} + +export function createPlanSeed( + plan: IHostPlan, + buildFolderPath: string +): IInternalHeftSessionPlanSeed | undefined { + const configuration: IHostPlanConfiguration | undefined = plan.config; + if (!configuration?.heftJson || configuration.buildFolderPath !== buildFolderPath) { + return undefined; + } + const heftConfigurationJson: IHeftConfigurationJson = restoreHostPlanObjectGraph( + configuration.heftJson, + getConfigurationFileFieldAnnotationSymbol() + ) as IHeftConfigurationJson; + seedPluginManifests(plan); + return { + heftConfigurationJson, + debugMessages: configuration.debugMessages ?? [], + pluginOptionsAreValidated: plan.optionsValidated === true && plan.plugins !== undefined + }; +} diff --git a/apps/heft/src/host/makeRequireStacksMatchTheHeftCommandLine.ts b/apps/heft/src/host/makeRequireStacksMatchTheHeftCommandLine.ts new file mode 100644 index 0000000000..1bb9802709 --- /dev/null +++ b/apps/heft/src/host/makeRequireStacksMatchTheHeftCommandLine.ts @@ -0,0 +1,115 @@ +import * as fs from 'node:fs'; +import nodeModule from 'node:module'; +import * as path from 'node:path'; + +interface IModuleWithParent { + readonly filename: string; + parent: unknown; +} + +type ModuleConstructor = new (id: string, parent: NodeModule | undefined) => NodeModule; + +const HEFT_PACKAGE_FOLDER: string = path.resolve(__dirname, '../..'); +const PENDING_DEPRECATION_FLAG: string = '--pending-deprecation'; + +let _standInParentsByHostModuleFilename: ReadonlyMap | undefined; +let _versionSelectorStandInModule: NodeModule | undefined; + +function createStandInModule( + pathInsideHeftPackage: string, + parentModule: NodeModule | undefined +): NodeModule { + const ModuleClass: ModuleConstructor = nodeModule as unknown as ModuleConstructor; + return new ModuleClass(path.join(HEFT_PACKAGE_FOLDER, pathInsideHeftPackage), parentModule); +} + +function getStandInParentsByHostModuleFilename(): ReadonlyMap { + if (!_standInParentsByHostModuleFilename) { + const binModule: NodeModule = createStandInModule('bin/heft', undefined); + const versionSelectorModule: NodeModule = createStandInModule( + 'lib-commonjs/startWithVersionSelector.js', + binModule + ); + _versionSelectorStandInModule = versionSelectorModule; + const startModule: NodeModule = createStandInModule('lib-commonjs/start.js', versionSelectorModule); + const commandLineParserModule: NodeModule = createStandInModule( + 'lib-commonjs/cli/HeftCommandLineParser.js', + startModule + ); + const leanCommandLineModule: NodeModule = createStandInModule( + 'lib-commonjs/cli/LeanHeftCommandLine.js', + commandLineParserModule + ); + const standInParentsByHostModuleName: ReadonlyArray = [ + ['HostEntry.js', startModule], + ['WarmHostEntry.js', startModule], + ['warmHostRun.js', startModule], + ['HostCommandLine.js', commandLineParserModule], + ['createPlanSeed.js', commandLineParserModule], + ['tryExecutePlannedCommandAsync.js', leanCommandLineModule], + ['plannedActions.js', leanCommandLineModule] + ]; + _standInParentsByHostModuleFilename = new Map( + standInParentsByHostModuleName.map(([hostModuleName, standInParent]) => [ + path.join(__dirname, hostModuleName), + standInParent + ]) + ); + } + return _standInParentsByHostModuleFilename; +} + +function isPendingDeprecationEnabled(): boolean { + return ( + process.execArgv.includes(PENDING_DEPRECATION_FLAG) || + (process.env.NODE_OPTIONS ?? '').includes(PENDING_DEPRECATION_FLAG) || + process.env.NODE_PENDING_DEPRECATION === '1' + ); +} + +export function makeRequireStacksMatchTheHeftCommandLine(): void { + if (isPendingDeprecationEnabled()) { + return; + } + const standInParentsByHostModuleFilename: ReadonlyMap = + getStandInParentsByHostModuleFilename(); + const startModuleFilename: string = path.join(HEFT_PACKAGE_FOLDER, 'lib-commonjs/start.js'); + for (const cachedModule of Object.values(require.cache)) { + const moduleWithParent: IModuleWithParent | undefined = cachedModule as unknown as + | IModuleWithParent + | undefined; + const parentFilename: string | undefined = (moduleWithParent?.parent as IModuleWithParent | undefined) + ?.filename; + const standInParent: NodeModule | undefined = + parentFilename === undefined ? undefined : standInParentsByHostModuleFilename.get(parentFilename); + if ( + moduleWithParent && + standInParent && + !standInParentsByHostModuleFilename.has(moduleWithParent.filename) + ) { + moduleWithParent.parent = + moduleWithParent.filename === startModuleFilename ? _versionSelectorStandInModule : standInParent; + } + } +} + +function tryGetRealPath(filePath: string): string { + try { + return fs.realpathSync(filePath); + } catch { + return filePath; + } +} + +export function presentTheMainModuleAsTheHeftBin(heftBinPath: string): void { + const mainModule: NodeModule | undefined = require.main; + if (mainModule) { + const heftBinRealPath: string = tryGetRealPath(heftBinPath); + const heftBinFolder: string = path.dirname(heftBinRealPath); + const moduleWithNodeModulePaths: { _nodeModulePaths(fromFolder: string): string[] } = + nodeModule as unknown as { _nodeModulePaths(fromFolder: string): string[] }; + mainModule.filename = heftBinRealPath; + (mainModule as { path: string }).path = heftBinFolder; + mainModule.paths = moduleWithNodeModulePaths._nodeModulePaths(heftBinFolder); + } +} diff --git a/apps/heft/src/host/plannedActions.ts b/apps/heft/src/host/plannedActions.ts new file mode 100644 index 0000000000..34fbc6db79 --- /dev/null +++ b/apps/heft/src/host/plannedActions.ts @@ -0,0 +1,118 @@ +import type { CommandLineParameter as PublicCommandLineParameter } from '@rushstack/ts-command-line'; +import type { CommandLineParameter } from '@rushstack/ts-command-line/lib/parameters/BaseClasses'; + +import { LeanParameterProvider } from '../cli/LeanParameterProvider'; +import type { IHeftAction, IHeftActionOptions } from '../cli/actions/IHeftAction'; +import type { HeftPhase } from '../pluginFramework/HeftPhase'; +import type { IHeftConfigurationJsonActionReference } from '../utilities/CoreConfigFiles'; +import type { HostPlanParameterValue, IHostPlanCommand } from './HostPlan'; + +const HEFT_TOOL_FILENAME: 'heft' = 'heft'; + +export class HostHeftAction extends LeanParameterProvider { + public readonly actionName: string; + public readonly watch: boolean; + #getSelectedPhases: (() => ReadonlySet) | undefined; + #selectedPhases: ReadonlySet | undefined; + #scopedParameters: ReadonlyArray | undefined; + + public constructor(actionName: string, watch: boolean) { + super(); + this.actionName = actionName; + this.watch = watch; + } + + public get selectedPhases(): ReadonlySet { + if (!this.#selectedPhases) { + this.#selectedPhases = this.#getSelectedPhases!(); + } + return this.#selectedPhases; + } + + public setSelectedPhasesFactory(getSelectedPhases: () => ReadonlySet): void { + this.#getSelectedPhases = getSelectedPhases; + } + + public override get parameters(): ReadonlyArray { + if (this.#scopedParameters) { + return [...super.parameters, ...this.#scopedParameters]; + } else { + return super.parameters; + } + } + + public setScopedParameters(scopedParameters: ReadonlyArray): void { + this.#scopedParameters = scopedParameters; + } + + public asHeftAction(): IHeftAction { + return this as unknown as IHeftAction; + } +} + +export function selectPhaseAndItsDependencies(phase: HeftPhase): Set { + const selectedPhases: Set = new Set([phase]); + for (const selectedPhase of selectedPhases) { + for (const dependencyPhase of selectedPhase.dependencyPhases) { + selectedPhases.add(dependencyPhase); + } + } + return selectedPhases; +} + +export function tryApplyPlannedParameterValues( + parameterProvider: LeanParameterProvider, + plannedValues: ReadonlyArray, + remainder?: ReadonlyArray +): boolean { + const definedParameters: ReadonlyArray = + parameterProvider.parameters as unknown as ReadonlyArray; + if (definedParameters.length !== plannedValues.length) { + return false; + } + const dataByParameter: Map = new Map(); + for (let parameterIndex: number = 0; parameterIndex < definedParameters.length; parameterIndex++) { + const definedParameter: CommandLineParameter = definedParameters[parameterIndex]; + const [parameterName, data] = plannedValues[parameterIndex]; + if ((definedParameter.scopedLongName ?? definedParameter.longName) !== parameterName) { + return false; + } + if (data !== null && data !== undefined) { + dataByParameter.set(definedParameter, data); + } + } + return parameterProvider.tryApplyValues(dataByParameter, remainder && [...remainder]); +} + +function getAliasExpansionMessage( + aliasName: string, + actionReference: IHeftConfigurationJsonActionReference +): string { + const defaultParametersString: string = (actionReference.defaultParameters ?? []).join(' '); + const expandedCommand: string = `${HEFT_TOOL_FILENAME} ${actionReference.actionName}${ + defaultParametersString ? ` ${defaultParametersString}` : '' + }`; + return `The "${HEFT_TOOL_FILENAME} ${aliasName}" alias was expanded to "${expandedCommand}".`; +} + +export function tryGetAliasExpansionMessage( + command: IHostPlanCommand, + actionOptions: IHeftActionOptions +): string | false | undefined { + if (command.commandName === command.unaliasedCommandName) { + return undefined; + } + const actionReference: IHeftConfigurationJsonActionReference | undefined = + actionOptions.internalHeftSession.actionReferencesByAlias.get(command.commandName); + if (actionReference?.actionName !== command.unaliasedCommandName) { + return false; + } + const aliasExpansionMessage: string = getAliasExpansionMessage(command.commandName, actionReference); + if ( + command.aliasExpansionMessage !== undefined && + command.aliasExpansionMessage !== aliasExpansionMessage + ) { + return false; + } + return aliasExpansionMessage; +} diff --git a/apps/heft/src/host/tryExecutePlannedCommandAsync.ts b/apps/heft/src/host/tryExecutePlannedCommandAsync.ts new file mode 100644 index 0000000000..b2c71941d4 --- /dev/null +++ b/apps/heft/src/host/tryExecutePlannedCommandAsync.ts @@ -0,0 +1,139 @@ +import type { HeftActionRunner } from '../cli/HeftActionRunner'; +import type { IHeftActionOptions } from '../cli/actions/IHeftAction'; +import type { IHeftCommandLineParserState } from '../cli/HeftCommandLineParser'; +import type { IScopingParameters } from '../cli/actions/PhaseScoping'; +import { LeanParameterProvider } from '../cli/LeanParameterProvider'; +import type { HeftPhase } from '../pluginFramework/HeftPhase'; +import type { IHostPlanCommand } from './HostPlan'; +import { makeRequireStacksMatchTheHeftCommandLine } from './makeRequireStacksMatchTheHeftCommandLine'; +import { + HostHeftAction, + selectPhaseAndItsDependencies, + tryApplyPlannedParameterValues, + tryGetAliasExpansionMessage +} from './plannedActions'; + +async function createActionRunnerAsync( + action: HostHeftAction, + actionOptions: IHeftActionOptions +): Promise { + const { HeftActionRunner: HeftActionRunnerClass } = await import('../cli/HeftActionRunner'); + return new HeftActionRunnerClass({ action: action.asHeftAction(), ...actionOptions }); +} + +async function tryPreparePlannedPhaseActionAsync( + command: IHostPlanCommand, + actionOptions: IHeftActionOptions +): Promise { + const phase: HeftPhase | undefined = + command.phaseName === undefined + ? undefined + : actionOptions.internalHeftSession.phasesByName.get(command.phaseName); + if (!phase || command.unaliasedCommandName !== `${phase.phaseName}${command.watch ? '-watch' : ''}`) { + return undefined; + } + const action: HostHeftAction = new HostHeftAction(command.unaliasedCommandName, command.watch); + action.setSelectedPhasesFactory(() => selectPhaseAndItsDependencies(phase)); + const actionRunner: HeftActionRunner = await createActionRunnerAsync(action, actionOptions); + actionRunner.defineParameters(); + return tryApplyPlannedParameterValues(action, command.values) ? actionRunner : undefined; +} + +function tryEvaluateSelectedPhases( + action: HostHeftAction, + scopingParameters: IScopingParameters, + actionOptions: IHeftActionOptions +): boolean { + const { phasesByName } = actionOptions.internalHeftSession; + const { onlyParameter, toParameter, toExceptParameter } = scopingParameters; + for (const scopingParameter of [onlyParameter, toParameter, toExceptParameter]) { + for (const phaseName of scopingParameter.values) { + if (!phasesByName.has(phaseName)) { + return false; + } + } + } + try { + return action.selectedPhases.size !== 0; + } catch { + return false; + } +} + +async function tryPreparePlannedRunActionAsync( + command: IHostPlanCommand, + actionOptions: IHeftActionOptions +): Promise { + if (command.unaliasedCommandName !== (command.watch ? 'run-watch' : 'run')) { + return undefined; + } + const { internalHeftSession, terminal } = actionOptions; + const { definePhaseScopingParameters, expandPhases } = await import('../cli/actions/PhaseScoping'); + const { SCOPED_ACTION_REMAINDER_DESCRIPTION } = await import('../cli/CliConstants'); + const action: HostHeftAction = new HostHeftAction(command.unaliasedCommandName, command.watch); + action.defineCommandLineRemainder({ description: SCOPED_ACTION_REMAINDER_DESCRIPTION }); + const scopingParameters: IScopingParameters = definePhaseScopingParameters(action); + const { onlyParameter, toParameter, toExceptParameter } = scopingParameters; + action.setSelectedPhasesFactory(() => + expandPhases(onlyParameter, toParameter, toExceptParameter, internalHeftSession, terminal) + ); + const actionRunner: HeftActionRunner = await createActionRunnerAsync(action, actionOptions); + const remainder: ReadonlyArray = command.remainder ?? []; + if ( + !tryApplyPlannedParameterValues(action, command.values, remainder) || + !tryEvaluateSelectedPhases(action, scopingParameters, actionOptions) + ) { + return undefined; + } + const scopedParameterProvider: LeanParameterProvider = new LeanParameterProvider(); + actionRunner.defineParameters(scopedParameterProvider.asCommandLineParameterProvider()); + if ( + (remainder.length && remainder[0] !== '--') || + !tryApplyPlannedParameterValues(scopedParameterProvider, command.scopedValues ?? []) + ) { + return undefined; + } + action.setScopedParameters(scopedParameterProvider.parameters); + return actionRunner; +} + +export async function tryExecutePlannedCommandAsync( + command: IHostPlanCommand, + actionOptions: IHeftActionOptions, + state: IHeftCommandLineParserState +): Promise { + const aliasExpansionMessage: string | false | undefined = tryGetAliasExpansionMessage( + command, + actionOptions + ); + if (aliasExpansionMessage === false) { + return undefined; + } + const actionRunner: HeftActionRunner | undefined = + command.actionKind === 'phase' + ? await tryPreparePlannedPhaseActionAsync(command, actionOptions) + : command.actionKind === 'run' + ? await tryPreparePlannedRunActionAsync(command, actionOptions) + : undefined; + if (!actionRunner) { + return undefined; + } + makeRequireStacksMatchTheHeftCommandLine(); + + try { + state.internalHeftSession.parsedCommandLine = { + commandName: command.commandName, + unaliasedCommandName: command.unaliasedCommandName + }; + state.childReporter?.setCommandName(command.commandName); + if (aliasExpansionMessage) { + actionOptions.terminal.writeLine(aliasExpansionMessage); + } + await actionRunner.executeAsync(); + } catch (error) { + await state.reportErrorAndSetExitCodeAsync(error as Error); + } + + process.exitCode = 0; + return true; +} diff --git a/apps/heft/src/host/warmHostFrames.ts b/apps/heft/src/host/warmHostFrames.ts new file mode 100644 index 0000000000..8a55c514c0 --- /dev/null +++ b/apps/heft/src/host/warmHostFrames.ts @@ -0,0 +1,80 @@ +import * as fs from 'node:fs'; + +export const RUN_FRAME_TYPE: number = 0x01; +export const SIGNAL_FRAME_TYPE: number = 0x02; +export const ACCEPT_FRAME_TYPE: number = 0x10; +export const REFUSE_FRAME_TYPE: number = 0x11; +export const STANDARD_OUTPUT_FRAME_TYPE: number = 0x12; +export const STANDARD_ERROR_FRAME_TYPE: number = 0x13; +export const EXIT_FRAME_TYPE: number = 0x14; + +const FRAME_HEADER_BYTES: number = 5; +const MAXIMUM_INCOMING_FRAME_PAYLOAD_BYTES: number = 16 * 1024 * 1024; +const WAIT_FOR_WRITABLE_SOCKET_MILLISECONDS: number = 1; + +export interface IWarmHostFrame { + readonly frameType: number; + readonly payload: Buffer; +} + +export class WarmHostFrameReader { + #pendingBytes: Buffer = Buffer.alloc(0); + + public pushBytes(incomingBytes: Buffer): IWarmHostFrame[] { + this.#pendingBytes = + this.#pendingBytes.length === 0 ? incomingBytes : Buffer.concat([this.#pendingBytes, incomingBytes]); + const completeFrames: IWarmHostFrame[] = []; + while (this.#pendingBytes.length >= FRAME_HEADER_BYTES) { + const payloadLength: number = this.#pendingBytes.readUInt32LE(0); + if (payloadLength > MAXIMUM_INCOMING_FRAME_PAYLOAD_BYTES) { + throw new Error(`A warm host frame of ${payloadLength} bytes exceeds the limit.`); + } + const frameLength: number = FRAME_HEADER_BYTES + payloadLength; + if (this.#pendingBytes.length < frameLength) { + break; + } + completeFrames.push({ + frameType: this.#pendingBytes[4], + payload: Buffer.from(this.#pendingBytes.subarray(FRAME_HEADER_BYTES, frameLength)) + }); + this.#pendingBytes = this.#pendingBytes.subarray(frameLength); + } + return completeFrames; + } +} + +function sleepSynchronously(milliseconds: number): void { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds); +} + +function writeAllBytesSynchronously(socketFileDescriptor: number, bytes: Uint8Array): void { + let writtenByteCount: number = 0; + while (writtenByteCount < bytes.length) { + try { + writtenByteCount += fs.writeSync(socketFileDescriptor, bytes, writtenByteCount); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EAGAIN') { + throw error; + } + sleepSynchronously(WAIT_FOR_WRITABLE_SOCKET_MILLISECONDS); + } + } +} + +export function writeFrameSynchronously( + socketFileDescriptor: number, + frameType: number, + payload: Uint8Array +): void { + const frameHeader: Buffer = Buffer.alloc(FRAME_HEADER_BYTES); + frameHeader.writeUInt32LE(payload.length, 0); + frameHeader[4] = frameType; + writeAllBytesSynchronously(socketFileDescriptor, frameHeader); + writeAllBytesSynchronously(socketFileDescriptor, payload); +} + +export function createExitFramePayload(exitCode: number): Buffer { + const exitFramePayload: Buffer = Buffer.alloc(4); + exitFramePayload.writeInt32LE(exitCode, 0); + return exitFramePayload; +} diff --git a/apps/heft/src/host/warmHostProcess.ts b/apps/heft/src/host/warmHostProcess.ts new file mode 100644 index 0000000000..4ec22cfda1 --- /dev/null +++ b/apps/heft/src/host/warmHostProcess.ts @@ -0,0 +1,170 @@ +import { execFileSync, spawn } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +const SOCKET_ARGUMENT_PREFIX: string = '--heft-warm-host-socket='; +const CAPTURE_PIPES_ARGUMENT_PREFIX: string = '--heft-warm-host-capture='; +const MODE_MODULUS_THAT_ISOLATES_GROUP_AND_OTHER_PERMISSIONS: number = 0o100; +const READY_MARKER_WAIT_MILLISECONDS: number = 5000; +const READY_MARKER_POLL_MILLISECONDS: number = 5; +const READ_ONLY_NON_BLOCKING_OPEN_FLAGS: number = fs.constants.O_RDONLY + fs.constants.O_NONBLOCK; +const CAPTURE_FILE_NAME_PATTERN: RegExp = /\.sock\.(\d+)\.\d+\.(out|err|ready)$/; + +export interface IWarmHostArguments { + readonly socketPath: string; + readonly capturePipesBasePath: string | undefined; +} + +export interface ICapturePipePaths { + readonly standardOutputPipePath: string; + readonly standardErrorPipePath: string; + readonly readyMarkerPath: string; +} + +function findArgumentValue(processArguments: ReadonlyArray, prefix: string): string | undefined { + for (const processArgument of processArguments) { + if (processArgument.startsWith(prefix)) { + return processArgument.slice(prefix.length); + } + } + return undefined; +} + +export function parseWarmHostArguments( + processArguments: ReadonlyArray +): IWarmHostArguments | undefined { + const socketPath: string | undefined = findArgumentValue(processArguments, SOCKET_ARGUMENT_PREFIX); + if (!socketPath || !path.isAbsolute(socketPath)) { + return undefined; + } + return { + socketPath, + capturePipesBasePath: findArgumentValue(processArguments, CAPTURE_PIPES_ARGUMENT_PREFIX) + }; +} + +export function getCapturePipePaths(capturePipesBasePath: string): ICapturePipePaths { + return { + standardOutputPipePath: `${capturePipesBasePath}.out`, + standardErrorPipePath: `${capturePipesBasePath}.err`, + readyMarkerPath: `${capturePipesBasePath}.ready` + }; +} + +export function isFolderPrivateToCurrentUser(folderPath: string): boolean { + try { + const folderStats: fs.Stats = fs.lstatSync(folderPath); + return ( + folderStats.isDirectory() && + folderStats.uid === process.getuid?.() && + folderStats.mode % MODE_MODULUS_THAT_ISOLATES_GROUP_AND_OTHER_PERMISSIONS === 0 + ); + } catch { + return false; + } +} + +function isProcessAlive(processId: number): boolean { + try { + process.kill(processId, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'EPERM'; + } +} + +function removeCaptureFilesOfDeadHosts(socketFolder: string): void { + for (const fileName of fs.readdirSync(socketFolder)) { + const captureFileNameMatch: RegExpExecArray | null = CAPTURE_FILE_NAME_PATTERN.exec(fileName); + if (captureFileNameMatch && !isProcessAlive(Number(captureFileNameMatch[1]))) { + fs.rmSync(path.join(socketFolder, fileName), { force: true }); + } + } +} + +export function removeCapturePipes(capturePipePaths: ICapturePipePaths): void { + for (const capturePath of Object.values(capturePipePaths)) { + fs.rmSync(capturePath, { force: true }); + } +} + +function waitForReadyMarker(readyMarkerPath: string): void { + const waitStartTime: number = Date.now(); + while (!fs.existsSync(readyMarkerPath) && Date.now() - waitStartTime < READY_MARKER_WAIT_MILLISECONDS) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, READY_MARKER_POLL_MILLISECONDS); + } + fs.rmSync(readyMarkerPath, { force: true }); +} + +function openPipeEnds(pipePath: string): { readonly keepAliveReadEnd: number; readonly writeEnd: number } { + const keepAliveReadEnd: number = fs.openSync(pipePath, READ_ONLY_NON_BLOCKING_OPEN_FLAGS); + return { keepAliveReadEnd, writeEnd: fs.openSync(pipePath, fs.constants.O_WRONLY) }; +} + +export function startWarmHostProcess( + warmHostEntryPath: string, + socketPath: string, + environment: NodeJS.ProcessEnv +): void { + const socketFolder: string = path.dirname(socketPath); + if (!isFolderPrivateToCurrentUser(socketFolder)) { + return; + } + removeCaptureFilesOfDeadHosts(socketFolder); + const capturePipesBasePath: string = `${socketPath}.${process.pid}.${Date.now()}`; + const capturePipePaths: ICapturePipePaths = getCapturePipePaths(capturePipesBasePath); + const { standardOutputPipePath, standardErrorPipePath } = capturePipePaths; + try { + execFileSync('mkfifo', ['-m', '600', standardOutputPipePath, standardErrorPipePath], { + env: { PATH: environment.PATH ?? process.env.PATH }, + stdio: 'ignore' + }); + const standardOutputEnds: ReturnType = openPipeEnds(standardOutputPipePath); + const standardErrorEnds: ReturnType = openPipeEnds(standardErrorPipePath); + spawn( + process.execPath, + [ + warmHostEntryPath, + `${SOCKET_ARGUMENT_PREFIX}${socketPath}`, + `${CAPTURE_PIPES_ARGUMENT_PREFIX}${capturePipesBasePath}` + ], + { + argv0: 'node', + cwd: '/', + detached: true, + env: environment, + stdio: ['ignore', standardOutputEnds.writeEnd, standardErrorEnds.writeEnd] + } + ).unref(); + for (const fileDescriptor of [standardOutputEnds.writeEnd, standardErrorEnds.writeEnd]) { + fs.closeSync(fileDescriptor); + } + waitForReadyMarker(capturePipePaths.readyMarkerPath); + for (const fileDescriptor of [standardOutputEnds.keepAliveReadEnd, standardErrorEnds.keepAliveReadEnd]) { + fs.closeSync(fileDescriptor); + } + } catch { + removeCapturePipes(capturePipePaths); + } +} + +export interface ICapturePipesOfThisHost { + readonly readEnds: readonly [number, number]; + readonly capturePipePaths: ICapturePipePaths; +} + +export function openCapturePipesForReading( + spawnerCapturePipePaths: ICapturePipePaths +): ICapturePipesOfThisHost { + const { standardOutputPipePath, standardErrorPipePath, readyMarkerPath } = spawnerCapturePipePaths; + const readEnds: readonly [number, number] = [ + fs.openSync(standardOutputPipePath, READ_ONLY_NON_BLOCKING_OPEN_FLAGS), + fs.openSync(standardErrorPipePath, READ_ONLY_NON_BLOCKING_OPEN_FLAGS) + ]; + const socketPath: string = standardOutputPipePath.replace(/\.sock\.\d+\.\d+\.out$/, '.sock'); + const capturePipePaths: ICapturePipePaths = getCapturePipePaths(`${socketPath}.${process.pid}.0`); + fs.renameSync(standardOutputPipePath, capturePipePaths.standardOutputPipePath); + fs.renameSync(standardErrorPipePath, capturePipePaths.standardErrorPipePath); + fs.closeSync(fs.openSync(readyMarkerPath, 'wx', 0o600)); + return { readEnds, capturePipePaths }; +} diff --git a/apps/heft/src/host/warmHostRun.ts b/apps/heft/src/host/warmHostRun.ts new file mode 100644 index 0000000000..8adc45b9e3 --- /dev/null +++ b/apps/heft/src/host/warmHostRun.ts @@ -0,0 +1,186 @@ +import type * as net from 'node:net'; +import * as path from 'node:path'; + +import type { HostCommandLine } from './HostCommandLine'; +import type { IHostPlan } from './HostPlan'; +import { getRefusalReason, tryParseRunRequest, type IWarmRunRequest } from './warmRunRequest'; +import { + makeRequireStacksMatchTheHeftCommandLine, + presentTheMainModuleAsTheHeftBin +} from './makeRequireStacksMatchTheHeftCommandLine'; +import type { CapturedOutputWorker } from './CapturedOutputWorker'; +import { + ACCEPT_FRAME_TYPE, + EXIT_FRAME_TYPE, + REFUSE_FRAME_TYPE, + RUN_FRAME_TYPE, + SIGNAL_FRAME_TYPE, + WarmHostFrameReader, + createExitFramePayload, + writeFrameSynchronously, + type IWarmHostFrame +} from './warmHostFrames'; +import { removeCapturePipes, startWarmHostProcess, type ICapturePipePaths } from './warmHostProcess'; + +const SIGNAL_NUMBER_FOR_HANG_UP: number = 1; + +export interface IWarmHostContext { + readonly warmHostEntryPath: string; + readonly socketPath: string; + readonly heftVersion: string; + readonly preloadEnvironment: Readonly; + readonly outputWorker: CapturedOutputWorker; + readonly capturePipePaths: ICapturePipePaths; + hasStaleModuleFiles(): boolean; +} + +function adoptClientEnvironment(clientEnvironment: Readonly>): void { + for (const variableName of Object.keys(process.env)) { + if (!Object.prototype.hasOwnProperty.call(clientEnvironment, variableName)) { + delete process.env[variableName]; + } + } + Object.assign(process.env, clientEnvironment); +} + +function tryWriteExitFrame(socketFileDescriptor: number, exitCode: number): void { + try { + writeFrameSynchronously(socketFileDescriptor, EXIT_FRAME_TYPE, createExitFramePayload(exitCode)); + } catch { + return; + } +} + +function writeExitFrameAndStartSuccessorWhenTheProcessExits( + socketFileDescriptor: number, + context: IWarmHostContext, + successorEnvironment: Readonly> +): void { + let exitFrameWasWritten: boolean = false; + const writeExitFrame: (exitCode: number) => void = (exitCode: number) => { + if (!exitFrameWasWritten) { + exitFrameWasWritten = true; + context.outputWorker.finishForwarding(); + tryWriteExitFrame(socketFileDescriptor, exitCode); + removeCapturePipes(context.capturePipePaths); + startWarmHostProcess(context.warmHostEntryPath, context.socketPath, successorEnvironment); + } + }; + process.on('exit', writeExitFrame); + const originalExit: typeof process.exit = process.exit; + process.exit = function exitAfterOtherExitListeners(exitCode?: string | number | null): never { + process.removeListener('exit', writeExitFrame); + process.on('exit', writeExitFrame); + return originalExit.call(process, exitCode); + } as typeof process.exit; +} + +function forwardSignalToProcessGroup(signalNumber: number): void { + try { + process.kill(-process.pid, signalNumber); + } catch { + process.kill(process.pid, signalNumber); + } +} + +function runPlan(plan: IHostPlan, context: IWarmHostContext): void { + const heftBinPath: string = + plan.heftBinPath ?? path.resolve(context.warmHostEntryPath, '../../../bin/heft'); + process.argv = [process.argv[0], heftBinPath, ...plan.argv]; + makeRequireStacksMatchTheHeftCommandLine(); + presentTheMainModuleAsTheHeftBin(heftBinPath); + const { HostCommandLine: HostCommandLineClass } = require('./HostCommandLine') as { + HostCommandLine: typeof HostCommandLine; + }; + const hostCommandLine: HostCommandLine = new HostCommandLineClass(plan); + hostCommandLine + .executeAsync() + .then(() => { + process.exit(process.exitCode === undefined ? 0 : process.exitCode); + }) + .catch((error) => { + hostCommandLine.globalTerminal.writeErrorLine(error.toString()); + process.exit(1); + }); +} + +let warmRunIsInProgress: boolean = false; + +function writeRefusal(socketFileDescriptor: number, refusalReason: string): void { + try { + writeFrameSynchronously( + socketFileDescriptor, + REFUSE_FRAME_TYPE, + Buffer.from(JSON.stringify({ reason: refusalReason })) + ); + } catch { + return; + } +} + +function startRun( + runRequest: IWarmRunRequest, + socketFileDescriptor: number, + context: IWarmHostContext +): void { + const refusalReason: string | undefined = getRefusalReason(runRequest, context); + if (refusalReason !== undefined) { + writeRefusal(socketFileDescriptor, refusalReason); + removeCapturePipes(context.capturePipePaths); + process.exit(0); + } + const acceptance: string = JSON.stringify({ + hostPid: process.pid, + nodeVersion: process.version, + heftVersion: context.heftVersion + }); + try { + writeFrameSynchronously(socketFileDescriptor, ACCEPT_FRAME_TYPE, Buffer.from(acceptance)); + } catch { + removeCapturePipes(context.capturePipePaths); + process.exit(0); + } + adoptClientEnvironment(runRequest.env!); + process.chdir(runRequest.cwd!); + context.outputWorker.startForwardingTo(socketFileDescriptor); + writeExitFrameAndStartSuccessorWhenTheProcessExits(socketFileDescriptor, context, { ...runRequest.env! }); + runPlan(runRequest.plan!, context); +} + +export interface IWarmRunListener { + readonly context: IWarmHostContext; + stopListening(): void; +} + +export function handleWarmHostConnection(socket: net.Socket, listener: IWarmRunListener): void { + const socketFileDescriptor: number = (socket as unknown as { _handle: { fd: number } })._handle.fd; + const frameReader: WarmHostFrameReader = new WarmHostFrameReader(); + let runWasStarted: boolean = false; + socket.on('error', () => undefined); + socket.on('data', (incomingBytes: Buffer) => { + let frames: IWarmHostFrame[]; + try { + frames = frameReader.pushBytes(incomingBytes); + } catch { + socket.destroy(); + return; + } + for (const frame of frames) { + if (runWasStarted) { + if (frame.frameType === SIGNAL_FRAME_TYPE && frame.payload.length === 1) { + forwardSignalToProcessGroup(frame.payload[0]); + } + } else if (frame.frameType !== RUN_FRAME_TYPE || warmRunIsInProgress) { + writeRefusal(socketFileDescriptor, frame.frameType === RUN_FRAME_TYPE ? 'busy' : 'frame'); + socket.destroy(); + return; + } else { + runWasStarted = true; + warmRunIsInProgress = true; + listener.stopListening(); + socket.on('close', () => forwardSignalToProcessGroup(SIGNAL_NUMBER_FOR_HANG_UP)); + startRun(tryParseRunRequest(frame.payload), socketFileDescriptor, listener.context); + } + } + }); +} diff --git a/apps/heft/src/host/warmRunRequest.ts b/apps/heft/src/host/warmRunRequest.ts new file mode 100644 index 0000000000..e2da9ed8b9 --- /dev/null +++ b/apps/heft/src/host/warmRunRequest.ts @@ -0,0 +1,119 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import type { IHostPlan } from './HostPlan'; + +export interface IWarmRunValidationContext { + readonly heftVersion: string; + readonly preloadEnvironment: Readonly; + hasStaleModuleFiles(): boolean; +} + +const WARM_PROTOCOL_VERSION: number = 1; +const ENVIRONMENT_VARIABLES_THAT_NAME_CLIENT_FILE_DESCRIPTORS: ReadonlyArray = [ + '_RUSH_REPORTER_CHILD_FD', + '_RUSH_REPORTER_CHILD_ACK_FD', + 'NODE_CHANNEL_FD' +]; +const ENVIRONMENT_VARIABLES_IGNORED_FOR_WARM_REUSE: ReadonlySet = new Set(['_', 'OLDPWD']); + +export interface IWarmRunRequest { + readonly protocolVersion?: number; + readonly plan?: IHostPlan; + readonly env?: Record; + readonly cwd?: string; + readonly nodeExecPath?: string; + readonly stdoutIsTTY?: boolean; + readonly stderrIsTTY?: boolean; + readonly umask?: string; +} + +function areEnvironmentsEquivalent( + clientEnvironment: Readonly, + hostEnvironment: Readonly +): boolean { + const variableNames: Set = new Set([ + ...Object.keys(clientEnvironment), + ...Object.keys(hostEnvironment) + ]); + for (const variableName of variableNames) { + if ( + !ENVIRONMENT_VARIABLES_IGNORED_FOR_WARM_REUSE.has(variableName) && + clientEnvironment[variableName] !== hostEnvironment[variableName] + ) { + return false; + } + } + return true; +} + +function tryReadOwnFileModeCreationMask(): number | undefined { + try { + const fileModeCreationMaskMatch: RegExpExecArray | null = /^Umask:\s*([0-7]+)$/m.exec( + fs.readFileSync('/proc/self/status', 'utf8') + ); + return fileModeCreationMaskMatch ? Number.parseInt(fileModeCreationMaskMatch[1], 8) : undefined; + } catch { + return undefined; + } +} + +function isFileModeCreationMaskDifferent(clientFileModeCreationMask: string | undefined): boolean { + return ( + clientFileModeCreationMask !== undefined && + Number.parseInt(clientFileModeCreationMask, 8) !== tryReadOwnFileModeCreationMask() + ); +} + +function tryGetRealPath(filePath: string | undefined): string | undefined { + try { + return filePath === undefined ? undefined : fs.realpathSync(filePath); + } catch { + return undefined; + } +} + +export function getRefusalReason( + runRequest: IWarmRunRequest, + context: IWarmRunValidationContext +): string | undefined { + const { plan, env, cwd } = runRequest; + if (runRequest.protocolVersion !== WARM_PROTOCOL_VERSION) { + return 'protocol version'; + } else if ( + plan?.kind !== 'heft-plan' || + plan.protocolVersion !== 1 || + plan.heftVersion !== context.heftVersion + ) { + return 'plan version'; + } else if (!plan.command || plan.command.watch || plan.argv.includes('--debug')) { + return 'command'; + } else if (runRequest.stdoutIsTTY !== false || runRequest.stderrIsTTY !== false) { + return 'terminal'; + } else if (!env || !cwd || !path.isAbsolute(cwd)) { + return 'request'; + } else if ( + ENVIRONMENT_VARIABLES_THAT_NAME_CLIENT_FILE_DESCRIPTORS.some( + (variableName: string) => env[variableName] !== undefined + ) + ) { + return 'client file descriptors'; + } else if (tryGetRealPath(runRequest.nodeExecPath) !== tryGetRealPath(process.execPath)) { + return 'node executable'; + } else if (!areEnvironmentsEquivalent(env, context.preloadEnvironment)) { + return 'environment'; + } else if (isFileModeCreationMaskDifferent(runRequest.umask)) { + return 'umask'; + } else if (context.hasStaleModuleFiles()) { + return 'stale modules'; + } + return undefined; +} + +export function tryParseRunRequest(payload: Buffer): IWarmRunRequest { + try { + return JSON.parse(payload.toString('utf8')); + } catch { + return {}; + } +} diff --git a/apps/heft/src/pluginFramework/HeftLifecycle.ts b/apps/heft/src/pluginFramework/HeftLifecycle.ts index efdafcdb4c..daa63eb5e1 100644 --- a/apps/heft/src/pluginFramework/HeftLifecycle.ts +++ b/apps/heft/src/pluginFramework/HeftLifecycle.ts @@ -162,7 +162,7 @@ export class HeftLifecycle extends HeftPluginHost { } } - public async ensureInitializedAsync(): Promise { + public async ensureInitializedAsync(pluginOptionsAreValidated: boolean = false): Promise { if (!this.#isInitialized) { this.#isInitialized = true; @@ -204,7 +204,9 @@ export class HeftLifecycle extends HeftPluginHost { // Validate the plugin options const pluginOptions: object | undefined = pluginSpecifier.options; - pluginDefinition.validateOptions(pluginOptions); + if (!pluginOptionsAreValidated) { + pluginDefinition.validateOptions(pluginOptions); + } // Partially populate the context. The session will be populated while applying the plugins. const lifecycleContext: IHeftLifecycleContext = { pluginOptions }; diff --git a/apps/heft/src/pluginFramework/HeftTask.ts b/apps/heft/src/pluginFramework/HeftTask.ts index ee9f987078..4af696b172 100644 --- a/apps/heft/src/pluginFramework/HeftTask.ts +++ b/apps/heft/src/pluginFramework/HeftTask.ts @@ -115,10 +115,12 @@ export class HeftTask implements IHeftTask { this.#validate(); } - public async ensureInitializedAsync(): Promise { + public async ensureInitializedAsync(pluginOptionsAreValidated: boolean = false): Promise { if (!this.#taskPluginDefinition) { this.#taskPluginDefinition = await this.#loadTaskPluginDefinitionAsync(); - this.pluginDefinition.validateOptions(this.pluginOptions); + if (!pluginOptionsAreValidated) { + this.pluginDefinition.validateOptions(this.pluginOptions); + } } } diff --git a/apps/heft/src/pluginFramework/InternalHeftSession.ts b/apps/heft/src/pluginFramework/InternalHeftSession.ts index 0d6071b97a..38ae17fb9b 100644 --- a/apps/heft/src/pluginFramework/InternalHeftSession.ts +++ b/apps/heft/src/pluginFramework/InternalHeftSession.ts @@ -7,10 +7,10 @@ import { Constants } from '../utilities/Constants'; import { HeftLifecycle } from './HeftLifecycle'; import type { HeftPhaseSession } from './HeftPhaseSession'; import { HeftPhase } from './HeftPhase'; -import { +import type { CoreConfigFiles, - type IHeftConfigurationJson, - type IHeftConfigurationJsonActionReference + IHeftConfigurationJson, + IHeftConfigurationJsonActionReference } from '../utilities/CoreConfigFiles'; import type { MetricsCollector } from '../metrics/MetricsCollector'; import type { LoggingManager } from './logging/LoggingManager'; @@ -20,12 +20,41 @@ import type { HeftTask } from './HeftTask'; import type { HeftParameterManager } from './HeftParameterManager'; import type { IHeftParsedCommandLine } from './HeftTaskSession'; +export interface IInternalHeftSessionPlanSeed { + readonly heftConfigurationJson: IHeftConfigurationJson; + readonly debugMessages: ReadonlyArray; + readonly pluginOptionsAreValidated: boolean; +} + export interface IInternalHeftSessionOptions { heftConfiguration: HeftConfiguration; loggingManager: LoggingManager; metricsCollector: MetricsCollector; debug: boolean; + + planSeed?: IInternalHeftSessionPlanSeed; +} + +async function loadHeftConfigurationJsonAsync( + options: IInternalHeftSessionOptions +): Promise { + const { heftConfiguration, planSeed } = options; + if (planSeed) { + for (const debugMessage of planSeed.debugMessages) { + heftConfiguration.globalTerminal.writeDebugLine(debugMessage); + } + return planSeed.heftConfigurationJson; + } + const { CoreConfigFiles: CoreConfigFilesClass } = require('../utilities/CoreConfigFiles') as { + CoreConfigFiles: typeof CoreConfigFiles; + }; + return await CoreConfigFilesClass.loadHeftConfigurationFileForProjectAsync( + heftConfiguration.globalTerminal, + heftConfiguration.buildFolderPath, + // Same data as heftConfiguration.rigConfig, without loading @rushstack/rig-package unless needed + getRigConfigForConfigLoading(heftConfiguration) + ); } function* getAllTasks(phases: Iterable): IterableIterator { @@ -65,25 +94,20 @@ export class InternalHeftSession { // Initialize the rig. Must be done before the HeftConfiguration.rigConfig is used. await options.heftConfiguration._checkForRigAsync(); - const heftConfigurationJson: IHeftConfigurationJson = - await CoreConfigFiles.loadHeftConfigurationFileForProjectAsync( - options.heftConfiguration.globalTerminal, - options.heftConfiguration.buildFolderPath, - // Same data as heftConfiguration.rigConfig, without loading @rushstack/rig-package unless needed - getRigConfigForConfigLoading(options.heftConfiguration) - ); + const heftConfigurationJson: IHeftConfigurationJson = await loadHeftConfigurationJsonAsync(options); + const pluginOptionsAreValidated: boolean = options.planSeed?.pluginOptionsAreValidated ?? false; const internalHeftSession: InternalHeftSession = new InternalHeftSession(heftConfigurationJson, options); // Initialize the lifecycle and the tasks. This will ensure that we throw an error if a plugin is improperly // specified, or if the options provided to a plugin are invalid. We will avoid loading the actual plugins // until they are needed. - await internalHeftSession.lifecycle.ensureInitializedAsync(); + await internalHeftSession.lifecycle.ensureInitializedAsync(pluginOptionsAreValidated); const tasks: Iterable = getAllTasks(internalHeftSession.phases); await Async.forEachAsync( tasks, async (task: HeftTask) => { - await task.ensureInitializedAsync(); + await task.ensureInitializedAsync(pluginOptionsAreValidated); }, { concurrency: Constants.maxParallelism } );