From c4b322246dfe43c1e063293efc3fcd33f6dfda38 Mon Sep 17 00:00:00 2001 From: Chayim Refael Friedman Date: Sun, 16 Aug 2026 03:09:04 +0300 Subject: [PATCH] Store token trees in the proc macro server in the representation of the tt crate And not in the representation of the proc macro bridge. We still do not use the contiguous representation the rest of r-a uses, and prefer a tree with `Rc`s (this changed from `Arc` because we don't need thread safety for the bridge) for cheap cloning, but the types are the same. The reason for that is that conversion to the bridge's types is a lossy conversion, in rustc as well: the internal token tree representation contains details that are lost in this conversion. This is a prerequisite to fix https://github.com/rust-lang/rust-analyzer/issues/23088 - the only way to fix it properly is to have an additional kind of token tree for doc comments, like rustc does, as evidenced by the fact that if a proc macro would create this macro (that has a doc comment in its matcher), the doc comment would *not* be ignored for matching, but if the macro's matcher would have been passed to the proc macro and it wouldn't have touch it (meaning, not even listing the `TokenTree`s then giving them back), the comment would still be ignored. It is also required for supporting invisible groups properly for the same reason (there're actually several kinds of invisible delimiters, and the proc macro bridge lossily converts them all into one). While we're at, I've also tried to untangle the mess of Cargo features of the proc macro server, and reduce the code it contains: There are now *two* somewhat-orthogonal axes: the `in-rust-tree` feature, and the `in-ra` vs. `in-proc-macro-srv` feature. The former only decides whether we should import rustc crates from crates.io or from the sysroot. The latter decides what code to enable - the proc macro server doesn't need all the code in tt and proc-macro-api. I also moved the proc macro server's `TokenStream` from proc-macro-srv into proc-macro-api and inverted their dependency, as was required for this work. You cannot enable `in-proc-macro-srv` and disable `in-rust-tree` (in practice this just disables all proc macro server code since such combination can arise when compiling r-a, also proc-macro-api supports this mode for tests), and you are not supposed to enable both `in-ra` and `in-proc-macro-srv`, except for tests. Enabling `in-rust-tree` and disabling `in-proc-macro-srv` is possible, though, and done when compiling r-a in-tree. In the future, this mode may use sysroot crates not just for the proc macro server, like was done in the past. --- Cargo.lock | 13 +- crates/hir-def/Cargo.toml | 3 + crates/hir-def/src/item_tree/attrs.rs | 10 +- crates/hir-def/src/lib.rs | 8 +- crates/load-cargo/src/lib.rs | 6 +- crates/mbe/Cargo.toml | 3 + crates/mbe/src/lib.rs | 7 +- crates/mbe/src/tests.rs | 76 +- crates/parser/Cargo.toml | 4 + crates/parser/src/lib.rs | 8 +- crates/proc-macro-api/Cargo.toml | 26 +- .../src/bidirectional_protocol.rs | 214 +--- .../src/bidirectional_protocol/msg.rs | 3 +- .../src/bidirectional_protocol/sender.rs | 210 ++++ crates/proc-macro-api/src/client.rs | 214 ++++ crates/proc-macro-api/src/flat.rs | 588 +++++++++ .../src/flat/proc_macro_srv_side.rs | 87 ++ crates/proc-macro-api/src/flat/ra_side.rs | 73 ++ crates/proc-macro-api/src/legacy_protocol.rs | 157 +-- .../proc-macro-api/src/legacy_protocol/msg.rs | 24 +- .../src/legacy_protocol/msg/flat.rs | 980 --------------- .../src/legacy_protocol/sender.rs | 156 +++ crates/proc-macro-api/src/lib.rs | 226 +--- crates/proc-macro-api/src/pool.rs | 5 +- crates/proc-macro-api/src/process.rs | 5 +- crates/proc-macro-api/src/token_stream.rs | 691 +++++++++++ crates/proc-macro-srv-cli/Cargo.toml | 19 +- crates/proc-macro-srv-cli/src/lib.rs | 81 ++ crates/proc-macro-srv-cli/src/main.rs | 97 +- crates/proc-macro-srv-cli/src/main_loop.rs | 102 +- .../tests/bidirectional_postcard.rs | 3 +- .../proc-macro-srv-cli/tests/common/utils.rs | 5 +- .../proc-macro-srv-cli/tests/legacy_json.rs | 3 +- crates/proc-macro-srv/Cargo.toml | 10 +- crates/proc-macro-srv/src/bridge.rs | 12 - crates/proc-macro-srv/src/dylib.rs | 3 +- .../proc-macro-srv/src/dylib/proc_macros.rs | 6 +- crates/proc-macro-srv/src/lib.rs | 71 +- crates/proc-macro-srv/src/server_impl.rs | 30 +- .../proc-macro-srv/src/server_impl/bridge.rs | 160 +++ .../src/server_impl/rust_analyzer_span.rs | 58 +- .../src/server_impl/token_id.rs | 68 +- crates/proc-macro-srv/src/tests/mod.rs | 932 +++++++-------- crates/proc-macro-srv/src/tests/utils.rs | 2 +- crates/proc-macro-srv/src/token_stream.rs | 767 ------------ crates/rust-analyzer/src/global_state.rs | 2 +- crates/rust-analyzer/src/reload.rs | 2 +- crates/stdx/src/lib.rs | 26 + crates/syntax-bridge/src/lib.rs | 4 +- crates/syntax/Cargo.toml | 3 + crates/syntax/src/lib.rs | 8 +- crates/tt/Cargo.toml | 22 +- crates/tt/src/leaf_types.rs | 398 +++++++ crates/tt/src/lib.rs | 1049 ++++++----------- 54 files changed, 3863 insertions(+), 3877 deletions(-) create mode 100644 crates/proc-macro-api/src/bidirectional_protocol/sender.rs create mode 100644 crates/proc-macro-api/src/client.rs create mode 100644 crates/proc-macro-api/src/flat.rs create mode 100644 crates/proc-macro-api/src/flat/proc_macro_srv_side.rs create mode 100644 crates/proc-macro-api/src/flat/ra_side.rs delete mode 100644 crates/proc-macro-api/src/legacy_protocol/msg/flat.rs create mode 100644 crates/proc-macro-api/src/legacy_protocol/sender.rs create mode 100644 crates/proc-macro-api/src/token_stream.rs delete mode 100644 crates/proc-macro-srv/src/bridge.rs create mode 100644 crates/proc-macro-srv/src/server_impl/bridge.rs delete mode 100644 crates/proc-macro-srv/src/token_stream.rs create mode 100644 crates/tt/src/leaf_types.rs diff --git a/Cargo.lock b/Cargo.lock index 5835ed9e552e..148e049be81e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1348,9 +1348,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libmimalloc-sys" @@ -1867,7 +1867,8 @@ dependencies = [ "intern", "paths", "postcard", - "proc-macro-srv", + "proc-macro-api", + "ra-ap-rustc_lexer", "rayon", "rustc-hash 2.1.2", "semver", @@ -1888,9 +1889,11 @@ dependencies = [ "intern", "line-index 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "paths", + "proc-macro-api", "proc-macro-test", "span", "stdx", + "tt", ] [[package]] @@ -2374,9 +2377,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" [[package]] name = "rustc-hash" diff --git a/crates/hir-def/Cargo.toml b/crates/hir-def/Cargo.toml index eb1e774b8f41..9319bb409143 100644 --- a/crates/hir-def/Cargo.toml +++ b/crates/hir-def/Cargo.toml @@ -56,3 +56,6 @@ in-rust-tree = ["hir-expand/in-rust-tree"] [lints] workspace = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_parse_format"] diff --git a/crates/hir-def/src/item_tree/attrs.rs b/crates/hir-def/src/item_tree/attrs.rs index 4ee2f56be19f..e69aa2a23075 100644 --- a/crates/hir-def/src/item_tree/attrs.rs +++ b/crates/hir-def/src/item_tree/attrs.rs @@ -21,7 +21,7 @@ use hir_expand::{ use intern::{Interned, Symbol, sym}; use syntax::{AstNode, ast}; use syntax_bridge::DocCommentDesugarMode; -use tt::token_to_literal; +use tt::literal_from_str; use crate::item_tree::lower::Ctx; @@ -64,11 +64,13 @@ impl AttrsOrCfg { ast::Meta::KeyValueMeta(meta) => { let span = span_map.span_for(path_range); let input = meta.expr().and_then(|value| { - if let ast::Expr::Literal(value) = value { - Some(Box::new(AttrInput::Literal(token_to_literal( + if let ast::Expr::Literal(value) = value + && let Ok(lit) = literal_from_str( value.token().text(), span_map.span_for(value.syntax().text_range()), - )))) + ) + { + Some(Box::new(AttrInput::Literal(lit))) } else { None } diff --git a/crates/hir-def/src/lib.rs b/crates/hir-def/src/lib.rs index 0712a025b49c..70b69597f9ca 100644 --- a/crates/hir-def/src/lib.rs +++ b/crates/hir-def/src/lib.rs @@ -9,11 +9,9 @@ #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] -#[cfg(feature = "in-rust-tree")] -extern crate rustc_parse_format; - -#[cfg(not(feature = "in-rust-tree"))] -extern crate ra_ap_rustc_parse_format as rustc_parse_format; +stdx::rustc_crates! { + extern crate rustc_parse_format or ra_ap_rustc_parse_format; +} pub extern crate ra_ap_rustc_abi as layout; pub extern crate ra_ap_rustc_abi as rustc_abi; diff --git a/crates/load-cargo/src/lib.rs b/crates/load-cargo/src/lib.rs index bf2331a40f9b..db2f6c5443f6 100644 --- a/crates/load-cargo/src/lib.rs +++ b/crates/load-cargo/src/lib.rs @@ -24,8 +24,8 @@ use ide_db::{ }; use itertools::Itertools; use proc_macro_api::{ - MacroDylib, ProcMacroClient, bidirectional_protocol::msg::{ParentSpan, SubRequest, SubResponse}, + client::{MacroDylib, ProcMacroClient}, }; use project_model::{CargoConfig, PackageRoot, ProjectManifest, ProjectWorkspace}; use span::{Span, SpanAnchor, SyntaxContext}; @@ -558,7 +558,7 @@ fn load_crate_graph_into_db( } fn expander_to_proc_macro( - expander: proc_macro_api::ProcMacro, + expander: proc_macro_api::client::ProcMacro, ignored_macros: &[Box], ) -> ProcMacro { let name = expander.name(); @@ -577,7 +577,7 @@ fn expander_to_proc_macro( } #[derive(Debug, PartialEq, Eq)] -struct Expander(proc_macro_api::ProcMacro); +struct Expander(proc_macro_api::client::ProcMacro); impl ProcMacroExpander for Expander { fn expand( diff --git a/crates/mbe/Cargo.toml b/crates/mbe/Cargo.toml index b6e55b9360a3..b2f587bf91fc 100644 --- a/crates/mbe/Cargo.toml +++ b/crates/mbe/Cargo.toml @@ -39,3 +39,6 @@ in-rust-tree = ["parser/in-rust-tree", "tt/in-rust-tree"] [lints] workspace = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_lexer"] diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index 6de9b4275ce2..13b71b5afdef 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -8,10 +8,9 @@ #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] -#[cfg(not(feature = "in-rust-tree"))] -extern crate ra_ap_rustc_lexer as rustc_lexer; -#[cfg(feature = "in-rust-tree")] -extern crate rustc_lexer; +stdx::rustc_crates! { + extern crate rustc_lexer or ra_ap_rustc_lexer; +} mod expander; mod macro_call_style; diff --git a/crates/mbe/src/tests.rs b/crates/mbe/src/tests.rs index 9e93d1869d04..fd382141290c 100644 --- a/crates/mbe/src/tests.rs +++ b/crates/mbe/src/tests.rs @@ -138,20 +138,20 @@ struct MyTraitMap2 IDENT MyTraitMap2 1:Root[0000, 0]@8..19#ROOT2024 SUBTREE {} 0:Root[0000, 0]@48..49#ROOT2024 0:Root[0000, 0]@100..101#ROOT2024 IDENT map 0:Root[0000, 0]@58..61#ROOT2024 - PUNCH : [alone] 0:Root[0000, 0]@61..62#ROOT2024 - PUNCH : [joint] 0:Root[0000, 0]@63..64#ROOT2024 - PUNCH : [alone] 0:Root[0000, 0]@64..65#ROOT2024 + PUNCT : [alone] 0:Root[0000, 0]@61..62#ROOT2024 + PUNCT : [joint] 0:Root[0000, 0]@63..64#ROOT2024 + PUNCT : [alone] 0:Root[0000, 0]@64..65#ROOT2024 IDENT std 0:Root[0000, 0]@65..68#ROOT2024 - PUNCH : [joint] 0:Root[0000, 0]@68..69#ROOT2024 - PUNCH : [alone] 0:Root[0000, 0]@69..70#ROOT2024 + PUNCT : [joint] 0:Root[0000, 0]@68..69#ROOT2024 + PUNCT : [alone] 0:Root[0000, 0]@69..70#ROOT2024 IDENT collections 0:Root[0000, 0]@70..81#ROOT2024 - PUNCH : [joint] 0:Root[0000, 0]@81..82#ROOT2024 - PUNCH : [alone] 0:Root[0000, 0]@82..83#ROOT2024 + PUNCT : [joint] 0:Root[0000, 0]@81..82#ROOT2024 + PUNCT : [alone] 0:Root[0000, 0]@82..83#ROOT2024 IDENT HashSet 0:Root[0000, 0]@83..90#ROOT2024 - PUNCH < [alone] 0:Root[0000, 0]@90..91#ROOT2024 + PUNCT < [alone] 0:Root[0000, 0]@90..91#ROOT2024 SUBTREE () 0:Root[0000, 0]@91..92#ROOT2024 0:Root[0000, 0]@92..93#ROOT2024 - PUNCH > [joint] 0:Root[0000, 0]@93..94#ROOT2024 - PUNCH , [alone] 0:Root[0000, 0]@94..95#ROOT2024 + PUNCT > [joint] 0:Root[0000, 0]@93..94#ROOT2024 + PUNCT , [alone] 0:Root[0000, 0]@94..95#ROOT2024 struct MyTraitMap2 { map: ::std::collections::HashSet<()>, @@ -186,22 +186,22 @@ fn main() { SUBTREE () 1:Root[0000, 0]@8..9#ROOT2024 1:Root[0000, 0]@9..10#ROOT2024 SUBTREE {} 1:Root[0000, 0]@11..12#ROOT2024 1:Root[0000, 0]@61..62#ROOT2024 LITERAL Integer 1 1:Root[0000, 0]@17..18#ROOT2024 - PUNCH ; [alone] 1:Root[0000, 0]@18..19#ROOT2024 + PUNCT ; [alone] 1:Root[0000, 0]@18..19#ROOT2024 LITERAL Float 1.0 1:Root[0000, 0]@24..27#ROOT2024 - PUNCH ; [alone] 1:Root[0000, 0]@27..28#ROOT2024 + PUNCT ; [alone] 1:Root[0000, 0]@27..28#ROOT2024 SUBTREE () 1:Root[0000, 0]@33..34#ROOT2024 1:Root[0000, 0]@39..40#ROOT2024 SUBTREE () 1:Root[0000, 0]@34..35#ROOT2024 1:Root[0000, 0]@37..38#ROOT2024 LITERAL Integer 1 1:Root[0000, 0]@35..36#ROOT2024 - PUNCH , [alone] 1:Root[0000, 0]@36..37#ROOT2024 - PUNCH , [alone] 1:Root[0000, 0]@38..39#ROOT2024 - PUNCH . [alone] 1:Root[0000, 0]@40..41#ROOT2024 + PUNCT , [alone] 1:Root[0000, 0]@36..37#ROOT2024 + PUNCT , [alone] 1:Root[0000, 0]@38..39#ROOT2024 + PUNCT . [alone] 1:Root[0000, 0]@40..41#ROOT2024 LITERAL Float 0.0 1:Root[0000, 0]@41..44#ROOT2024 - PUNCH ; [alone] 1:Root[0000, 0]@44..45#ROOT2024 + PUNCT ; [alone] 1:Root[0000, 0]@44..45#ROOT2024 IDENT let 1:Root[0000, 0]@50..53#ROOT2024 IDENT x 1:Root[0000, 0]@54..55#ROOT2024 - PUNCH = [alone] 1:Root[0000, 0]@56..57#ROOT2024 + PUNCT = [alone] 1:Root[0000, 0]@56..57#ROOT2024 LITERAL Integer 1 1:Root[0000, 0]@58..59#ROOT2024 - PUNCH ; [alone] 1:Root[0000, 0]@59..60#ROOT2024 + PUNCT ; [alone] 1:Root[0000, 0]@59..60#ROOT2024 fn main(){ 1; @@ -229,12 +229,12 @@ fn expr_2021() { expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..25#ROOT2024 1:Root[0000, 0]@0..25#ROOT2024 IDENT _ 1:Root[0000, 0]@5..6#ROOT2024 - PUNCH ; [joint] 0:Root[0000, 0]@36..37#ROOT2024 + PUNCT ; [joint] 0:Root[0000, 0]@36..37#ROOT2024 SUBTREE () 0:Root[0000, 0]@34..35#ROOT2024 0:Root[0000, 0]@34..35#ROOT2024 IDENT const 1:Root[0000, 0]@12..17#ROOT2024 SUBTREE {} 1:Root[0000, 0]@18..19#ROOT2024 1:Root[0000, 0]@22..23#ROOT2024 LITERAL Integer 1 1:Root[0000, 0]@20..21#ROOT2024 - PUNCH ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 + PUNCT ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 _; (const { @@ -261,7 +261,7 @@ fn expr_2021() { } SUBTREE $$ 1:Root[0000, 0]@0..8#ROOT2024 1:Root[0000, 0]@0..8#ROOT2024 - PUNCH ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 + PUNCT ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 ;"#]], ); @@ -285,7 +285,7 @@ fn expr_2021() { } SUBTREE $$ 1:Root[0000, 0]@0..18#ROOT2024 1:Root[0000, 0]@0..18#ROOT2024 - PUNCH ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 + PUNCT ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 ;"#]], ); @@ -307,24 +307,24 @@ fn expr_2021() { expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..76#ROOT2024 1:Root[0000, 0]@0..76#ROOT2024 LITERAL Integer 4 1:Root[0000, 0]@5..6#ROOT2024 - PUNCH ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 + PUNCT ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 LITERAL Str literal 1:Root[0000, 0]@12..21#ROOT2024 - PUNCH ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 + PUNCT ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 SUBTREE () 0:Root[0000, 0]@39..40#ROOT2024 0:Root[0000, 0]@39..40#ROOT2024 IDENT funcall 1:Root[0000, 0]@27..34#ROOT2024 SUBTREE () 1:Root[0000, 0]@34..35#ROOT2024 1:Root[0000, 0]@35..36#ROOT2024 - PUNCH ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 + PUNCT ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 SUBTREE () 0:Root[0000, 0]@39..40#ROOT2024 0:Root[0000, 0]@39..40#ROOT2024 IDENT future 1:Root[0000, 0]@42..48#ROOT2024 - PUNCH . [alone] 1:Root[0000, 0]@48..49#ROOT2024 + PUNCT . [alone] 1:Root[0000, 0]@48..49#ROOT2024 IDENT await 1:Root[0000, 0]@49..54#ROOT2024 - PUNCH ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 + PUNCT ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 SUBTREE () 0:Root[0000, 0]@39..40#ROOT2024 0:Root[0000, 0]@39..40#ROOT2024 IDENT break 1:Root[0000, 0]@60..65#ROOT2024 - PUNCH ' [joint] 1:Root[0000, 0]@66..67#ROOT2024 + PUNCT ' [joint] 1:Root[0000, 0]@66..67#ROOT2024 IDENT foo 1:Root[0000, 0]@67..70#ROOT2024 IDENT bar 1:Root[0000, 0]@71..74#ROOT2024 - PUNCH ; [alone] 0:Root[0000, 0]@44..45#ROOT2024 + PUNCT ; [alone] 0:Root[0000, 0]@44..45#ROOT2024 4; "literal"; @@ -352,7 +352,7 @@ fn expr_2021() { } SUBTREE $$ 1:Root[0000, 0]@0..8#ROOT2024 1:Root[0000, 0]@0..8#ROOT2024 - PUNCH ; [alone] 0:Root[0000, 0]@44..45#ROOT2024 + PUNCT ; [alone] 0:Root[0000, 0]@44..45#ROOT2024 ;"#]], ); @@ -371,7 +371,7 @@ fn minus_belongs_to_literal() { "-1", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..2#ROOT2024 1:Root[0000, 0]@0..2#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@10..11#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@10..11#ROOT2024 LITERAL Integer 1 0:Root[0000, 0]@11..12#ROOT2024 -1"#]], @@ -380,7 +380,7 @@ fn minus_belongs_to_literal() { "- 1", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..3#ROOT2024 1:Root[0000, 0]@0..3#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@10..11#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@10..11#ROOT2024 LITERAL Integer 1 0:Root[0000, 0]@11..12#ROOT2024 -1"#]], @@ -389,7 +389,7 @@ fn minus_belongs_to_literal() { "-2", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..2#ROOT2024 1:Root[0000, 0]@0..2#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@25..26#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@25..26#ROOT2024 LITERAL Integer 2 0:Root[0000, 0]@27..28#ROOT2024 -2"#]], @@ -398,7 +398,7 @@ fn minus_belongs_to_literal() { "- 2", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..3#ROOT2024 1:Root[0000, 0]@0..3#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@25..26#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@25..26#ROOT2024 LITERAL Integer 2 0:Root[0000, 0]@27..28#ROOT2024 -2"#]], @@ -407,7 +407,7 @@ fn minus_belongs_to_literal() { "-3.0", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..4#ROOT2024 1:Root[0000, 0]@0..4#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@43..44#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@43..44#ROOT2024 LITERAL Float 3.0 0:Root[0000, 0]@45..48#ROOT2024 -3.0"#]], @@ -416,7 +416,7 @@ fn minus_belongs_to_literal() { "- 3.0", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..5#ROOT2024 1:Root[0000, 0]@0..5#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@43..44#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@43..44#ROOT2024 LITERAL Float 3.0 0:Root[0000, 0]@45..48#ROOT2024 -3.0"#]], @@ -433,7 +433,7 @@ fn minus_belongs_to_literal() { "@-1", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..3#ROOT2024 1:Root[0000, 0]@0..3#ROOT2024 - PUNCH - [alone] 1:Root[0000, 0]@1..2#ROOT2024 + PUNCT - [alone] 1:Root[0000, 0]@1..2#ROOT2024 LITERAL Integer 1 1:Root[0000, 0]@2..3#ROOT2024 -1"#]], @@ -450,7 +450,7 @@ fn minus_belongs_to_literal() { "@-1.0", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..5#ROOT2024 1:Root[0000, 0]@0..5#ROOT2024 - PUNCH - [alone] 1:Root[0000, 0]@1..2#ROOT2024 + PUNCT - [alone] 1:Root[0000, 0]@1..2#ROOT2024 LITERAL Float 1.0 1:Root[0000, 0]@2..5#ROOT2024 -1.0"#]], diff --git a/crates/parser/Cargo.toml b/crates/parser/Cargo.toml index 2bdf8d76fbc6..8710f2c51228 100644 --- a/crates/parser/Cargo.toml +++ b/crates/parser/Cargo.toml @@ -19,6 +19,7 @@ rustc-literal-escaper.workspace = true tracing.workspace = true edition.workspace = true +stdx.workspace = true winnow = { version = "0.7.13", default-features = false } [dev-dependencies] @@ -32,3 +33,6 @@ in-rust-tree = [] [lints] workspace = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_lexer"] diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index 5900d7cfeda9..ba9b3d39a8f5 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -21,12 +21,12 @@ #![allow(rustdoc::private_intra_doc_links)] #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] -#[cfg(not(feature = "in-rust-tree"))] -extern crate ra_ap_rustc_lexer as rustc_lexer; #[cfg(feature = "in-rust-tree")] extern crate rustc_driver as _; -#[cfg(feature = "in-rust-tree")] -extern crate rustc_lexer; + +stdx::rustc_crates! { + extern crate rustc_lexer or ra_ap_rustc_lexer; +} mod event; mod frontmatter; diff --git a/crates/proc-macro-api/Cargo.toml b/crates/proc-macro-api/Cargo.toml index 7342e0ecdcf6..e88f43178868 100644 --- a/crates/proc-macro-api/Cargo.toml +++ b/crates/proc-macro-api/Cargo.toml @@ -22,20 +22,36 @@ indexmap.workspace = true # local deps paths = { workspace = true, features = ["serde1"] } -tt.workspace = true +tt = { path = "../tt", version = "0.0.0", default-features = false } stdx.workspace = true -proc-macro-srv = {workspace = true, optional = true} # span = {workspace = true, default-features = false} does not work -span = { path = "../span", version = "0.0.0", default-features = false} +span = { path = "../span", version = "0.0.0", default-features = false } + +ra-ap-rustc_lexer.workspace = true intern.workspace = true postcard.workspace = true semver.workspace = true rayon.workspace = true +[dev-dependencies] +# Enable both features for test. +proc-macro-api = { path = "../proc-macro-api", features = [ + "in-ra", + "in-proc-macro-srv", +] } + [features] -in-rust-tree = ["proc-macro-srv", "proc-macro-srv/in-rust-tree"] -default = [] +default = ["in-ra"] +in-rust-tree = ["tt/in-rust-tree"] +in-ra = ["tt/in-ra"] +in-proc-macro-srv = [] [lints] workspace = true + +[package.metadata.rust-analyzer] +rustc_private = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_lexer"] diff --git a/crates/proc-macro-api/src/bidirectional_protocol.rs b/crates/proc-macro-api/src/bidirectional_protocol.rs index f070b1c9a334..c475ebfbbe43 100644 --- a/crates/proc-macro-api/src/bidirectional_protocol.rs +++ b/crates/proc-macro-api/src/bidirectional_protocol.rs @@ -1,214 +1,8 @@ //! Bidirectional protocol methods -use std::{ - io::{self, BufRead, Write}, - panic::{AssertUnwindSafe, catch_unwind}, - sync::Arc, -}; - -use paths::AbsPath; -use span::Span; - -use crate::{ - ProcMacro, ProcMacroKind, ServerError, - bidirectional_protocol::msg::{ - ApiVersionCheck, BidirectionalMessage, ExpandMacro, ExpandMacroData, ExpnGlobals, - ListMacros, Request, Response, SubRequest, SubResponse, - }, - legacy_protocol::{ - SpanMode, - msg::{ - FlatTree, ServerConfig, SpanDataIndexMap, deserialize_span_data_index_map, - serialize_span_data_index_map, - }, - }, - process::ProcMacroServerProcess, - transport::postcard, -}; - pub mod msg; +#[cfg(feature = "in-ra")] +mod sender; -pub type SubCallback<'a> = &'a dyn Fn(SubRequest) -> Result; - -pub fn run_conversation( - writer: &mut dyn Write, - reader: &mut dyn BufRead, - buf: &mut Vec, - msg: BidirectionalMessage, - callback: SubCallback<'_>, -) -> Result { - let encoded = postcard::encode(&msg).map_err(wrap_encode)?; - postcard::write(writer, &encoded).map_err(wrap_io("failed to write initial request"))?; - - loop { - let maybe_buf = postcard::read(reader, buf).map_err(wrap_io("failed to read message"))?; - let Some(b) = maybe_buf else { - return Err(ServerError { - message: "proc-macro server closed the stream".into(), - io: Some(Arc::new(io::Error::new(io::ErrorKind::UnexpectedEof, "closed"))), - }); - }; - - let msg: BidirectionalMessage = postcard::decode(b).map_err(wrap_decode)?; - - match msg { - BidirectionalMessage::Response(response) => { - return Ok(BidirectionalMessage::Response(response)); - } - BidirectionalMessage::SubRequest(sr) => { - // TODO: Avoid `AssertUnwindSafe` by making the callback `UnwindSafe` once `SourceDatabase` - // becomes unwind-safe (currently blocked by `parking_lot::RwLock` in the VFS). - let resp = match catch_unwind(AssertUnwindSafe(|| callback(sr))) { - Ok(Ok(resp)) => BidirectionalMessage::SubResponse(resp), - Ok(Err(err)) => BidirectionalMessage::SubResponse(SubResponse::Cancel { - reason: err.to_string(), - }), - Err(_) => BidirectionalMessage::SubResponse(SubResponse::Cancel { - reason: "callback panicked or was cancelled".into(), - }), - }; - - let encoded = postcard::encode(&resp).map_err(wrap_encode)?; - postcard::write(writer, &encoded) - .map_err(wrap_io("failed to write sub-response"))?; - } - _ => { - return Err(ServerError { - message: format!("unexpected message {:?}", msg), - io: None, - }); - } - } - } -} - -fn wrap_io(msg: &'static str) -> impl Fn(io::Error) -> ServerError { - move |err| ServerError { message: msg.into(), io: Some(Arc::new(err)) } -} - -fn wrap_encode(err: io::Error) -> ServerError { - ServerError { message: "failed to encode message".into(), io: Some(Arc::new(err)) } -} - -fn wrap_decode(err: io::Error) -> ServerError { - ServerError { message: "failed to decode message".into(), io: Some(Arc::new(err)) } -} - -pub(crate) fn version_check( - srv: &ProcMacroServerProcess, - callback: SubCallback<'_>, -) -> Result { - let request = BidirectionalMessage::Request(Request::ApiVersionCheck(ApiVersionCheck {})); - - let response_payload = run_request(srv, request, callback)?; - - match response_payload { - BidirectionalMessage::Response(Response::ApiVersionCheck(version)) => Ok(version), - other => { - Err(ServerError { message: format!("unexpected response: {:?}", other), io: None }) - } - } -} - -/// Enable support for rust-analyzer span mode if the server supports it. -pub(crate) fn enable_rust_analyzer_spans( - srv: &ProcMacroServerProcess, - callback: SubCallback<'_>, -) -> Result { - let request = BidirectionalMessage::Request(Request::SetConfig(ServerConfig { - span_mode: SpanMode::RustAnalyzer, - })); - - let response_payload = run_request(srv, request, callback)?; - - match response_payload { - BidirectionalMessage::Response(Response::SetConfig(ServerConfig { span_mode })) => { - Ok(span_mode) - } - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -/// Finds proc-macros in a given dynamic library. -pub(crate) fn find_proc_macros( - srv: &ProcMacroServerProcess, - dylib_path: &AbsPath, - callback: SubCallback<'_>, -) -> Result, String>, ServerError> { - let request = BidirectionalMessage::Request(Request::ListMacros(ListMacros { - dylib_path: dylib_path.to_path_buf().into(), - })); - - let response_payload = run_request(srv, request, callback)?; - - match response_payload { - BidirectionalMessage::Response(Response::ListMacros(it)) => Ok(it), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -pub(crate) fn expand( - proc_macro: &ProcMacro, - process: &ProcMacroServerProcess, - subtree: tt::SubtreeView<'_>, - attr: Option>, - env: Vec<(String, String)>, - def_site: Span, - call_site: Span, - mixed_site: Span, - current_dir: String, - callback: SubCallback<'_>, -) -> Result, crate::ServerError> { - let version = process.version(); - let mut span_data_table = SpanDataIndexMap::default(); - let def_site = span_data_table.insert_full(def_site).0; - let call_site = span_data_table.insert_full(call_site).0; - let mixed_site = span_data_table.insert_full(mixed_site).0; - let task = BidirectionalMessage::Request(Request::ExpandMacro(Box::new(ExpandMacro { - data: ExpandMacroData { - macro_body: FlatTree::from_subtree(subtree, version, &mut span_data_table), - macro_name: proc_macro.name.to_string(), - attributes: attr - .map(|subtree| FlatTree::from_subtree(subtree, version, &mut span_data_table)), - has_global_spans: ExpnGlobals { def_site, call_site, mixed_site }, - span_data_table: if process.rust_analyzer_spans() { - serialize_span_data_index_map(&span_data_table) - } else { - Vec::new() - }, - }, - lib: proc_macro.dylib_path.to_path_buf().into(), - env, - current_dir: Some(current_dir), - }))); - - let response_payload = run_request(process, task, callback)?; - - match response_payload { - BidirectionalMessage::Response(Response::ExpandMacro(it)) => Ok(it - .map(|resp| { - FlatTree::to_subtree_resolved( - resp.tree, - version, - &deserialize_span_data_index_map(&resp.span_data_table), - ) - }) - .map_err(|msg| msg.0)), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -fn run_request( - srv: &ProcMacroServerProcess, - msg: BidirectionalMessage, - callback: SubCallback<'_>, -) -> Result { - if let Some(err) = srv.exited() { - return Err(err.clone()); - } - srv.run_bidirectional(msg, callback) -} - -pub fn reject_subrequests(req: SubRequest) -> Result { - Err(ServerError { message: format!("{req:?} sub-request not supported here"), io: None }) -} +#[cfg(feature = "in-ra")] +pub use self::sender::*; diff --git a/crates/proc-macro-api/src/bidirectional_protocol/msg.rs b/crates/proc-macro-api/src/bidirectional_protocol/msg.rs index e516297f0619..5094ba0b0453 100644 --- a/crates/proc-macro-api/src/bidirectional_protocol/msg.rs +++ b/crates/proc-macro-api/src/bidirectional_protocol/msg.rs @@ -12,7 +12,8 @@ use serde::{Deserialize, Serialize}; use crate::{ ProcMacroKind, - legacy_protocol::msg::{FlatTree, Message, PanicMessage, ServerConfig}, + flat::FlatTree, + legacy_protocol::msg::{Message, PanicMessage, ServerConfig}, transport::postcard, }; diff --git a/crates/proc-macro-api/src/bidirectional_protocol/sender.rs b/crates/proc-macro-api/src/bidirectional_protocol/sender.rs new file mode 100644 index 000000000000..0bf571b3bfb2 --- /dev/null +++ b/crates/proc-macro-api/src/bidirectional_protocol/sender.rs @@ -0,0 +1,210 @@ +//! Functions for the sender side, i.e. the rust-analyzer side. + +use std::{ + io::{self, BufRead, Write}, + panic::{AssertUnwindSafe, catch_unwind}, + sync::Arc, +}; + +use paths::AbsPath; +use span::Span; + +use crate::{ + ProcMacroKind, + bidirectional_protocol::msg::{ + ApiVersionCheck, BidirectionalMessage, ExpandMacro, ExpandMacroData, ExpnGlobals, + ListMacros, Request, Response, SubRequest, SubResponse, + }, + client::{ProcMacro, ServerError}, + flat::{ + FlatTree, SpanDataIndexMap, deserialize_span_data_index_map, serialize_span_data_index_map, + }, + legacy_protocol::msg::{ServerConfig, SpanMode}, + process::ProcMacroServerProcess, + transport::postcard, +}; + +pub type SubCallback<'a> = &'a dyn Fn(SubRequest) -> Result; + +pub fn run_conversation( + writer: &mut dyn Write, + reader: &mut dyn BufRead, + buf: &mut Vec, + msg: BidirectionalMessage, + callback: SubCallback<'_>, +) -> Result { + let encoded = postcard::encode(&msg).map_err(wrap_encode)?; + postcard::write(writer, &encoded).map_err(wrap_io("failed to write initial request"))?; + + loop { + let maybe_buf = postcard::read(reader, buf).map_err(wrap_io("failed to read message"))?; + let Some(b) = maybe_buf else { + return Err(ServerError { + message: "proc-macro server closed the stream".into(), + io: Some(Arc::new(io::Error::new(io::ErrorKind::UnexpectedEof, "closed"))), + }); + }; + + let msg: BidirectionalMessage = postcard::decode(b).map_err(wrap_decode)?; + + match msg { + BidirectionalMessage::Response(response) => { + return Ok(BidirectionalMessage::Response(response)); + } + BidirectionalMessage::SubRequest(sr) => { + // TODO: Avoid `AssertUnwindSafe` by making the callback `UnwindSafe` once `SourceDatabase` + // becomes unwind-safe (currently blocked by `parking_lot::RwLock` in the VFS). + let resp = match catch_unwind(AssertUnwindSafe(|| callback(sr))) { + Ok(Ok(resp)) => BidirectionalMessage::SubResponse(resp), + Ok(Err(err)) => BidirectionalMessage::SubResponse(SubResponse::Cancel { + reason: err.to_string(), + }), + Err(_) => BidirectionalMessage::SubResponse(SubResponse::Cancel { + reason: "callback panicked or was cancelled".into(), + }), + }; + + let encoded = postcard::encode(&resp).map_err(wrap_encode)?; + postcard::write(writer, &encoded) + .map_err(wrap_io("failed to write sub-response"))?; + } + _ => { + return Err(ServerError { + message: format!("unexpected message {:?}", msg), + io: None, + }); + } + } + } +} + +fn wrap_io(msg: &'static str) -> impl Fn(io::Error) -> ServerError { + move |err| ServerError { message: msg.into(), io: Some(Arc::new(err)) } +} + +fn wrap_encode(err: io::Error) -> ServerError { + ServerError { message: "failed to encode message".into(), io: Some(Arc::new(err)) } +} + +fn wrap_decode(err: io::Error) -> ServerError { + ServerError { message: "failed to decode message".into(), io: Some(Arc::new(err)) } +} + +pub(crate) fn version_check( + srv: &ProcMacroServerProcess, + callback: SubCallback<'_>, +) -> Result { + let request = BidirectionalMessage::Request(Request::ApiVersionCheck(ApiVersionCheck {})); + + let response_payload = run_request(srv, request, callback)?; + + match response_payload { + BidirectionalMessage::Response(Response::ApiVersionCheck(version)) => Ok(version), + other => { + Err(ServerError { message: format!("unexpected response: {:?}", other), io: None }) + } + } +} + +/// Enable support for rust-analyzer span mode if the server supports it. +pub(crate) fn enable_rust_analyzer_spans( + srv: &ProcMacroServerProcess, + callback: SubCallback<'_>, +) -> Result { + let request = BidirectionalMessage::Request(Request::SetConfig(ServerConfig { + span_mode: SpanMode::RustAnalyzer, + })); + + let response_payload = run_request(srv, request, callback)?; + + match response_payload { + BidirectionalMessage::Response(Response::SetConfig(ServerConfig { span_mode })) => { + Ok(span_mode) + } + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +/// Finds proc-macros in a given dynamic library. +pub(crate) fn find_proc_macros( + srv: &ProcMacroServerProcess, + dylib_path: &AbsPath, + callback: SubCallback<'_>, +) -> Result, String>, ServerError> { + let request = BidirectionalMessage::Request(Request::ListMacros(ListMacros { + dylib_path: dylib_path.to_path_buf().into(), + })); + + let response_payload = run_request(srv, request, callback)?; + + match response_payload { + BidirectionalMessage::Response(Response::ListMacros(it)) => Ok(it), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +pub(crate) fn expand( + proc_macro: &ProcMacro, + process: &ProcMacroServerProcess, + subtree: tt::SubtreeView<'_>, + attr: Option>, + env: Vec<(String, String)>, + def_site: Span, + call_site: Span, + mixed_site: Span, + current_dir: String, + callback: SubCallback<'_>, +) -> Result, ServerError> { + let version = process.version(); + let mut span_data_table = SpanDataIndexMap::default(); + let def_site = span_data_table.insert_full(def_site).0; + let call_site = span_data_table.insert_full(call_site).0; + let mixed_site = span_data_table.insert_full(mixed_site).0; + let task = BidirectionalMessage::Request(Request::ExpandMacro(Box::new(ExpandMacro { + data: ExpandMacroData { + macro_body: FlatTree::from_subtree(subtree, version, &mut span_data_table), + macro_name: proc_macro.name.to_string(), + attributes: attr + .map(|subtree| FlatTree::from_subtree(subtree, version, &mut span_data_table)), + has_global_spans: ExpnGlobals { def_site, call_site, mixed_site }, + span_data_table: if process.rust_analyzer_spans() { + serialize_span_data_index_map(&span_data_table) + } else { + Vec::new() + }, + }, + lib: proc_macro.dylib_path.to_path_buf().into(), + env, + current_dir: Some(current_dir), + }))); + + let response_payload = run_request(process, task, callback)?; + + match response_payload { + BidirectionalMessage::Response(Response::ExpandMacro(it)) => Ok(it + .map(|resp| { + FlatTree::to_subtree( + resp.tree, + version, + &deserialize_span_data_index_map(&resp.span_data_table), + ) + }) + .map_err(|msg| msg.0)), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +fn run_request( + srv: &ProcMacroServerProcess, + msg: BidirectionalMessage, + callback: SubCallback<'_>, +) -> Result { + if let Some(err) = srv.exited() { + return Err(err.clone()); + } + srv.run_bidirectional(msg, callback) +} + +pub fn reject_subrequests(req: SubRequest) -> Result { + Err(ServerError { message: format!("{req:?} sub-request not supported here"), io: None }) +} diff --git a/crates/proc-macro-api/src/client.rs b/crates/proc-macro-api/src/client.rs new file mode 100644 index 000000000000..23638850331f --- /dev/null +++ b/crates/proc-macro-api/src/client.rs @@ -0,0 +1,214 @@ +//! Definitions and operations for the proc macro client operated in rust-analyzer. + +use paths::{AbsPath, AbsPathBuf}; +use semver::Version; +use span::{ErasedFileAstId, FIXUP_ERASED_FILE_AST_ID_MARKER, Span}; +use std::{fmt, io, sync::Arc, time::SystemTime}; + +use crate::{ + ProcMacroKind, ProtocolFormat, + bidirectional_protocol::SubCallback, + pool::ProcMacroServerPool, + process::{self, ProcMacroServerProcess}, + version, +}; + +/// A handle to proc-macro server process pool which load dylibs with macros (.so or .dll) +/// and runs actual macro expansion functions. +#[derive(Debug, Clone)] +pub struct ProcMacroClient { + /// Currently, the proc macro process expands all procedural macros sequentially. + /// + /// That means that concurrent salsa requests may block each other when expanding proc macros, + /// which is unfortunate, but simple and good enough for the time being. + pool: Arc, + /// The path to the proc-macro server binary. + path: AbsPathBuf, +} + +/// Represents a dynamically loaded library containing procedural macros. +pub struct MacroDylib { + pub(crate) path: AbsPathBuf, +} + +impl MacroDylib { + /// Creates a new MacroDylib instance with the given path. + pub fn new(path: AbsPathBuf) -> MacroDylib { + MacroDylib { path } + } +} + +/// A handle to a specific proc-macro (a `#[proc_macro]` annotated function). +/// +/// It exists within the context of a specific proc-macro server -- currently +/// we share a single expander process for all macros within a workspace. +#[derive(Debug, Clone)] +pub struct ProcMacro { + pub(crate) pool: ProcMacroServerPool, + pub(crate) dylib_path: Arc, + pub(crate) name: Box, + pub(crate) kind: ProcMacroKind, + pub(crate) dylib_last_modified: Option, +} + +impl Eq for ProcMacro {} +impl PartialEq for ProcMacro { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + && self.kind == other.kind + && self.dylib_path == other.dylib_path + && self.dylib_last_modified == other.dylib_last_modified + } +} + +/// Represents errors encountered when communicating with the proc-macro server. +#[derive(Clone, Debug)] +pub struct ServerError { + pub message: String, + pub io: Option>, +} + +impl fmt::Display for ServerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.message.fmt(f)?; + if let Some(io) = &self.io { + f.write_str(": ")?; + io.fmt(f)?; + } + Ok(()) + } +} + +impl ProcMacroClient { + /// Spawns an external process as the proc macro server and returns a client connected to it. + pub fn spawn<'a>( + process_path: &AbsPath, + env: impl IntoIterator< + Item = (impl AsRef, &'a Option>), + > + Clone, + version: Option<&Version>, + num_process: usize, + ) -> io::Result { + let pool_size = num_process; + let mut workers = Vec::with_capacity(pool_size); + for _ in 0..pool_size { + let worker = ProcMacroServerProcess::spawn(process_path, env.clone(), version)?; + workers.push(worker); + } + + let pool = ProcMacroServerPool::new(workers); + Ok(ProcMacroClient { pool: Arc::new(pool), path: process_path.to_owned() }) + } + + /// Invokes `spawn` and returns a client connected to the resulting read and write handles. + /// + /// The `process_path` is used for `Self::server_path`. This function is mainly used for testing. + pub fn with_io_channels( + process_path: &AbsPath, + spawn: impl Fn( + Option, + ) -> io::Result<( + Box, + Box, + Box, + )> + Clone, + version: Option<&Version>, + num_process: usize, + ) -> io::Result { + let pool_size = num_process; + let mut workers = Vec::with_capacity(pool_size); + for _ in 0..pool_size { + let worker = + ProcMacroServerProcess::run(spawn.clone(), version, || "".to_owned())?; + workers.push(worker); + } + + let pool = ProcMacroServerPool::new(workers); + Ok(ProcMacroClient { pool: Arc::new(pool), path: process_path.to_owned() }) + } + + /// Returns the absolute path to the proc-macro server. + pub fn server_path(&self) -> &AbsPath { + &self.path + } + + /// Loads a proc-macro dylib into the server process returning a list of `ProcMacro`s loaded. + pub fn load_dylib(&self, dylib: MacroDylib) -> Result, ServerError> { + self.pool.load_dylib(&dylib) + } + + /// Checks if the proc-macro server has exited. + pub fn exited(&self) -> Option<&ServerError> { + self.pool.exited() + } +} + +impl ProcMacro { + /// Returns the name of the procedural macro. + pub fn name(&self) -> &str { + &self.name + } + + /// Returns the type of procedural macro. + pub fn kind(&self) -> ProcMacroKind { + self.kind + } + + pub(crate) fn needs_fixup_change(&self) -> bool { + let version = self.pool.version(); + (version::RUST_ANALYZER_SPAN_SUPPORT..version::HASHED_AST_ID).contains(&version) + } + + /// On some server versions, the fixup ast id is different than ours. So change it to match. + pub(crate) fn change_fixup_to_match_old_server(&self, tt: &mut tt::TopSubtree) { + const OLD_FIXUP_AST_ID: ErasedFileAstId = ErasedFileAstId::from_raw(!0 - 1); + tt.change_every_ast_id(|ast_id| { + if *ast_id == FIXUP_ERASED_FILE_AST_ID_MARKER { + *ast_id = OLD_FIXUP_AST_ID; + } else if *ast_id == OLD_FIXUP_AST_ID { + // Swap between them, that means no collision plus the change can be reversed by doing itself. + *ast_id = FIXUP_ERASED_FILE_AST_ID_MARKER; + } + }); + } + + /// Expands the procedural macro by sending an expansion request to the server. + /// This includes span information and environmental context. + pub fn expand( + &self, + subtree: tt::SubtreeView<'_>, + attr: Option>, + env: Vec<(String, String)>, + def_site: Span, + call_site: Span, + mixed_site: Span, + current_dir: String, + callback: Option>, + ) -> Result, ServerError> { + let (mut subtree, mut attr) = (subtree, attr); + let (mut subtree_changed, mut attr_changed); + if self.needs_fixup_change() { + subtree_changed = tt::TopSubtree::from_subtree(subtree); + self.change_fixup_to_match_old_server(&mut subtree_changed); + subtree = subtree_changed.view(); + + if let Some(attr) = &mut attr { + attr_changed = tt::TopSubtree::from_subtree(*attr); + self.change_fixup_to_match_old_server(&mut attr_changed); + *attr = attr_changed.view(); + } + } + + self.pool.pick_process()?.expand( + self, + subtree, + attr, + env, + def_site, + call_site, + mixed_site, + current_dir, + callback, + ) + } +} diff --git a/crates/proc-macro-api/src/flat.rs b/crates/proc-macro-api/src/flat.rs new file mode 100644 index 000000000000..b73792653641 --- /dev/null +++ b/crates/proc-macro-api/src/flat.rs @@ -0,0 +1,588 @@ +//! Serialization-friendly representation of `tt::TopSubtree`. +//! +//! It is possible to serialize `TopSubtree` recursively, as a tree, but using +//! arbitrary-nested trees in JSON is problematic, as they can cause the JSON +//! parser to overflow the stack. +//! +//! Additionally, such implementation would be pretty verbose, and we do care +//! about performance here a bit. +//! +//! So what this module does is dumping a `tt::TopSubtree` into a bunch of flat +//! array of numbers. +//! +//! ```json +//! { +//! // Array of subtrees, each subtree is represented by 4 numbers: +//! // id of delimiter, delimiter kind, index of first child in `token_tree`, +//! // index of last child in `token_tree` +//! "subtree":[4294967295,0,0,5,2,2,5,5], +//! // 2 ints per literal: [token id, index into `text`] +//! "literal":[4294967295,1], +//! // 3 ints per punct: [token id, char, spacing] +//! "punct":[4294967295,64,1], +//! // 2 ints per ident: [token id, index into `text`] +//! "ident": [0,0,1,1], +//! // children of all subtrees, concatenated. Each child is represented as `index << shift_indices_by | tag` +//! // where tag denotes one of subtree, literal, punct or ident. +//! "token_tree":[3,7,1,4], +//! // Strings shared by idents and literals +//! "text": ["struct","Foo"] +//! } +//! ``` +//! +//! We probably should replace most of the code here with bincode someday, but, +//! as we don't have bincode in Cargo.toml yet, let's stick with serde_json for +//! the time being. + +#[cfg(feature = "in-proc-macro-srv")] +mod proc_macro_srv_side; +#[cfg(feature = "in-ra")] +mod ra_side; + +use std::{borrow::Borrow, collections::VecDeque, marker::PhantomData}; + +use intern::Symbol; +use rustc_hash::FxHashMap; +use serde_derive::{Deserialize, Serialize}; +use span::{EditionedFileId, ErasedFileAstId, Span, SpanAnchor, SyntaxContext, TextRange}; + +use crate::{ + legacy_protocol::SpanId, + version::{ENCODE_CLOSE_SPAN_VERSION, EXTENDED_LEAF_DATA}, +}; + +pub type SpanDataIndexMap = + indexmap::IndexSet>; + +pub fn serialize_span_data_index_map(map: &SpanDataIndexMap) -> Vec { + map.iter() + .map(|span| { + [ + span.anchor.file_id.as_u32(), + span.anchor.ast_id.into_raw(), + span.range.start().into(), + span.range.end().into(), + span.ctx.into_u32(), + ] + }) + .collect::>() + .into_flattened() +} + +pub fn deserialize_span_data_index_map(map: &[u32]) -> SpanDataIndexMap { + let (chunks, remainder) = map.as_chunks(); + assert!(remainder.is_empty()); + chunks + .iter() + .map(|&[file_id, ast_id, start, end, e]| { + Span { + anchor: SpanAnchor { + file_id: EditionedFileId::from_raw(file_id), + ast_id: ErasedFileAstId::from_raw(ast_id), + }, + range: TextRange::new(start.into(), end.into()), + // SAFETY: We only receive spans from the server. If someone mess up the communication UB can happen, + // but that will be their problem. + ctx: unsafe { SyntaxContext::from_u32(e) }, + } + }) + .collect() +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct FlatTree { + subtree: Vec, + literal: Vec, + punct: Vec, + ident: Vec, + token_tree: Vec, + text: Vec, +} + +impl FlatTree { + fn deserialize<'a, ST: SpanTransformer, W: WriterTrait<'a, ST::Span>>( + top_subtree: W::Subtree, + version: u32, + span_data_table: &mut ST::Table, + ) -> FlatTree { + let mut w = Writer:: { + string_table: FxHashMap::default(), + work: VecDeque::new(), + span_data_table, + + subtree: Vec::new(), + literal: Vec::new(), + punct: Vec::new(), + ident: Vec::new(), + token_tree: Vec::new(), + text: Vec::new(), + version, + }; + w.write_subtree(top_subtree); + + FlatTree { + subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { + write_vec(w.subtree, SubtreeRepr::write_with_close_span) + } else { + write_vec(w.subtree, SubtreeRepr::write) + }, + literal: if version >= EXTENDED_LEAF_DATA { + write_vec(w.literal, LiteralRepr::write_with_kind) + } else { + write_vec(w.literal, LiteralRepr::write) + }, + punct: write_vec(w.punct, PunctRepr::write), + ident: if version >= EXTENDED_LEAF_DATA { + write_vec(w.ident, IdentRepr::write_with_rawness) + } else { + write_vec(w.ident, IdentRepr::write) + }, + token_tree: w.token_tree, + text: w.text, + } + } + + fn serialize>( + self, + version: u32, + span_data_table: &ST::Table, + ) -> (tt::Delimiter, Vec) { + Reader:: { + subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { + read_vec(self.subtree, SubtreeRepr::read_with_close_span) + } else { + read_vec(self.subtree, SubtreeRepr::read) + }, + literal: if version >= EXTENDED_LEAF_DATA { + read_vec(self.literal, LiteralRepr::read_with_kind) + } else { + read_vec(self.literal, LiteralRepr::read) + }, + punct: read_vec(self.punct, PunctRepr::read), + ident: if version >= EXTENDED_LEAF_DATA { + read_vec(self.ident, IdentRepr::read_with_rawness) + } else { + read_vec(self.ident, IdentRepr::read) + }, + token_tree: self.token_tree, + text: self.text, + span_data_table, + version, + _marker: PhantomData, + } + .read() + } +} + +#[derive(Debug)] +struct SubtreeRepr { + open: SpanId, + close: SpanId, + kind: tt::DelimiterKind, + tt: [u32; 2], +} + +#[derive(Debug)] +struct LiteralRepr { + id: SpanId, + text: u32, + suffix: u32, + kind: u16, +} + +#[derive(Debug)] +struct PunctRepr { + id: SpanId, + char: char, + spacing: tt::Spacing, +} + +#[derive(Debug)] +struct IdentRepr { + id: SpanId, + text: u32, + is_raw: bool, +} + +fn read_vec T, const N: usize>(xs: Vec, f: F) -> Vec { + let (chunks, remainder) = xs.as_chunks(); + assert!(remainder.is_empty()); + chunks.iter().map(|chunk| f(*chunk)).collect() +} + +fn write_vec [u32; N], const N: usize>(xs: Vec, f: F) -> Vec { + xs.into_iter().map(f).collect::>().into_flattened() +} + +impl SubtreeRepr { + fn write(self) -> [u32; 4] { + let kind = match self.kind { + tt::DelimiterKind::Invisible => 0, + tt::DelimiterKind::Parenthesis => 1, + tt::DelimiterKind::Brace => 2, + tt::DelimiterKind::Bracket => 3, + }; + [self.open.0, kind, self.tt[0], self.tt[1]] + } + fn read([open, kind, lo, len]: [u32; 4]) -> SubtreeRepr { + let kind = match kind { + 0 => tt::DelimiterKind::Invisible, + 1 => tt::DelimiterKind::Parenthesis, + 2 => tt::DelimiterKind::Brace, + 3 => tt::DelimiterKind::Bracket, + other => panic!("bad kind {other}"), + }; + SubtreeRepr { open: SpanId(open), close: SpanId(!0), kind, tt: [lo, len] } + } + fn write_with_close_span(self) -> [u32; 5] { + let kind = match self.kind { + tt::DelimiterKind::Invisible => 0, + tt::DelimiterKind::Parenthesis => 1, + tt::DelimiterKind::Brace => 2, + tt::DelimiterKind::Bracket => 3, + }; + [self.open.0, self.close.0, kind, self.tt[0], self.tt[1]] + } + fn read_with_close_span([open, close, kind, lo, len]: [u32; 5]) -> SubtreeRepr { + let kind = match kind { + 0 => tt::DelimiterKind::Invisible, + 1 => tt::DelimiterKind::Parenthesis, + 2 => tt::DelimiterKind::Brace, + 3 => tt::DelimiterKind::Bracket, + other => panic!("bad kind {other}"), + }; + SubtreeRepr { open: SpanId(open), close: SpanId(close), kind, tt: [lo, len] } + } +} + +impl LiteralRepr { + fn write(self) -> [u32; 2] { + [self.id.0, self.text] + } + fn read([id, text]: [u32; 2]) -> LiteralRepr { + LiteralRepr { id: SpanId(id), text, kind: 0, suffix: !0 } + } + fn write_with_kind(self) -> [u32; 4] { + [self.id.0, self.text, self.kind as u32, self.suffix] + } + fn read_with_kind([id, text, kind, suffix]: [u32; 4]) -> LiteralRepr { + LiteralRepr { id: SpanId(id), text, kind: kind as u16, suffix } + } +} + +impl PunctRepr { + fn write(self) -> [u32; 3] { + let spacing = match self.spacing { + tt::Spacing::Alone | tt::Spacing::JointHidden => 0, + tt::Spacing::Joint => 1, + }; + [self.id.0, self.char as u32, spacing] + } + fn read([id, char, spacing]: [u32; 3]) -> PunctRepr { + let spacing = match spacing { + 0 => tt::Spacing::Alone, + 1 => tt::Spacing::Joint, + other => panic!("bad spacing {other}"), + }; + PunctRepr { id: SpanId(id), char: char.try_into().unwrap(), spacing } + } +} + +impl IdentRepr { + fn write(self) -> [u32; 2] { + [self.id.0, self.text] + } + fn read(data: [u32; 2]) -> IdentRepr { + IdentRepr { id: SpanId(data[0]), text: data[1], is_raw: false } + } + fn write_with_rawness(self) -> [u32; 3] { + [self.id.0, self.text, self.is_raw as u32] + } + fn read_with_rawness([id, text, is_raw]: [u32; 3]) -> IdentRepr { + IdentRepr { id: SpanId(id), text, is_raw: is_raw == 1 } + } +} + +pub trait SpanTransformer { + type Table; + type Span: Copy + 'static; + fn token_id_of(table: &mut Self::Table, s: Self::Span) -> SpanId; + fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span; +} +impl SpanTransformer for SpanId { + type Table = (); + type Span = Self; + fn token_id_of((): &mut Self::Table, token_id: Self::Span) -> SpanId { + token_id + } + + fn span_for_token_id((): &Self::Table, id: SpanId) -> Self::Span { + id + } +} +impl SpanTransformer for Span { + type Table = SpanDataIndexMap; + type Span = Self; + fn token_id_of(table: &mut Self::Table, span: Self::Span) -> SpanId { + SpanId(table.insert_full(span).0 as u32) + } + fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span { + *table.get_index(id.0 as usize).unwrap_or_else(|| &table[0]) + } +} + +enum SubtreeOrLeafRef<'a, Span, W: WriterTrait<'a, Span>> { + Subtree(W::Subtree), + Leaf(W::Leaf), +} + +trait WriterTrait<'a, Span>: Sized { + type Subtree; + type Leaf: Borrow>; + + type SubtreeIter; + + fn subtree_data(subtree: Self::Subtree) -> (usize, tt::Delimiter, Self::SubtreeIter); + + fn subtree_iter_next(iter: &mut Self::SubtreeIter) -> Option>; +} + +struct Writer<'a, 'span, ST: SpanTransformer, W: WriterTrait<'a, ST::Span>> { + work: VecDeque<(usize, usize, W::SubtreeIter)>, + string_table: FxHashMap, u32>, + span_data_table: &'span mut ST::Table, + version: u32, + + subtree: Vec, + literal: Vec, + punct: Vec, + ident: Vec, + token_tree: Vec, + text: Vec, +} + +impl<'a, ST: SpanTransformer, W: WriterTrait<'a, ST::Span>> Writer<'a, '_, ST, W> { + fn write_subtree(&mut self, root: W::Subtree) { + self.enqueue(root); + while let Some((idx, len, subtree)) = self.work.pop_front() { + self.subtree(idx, len, subtree); + } + } + + fn subtree(&mut self, idx: usize, n_tt: usize, mut subtree: W::SubtreeIter) { + let mut first_tt = self.token_tree.len(); + self.token_tree.resize(first_tt + n_tt, !0); + + self.subtree[idx].tt = [first_tt as u32, (first_tt + n_tt) as u32]; + + while let Some(child) = W::subtree_iter_next(&mut subtree) { + let idx_tag = match child { + SubtreeOrLeafRef::Subtree(subtree) => { + let idx = self.enqueue(subtree); + idx << 2 + } + SubtreeOrLeafRef::Leaf(leaf) => match leaf.borrow() { + tt::Leaf::Literal(lit) => { + let idx = self.literal.len() as u32; + let id = self.token_id_of(lit.span); + let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA { + let (text, suffix) = lit.text_and_suffix(); + ( + self.intern_owned(text.to_owned()), + if suffix.is_empty() { + !0 + } else { + self.intern_owned(suffix.to_owned()) + }, + ) + } else { + (self.intern_owned(format!("{lit}")), !0) + }; + self.literal.push(LiteralRepr { + id, + text, + kind: u16::from_le_bytes(match lit.kind { + tt::LitKind::Err(_) => [0, 0], + tt::LitKind::Byte => [1, 0], + tt::LitKind::Char => [2, 0], + tt::LitKind::Integer => [3, 0], + tt::LitKind::Float => [4, 0], + tt::LitKind::Str => [5, 0], + tt::LitKind::StrRaw(r) => [6, r], + tt::LitKind::ByteStr => [7, 0], + tt::LitKind::ByteStrRaw(r) => [8, r], + tt::LitKind::CStr => [9, 0], + tt::LitKind::CStrRaw(r) => [10, r], + }), + suffix, + }); + (idx << 2) | 0b01 + } + tt::Leaf::Punct(punct) => { + let idx = self.punct.len() as u32; + let id = self.token_id_of(punct.span); + self.punct.push(PunctRepr { char: punct.char, spacing: punct.spacing, id }); + (idx << 2) | 0b10 + } + tt::Leaf::Ident(ident) => { + let idx = self.ident.len() as u32; + let id = self.token_id_of(ident.span); + let text = if self.version >= EXTENDED_LEAF_DATA { + self.intern_owned(ident.sym.as_str().to_owned()) + } else if ident.is_raw.yes() { + self.intern_owned(format!("r#{}", ident.sym.as_str(),)) + } else { + self.intern_owned(ident.sym.as_str().to_owned()) + }; + self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw.yes() }); + (idx << 2) | 0b11 + } + }, + }; + self.token_tree[first_tt] = idx_tag; + first_tt += 1; + } + } + + fn enqueue(&mut self, subtree: W::Subtree) -> u32 { + let idx = self.subtree.len(); + let (len, delimiter, contents) = W::subtree_data(subtree); + let open = self.token_id_of(delimiter.open); + let close = self.token_id_of(delimiter.close); + let delimiter_kind = delimiter.kind; + self.subtree.push(SubtreeRepr { open, close, kind: delimiter_kind, tt: [!0, !0] }); + self.work.push_back((idx, len, contents)); + idx as u32 + } +} + +impl<'a, ST: SpanTransformer, W: WriterTrait<'a, ST::Span>> Writer<'a, '_, ST, W> { + fn token_id_of(&mut self, span: ST::Span) -> SpanId { + ST::token_id_of(self.span_data_table, span) + } + + pub(crate) fn intern_owned(&mut self, text: String) -> u32 { + let table = &mut self.text; + *self.string_table.entry(text.clone().into()).or_insert_with(|| { + let idx = table.len(); + table.push(text); + idx as u32 + }) + } +} + +trait ReaderTrait { + type TokenTree; + + fn leaf(leaf: tt::Leaf) -> Self::TokenTree; + + fn append_subtree( + delimiter: tt::Delimiter, + children: Vec, + insert_into: &mut Vec, + ); +} + +struct Reader<'span, ST: SpanTransformer, R: ReaderTrait> { + version: u32, + subtree: Vec, + literal: Vec, + punct: Vec, + ident: Vec, + token_tree: Vec, + text: Vec, + span_data_table: &'span ST::Table, + _marker: PhantomData, +} + +impl> Reader<'_, ST, R> { + pub(crate) fn read(self) -> (tt::Delimiter, Vec) { + let mut res: Vec, Vec)>> = + (0..self.subtree.len()).map(|_| None).collect(); + let read_span = |id| ST::span_for_token_id(self.span_data_table, id); + for i in (0..self.subtree.len()).rev() { + let repr = &self.subtree[i]; + let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize]; + let delimiter = tt::Delimiter { + open: read_span(repr.open), + close: read_span(repr.close), + kind: repr.kind, + }; + let mut s = Vec::new(); + for &idx_tag in token_trees { + let tag = idx_tag & 0b11; + let idx = (idx_tag >> 2) as usize; + match tag { + // XXX: we iterate subtrees in reverse to guarantee + // that this unwrap doesn't fire. + 0b00 => { + let (delimiter, subtree) = res[idx].take().unwrap(); + R::append_subtree(delimiter, subtree, &mut s); + } + 0b01 => { + use tt::LitKind::*; + let repr = &self.literal[idx]; + let text = self.text[repr.text as usize].as_str(); + let span = read_span(repr.id); + s.push(R::leaf(tt::Leaf::Literal(if self.version >= EXTENDED_LEAF_DATA { + tt::Literal::new( + text, + span, + match u16::to_le_bytes(repr.kind) { + [0, _] => Err(()), + [1, _] => Byte, + [2, _] => Char, + [3, _] => Integer, + [4, _] => Float, + [5, _] => Str, + [6, r] => StrRaw(r), + [7, _] => ByteStr, + [8, r] => ByteStrRaw(r), + [9, _] => CStr, + [10, r] => CStrRaw(r), + _ => unreachable!(), + }, + if repr.suffix != !0 { + self.text[repr.suffix as usize].as_str() + } else { + "" + }, + ) + } else { + tt::literal_from_str_or_err(text, span) + }))) + } + 0b10 => { + let repr = &self.punct[idx]; + s.push(R::leaf(tt::Leaf::Punct(tt::Punct { + char: repr.char, + spacing: repr.spacing, + span: read_span(repr.id), + }))) + } + 0b11 => { + let repr = &self.ident[idx]; + let text = self.text[repr.text as usize].as_str(); + let (is_raw, text) = if self.version >= EXTENDED_LEAF_DATA { + ( + if repr.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No }, + text, + ) + } else { + tt::IdentIsRaw::split_from_symbol(text) + }; + s.push(R::leaf(tt::Leaf::Ident(tt::Ident { + sym: Symbol::intern(text), + span: read_span(repr.id), + is_raw, + }))) + } + other => panic!("bad tag: {other}"), + } + } + res[i] = Some((delimiter, s)); + } + + res[0].take().unwrap() + } +} diff --git a/crates/proc-macro-api/src/flat/proc_macro_srv_side.rs b/crates/proc-macro-api/src/flat/proc_macro_srv_side.rs new file mode 100644 index 000000000000..03701f3f2660 --- /dev/null +++ b/crates/proc-macro-api/src/flat/proc_macro_srv_side.rs @@ -0,0 +1,87 @@ +//! Conversion from/to the flat tree for the proc macro server. + +use crate::{ + flat::{FlatTree, ReaderTrait, SpanTransformer, SubtreeOrLeafRef, WriterTrait}, + token_stream::{Group, SpanLike, TokenStream, TokenTree}, +}; + +struct Writer; + +impl<'a, Span: Copy + 'a> WriterTrait<'a, Span> for Writer { + type Subtree = &'a Group; + type Leaf = &'a tt::Leaf; + + type SubtreeIter = Option>>; + + fn subtree_data(subtree: Self::Subtree) -> (usize, tt::Delimiter, Self::SubtreeIter) { + (subtree.stream_len(), subtree.delimiter, subtree.stream.as_ref().map(|it| it.iter())) + } + + fn subtree_iter_next(iter: &mut Self::SubtreeIter) -> Option> { + iter.as_mut()?.next().map(|item| match item { + TokenTree::Leaf(leaf) => SubtreeOrLeafRef::Leaf(leaf), + TokenTree::Group(group) => SubtreeOrLeafRef::Subtree(group), + }) + } +} + +struct Reader; + +impl ReaderTrait for Reader { + type TokenTree = TokenTree; + + fn leaf(leaf: tt::Leaf) -> Self::TokenTree { + TokenTree::Leaf(leaf) + } + + fn append_subtree( + delimiter: tt::Delimiter, + children: Vec, + insert_into: &mut Vec, + ) { + insert_into.push(TokenTree::Group(Group { + delimiter, + stream: TokenStream::new_or_empty(children), + })); + } +} + +impl FlatTree { + pub fn from_tokenstream( + tokenstream: TokenStream, + call_site: ST::Span, + version: u32, + span_data_table: &mut ST::Table, + ) -> FlatTree { + let root = if let Some(group) = tokenstream.as_single_group() { + group.clone() + } else { + Group { + delimiter: tt::Delimiter { + open: call_site, + close: call_site, + kind: tt::DelimiterKind::Invisible, + }, + stream: Some(tokenstream), + } + }; + FlatTree::deserialize::(&root, version, span_data_table) + } + + pub fn to_tokenstream>( + self, + version: u32, + span_data_table: &ST::Table, + ) -> TokenStream { + let (top_delimiter, top_children) = self.serialize::(version, span_data_table); + let result = if top_delimiter.kind == tt::DelimiterKind::Invisible { + top_children + } else { + vec![TokenTree::Group(Group { + delimiter: top_delimiter, + stream: TokenStream::new_or_empty(top_children), + })] + }; + TokenStream::new(result) + } +} diff --git a/crates/proc-macro-api/src/flat/ra_side.rs b/crates/proc-macro-api/src/flat/ra_side.rs new file mode 100644 index 000000000000..bd42f4461463 --- /dev/null +++ b/crates/proc-macro-api/src/flat/ra_side.rs @@ -0,0 +1,73 @@ +//! Conversion from/to the flat tree for rust-analyzer. + +use tt::Span; + +use crate::flat::{FlatTree, ReaderTrait, SpanDataIndexMap, SubtreeOrLeafRef, WriterTrait}; + +struct Writer; + +impl<'a> WriterTrait<'a, Span> for Writer { + type Subtree = (tt::Subtree, tt::TtIter<'a>); + type Leaf = tt::Leaf; + + type SubtreeIter = tt::TtIter<'a>; + + fn subtree_data((subtree, iter): Self::Subtree) -> (usize, tt::Delimiter, Self::SubtreeIter) { + // FIXME: `count()` walks over the iterator. + (iter.clone().count(), subtree.delimiter, iter) + } + + fn subtree_iter_next(iter: &mut Self::SubtreeIter) -> Option> { + iter.next().map(|item| match item { + tt::TtElement::Leaf(leaf) => SubtreeOrLeafRef::Leaf(leaf), + tt::TtElement::Subtree(subtree, iter) => SubtreeOrLeafRef::Subtree((subtree, iter)), + }) + } +} + +struct Reader; + +impl ReaderTrait for Reader { + type TokenTree = tt::TokenTree; + + fn leaf(leaf: tt::Leaf) -> Self::TokenTree { + tt::TokenTree::Leaf(leaf) + } + + fn append_subtree( + delimiter: tt::Delimiter, + children: Vec, + insert_into: &mut Vec, + ) { + insert_into + .push(tt::TokenTree::Subtree(tt::Subtree { delimiter, len: children.len() as u32 })); + insert_into.extend(children); + } +} + +impl FlatTree { + pub fn from_subtree( + subtree: tt::SubtreeView<'_>, + version: u32, + span_data_table: &mut SpanDataIndexMap, + ) -> FlatTree { + FlatTree::deserialize::( + (subtree.top_subtree(), subtree.iter()), + version, + span_data_table, + ) + } + + pub fn to_subtree(self, version: u32, span_data_table: &SpanDataIndexMap) -> tt::TopSubtree { + let (top_delimiter, mut top_children) = + self.serialize::(version, span_data_table); + top_children.insert( + 0, + tt::TokenTree::Subtree(tt::Subtree { + delimiter: top_delimiter, + len: top_children.len() as u32, + }), + ); + tt::TopSubtree::from_serialized(top_children) + } +} diff --git a/crates/proc-macro-api/src/legacy_protocol.rs b/crates/proc-macro-api/src/legacy_protocol.rs index ee1795d39c2e..929729e0492f 100644 --- a/crates/proc-macro-api/src/legacy_protocol.rs +++ b/crates/proc-macro-api/src/legacy_protocol.rs @@ -1,27 +1,11 @@ //! The initial proc-macro-srv protocol, soon to be deprecated. pub mod msg; +#[cfg(feature = "in-ra")] +mod sender; -use std::{ - io::{BufRead, Write}, - sync::Arc, -}; - -use paths::AbsPath; -use span::Span; - -use crate::{ - ProcMacro, ProcMacroKind, ServerError, - legacy_protocol::msg::{ - ExpandMacro, ExpandMacroData, ExpnGlobals, FlatTree, Message, Request, Response, - ServerConfig, SpanDataIndexMap, deserialize_span_data_index_map, - flat::serialize_span_data_index_map, - }, - process::ProcMacroServerProcess, - version, -}; - -pub(crate) use crate::legacy_protocol::msg::SpanMode; +#[cfg(feature = "in-ra")] +pub(crate) use self::sender::*; /// Legacy span type, only defined here as it is still used by the proc-macro server. /// While rust-analyzer doesn't use this anymore at all, RustRover relies on the legacy type for @@ -34,136 +18,3 @@ impl std::fmt::Debug for SpanId { self.0.fmt(f) } } - -pub(crate) fn version_check(srv: &ProcMacroServerProcess) -> Result { - let request = Request::ApiVersionCheck {}; - let response = send_task(srv, request)?; - - match response { - Response::ApiVersionCheck(version) => Ok(version), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -/// Enable support for rust-analyzer span mode if the server supports it. -pub(crate) fn enable_rust_analyzer_spans( - srv: &ProcMacroServerProcess, -) -> Result { - let request = Request::SetConfig(ServerConfig { span_mode: SpanMode::RustAnalyzer }); - let response = send_task(srv, request)?; - - match response { - Response::SetConfig(ServerConfig { span_mode }) => Ok(span_mode), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -/// Finds proc-macros in a given dynamic library. -pub(crate) fn find_proc_macros( - srv: &ProcMacroServerProcess, - dylib_path: &AbsPath, -) -> Result, String>, ServerError> { - let request = Request::ListMacros { dylib_path: dylib_path.to_path_buf().into() }; - - let response = send_task(srv, request)?; - - match response { - Response::ListMacros(it) => Ok(it), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -pub(crate) fn expand( - proc_macro: &ProcMacro, - process: &ProcMacroServerProcess, - subtree: tt::SubtreeView<'_>, - attr: Option>, - env: Vec<(String, String)>, - def_site: Span, - call_site: Span, - mixed_site: Span, - current_dir: String, -) -> Result, crate::ServerError> { - let version = process.version(); - let mut span_data_table = SpanDataIndexMap::default(); - let def_site = span_data_table.insert_full(def_site).0; - let call_site = span_data_table.insert_full(call_site).0; - let mixed_site = span_data_table.insert_full(mixed_site).0; - let task = ExpandMacro { - data: ExpandMacroData { - macro_body: FlatTree::from_subtree(subtree, version, &mut span_data_table), - macro_name: proc_macro.name.to_string(), - attributes: attr - .map(|subtree| FlatTree::from_subtree(subtree, version, &mut span_data_table)), - has_global_spans: ExpnGlobals { - serialize: version >= version::HAS_GLOBAL_SPANS, - def_site, - call_site, - mixed_site, - }, - span_data_table: if process.rust_analyzer_spans() { - serialize_span_data_index_map(&span_data_table) - } else { - Vec::new() - }, - }, - lib: proc_macro.dylib_path.to_path_buf().into(), - env, - current_dir: Some(current_dir), - }; - - let response = send_task(process, Request::ExpandMacro(Box::new(task)))?; - - match response { - Response::ExpandMacro(it) => Ok(it - .map(|tree| { - let mut expanded = FlatTree::to_subtree_resolved(tree, version, &span_data_table); - if proc_macro.needs_fixup_change() { - proc_macro.change_fixup_to_match_old_server(&mut expanded); - } - expanded - }) - .map_err(|msg| msg.0)), - Response::ExpandMacroExtended(it) => Ok(it - .map(|resp| { - let mut expanded = FlatTree::to_subtree_resolved( - resp.tree, - version, - &deserialize_span_data_index_map(&resp.span_data_table), - ); - if proc_macro.needs_fixup_change() { - proc_macro.change_fixup_to_match_old_server(&mut expanded); - } - expanded - }) - .map_err(|msg| msg.0)), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -/// Sends a request to the proc-macro server and waits for a response. -fn send_task(srv: &ProcMacroServerProcess, req: Request) -> Result { - if let Some(server_error) = srv.exited() { - return Err(server_error.clone()); - } - - srv.send_task_legacy::<_, _>(send_request, req) -} - -/// Sends a request to the server and reads the response. -fn send_request( - mut writer: &mut dyn Write, - mut reader: &mut dyn BufRead, - req: Request, - buf: &mut String, -) -> Result, ServerError> { - req.write(&mut writer).map_err(|err| ServerError { - message: "failed to write request".into(), - io: Some(Arc::new(err)), - })?; - let res = Response::read(&mut reader, buf).map_err(|err| ServerError { - message: "failed to read response".into(), - io: Some(Arc::new(err)), - })?; - Ok(res) -} diff --git a/crates/proc-macro-api/src/legacy_protocol/msg.rs b/crates/proc-macro-api/src/legacy_protocol/msg.rs index 9b71a8b70c8f..9babf6a3dbf5 100644 --- a/crates/proc-macro-api/src/legacy_protocol/msg.rs +++ b/crates/proc-macro-api/src/legacy_protocol/msg.rs @@ -1,6 +1,4 @@ //! Defines messages for cross-process message passing based on `ndjson` wire protocol -pub(crate) mod flat; -pub use self::flat::*; use std::io::{self, BufRead, Write}; @@ -8,7 +6,7 @@ use paths::Utf8PathBuf; use serde::de::DeserializeOwned; use serde_derive::{Deserialize, Serialize}; -use crate::{ProcMacroKind, transport::json}; +use crate::{ProcMacroKind, flat::FlatTree, transport::json}; /// Represents requests sent from the client to the proc-macro-srv. #[derive(Debug, Serialize, Deserialize)] @@ -376,7 +374,7 @@ mod tests { assert_eq!( tt, - back.data.macro_body.to_subtree_resolved(v, &span_data_table), + back.data.macro_body.to_subtree(v, &span_data_table), "version: {v}" ); } @@ -384,7 +382,6 @@ mod tests { } #[test] - #[cfg(feature = "in-rust-tree")] fn test_proc_macro_rpc_works_ts() { for tt in [ fixture_token_tree_top_many_none, @@ -395,18 +392,19 @@ mod tests { for v in version::RUST_ANALYZER_SPAN_SUPPORT..=version::CURRENT_API_VERSION { let mut span_data_table = Default::default(); let flat_tree = FlatTree::from_subtree(tt.view(), v, &mut span_data_table); - assert_eq!( - tt, - flat_tree.clone().to_subtree_resolved(v, &span_data_table), - "version: {v}" - ); - let ts = flat_tree.to_tokenstream_resolved(v, &span_data_table, |a, b| a.cover(b)); + assert_eq!(tt, flat_tree.clone().to_subtree(v, &span_data_table), "version: {v}"); + let ts = flat_tree.to_tokenstream::(v, &span_data_table); let call_site = *span_data_table.first().unwrap(); let mut span_data_table = Default::default(); assert_eq!( tt, - FlatTree::from_tokenstream(ts.clone(), v, call_site, &mut span_data_table) - .to_subtree_resolved(v, &span_data_table), + FlatTree::from_tokenstream::( + ts.clone(), + call_site, + v, + &mut span_data_table + ) + .to_subtree(v, &span_data_table), "version: {v}, ts:\n{ts:#?}" ); } diff --git a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs b/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs deleted file mode 100644 index b9b6247b54fa..000000000000 --- a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs +++ /dev/null @@ -1,980 +0,0 @@ -//! Serialization-friendly representation of `tt::TopSubtree`. -//! -//! It is possible to serialize `TopSubtree` recursively, as a tree, but using -//! arbitrary-nested trees in JSON is problematic, as they can cause the JSON -//! parser to overflow the stack. -//! -//! Additionally, such implementation would be pretty verbose, and we do care -//! about performance here a bit. -//! -//! So what this module does is dumping a `tt::TopSubtree` into a bunch of flat -//! array of numbers. -//! -//! ```json -//! { -//! // Array of subtrees, each subtree is represented by 4 numbers: -//! // id of delimiter, delimiter kind, index of first child in `token_tree`, -//! // index of last child in `token_tree` -//! "subtree":[4294967295,0,0,5,2,2,5,5], -//! // 2 ints per literal: [token id, index into `text`] -//! "literal":[4294967295,1], -//! // 3 ints per punct: [token id, char, spacing] -//! "punct":[4294967295,64,1], -//! // 2 ints per ident: [token id, index into `text`] -//! "ident": [0,0,1,1], -//! // children of all subtrees, concatenated. Each child is represented as `index << 2 | tag` -//! // where tag denotes one of subtree, literal, punct or ident. -//! "token_tree":[3,7,1,4], -//! // Strings shared by idents and literals -//! "text": ["struct","Foo"] -//! } -//! ``` -//! -//! We probably should replace most of the code here with bincode someday, but, -//! as we don't have bincode in Cargo.toml yet, let's stick with serde_json for -//! the time being. - -#[cfg(feature = "in-rust-tree")] -use proc_macro_srv::TokenStream; - -use std::collections::VecDeque; - -use intern::Symbol; -use rustc_hash::FxHashMap; -use serde_derive::{Deserialize, Serialize}; -use span::{EditionedFileId, ErasedFileAstId, Span, SpanAnchor, SyntaxContext, TextRange}; - -use crate::{ - legacy_protocol::SpanId, - version::{ENCODE_CLOSE_SPAN_VERSION, EXTENDED_LEAF_DATA}, -}; - -pub type SpanDataIndexMap = - indexmap::IndexSet>; - -pub fn serialize_span_data_index_map(map: &SpanDataIndexMap) -> Vec { - map.iter() - .map(|span| { - [ - span.anchor.file_id.as_u32(), - span.anchor.ast_id.into_raw(), - span.range.start().into(), - span.range.end().into(), - span.ctx.into_u32(), - ] - }) - .collect::>() - .into_flattened() -} - -pub fn deserialize_span_data_index_map(map: &[u32]) -> SpanDataIndexMap { - let (chunks, remainder) = map.as_chunks(); - assert!(remainder.is_empty()); - chunks - .iter() - .map(|&[file_id, ast_id, start, end, e]| { - Span { - anchor: SpanAnchor { - file_id: EditionedFileId::from_raw(file_id), - ast_id: ErasedFileAstId::from_raw(ast_id), - }, - range: TextRange::new(start.into(), end.into()), - // SAFETY: We only receive spans from the server. If someone mess up the communication UB can happen, - // but that will be their problem. - ctx: unsafe { SyntaxContext::from_u32(e) }, - } - }) - .collect() -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct FlatTree { - subtree: Vec, - literal: Vec, - punct: Vec, - ident: Vec, - token_tree: Vec, - text: Vec, -} - -struct SubtreeRepr { - open: SpanId, - close: SpanId, - kind: tt::DelimiterKind, - tt: [u32; 2], -} - -struct LiteralRepr { - id: SpanId, - text: u32, - suffix: u32, - kind: u16, -} - -struct PunctRepr { - id: SpanId, - char: char, - spacing: tt::Spacing, -} - -struct IdentRepr { - id: SpanId, - text: u32, - is_raw: bool, -} - -impl FlatTree { - pub fn from_subtree( - subtree: tt::SubtreeView<'_>, - version: u32, - span_data_table: &mut SpanDataIndexMap, - ) -> FlatTree { - let mut w = Writer:: { - string_table: FxHashMap::default(), - work: VecDeque::new(), - span_data_table, - - subtree: Vec::new(), - literal: Vec::new(), - punct: Vec::new(), - ident: Vec::new(), - token_tree: Vec::new(), - text: Vec::new(), - version, - }; - w.write_subtree(subtree); - - FlatTree { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - write_vec(w.subtree, SubtreeRepr::write_with_close_span) - } else { - write_vec(w.subtree, SubtreeRepr::write) - }, - literal: if version >= EXTENDED_LEAF_DATA { - write_vec(w.literal, LiteralRepr::write_with_kind) - } else { - write_vec(w.literal, LiteralRepr::write) - }, - punct: write_vec(w.punct, PunctRepr::write), - ident: if version >= EXTENDED_LEAF_DATA { - write_vec(w.ident, IdentRepr::write_with_rawness) - } else { - write_vec(w.ident, IdentRepr::write) - }, - token_tree: w.token_tree, - text: w.text, - } - } - - pub fn to_subtree_resolved( - self, - version: u32, - span_data_table: &SpanDataIndexMap, - ) -> tt::TopSubtree { - Reader:: { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - read_vec(self.subtree, SubtreeRepr::read_with_close_span) - } else { - read_vec(self.subtree, SubtreeRepr::read) - }, - literal: if version >= EXTENDED_LEAF_DATA { - read_vec(self.literal, LiteralRepr::read_with_kind) - } else { - read_vec(self.literal, LiteralRepr::read) - }, - punct: read_vec(self.punct, PunctRepr::read), - ident: if version >= EXTENDED_LEAF_DATA { - read_vec(self.ident, IdentRepr::read_with_rawness) - } else { - read_vec(self.ident, IdentRepr::read) - }, - token_tree: self.token_tree, - text: self.text, - span_data_table, - version, - } - .read_subtree() - } -} - -#[cfg(feature = "in-rust-tree")] -impl FlatTree { - pub fn from_tokenstream( - tokenstream: proc_macro_srv::TokenStream, - version: u32, - call_site: Span, - span_data_table: &mut SpanDataIndexMap, - ) -> FlatTree { - let mut w = Writer:: { - string_table: FxHashMap::default(), - work: VecDeque::new(), - span_data_table, - - subtree: Vec::new(), - literal: Vec::new(), - punct: Vec::new(), - ident: Vec::new(), - token_tree: Vec::new(), - text: Vec::new(), - version, - }; - w.write_tokenstream(call_site, &tokenstream); - - FlatTree { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - write_vec(w.subtree, SubtreeRepr::write_with_close_span) - } else { - write_vec(w.subtree, SubtreeRepr::write) - }, - literal: if version >= EXTENDED_LEAF_DATA { - write_vec(w.literal, LiteralRepr::write_with_kind) - } else { - write_vec(w.literal, LiteralRepr::write) - }, - punct: write_vec(w.punct, PunctRepr::write), - ident: if version >= EXTENDED_LEAF_DATA { - write_vec(w.ident, IdentRepr::write_with_rawness) - } else { - write_vec(w.ident, IdentRepr::write) - }, - token_tree: w.token_tree, - text: w.text, - } - } - - pub fn from_tokenstream_raw>( - tokenstream: proc_macro_srv::TokenStream, - call_site: T::Span, - version: u32, - ) -> FlatTree { - let mut w = Writer:: { - string_table: FxHashMap::default(), - work: VecDeque::new(), - span_data_table: &mut (), - - subtree: Vec::new(), - literal: Vec::new(), - punct: Vec::new(), - ident: Vec::new(), - token_tree: Vec::new(), - text: Vec::new(), - version, - }; - w.write_tokenstream(call_site, &tokenstream); - - FlatTree { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - write_vec(w.subtree, SubtreeRepr::write_with_close_span) - } else { - write_vec(w.subtree, SubtreeRepr::write) - }, - literal: if version >= EXTENDED_LEAF_DATA { - write_vec(w.literal, LiteralRepr::write_with_kind) - } else { - write_vec(w.literal, LiteralRepr::write) - }, - punct: write_vec(w.punct, PunctRepr::write), - ident: if version >= EXTENDED_LEAF_DATA { - write_vec(w.ident, IdentRepr::write_with_rawness) - } else { - write_vec(w.ident, IdentRepr::write) - }, - token_tree: w.token_tree, - text: w.text, - } - } - - pub fn to_tokenstream_unresolved>( - self, - version: u32, - span_join: impl Fn(T::Span, T::Span) -> T::Span, - ) -> proc_macro_srv::TokenStream { - Reader:: { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - read_vec(self.subtree, SubtreeRepr::read_with_close_span) - } else { - read_vec(self.subtree, SubtreeRepr::read) - }, - literal: if version >= EXTENDED_LEAF_DATA { - read_vec(self.literal, LiteralRepr::read_with_kind) - } else { - read_vec(self.literal, LiteralRepr::read) - }, - punct: read_vec(self.punct, PunctRepr::read), - ident: if version >= EXTENDED_LEAF_DATA { - read_vec(self.ident, IdentRepr::read_with_rawness) - } else { - read_vec(self.ident, IdentRepr::read) - }, - token_tree: self.token_tree, - text: self.text, - span_data_table: &(), - version, - } - .read_tokenstream(span_join) - } - - pub fn to_tokenstream_resolved( - self, - version: u32, - span_data_table: &SpanDataIndexMap, - span_join: impl Fn(Span, Span) -> Span, - ) -> proc_macro_srv::TokenStream { - Reader:: { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - read_vec(self.subtree, SubtreeRepr::read_with_close_span) - } else { - read_vec(self.subtree, SubtreeRepr::read) - }, - literal: if version >= EXTENDED_LEAF_DATA { - read_vec(self.literal, LiteralRepr::read_with_kind) - } else { - read_vec(self.literal, LiteralRepr::read) - }, - punct: read_vec(self.punct, PunctRepr::read), - ident: if version >= EXTENDED_LEAF_DATA { - read_vec(self.ident, IdentRepr::read_with_rawness) - } else { - read_vec(self.ident, IdentRepr::read) - }, - token_tree: self.token_tree, - text: self.text, - span_data_table, - version, - } - .read_tokenstream(span_join) - } -} - -fn read_vec T, const N: usize>(xs: Vec, f: F) -> Vec { - let (chunks, remainder) = xs.as_chunks(); - assert!(remainder.is_empty()); - chunks.iter().map(|chunk| f(*chunk)).collect() -} - -fn write_vec [u32; N], const N: usize>(xs: Vec, f: F) -> Vec { - xs.into_iter().map(f).collect::>().into_flattened() -} - -impl SubtreeRepr { - fn write(self) -> [u32; 4] { - let kind = match self.kind { - tt::DelimiterKind::Invisible => 0, - tt::DelimiterKind::Parenthesis => 1, - tt::DelimiterKind::Brace => 2, - tt::DelimiterKind::Bracket => 3, - }; - [self.open.0, kind, self.tt[0], self.tt[1]] - } - fn read([open, kind, lo, len]: [u32; 4]) -> SubtreeRepr { - let kind = match kind { - 0 => tt::DelimiterKind::Invisible, - 1 => tt::DelimiterKind::Parenthesis, - 2 => tt::DelimiterKind::Brace, - 3 => tt::DelimiterKind::Bracket, - other => panic!("bad kind {other}"), - }; - SubtreeRepr { open: SpanId(open), close: SpanId(!0), kind, tt: [lo, len] } - } - fn write_with_close_span(self) -> [u32; 5] { - let kind = match self.kind { - tt::DelimiterKind::Invisible => 0, - tt::DelimiterKind::Parenthesis => 1, - tt::DelimiterKind::Brace => 2, - tt::DelimiterKind::Bracket => 3, - }; - [self.open.0, self.close.0, kind, self.tt[0], self.tt[1]] - } - fn read_with_close_span([open, close, kind, lo, len]: [u32; 5]) -> SubtreeRepr { - let kind = match kind { - 0 => tt::DelimiterKind::Invisible, - 1 => tt::DelimiterKind::Parenthesis, - 2 => tt::DelimiterKind::Brace, - 3 => tt::DelimiterKind::Bracket, - other => panic!("bad kind {other}"), - }; - SubtreeRepr { open: SpanId(open), close: SpanId(close), kind, tt: [lo, len] } - } -} - -impl LiteralRepr { - fn write(self) -> [u32; 2] { - [self.id.0, self.text] - } - fn read([id, text]: [u32; 2]) -> LiteralRepr { - LiteralRepr { id: SpanId(id), text, kind: 0, suffix: !0 } - } - fn write_with_kind(self) -> [u32; 4] { - [self.id.0, self.text, self.kind as u32, self.suffix] - } - fn read_with_kind([id, text, kind, suffix]: [u32; 4]) -> LiteralRepr { - LiteralRepr { id: SpanId(id), text, kind: kind as u16, suffix } - } -} - -impl PunctRepr { - fn write(self) -> [u32; 3] { - let spacing = match self.spacing { - tt::Spacing::Alone | tt::Spacing::JointHidden => 0, - tt::Spacing::Joint => 1, - }; - [self.id.0, self.char as u32, spacing] - } - fn read([id, char, spacing]: [u32; 3]) -> PunctRepr { - let spacing = match spacing { - 0 => tt::Spacing::Alone, - 1 => tt::Spacing::Joint, - other => panic!("bad spacing {other}"), - }; - PunctRepr { id: SpanId(id), char: char.try_into().unwrap(), spacing } - } -} - -impl IdentRepr { - fn write(self) -> [u32; 2] { - [self.id.0, self.text] - } - fn read(data: [u32; 2]) -> IdentRepr { - IdentRepr { id: SpanId(data[0]), text: data[1], is_raw: false } - } - fn write_with_rawness(self) -> [u32; 3] { - [self.id.0, self.text, self.is_raw as u32] - } - fn read_with_rawness([id, text, is_raw]: [u32; 3]) -> IdentRepr { - IdentRepr { id: SpanId(id), text, is_raw: is_raw == 1 } - } -} - -pub trait SpanTransformer { - type Table; - type Span: Copy; - fn token_id_of(table: &mut Self::Table, s: Self::Span) -> SpanId; - fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span; -} -impl SpanTransformer for SpanId { - type Table = (); - type Span = Self; - fn token_id_of((): &mut Self::Table, token_id: Self::Span) -> SpanId { - token_id - } - - fn span_for_token_id((): &Self::Table, id: SpanId) -> Self::Span { - id - } -} -impl SpanTransformer for Span { - type Table = SpanDataIndexMap; - type Span = Self; - fn token_id_of(table: &mut Self::Table, span: Self::Span) -> SpanId { - SpanId(table.insert_full(span).0 as u32) - } - fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span { - *table.get_index(id.0 as usize).unwrap_or_else(|| &table[0]) - } -} - -struct Writer<'a, 'span, S: SpanTransformer, W> { - work: VecDeque<(usize, usize, W)>, - string_table: FxHashMap, u32>, - span_data_table: &'span mut S::Table, - version: u32, - - subtree: Vec, - literal: Vec, - punct: Vec, - ident: Vec, - token_tree: Vec, - text: Vec, -} - -impl<'a, T: SpanTransformer> Writer<'a, '_, T, tt::iter::TtIter<'a>> { - fn write_subtree(&mut self, root: tt::SubtreeView<'a>) { - let subtree = root.top_subtree(); - self.enqueue(&subtree, root.iter()); - while let Some((idx, len, subtree)) = self.work.pop_front() { - self.subtree(idx, len, subtree); - } - } - - #[expect( - clippy::explicit_counter_loop, - reason = "it looks better the current way since we use `first_tt` before the loop" - )] - fn subtree(&mut self, idx: usize, n_tt: usize, subtree: tt::iter::TtIter<'a>) { - let mut first_tt = self.token_tree.len(); - self.token_tree.resize(first_tt + n_tt, !0); - - self.subtree[idx].tt = [first_tt as u32, (first_tt + n_tt) as u32]; - - for child in subtree { - let idx_tag = match child { - tt::iter::TtElement::Subtree(subtree, subtree_iter) => { - let idx = self.enqueue(&subtree, subtree_iter); - idx << 2 - } - tt::iter::TtElement::Leaf(leaf) => match leaf { - tt::Leaf::Literal(lit) => { - let idx = self.literal.len() as u32; - let id = self.token_id_of(lit.span); - let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA { - let (text, suffix) = lit.text_and_suffix(); - ( - self.intern_owned(text.to_owned()), - if suffix.is_empty() { - !0 - } else { - self.intern_owned(suffix.to_owned()) - }, - ) - } else { - (self.intern_owned(format!("{lit}")), !0) - }; - self.literal.push(LiteralRepr { - id, - text, - kind: u16::from_le_bytes(match lit.kind { - tt::LitKind::Err(_) => [0, 0], - tt::LitKind::Byte => [1, 0], - tt::LitKind::Char => [2, 0], - tt::LitKind::Integer => [3, 0], - tt::LitKind::Float => [4, 0], - tt::LitKind::Str => [5, 0], - tt::LitKind::StrRaw(r) => [6, r], - tt::LitKind::ByteStr => [7, 0], - tt::LitKind::ByteStrRaw(r) => [8, r], - tt::LitKind::CStr => [9, 0], - tt::LitKind::CStrRaw(r) => [10, r], - }), - suffix, - }); - (idx << 2) | 0b01 - } - tt::Leaf::Punct(punct) => { - let idx = self.punct.len() as u32; - let id = self.token_id_of(punct.span); - self.punct.push(PunctRepr { char: punct.char, spacing: punct.spacing, id }); - (idx << 2) | 0b10 - } - tt::Leaf::Ident(ident) => { - let idx = self.ident.len() as u32; - let id = self.token_id_of(ident.span); - let text = if self.version >= EXTENDED_LEAF_DATA { - self.intern_owned(ident.sym.as_str().to_owned()) - } else if ident.is_raw.yes() { - self.intern_owned(format!("r#{}", ident.sym.as_str(),)) - } else { - self.intern_owned(ident.sym.as_str().to_owned()) - }; - self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw.yes() }); - (idx << 2) | 0b11 - } - }, - }; - self.token_tree[first_tt] = idx_tag; - first_tt += 1; - } - } - - fn enqueue(&mut self, subtree: &tt::Subtree, contents: tt::iter::TtIter<'a>) -> u32 { - let idx = self.subtree.len(); - let open = self.token_id_of(subtree.delimiter.open); - let close = self.token_id_of(subtree.delimiter.close); - let delimiter_kind = subtree.delimiter.kind; - self.subtree.push(SubtreeRepr { open, close, kind: delimiter_kind, tt: [!0, !0] }); - // FIXME: `count()` walks over the entire iterator. - self.work.push_back((idx, contents.clone().count(), contents)); - idx as u32 - } -} - -impl<'a, T: SpanTransformer, U> Writer<'a, '_, T, U> { - fn token_id_of(&mut self, span: T::Span) -> SpanId { - T::token_id_of(self.span_data_table, span) - } - - #[cfg(feature = "in-rust-tree")] - pub(crate) fn intern(&mut self, text: &'a str) -> u32 { - let table = &mut self.text; - *self.string_table.entry(text.into()).or_insert_with(|| { - let idx = table.len(); - table.push(text.to_owned()); - idx as u32 - }) - } - - pub(crate) fn intern_owned(&mut self, text: String) -> u32 { - let table = &mut self.text; - *self.string_table.entry(text.clone().into()).or_insert_with(|| { - let idx = table.len(); - table.push(text); - idx as u32 - }) - } -} - -#[cfg(feature = "in-rust-tree")] -impl<'a, T: SpanTransformer> - Writer<'a, '_, T, Option>> -{ - fn write_tokenstream( - &mut self, - call_site: T::Span, - root: &'a proc_macro_srv::TokenStream, - ) { - let call_site = self.token_id_of(call_site); - if let Some(group) = root.as_single_group() { - self.enqueue(group); - } else { - self.subtree.push(SubtreeRepr { - open: call_site, - close: call_site, - kind: tt::DelimiterKind::Invisible, - tt: [!0, !0], - }); - self.work.push_back((0, root.len(), Some(root.iter()))); - } - while let Some((idx, len, group)) = self.work.pop_front() { - self.group(idx, len, group); - } - } - - fn group( - &mut self, - idx: usize, - n_tt: usize, - group: Option>, - ) { - let mut first_tt = self.token_tree.len(); - self.token_tree.resize(first_tt + n_tt, !0); - - self.subtree[idx].tt = [first_tt as u32, (first_tt + n_tt) as u32]; - - for tt in group.into_iter().flatten() { - let idx_tag = match tt { - proc_macro_srv::TokenTree::Group(group) => { - let idx = self.enqueue(group); - idx << 2 - } - proc_macro_srv::TokenTree::Literal(lit) => { - let idx = self.literal.len() as u32; - let id = self.token_id_of(lit.span); - let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA { - ( - self.intern(lit.symbol.as_str()), - lit.suffix.as_ref().map(|s| self.intern(s.as_str())).unwrap_or(!0), - ) - } else { - (self.intern_owned(proc_macro_srv::literal_to_string(lit)), !0) - }; - self.literal.push(LiteralRepr { - id, - text, - kind: u16::from_le_bytes(match lit.kind { - proc_macro_srv::LitKind::ErrWithGuar => [0, 0], - proc_macro_srv::LitKind::Byte => [1, 0], - proc_macro_srv::LitKind::Char => [2, 0], - proc_macro_srv::LitKind::Integer => [3, 0], - proc_macro_srv::LitKind::Float => [4, 0], - proc_macro_srv::LitKind::Str => [5, 0], - proc_macro_srv::LitKind::StrRaw(r) => [6, r], - proc_macro_srv::LitKind::ByteStr => [7, 0], - proc_macro_srv::LitKind::ByteStrRaw(r) => [8, r], - proc_macro_srv::LitKind::CStr => [9, 0], - proc_macro_srv::LitKind::CStrRaw(r) => [10, r], - }), - suffix, - }); - (idx << 2) | 0b01 - } - proc_macro_srv::TokenTree::Punct(punct) => { - let idx = self.punct.len() as u32; - let id = self.token_id_of(punct.span); - self.punct.push(PunctRepr { - char: punct.ch as char, - spacing: if punct.joint { tt::Spacing::Joint } else { tt::Spacing::Alone }, - id, - }); - (idx << 2) | 0b10 - } - proc_macro_srv::TokenTree::Ident(ident) => { - let idx = self.ident.len() as u32; - let id = self.token_id_of(ident.span); - let text = if self.version >= EXTENDED_LEAF_DATA { - self.intern(ident.sym.as_str()) - } else if ident.is_raw { - self.intern_owned(format!("r#{}", ident.sym.as_str(),)) - } else { - self.intern(ident.sym.as_str()) - }; - self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw }); - (idx << 2) | 0b11 - } - }; - self.token_tree[first_tt] = idx_tag; - first_tt += 1; - } - } - - fn enqueue(&mut self, group: &'a proc_macro_srv::Group) -> u32 { - let idx = self.subtree.len(); - let open = self.token_id_of(group.span.open); - let close = self.token_id_of(group.span.close); - let delimiter_kind = match group.delimiter { - proc_macro_srv::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis, - proc_macro_srv::Delimiter::Brace => tt::DelimiterKind::Brace, - proc_macro_srv::Delimiter::Bracket => tt::DelimiterKind::Bracket, - proc_macro_srv::Delimiter::None => tt::DelimiterKind::Invisible, - }; - self.subtree.push(SubtreeRepr { open, close, kind: delimiter_kind, tt: [!0, !0] }); - self.work.push_back(( - idx, - group.stream.as_ref().map_or(0, |stream| stream.len()), - group.stream.as_ref().map(|ts| ts.iter()), - )); - idx as u32 - } -} - -struct Reader<'span, S: SpanTransformer> { - version: u32, - subtree: Vec, - literal: Vec, - punct: Vec, - ident: Vec, - token_tree: Vec, - text: Vec, - span_data_table: &'span S::Table, -} - -impl> Reader<'_, T> { - pub(crate) fn read_subtree(self) -> tt::TopSubtree { - let mut res: Vec)>> = - vec![None; self.subtree.len()]; - let read_span = |id| T::span_for_token_id(self.span_data_table, id); - for i in (0..self.subtree.len()).rev() { - let repr = &self.subtree[i]; - let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize]; - let delimiter = tt::Delimiter { - open: read_span(repr.open), - close: read_span(repr.close), - kind: repr.kind, - }; - let mut s = Vec::new(); - for &idx_tag in token_trees { - let tag = idx_tag & 0b11; - let idx = (idx_tag >> 2) as usize; - match tag { - // XXX: we iterate subtrees in reverse to guarantee - // that this unwrap doesn't fire. - 0b00 => { - let (delimiter, subtree) = res[idx].take().unwrap(); - s.push(tt::TokenTree::Subtree(tt::Subtree { - delimiter, - len: subtree.len() as u32, - })); - s.extend(subtree) - } - 0b01 => { - use tt::LitKind::*; - let repr = &self.literal[idx]; - let text = self.text[repr.text as usize].as_str(); - let span = read_span(repr.id); - s.push( - tt::Leaf::Literal(if self.version >= EXTENDED_LEAF_DATA { - tt::Literal::new( - text, - span, - match u16::to_le_bytes(repr.kind) { - [0, _] => Err(()), - [1, _] => Byte, - [2, _] => Char, - [3, _] => Integer, - [4, _] => Float, - [5, _] => Str, - [6, r] => StrRaw(r), - [7, _] => ByteStr, - [8, r] => ByteStrRaw(r), - [9, _] => CStr, - [10, r] => CStrRaw(r), - _ => unreachable!(), - }, - if repr.suffix != !0 { - self.text[repr.suffix as usize].as_str() - } else { - "" - }, - ) - } else { - tt::token_to_literal(text, span) - }) - .into(), - ) - } - 0b10 => { - let repr = &self.punct[idx]; - s.push( - tt::Leaf::Punct(tt::Punct { - char: repr.char, - spacing: repr.spacing, - span: read_span(repr.id), - }) - .into(), - ) - } - 0b11 => { - let repr = &self.ident[idx]; - let text = self.text[repr.text as usize].as_str(); - let (is_raw, text) = if self.version >= EXTENDED_LEAF_DATA { - ( - if repr.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No }, - text, - ) - } else { - tt::IdentIsRaw::split_from_symbol(text) - }; - s.push( - tt::Leaf::Ident(tt::Ident { - sym: Symbol::intern(text), - span: read_span(repr.id), - is_raw, - }) - .into(), - ) - } - other => panic!("bad tag: {other}"), - } - } - res[i] = Some((delimiter, s)); - } - - let (delimiter, mut res) = res[0].take().unwrap(); - res.insert(0, tt::TokenTree::Subtree(tt::Subtree { delimiter, len: res.len() as u32 })); - tt::TopSubtree::from_serialized(res) - } -} - -#[cfg(feature = "in-rust-tree")] -impl Reader<'_, T> { - pub(crate) fn read_tokenstream( - self, - span_join: impl Fn(T::Span, T::Span) -> T::Span, - ) -> proc_macro_srv::TokenStream { - let mut res: Vec>> = vec![None; self.subtree.len()]; - let read_span = |id| T::span_for_token_id(self.span_data_table, id); - for i in (0..self.subtree.len()).rev() { - let repr = &self.subtree[i]; - let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize]; - - let stream = token_trees - .iter() - .copied() - .map(|idx_tag| { - let tag = idx_tag & 0b11; - let idx = (idx_tag >> 2) as usize; - match tag { - // XXX: we iterate subtrees in reverse to guarantee - // that this unwrap doesn't fire. - 0b00 => proc_macro_srv::TokenTree::Group(res[idx].take().unwrap()), - 0b01 => { - let repr = &self.literal[idx]; - let text = self.text[repr.text as usize].as_str(); - let span = read_span(repr.id); - proc_macro_srv::TokenTree::Literal( - if self.version >= EXTENDED_LEAF_DATA { - proc_macro_srv::Literal { - symbol: Symbol::intern(text), - span, - kind: match u16::to_le_bytes(repr.kind) { - [0, _] => proc_macro_srv::LitKind::ErrWithGuar, - [1, _] => proc_macro_srv::LitKind::Byte, - [2, _] => proc_macro_srv::LitKind::Char, - [3, _] => proc_macro_srv::LitKind::Integer, - [4, _] => proc_macro_srv::LitKind::Float, - [5, _] => proc_macro_srv::LitKind::Str, - [6, r] => proc_macro_srv::LitKind::StrRaw(r), - [7, _] => proc_macro_srv::LitKind::ByteStr, - [8, r] => proc_macro_srv::LitKind::ByteStrRaw(r), - [9, _] => proc_macro_srv::LitKind::CStr, - [10, r] => proc_macro_srv::LitKind::CStrRaw(r), - _ => unreachable!(), - }, - suffix: if repr.suffix != !0 { - Some(Symbol::intern( - self.text[repr.suffix as usize].as_str(), - )) - } else { - None - }, - } - } else { - proc_macro_srv::literal_from_str(text, span).unwrap_or_else( - |_| proc_macro_srv::Literal { - symbol: Symbol::intern("internal error"), - span, - kind: proc_macro_srv::LitKind::ErrWithGuar, - suffix: None, - }, - ) - }, - ) - } - 0b10 => { - let repr = &self.punct[idx]; - proc_macro_srv::TokenTree::Punct(proc_macro_srv::Punct { - ch: repr.char as u8, - joint: repr.spacing == tt::Spacing::Joint, - span: read_span(repr.id), - }) - } - 0b11 => { - let repr = &self.ident[idx]; - let text = self.text[repr.text as usize].as_str(); - let (is_raw, text) = if self.version >= EXTENDED_LEAF_DATA { - ( - if repr.is_raw { - tt::IdentIsRaw::Yes - } else { - tt::IdentIsRaw::No - }, - text, - ) - } else { - tt::IdentIsRaw::split_from_symbol(text) - }; - proc_macro_srv::TokenTree::Ident(proc_macro_srv::Ident { - sym: Symbol::intern(text), - span: read_span(repr.id), - is_raw: is_raw.yes(), - }) - } - other => panic!("bad tag: {other}"), - } - }) - .collect::>(); - let open = read_span(repr.open); - let close = read_span(repr.close); - let g = proc_macro_srv::Group { - delimiter: match repr.kind { - tt::DelimiterKind::Parenthesis => proc_macro_srv::Delimiter::Parenthesis, - tt::DelimiterKind::Brace => proc_macro_srv::Delimiter::Brace, - tt::DelimiterKind::Bracket => proc_macro_srv::Delimiter::Bracket, - tt::DelimiterKind::Invisible => proc_macro_srv::Delimiter::None, - }, - stream: if stream.is_empty() { None } else { Some(TokenStream::new(stream)) }, - span: proc_macro_srv::DelimSpan { - open, - close, - // FIXME: The protocol does not yet encode entire spans ... - entire: span_join(open, close), - }, - }; - res[i] = Some(g); - } - let group = res[0].take().unwrap(); - if group.delimiter == proc_macro_srv::Delimiter::None { - group.stream.unwrap_or_default() - } else { - TokenStream::new(vec![proc_macro_srv::TokenTree::Group(group)]) - } - } -} diff --git a/crates/proc-macro-api/src/legacy_protocol/sender.rs b/crates/proc-macro-api/src/legacy_protocol/sender.rs new file mode 100644 index 000000000000..9b7a2f8f254b --- /dev/null +++ b/crates/proc-macro-api/src/legacy_protocol/sender.rs @@ -0,0 +1,156 @@ +//! Functions for the sender side, i.e. the rust-analyzer side. + +use std::{ + io::{BufRead, Write}, + sync::Arc, +}; + +use paths::AbsPath; +use span::Span; + +use crate::{ + ProcMacroKind, + client::{ProcMacro, ServerError}, + flat::{ + FlatTree, SpanDataIndexMap, deserialize_span_data_index_map, serialize_span_data_index_map, + }, + legacy_protocol::msg::{ + ExpandMacro, ExpandMacroData, ExpnGlobals, Message, Request, Response, ServerConfig, + SpanMode, + }, + process::ProcMacroServerProcess, + version, +}; + +pub(crate) fn version_check(srv: &ProcMacroServerProcess) -> Result { + let request = Request::ApiVersionCheck {}; + let response = send_task(srv, request)?; + + match response { + Response::ApiVersionCheck(version) => Ok(version), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +/// Enable support for rust-analyzer span mode if the server supports it. +pub(crate) fn enable_rust_analyzer_spans( + srv: &ProcMacroServerProcess, +) -> Result { + let request = Request::SetConfig(ServerConfig { span_mode: SpanMode::RustAnalyzer }); + let response = send_task(srv, request)?; + + match response { + Response::SetConfig(ServerConfig { span_mode }) => Ok(span_mode), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +/// Finds proc-macros in a given dynamic library. +pub(crate) fn find_proc_macros( + srv: &ProcMacroServerProcess, + dylib_path: &AbsPath, +) -> Result, String>, ServerError> { + let request = Request::ListMacros { dylib_path: dylib_path.to_path_buf().into() }; + + let response = send_task(srv, request)?; + + match response { + Response::ListMacros(it) => Ok(it), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +pub(crate) fn expand( + proc_macro: &ProcMacro, + process: &ProcMacroServerProcess, + subtree: tt::SubtreeView<'_>, + attr: Option>, + env: Vec<(String, String)>, + def_site: Span, + call_site: Span, + mixed_site: Span, + current_dir: String, +) -> Result, ServerError> { + let version = process.version(); + let mut span_data_table = SpanDataIndexMap::default(); + let def_site = span_data_table.insert_full(def_site).0; + let call_site = span_data_table.insert_full(call_site).0; + let mixed_site = span_data_table.insert_full(mixed_site).0; + let task = ExpandMacro { + data: ExpandMacroData { + macro_body: FlatTree::from_subtree(subtree, version, &mut span_data_table), + macro_name: proc_macro.name.to_string(), + attributes: attr + .map(|subtree| FlatTree::from_subtree(subtree, version, &mut span_data_table)), + has_global_spans: ExpnGlobals { + serialize: version >= version::HAS_GLOBAL_SPANS, + def_site, + call_site, + mixed_site, + }, + span_data_table: if process.rust_analyzer_spans() { + serialize_span_data_index_map(&span_data_table) + } else { + Vec::new() + }, + }, + lib: proc_macro.dylib_path.to_path_buf().into(), + env, + current_dir: Some(current_dir), + }; + + let response = send_task(process, Request::ExpandMacro(Box::new(task)))?; + + match response { + Response::ExpandMacro(it) => Ok(it + .map(|tree| { + let mut expanded = FlatTree::to_subtree(tree, version, &span_data_table); + if proc_macro.needs_fixup_change() { + proc_macro.change_fixup_to_match_old_server(&mut expanded); + } + expanded + }) + .map_err(|msg| msg.0)), + Response::ExpandMacroExtended(it) => Ok(it + .map(|resp| { + let mut expanded = FlatTree::to_subtree( + resp.tree, + version, + &deserialize_span_data_index_map(&resp.span_data_table), + ); + if proc_macro.needs_fixup_change() { + proc_macro.change_fixup_to_match_old_server(&mut expanded); + } + expanded + }) + .map_err(|msg| msg.0)), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +/// Sends a request to the proc-macro server and waits for a response. +fn send_task(srv: &ProcMacroServerProcess, req: Request) -> Result { + if let Some(server_error) = srv.exited() { + return Err(server_error.clone()); + } + + srv.send_task_legacy::<_, _>(send_request, req) +} + +/// Sends a request to the server and reads the response. +fn send_request( + mut writer: &mut dyn Write, + mut reader: &mut dyn BufRead, + req: Request, + buf: &mut String, +) -> Result, ServerError> { + req.write(&mut writer).map_err(|err| ServerError { + message: "failed to write request".into(), + io: Some(Arc::new(err)), + })?; + let res = Response::read(&mut reader, buf).map_err(|err| ServerError { + message: "failed to read response".into(), + io: Some(Arc::new(err)), + })?; + Ok(res) +} diff --git a/crates/proc-macro-api/src/lib.rs b/crates/proc-macro-api/src/lib.rs index 4b5e25e48801..1869a019df6d 100644 --- a/crates/proc-macro-api/src/lib.rs +++ b/crates/proc-macro-api/src/lib.rs @@ -5,30 +5,34 @@ //! is used to provide basic infrastructure for communication between two //! processes: Client (RA itself), Server (the external program) -#![cfg_attr(not(feature = "in-rust-tree"), allow(unused_crate_dependencies))] +#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] #![cfg_attr( - feature = "in-rust-tree", - feature(proc_macro_internals, proc_macro_diagnostic, proc_macro_span, rustc_private) + all(feature = "in-rust-tree", feature = "in-proc-macro-srv"), + feature(proc_macro_internals, proc_macro_diagnostic, proc_macro_span) )] #![allow(internal_features, unused_features)] #[cfg(feature = "in-rust-tree")] extern crate rustc_driver as _; +stdx::rustc_crates! { + extern crate rustc_lexer or ra_ap_rustc_lexer; +} + pub mod bidirectional_protocol; +#[cfg(feature = "in-ra")] +pub mod client; +pub mod flat; pub mod legacy_protocol; +#[cfg(feature = "in-ra")] pub mod pool; +#[cfg(feature = "in-ra")] pub mod process; +#[cfg(feature = "in-proc-macro-srv")] +pub mod token_stream; pub mod transport; -use paths::{AbsPath, AbsPathBuf}; -use semver::Version; -use span::{ErasedFileAstId, FIXUP_ERASED_FILE_AST_ID_MARKER, Span}; -use std::{fmt, io, sync::Arc, time::SystemTime}; - -use crate::{ - bidirectional_protocol::SubCallback, pool::ProcMacroServerPool, process::ProcMacroServerProcess, -}; +use std::fmt; /// The versions of the server protocol pub mod version { @@ -77,203 +81,3 @@ pub enum ProcMacroKind { #[serde(rename(serialize = "FuncLike", deserialize = "FuncLike"))] Bang, } - -/// A handle to proc-macro server process pool which load dylibs with macros (.so or .dll) -/// and runs actual macro expansion functions. -#[derive(Debug, Clone)] -pub struct ProcMacroClient { - /// Currently, the proc macro process expands all procedural macros sequentially. - /// - /// That means that concurrent salsa requests may block each other when expanding proc macros, - /// which is unfortunate, but simple and good enough for the time being. - pool: Arc, - /// The path to the proc-macro server binary. - path: AbsPathBuf, -} - -/// Represents a dynamically loaded library containing procedural macros. -pub struct MacroDylib { - path: AbsPathBuf, -} - -impl MacroDylib { - /// Creates a new MacroDylib instance with the given path. - pub fn new(path: AbsPathBuf) -> MacroDylib { - MacroDylib { path } - } -} - -/// A handle to a specific proc-macro (a `#[proc_macro]` annotated function). -/// -/// It exists within the context of a specific proc-macro server -- currently -/// we share a single expander process for all macros within a workspace. -#[derive(Debug, Clone)] -pub struct ProcMacro { - pool: ProcMacroServerPool, - dylib_path: Arc, - name: Box, - kind: ProcMacroKind, - dylib_last_modified: Option, -} - -impl Eq for ProcMacro {} -impl PartialEq for ProcMacro { - fn eq(&self, other: &Self) -> bool { - self.name == other.name - && self.kind == other.kind - && self.dylib_path == other.dylib_path - && self.dylib_last_modified == other.dylib_last_modified - } -} - -/// Represents errors encountered when communicating with the proc-macro server. -#[derive(Clone, Debug)] -pub struct ServerError { - pub message: String, - pub io: Option>, -} - -impl fmt::Display for ServerError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.message.fmt(f)?; - if let Some(io) = &self.io { - f.write_str(": ")?; - io.fmt(f)?; - } - Ok(()) - } -} - -impl ProcMacroClient { - /// Spawns an external process as the proc macro server and returns a client connected to it. - pub fn spawn<'a>( - process_path: &AbsPath, - env: impl IntoIterator< - Item = (impl AsRef, &'a Option>), - > + Clone, - version: Option<&Version>, - num_process: usize, - ) -> io::Result { - let pool_size = num_process; - let mut workers = Vec::with_capacity(pool_size); - for _ in 0..pool_size { - let worker = ProcMacroServerProcess::spawn(process_path, env.clone(), version)?; - workers.push(worker); - } - - let pool = ProcMacroServerPool::new(workers); - Ok(ProcMacroClient { pool: Arc::new(pool), path: process_path.to_owned() }) - } - - /// Invokes `spawn` and returns a client connected to the resulting read and write handles. - /// - /// The `process_path` is used for `Self::server_path`. This function is mainly used for testing. - pub fn with_io_channels( - process_path: &AbsPath, - spawn: impl Fn( - Option, - ) -> io::Result<( - Box, - Box, - Box, - )> + Clone, - version: Option<&Version>, - num_process: usize, - ) -> io::Result { - let pool_size = num_process; - let mut workers = Vec::with_capacity(pool_size); - for _ in 0..pool_size { - let worker = - ProcMacroServerProcess::run(spawn.clone(), version, || "".to_owned())?; - workers.push(worker); - } - - let pool = ProcMacroServerPool::new(workers); - Ok(ProcMacroClient { pool: Arc::new(pool), path: process_path.to_owned() }) - } - - /// Returns the absolute path to the proc-macro server. - pub fn server_path(&self) -> &AbsPath { - &self.path - } - - /// Loads a proc-macro dylib into the server process returning a list of `ProcMacro`s loaded. - pub fn load_dylib(&self, dylib: MacroDylib) -> Result, ServerError> { - self.pool.load_dylib(&dylib) - } - - /// Checks if the proc-macro server has exited. - pub fn exited(&self) -> Option<&ServerError> { - self.pool.exited() - } -} - -impl ProcMacro { - /// Returns the name of the procedural macro. - pub fn name(&self) -> &str { - &self.name - } - - /// Returns the type of procedural macro. - pub fn kind(&self) -> ProcMacroKind { - self.kind - } - - fn needs_fixup_change(&self) -> bool { - let version = self.pool.version(); - (version::RUST_ANALYZER_SPAN_SUPPORT..version::HASHED_AST_ID).contains(&version) - } - - /// On some server versions, the fixup ast id is different than ours. So change it to match. - fn change_fixup_to_match_old_server(&self, tt: &mut tt::TopSubtree) { - const OLD_FIXUP_AST_ID: ErasedFileAstId = ErasedFileAstId::from_raw(!0 - 1); - tt.change_every_ast_id(|ast_id| { - if *ast_id == FIXUP_ERASED_FILE_AST_ID_MARKER { - *ast_id = OLD_FIXUP_AST_ID; - } else if *ast_id == OLD_FIXUP_AST_ID { - // Swap between them, that means no collision plus the change can be reversed by doing itself. - *ast_id = FIXUP_ERASED_FILE_AST_ID_MARKER; - } - }); - } - - /// Expands the procedural macro by sending an expansion request to the server. - /// This includes span information and environmental context. - pub fn expand( - &self, - subtree: tt::SubtreeView<'_>, - attr: Option>, - env: Vec<(String, String)>, - def_site: Span, - call_site: Span, - mixed_site: Span, - current_dir: String, - callback: Option>, - ) -> Result, ServerError> { - let (mut subtree, mut attr) = (subtree, attr); - let (mut subtree_changed, mut attr_changed); - if self.needs_fixup_change() { - subtree_changed = tt::TopSubtree::from_subtree(subtree); - self.change_fixup_to_match_old_server(&mut subtree_changed); - subtree = subtree_changed.view(); - - if let Some(attr) = &mut attr { - attr_changed = tt::TopSubtree::from_subtree(*attr); - self.change_fixup_to_match_old_server(&mut attr_changed); - *attr = attr_changed.view(); - } - } - - self.pool.pick_process()?.expand( - self, - subtree, - attr, - env, - def_site, - call_site, - mixed_site, - current_dir, - callback, - ) - } -} diff --git a/crates/proc-macro-api/src/pool.rs b/crates/proc-macro-api/src/pool.rs index e6541823da58..1f449a12399e 100644 --- a/crates/proc-macro-api/src/pool.rs +++ b/crates/proc-macro-api/src/pool.rs @@ -3,7 +3,10 @@ use std::sync::Arc; use rayon::iter::{IntoParallelIterator, ParallelIterator}; -use crate::{MacroDylib, ProcMacro, ServerError, process::ProcMacroServerProcess}; +use crate::{ + client::{MacroDylib, ProcMacro, ServerError}, + process::ProcMacroServerProcess, +}; #[derive(Debug, Clone)] pub(crate) struct ProcMacroServerPool { diff --git a/crates/proc-macro-api/src/process.rs b/crates/proc-macro-api/src/process.rs index 035c12669c8f..2781d2b4db0b 100644 --- a/crates/proc-macro-api/src/process.rs +++ b/crates/proc-macro-api/src/process.rs @@ -17,13 +17,14 @@ use span::Span; use stdx::JodChild; use crate::{ - ProcMacro, ProcMacroKind, ProtocolFormat, ServerError, + ProcMacroKind, ProtocolFormat, bidirectional_protocol::{ self, SubCallback, msg::{BidirectionalMessage, SubResponse}, reject_subrequests, }, - legacy_protocol::{self, SpanMode}, + client::{ProcMacro, ServerError}, + legacy_protocol::{self, msg::SpanMode}, version, }; diff --git a/crates/proc-macro-api/src/token_stream.rs b/crates/proc-macro-api/src/token_stream.rs new file mode 100644 index 000000000000..d51662b4eef3 --- /dev/null +++ b/crates/proc-macro-api/src/token_stream.rs @@ -0,0 +1,691 @@ +//! The proc-macro server token stream implementation. + +use core::fmt; +use std::{mem, rc::Rc}; + +use intern::{Symbol, sym}; +use tt::{ + Delimiter, DelimiterKind, Ident, IdentIsRaw, Leaf, LitKind, Literal, Punct, Spacing, + literal_from_lexer, +}; + +/// Trait for allowing tests to parse tokenstreams with dynamic span ranges +pub trait SpanLike: Copy { + fn derive_ranged(&self, range: std::ops::Range) -> Self; + + fn cover(self, other: Self) -> Self; +} + +#[derive(Debug, Clone)] +pub struct Group { + pub delimiter: Delimiter, + pub stream: Option>, +} + +impl Group { + pub fn stream_len(&self) -> usize { + self.stream.as_ref().map_or(0, |it| it.len()) + } +} + +#[derive(Clone)] +pub enum TokenTree { + Leaf(Leaf), + Group(Group), +} + +#[derive(Clone)] +#[expect(clippy::rc_buffer, reason = "we commonly mutate this via `Rc::make_mut()`")] +pub struct TokenStream(Rc>>); + +impl Default for TokenStream { + fn default() -> Self { + Self(Default::default()) + } +} + +impl TokenStream { + #[inline] + pub fn new(tts: Vec>) -> TokenStream { + TokenStream(Rc::new(tts)) + } + + #[inline] + pub fn new_or_empty(tts: Vec>) -> Option> { + if tts.is_empty() { None } else { Some(TokenStream(Rc::new(tts))) } + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + #[inline] + pub fn len(&self) -> usize { + self.0.len() + } + + #[inline] + pub fn iter(&self) -> std::slice::Iter<'_, TokenTree> { + self.0.iter() + } + + #[inline] + pub fn as_slice(&self) -> &[TokenTree] { + &self.0 + } + + #[inline] + pub fn as_single_group(&self) -> Option<&Group> { + match &**self.0 { + [TokenTree::Group(group)] => Some(group), + _ => None, + } + } + + pub fn from_str(s: &str, span: S) -> Result + where + S: SpanLike + Copy, + { + let mut groups = Vec::new(); + groups.push((DelimiterKind::Invisible, 0..0, vec![])); + let mut offset = 0; + let mut tokens = rustc_lexer::tokenize(s, rustc_lexer::FrontmatterAllowed::No).peekable(); + while let Some(token) = tokens.next() { + let range = offset..offset + token.len as usize; + offset += token.len as usize; + + let mut spacing = || { + let is_joint = tokens.peek().is_some_and(|token| { + matches!( + token.kind, + rustc_lexer::TokenKind::RawLifetime + | rustc_lexer::TokenKind::GuardedStrPrefix + | rustc_lexer::TokenKind::Lifetime { .. } + | rustc_lexer::TokenKind::Semi + | rustc_lexer::TokenKind::Comma + | rustc_lexer::TokenKind::Dot + | rustc_lexer::TokenKind::OpenParen + | rustc_lexer::TokenKind::CloseParen + | rustc_lexer::TokenKind::OpenBrace + | rustc_lexer::TokenKind::CloseBrace + | rustc_lexer::TokenKind::OpenBracket + | rustc_lexer::TokenKind::CloseBracket + | rustc_lexer::TokenKind::At + | rustc_lexer::TokenKind::Pound + | rustc_lexer::TokenKind::Tilde + | rustc_lexer::TokenKind::Question + | rustc_lexer::TokenKind::Colon + | rustc_lexer::TokenKind::Dollar + | rustc_lexer::TokenKind::Eq + | rustc_lexer::TokenKind::Bang + | rustc_lexer::TokenKind::Lt + | rustc_lexer::TokenKind::Gt + | rustc_lexer::TokenKind::Minus + | rustc_lexer::TokenKind::And + | rustc_lexer::TokenKind::Or + | rustc_lexer::TokenKind::Plus + | rustc_lexer::TokenKind::Star + | rustc_lexer::TokenKind::Slash + | rustc_lexer::TokenKind::Percent + | rustc_lexer::TokenKind::Caret + ) + }); + if is_joint { Spacing::Joint } else { Spacing::Alone } + }; + + let Some((open_delim, _, tokenstream)) = groups.last_mut() else { + return Err("Unbalanced delimiters".to_owned()); + }; + match token.kind { + rustc_lexer::TokenKind::OpenParen => { + groups.push((DelimiterKind::Parenthesis, range, vec![])) + } + rustc_lexer::TokenKind::CloseParen if *open_delim != DelimiterKind::Parenthesis => { + return if *open_delim == DelimiterKind::Invisible { + Err("Unexpected ')'".to_owned()) + } else { + Err("Expected ')'".to_owned()) + }; + } + rustc_lexer::TokenKind::CloseParen => { + let (delimiter, open_range, stream) = groups.pop().unwrap(); + groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( + TokenTree::Group(Group { + delimiter: Delimiter { + open: span.derive_ranged(open_range), + close: span.derive_ranged(range), + kind: delimiter, + }, + stream: TokenStream::new_or_empty(stream), + }), + ); + } + rustc_lexer::TokenKind::OpenBrace => { + groups.push((DelimiterKind::Brace, range, vec![])) + } + rustc_lexer::TokenKind::CloseBrace if *open_delim != DelimiterKind::Brace => { + return if *open_delim == DelimiterKind::Invisible { + Err("Unexpected '}'".to_owned()) + } else { + Err("Expected '}'".to_owned()) + }; + } + rustc_lexer::TokenKind::CloseBrace => { + let (delimiter, open_range, stream) = groups.pop().unwrap(); + groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( + TokenTree::Group(Group { + delimiter: Delimiter { + open: span.derive_ranged(open_range), + close: span.derive_ranged(range), + kind: delimiter, + }, + stream: TokenStream::new_or_empty(stream), + }), + ); + } + rustc_lexer::TokenKind::OpenBracket => { + groups.push((DelimiterKind::Bracket, range, vec![])) + } + rustc_lexer::TokenKind::CloseBracket if *open_delim != DelimiterKind::Bracket => { + return if *open_delim == DelimiterKind::Invisible { + Err("Unexpected ']'".to_owned()) + } else { + Err("Expected ']'".to_owned()) + }; + } + rustc_lexer::TokenKind::CloseBracket => { + let (delimiter, open_range, stream) = groups.pop().unwrap(); + groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( + TokenTree::Group(Group { + delimiter: Delimiter { + open: span.derive_ranged(open_range), + close: span.derive_ranged(range), + kind: delimiter, + }, + stream: TokenStream::new_or_empty(stream), + }), + ); + } + rustc_lexer::TokenKind::LineComment { doc_style: None } + | rustc_lexer::TokenKind::BlockComment { doc_style: None, terminated: _ } => { + continue; + } + rustc_lexer::TokenKind::LineComment { doc_style: Some(doc_style) } => { + let text = &s[range.start + 3..range.end]; + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '#', + spacing: Spacing::Alone, + span, + }))); + if doc_style == rustc_lexer::DocStyle::Inner { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '!', + spacing: Spacing::Alone, + span, + }))); + } + let span = span.derive_ranged(range); + tokenstream.push(TokenTree::Group(Group { + delimiter: Delimiter { + open: span, + close: span, + kind: DelimiterKind::Bracket, + }, + stream: TokenStream::new_or_empty(vec![ + TokenTree::Leaf(Leaf::Ident(Ident { + sym: sym::doc, + is_raw: IdentIsRaw::No, + span, + })), + TokenTree::Leaf(Leaf::Punct(Punct { + char: '=', + spacing: Spacing::Alone, + span, + })), + TokenTree::Leaf(Leaf::Literal(Literal::new_no_suffix( + &text.escape_debug().to_string(), + span, + LitKind::Str, + ))), + ]), + })); + } + rustc_lexer::TokenKind::BlockComment { doc_style: Some(doc_style), terminated } => { + let text = + &s[range.start + 3..if terminated { range.end - 2 } else { range.end }]; + let span = span.derive_ranged(range); + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '#', + spacing: Spacing::Alone, + span, + }))); + if doc_style == rustc_lexer::DocStyle::Inner { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '!', + spacing: Spacing::Alone, + span, + }))); + } + tokenstream.push(TokenTree::Group(Group { + delimiter: Delimiter { + open: span, + close: span, + kind: DelimiterKind::Bracket, + }, + stream: TokenStream::new_or_empty(vec![ + TokenTree::Leaf(Leaf::Ident(Ident { + sym: sym::doc, + is_raw: IdentIsRaw::No, + span, + })), + TokenTree::Leaf(Leaf::Punct(Punct { + char: '=', + spacing: Spacing::Alone, + span, + })), + TokenTree::Leaf(Leaf::Literal(Literal::new_no_suffix( + &text.escape_debug().to_string(), + span, + LitKind::Str, + ))), + ]), + })); + } + rustc_lexer::TokenKind::Whitespace => continue, + rustc_lexer::TokenKind::Frontmatter { .. } => unreachable!(), + rustc_lexer::TokenKind::Unknown => { + return Err(format!("Unknown token: `{}`", &s[range])); + } + rustc_lexer::TokenKind::UnknownPrefix => { + return Err(format!("Unknown prefix: `{}`", &s[range])); + } + rustc_lexer::TokenKind::UnknownPrefixLifetime => { + return Err(format!("Unknown lifetime prefix: `{}`", &s[range])); + } + // FIXME: Error on edition >= 2024 ... I dont think the proc-macro server can fetch editions currently + // and whose edition is this? + rustc_lexer::TokenKind::GuardedStrPrefix => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: s.as_bytes()[range.start].into(), + spacing: Spacing::Joint, + span: span.derive_ranged(range.start..range.start + 1), + }))); + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: s.as_bytes()[range.start + 1].into(), + spacing: spacing(), + span: span.derive_ranged(range.start + 1..range.end), + }))) + } + rustc_lexer::TokenKind::Ident => { + tokenstream.push(TokenTree::Leaf(Leaf::Ident(Ident { + sym: Symbol::intern(&s[range.clone()]), + is_raw: IdentIsRaw::No, + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::InvalidIdent => { + return Err(format!("Invalid identifier: `{}`", &s[range])); + } + rustc_lexer::TokenKind::RawIdent => { + let range = range.start + 2..range.end; + tokenstream.push(TokenTree::Leaf(Leaf::Ident(Ident { + sym: Symbol::intern(&s[range.clone()]), + is_raw: IdentIsRaw::Yes, + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Literal { kind, suffix_start } => { + tokenstream.push(TokenTree::Leaf(Leaf::Literal(literal_from_lexer( + &s[range.clone()], + span.derive_ranged(range), + kind, + suffix_start, + )))) + } + rustc_lexer::TokenKind::RawLifetime => { + let range = range.start + 1 + 2..range.end; + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '\'', + spacing: Spacing::Joint, + span: span.derive_ranged(range.start..range.start + 1), + }))); + tokenstream.push(TokenTree::Leaf(Leaf::Ident(Ident { + sym: Symbol::intern(&s[range.clone()]), + is_raw: IdentIsRaw::Yes, + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Lifetime { starts_with_number } => { + if starts_with_number { + return Err("Lifetime cannot start with a number".to_owned()); + } + let range = range.start + 1..range.end; + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '\'', + spacing: Spacing::Joint, + span: span.derive_ranged(range.start..range.start + 1), + }))); + tokenstream.push(TokenTree::Leaf(Leaf::Ident(Ident { + sym: Symbol::intern(&s[range.clone()]), + is_raw: IdentIsRaw::No, + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Semi => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: ';', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Comma => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: ',', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Dot => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '.', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::At => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '@', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Pound => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '#', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Tilde => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '~', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Question => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '?', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Colon => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: ':', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Dollar => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '$', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Eq => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '=', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Bang => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '!', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Lt => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '<', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Gt => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '>', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Minus => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '-', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::And => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '&', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Or => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '|', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Plus => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '+', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Star => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '*', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Slash => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '/', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Caret => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '^', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Percent => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '%', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Eof => break, + } + } + if let Some((DelimiterKind::Invisible, _, tokentrees)) = groups.pop() + && groups.is_empty() + { + Ok(TokenStream::new(tokentrees)) + } else { + Err("Mismatched token groups".to_owned()) + } + } +} + +impl fmt::Display for TokenStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut emit_whitespace = false; + for tt in self.0.iter() { + display_token_tree(tt, &mut emit_whitespace, f)?; + } + Ok(()) + } +} + +fn display_token_tree( + tt: &TokenTree, + emit_whitespace: &mut bool, + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + if mem::take(emit_whitespace) { + write!(f, " ")?; + } + match tt { + TokenTree::Group(Group { delimiter, stream }) => { + let (open, close) = delimiter.kind.display_open_close(); + write!(f, "{open}")?; + if let Some(stream) = stream { + write!(f, "{stream}")?; + } + write!(f, "{close}")?; + } + TokenTree::Leaf(leaf) => { + fmt::Display::fmt(leaf, f)?; + *emit_whitespace = match leaf { + Leaf::Literal(literal) => !matches!( + literal.kind, + LitKind::Str + | LitKind::StrRaw(_) + | LitKind::ByteStr + | LitKind::ByteStrRaw(_) + | LitKind::CStr + | LitKind::CStrRaw(_) + ), + Leaf::Punct(punct) => punct.spacing == Spacing::Alone, + Leaf::Ident(_) => true, + }; + } + } + Ok(()) +} + +impl fmt::Debug for TokenStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + debug_token_stream(self, 0, f) + } +} + +fn debug_token_stream( + ts: &TokenStream, + depth: usize, + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + for tt in ts.0.iter() { + debug_token_tree(tt, depth, f)?; + } + Ok(()) +} + +fn debug_token_tree( + tt: &TokenTree, + depth: usize, + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + write!(f, "{:indent$}", "", indent = depth * 2)?; + + match tt { + TokenTree::Group(Group { delimiter, stream }) => { + writeln!( + f, + "GROUP {} {:#?} {:#?}", + delimiter.kind.debug_view(), + delimiter.open, + delimiter.close, + )?; + if let Some(stream) = stream { + debug_token_stream(stream, depth + 1, f)?; + } + return Ok(()); + } + TokenTree::Leaf(leaf) => leaf.print_debug(f)?, + } + writeln!(f) +} + +impl TokenStream { + pub fn extend_with_streams(&mut self, streams: std::vec::IntoIter>) { + let vec_mut = Rc::make_mut(&mut self.0); + + vec_mut.reserve(streams.as_slice().iter().map(|item| item.len()).sum()); + streams.into_iter().for_each(|item| vec_mut.extend(item.iter().cloned())); + } +} + +impl FromIterator> for TokenStream { + fn from_iter>>(iter: I) -> Self { + TokenStream::new(Vec::from_iter(iter)) + } +} + +impl Extend> for TokenStream { + fn extend>>(&mut self, iter: T) { + let vec_mut = Rc::make_mut(&mut self.0); + vec_mut.extend(iter); + } +} + +impl SpanLike for () { + fn derive_ranged(&self, _: std::ops::Range) -> Self { + *self + } + + fn cover(self, _other: Self) -> Self { + self + } +} + +impl SpanLike for span::Span { + fn derive_ranged(&self, range: std::ops::Range) -> Self { + span::Span { + range: span::TextRange::new( + span::TextSize::new(range.start as u32), + span::TextSize::new(range.end as u32), + ), + anchor: self.anchor, + ctx: self.ctx, + } + } + + fn cover(self, other: Self) -> Self { + self.cover(other) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ts_to_string() { + let token_stream = + TokenStream::from_str("{} () [] <> ;/., \"gfhdgfuiofghd\" 0f32 r#\"dff\"# 'r#lt", ()) + .unwrap(); + assert_eq!(token_stream.to_string(), "{}()[]<> ;/., \"gfhdgfuiofghd\"0f32 r#\"dff\"#'r#lt"); + } + + #[test] + fn doc_comment_from_str() { + let token_stream = TokenStream::from_str("/// foo", ()).unwrap(); + assert_eq!(token_stream.to_string(), r#"# [doc = " foo"]"#); + } +} diff --git a/crates/proc-macro-srv-cli/Cargo.toml b/crates/proc-macro-srv-cli/Cargo.toml index 44e19f2d3c3b..cfc157596cf7 100644 --- a/crates/proc-macro-srv-cli/Cargo.toml +++ b/crates/proc-macro-srv-cli/Cargo.toml @@ -15,26 +15,33 @@ doctest = false [dependencies] proc-macro-srv.workspace = true -proc-macro-api.workspace = true -clap = {version = "4.5.42", default-features = false, features = ["std"]} +proc-macro-api = { path = "../proc-macro-api", version = "0.0.0", default-features = false, features = [ + "in-proc-macro-srv", +] } +clap = { version = "4.5.42", default-features = false, features = ["std"] } +# span = { workspace = true, default-features = false } does not work +span = { path = "../span", version = "0.0.0", default-features = false } [dev-dependencies] expect-test.workspace = true paths.workspace = true -# span = {workspace = true, default-features = false} does not work -span = { path = "../span", default-features = false} -tt.workspace = true intern.workspace = true # used as proc macro test target proc-macro-test.path = "../proc-macro-srv/proc-macro-test" +# Enable the in-ra feature for tests. +proc-macro-api = { path = "../proc-macro-api", default-features = false, features = [ + "in-proc-macro-srv", + "in-ra", +] } +tt.workspace = true + [features] default = [] # default = ["in-rust-tree"] in-rust-tree = ["proc-macro-srv/in-rust-tree", "proc-macro-api/in-rust-tree"] - [[bin]] name = "rust-analyzer-proc-macro-srv" path = "src/main.rs" diff --git a/crates/proc-macro-srv-cli/src/lib.rs b/crates/proc-macro-srv-cli/src/lib.rs index c330928fbc9c..e51708a4a025 100644 --- a/crates/proc-macro-srv-cli/src/lib.rs +++ b/crates/proc-macro-srv-cli/src/lib.rs @@ -4,7 +4,88 @@ #![cfg(feature = "in-rust-tree")] #![feature(rustc_private)] +#![expect(clippy::print_stdout, clippy::print_stderr)] extern crate rustc_driver as _; pub mod main_loop; +mod version; + +use clap::{Command, ValueEnum}; +use proc_macro_api::ProtocolFormat; + +pub fn main() -> std::io::Result<()> { + let v = std::env::var("RUST_ANALYZER_INTERNALS_DO_NOT_USE"); + if v.is_err() { + eprintln!( + "This is an IDE implementation detail, you can use this tool by exporting RUST_ANALYZER_INTERNALS_DO_NOT_USE." + ); + eprintln!( + "Note that this tool's API is highly unstable and may break without prior notice" + ); + std::process::exit(122); + } + let matches = Command::new("proc-macro-srv") + .args(&[ + clap::Arg::new("format") + .long("format") + .action(clap::ArgAction::Set) + .default_value("json-legacy") + .value_parser(clap::builder::EnumValueParser::::new()), + clap::Arg::new("version") + .long("version") + .action(clap::ArgAction::SetTrue) + .help("Prints the version of the proc-macro-srv"), + ]) + .get_matches(); + if matches.get_flag("version") { + println!("rust-analyzer-proc-macro-srv {}", version::version()); + return Ok(()); + } + let &format = matches + .get_one::("format") + .expect("format value should always be present"); + + let mut stdin = std::io::BufReader::new(std::io::stdin()); + let mut stdout = std::io::stdout(); + + main_loop::run(&mut stdin, &mut stdout, format.into()) +} + +/// Wrapper for CLI argument parsing that implements `ValueEnum`. +#[derive(Copy, Clone)] +struct ProtocolFormatArg(ProtocolFormat); + +impl From for ProtocolFormat { + fn from(arg: ProtocolFormatArg) -> Self { + arg.0 + } +} + +impl ValueEnum for ProtocolFormatArg { + fn value_variants<'a>() -> &'a [Self] { + &[ + ProtocolFormatArg(ProtocolFormat::JsonLegacy), + ProtocolFormatArg(ProtocolFormat::BidirectionalPostcardPrototype), + ] + } + + fn to_possible_value(&self) -> Option { + match self.0 { + ProtocolFormat::JsonLegacy => Some(clap::builder::PossibleValue::new("json-legacy")), + ProtocolFormat::BidirectionalPostcardPrototype => { + Some(clap::builder::PossibleValue::new("bidirectional-postcard-prototype")) + } + } + } + + fn from_str(input: &str, _ignore_case: bool) -> Result { + match input { + "json-legacy" => Ok(ProtocolFormatArg(ProtocolFormat::JsonLegacy)), + "bidirectional-postcard-prototype" => { + Ok(ProtocolFormatArg(ProtocolFormat::BidirectionalPostcardPrototype)) + } + _ => Err(format!("unknown protocol format: {input}")), + } + } +} diff --git a/crates/proc-macro-srv-cli/src/main.rs b/crates/proc-macro-srv-cli/src/main.rs index 926633df628d..46181aceaa89 100644 --- a/crates/proc-macro-srv-cli/src/main.rs +++ b/crates/proc-macro-srv-cli/src/main.rs @@ -2,104 +2,13 @@ //! Driver for proc macro server #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] #![cfg_attr(not(feature = "in-rust-tree"), allow(unused_crate_dependencies))] -#![allow(clippy::print_stdout, clippy::print_stderr)] #[cfg(feature = "in-rust-tree")] extern crate rustc_driver as _; -mod version; - -use clap::{Command, ValueEnum}; -use proc_macro_api::ProtocolFormat; - -#[cfg(feature = "in-rust-tree")] -use proc_macro_srv_cli::main_loop::run; - fn main() -> std::io::Result<()> { - let v = std::env::var("RUST_ANALYZER_INTERNALS_DO_NOT_USE"); - if v.is_err() { - eprintln!( - "This is an IDE implementation detail, you can use this tool by exporting RUST_ANALYZER_INTERNALS_DO_NOT_USE." - ); - eprintln!( - "Note that this tool's API is highly unstable and may break without prior notice" - ); - std::process::exit(122); - } - let matches = Command::new("proc-macro-srv") - .args(&[ - clap::Arg::new("format") - .long("format") - .action(clap::ArgAction::Set) - .default_value("json-legacy") - .value_parser(clap::builder::EnumValueParser::::new()), - clap::Arg::new("version") - .long("version") - .action(clap::ArgAction::SetTrue) - .help("Prints the version of the proc-macro-srv"), - ]) - .get_matches(); - if matches.get_flag("version") { - println!("rust-analyzer-proc-macro-srv {}", version::version()); - return Ok(()); - } - let &format = matches - .get_one::("format") - .expect("format value should always be present"); - - let mut stdin = std::io::BufReader::new(std::io::stdin()); - let mut stdout = std::io::stdout(); - - run(&mut stdin, &mut stdout, format.into()) -} - -/// Wrapper for CLI argument parsing that implements `ValueEnum`. -#[derive(Copy, Clone)] -struct ProtocolFormatArg(ProtocolFormat); - -impl From for ProtocolFormat { - fn from(arg: ProtocolFormatArg) -> Self { - arg.0 - } -} - -impl ValueEnum for ProtocolFormatArg { - fn value_variants<'a>() -> &'a [Self] { - &[ - ProtocolFormatArg(ProtocolFormat::JsonLegacy), - ProtocolFormatArg(ProtocolFormat::BidirectionalPostcardPrototype), - ] - } - - fn to_possible_value(&self) -> Option { - match self.0 { - ProtocolFormat::JsonLegacy => Some(clap::builder::PossibleValue::new("json-legacy")), - ProtocolFormat::BidirectionalPostcardPrototype => { - Some(clap::builder::PossibleValue::new("bidirectional-postcard-prototype")) - } - } - } - - fn from_str(input: &str, _ignore_case: bool) -> Result { - match input { - "json-legacy" => Ok(ProtocolFormatArg(ProtocolFormat::JsonLegacy)), - "bidirectional-postcard-prototype" => { - Ok(ProtocolFormatArg(ProtocolFormat::BidirectionalPostcardPrototype)) - } - _ => Err(format!("unknown protocol format: {input}")), - } + cfg_select! { + feature = "in-rust-tree" => proc_macro_srv_cli::main(), + _ => Ok(()), } } - -#[cfg(not(feature = "in-rust-tree"))] -fn run( - _: &mut std::io::BufReader, - _: &mut std::io::Stdout, - _: ProtocolFormat, -) -> std::io::Result<()> { - Err(std::io::Error::new( - std::io::ErrorKind::Unsupported, - "proc-macro-srv-cli needs to be compiled with the `in-rust-tree` feature to function" - .to_owned(), - )) -} diff --git a/crates/proc-macro-srv-cli/src/main_loop.rs b/crates/proc-macro-srv-cli/src/main_loop.rs index 6697b6380dd2..4b92fccc0922 100644 --- a/crates/proc-macro-srv-cli/src/main_loop.rs +++ b/crates/proc-macro-srv-cli/src/main_loop.rs @@ -1,22 +1,24 @@ //! The main loop of the proc-macro server. -use proc_macro_api::bidirectional_protocol::msg::{ApiVersionCheck, ListMacros}; -use proc_macro_api::{ - ProtocolFormat, bidirectional_protocol::msg as bidirectional, legacy_protocol::msg as legacy, - version::CURRENT_API_VERSION, -}; + use std::panic::{panic_any, resume_unwind}; use std::{ io::{self, BufRead, Write}, ops::Range, }; -use legacy::Message; - +use proc_macro_api::{ + ProtocolFormat, + bidirectional_protocol::msg::{self as bidirectional, ApiVersionCheck, ListMacros}, + flat::{self, SpanTransformer}, + legacy_protocol::msg::{self as legacy, Message}, + version::CURRENT_API_VERSION, +}; use proc_macro_srv::{EnvSnapshot, ProcMacroClientError, ProcMacroPanicMarker, SpanId}; +use span::Span; struct SpanTrans; -impl legacy::SpanTransformer for SpanTrans { +impl SpanTransformer for SpanTrans { type Table = (); type Span = SpanId; fn token_id_of( @@ -138,14 +140,9 @@ fn handle_expand_id( let call_site = SpanId(call_site as u32); let mixed_site = SpanId(mixed_site as u32); - let macro_body = - macro_body.to_tokenstream_unresolved::(CURRENT_API_VERSION, |_, b| b); - let attributes = attributes - .map(|it| it.to_tokenstream_unresolved::(CURRENT_API_VERSION, |_, b| b)); - let mut tracked_env = Default::default(); let res = srv - .expand( + .expand::( lib, &env, current_dir, @@ -156,11 +153,9 @@ fn handle_expand_id( call_site, mixed_site, &mut tracked_env, + &mut (), None, ) - .map(|it| { - legacy::FlatTree::from_tokenstream_raw::(it, call_site, CURRENT_API_VERSION) - }) .map(|tree| bidirectional::ExpandMacroResponse { tree, span_data_table: vec![], @@ -415,27 +410,16 @@ fn handle_expand_ra( }, } = task; - let mut span_data_table = legacy::deserialize_span_data_index_map(&span_data_table); + let mut span_data_table = flat::deserialize_span_data_index_map(&span_data_table); let def_site = span_data_table[def_site]; let call_site = span_data_table[call_site]; let mixed_site = span_data_table[mixed_site]; - let macro_body = - macro_body.to_tokenstream_resolved(CURRENT_API_VERSION, &span_data_table, |a, b| { - srv.join_spans(a, b).unwrap_or(b) - }); - - let attributes = attributes.map(|it| { - it.to_tokenstream_resolved(CURRENT_API_VERSION, &span_data_table, |a, b| { - srv.join_spans(a, b).unwrap_or(b) - }) - }); - let mut tracked_env = Default::default(); let res = srv - .expand( + .expand::( lib, &env, current_dir, @@ -446,19 +430,10 @@ fn handle_expand_ra( call_site, mixed_site, &mut tracked_env, + &mut span_data_table, Some(&mut ProcMacroClientHandle { stdin, stdout, buf }), ) - .map(|it| { - ( - legacy::FlatTree::from_tokenstream( - it, - CURRENT_API_VERSION, - call_site, - &mut span_data_table, - ), - legacy::serialize_span_data_index_map(&span_data_table), - ) - }) + .map(|it| (it, flat::serialize_span_data_index_map(&span_data_table))) .map(|(tree, span_data_table)| bidirectional::ExpandMacroResponse { tree, span_data_table, @@ -521,13 +496,7 @@ fn run_old( let call_site = SpanId(call_site as u32); let mixed_site = SpanId(mixed_site as u32); - let macro_body = macro_body - .to_tokenstream_unresolved::(CURRENT_API_VERSION, |_, b| b); - let attributes = attributes.map(|it| { - it.to_tokenstream_unresolved::(CURRENT_API_VERSION, |_, b| b) - }); - - srv.expand( + srv.expand::( lib, &env, current_dir, @@ -538,39 +507,21 @@ fn run_old( call_site, mixed_site, &mut Default::default(), + &mut (), None, ) - .map(|it| { - legacy::FlatTree::from_tokenstream_raw::( - it, - call_site, - CURRENT_API_VERSION, - ) - }) .map_err(|e| e.into_string().unwrap_or_default()) .map_err(legacy::PanicMessage) }), legacy::SpanMode::RustAnalyzer => legacy::Response::ExpandMacroExtended({ let mut span_data_table = - legacy::deserialize_span_data_index_map(&span_data_table); + flat::deserialize_span_data_index_map(&span_data_table); let def_site = span_data_table[def_site]; let call_site = span_data_table[call_site]; let mixed_site = span_data_table[mixed_site]; - let macro_body = macro_body.to_tokenstream_resolved( - CURRENT_API_VERSION, - &span_data_table, - |a, b| srv.join_spans(a, b).unwrap_or(b), - ); - let attributes = attributes.map(|it| { - it.to_tokenstream_resolved( - CURRENT_API_VERSION, - &span_data_table, - |a, b| srv.join_spans(a, b).unwrap_or(b), - ) - }); - srv.expand( + srv.expand::( lib, &env, current_dir, @@ -581,19 +532,10 @@ fn run_old( call_site, mixed_site, &mut Default::default(), + &mut span_data_table, None, ) - .map(|it| { - ( - legacy::FlatTree::from_tokenstream( - it, - CURRENT_API_VERSION, - call_site, - &mut span_data_table, - ), - legacy::serialize_span_data_index_map(&span_data_table), - ) - }) + .map(|it| (it, flat::serialize_span_data_index_map(&span_data_table))) .map(|(tree, span_data_table)| legacy::ExpandMacroExtended { tree, span_data_table, diff --git a/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs b/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs index 9c55eed08e52..9f4b62daa6ac 100644 --- a/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs +++ b/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs @@ -20,7 +20,8 @@ use proc_macro_api::{ }, reject_subrequests, }, - legacy_protocol::msg::{PanicMessage, ServerConfig, SpanDataIndexMap, SpanMode}, + flat::SpanDataIndexMap, + legacy_protocol::msg::{PanicMessage, ServerConfig, SpanMode}, version::CURRENT_API_VERSION, }; diff --git a/crates/proc-macro-srv-cli/tests/common/utils.rs b/crates/proc-macro-srv-cli/tests/common/utils.rs index b78e10745274..7c7dae39b157 100644 --- a/crates/proc-macro-srv-cli/tests/common/utils.rs +++ b/crates/proc-macro-srv-cli/tests/common/utils.rs @@ -7,11 +7,12 @@ use std::{ use paths::Utf8PathBuf; use proc_macro_api::{ - ServerError, bidirectional_protocol::msg::{ BidirectionalMessage, Request as BiRequest, Response as BiResponse, SubRequest, SubResponse, }, - legacy_protocol::msg::{FlatTree, Message, Request, Response, SpanDataIndexMap}, + client::ServerError, + flat::{FlatTree, SpanDataIndexMap}, + legacy_protocol::msg::{Message, Request, Response}, }; use span::{Edition, EditionedFileId, FileId, Span, SpanAnchor, SyntaxContext, TextRange}; use tt::{Delimiter, DelimiterKind, TopSubtreeBuilder}; diff --git a/crates/proc-macro-srv-cli/tests/legacy_json.rs b/crates/proc-macro-srv-cli/tests/legacy_json.rs index f5cbaa7421eb..3dbe67c595c3 100644 --- a/crates/proc-macro-srv-cli/tests/legacy_json.rs +++ b/crates/proc-macro-srv-cli/tests/legacy_json.rs @@ -18,9 +18,10 @@ use common::utils::{ use expect_test::expect; use proc_macro_api::{ ProtocolFormat::JsonLegacy, + flat::SpanDataIndexMap, legacy_protocol::msg::{ ExpandMacro, ExpandMacroData, ExpnGlobals, PanicMessage, Request, Response, ServerConfig, - SpanDataIndexMap, SpanMode, + SpanMode, }, version::CURRENT_API_VERSION, }; diff --git a/crates/proc-macro-srv/Cargo.toml b/crates/proc-macro-srv/Cargo.toml index 05e0012586d5..68ae00760207 100644 --- a/crates/proc-macro-srv/Cargo.toml +++ b/crates/proc-macro-srv/Cargo.toml @@ -15,8 +15,12 @@ doctest = false [dependencies] paths.workspace = true # span = {workspace = true, default-features = false} does not work -span = { path = "../span", version = "0.0.0", default-features = false} +span = { path = "../span", version = "0.0.0", default-features = false } intern.workspace = true +tt = { path = "../tt", version = "0.0.0", default-features = false } +proc-macro-api = { path = "../proc-macro-api", version = "0.0.0", default-features = false, features = [ + "in-proc-macro-srv", +] } stdx.workspace = true [dev-dependencies] @@ -28,10 +32,10 @@ proc-macro-test.path = "./proc-macro-test" [features] default = [] -in-rust-tree = [] +in-rust-tree = ["tt/in-rust-tree", "proc-macro-api/in-rust-tree"] [lints] workspace = true [package.metadata.rust-analyzer] -rustc_private=true +rustc_private = true diff --git a/crates/proc-macro-srv/src/bridge.rs b/crates/proc-macro-srv/src/bridge.rs deleted file mode 100644 index fc62f9413a34..000000000000 --- a/crates/proc-macro-srv/src/bridge.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! `proc_macro::bridge` newtypes. - -use rustc_proc_macro::bridge as pm_bridge; - -pub use pm_bridge::{DelimSpan, Diagnostic, ExpnGlobals, LitKind}; - -pub type TokenTree = - pm_bridge::TokenTree, S, intern::Symbol>; -pub type Literal = pm_bridge::Literal; -pub type Group = pm_bridge::Group, S>; -pub type Punct = pm_bridge::Punct; -pub type Ident = pm_bridge::Ident; diff --git a/crates/proc-macro-srv/src/dylib.rs b/crates/proc-macro-srv/src/dylib.rs index 2a5e79d9e5cc..8da34f63688b 100644 --- a/crates/proc-macro-srv/src/dylib.rs +++ b/crates/proc-macro-srv/src/dylib.rs @@ -3,6 +3,7 @@ mod proc_macros; use paths::{Utf8Path, Utf8PathBuf}; +use proc_macro_api::token_stream::TokenStream; use rustc_codegen_ssa::back::metadata::DefaultMetadataLoader; use rustc_interface::util::rustc_version_str; use rustc_proc_macro::bridge; @@ -11,7 +12,7 @@ use stdx::tempfile::NamedTempFile; use crate::{ PanicMessage, ProcMacroClientHandle, ProcMacroKind, ProcMacroSrvSpan, TrackedEnv, - dylib::proc_macros::ProcMacros, token_stream::TokenStream, + dylib::proc_macros::ProcMacros, }; pub(crate) struct Expander { diff --git a/crates/proc-macro-srv/src/dylib/proc_macros.rs b/crates/proc-macro-srv/src/dylib/proc_macros.rs index 4976298d5fcd..5324a02a54ac 100644 --- a/crates/proc-macro-srv/src/dylib/proc_macros.rs +++ b/crates/proc-macro-srv/src/dylib/proc_macros.rs @@ -1,9 +1,9 @@ //! Proc macro ABI -use crate::{ - ProcMacroClientHandle, ProcMacroKind, ProcMacroSrvSpan, TrackedEnv, token_stream::TokenStream, -}; +use proc_macro_api::token_stream::TokenStream; use rustc_proc_macro::bridge; +use crate::{ProcMacroClientHandle, ProcMacroKind, ProcMacroSrvSpan, TrackedEnv}; + impl From for crate::PanicMessage { fn from(p: bridge::PanicMessage) -> Self { Self { message: p.into_string() } diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs index 7fc04a05155f..6fb4e618e8f8 100644 --- a/crates/proc-macro-srv/src/lib.rs +++ b/crates/proc-macro-srv/src/lib.rs @@ -18,15 +18,12 @@ extern crate rustc_codegen_ssa; extern crate rustc_driver as _; extern crate rustc_interface; -extern crate rustc_lexer; extern crate rustc_metadata; extern crate rustc_proc_macro; extern crate rustc_span; -mod bridge; mod dylib; mod server_impl; -mod token_stream; use std::{ collections::{HashMap, HashSet, hash_map::Entry}, @@ -40,16 +37,19 @@ use std::{ }; use paths::{Utf8Path, Utf8PathBuf}; +use proc_macro_api::{ + flat::{FlatTree, SpanTransformer}, + token_stream::SpanLike, + version::CURRENT_API_VERSION, +}; use span::{FIXUP_ERASED_FILE_AST_ID_MARKER, Span}; -pub use crate::server_impl::token_id::SpanId; - pub use rustc_proc_macro::Delimiter; pub use span; -pub use crate::bridge::*; -pub use crate::server_impl::literal_from_str; -pub use crate::token_stream::{TokenStream, TokenStreamIter, literal_to_string}; +pub use tt::literal_from_str; + +pub use crate::server_impl::token_id::SpanId; #[derive(Copy, Clone, Eq, PartialEq, Debug)] pub enum ProcMacroKind { @@ -127,20 +127,26 @@ impl ExpandError { } impl ProcMacroSrv<'_> { - pub fn expand<'a, S: ProcMacroSrvSpan + 'a>( + pub fn expand<'a, ST>( &self, lib: impl AsRef, env: &[(String, String)], current_dir: Option>, macro_name: &str, - macro_body: token_stream::TokenStream, - attribute: Option>, - def_site: S, - call_site: S, - mixed_site: S, + macro_body: FlatTree, + attribute: Option, + def_site: ST::Span, + call_site: ST::Span, + mixed_site: ST::Span, tracked_env: &'a mut TrackedEnv, + span_data_table: &mut ST::Table, callback: Option>, - ) -> Result, ExpandError> { + ) -> Result + where + ST: SpanTransformer, + ST::Span: ProcMacroSrvSpan + SpanLike + 'a, + ST::Table: Send, + { let snapped_env = self.env; let expander = self.expander(lib.as_ref()).map_err(|err| ExpandError::Internal { reason: Some(format!("failed to load macro: {err}")), @@ -155,16 +161,29 @@ impl ProcMacroSrv<'_> { .stack_size(EXPANDER_STACK_SIZE) .name(macro_name.to_owned()) .spawn_scoped(s, move || { - expander.expand( - macro_name, - macro_body, - attribute, - def_site, - call_site, - mixed_site, - tracked_env, - callback, - ) + let macro_body = + macro_body.to_tokenstream::(CURRENT_API_VERSION, span_data_table); + let attribute = attribute + .map(|it| it.to_tokenstream::(CURRENT_API_VERSION, span_data_table)); + expander + .expand( + macro_name, + macro_body, + attribute, + def_site, + call_site, + mixed_site, + tracked_env, + callback, + ) + .map(|result| { + FlatTree::from_tokenstream::( + result, + call_site, + CURRENT_API_VERSION, + span_data_table, + ) + }) }); match thread.unwrap().join() { Ok(res) => res.map_err(ExpandError::Panic), @@ -232,7 +251,7 @@ pub struct TrackedEnv { pub trait ProcMacroSrvSpan: Copy + Send + Sync { type Server<'a>: rustc_proc_macro::bridge::server::Server< - TokenStream = crate::token_stream::TokenStream, + TokenStream = proc_macro_api::token_stream::TokenStream, >; fn make_server<'a>( call_site: Self, diff --git a/crates/proc-macro-srv/src/server_impl.rs b/crates/proc-macro-srv/src/server_impl.rs index bacead1a88da..aaf8f540d9f4 100644 --- a/crates/proc-macro-srv/src/server_impl.rs +++ b/crates/proc-macro-srv/src/server_impl.rs @@ -6,34 +6,6 @@ //! The original idea from fedochet is using proc-macro2 as backend, //! we use tt instead for better integration with RA. +mod bridge; pub(crate) mod rust_analyzer_span; pub(crate) mod token_id; - -pub fn literal_from_str( - s: &str, - span: Span, -) -> Result, ()> { - use rustc_lexer::{LiteralKind, Token, TokenKind}; - let mut tokens = rustc_lexer::tokenize(s, rustc_lexer::FrontmatterAllowed::No); - let minus_or_lit = tokens.next().unwrap_or(Token { kind: TokenKind::Eof, len: 0 }); - - let lit = if minus_or_lit.kind == TokenKind::Minus { - let lit = tokens.next().ok_or(())?; - if !matches!( - lit.kind, - TokenKind::Literal { kind: LiteralKind::Int { .. } | LiteralKind::Float { .. }, .. } - ) { - return Err(()); - } - lit - } else { - minus_or_lit - }; - - if tokens.next().is_some() { - return Err(()); - } - - let TokenKind::Literal { kind, suffix_start } = lit.kind else { return Err(()) }; - Ok(crate::token_stream::literal_from_lexer(s, span, kind, suffix_start)) -} diff --git a/crates/proc-macro-srv/src/server_impl/bridge.rs b/crates/proc-macro-srv/src/server_impl/bridge.rs new file mode 100644 index 000000000000..34b0a8547911 --- /dev/null +++ b/crates/proc-macro-srv/src/server_impl/bridge.rs @@ -0,0 +1,160 @@ +//! Conversions between proc_macro bridge types and tt types. + +use proc_macro_api::token_stream::SpanLike; + +pub(super) mod ours { + pub(crate) type Literal = tt::Literal; + pub(crate) type Punct = tt::Punct; + pub(crate) type Ident = tt::Ident; + pub(crate) type Leaf = tt::Leaf; + pub(crate) type Group = proc_macro_api::token_stream::Group; + pub(crate) type TokenTree = proc_macro_api::token_stream::TokenTree; + pub(crate) type TokenStream = proc_macro_api::token_stream::TokenStream; +} + +#[expect(clippy::module_inception, reason = "this is not a mistake")] +pub(super) mod bridge { + use rustc_proc_macro::bridge as pm_bridge; + + use super::ours; + + pub(crate) use pm_bridge::*; + + pub(crate) type TokenTree = + pm_bridge::TokenTree, Span, intern::Symbol>; + pub(crate) type Literal = pm_bridge::Literal; + pub(crate) type Group = pm_bridge::Group, Span>; + pub(crate) type Punct = pm_bridge::Punct; + pub(crate) type Ident = pm_bridge::Ident; +} + +pub(super) fn literal_into_bridge(literal: ours::Literal) -> bridge::Literal { + let kind = match literal.kind { + tt::LitKind::Byte => bridge::LitKind::Byte, + tt::LitKind::Char => bridge::LitKind::Char, + tt::LitKind::Integer => bridge::LitKind::Integer, + tt::LitKind::Float => bridge::LitKind::Float, + tt::LitKind::Str => bridge::LitKind::Str, + tt::LitKind::StrRaw(count) => bridge::LitKind::StrRaw(count), + tt::LitKind::ByteStr => bridge::LitKind::ByteStr, + tt::LitKind::ByteStrRaw(count) => bridge::LitKind::ByteStrRaw(count), + tt::LitKind::CStr => bridge::LitKind::CStr, + tt::LitKind::CStrRaw(count) => bridge::LitKind::CStrRaw(count), + tt::LitKind::Err(()) => bridge::LitKind::ErrWithGuar, + }; + let (symbol, suffix) = literal.text_and_suffix_symbols(); + bridge::Literal { kind, symbol, suffix, span: literal.span } +} + +pub(super) fn punct_into_bridge(punct: ours::Punct) -> bridge::Punct { + bridge::Punct { + // FIXME: Is `as u8` correct here? + ch: punct.char as u8, + joint: punct.spacing != tt::Spacing::Alone, + span: punct.span, + } +} + +pub(super) fn ident_into_bridge(ident: ours::Ident) -> bridge::Ident { + bridge::Ident { sym: ident.sym, is_raw: ident.is_raw.yes(), span: ident.span } +} + +pub(super) fn group_into_bridge(group: ours::Group) -> bridge::Group { + let delimiter = match group.delimiter.kind { + tt::DelimiterKind::Parenthesis => rustc_proc_macro::Delimiter::Parenthesis, + tt::DelimiterKind::Brace => rustc_proc_macro::Delimiter::Brace, + tt::DelimiterKind::Bracket => rustc_proc_macro::Delimiter::Bracket, + tt::DelimiterKind::Invisible => rustc_proc_macro::Delimiter::None, + }; + let span = bridge::DelimSpan { + open: group.delimiter.open, + close: group.delimiter.close, + entire: group.delimiter.open.cover(group.delimiter.close), + }; + bridge::Group { delimiter, stream: group.stream, span } +} + +pub(super) fn token_tree_into_bridge( + token_tree: ours::TokenTree, +) -> bridge::TokenTree { + match token_tree { + ours::TokenTree::Leaf(ours::Leaf::Literal(literal)) => { + bridge::TokenTree::Literal(literal_into_bridge(literal)) + } + ours::TokenTree::Leaf(ours::Leaf::Ident(ident)) => { + bridge::TokenTree::Ident(ident_into_bridge(ident)) + } + ours::TokenTree::Leaf(ours::Leaf::Punct(punct)) => { + bridge::TokenTree::Punct(punct_into_bridge(punct)) + } + ours::TokenTree::Group(group) => bridge::TokenTree::Group(group_into_bridge(group)), + } +} + +pub(super) fn literal_from_bridge(literal: bridge::Literal) -> ours::Literal { + let kind = match literal.kind { + bridge::LitKind::Byte => tt::LitKind::Byte, + bridge::LitKind::Char => tt::LitKind::Char, + bridge::LitKind::Integer => tt::LitKind::Integer, + bridge::LitKind::Float => tt::LitKind::Float, + bridge::LitKind::Str => tt::LitKind::Str, + bridge::LitKind::StrRaw(count) => tt::LitKind::StrRaw(count), + bridge::LitKind::ByteStr => tt::LitKind::ByteStr, + bridge::LitKind::ByteStrRaw(count) => tt::LitKind::ByteStrRaw(count), + bridge::LitKind::CStr => tt::LitKind::CStr, + bridge::LitKind::CStrRaw(count) => tt::LitKind::CStrRaw(count), + bridge::LitKind::ErrWithGuar => tt::LitKind::Err(()), + }; + match literal.suffix { + Some(suffix) => { + tt::Literal::new(literal.symbol.as_str(), literal.span, kind, suffix.as_str()) + } + None => { + tt::Literal { text_and_suffix: literal.symbol, span: literal.span, kind, suffix_len: 0 } + } + } +} + +pub(super) fn punct_from_bridge(punct: bridge::Punct) -> ours::Punct { + ours::Punct { + char: char::from(punct.ch), + spacing: if punct.joint { tt::Spacing::Joint } else { tt::Spacing::Alone }, + span: punct.span, + } +} + +pub(super) fn ident_from_bridge(ident: bridge::Ident) -> ours::Ident { + ours::Ident { + sym: ident.sym, + is_raw: if ident.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No }, + span: ident.span, + } +} + +pub(super) fn group_from_bridge(group: bridge::Group) -> ours::Group { + let kind = match group.delimiter { + rustc_proc_macro::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis, + rustc_proc_macro::Delimiter::Brace => tt::DelimiterKind::Brace, + rustc_proc_macro::Delimiter::Bracket => tt::DelimiterKind::Bracket, + rustc_proc_macro::Delimiter::None => tt::DelimiterKind::Invisible, + }; + let delimiter = tt::Delimiter { open: group.span.open, close: group.span.close, kind }; + ours::Group { delimiter, stream: group.stream } +} + +pub(super) fn token_tree_from_bridge( + token_tree: bridge::TokenTree, +) -> ours::TokenTree { + match token_tree { + bridge::TokenTree::Literal(literal) => { + ours::TokenTree::Leaf(ours::Leaf::Literal(literal_from_bridge(literal))) + } + bridge::TokenTree::Ident(ident) => { + ours::TokenTree::Leaf(ours::Leaf::Ident(ident_from_bridge(ident))) + } + bridge::TokenTree::Punct(punct) => { + ours::TokenTree::Leaf(ours::Leaf::Punct(punct_from_bridge(punct))) + } + bridge::TokenTree::Group(group) => ours::TokenTree::Group(group_from_bridge(group)), + } +} diff --git a/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs b/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs index bdac9c920c43..ec8882e5b2b1 100644 --- a/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs +++ b/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs @@ -9,13 +9,28 @@ use std::ops::{Bound, Range}; use intern::Symbol; use rustc_proc_macro::bridge::server; use span::{ErasedFileAstId, Span, TextRange, TextSize}; +use tt::literal_from_str; use crate::{ ProcMacroClientHandle, TrackedEnv, - bridge::{Diagnostic, ExpnGlobals, Literal, TokenTree}, - server_impl::literal_from_str, + server_impl::bridge::{literal_into_bridge, token_tree_from_bridge, token_tree_into_bridge}, }; +mod ours { + use span::Span; + + pub(super) type TokenStream = crate::server_impl::bridge::ours::TokenStream; +} + +mod bridge { + use span::Span; + + pub(super) use crate::server_impl::bridge::bridge::*; + + pub(super) type TokenTree = crate::server_impl::bridge::bridge::TokenTree; + pub(super) type Literal = crate::server_impl::bridge::bridge::Literal; +} + pub struct RaSpanServer<'a> { pub tracked_env: &'a mut TrackedEnv, pub call_site: Span, @@ -26,12 +41,12 @@ pub struct RaSpanServer<'a> { } impl server::Server for RaSpanServer<'_> { - type TokenStream = crate::token_stream::TokenStream; + type TokenStream = ours::TokenStream; type Span = Span; type Symbol = Symbol; - fn globals(&mut self) -> ExpnGlobals { - ExpnGlobals { + fn globals(&mut self) -> bridge::ExpnGlobals { + bridge::ExpnGlobals { def_site: self.def_site, call_site: self.call_site, mixed_site: self.mixed_site, @@ -57,12 +72,13 @@ impl server::Server for RaSpanServer<'_> { self.tracked_env.paths.insert(path.into()); } - fn literal_from_str(&mut self, s: &str) -> Result, String> { + fn literal_from_str(&mut self, s: &str) -> Result { literal_from_str(s, self.call_site) - .map_err(|()| "cannot parse string into literal".to_string()) + .map(literal_into_bridge) + .map_err(|()| "cannot parse string into literal".to_owned()) } - fn emit_diagnostic(&mut self, _: Diagnostic) { + fn emit_diagnostic(&mut self, _: bridge::Diagnostic) { // FIXME handle diagnostic } @@ -85,8 +101,8 @@ impl server::Server for RaSpanServer<'_> { stream.to_string() } - fn ts_from_token_tree(&mut self, tree: TokenTree) -> Self::TokenStream { - Self::TokenStream::new(vec![tree]) + fn ts_from_token_tree(&mut self, tree: bridge::TokenTree) -> Self::TokenStream { + ours::TokenStream::new(vec![token_tree_from_bridge(tree)]) } fn ts_expand_expr(&mut self, self_: &Self::TokenStream) -> Result { @@ -100,17 +116,16 @@ impl server::Server for RaSpanServer<'_> { fn ts_concat_trees( &mut self, - base: Option, - trees: Vec>, + base: Option, + trees: Vec, ) -> Self::TokenStream { + let trees = trees.into_iter().map(token_tree_from_bridge); match base { Some(mut base) => { - for tt in trees { - base.push_tree(tt); - } + base.extend(trees); base } - None => Self::TokenStream::new(trees), + None => trees.collect(), } } @@ -119,15 +134,14 @@ impl server::Server for RaSpanServer<'_> { base: Option, streams: Vec, ) -> Self::TokenStream { - let mut stream = base.unwrap_or_default(); - for s in streams { - stream.push_stream(s); - } + let mut streams = streams.into_iter(); + let mut stream = base.or_else(|| streams.next()).unwrap_or_default(); + stream.extend_with_streams(streams); stream } - fn ts_into_trees(&mut self, stream: Self::TokenStream) -> Vec> { - (*stream.0).clone() + fn ts_into_trees(&mut self, stream: ours::TokenStream) -> Vec { + stream.iter().cloned().map(token_tree_into_bridge).collect() } fn span_debug(&mut self, span: Self::Span) -> String { diff --git a/crates/proc-macro-srv/src/server_impl/token_id.rs b/crates/proc-macro-srv/src/server_impl/token_id.rs index 6c393b8befb1..266809be985a 100644 --- a/crates/proc-macro-srv/src/server_impl/token_id.rs +++ b/crates/proc-macro-srv/src/server_impl/token_id.rs @@ -4,23 +4,48 @@ use std::ops::{Bound, Range}; use intern::Symbol; use rustc_proc_macro::bridge::server; +use tt::literal_from_str; use crate::{ ProcMacroClientHandle, - bridge::{Diagnostic, ExpnGlobals, Literal, TokenTree}, - server_impl::literal_from_str, + server_impl::bridge::{literal_into_bridge, token_tree_from_bridge, token_tree_into_bridge}, }; #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct SpanId(pub u32); +impl proc_macro_api::token_stream::SpanLike for crate::SpanId { + fn derive_ranged(&self, _: std::ops::Range) -> Self { + *self + } + + fn cover(self, _other: Self) -> Self { + self + } +} + impl std::fmt::Debug for SpanId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.0.fmt(f) } } -type Span = SpanId; +use SpanId as Span; + +mod ours { + use super::Span; + + pub(super) type TokenStream = crate::server_impl::bridge::ours::TokenStream; +} + +mod bridge { + use super::Span; + + pub(super) use crate::server_impl::bridge::bridge::*; + + pub(super) type TokenTree = crate::server_impl::bridge::bridge::TokenTree; + pub(super) type Literal = crate::server_impl::bridge::bridge::Literal; +} pub struct SpanIdServer<'a> { pub call_site: Span, @@ -30,12 +55,12 @@ pub struct SpanIdServer<'a> { } impl server::Server for SpanIdServer<'_> { - type TokenStream = crate::token_stream::TokenStream; + type TokenStream = ours::TokenStream; type Span = Span; type Symbol = Symbol; - fn globals(&mut self) -> ExpnGlobals { - ExpnGlobals { + fn globals(&mut self) -> bridge::ExpnGlobals { + bridge::ExpnGlobals { def_site: self.def_site, call_site: self.call_site, mixed_site: self.mixed_site, @@ -57,12 +82,13 @@ impl server::Server for SpanIdServer<'_> { fn track_path(&mut self, _: &str) {} - fn literal_from_str(&mut self, s: &str) -> Result, String> { + fn literal_from_str(&mut self, s: &str) -> Result { literal_from_str(s, self.call_site) - .map_err(|()| "cannot parse string into literal".to_string()) + .map(literal_into_bridge) + .map_err(|()| "cannot parse string into literal".to_owned()) } - fn emit_diagnostic(&mut self, _: Diagnostic) {} + fn emit_diagnostic(&mut self, _: bridge::Diagnostic) {} fn ts_drop(&mut self, stream: Self::TokenStream) { drop(stream); @@ -82,8 +108,8 @@ impl server::Server for SpanIdServer<'_> { fn ts_to_string(&mut self, stream: &Self::TokenStream) -> String { stream.to_string() } - fn ts_from_token_tree(&mut self, tree: TokenTree) -> Self::TokenStream { - Self::TokenStream::new(vec![tree]) + fn ts_from_token_tree(&mut self, tree: bridge::TokenTree) -> Self::TokenStream { + Self::TokenStream::new(vec![token_tree_from_bridge(tree)]) } fn ts_expand_expr(&mut self, self_: &Self::TokenStream) -> Result { @@ -93,16 +119,15 @@ impl server::Server for SpanIdServer<'_> { fn ts_concat_trees( &mut self, base: Option, - trees: Vec>, + trees: Vec, ) -> Self::TokenStream { + let trees = trees.into_iter().map(token_tree_from_bridge); match base { Some(mut base) => { - for tt in trees { - base.push_tree(tt); - } + base.extend(trees); base } - None => Self::TokenStream::new(trees), + None => trees.collect(), } } @@ -111,15 +136,14 @@ impl server::Server for SpanIdServer<'_> { base: Option, streams: Vec, ) -> Self::TokenStream { - let mut stream = base.unwrap_or_default(); - for s in streams { - stream.push_stream(s); - } + let mut streams = streams.into_iter(); + let mut stream = base.or_else(|| streams.next()).unwrap_or_default(); + stream.extend_with_streams(streams); stream } - fn ts_into_trees(&mut self, stream: Self::TokenStream) -> Vec> { - (*stream.0).clone() + fn ts_into_trees(&mut self, stream: ours::TokenStream) -> Vec { + stream.iter().cloned().map(token_tree_into_bridge).collect() } fn span_debug(&mut self, span: Self::Span) -> String { diff --git a/crates/proc-macro-srv/src/tests/mod.rs b/crates/proc-macro-srv/src/tests/mod.rs index 2cb6817fb916..d3f3cdd57bbb 100644 --- a/crates/proc-macro-srv/src/tests/mod.rs +++ b/crates/proc-macro-srv/src/tests/mod.rs @@ -12,42 +12,42 @@ fn test_derive_empty() { "DeriveEmpty", r#"struct S { field: &'r#lt fn(u32) -> &'a r#u32 }"#, expect![[r#" - IDENT 1 struct - IDENT 1 S - GROUP {} 1 1 1 - IDENT 1 field - PUNCT 1 : [alone] - PUNCT 1 & [joint] - PUNCT 1 ' [joint] - IDENT 1 r#lt - IDENT 1 fn - GROUP () 1 1 1 - IDENT 1 u32 - PUNCT 1 - [joint] - PUNCT 1 > [alone] - PUNCT 1 & [joint] - PUNCT 1 ' [joint] - IDENT 1 a - IDENT 1 r#u32 + IDENT struct 1 + IDENT S 1 + GROUP {} 1 1 + IDENT field 1 + PUNCT : [alone] 1 + PUNCT & [joint] 1 + PUNCT ' [joint] 1 + IDENT r#lt 1 + IDENT fn 1 + GROUP () 1 1 + IDENT u32 1 + PUNCT - [joint] 1 + PUNCT > [alone] 1 + PUNCT & [joint] 1 + PUNCT ' [joint] 1 + IDENT a 1 + IDENT r#u32 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..6#0 struct - IDENT 42:Root[0000, 0]@7..8#0 S - GROUP {} 42:Root[0000, 0]@9..10#0 42:Root[0000, 0]@46..47#0 42:Root[0000, 0]@9..47#0 - IDENT 42:Root[0000, 0]@11..16#0 field - PUNCT 42:Root[0000, 0]@16..17#0 : [alone] - PUNCT 42:Root[0000, 0]@18..19#0 & [joint] - PUNCT 42:Root[0000, 0]@22..23#0 ' [joint] - IDENT 42:Root[0000, 0]@22..24#0 r#lt - IDENT 42:Root[0000, 0]@25..27#0 fn - GROUP () 42:Root[0000, 0]@27..28#0 42:Root[0000, 0]@31..32#0 42:Root[0000, 0]@27..32#0 - IDENT 42:Root[0000, 0]@28..31#0 u32 - PUNCT 42:Root[0000, 0]@33..34#0 - [joint] - PUNCT 42:Root[0000, 0]@34..35#0 > [alone] - PUNCT 42:Root[0000, 0]@36..37#0 & [joint] - PUNCT 42:Root[0000, 0]@38..39#0 ' [joint] - IDENT 42:Root[0000, 0]@38..39#0 a - IDENT 42:Root[0000, 0]@42..45#0 r#u32 + IDENT struct 42:Root[0000, 0]@0..6#0 + IDENT S 42:Root[0000, 0]@7..8#0 + GROUP {} 42:Root[0000, 0]@9..10#0 42:Root[0000, 0]@46..47#0 + IDENT field 42:Root[0000, 0]@11..16#0 + PUNCT : [alone] 42:Root[0000, 0]@16..17#0 + PUNCT & [joint] 42:Root[0000, 0]@18..19#0 + PUNCT ' [joint] 42:Root[0000, 0]@22..23#0 + IDENT r#lt 42:Root[0000, 0]@22..24#0 + IDENT fn 42:Root[0000, 0]@25..27#0 + GROUP () 42:Root[0000, 0]@27..28#0 42:Root[0000, 0]@31..32#0 + IDENT u32 42:Root[0000, 0]@28..31#0 + PUNCT - [joint] 42:Root[0000, 0]@33..34#0 + PUNCT > [alone] 42:Root[0000, 0]@34..35#0 + PUNCT & [joint] 42:Root[0000, 0]@36..37#0 + PUNCT ' [joint] 42:Root[0000, 0]@38..39#0 + IDENT a 42:Root[0000, 0]@38..39#0 + IDENT r#u32 42:Root[0000, 0]@42..45#0 "#]], ); } @@ -65,148 +65,148 @@ pub struct Foo { } "#, expect![[r#" - PUNCT 1 # [joint] - GROUP [] 1 1 1 - IDENT 1 helper - GROUP () 1 1 1 - IDENT 1 build_fn - GROUP () 1 1 1 - IDENT 1 private - PUNCT 1 , [alone] - IDENT 1 name - PUNCT 1 = [alone] - LITER 1 Str partial_build - IDENT 1 pub - IDENT 1 struct - IDENT 1 Foo - GROUP {} 1 1 1 - PUNCT 1 # [alone] - GROUP [] 1 1 1 - IDENT 1 doc - PUNCT 1 = [alone] - LITER 1 Str The domain where this federated instance is running - PUNCT 1 # [joint] - GROUP [] 1 1 1 - IDENT 1 helper - GROUP () 1 1 1 - IDENT 1 setter - GROUP () 1 1 1 - IDENT 1 into - IDENT 1 pub - GROUP () 1 1 1 - IDENT 1 crate - IDENT 1 domain - PUNCT 1 : [alone] - IDENT 1 String - PUNCT 1 , [alone] - - - PUNCT 1 # [joint] - GROUP [] 1 1 1 - IDENT 1 helper - GROUP () 1 1 1 - IDENT 1 build_fn - GROUP () 1 1 1 - IDENT 1 private - PUNCT 1 , [alone] - IDENT 1 name - PUNCT 1 = [alone] - LITER 1 Str partial_build - IDENT 1 pub - IDENT 1 struct - IDENT 1 Foo - GROUP {} 1 1 1 - PUNCT 1 # [alone] - GROUP [] 1 1 1 - IDENT 1 doc - PUNCT 1 = [alone] - LITER 1 Str The domain where this federated instance is running - PUNCT 1 # [joint] - GROUP [] 1 1 1 - IDENT 1 helper - GROUP () 1 1 1 - IDENT 1 setter - GROUP () 1 1 1 - IDENT 1 into - IDENT 1 pub - GROUP () 1 1 1 - IDENT 1 crate - IDENT 1 domain - PUNCT 1 : [alone] - IDENT 1 String - PUNCT 1 , [alone] + PUNCT # [joint] 1 + GROUP [] 1 1 + IDENT helper 1 + GROUP () 1 1 + IDENT build_fn 1 + GROUP () 1 1 + IDENT private 1 + PUNCT , [alone] 1 + IDENT name 1 + PUNCT = [alone] 1 + LITERAL Str partial_build 1 + IDENT pub 1 + IDENT struct 1 + IDENT Foo 1 + GROUP {} 1 1 + PUNCT # [alone] 1 + GROUP [] 1 1 + IDENT doc 1 + PUNCT = [alone] 1 + LITERAL Str The domain where this federated instance is running 1 + PUNCT # [joint] 1 + GROUP [] 1 1 + IDENT helper 1 + GROUP () 1 1 + IDENT setter 1 + GROUP () 1 1 + IDENT into 1 + IDENT pub 1 + GROUP () 1 1 + IDENT crate 1 + IDENT domain 1 + PUNCT : [alone] 1 + IDENT String 1 + PUNCT , [alone] 1 + + + PUNCT # [joint] 1 + GROUP [] 1 1 + IDENT helper 1 + GROUP () 1 1 + IDENT build_fn 1 + GROUP () 1 1 + IDENT private 1 + PUNCT , [alone] 1 + IDENT name 1 + PUNCT = [alone] 1 + LITERAL Str partial_build 1 + IDENT pub 1 + IDENT struct 1 + IDENT Foo 1 + GROUP {} 1 1 + PUNCT # [alone] 1 + GROUP [] 1 1 + IDENT doc 1 + PUNCT = [alone] 1 + LITERAL Str The domain where this federated instance is running 1 + PUNCT # [joint] 1 + GROUP [] 1 1 + IDENT helper 1 + GROUP () 1 1 + IDENT setter 1 + GROUP () 1 1 + IDENT into 1 + IDENT pub 1 + GROUP () 1 1 + IDENT crate 1 + IDENT domain 1 + PUNCT : [alone] 1 + IDENT String 1 + PUNCT , [alone] 1 "#]], expect![[r#" - PUNCT 42:Root[0000, 0]@1..2#0 # [joint] - GROUP [] 42:Root[0000, 0]@2..3#0 42:Root[0000, 0]@52..53#0 42:Root[0000, 0]@2..53#0 - IDENT 42:Root[0000, 0]@3..9#0 helper - GROUP () 42:Root[0000, 0]@9..10#0 42:Root[0000, 0]@51..52#0 42:Root[0000, 0]@9..52#0 - IDENT 42:Root[0000, 0]@10..18#0 build_fn - GROUP () 42:Root[0000, 0]@18..19#0 42:Root[0000, 0]@50..51#0 42:Root[0000, 0]@18..51#0 - IDENT 42:Root[0000, 0]@19..26#0 private - PUNCT 42:Root[0000, 0]@26..27#0 , [alone] - IDENT 42:Root[0000, 0]@28..32#0 name - PUNCT 42:Root[0000, 0]@33..34#0 = [alone] - LITER 42:Root[0000, 0]@35..50#0 Str partial_build - IDENT 42:Root[0000, 0]@54..57#0 pub - IDENT 42:Root[0000, 0]@58..64#0 struct - IDENT 42:Root[0000, 0]@65..68#0 Foo - GROUP {} 42:Root[0000, 0]@69..70#0 42:Root[0000, 0]@190..191#0 42:Root[0000, 0]@69..191#0 - PUNCT 42:Root[0000, 0]@0..0#0 # [alone] - GROUP [] 42:Root[0000, 0]@0..0#0 42:Root[0000, 0]@0..0#0 42:Root[0000, 0]@0..0#0 - IDENT 42:Root[0000, 0]@0..0#0 doc - PUNCT 42:Root[0000, 0]@0..0#0 = [alone] - LITER 42:Root[0000, 0]@75..130#0 Str The domain where this federated instance is running - PUNCT 42:Root[0000, 0]@135..136#0 # [joint] - GROUP [] 42:Root[0000, 0]@136..137#0 42:Root[0000, 0]@157..158#0 42:Root[0000, 0]@136..158#0 - IDENT 42:Root[0000, 0]@137..143#0 helper - GROUP () 42:Root[0000, 0]@143..144#0 42:Root[0000, 0]@156..157#0 42:Root[0000, 0]@143..157#0 - IDENT 42:Root[0000, 0]@144..150#0 setter - GROUP () 42:Root[0000, 0]@150..151#0 42:Root[0000, 0]@155..156#0 42:Root[0000, 0]@150..156#0 - IDENT 42:Root[0000, 0]@151..155#0 into - IDENT 42:Root[0000, 0]@163..166#0 pub - GROUP () 42:Root[0000, 0]@166..167#0 42:Root[0000, 0]@172..173#0 42:Root[0000, 0]@166..173#0 - IDENT 42:Root[0000, 0]@167..172#0 crate - IDENT 42:Root[0000, 0]@174..180#0 domain - PUNCT 42:Root[0000, 0]@180..181#0 : [alone] - IDENT 42:Root[0000, 0]@182..188#0 String - PUNCT 42:Root[0000, 0]@188..189#0 , [alone] - - - PUNCT 42:Root[0000, 0]@1..2#0 # [joint] - GROUP [] 42:Root[0000, 0]@2..3#0 42:Root[0000, 0]@52..53#0 42:Root[0000, 0]@2..53#0 - IDENT 42:Root[0000, 0]@3..9#0 helper - GROUP () 42:Root[0000, 0]@9..10#0 42:Root[0000, 0]@51..52#0 42:Root[0000, 0]@9..52#0 - IDENT 42:Root[0000, 0]@10..18#0 build_fn - GROUP () 42:Root[0000, 0]@18..19#0 42:Root[0000, 0]@50..51#0 42:Root[0000, 0]@18..51#0 - IDENT 42:Root[0000, 0]@19..26#0 private - PUNCT 42:Root[0000, 0]@26..27#0 , [alone] - IDENT 42:Root[0000, 0]@28..32#0 name - PUNCT 42:Root[0000, 0]@33..34#0 = [alone] - LITER 42:Root[0000, 0]@35..50#0 Str partial_build - IDENT 42:Root[0000, 0]@54..57#0 pub - IDENT 42:Root[0000, 0]@58..64#0 struct - IDENT 42:Root[0000, 0]@65..68#0 Foo - GROUP {} 42:Root[0000, 0]@69..70#0 42:Root[0000, 0]@190..191#0 42:Root[0000, 0]@69..191#0 - PUNCT 42:Root[0000, 0]@0..0#0 # [alone] - GROUP [] 42:Root[0000, 0]@0..0#0 42:Root[0000, 0]@0..0#0 42:Root[0000, 0]@0..0#0 - IDENT 42:Root[0000, 0]@0..0#0 doc - PUNCT 42:Root[0000, 0]@0..0#0 = [alone] - LITER 42:Root[0000, 0]@75..130#0 Str The domain where this federated instance is running - PUNCT 42:Root[0000, 0]@135..136#0 # [joint] - GROUP [] 42:Root[0000, 0]@136..137#0 42:Root[0000, 0]@157..158#0 42:Root[0000, 0]@136..158#0 - IDENT 42:Root[0000, 0]@137..143#0 helper - GROUP () 42:Root[0000, 0]@143..144#0 42:Root[0000, 0]@156..157#0 42:Root[0000, 0]@143..157#0 - IDENT 42:Root[0000, 0]@144..150#0 setter - GROUP () 42:Root[0000, 0]@150..151#0 42:Root[0000, 0]@155..156#0 42:Root[0000, 0]@150..156#0 - IDENT 42:Root[0000, 0]@151..155#0 into - IDENT 42:Root[0000, 0]@163..166#0 pub - GROUP () 42:Root[0000, 0]@166..167#0 42:Root[0000, 0]@172..173#0 42:Root[0000, 0]@166..173#0 - IDENT 42:Root[0000, 0]@167..172#0 crate - IDENT 42:Root[0000, 0]@174..180#0 domain - PUNCT 42:Root[0000, 0]@180..181#0 : [alone] - IDENT 42:Root[0000, 0]@182..188#0 String - PUNCT 42:Root[0000, 0]@188..189#0 , [alone] + PUNCT # [joint] 42:Root[0000, 0]@1..2#0 + GROUP [] 42:Root[0000, 0]@2..3#0 42:Root[0000, 0]@52..53#0 + IDENT helper 42:Root[0000, 0]@3..9#0 + GROUP () 42:Root[0000, 0]@9..10#0 42:Root[0000, 0]@51..52#0 + IDENT build_fn 42:Root[0000, 0]@10..18#0 + GROUP () 42:Root[0000, 0]@18..19#0 42:Root[0000, 0]@50..51#0 + IDENT private 42:Root[0000, 0]@19..26#0 + PUNCT , [alone] 42:Root[0000, 0]@26..27#0 + IDENT name 42:Root[0000, 0]@28..32#0 + PUNCT = [alone] 42:Root[0000, 0]@33..34#0 + LITERAL Str partial_build 42:Root[0000, 0]@35..50#0 + IDENT pub 42:Root[0000, 0]@54..57#0 + IDENT struct 42:Root[0000, 0]@58..64#0 + IDENT Foo 42:Root[0000, 0]@65..68#0 + GROUP {} 42:Root[0000, 0]@69..70#0 42:Root[0000, 0]@190..191#0 + PUNCT # [alone] 42:Root[0000, 0]@0..0#0 + GROUP [] 42:Root[0000, 0]@75..130#0 42:Root[0000, 0]@75..130#0 + IDENT doc 42:Root[0000, 0]@75..130#0 + PUNCT = [alone] 42:Root[0000, 0]@75..130#0 + LITERAL Str The domain where this federated instance is running 42:Root[0000, 0]@75..130#0 + PUNCT # [joint] 42:Root[0000, 0]@135..136#0 + GROUP [] 42:Root[0000, 0]@136..137#0 42:Root[0000, 0]@157..158#0 + IDENT helper 42:Root[0000, 0]@137..143#0 + GROUP () 42:Root[0000, 0]@143..144#0 42:Root[0000, 0]@156..157#0 + IDENT setter 42:Root[0000, 0]@144..150#0 + GROUP () 42:Root[0000, 0]@150..151#0 42:Root[0000, 0]@155..156#0 + IDENT into 42:Root[0000, 0]@151..155#0 + IDENT pub 42:Root[0000, 0]@163..166#0 + GROUP () 42:Root[0000, 0]@166..167#0 42:Root[0000, 0]@172..173#0 + IDENT crate 42:Root[0000, 0]@167..172#0 + IDENT domain 42:Root[0000, 0]@174..180#0 + PUNCT : [alone] 42:Root[0000, 0]@180..181#0 + IDENT String 42:Root[0000, 0]@182..188#0 + PUNCT , [alone] 42:Root[0000, 0]@188..189#0 + + + PUNCT # [joint] 42:Root[0000, 0]@1..2#0 + GROUP [] 42:Root[0000, 0]@2..3#0 42:Root[0000, 0]@52..53#0 + IDENT helper 42:Root[0000, 0]@3..9#0 + GROUP () 42:Root[0000, 0]@9..10#0 42:Root[0000, 0]@51..52#0 + IDENT build_fn 42:Root[0000, 0]@10..18#0 + GROUP () 42:Root[0000, 0]@18..19#0 42:Root[0000, 0]@50..51#0 + IDENT private 42:Root[0000, 0]@19..26#0 + PUNCT , [alone] 42:Root[0000, 0]@26..27#0 + IDENT name 42:Root[0000, 0]@28..32#0 + PUNCT = [alone] 42:Root[0000, 0]@33..34#0 + LITERAL Str partial_build 42:Root[0000, 0]@35..50#0 + IDENT pub 42:Root[0000, 0]@54..57#0 + IDENT struct 42:Root[0000, 0]@58..64#0 + IDENT Foo 42:Root[0000, 0]@65..68#0 + GROUP {} 42:Root[0000, 0]@69..70#0 42:Root[0000, 0]@190..191#0 + PUNCT # [alone] 42:Root[0000, 0]@0..0#0 + GROUP [] 42:Root[0000, 0]@75..130#0 42:Root[0000, 0]@75..130#0 + IDENT doc 42:Root[0000, 0]@75..130#0 + PUNCT = [alone] 42:Root[0000, 0]@75..130#0 + LITERAL Str The domain where this federated instance is running 42:Root[0000, 0]@75..130#0 + PUNCT # [joint] 42:Root[0000, 0]@135..136#0 + GROUP [] 42:Root[0000, 0]@136..137#0 42:Root[0000, 0]@157..158#0 + IDENT helper 42:Root[0000, 0]@137..143#0 + GROUP () 42:Root[0000, 0]@143..144#0 42:Root[0000, 0]@156..157#0 + IDENT setter 42:Root[0000, 0]@144..150#0 + GROUP () 42:Root[0000, 0]@150..151#0 42:Root[0000, 0]@155..156#0 + IDENT into 42:Root[0000, 0]@151..155#0 + IDENT pub 42:Root[0000, 0]@163..166#0 + GROUP () 42:Root[0000, 0]@166..167#0 42:Root[0000, 0]@172..173#0 + IDENT crate 42:Root[0000, 0]@167..172#0 + IDENT domain 42:Root[0000, 0]@174..180#0 + PUNCT : [alone] 42:Root[0000, 0]@180..181#0 + IDENT String 42:Root[0000, 0]@182..188#0 + PUNCT , [alone] 42:Root[0000, 0]@188..189#0 "#]], ); } @@ -217,34 +217,34 @@ fn test_derive_error() { "DeriveError", r#"struct S { field: u32 }"#, expect![[r#" - IDENT 1 struct - IDENT 1 S - GROUP {} 1 1 1 - IDENT 1 field - PUNCT 1 : [alone] - IDENT 1 u32 - - - IDENT 1 compile_error - PUNCT 1 ! [joint] - GROUP () 1 1 1 - LITER 1 Str #[derive(DeriveError)] struct S {field : u32} - PUNCT 1 ; [alone] + IDENT struct 1 + IDENT S 1 + GROUP {} 1 1 + IDENT field 1 + PUNCT : [alone] 1 + IDENT u32 1 + + + IDENT compile_error 1 + PUNCT ! [joint] 1 + GROUP () 1 1 + LITERAL Str #[derive(DeriveError)] struct S {field : u32} 1 + PUNCT ; [alone] 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..6#0 struct - IDENT 42:Root[0000, 0]@7..8#0 S - GROUP {} 42:Root[0000, 0]@9..10#0 42:Root[0000, 0]@22..23#0 42:Root[0000, 0]@9..23#0 - IDENT 42:Root[0000, 0]@11..16#0 field - PUNCT 42:Root[0000, 0]@16..17#0 : [alone] - IDENT 42:Root[0000, 0]@18..21#0 u32 - - - IDENT 42:Root[0000, 0]@0..13#0 compile_error - PUNCT 42:Root[0000, 0]@13..14#0 ! [joint] - GROUP () 42:Root[0000, 0]@14..15#0 42:Root[0000, 0]@62..63#0 42:Root[0000, 0]@14..63#0 - LITER 42:Root[0000, 0]@15..62#0 Str #[derive(DeriveError)] struct S {field : u32} - PUNCT 42:Root[0000, 0]@63..64#0 ; [alone] + IDENT struct 42:Root[0000, 0]@0..6#0 + IDENT S 42:Root[0000, 0]@7..8#0 + GROUP {} 42:Root[0000, 0]@9..10#0 42:Root[0000, 0]@22..23#0 + IDENT field 42:Root[0000, 0]@11..16#0 + PUNCT : [alone] 42:Root[0000, 0]@16..17#0 + IDENT u32 42:Root[0000, 0]@18..21#0 + + + IDENT compile_error 42:Root[0000, 0]@0..13#0 + PUNCT ! [joint] 42:Root[0000, 0]@13..14#0 + GROUP () 42:Root[0000, 0]@14..15#0 42:Root[0000, 0]@62..63#0 + LITERAL Str #[derive(DeriveError)] struct S {field : u32} 42:Root[0000, 0]@15..62#0 + PUNCT ; [alone] 42:Root[0000, 0]@63..64#0 "#]], ); } @@ -255,40 +255,40 @@ fn test_fn_like_macro_noop() { "fn_like_noop", r#"ident, 0, 1, []"#, expect![[r#" - IDENT 1 ident - PUNCT 1 , [alone] - LITER 1 Integer 0 - PUNCT 1 , [alone] - LITER 1 Integer 1 - PUNCT 1 , [alone] - GROUP [] 1 1 1 - - - IDENT 1 ident - PUNCT 1 , [alone] - LITER 1 Integer 0 - PUNCT 1 , [alone] - LITER 1 Integer 1 - PUNCT 1 , [alone] - GROUP [] 1 1 1 + IDENT ident 1 + PUNCT , [alone] 1 + LITERAL Integer 0 1 + PUNCT , [alone] 1 + LITERAL Integer 1 1 + PUNCT , [alone] 1 + GROUP [] 1 1 + + + IDENT ident 1 + PUNCT , [alone] 1 + LITERAL Integer 0 1 + PUNCT , [alone] 1 + LITERAL Integer 1 1 + PUNCT , [alone] 1 + GROUP [] 1 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..5#0 ident - PUNCT 42:Root[0000, 0]@5..6#0 , [alone] - LITER 42:Root[0000, 0]@7..8#0 Integer 0 - PUNCT 42:Root[0000, 0]@8..9#0 , [alone] - LITER 42:Root[0000, 0]@10..11#0 Integer 1 - PUNCT 42:Root[0000, 0]@11..12#0 , [alone] - GROUP [] 42:Root[0000, 0]@13..14#0 42:Root[0000, 0]@14..15#0 42:Root[0000, 0]@13..15#0 - - - IDENT 42:Root[0000, 0]@0..5#0 ident - PUNCT 42:Root[0000, 0]@5..6#0 , [alone] - LITER 42:Root[0000, 0]@7..8#0 Integer 0 - PUNCT 42:Root[0000, 0]@8..9#0 , [alone] - LITER 42:Root[0000, 0]@10..11#0 Integer 1 - PUNCT 42:Root[0000, 0]@11..12#0 , [alone] - GROUP [] 42:Root[0000, 0]@13..14#0 42:Root[0000, 0]@14..15#0 42:Root[0000, 0]@13..15#0 + IDENT ident 42:Root[0000, 0]@0..5#0 + PUNCT , [alone] 42:Root[0000, 0]@5..6#0 + LITERAL Integer 0 42:Root[0000, 0]@7..8#0 + PUNCT , [alone] 42:Root[0000, 0]@8..9#0 + LITERAL Integer 1 42:Root[0000, 0]@10..11#0 + PUNCT , [alone] 42:Root[0000, 0]@11..12#0 + GROUP [] 42:Root[0000, 0]@13..14#0 42:Root[0000, 0]@14..15#0 + + + IDENT ident 42:Root[0000, 0]@0..5#0 + PUNCT , [alone] 42:Root[0000, 0]@5..6#0 + LITERAL Integer 0 42:Root[0000, 0]@7..8#0 + PUNCT , [alone] 42:Root[0000, 0]@8..9#0 + LITERAL Integer 1 42:Root[0000, 0]@10..11#0 + PUNCT , [alone] 42:Root[0000, 0]@11..12#0 + GROUP [] 42:Root[0000, 0]@13..14#0 42:Root[0000, 0]@14..15#0 "#]], ); } @@ -299,36 +299,36 @@ fn test_fn_like_macro_clone_ident_subtree() { "fn_like_clone_tokens", r#"ident, [ident2, ident3]"#, expect![[r#" - IDENT 1 ident - PUNCT 1 , [alone] - GROUP [] 1 1 1 - IDENT 1 ident2 - PUNCT 1 , [alone] - IDENT 1 ident3 - - - IDENT 1 ident - PUNCT 1 , [alone] - GROUP [] 1 1 1 - IDENT 1 ident2 - PUNCT 1 , [alone] - IDENT 1 ident3 + IDENT ident 1 + PUNCT , [alone] 1 + GROUP [] 1 1 + IDENT ident2 1 + PUNCT , [alone] 1 + IDENT ident3 1 + + + IDENT ident 1 + PUNCT , [alone] 1 + GROUP [] 1 1 + IDENT ident2 1 + PUNCT , [alone] 1 + IDENT ident3 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..5#0 ident - PUNCT 42:Root[0000, 0]@5..6#0 , [alone] - GROUP [] 42:Root[0000, 0]@7..8#0 42:Root[0000, 0]@22..23#0 42:Root[0000, 0]@7..23#0 - IDENT 42:Root[0000, 0]@8..14#0 ident2 - PUNCT 42:Root[0000, 0]@14..15#0 , [alone] - IDENT 42:Root[0000, 0]@16..22#0 ident3 - - - IDENT 42:Root[0000, 0]@0..5#0 ident - PUNCT 42:Root[0000, 0]@5..6#0 , [alone] - GROUP [] 42:Root[0000, 0]@7..23#0 42:Root[0000, 0]@7..23#0 42:Root[0000, 0]@7..23#0 - IDENT 42:Root[0000, 0]@8..14#0 ident2 - PUNCT 42:Root[0000, 0]@14..15#0 , [alone] - IDENT 42:Root[0000, 0]@16..22#0 ident3 + IDENT ident 42:Root[0000, 0]@0..5#0 + PUNCT , [alone] 42:Root[0000, 0]@5..6#0 + GROUP [] 42:Root[0000, 0]@7..8#0 42:Root[0000, 0]@22..23#0 + IDENT ident2 42:Root[0000, 0]@8..14#0 + PUNCT , [alone] 42:Root[0000, 0]@14..15#0 + IDENT ident3 42:Root[0000, 0]@16..22#0 + + + IDENT ident 42:Root[0000, 0]@0..5#0 + PUNCT , [alone] 42:Root[0000, 0]@5..6#0 + GROUP [] 42:Root[0000, 0]@7..23#0 42:Root[0000, 0]@7..23#0 + IDENT ident2 42:Root[0000, 0]@8..14#0 + PUNCT , [alone] 42:Root[0000, 0]@14..15#0 + IDENT ident3 42:Root[0000, 0]@16..22#0 "#]], ); } @@ -339,16 +339,16 @@ fn test_fn_like_macro_clone_raw_ident() { "fn_like_clone_tokens", "r#async", expect![[r#" - IDENT 1 r#async + IDENT r#async 1 - IDENT 1 r#async + IDENT r#async 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@2..7#0 r#async + IDENT r#async 42:Root[0000, 0]@2..7#0 - IDENT 42:Root[0000, 0]@2..7#0 r#async + IDENT r#async 42:Root[0000, 0]@2..7#0 "#]], ); } @@ -359,18 +359,18 @@ fn test_fn_like_fn_like_span_join() { "fn_like_span_join", "foo bar", expect![[r#" - IDENT 1 foo - IDENT 1 bar + IDENT foo 1 + IDENT bar 1 - IDENT 1 r#joined + IDENT r#joined 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..3#0 foo - IDENT 42:Root[0000, 0]@8..11#0 bar + IDENT foo 42:Root[0000, 0]@0..3#0 + IDENT bar 42:Root[0000, 0]@8..11#0 - IDENT 42:Root[0000, 0]@0..11#0 r#joined + IDENT r#joined 42:Root[0000, 0]@0..11#0 "#]], ); } @@ -381,24 +381,24 @@ fn test_fn_like_fn_like_span_ops() { "fn_like_span_ops", "set_def_site resolved_at_def_site start_span", expect![[r#" - IDENT 1 set_def_site - IDENT 1 resolved_at_def_site - IDENT 1 start_span + IDENT set_def_site 1 + IDENT resolved_at_def_site 1 + IDENT start_span 1 - IDENT 0 set_def_site - IDENT 1 resolved_at_def_site - IDENT 1 start_span + IDENT set_def_site 0 + IDENT resolved_at_def_site 1 + IDENT start_span 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..12#0 set_def_site - IDENT 42:Root[0000, 0]@13..33#0 resolved_at_def_site - IDENT 42:Root[0000, 0]@34..44#0 start_span + IDENT set_def_site 42:Root[0000, 0]@0..12#0 + IDENT resolved_at_def_site 42:Root[0000, 0]@13..33#0 + IDENT start_span 42:Root[0000, 0]@34..44#0 - IDENT 41:Root[0000, 0]@0..150#0 set_def_site - IDENT 42:Root[0000, 0]@13..33#0 resolved_at_def_site - IDENT 42:Root[0000, 0]@34..34#0 start_span + IDENT set_def_site 41:Root[0000, 0]@0..150#0 + IDENT resolved_at_def_site 42:Root[0000, 0]@13..33#0 + IDENT start_span 42:Root[0000, 0]@34..34#0 "#]], ); } @@ -411,36 +411,36 @@ fn test_fn_like_mk_literals() { expect![[r#" - LITER 1 ByteStr byte_string - LITER 1 Char c - LITER 1 Str string - LITER 1 Str -string - LITER 1 CStr cstring - LITER 1 Float 3.14f64 - LITER 1 Float -3.14f64 - LITER 1 Float 3.14 - LITER 1 Float -3.14 - LITER 1 Integer 123i64 - LITER 1 Integer -123i64 - LITER 1 Integer 123 - LITER 1 Integer -123 + LITERAL ByteStr byte_string 1 + LITERAL Char c 1 + LITERAL Str string 1 + LITERAL Str -string 1 + LITERAL CStr cstring 1 + LITERAL Float 3.14f64 1 + LITERAL Float -3.14f64 1 + LITERAL Float 3.14 1 + LITERAL Float -3.14 1 + LITERAL Integer 123i64 1 + LITERAL Integer -123i64 1 + LITERAL Integer 123 1 + LITERAL Integer -123 1 "#]], expect![[r#" - LITER 42:Root[0000, 0]@0..100#0 ByteStr byte_string - LITER 42:Root[0000, 0]@0..100#0 Char c - LITER 42:Root[0000, 0]@0..100#0 Str string - LITER 42:Root[0000, 0]@0..100#0 Str -string - LITER 42:Root[0000, 0]@0..100#0 CStr cstring - LITER 42:Root[0000, 0]@0..100#0 Float 3.14f64 - LITER 42:Root[0000, 0]@0..100#0 Float -3.14f64 - LITER 42:Root[0000, 0]@0..100#0 Float 3.14 - LITER 42:Root[0000, 0]@0..100#0 Float -3.14 - LITER 42:Root[0000, 0]@0..100#0 Integer 123i64 - LITER 42:Root[0000, 0]@0..100#0 Integer -123i64 - LITER 42:Root[0000, 0]@0..100#0 Integer 123 - LITER 42:Root[0000, 0]@0..100#0 Integer -123 + LITERAL ByteStr byte_string 42:Root[0000, 0]@0..100#0 + LITERAL Char c 42:Root[0000, 0]@0..100#0 + LITERAL Str string 42:Root[0000, 0]@0..100#0 + LITERAL Str -string 42:Root[0000, 0]@0..100#0 + LITERAL CStr cstring 42:Root[0000, 0]@0..100#0 + LITERAL Float 3.14f64 42:Root[0000, 0]@0..100#0 + LITERAL Float -3.14f64 42:Root[0000, 0]@0..100#0 + LITERAL Float 3.14 42:Root[0000, 0]@0..100#0 + LITERAL Float -3.14 42:Root[0000, 0]@0..100#0 + LITERAL Integer 123i64 42:Root[0000, 0]@0..100#0 + LITERAL Integer -123i64 42:Root[0000, 0]@0..100#0 + LITERAL Integer 123 42:Root[0000, 0]@0..100#0 + LITERAL Integer -123 42:Root[0000, 0]@0..100#0 "#]], ); } @@ -453,14 +453,14 @@ fn test_fn_like_mk_idents() { expect![[r#" - IDENT 1 standard - IDENT 1 r#raw + IDENT standard 1 + IDENT r#raw 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..100#0 standard - IDENT 42:Root[0000, 0]@0..100#0 r#raw + IDENT standard 42:Root[0000, 0]@0..100#0 + IDENT r#raw 42:Root[0000, 0]@0..100#0 "#]], ); } @@ -471,92 +471,92 @@ fn test_fn_like_macro_clone_literals() { "fn_like_clone_tokens", r###"1u16, 2_u32, -4i64, 3.14f32, "hello bridge", "suffixed"suffix, r##"raw"##, 'a', b'b', c"null""###, expect![[r#" - LITER 1 Integer 1u16 - PUNCT 1 , [alone] - LITER 1 Integer 2_u32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Integer 4i64 - PUNCT 1 , [alone] - LITER 1 Float 3.14f32 - PUNCT 1 , [alone] - LITER 1 Str hello bridge - PUNCT 1 , [alone] - LITER 1 Str suffixedsuffix - PUNCT 1 , [alone] - LITER 1 StrRaw(2) raw - PUNCT 1 , [alone] - LITER 1 Char a - PUNCT 1 , [alone] - LITER 1 Byte b - PUNCT 1 , [alone] - LITER 1 CStr null - - - LITER 1 Integer 1u16 - PUNCT 1 , [alone] - LITER 1 Integer 2_u32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Integer 4i64 - PUNCT 1 , [alone] - LITER 1 Float 3.14f32 - PUNCT 1 , [alone] - LITER 1 Str hello bridge - PUNCT 1 , [alone] - LITER 1 Str suffixedsuffix - PUNCT 1 , [alone] - LITER 1 StrRaw(2) raw - PUNCT 1 , [alone] - LITER 1 Char a - PUNCT 1 , [alone] - LITER 1 Byte b - PUNCT 1 , [alone] - LITER 1 CStr null + LITERAL Integer 1u16 1 + PUNCT , [alone] 1 + LITERAL Integer 2_u32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Integer 4i64 1 + PUNCT , [alone] 1 + LITERAL Float 3.14f32 1 + PUNCT , [alone] 1 + LITERAL Str hello bridge 1 + PUNCT , [alone] 1 + LITERAL Err(()) "suffixed"suffix 1 + PUNCT , [alone] 1 + LITERAL StrRaw(2) raw 1 + PUNCT , [alone] 1 + LITERAL Char a 1 + PUNCT , [alone] 1 + LITERAL Byte b 1 + PUNCT , [alone] 1 + LITERAL CStr null 1 + + + LITERAL Integer 1u16 1 + PUNCT , [alone] 1 + LITERAL Integer 2_u32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Integer 4i64 1 + PUNCT , [alone] 1 + LITERAL Float 3.14f32 1 + PUNCT , [alone] 1 + LITERAL Str hello bridge 1 + PUNCT , [alone] 1 + LITERAL Err(()) "suffixed"suffix 1 + PUNCT , [alone] 1 + LITERAL StrRaw(2) raw 1 + PUNCT , [alone] 1 + LITERAL Char a 1 + PUNCT , [alone] 1 + LITERAL Byte b 1 + PUNCT , [alone] 1 + LITERAL CStr null 1 "#]], expect![[r#" - LITER 42:Root[0000, 0]@0..4#0 Integer 1u16 - PUNCT 42:Root[0000, 0]@4..5#0 , [alone] - LITER 42:Root[0000, 0]@6..11#0 Integer 2_u32 - PUNCT 42:Root[0000, 0]@11..12#0 , [alone] - PUNCT 42:Root[0000, 0]@13..14#0 - [alone] - LITER 42:Root[0000, 0]@14..18#0 Integer 4i64 - PUNCT 42:Root[0000, 0]@18..19#0 , [alone] - LITER 42:Root[0000, 0]@20..27#0 Float 3.14f32 - PUNCT 42:Root[0000, 0]@27..28#0 , [alone] - LITER 42:Root[0000, 0]@29..43#0 Str hello bridge - PUNCT 42:Root[0000, 0]@43..44#0 , [alone] - LITER 42:Root[0000, 0]@45..61#0 Str suffixedsuffix - PUNCT 42:Root[0000, 0]@61..62#0 , [alone] - LITER 42:Root[0000, 0]@63..73#0 StrRaw(2) raw - PUNCT 42:Root[0000, 0]@73..74#0 , [alone] - LITER 42:Root[0000, 0]@75..78#0 Char a - PUNCT 42:Root[0000, 0]@78..79#0 , [alone] - LITER 42:Root[0000, 0]@80..84#0 Byte b - PUNCT 42:Root[0000, 0]@84..85#0 , [alone] - LITER 42:Root[0000, 0]@86..93#0 CStr null - - - LITER 42:Root[0000, 0]@0..4#0 Integer 1u16 - PUNCT 42:Root[0000, 0]@4..5#0 , [alone] - LITER 42:Root[0000, 0]@6..11#0 Integer 2_u32 - PUNCT 42:Root[0000, 0]@11..12#0 , [alone] - PUNCT 42:Root[0000, 0]@13..14#0 - [alone] - LITER 42:Root[0000, 0]@14..18#0 Integer 4i64 - PUNCT 42:Root[0000, 0]@18..19#0 , [alone] - LITER 42:Root[0000, 0]@20..27#0 Float 3.14f32 - PUNCT 42:Root[0000, 0]@27..28#0 , [alone] - LITER 42:Root[0000, 0]@29..43#0 Str hello bridge - PUNCT 42:Root[0000, 0]@43..44#0 , [alone] - LITER 42:Root[0000, 0]@45..61#0 Str suffixedsuffix - PUNCT 42:Root[0000, 0]@61..62#0 , [alone] - LITER 42:Root[0000, 0]@63..73#0 StrRaw(2) raw - PUNCT 42:Root[0000, 0]@73..74#0 , [alone] - LITER 42:Root[0000, 0]@75..78#0 Char a - PUNCT 42:Root[0000, 0]@78..79#0 , [alone] - LITER 42:Root[0000, 0]@80..84#0 Byte b - PUNCT 42:Root[0000, 0]@84..85#0 , [alone] - LITER 42:Root[0000, 0]@86..93#0 CStr null + LITERAL Integer 1u16 42:Root[0000, 0]@0..4#0 + PUNCT , [alone] 42:Root[0000, 0]@4..5#0 + LITERAL Integer 2_u32 42:Root[0000, 0]@6..11#0 + PUNCT , [alone] 42:Root[0000, 0]@11..12#0 + PUNCT - [alone] 42:Root[0000, 0]@13..14#0 + LITERAL Integer 4i64 42:Root[0000, 0]@14..18#0 + PUNCT , [alone] 42:Root[0000, 0]@18..19#0 + LITERAL Float 3.14f32 42:Root[0000, 0]@20..27#0 + PUNCT , [alone] 42:Root[0000, 0]@27..28#0 + LITERAL Str hello bridge 42:Root[0000, 0]@29..43#0 + PUNCT , [alone] 42:Root[0000, 0]@43..44#0 + LITERAL Err(()) "suffixed"suffix 42:Root[0000, 0]@45..61#0 + PUNCT , [alone] 42:Root[0000, 0]@61..62#0 + LITERAL StrRaw(2) raw 42:Root[0000, 0]@63..73#0 + PUNCT , [alone] 42:Root[0000, 0]@73..74#0 + LITERAL Char a 42:Root[0000, 0]@75..78#0 + PUNCT , [alone] 42:Root[0000, 0]@78..79#0 + LITERAL Byte b 42:Root[0000, 0]@80..84#0 + PUNCT , [alone] 42:Root[0000, 0]@84..85#0 + LITERAL CStr null 42:Root[0000, 0]@86..93#0 + + + LITERAL Integer 1u16 42:Root[0000, 0]@0..4#0 + PUNCT , [alone] 42:Root[0000, 0]@4..5#0 + LITERAL Integer 2_u32 42:Root[0000, 0]@6..11#0 + PUNCT , [alone] 42:Root[0000, 0]@11..12#0 + PUNCT - [alone] 42:Root[0000, 0]@13..14#0 + LITERAL Integer 4i64 42:Root[0000, 0]@14..18#0 + PUNCT , [alone] 42:Root[0000, 0]@18..19#0 + LITERAL Float 3.14f32 42:Root[0000, 0]@20..27#0 + PUNCT , [alone] 42:Root[0000, 0]@27..28#0 + LITERAL Str hello bridge 42:Root[0000, 0]@29..43#0 + PUNCT , [alone] 42:Root[0000, 0]@43..44#0 + LITERAL Err(()) "suffixed"suffix 42:Root[0000, 0]@45..61#0 + PUNCT , [alone] 42:Root[0000, 0]@61..62#0 + LITERAL StrRaw(2) raw 42:Root[0000, 0]@63..73#0 + PUNCT , [alone] 42:Root[0000, 0]@73..74#0 + LITERAL Char a 42:Root[0000, 0]@75..78#0 + PUNCT , [alone] 42:Root[0000, 0]@78..79#0 + LITERAL Byte b 42:Root[0000, 0]@80..84#0 + PUNCT , [alone] 42:Root[0000, 0]@84..85#0 + LITERAL CStr null 42:Root[0000, 0]@86..93#0 "#]], ); } @@ -567,56 +567,56 @@ fn test_fn_like_macro_negative_literals() { "fn_like_clone_tokens", r###"-1u16, - 2_u32, -3.14f32, - 2.7"###, expect![[r#" - PUNCT 1 - [alone] - LITER 1 Integer 1u16 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Integer 2_u32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Float 3.14f32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Float 2.7 - - - PUNCT 1 - [alone] - LITER 1 Integer 1u16 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Integer 2_u32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Float 3.14f32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Float 2.7 + PUNCT - [alone] 1 + LITERAL Integer 1u16 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Integer 2_u32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Float 3.14f32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Float 2.7 1 + + + PUNCT - [alone] 1 + LITERAL Integer 1u16 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Integer 2_u32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Float 3.14f32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Float 2.7 1 "#]], expect![[r#" - PUNCT 42:Root[0000, 0]@0..1#0 - [alone] - LITER 42:Root[0000, 0]@1..5#0 Integer 1u16 - PUNCT 42:Root[0000, 0]@5..6#0 , [alone] - PUNCT 42:Root[0000, 0]@7..8#0 - [alone] - LITER 42:Root[0000, 0]@9..14#0 Integer 2_u32 - PUNCT 42:Root[0000, 0]@14..15#0 , [alone] - PUNCT 42:Root[0000, 0]@16..17#0 - [alone] - LITER 42:Root[0000, 0]@17..24#0 Float 3.14f32 - PUNCT 42:Root[0000, 0]@24..25#0 , [alone] - PUNCT 42:Root[0000, 0]@26..27#0 - [alone] - LITER 42:Root[0000, 0]@28..31#0 Float 2.7 - - - PUNCT 42:Root[0000, 0]@0..1#0 - [alone] - LITER 42:Root[0000, 0]@1..5#0 Integer 1u16 - PUNCT 42:Root[0000, 0]@5..6#0 , [alone] - PUNCT 42:Root[0000, 0]@7..8#0 - [alone] - LITER 42:Root[0000, 0]@9..14#0 Integer 2_u32 - PUNCT 42:Root[0000, 0]@14..15#0 , [alone] - PUNCT 42:Root[0000, 0]@16..17#0 - [alone] - LITER 42:Root[0000, 0]@17..24#0 Float 3.14f32 - PUNCT 42:Root[0000, 0]@24..25#0 , [alone] - PUNCT 42:Root[0000, 0]@26..27#0 - [alone] - LITER 42:Root[0000, 0]@28..31#0 Float 2.7 + PUNCT - [alone] 42:Root[0000, 0]@0..1#0 + LITERAL Integer 1u16 42:Root[0000, 0]@1..5#0 + PUNCT , [alone] 42:Root[0000, 0]@5..6#0 + PUNCT - [alone] 42:Root[0000, 0]@7..8#0 + LITERAL Integer 2_u32 42:Root[0000, 0]@9..14#0 + PUNCT , [alone] 42:Root[0000, 0]@14..15#0 + PUNCT - [alone] 42:Root[0000, 0]@16..17#0 + LITERAL Float 3.14f32 42:Root[0000, 0]@17..24#0 + PUNCT , [alone] 42:Root[0000, 0]@24..25#0 + PUNCT - [alone] 42:Root[0000, 0]@26..27#0 + LITERAL Float 2.7 42:Root[0000, 0]@28..31#0 + + + PUNCT - [alone] 42:Root[0000, 0]@0..1#0 + LITERAL Integer 1u16 42:Root[0000, 0]@1..5#0 + PUNCT , [alone] 42:Root[0000, 0]@5..6#0 + PUNCT - [alone] 42:Root[0000, 0]@7..8#0 + LITERAL Integer 2_u32 42:Root[0000, 0]@9..14#0 + PUNCT , [alone] 42:Root[0000, 0]@14..15#0 + PUNCT - [alone] 42:Root[0000, 0]@16..17#0 + LITERAL Float 3.14f32 42:Root[0000, 0]@17..24#0 + PUNCT , [alone] 42:Root[0000, 0]@24..25#0 + PUNCT - [alone] 42:Root[0000, 0]@26..27#0 + LITERAL Float 2.7 42:Root[0000, 0]@28..31#0 "#]], ); } @@ -631,36 +631,36 @@ fn test_attr_macro() { r#"mod m {}"#, r#"some arguments"#, expect![[r#" - IDENT 1 mod - IDENT 1 m - GROUP {} 1 1 1 + IDENT mod 1 + IDENT m 1 + GROUP {} 1 1 - IDENT 1 some - IDENT 1 arguments + IDENT some 1 + IDENT arguments 1 - IDENT 1 compile_error - PUNCT 1 ! [joint] - GROUP () 1 1 1 - LITER 1 Str #[attr_error(some arguments)] mod m {} - PUNCT 1 ; [alone] + IDENT compile_error 1 + PUNCT ! [joint] 1 + GROUP () 1 1 + LITERAL Str #[attr_error(some arguments)] mod m {} 1 + PUNCT ; [alone] 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..3#0 mod - IDENT 42:Root[0000, 0]@4..5#0 m - GROUP {} 42:Root[0000, 0]@6..7#0 42:Root[0000, 0]@7..8#0 42:Root[0000, 0]@6..8#0 + IDENT mod 42:Root[0000, 0]@0..3#0 + IDENT m 42:Root[0000, 0]@4..5#0 + GROUP {} 42:Root[0000, 0]@6..7#0 42:Root[0000, 0]@7..8#0 - IDENT 42:Root[0000, 0]@0..4#0 some - IDENT 42:Root[0000, 0]@5..14#0 arguments + IDENT some 42:Root[0000, 0]@0..4#0 + IDENT arguments 42:Root[0000, 0]@5..14#0 - IDENT 42:Root[0000, 0]@0..13#0 compile_error - PUNCT 42:Root[0000, 0]@13..14#0 ! [joint] - GROUP () 42:Root[0000, 0]@14..15#0 42:Root[0000, 0]@55..56#0 42:Root[0000, 0]@14..56#0 - LITER 42:Root[0000, 0]@15..55#0 Str #[attr_error(some arguments)] mod m {} - PUNCT 42:Root[0000, 0]@56..57#0 ; [alone] + IDENT compile_error 42:Root[0000, 0]@0..13#0 + PUNCT ! [joint] 42:Root[0000, 0]@13..14#0 + GROUP () 42:Root[0000, 0]@14..15#0 42:Root[0000, 0]@55..56#0 + LITERAL Str #[attr_error(some arguments)] mod m {} 42:Root[0000, 0]@15..55#0 + PUNCT ; [alone] 42:Root[0000, 0]@56..57#0 "#]], ); } @@ -722,8 +722,8 @@ fn test_fn_like_span_line_column() { " hello", expect![[r#" - LITER 42:Root[0000, 0]@0..100#0 Integer 2 - LITER 42:Root[0000, 0]@0..100#0 Integer 1 + LITERAL Integer 2 42:Root[0000, 0]@0..100#0 + LITERAL Integer 1 42:Root[0000, 0]@0..100#0 "#]], ); } diff --git a/crates/proc-macro-srv/src/tests/utils.rs b/crates/proc-macro-srv/src/tests/utils.rs index 7f92c66fb69d..b514e8e3d8f9 100644 --- a/crates/proc-macro-srv/src/tests/utils.rs +++ b/crates/proc-macro-srv/src/tests/utils.rs @@ -1,6 +1,7 @@ //! utils used in proc-macro tests use expect_test::Expect; +use proc_macro_api::token_stream::TokenStream; use span::{ EditionedFileId, FileId, ROOT_ERASED_FILE_AST_ID, Span, SpanAnchor, SyntaxContext, TextRange, }; @@ -8,7 +9,6 @@ use std::ops::Range; use crate::{ EnvSnapshot, ProcMacroClientInterface, ProcMacroSrv, SpanId, dylib, proc_macro_test_dylib_path, - token_stream::TokenStream, }; fn make_ctx() -> SyntaxContext { diff --git a/crates/proc-macro-srv/src/token_stream.rs b/crates/proc-macro-srv/src/token_stream.rs deleted file mode 100644 index 5201bb6aeb86..000000000000 --- a/crates/proc-macro-srv/src/token_stream.rs +++ /dev/null @@ -1,767 +0,0 @@ -//! The proc-macro server token stream implementation. - -use core::fmt; -use std::{mem, sync::Arc}; - -use intern::Symbol; -use rustc_lexer::{DocStyle, LiteralKind}; -use rustc_proc_macro::Delimiter; - -use crate::bridge::{DelimSpan, Group, Ident, LitKind, Literal, Punct, TokenTree}; - -/// Trait for allowing tests to parse tokenstreams with dynamic span ranges -pub(crate) trait SpanLike { - fn derive_ranged(&self, range: std::ops::Range) -> Self; -} - -#[derive(Clone)] -pub struct TokenStream(pub(crate) Arc>>); - -impl Default for TokenStream { - fn default() -> Self { - Self(Default::default()) - } -} - -impl TokenStream { - pub fn new(tts: Vec>) -> TokenStream { - TokenStream(Arc::new(tts)) - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - pub fn len(&self) -> usize { - self.0.len() - } - - pub fn iter(&self) -> TokenStreamIter<'_, S> { - TokenStreamIter::new(self) - } - - pub fn as_single_group(&self) -> Option<&Group> { - match &**self.0 { - [TokenTree::Group(group)] => Some(group), - _ => None, - } - } - - pub(crate) fn from_str(s: &str, span: S) -> Result - where - S: SpanLike + Copy, - { - let mut groups = Vec::new(); - groups.push((rustc_proc_macro::Delimiter::None, 0..0, vec![])); - let mut offset = 0; - let mut tokens = rustc_lexer::tokenize(s, rustc_lexer::FrontmatterAllowed::No).peekable(); - while let Some(token) = tokens.next() { - let range = offset..offset + token.len as usize; - offset += token.len as usize; - - let mut is_joint = || { - tokens.peek().is_some_and(|token| { - matches!( - token.kind, - rustc_lexer::TokenKind::RawLifetime - | rustc_lexer::TokenKind::GuardedStrPrefix - | rustc_lexer::TokenKind::Lifetime { .. } - | rustc_lexer::TokenKind::Semi - | rustc_lexer::TokenKind::Comma - | rustc_lexer::TokenKind::Dot - | rustc_lexer::TokenKind::OpenParen - | rustc_lexer::TokenKind::CloseParen - | rustc_lexer::TokenKind::OpenBrace - | rustc_lexer::TokenKind::CloseBrace - | rustc_lexer::TokenKind::OpenBracket - | rustc_lexer::TokenKind::CloseBracket - | rustc_lexer::TokenKind::At - | rustc_lexer::TokenKind::Pound - | rustc_lexer::TokenKind::Tilde - | rustc_lexer::TokenKind::Question - | rustc_lexer::TokenKind::Colon - | rustc_lexer::TokenKind::Dollar - | rustc_lexer::TokenKind::Eq - | rustc_lexer::TokenKind::Bang - | rustc_lexer::TokenKind::Lt - | rustc_lexer::TokenKind::Gt - | rustc_lexer::TokenKind::Minus - | rustc_lexer::TokenKind::And - | rustc_lexer::TokenKind::Or - | rustc_lexer::TokenKind::Plus - | rustc_lexer::TokenKind::Star - | rustc_lexer::TokenKind::Slash - | rustc_lexer::TokenKind::Percent - | rustc_lexer::TokenKind::Caret - ) - }) - }; - - let Some((open_delim, _, tokenstream)) = groups.last_mut() else { - return Err("Unbalanced delimiters".to_owned()); - }; - match token.kind { - rustc_lexer::TokenKind::OpenParen => { - groups.push((rustc_proc_macro::Delimiter::Parenthesis, range, vec![])) - } - rustc_lexer::TokenKind::CloseParen if *open_delim != Delimiter::Parenthesis => { - return if *open_delim == Delimiter::None { - Err("Unexpected ')'".to_owned()) - } else { - Err("Expected ')'".to_owned()) - }; - } - rustc_lexer::TokenKind::CloseParen => { - let (delimiter, open_range, stream) = groups.pop().unwrap(); - groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( - TokenTree::Group(Group { - delimiter, - stream: if stream.is_empty() { - None - } else { - Some(TokenStream::new(stream)) - }, - span: DelimSpan { - entire: span.derive_ranged(open_range.start..range.end), - open: span.derive_ranged(open_range), - close: span.derive_ranged(range), - }, - }), - ); - } - rustc_lexer::TokenKind::OpenBrace => { - groups.push((rustc_proc_macro::Delimiter::Brace, range, vec![])) - } - rustc_lexer::TokenKind::CloseBrace if *open_delim != Delimiter::Brace => { - return if *open_delim == Delimiter::None { - Err("Unexpected '}'".to_owned()) - } else { - Err("Expected '}'".to_owned()) - }; - } - rustc_lexer::TokenKind::CloseBrace => { - let (delimiter, open_range, stream) = groups.pop().unwrap(); - groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( - TokenTree::Group(Group { - delimiter, - stream: if stream.is_empty() { - None - } else { - Some(TokenStream::new(stream)) - }, - span: DelimSpan { - entire: span.derive_ranged(open_range.start..range.end), - open: span.derive_ranged(open_range), - close: span.derive_ranged(range), - }, - }), - ); - } - rustc_lexer::TokenKind::OpenBracket => { - groups.push((rustc_proc_macro::Delimiter::Bracket, range, vec![])) - } - rustc_lexer::TokenKind::CloseBracket if *open_delim != Delimiter::Bracket => { - return if *open_delim == Delimiter::None { - Err("Unexpected ']'".to_owned()) - } else { - Err("Expected ']'".to_owned()) - }; - } - rustc_lexer::TokenKind::CloseBracket => { - let (delimiter, open_range, stream) = groups.pop().unwrap(); - groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( - TokenTree::Group(Group { - delimiter, - stream: if stream.is_empty() { - None - } else { - Some(TokenStream::new(stream)) - }, - span: DelimSpan { - entire: span.derive_ranged(open_range.start..range.end), - open: span.derive_ranged(open_range), - close: span.derive_ranged(range), - }, - }), - ); - } - rustc_lexer::TokenKind::LineComment { doc_style: None } - | rustc_lexer::TokenKind::BlockComment { doc_style: None, terminated: _ } => { - continue; - } - rustc_lexer::TokenKind::LineComment { doc_style: Some(doc_style) } => { - let text = &s[range.start + 3..range.end]; - tokenstream.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span })); - if doc_style == DocStyle::Inner { - tokenstream.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span })); - } - tokenstream.push(TokenTree::Group(Group { - delimiter: Delimiter::Bracket, - stream: Some(TokenStream::new(vec![ - TokenTree::Ident(Ident { - sym: Symbol::intern("doc"), - is_raw: false, - span, - }), - TokenTree::Punct(Punct { ch: b'=', joint: false, span }), - TokenTree::Literal(Literal { - kind: LitKind::Str, - symbol: Symbol::intern(&text.escape_debug().to_string()), - suffix: None, - span: span.derive_ranged(range), - }), - ])), - span: DelimSpan { open: span, close: span, entire: span }, - })); - } - rustc_lexer::TokenKind::BlockComment { doc_style: Some(doc_style), terminated } => { - let text = - &s[range.start + 3..if terminated { range.end - 2 } else { range.end }]; - tokenstream.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span })); - if doc_style == DocStyle::Inner { - tokenstream.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span })); - } - tokenstream.push(TokenTree::Group(Group { - delimiter: Delimiter::Bracket, - stream: Some(TokenStream::new(vec![ - TokenTree::Ident(Ident { - sym: Symbol::intern("doc"), - is_raw: false, - span, - }), - TokenTree::Punct(Punct { ch: b'=', joint: false, span }), - TokenTree::Literal(Literal { - kind: LitKind::Str, - symbol: Symbol::intern(&text.escape_debug().to_string()), - suffix: None, - span: span.derive_ranged(range), - }), - ])), - span: DelimSpan { open: span, close: span, entire: span }, - })); - } - rustc_lexer::TokenKind::Whitespace => continue, - rustc_lexer::TokenKind::Frontmatter { .. } => unreachable!(), - rustc_lexer::TokenKind::Unknown => { - return Err(format!("Unknown token: `{}`", &s[range])); - } - rustc_lexer::TokenKind::UnknownPrefix => { - return Err(format!("Unknown prefix: `{}`", &s[range])); - } - rustc_lexer::TokenKind::UnknownPrefixLifetime => { - return Err(format!("Unknown lifetime prefix: `{}`", &s[range])); - } - // FIXME: Error on edition >= 2024 ... I dont think the proc-macro server can fetch editions currently - // and whose edition is this? - rustc_lexer::TokenKind::GuardedStrPrefix => { - tokenstream.push(TokenTree::Punct(Punct { - ch: s.as_bytes()[range.start], - joint: true, - span: span.derive_ranged(range.start..range.start + 1), - })); - tokenstream.push(TokenTree::Punct(Punct { - ch: s.as_bytes()[range.start + 1], - joint: is_joint(), - span: span.derive_ranged(range.start + 1..range.end), - })) - } - rustc_lexer::TokenKind::Ident => tokenstream.push(TokenTree::Ident(Ident { - sym: Symbol::intern(&s[range.clone()]), - is_raw: false, - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::InvalidIdent => { - return Err(format!("Invalid identifier: `{}`", &s[range])); - } - rustc_lexer::TokenKind::RawIdent => { - let range = range.start + 2..range.end; - tokenstream.push(TokenTree::Ident(Ident { - sym: Symbol::intern(&s[range.clone()]), - is_raw: true, - span: span.derive_ranged(range), - })) - } - rustc_lexer::TokenKind::Literal { kind, suffix_start } => { - tokenstream.push(TokenTree::Literal(literal_from_lexer( - &s[range.clone()], - span.derive_ranged(range), - kind, - suffix_start, - ))) - } - rustc_lexer::TokenKind::RawLifetime => { - let range = range.start + 1 + 2..range.end; - tokenstream.push(TokenTree::Punct(Punct { - ch: b'\'', - joint: true, - span: span.derive_ranged(range.start..range.start + 1), - })); - tokenstream.push(TokenTree::Ident(Ident { - sym: Symbol::intern(&s[range.clone()]), - is_raw: true, - span: span.derive_ranged(range), - })) - } - rustc_lexer::TokenKind::Lifetime { starts_with_number } => { - if starts_with_number { - return Err("Lifetime cannot start with a number".to_owned()); - } - let range = range.start + 1..range.end; - tokenstream.push(TokenTree::Punct(Punct { - ch: b'\'', - joint: true, - span: span.derive_ranged(range.start..range.start + 1), - })); - tokenstream.push(TokenTree::Ident(Ident { - sym: Symbol::intern(&s[range.clone()]), - is_raw: false, - span: span.derive_ranged(range), - })) - } - rustc_lexer::TokenKind::Semi => tokenstream.push(TokenTree::Punct(Punct { - ch: b';', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Comma => tokenstream.push(TokenTree::Punct(Punct { - ch: b',', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Dot => tokenstream.push(TokenTree::Punct(Punct { - ch: b'.', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::At => tokenstream.push(TokenTree::Punct(Punct { - ch: b'@', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Pound => tokenstream.push(TokenTree::Punct(Punct { - ch: b'#', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Tilde => tokenstream.push(TokenTree::Punct(Punct { - ch: b'~', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Question => tokenstream.push(TokenTree::Punct(Punct { - ch: b'?', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Colon => tokenstream.push(TokenTree::Punct(Punct { - ch: b':', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Dollar => tokenstream.push(TokenTree::Punct(Punct { - ch: b'$', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Eq => tokenstream.push(TokenTree::Punct(Punct { - ch: b'=', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Bang => tokenstream.push(TokenTree::Punct(Punct { - ch: b'!', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Lt => tokenstream.push(TokenTree::Punct(Punct { - ch: b'<', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Gt => tokenstream.push(TokenTree::Punct(Punct { - ch: b'>', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Minus => tokenstream.push(TokenTree::Punct(Punct { - ch: b'-', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::And => tokenstream.push(TokenTree::Punct(Punct { - ch: b'&', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Or => tokenstream.push(TokenTree::Punct(Punct { - ch: b'|', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Plus => tokenstream.push(TokenTree::Punct(Punct { - ch: b'+', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Star => tokenstream.push(TokenTree::Punct(Punct { - ch: b'*', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Slash => tokenstream.push(TokenTree::Punct(Punct { - ch: b'/', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Caret => tokenstream.push(TokenTree::Punct(Punct { - ch: b'^', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Percent => tokenstream.push(TokenTree::Punct(Punct { - ch: b'%', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Eof => break, - } - } - if let Some((Delimiter::None, _, tokentrees)) = groups.pop() - && groups.is_empty() - { - Ok(TokenStream::new(tokentrees)) - } else { - Err("Mismatched token groups".to_owned()) - } - } -} - -impl fmt::Display for TokenStream { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut emit_whitespace = false; - for tt in self.0.iter() { - display_token_tree(tt, &mut emit_whitespace, f)?; - } - Ok(()) - } -} - -fn display_token_tree( - tt: &TokenTree, - emit_whitespace: &mut bool, - f: &mut std::fmt::Formatter<'_>, -) -> std::fmt::Result { - if mem::take(emit_whitespace) { - write!(f, " ")?; - } - match tt { - TokenTree::Group(Group { delimiter, stream, span: _ }) => { - write!( - f, - "{}", - match delimiter { - rustc_proc_macro::Delimiter::Parenthesis => "(", - rustc_proc_macro::Delimiter::Brace => "{", - rustc_proc_macro::Delimiter::Bracket => "[", - rustc_proc_macro::Delimiter::None => "", - } - )?; - if let Some(stream) = stream { - write!(f, "{stream}")?; - } - write!( - f, - "{}", - match delimiter { - rustc_proc_macro::Delimiter::Parenthesis => ")", - rustc_proc_macro::Delimiter::Brace => "}", - rustc_proc_macro::Delimiter::Bracket => "]", - rustc_proc_macro::Delimiter::None => "", - } - )?; - } - TokenTree::Punct(Punct { ch, joint, span: _ }) => { - *emit_whitespace = !*joint; - write!(f, "{}", *ch as char)?; - } - TokenTree::Ident(Ident { sym, is_raw, span: _ }) => { - if *is_raw { - write!(f, "r#")?; - } - write!(f, "{sym}")?; - *emit_whitespace = true; - } - TokenTree::Literal(lit) => { - display_fmt_literal(lit, f)?; - let joint = match lit.kind { - LitKind::Str - | LitKind::StrRaw(_) - | LitKind::ByteStr - | LitKind::ByteStrRaw(_) - | LitKind::CStr - | LitKind::CStrRaw(_) => true, - _ => false, - }; - *emit_whitespace = !joint; - } - } - Ok(()) -} - -pub fn literal_to_string(literal: &Literal) -> String { - let mut buf = String::new(); - display_fmt_literal(literal, &mut buf).unwrap(); - buf -} - -fn display_fmt_literal(literal: &Literal, f: &mut impl std::fmt::Write) -> fmt::Result { - match literal.kind { - LitKind::Byte => write!(f, "b'{}'", literal.symbol), - LitKind::Char => write!(f, "'{}'", literal.symbol), - LitKind::Integer | LitKind::Float | LitKind::ErrWithGuar => { - write!(f, "{}", literal.symbol) - } - LitKind::Str => write!(f, "\"{}\"", literal.symbol), - LitKind::ByteStr => write!(f, "b\"{}\"", literal.symbol), - LitKind::CStr => write!(f, "c\"{}\"", literal.symbol), - LitKind::StrRaw(num_of_hashes) => { - let num_of_hashes = num_of_hashes as usize; - write!( - f, - r#"r{0:# { - let num_of_hashes = num_of_hashes as usize; - write!( - f, - r#"br{0:# { - let num_of_hashes = num_of_hashes as usize; - write!( - f, - r#"cr{0:# fmt::Debug for TokenStream { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - debug_token_stream(self, 0, f) - } -} - -fn debug_token_stream( - ts: &TokenStream, - depth: usize, - f: &mut std::fmt::Formatter<'_>, -) -> std::fmt::Result { - for tt in ts.0.iter() { - debug_token_tree(tt, depth, f)?; - } - Ok(()) -} - -fn debug_token_tree( - tt: &TokenTree, - depth: usize, - f: &mut std::fmt::Formatter<'_>, -) -> std::fmt::Result { - write!(f, "{:indent$}", "", indent = depth * 2)?; - match tt { - TokenTree::Group(Group { delimiter, stream, span }) => { - writeln!( - f, - "GROUP {}{} {:#?} {:#?} {:#?}", - match delimiter { - rustc_proc_macro::Delimiter::Parenthesis => "(", - rustc_proc_macro::Delimiter::Brace => "{", - rustc_proc_macro::Delimiter::Bracket => "[", - rustc_proc_macro::Delimiter::None => "$", - }, - match delimiter { - rustc_proc_macro::Delimiter::Parenthesis => ")", - rustc_proc_macro::Delimiter::Brace => "}", - rustc_proc_macro::Delimiter::Bracket => "]", - rustc_proc_macro::Delimiter::None => "$", - }, - span.open, - span.close, - span.entire, - )?; - if let Some(stream) = stream { - debug_token_stream(stream, depth + 1, f)?; - } - return Ok(()); - } - TokenTree::Punct(Punct { ch, joint, span }) => write!( - f, - "PUNCT {span:#?} {} {}", - *ch as char, - if *joint { "[joint]" } else { "[alone]" } - )?, - TokenTree::Ident(Ident { sym, is_raw, span }) => { - write!(f, "IDENT {span:#?} ")?; - if *is_raw { - write!(f, "r#")?; - } - write!(f, "{sym}")?; - } - TokenTree::Literal(Literal { kind, symbol, suffix, span }) => write!( - f, - "LITER {span:#?} {kind:?} {symbol}{}", - match suffix { - Some(suffix) => suffix.clone(), - None => Symbol::intern(""), - } - )?, - } - writeln!(f) -} - -impl TokenStream { - /// Push `tt` onto the end of the stream, possibly gluing it to the last - /// token. Uses `make_mut` to maximize efficiency. - pub(crate) fn push_tree(&mut self, tt: TokenTree) { - let vec_mut = Arc::make_mut(&mut self.0); - vec_mut.push(tt); - } - - /// Push `stream` onto the end of the stream, possibly gluing the first - /// token tree to the last token. (No other token trees will be glued.) - /// Uses `make_mut` to maximize efficiency. - pub(crate) fn push_stream(&mut self, stream: TokenStream) { - let vec_mut = Arc::make_mut(&mut self.0); - - let stream_iter = stream.0.iter().cloned(); - - vec_mut.extend(stream_iter); - } -} - -impl FromIterator> for TokenStream { - fn from_iter>>(iter: I) -> Self { - TokenStream::new(iter.into_iter().collect::>>()) - } -} - -#[derive(Clone)] -pub struct TokenStreamIter<'t, S> { - stream: &'t TokenStream, - index: usize, -} - -impl<'t, S> TokenStreamIter<'t, S> { - fn new(stream: &'t TokenStream) -> Self { - TokenStreamIter { stream, index: 0 } - } -} - -impl<'t, S> Iterator for TokenStreamIter<'t, S> { - type Item = &'t TokenTree; - - fn next(&mut self) -> Option<&'t TokenTree> { - self.stream.0.get(self.index).map(|tree| { - self.index += 1; - tree - }) - } -} - -pub(super) fn literal_from_lexer( - s: &str, - span: Span, - kind: rustc_lexer::LiteralKind, - suffix_start: u32, -) -> Literal { - let (kind, start_offset, end_offset) = match kind { - LiteralKind::Int { .. } => (LitKind::Integer, 0, 0), - LiteralKind::Float { .. } => (LitKind::Float, 0, 0), - LiteralKind::Char { terminated } => (LitKind::Char, 1, terminated as usize), - LiteralKind::Byte { terminated } => (LitKind::Byte, 2, terminated as usize), - LiteralKind::Str { terminated } => (LitKind::Str, 1, terminated as usize), - LiteralKind::ByteStr { terminated } => (LitKind::ByteStr, 2, terminated as usize), - LiteralKind::CStr { terminated } => (LitKind::CStr, 2, terminated as usize), - LiteralKind::RawStr { n_hashes } => ( - LitKind::StrRaw(n_hashes.unwrap_or_default()), - 2 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - LiteralKind::RawByteStr { n_hashes } => ( - LitKind::ByteStrRaw(n_hashes.unwrap_or_default()), - 3 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - LiteralKind::RawCStr { n_hashes } => ( - LitKind::CStrRaw(n_hashes.unwrap_or_default()), - 3 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - }; - - let (lit, suffix) = s.split_at(suffix_start as usize); - let lit = &lit[start_offset..lit.len() - end_offset]; - let suffix = match suffix { - "" | "_" => None, - suffix => Some(Symbol::intern(suffix)), - }; - - Literal { kind, symbol: Symbol::intern(lit), suffix, span } -} - -impl SpanLike for crate::SpanId { - fn derive_ranged(&self, _: std::ops::Range) -> Self { - *self - } -} - -impl SpanLike for () { - fn derive_ranged(&self, _: std::ops::Range) -> Self { - *self - } -} - -impl SpanLike for crate::Span { - fn derive_ranged(&self, range: std::ops::Range) -> Self { - crate::Span { - range: span::TextRange::new( - span::TextSize::new(range.start as u32), - span::TextSize::new(range.end as u32), - ), - anchor: self.anchor, - ctx: self.ctx, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn ts_to_string() { - let token_stream = - TokenStream::from_str("{} () [] <> ;/., \"gfhdgfuiofghd\" 0f32 r#\"dff\"# 'r#lt", ()) - .unwrap(); - assert_eq!(token_stream.to_string(), "{}()[]<> ;/., \"gfhdgfuiofghd\"0f32 r#\"dff\"#'r#lt"); - } - - #[test] - fn doc_comment_from_str() { - let token_stream = TokenStream::from_str("/// foo", ()).unwrap(); - assert_eq!(token_stream.to_string(), r#"# [doc = " foo"]"#); - } -} diff --git a/crates/rust-analyzer/src/global_state.rs b/crates/rust-analyzer/src/global_state.rs index c0d87104fbe1..3659c718b177 100644 --- a/crates/rust-analyzer/src/global_state.rs +++ b/crates/rust-analyzer/src/global_state.rs @@ -23,7 +23,7 @@ use parking_lot::{ MappedRwLockReadGuard, Mutex, RwLock, RwLockReadGuard, RwLockUpgradableReadGuard, RwLockWriteGuard, }; -use proc_macro_api::ProcMacroClient; +use proc_macro_api::client::ProcMacroClient; use project_model::{ ManifestPath, ProjectWorkspace, ProjectWorkspaceKind, TargetKind, WorkspaceBuildScripts, }; diff --git a/crates/rust-analyzer/src/reload.rs b/crates/rust-analyzer/src/reload.rs index 039fbeff828e..91014f0f7198 100644 --- a/crates/rust-analyzer/src/reload.rs +++ b/crates/rust-analyzer/src/reload.rs @@ -24,7 +24,7 @@ use itertools::Itertools; use load_cargo::{ProjectFolders, load_proc_macro}; use lsp_types::FileSystemWatcher; use paths::Utf8Path; -use proc_macro_api::ProcMacroClient; +use proc_macro_api::client::ProcMacroClient; use project_model::{ ManifestPath, ProjectWorkspace, ProjectWorkspaceKind, WorkspaceBuildScripts, project_json, }; diff --git a/crates/stdx/src/lib.rs b/crates/stdx/src/lib.rs index dcba06415b5f..7d844c041b0a 100644 --- a/crates/stdx/src/lib.rs +++ b/crates/stdx/src/lib.rs @@ -363,6 +363,32 @@ pub fn slice_tails(this: &[T]) -> impl Iterator { (0..this.len()).map(|i| &this[i..]) } +/// Imports a sysroot crate from the sysroot or from crates.io, depending on whether the `in-rust-tree` +/// feature is active. +/// +/// Syntax: +/// ``` +/// extern crate sysroot_crate or crates_io_crate; +/// ``` +// FIXME: Should this really be in `stdx`? +#[macro_export] +macro_rules! rustc_crates { + ( + $( + extern crate $sysroot_crate:ident or $crates_io_crate:ident; + )* + ) => { + ::std::cfg_select! { + feature = "in-rust-tree" => { + $( extern crate $sysroot_crate; )* + } + _ => { + $( extern crate $crates_io_crate as $sysroot_crate; )* + } + } + }; +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/syntax-bridge/src/lib.rs b/crates/syntax-bridge/src/lib.rs index 181f9a14e764..f0e1c5cf27ee 100644 --- a/crates/syntax-bridge/src/lib.rs +++ b/crates/syntax-bridge/src/lib.rs @@ -18,7 +18,7 @@ use syntax::{ ast::{self, make::tokens::doc_comment}, format_smolstr, }; -use tt::{Punct, buffer::Cursor, token_to_literal}; +use tt::{Punct, buffer::Cursor}; pub mod prettify_macro_expansion; mod to_parser_input; @@ -318,7 +318,7 @@ where k if k.is_literal() => { let text = token.to_text(conv); let span = conv.span_for(abs_range); - token_to_literal(&text, span).into() + tt::literal_from_str_or_err(&text, span).into() } LIFETIME_IDENT => { let apostrophe = tt::Leaf::from(tt::Punct { diff --git a/crates/syntax/Cargo.toml b/crates/syntax/Cargo.toml index a9df1acdae9a..8bf7bb582942 100644 --- a/crates/syntax/Cargo.toml +++ b/crates/syntax/Cargo.toml @@ -40,3 +40,6 @@ in-rust-tree = [] [lints] workspace = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_lexer"] diff --git a/crates/syntax/src/lib.rs b/crates/syntax/src/lib.rs index dc7d7af7a2b9..8f46683da748 100644 --- a/crates/syntax/src/lib.rs +++ b/crates/syntax/src/lib.rs @@ -21,12 +21,12 @@ #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] -#[cfg(not(feature = "in-rust-tree"))] -extern crate ra_ap_rustc_lexer as rustc_lexer; #[cfg(feature = "in-rust-tree")] extern crate rustc_driver as _; -#[cfg(feature = "in-rust-tree")] -extern crate rustc_lexer; + +stdx::rustc_crates! { + extern crate rustc_lexer or ra_ap_rustc_lexer; +} mod parsing; mod ptr; diff --git a/crates/tt/Cargo.toml b/crates/tt/Cargo.toml index bd8f740b2f50..d7465d09c129 100644 --- a/crates/tt/Cargo.toml +++ b/crates/tt/Cargo.toml @@ -14,16 +14,30 @@ doctest = false [dependencies] arrayvec.workspace = true -text-size.workspace = true -rustc-hash.workspace = true +text-size = { workspace = true, optional = true } +rustc-hash = { workspace = true, optional = true } -span = { path = "../span", version = "0.0", default-features = false } -stdx.workspace = true +span = { path = "../span", version = "0.0", default-features = false, optional = true } +stdx = { workspace = true, optional = true } intern.workspace = true ra-ap-rustc_lexer.workspace = true [features] +default = ["in-ra"] in-rust-tree = [] +# Inside rust-analyzer and not the proc macro server. Being in the proc macro server disables everything but the TokenTree types. +in-ra = [ + "dep:text-size", + "dep:rustc-hash", + "dep:span", + "dep:stdx", +] [lints] workspace = true + +[package.metadata.rust-analyzer] +rustc_private = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_lexer"] diff --git a/crates/tt/src/leaf_types.rs b/crates/tt/src/leaf_types.rs new file mode 100644 index 000000000000..a8d48d097e6b --- /dev/null +++ b/crates/tt/src/leaf_types.rs @@ -0,0 +1,398 @@ +//! Types that are shared between rust-analyzer and the proc macro server. + +use std::fmt; + +use arrayvec::ArrayString; +use intern::Symbol; + +#[cfg(feature = "in-ra")] +type DefaultSpan = span::Span; +#[cfg(not(feature = "in-ra"))] +pub enum DefaultSpan {} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +#[repr(u8)] +// The discriminants are important for `storage.rs` decoding. +pub enum IdentIsRaw { + No = 0, + Yes = 1, +} + +impl IdentIsRaw { + pub fn yes(self) -> bool { + matches!(self, IdentIsRaw::Yes) + } + pub fn no(&self) -> bool { + matches!(self, IdentIsRaw::No) + } + pub fn as_str(self) -> &'static str { + match self { + IdentIsRaw::No => "", + IdentIsRaw::Yes => "r#", + } + } + pub fn split_from_symbol(sym: &str) -> (Self, &str) { + if let Some(sym) = sym.strip_prefix("r#") { + (IdentIsRaw::Yes, sym) + } else { + (IdentIsRaw::No, sym) + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +pub enum LitKind { + Byte, + Char, + Integer, // e.g. `1`, `1u8`, `1f32` + Float, // e.g. `1.`, `1.0`, `1e3f32` + Str, + StrRaw(u8), // raw string delimited by `n` hash symbols + ByteStr, + ByteStrRaw(u8), // raw byte string delimited by `n` hash symbols + CStr, + CStrRaw(u8), + Err(()), +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Leaf { + Literal(Literal), + Punct(Punct), + Ident(Ident), +} + +#[derive(Debug, Copy, Clone, PartialEq)] +pub struct DelimSpan { + pub open: Span, + pub close: Span, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Delimiter { + pub open: Span, + pub close: Span, + pub kind: DelimiterKind, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[repr(u8)] +// The discriminants are important for decoding for `storage.rs`. +pub enum DelimiterKind { + Parenthesis = 0, + Brace = 1, + Bracket = 2, + Invisible = 3, +} + +impl DelimiterKind { + pub fn display_open_close(self) -> (&'static str, &'static str) { + match self { + DelimiterKind::Brace => ("{", "}"), + DelimiterKind::Bracket => ("[", "]"), + DelimiterKind::Parenthesis => ("(", ")"), + DelimiterKind::Invisible => ("", ""), + } + } + + pub fn debug_view(self) -> &'static str { + match self { + DelimiterKind::Invisible => "$$", + DelimiterKind::Parenthesis => "()", + DelimiterKind::Brace => "{}", + DelimiterKind::Bracket => "[]", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Literal { + /// Escaped, text then suffix concatenated. + pub text_and_suffix: Symbol, + pub span: Span, + pub kind: LitKind, + pub suffix_len: u8, +} + +impl Literal { + #[inline] + pub fn text_and_suffix(&self) -> (&str, &str) { + let text_and_suffix = self.text_and_suffix.as_str(); + text_and_suffix.split_at(text_and_suffix.len() - usize::from(self.suffix_len)) + } + + pub fn text_and_suffix_symbols(&self) -> (Symbol, Option) { + if self.suffix_len == 0 { + (self.text_and_suffix.clone(), None) + } else { + let (text, suffix) = self.text_and_suffix(); + (Symbol::intern(text), Some(Symbol::intern(suffix))) + } + } + + #[inline] + pub fn text(&self) -> &str { + self.text_and_suffix().0 + } + + #[inline] + pub fn suffix(&self) -> &str { + self.text_and_suffix().1 + } + + pub fn new(text: &str, span: Span, kind: LitKind, suffix: &str) -> Self { + const MAX_INLINE_CAPACITY: usize = 30; + let text_and_suffix = if suffix.is_empty() { + Symbol::intern(text) + } else if (text.len() + suffix.len()) < MAX_INLINE_CAPACITY { + let mut text_and_suffix = ArrayString::::new(); + text_and_suffix.push_str(text); + text_and_suffix.push_str(suffix); + Symbol::intern(&text_and_suffix) + } else { + let mut text_and_suffix = String::with_capacity(text.len() + suffix.len()); + text_and_suffix.push_str(text); + text_and_suffix.push_str(suffix); + Symbol::intern(&text_and_suffix) + }; + + Self { text_and_suffix, span, kind, suffix_len: suffix.len().try_into().unwrap() } + } + + #[inline] + pub fn new_no_suffix(text: &str, span: Span, kind: LitKind) -> Self { + Self { text_and_suffix: Symbol::intern(text), span, kind, suffix_len: 0 } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Punct { + pub char: char, + pub spacing: Spacing, + pub span: Span, +} + +/// Indicates whether a token can join with the following token to form a +/// compound token. Used for conversions to `proc_macro::Spacing`. Also used to +/// guide pretty-printing, which is where the `JointHidden` value (which isn't +/// part of `proc_macro::Spacing`) comes in useful. +// The discriminants are important for decoding for `storage.rs`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(u8)] +pub enum Spacing { + /// The token cannot join with the following token to form a compound + /// token. + /// + /// In token streams parsed from source code, the compiler will use `Alone` + /// for any token immediately followed by whitespace, a non-doc comment, or + /// EOF. + /// + /// When constructing token streams within the compiler, use this for each + /// token that (a) should be pretty-printed with a space after it, or (b) + /// is the last token in the stream. (In the latter case the choice of + /// spacing doesn't matter because it is never used for the last token. We + /// arbitrarily use `Alone`.) + /// + /// Converts to `proc_macro::Spacing::Alone`, and + /// `proc_macro::Spacing::Alone` converts back to this. + Alone = 0, + + /// The token can join with the following token to form a compound token. + /// + /// In token streams parsed from source code, the compiler will use `Joint` + /// for any token immediately followed by punctuation (as determined by + /// `Token::is_punct`). + /// + /// When constructing token streams within the compiler, use this for each + /// token that (a) should be pretty-printed without a space after it, and + /// (b) is followed by a punctuation token. + /// + /// Converts to `proc_macro::Spacing::Joint`, and + /// `proc_macro::Spacing::Joint` converts back to this. + Joint = 1, + + /// The token can join with the following token to form a compound token, + /// but this will not be visible at the proc macro level. (This is what the + /// `Hidden` means; see below.) + /// + /// In token streams parsed from source code, the compiler will use + /// `JointHidden` for any token immediately followed by anything not + /// covered by the `Alone` and `Joint` cases: an identifier, lifetime, + /// literal, delimiter, doc comment. + /// + /// When constructing token streams, use this for each token that (a) + /// should be pretty-printed without a space after it, and (b) is followed + /// by a non-punctuation token. + /// + /// Converts to `proc_macro::Spacing::Alone`, but + /// `proc_macro::Spacing::Alone` converts back to `token::Spacing::Alone`. + /// Because of that, pretty-printing of `TokenStream`s produced by proc + /// macros is unavoidably uglier (with more whitespace between tokens) than + /// pretty-printing of `TokenStream`'s produced by other means (i.e. parsed + /// source code, internally constructed token streams, and token streams + /// produced by declarative macros). + JointHidden = 2, +} + +/// Identifier or keyword. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Ident { + pub sym: Symbol, + pub span: Span, + pub is_raw: IdentIsRaw, +} + +impl fmt::Display for Leaf { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Leaf::Ident(it) => fmt::Display::fmt(it, f), + Leaf::Literal(it) => fmt::Display::fmt(it, f), + Leaf::Punct(it) => fmt::Display::fmt(it, f), + } + } +} + +impl fmt::Display for Ident { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.is_raw.as_str(), f)?; + fmt::Display::fmt(&self.sym, f) + } +} + +impl fmt::Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (text, suffix) = self.text_and_suffix(); + match self.kind { + LitKind::Byte => write!(f, "b'{}'", text), + LitKind::Char => write!(f, "'{}'", text), + LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", text), + LitKind::Str => write!(f, "\"{}\"", text), + LitKind::ByteStr => write!(f, "b\"{}\"", text), + LitKind::CStr => write!(f, "c\"{}\"", text), + LitKind::StrRaw(num_of_hashes) => { + let num_of_hashes = num_of_hashes as usize; + write!(f, r#"r{0:# { + let num_of_hashes = num_of_hashes as usize; + write!(f, r#"br{0:# { + let num_of_hashes = num_of_hashes as usize; + write!(f, r#"cr{0:# fmt::Display for Punct { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.char, f) + } +} + +impl Leaf { + pub fn print_debug(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Leaf::Literal(lit) => { + let (text, suffix) = lit.text_and_suffix(); + write!(f, "LITERAL {:?} {}{} {:#?}", lit.kind, text, suffix, lit.span)?; + } + Leaf::Punct(punct) => { + write!( + f, + "PUNCT {} [{}] {:#?}", + punct.char, + if punct.spacing == Spacing::Alone { "alone" } else { "joint" }, + punct.span + )?; + } + Leaf::Ident(ident) => { + write!(f, "IDENT {}{} {:#?}", ident.is_raw.as_str(), ident.sym, ident.span)?; + } + } + + Ok(()) + } +} + +pub fn literal_from_lexer( + text: &str, + span: Span, + kind: rustc_lexer::LiteralKind, + suffix_start: u32, +) -> Literal { + use rustc_lexer::LiteralKind; + + let (kind, start_offset, end_offset) = match kind { + LiteralKind::Int { .. } => (LitKind::Integer, 0, 0), + LiteralKind::Float { .. } => (LitKind::Float, 0, 0), + LiteralKind::Char { terminated } => (LitKind::Char, 1, terminated as usize), + LiteralKind::Byte { terminated } => (LitKind::Byte, 2, terminated as usize), + LiteralKind::Str { terminated } => (LitKind::Str, 1, terminated as usize), + LiteralKind::ByteStr { terminated } => (LitKind::ByteStr, 2, terminated as usize), + LiteralKind::CStr { terminated } => (LitKind::CStr, 2, terminated as usize), + LiteralKind::RawStr { n_hashes } => ( + LitKind::StrRaw(n_hashes.unwrap_or_default()), + 2 + n_hashes.unwrap_or_default() as usize, + 1 + n_hashes.unwrap_or_default() as usize, + ), + LiteralKind::RawByteStr { n_hashes } => ( + LitKind::ByteStrRaw(n_hashes.unwrap_or_default()), + 3 + n_hashes.unwrap_or_default() as usize, + 1 + n_hashes.unwrap_or_default() as usize, + ), + LiteralKind::RawCStr { n_hashes } => ( + LitKind::CStrRaw(n_hashes.unwrap_or_default()), + 3 + n_hashes.unwrap_or_default() as usize, + 1 + n_hashes.unwrap_or_default() as usize, + ), + }; + + let (lit, suffix) = text.split_at(suffix_start as usize); + let lit = &lit[start_offset..lit.len() - end_offset]; + let suffix = match suffix { + "" | "_" => "", + // ill-suffixed literals + _ if !matches!(kind, LitKind::Integer | LitKind::Float | LitKind::Err(_)) => { + return Literal::new_no_suffix(text, span, LitKind::Err(())); + } + suffix => suffix, + }; + + Literal::new(lit, span, kind, suffix) +} + +pub fn literal_from_str(text: &str, span: Span) -> Result, ()> { + use rustc_lexer::{LiteralKind, Token, TokenKind}; + + let mut tokens = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No); + let minus_or_lit = tokens.next().unwrap_or(Token { kind: TokenKind::Eof, len: 0 }); + + let lit = if minus_or_lit.kind == TokenKind::Minus { + let lit = tokens.next().ok_or(())?; + if !matches!( + lit.kind, + TokenKind::Literal { kind: LiteralKind::Int { .. } | LiteralKind::Float { .. }, .. } + ) { + return Err(()); + } + lit + } else { + minus_or_lit + }; + + if tokens.next().is_some() { + return Err(()); + } + + let TokenKind::Literal { kind, suffix_start } = lit.kind else { return Err(()) }; + Ok(literal_from_lexer(text, span, kind, suffix_start)) +} + +pub fn literal_from_str_or_err(text: &str, span: Span) -> Literal { + literal_from_str(text, span) + .unwrap_or_else(|_| Literal::new_no_suffix(text, span, LitKind::Err(()))) +} diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs index 2bc2b64cd4fd..d41f9a9c58f7 100644 --- a/crates/tt/src/lib.rs +++ b/crates/tt/src/lib.rs @@ -8,812 +8,481 @@ #[cfg(feature = "in-rust-tree")] extern crate rustc_driver as _; -#[cfg(not(feature = "in-rust-tree"))] -extern crate ra_ap_rustc_lexer as rustc_lexer; -#[cfg(feature = "in-rust-tree")] -extern crate rustc_lexer; +stdx::rustc_crates! { + extern crate rustc_lexer or ra_ap_rustc_lexer; +} +#[cfg(feature = "in-ra")] pub mod buffer; +#[cfg(feature = "in-ra")] pub mod iter; +mod leaf_types; +#[cfg(feature = "in-ra")] mod storage; -use std::fmt; +#[cfg(feature = "in-ra")] +pub use self::in_ra::*; +pub use self::leaf_types::*; -use arrayvec::ArrayString; -use buffer::Cursor; -use intern::Symbol; -use stdx::{impl_from, itertools::Itertools as _}; +#[cfg(feature = "in-ra")] +mod in_ra { + use std::fmt; -pub use span::Span; -pub use text_size::{TextRange, TextSize}; + use intern::Symbol; + use stdx::impl_from; -use crate::storage::TokenTreesSlice; + pub use span::Span; + pub use text_size::{TextRange, TextSize}; -pub use self::iter::{TtElement, TtIter}; -pub use self::storage::{TopSubtree, TopSubtreeBuilder}; + use crate::{leaf_types::*, storage::TokenTreesSlice}; -pub const MAX_GLUED_PUNCT_LEN: usize = 3; + pub use crate::{ + buffer::Cursor, + iter::{TtElement, TtIter}, + storage::{TopSubtree, TopSubtreeBuilder}, + }; -#[derive(Clone, PartialEq, Debug)] -pub struct Lit { - pub kind: LitKind, - pub symbol: Symbol, - pub suffix: Option, -} + pub const MAX_GLUED_PUNCT_LEN: usize = 3; -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -#[repr(u8)] -// The discriminants are important for `storage.rs` decoding. -pub enum IdentIsRaw { - No = 0, - Yes = 1, -} -impl IdentIsRaw { - pub fn yes(self) -> bool { - matches!(self, IdentIsRaw::Yes) - } - pub fn no(&self) -> bool { - matches!(self, IdentIsRaw::No) + #[derive(Debug, Clone, PartialEq, Eq, Hash)] + pub enum TokenTree { + Leaf(Leaf), + Subtree(Subtree), } - pub fn as_str(self) -> &'static str { - match self { - IdentIsRaw::No => "", - IdentIsRaw::Yes => "r#", - } - } - pub fn split_from_symbol(sym: &str) -> (Self, &str) { - if let Some(sym) = sym.strip_prefix("r#") { - (IdentIsRaw::Yes, sym) - } else { - (IdentIsRaw::No, sym) + impl_from!(Leaf, Subtree for TokenTree); + impl TokenTree { + pub fn first_span(&self) -> Span { + match self { + TokenTree::Leaf(l) => *l.span(), + TokenTree::Subtree(s) => s.delimiter.open, + } } } -} -#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] -pub enum LitKind { - Byte, - Char, - Integer, // e.g. `1`, `1u8`, `1f32` - Float, // e.g. `1.`, `1.0`, `1e3f32` - Str, - StrRaw(u8), // raw string delimited by `n` hash symbols - ByteStr, - ByteStrRaw(u8), // raw byte string delimited by `n` hash symbols - CStr, - CStrRaw(u8), - Err(()), -} + impl Leaf { + pub fn span(&self) -> &Span { + match self { + Leaf::Literal(it) => &it.span, + Leaf::Punct(it) => &it.span, + Leaf::Ident(it) => &it.span, + } + } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum TokenTree { - Leaf(Leaf), - Subtree(Subtree), -} -impl_from!(Leaf, Subtree for TokenTree); -impl TokenTree { - pub fn first_span(&self) -> Span { - match self { - TokenTree::Leaf(l) => *l.span(), - TokenTree::Subtree(s) => s.delimiter.open, + pub(crate) fn symbol(&self) -> Option<&Symbol> { + match self { + Leaf::Literal(Literal { text_and_suffix: symbol, .. }) + | Leaf::Ident(Ident { sym: symbol, .. }) => Some(symbol), + Leaf::Punct(_) => None, + } } } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum Leaf { - Literal(Literal), - Punct(Punct), - Ident(Ident), -} + impl_from!(Literal, Punct, Ident for Leaf); -impl Leaf { - pub fn span(&self) -> &Span { - match self { - Leaf::Literal(it) => &it.span, - Leaf::Punct(it) => &it.span, - Leaf::Ident(it) => &it.span, - } + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub struct Subtree { + pub delimiter: Delimiter, + /// Number of following token trees that belong to this subtree, excluding this subtree. + pub len: u32, } - fn symbol(&self) -> Option<&Symbol> { - match self { - Leaf::Literal(Literal { text_and_suffix: symbol, .. }) - | Leaf::Ident(Ident { sym: symbol, .. }) => Some(symbol), - Leaf::Punct(_) => None, + impl Subtree { + pub fn usize_len(&self) -> usize { + self.len as usize } } -} -impl_from!(Literal, Punct, Ident for Leaf); -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct Subtree { - pub delimiter: Delimiter, - /// Number of following token trees that belong to this subtree, excluding this subtree. - pub len: u32, -} - -impl Subtree { - pub fn usize_len(&self) -> usize { - self.len as usize + #[derive(Clone, Copy)] + pub struct TokenTreesView<'a> { + pub(crate) slice: TokenTreesSlice<'a>, + pub(crate) len: usize, } -} -#[derive(Clone, Copy)] -pub struct TokenTreesView<'a> { - slice: TokenTreesSlice<'a>, - len: usize, -} - -impl<'a> TokenTreesView<'a> { - #[inline] - pub fn empty() -> Self { - Self { slice: TokenTreesSlice::empty(), len: 0 } - } + impl<'a> TokenTreesView<'a> { + #[inline] + pub fn empty() -> Self { + Self { slice: TokenTreesSlice::empty(), len: 0 } + } - pub fn iter(&self) -> TtIter<'a> { - TtIter::new(*self) - } + pub fn iter(&self) -> TtIter<'a> { + TtIter::new(*self) + } - pub fn cursor(&self) -> Cursor<'a> { - Cursor::new(*self) - } + pub fn cursor(&self) -> Cursor<'a> { + Cursor::new(*self) + } - pub fn len(&self) -> usize { - self.len - } + pub fn len(&self) -> usize { + self.len + } - pub fn is_empty(&self) -> bool { - self.len() == 0 - } + pub fn is_empty(&self) -> bool { + self.len() == 0 + } - pub fn try_into_subtree(self) -> Option> { - let is_subtree = self.iter_flat_tokens().next().is_some_and( + pub fn try_into_subtree(self) -> Option> { + let is_subtree = self.iter_flat_tokens().next().is_some_and( |it| matches!(it, TokenTree::Subtree(subtree) if subtree.usize_len() == self.len - 1), ); - if is_subtree { Some(SubtreeView(self)) } else { None } - } + if is_subtree { Some(SubtreeView(self)) } else { None } + } - pub fn strip_invisible(self) -> TokenTreesView<'a> { - self.try_into_subtree().map(|subtree| subtree.strip_invisible()).unwrap_or(self) - } + pub fn strip_invisible(self) -> TokenTreesView<'a> { + self.try_into_subtree().map(|subtree| subtree.strip_invisible()).unwrap_or(self) + } - pub fn split( - self, - mut split_fn: impl FnMut(TtElement<'a>) -> bool, - ) -> impl Iterator> { - let mut subtree_iter = self.iter(); - let mut need_to_yield_even_if_empty = true; + pub fn split( + self, + mut split_fn: impl FnMut(TtElement<'a>) -> bool, + ) -> impl Iterator> { + let mut subtree_iter = self.iter(); + let mut need_to_yield_even_if_empty = true; - std::iter::from_fn(move || { - if subtree_iter.is_empty() && !need_to_yield_even_if_empty { - return None; - }; + std::iter::from_fn(move || { + if subtree_iter.is_empty() && !need_to_yield_even_if_empty { + return None; + }; - need_to_yield_even_if_empty = false; - let savepoint = subtree_iter.savepoint(); - let mut result = subtree_iter.from_savepoint(savepoint); - while let Some(tt) = subtree_iter.next() { - if split_fn(tt) { - need_to_yield_even_if_empty = true; - break; + need_to_yield_even_if_empty = false; + let savepoint = subtree_iter.savepoint(); + let mut result = subtree_iter.from_savepoint(savepoint); + while let Some(tt) = subtree_iter.next() { + if split_fn(tt) { + need_to_yield_even_if_empty = true; + break; + } + result = subtree_iter.from_savepoint(savepoint); } - result = subtree_iter.from_savepoint(savepoint); - } - Some(result) - }) - } + Some(result) + }) + } - pub fn first_span(&self) -> Option { - self.iter_flat_tokens().next().map(|it| it.first_span()) - } + pub fn first_span(&self) -> Option { + self.iter_flat_tokens().next().map(|it| it.first_span()) + } - /// Note: this is quite expensive, this needs to decode the whole view, - /// although it "tricks" by skipping subtrees (since we know their byte length). - pub fn last_span(&self) -> Option { - let mut iter = self.iter(); - loop { - match iter.last()? { - TtElement::Leaf(leaf) => return Some(*leaf.span()), - TtElement::Subtree(subtree, tt_iter) => { - if subtree.len == 0 { - return Some(subtree.delimiter.close); - } else { - iter = tt_iter; + /// Note: this is quite expensive, this needs to decode the whole view, + /// although it "tricks" by skipping subtrees (since we know their byte length). + pub fn last_span(&self) -> Option { + let mut iter = self.iter(); + loop { + match iter.last()? { + TtElement::Leaf(leaf) => return Some(*leaf.span()), + TtElement::Subtree(subtree, tt_iter) => { + if subtree.len == 0 { + return Some(subtree.delimiter.close); + } else { + iter = tt_iter; + } } } } } - } - pub fn iter_flat_tokens(&self) -> impl Iterator + use<'a> { - self.slice.iter().take(self.len) + pub fn iter_flat_tokens(&self) -> impl Iterator + use<'a> { + self.slice.iter().take(self.len) + } } -} -impl fmt::Debug for TokenTreesView<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut iter = self.iter(); - while let Some(tt) = iter.next() { - print_debug_token(f, 0, tt)?; - if !iter.is_empty() { - writeln!(f)?; + impl fmt::Debug for TokenTreesView<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut iter = self.iter(); + while let Some(tt) = iter.next() { + print_debug_token(f, 0, tt)?; + if !iter.is_empty() { + writeln!(f)?; + } } + Ok(()) } - Ok(()) } -} -impl fmt::Display for TokenTreesView<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - return token_trees_display(f, self.iter()); - - fn subtree_display( - subtree: &Subtree, - f: &mut fmt::Formatter<'_>, - iter: TtIter<'_>, - ) -> fmt::Result { - let (l, r) = match subtree.delimiter.kind { - DelimiterKind::Parenthesis => ("(", ")"), - DelimiterKind::Brace => ("{", "}"), - DelimiterKind::Bracket => ("[", "]"), - DelimiterKind::Invisible => ("", ""), - }; - f.write_str(l)?; - token_trees_display(f, iter)?; - f.write_str(r)?; - Ok(()) - } + impl fmt::Display for TokenTreesView<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + return token_trees_display(f, self.iter()); - fn token_trees_display(f: &mut fmt::Formatter<'_>, iter: TtIter<'_>) -> fmt::Result { - let mut needs_space = false; - for child in iter { - if needs_space { - f.write_str(" ")?; - } - needs_space = true; + fn subtree_display( + subtree: &Subtree, + f: &mut fmt::Formatter<'_>, + iter: TtIter<'_>, + ) -> fmt::Result { + let (l, r) = match subtree.delimiter.kind { + DelimiterKind::Parenthesis => ("(", ")"), + DelimiterKind::Brace => ("{", "}"), + DelimiterKind::Bracket => ("[", "]"), + DelimiterKind::Invisible => ("", ""), + }; + f.write_str(l)?; + token_trees_display(f, iter)?; + f.write_str(r)?; + Ok(()) + } - match child { - TtElement::Leaf(Leaf::Punct(p)) => { - needs_space = p.spacing == Spacing::Alone; - fmt::Display::fmt(&p, f)?; + fn token_trees_display(f: &mut fmt::Formatter<'_>, iter: TtIter<'_>) -> fmt::Result { + let mut needs_space = false; + for child in iter { + if needs_space { + f.write_str(" ")?; } - TtElement::Leaf(leaf) => fmt::Display::fmt(&leaf, f)?, - TtElement::Subtree(subtree, subtree_iter) => { - subtree_display(&subtree, f, subtree_iter)? + needs_space = true; + + match child { + TtElement::Leaf(Leaf::Punct(p)) => { + needs_space = p.spacing == Spacing::Alone; + fmt::Display::fmt(&p, f)?; + } + TtElement::Leaf(leaf) => fmt::Display::fmt(&leaf, f)?, + TtElement::Subtree(subtree, subtree_iter) => { + subtree_display(&subtree, f, subtree_iter)? + } } } + Ok(()) } - Ok(()) } } -} - -#[derive(Clone, Copy)] -// Invariant: always starts with `Subtree` that covers the entire thing. -pub struct SubtreeView<'a>(TokenTreesView<'a>); - -impl<'a> SubtreeView<'a> { - pub fn as_token_trees(self) -> TokenTreesView<'a> { - self.0 - } - pub fn iter(&self) -> TtIter<'a> { - self.token_trees().iter() - } - - pub fn top_subtree(&self) -> Subtree { - let Some(TokenTree::Subtree(subtree)) = self.0.iter_flat_tokens().next() else { - unreachable!("the first token tree is always the top subtree"); - }; - subtree - } + #[derive(Clone, Copy)] + // Invariant: always starts with `Subtree` that covers the entire thing. + pub struct SubtreeView<'a>(pub(crate) TokenTreesView<'a>); - pub fn strip_invisible(&self) -> TokenTreesView<'a> { - if self.top_subtree().delimiter.kind == DelimiterKind::Invisible { - self.token_trees() - } else { + impl<'a> SubtreeView<'a> { + pub fn as_token_trees(self) -> TokenTreesView<'a> { self.0 } - } - - pub fn token_trees(&self) -> TokenTreesView<'a> { - let mut result = self.0; - result.slice.advance(); - result.len -= 1; - result - } -} - -impl fmt::Debug for SubtreeView<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&self.0, f) - } -} -impl fmt::Display for SubtreeView<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.0, f) - } -} - -#[derive(Debug, Copy, Clone, PartialEq)] -pub struct DelimSpan { - pub open: Span, - pub close: Span, -} + pub fn iter(&self) -> TtIter<'a> { + self.token_trees().iter() + } -impl DelimSpan { - pub fn from_single(sp: Span) -> Self { - DelimSpan { open: sp, close: sp } - } + pub fn top_subtree(&self) -> Subtree { + let Some(TokenTree::Subtree(subtree)) = self.0.iter_flat_tokens().next() else { + unreachable!("the first token tree is always the top subtree"); + }; + subtree + } - pub fn from_pair(open: Span, close: Span) -> Self { - DelimSpan { open, close } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct Delimiter { - pub open: Span, - pub close: Span, - pub kind: DelimiterKind, -} + pub fn strip_invisible(&self) -> TokenTreesView<'a> { + if self.top_subtree().delimiter.kind == DelimiterKind::Invisible { + self.token_trees() + } else { + self.0 + } + } -impl Delimiter { - pub const fn invisible_spanned(span: Span) -> Self { - Delimiter { open: span, close: span, kind: DelimiterKind::Invisible } + pub fn token_trees(&self) -> TokenTreesView<'a> { + let mut result = self.0; + result.slice.advance(); + result.len -= 1; + result + } } - pub const fn invisible_delim_spanned(span: DelimSpan) -> Self { - Delimiter { open: span.open, close: span.close, kind: DelimiterKind::Invisible } + impl fmt::Debug for SubtreeView<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.0, f) + } } - pub fn delim_span(&self) -> DelimSpan { - DelimSpan { open: self.open, close: self.close } + impl fmt::Display for SubtreeView<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -#[repr(u8)] -// The discriminants are important for decoding for `storage.rs`. -pub enum DelimiterKind { - Parenthesis = 0, - Brace = 1, - Bracket = 2, - Invisible = 3, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Literal { - /// Escaped, text then suffix concatenated. - pub text_and_suffix: Symbol, - pub span: Span, - pub kind: LitKind, - pub suffix_len: u8, -} -impl Literal { - #[inline] - pub fn text_and_suffix(&self) -> (&str, &str) { - let text_and_suffix = self.text_and_suffix.as_str(); - text_and_suffix.split_at(text_and_suffix.len() - usize::from(self.suffix_len)) - } + impl DelimSpan { + pub fn from_single(sp: Span) -> Self { + DelimSpan { open: sp, close: sp } + } - #[inline] - pub fn text(&self) -> &str { - self.text_and_suffix().0 + pub fn from_pair(open: Span, close: Span) -> Self { + DelimSpan { open, close } + } } - #[inline] - pub fn suffix(&self) -> &str { - self.text_and_suffix().1 - } + impl Delimiter { + pub const fn invisible_spanned(span: Span) -> Self { + Delimiter { open: span, close: span, kind: DelimiterKind::Invisible } + } - pub fn new(text: &str, span: Span, kind: LitKind, suffix: &str) -> Self { - const MAX_INLINE_CAPACITY: usize = 30; - let text_and_suffix = if suffix.is_empty() { - Symbol::intern(text) - } else if (text.len() + suffix.len()) < MAX_INLINE_CAPACITY { - let mut text_and_suffix = ArrayString::::new(); - text_and_suffix.push_str(text); - text_and_suffix.push_str(suffix); - Symbol::intern(&text_and_suffix) - } else { - let mut text_and_suffix = String::with_capacity(text.len() + suffix.len()); - text_and_suffix.push_str(text); - text_and_suffix.push_str(suffix); - Symbol::intern(&text_and_suffix) - }; - - Self { text_and_suffix, span, kind, suffix_len: suffix.len().try_into().unwrap() } - } + pub const fn invisible_delim_spanned(span: DelimSpan) -> Self { + Delimiter { open: span.open, close: span.close, kind: DelimiterKind::Invisible } + } - #[inline] - pub fn new_no_suffix(text: &str, span: Span, kind: LitKind) -> Self { - Self { text_and_suffix: Symbol::intern(text), span, kind, suffix_len: 0 } + pub fn delim_span(&self) -> DelimSpan { + DelimSpan { open: self.open, close: self.close } + } } -} - -pub fn token_to_literal(text: &str, span: Span) -> Literal { - use rustc_lexer::LiteralKind; - - let token = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No).next_tuple(); - let Some((rustc_lexer::Token { - kind: rustc_lexer::TokenKind::Literal { kind, suffix_start }, - .. - },)) = token - else { - return Literal::new_no_suffix(text, span, LitKind::Err(())); - }; - - let (kind, start_offset, end_offset) = match kind { - LiteralKind::Int { .. } => (LitKind::Integer, 0, 0), - LiteralKind::Float { .. } => (LitKind::Float, 0, 0), - LiteralKind::Char { terminated } => (LitKind::Char, 1, terminated as usize), - LiteralKind::Byte { terminated } => (LitKind::Byte, 2, terminated as usize), - LiteralKind::Str { terminated } => (LitKind::Str, 1, terminated as usize), - LiteralKind::ByteStr { terminated } => (LitKind::ByteStr, 2, terminated as usize), - LiteralKind::CStr { terminated } => (LitKind::CStr, 2, terminated as usize), - LiteralKind::RawStr { n_hashes } => ( - LitKind::StrRaw(n_hashes.unwrap_or_default()), - 2 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - LiteralKind::RawByteStr { n_hashes } => ( - LitKind::ByteStrRaw(n_hashes.unwrap_or_default()), - 3 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - LiteralKind::RawCStr { n_hashes } => ( - LitKind::CStrRaw(n_hashes.unwrap_or_default()), - 3 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - }; - let (lit, suffix) = text.split_at(suffix_start as usize); - let lit = &lit[start_offset..lit.len() - end_offset]; - let suffix = match suffix { - "" | "_" => "", - // ill-suffixed literals - _ if !matches!(kind, LitKind::Integer | LitKind::Float | LitKind::Err(_)) => { - return Literal::new_no_suffix(text, span, LitKind::Err(())); + impl Ident { + pub fn new(text: &str, span: Span) -> Self { + // let raw_stripped = IdentIsRaw::split_from_symbol(text.as_ref()); + let (is_raw, text) = IdentIsRaw::split_from_symbol(text); + Ident { sym: Symbol::intern(text), span, is_raw } } - suffix => suffix, - }; - - Literal::new(lit, span, kind, suffix) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct Punct { - pub char: char, - pub spacing: Spacing, - pub span: Span, -} - -/// Indicates whether a token can join with the following token to form a -/// compound token. Used for conversions to `proc_macro::Spacing`. Also used to -/// guide pretty-printing, which is where the `JointHidden` value (which isn't -/// part of `proc_macro::Spacing`) comes in useful. -// The discriminants are important for decoding for `storage.rs`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -pub enum Spacing { - /// The token cannot join with the following token to form a compound - /// token. - /// - /// In token streams parsed from source code, the compiler will use `Alone` - /// for any token immediately followed by whitespace, a non-doc comment, or - /// EOF. - /// - /// When constructing token streams within the compiler, use this for each - /// token that (a) should be pretty-printed with a space after it, or (b) - /// is the last token in the stream. (In the latter case the choice of - /// spacing doesn't matter because it is never used for the last token. We - /// arbitrarily use `Alone`.) - /// - /// Converts to `proc_macro::Spacing::Alone`, and - /// `proc_macro::Spacing::Alone` converts back to this. - Alone = 0, - - /// The token can join with the following token to form a compound token. - /// - /// In token streams parsed from source code, the compiler will use `Joint` - /// for any token immediately followed by punctuation (as determined by - /// `Token::is_punct`). - /// - /// When constructing token streams within the compiler, use this for each - /// token that (a) should be pretty-printed without a space after it, and - /// (b) is followed by a punctuation token. - /// - /// Converts to `proc_macro::Spacing::Joint`, and - /// `proc_macro::Spacing::Joint` converts back to this. - Joint = 1, - - /// The token can join with the following token to form a compound token, - /// but this will not be visible at the proc macro level. (This is what the - /// `Hidden` means; see below.) - /// - /// In token streams parsed from source code, the compiler will use - /// `JointHidden` for any token immediately followed by anything not - /// covered by the `Alone` and `Joint` cases: an identifier, lifetime, - /// literal, delimiter, doc comment. - /// - /// When constructing token streams, use this for each token that (a) - /// should be pretty-printed without a space after it, and (b) is followed - /// by a non-punctuation token. - /// - /// Converts to `proc_macro::Spacing::Alone`, but - /// `proc_macro::Spacing::Alone` converts back to `token::Spacing::Alone`. - /// Because of that, pretty-printing of `TokenStream`s produced by proc - /// macros is unavoidably uglier (with more whitespace between tokens) than - /// pretty-printing of `TokenStream`'s produced by other means (i.e. parsed - /// source code, internally constructed token streams, and token streams - /// produced by declarative macros). - JointHidden = 2, -} - -/// Identifier or keyword. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Ident { - pub sym: Symbol, - pub span: Span, - pub is_raw: IdentIsRaw, -} - -impl Ident { - pub fn new(text: &str, span: Span) -> Self { - // let raw_stripped = IdentIsRaw::split_from_symbol(text.as_ref()); - let (is_raw, text) = IdentIsRaw::split_from_symbol(text); - Ident { sym: Symbol::intern(text), span, is_raw } } -} -fn print_debug_subtree( - f: &mut fmt::Formatter<'_>, - subtree: &Subtree, - level: usize, - iter: TtIter<'_>, -) -> fmt::Result { - let align = " ".repeat(level); - - let Delimiter { kind, open, close } = &subtree.delimiter; - let delim = match kind { - DelimiterKind::Invisible => "$$", - DelimiterKind::Parenthesis => "()", - DelimiterKind::Brace => "{}", - DelimiterKind::Bracket => "[]", - }; + fn print_debug_subtree( + f: &mut fmt::Formatter<'_>, + subtree: &Subtree, + level: usize, + iter: TtIter<'_>, + ) -> fmt::Result { + let Delimiter { kind, open, close } = &subtree.delimiter; + let delim = kind.debug_view(); + + write!(f, "SUBTREE {delim} ",)?; + write!(f, "{open:#?}")?; + write!(f, " ")?; + write!(f, "{close:#?}")?; + for child in iter { + writeln!(f)?; + print_debug_token(f, level + 1, child)?; + } - write!(f, "{align}SUBTREE {delim} ",)?; - write!(f, "{open:#?}")?; - write!(f, " ")?; - write!(f, "{close:#?}")?; - for child in iter { - writeln!(f)?; - print_debug_token(f, level + 1, child)?; + Ok(()) } - Ok(()) -} - -fn print_debug_token(f: &mut fmt::Formatter<'_>, level: usize, tt: TtElement<'_>) -> fmt::Result { - let align = " ".repeat(level); + fn print_debug_token( + f: &mut fmt::Formatter<'_>, + level: usize, + tt: TtElement<'_>, + ) -> fmt::Result { + write!(f, "{:indent$}", "", indent = level * 2)?; - match tt { - TtElement::Leaf(leaf) => match leaf { - Leaf::Literal(lit) => { - let (text, suffix) = lit.text_and_suffix(); - write!(f, "{}LITERAL {:?} {}{} {:#?}", align, lit.kind, text, suffix, lit.span)?; - } - Leaf::Punct(punct) => { - write!( - f, - "{}PUNCH {} [{}] {:#?}", - align, - punct.char, - if punct.spacing == Spacing::Alone { "alone" } else { "joint" }, - punct.span - )?; - } - Leaf::Ident(ident) => { - write!( - f, - "{}IDENT {}{} {:#?}", - align, - ident.is_raw.as_str(), - ident.sym, - ident.span - )?; + match tt { + TtElement::Leaf(leaf) => leaf.print_debug(f), + TtElement::Subtree(subtree, subtree_iter) => { + print_debug_subtree(f, &subtree, level, subtree_iter) } - }, - TtElement::Subtree(subtree, subtree_iter) => { - print_debug_subtree(f, &subtree, level, subtree_iter)?; } } - Ok(()) -} - -impl fmt::Debug for TopSubtree { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&self.view(), f) - } -} - -impl fmt::Display for TopSubtree { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.view(), f) - } -} - -impl fmt::Display for Leaf { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Leaf::Ident(it) => fmt::Display::fmt(it, f), - Leaf::Literal(it) => fmt::Display::fmt(it, f), - Leaf::Punct(it) => fmt::Display::fmt(it, f), + impl fmt::Debug for TopSubtree { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.view(), f) } } -} - -impl fmt::Display for Ident { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.is_raw.as_str(), f)?; - fmt::Display::fmt(&self.sym, f) - } -} -impl fmt::Display for Literal { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let (text, suffix) = self.text_and_suffix(); - match self.kind { - LitKind::Byte => write!(f, "b'{}'", text), - LitKind::Char => write!(f, "'{}'", text), - LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", text), - LitKind::Str => write!(f, "\"{}\"", text), - LitKind::ByteStr => write!(f, "b\"{}\"", text), - LitKind::CStr => write!(f, "c\"{}\"", text), - LitKind::StrRaw(num_of_hashes) => { - let num_of_hashes = num_of_hashes as usize; - write!(f, r#"r{0:# { - let num_of_hashes = num_of_hashes as usize; - write!(f, r#"br{0:# { - let num_of_hashes = num_of_hashes as usize; - write!(f, r#"cr{0:#) -> fmt::Result { - fmt::Display::fmt(&self.char, f) + impl fmt::Display for TopSubtree { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.view(), f) + } } -} -impl Subtree { - /// Count the number of tokens recursively - pub fn count(&self) -> usize { - self.usize_len() + impl Subtree { + /// Count the number of tokens recursively + pub fn count(&self) -> usize { + self.usize_len() + } } -} -pub fn pretty(tkns: TokenTreesView<'_>) -> String { - return pretty_impl(tkns.iter()); + pub fn pretty(tkns: TokenTreesView<'_>) -> String { + return pretty_impl(tkns.iter()); - fn tokentree_to_text(tkn: TtElement<'_>) -> String { - match tkn { - TtElement::Leaf(leaf) => { - format!("{}", leaf) - } - TtElement::Subtree(Subtree { delimiter, .. }, subtree_content) => { - let content = pretty_impl(subtree_content); - let (open, close) = match delimiter.kind { - DelimiterKind::Brace => ("{", "}"), - DelimiterKind::Bracket => ("[", "]"), - DelimiterKind::Parenthesis => ("(", ")"), - DelimiterKind::Invisible => ("", ""), - }; - format!("{open}{content}{close}") + fn tokentree_to_text(tkn: TtElement<'_>) -> String { + match tkn { + TtElement::Leaf(leaf) => { + format!("{}", leaf) + } + TtElement::Subtree(Subtree { delimiter, .. }, subtree_content) => { + let content = pretty_impl(subtree_content); + let (open, close) = delimiter.kind.display_open_close(); + format!("{open}{content}{close}") + } } } - } - fn pretty_impl(tkns: TtIter<'_>) -> String { - let mut last = String::new(); - let mut last_to_joint = true; - - for tkn in tkns { - last = - [last, tokentree_to_text(tkn.clone())].join(if last_to_joint { "" } else { " " }); - last_to_joint = false; - if let TtElement::Leaf(Leaf::Punct(Punct { spacing, .. })) = tkn - && spacing == Spacing::Joint - { - last_to_joint = true; + fn pretty_impl(tkns: TtIter<'_>) -> String { + let mut last = String::new(); + let mut last_to_joint = true; + + for tkn in tkns { + last = [last, tokentree_to_text(tkn.clone())].join(if last_to_joint { + "" + } else { + " " + }); + last_to_joint = false; + if let TtElement::Leaf(Leaf::Punct(Punct { spacing, .. })) = tkn + && spacing == Spacing::Joint + { + last_to_joint = true; + } } + last } - last } -} -#[derive(Debug)] -pub enum TransformTtAction<'a> { - Keep, - ReplaceWith(TokenTreesView<'a>), -} - -impl TransformTtAction<'_> { - #[inline] - pub fn remove() -> Self { - Self::ReplaceWith(TokenTreesView::empty()) + #[derive(Debug)] + pub enum TransformTtAction<'a> { + Keep, + ReplaceWith(TokenTreesView<'a>), } -} -/// This function takes a token tree, and calls `callback` with each token tree in it. -/// Then it does what the callback says: keeps the tt or replaces it with a (possibly empty) -/// tts view. -pub fn transform_tt<'b>( - tt: &mut TopSubtree, - mut callback: impl FnMut(&TokenTree) -> TransformTtAction<'b>, -) { - let mut tt_vec = tt.as_token_trees().iter_flat_tokens().collect::>(); - - // We need to keep a stack of the currently open subtrees, because we need to update - // them if we change the number of items in them. - let mut subtrees_stack = Vec::new(); - let mut i = 0; - while i < tt_vec.len() { - 'pop_finished_subtrees: while let Some(&subtree_idx) = subtrees_stack.last() { - let TokenTree::Subtree(subtree) = &tt_vec[subtree_idx] else { - unreachable!("non-subtree on subtrees stack"); - }; - if i >= subtree_idx + 1 + subtree.usize_len() { - subtrees_stack.pop(); - } else { - break 'pop_finished_subtrees; - } + impl TransformTtAction<'_> { + #[inline] + pub fn remove() -> Self { + Self::ReplaceWith(TokenTreesView::empty()) } + } - let current = &tt_vec[i]; - let action = callback(current); - match action { - TransformTtAction::Keep => { - // This cannot be shared with the replaced case, because then we may push the same subtree - // twice, and will update it twice which will lead to errors. - if let TokenTree::Subtree(_) = current { - subtrees_stack.push(i); + /// This function takes a token tree, and calls `callback` with each token tree in it. + /// Then it does what the callback says: keeps the tt or replaces it with a (possibly empty) + /// tts view. + pub fn transform_tt<'b>( + tt: &mut TopSubtree, + mut callback: impl FnMut(&TokenTree) -> TransformTtAction<'b>, + ) { + let mut tt_vec = tt.as_token_trees().iter_flat_tokens().collect::>(); + + // We need to keep a stack of the currently open subtrees, because we need to update + // them if we change the number of items in them. + let mut subtrees_stack = Vec::new(); + let mut i = 0; + while i < tt_vec.len() { + 'pop_finished_subtrees: while let Some(&subtree_idx) = subtrees_stack.last() { + let TokenTree::Subtree(subtree) = &tt_vec[subtree_idx] else { + unreachable!("non-subtree on subtrees stack"); + }; + if i >= subtree_idx + 1 + subtree.usize_len() { + subtrees_stack.pop(); + } else { + break 'pop_finished_subtrees; } - - i += 1; } - TransformTtAction::ReplaceWith(replacement) => { - let old_len = 1 + match current { - TokenTree::Leaf(_) => 0, - TokenTree::Subtree(subtree) => subtree.usize_len(), - }; - let len_diff = replacement.len() as i64 - old_len as i64; - tt_vec.splice(i..i + old_len, replacement.iter_flat_tokens()); - // Skip the newly inserted replacement, we don't want to visit it. - i += replacement.len(); - - for &subtree_idx in &subtrees_stack { - let TokenTree::Subtree(subtree) = &mut tt_vec[subtree_idx] else { - unreachable!("non-subtree on subtrees stack"); + + let current = &tt_vec[i]; + let action = callback(current); + match action { + TransformTtAction::Keep => { + // This cannot be shared with the replaced case, because then we may push the same subtree + // twice, and will update it twice which will lead to errors. + if let TokenTree::Subtree(_) = current { + subtrees_stack.push(i); + } + + i += 1; + } + TransformTtAction::ReplaceWith(replacement) => { + let old_len = 1 + match current { + TokenTree::Leaf(_) => 0, + TokenTree::Subtree(subtree) => subtree.usize_len(), }; - subtree.len = (i64::from(subtree.len) + len_diff).try_into().unwrap(); + let len_diff = replacement.len() as i64 - old_len as i64; + tt_vec.splice(i..i + old_len, replacement.iter_flat_tokens()); + // Skip the newly inserted replacement, we don't want to visit it. + i += replacement.len(); + + for &subtree_idx in &subtrees_stack { + let TokenTree::Subtree(subtree) = &mut tt_vec[subtree_idx] else { + unreachable!("non-subtree on subtrees stack"); + }; + subtree.len = (i64::from(subtree.len) + len_diff).try_into().unwrap(); + } } } } - } - *tt = TopSubtree::from_serialized(tt_vec); + *tt = TopSubtree::from_serialized(tt_vec); + } }