diff --git a/Cargo.lock b/Cargo.lock index 5835ed9e552e..148e049be81e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1348,9 +1348,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libmimalloc-sys" @@ -1867,7 +1867,8 @@ dependencies = [ "intern", "paths", "postcard", - "proc-macro-srv", + "proc-macro-api", + "ra-ap-rustc_lexer", "rayon", "rustc-hash 2.1.2", "semver", @@ -1888,9 +1889,11 @@ dependencies = [ "intern", "line-index 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "paths", + "proc-macro-api", "proc-macro-test", "span", "stdx", + "tt", ] [[package]] @@ -2374,9 +2377,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" [[package]] name = "rustc-hash" diff --git a/crates/hir-def/Cargo.toml b/crates/hir-def/Cargo.toml index eb1e774b8f41..9319bb409143 100644 --- a/crates/hir-def/Cargo.toml +++ b/crates/hir-def/Cargo.toml @@ -56,3 +56,6 @@ in-rust-tree = ["hir-expand/in-rust-tree"] [lints] workspace = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_parse_format"] diff --git a/crates/hir-def/src/item_tree/attrs.rs b/crates/hir-def/src/item_tree/attrs.rs index 4ee2f56be19f..e69aa2a23075 100644 --- a/crates/hir-def/src/item_tree/attrs.rs +++ b/crates/hir-def/src/item_tree/attrs.rs @@ -21,7 +21,7 @@ use hir_expand::{ use intern::{Interned, Symbol, sym}; use syntax::{AstNode, ast}; use syntax_bridge::DocCommentDesugarMode; -use tt::token_to_literal; +use tt::literal_from_str; use crate::item_tree::lower::Ctx; @@ -64,11 +64,13 @@ impl AttrsOrCfg { ast::Meta::KeyValueMeta(meta) => { let span = span_map.span_for(path_range); let input = meta.expr().and_then(|value| { - if let ast::Expr::Literal(value) = value { - Some(Box::new(AttrInput::Literal(token_to_literal( + if let ast::Expr::Literal(value) = value + && let Ok(lit) = literal_from_str( value.token().text(), span_map.span_for(value.syntax().text_range()), - )))) + ) + { + Some(Box::new(AttrInput::Literal(lit))) } else { None } diff --git a/crates/hir-def/src/lib.rs b/crates/hir-def/src/lib.rs index 0712a025b49c..70b69597f9ca 100644 --- a/crates/hir-def/src/lib.rs +++ b/crates/hir-def/src/lib.rs @@ -9,11 +9,9 @@ #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] -#[cfg(feature = "in-rust-tree")] -extern crate rustc_parse_format; - -#[cfg(not(feature = "in-rust-tree"))] -extern crate ra_ap_rustc_parse_format as rustc_parse_format; +stdx::rustc_crates! { + extern crate rustc_parse_format or ra_ap_rustc_parse_format; +} pub extern crate ra_ap_rustc_abi as layout; pub extern crate ra_ap_rustc_abi as rustc_abi; diff --git a/crates/load-cargo/src/lib.rs b/crates/load-cargo/src/lib.rs index bf2331a40f9b..db2f6c5443f6 100644 --- a/crates/load-cargo/src/lib.rs +++ b/crates/load-cargo/src/lib.rs @@ -24,8 +24,8 @@ use ide_db::{ }; use itertools::Itertools; use proc_macro_api::{ - MacroDylib, ProcMacroClient, bidirectional_protocol::msg::{ParentSpan, SubRequest, SubResponse}, + client::{MacroDylib, ProcMacroClient}, }; use project_model::{CargoConfig, PackageRoot, ProjectManifest, ProjectWorkspace}; use span::{Span, SpanAnchor, SyntaxContext}; @@ -558,7 +558,7 @@ fn load_crate_graph_into_db( } fn expander_to_proc_macro( - expander: proc_macro_api::ProcMacro, + expander: proc_macro_api::client::ProcMacro, ignored_macros: &[Box], ) -> ProcMacro { let name = expander.name(); @@ -577,7 +577,7 @@ fn expander_to_proc_macro( } #[derive(Debug, PartialEq, Eq)] -struct Expander(proc_macro_api::ProcMacro); +struct Expander(proc_macro_api::client::ProcMacro); impl ProcMacroExpander for Expander { fn expand( diff --git a/crates/mbe/Cargo.toml b/crates/mbe/Cargo.toml index b6e55b9360a3..b2f587bf91fc 100644 --- a/crates/mbe/Cargo.toml +++ b/crates/mbe/Cargo.toml @@ -39,3 +39,6 @@ in-rust-tree = ["parser/in-rust-tree", "tt/in-rust-tree"] [lints] workspace = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_lexer"] diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index 6de9b4275ce2..13b71b5afdef 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -8,10 +8,9 @@ #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] -#[cfg(not(feature = "in-rust-tree"))] -extern crate ra_ap_rustc_lexer as rustc_lexer; -#[cfg(feature = "in-rust-tree")] -extern crate rustc_lexer; +stdx::rustc_crates! { + extern crate rustc_lexer or ra_ap_rustc_lexer; +} mod expander; mod macro_call_style; diff --git a/crates/mbe/src/tests.rs b/crates/mbe/src/tests.rs index 9e93d1869d04..fd382141290c 100644 --- a/crates/mbe/src/tests.rs +++ b/crates/mbe/src/tests.rs @@ -138,20 +138,20 @@ struct MyTraitMap2 IDENT MyTraitMap2 1:Root[0000, 0]@8..19#ROOT2024 SUBTREE {} 0:Root[0000, 0]@48..49#ROOT2024 0:Root[0000, 0]@100..101#ROOT2024 IDENT map 0:Root[0000, 0]@58..61#ROOT2024 - PUNCH : [alone] 0:Root[0000, 0]@61..62#ROOT2024 - PUNCH : [joint] 0:Root[0000, 0]@63..64#ROOT2024 - PUNCH : [alone] 0:Root[0000, 0]@64..65#ROOT2024 + PUNCT : [alone] 0:Root[0000, 0]@61..62#ROOT2024 + PUNCT : [joint] 0:Root[0000, 0]@63..64#ROOT2024 + PUNCT : [alone] 0:Root[0000, 0]@64..65#ROOT2024 IDENT std 0:Root[0000, 0]@65..68#ROOT2024 - PUNCH : [joint] 0:Root[0000, 0]@68..69#ROOT2024 - PUNCH : [alone] 0:Root[0000, 0]@69..70#ROOT2024 + PUNCT : [joint] 0:Root[0000, 0]@68..69#ROOT2024 + PUNCT : [alone] 0:Root[0000, 0]@69..70#ROOT2024 IDENT collections 0:Root[0000, 0]@70..81#ROOT2024 - PUNCH : [joint] 0:Root[0000, 0]@81..82#ROOT2024 - PUNCH : [alone] 0:Root[0000, 0]@82..83#ROOT2024 + PUNCT : [joint] 0:Root[0000, 0]@81..82#ROOT2024 + PUNCT : [alone] 0:Root[0000, 0]@82..83#ROOT2024 IDENT HashSet 0:Root[0000, 0]@83..90#ROOT2024 - PUNCH < [alone] 0:Root[0000, 0]@90..91#ROOT2024 + PUNCT < [alone] 0:Root[0000, 0]@90..91#ROOT2024 SUBTREE () 0:Root[0000, 0]@91..92#ROOT2024 0:Root[0000, 0]@92..93#ROOT2024 - PUNCH > [joint] 0:Root[0000, 0]@93..94#ROOT2024 - PUNCH , [alone] 0:Root[0000, 0]@94..95#ROOT2024 + PUNCT > [joint] 0:Root[0000, 0]@93..94#ROOT2024 + PUNCT , [alone] 0:Root[0000, 0]@94..95#ROOT2024 struct MyTraitMap2 { map: ::std::collections::HashSet<()>, @@ -186,22 +186,22 @@ fn main() { SUBTREE () 1:Root[0000, 0]@8..9#ROOT2024 1:Root[0000, 0]@9..10#ROOT2024 SUBTREE {} 1:Root[0000, 0]@11..12#ROOT2024 1:Root[0000, 0]@61..62#ROOT2024 LITERAL Integer 1 1:Root[0000, 0]@17..18#ROOT2024 - PUNCH ; [alone] 1:Root[0000, 0]@18..19#ROOT2024 + PUNCT ; [alone] 1:Root[0000, 0]@18..19#ROOT2024 LITERAL Float 1.0 1:Root[0000, 0]@24..27#ROOT2024 - PUNCH ; [alone] 1:Root[0000, 0]@27..28#ROOT2024 + PUNCT ; [alone] 1:Root[0000, 0]@27..28#ROOT2024 SUBTREE () 1:Root[0000, 0]@33..34#ROOT2024 1:Root[0000, 0]@39..40#ROOT2024 SUBTREE () 1:Root[0000, 0]@34..35#ROOT2024 1:Root[0000, 0]@37..38#ROOT2024 LITERAL Integer 1 1:Root[0000, 0]@35..36#ROOT2024 - PUNCH , [alone] 1:Root[0000, 0]@36..37#ROOT2024 - PUNCH , [alone] 1:Root[0000, 0]@38..39#ROOT2024 - PUNCH . [alone] 1:Root[0000, 0]@40..41#ROOT2024 + PUNCT , [alone] 1:Root[0000, 0]@36..37#ROOT2024 + PUNCT , [alone] 1:Root[0000, 0]@38..39#ROOT2024 + PUNCT . [alone] 1:Root[0000, 0]@40..41#ROOT2024 LITERAL Float 0.0 1:Root[0000, 0]@41..44#ROOT2024 - PUNCH ; [alone] 1:Root[0000, 0]@44..45#ROOT2024 + PUNCT ; [alone] 1:Root[0000, 0]@44..45#ROOT2024 IDENT let 1:Root[0000, 0]@50..53#ROOT2024 IDENT x 1:Root[0000, 0]@54..55#ROOT2024 - PUNCH = [alone] 1:Root[0000, 0]@56..57#ROOT2024 + PUNCT = [alone] 1:Root[0000, 0]@56..57#ROOT2024 LITERAL Integer 1 1:Root[0000, 0]@58..59#ROOT2024 - PUNCH ; [alone] 1:Root[0000, 0]@59..60#ROOT2024 + PUNCT ; [alone] 1:Root[0000, 0]@59..60#ROOT2024 fn main(){ 1; @@ -229,12 +229,12 @@ fn expr_2021() { expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..25#ROOT2024 1:Root[0000, 0]@0..25#ROOT2024 IDENT _ 1:Root[0000, 0]@5..6#ROOT2024 - PUNCH ; [joint] 0:Root[0000, 0]@36..37#ROOT2024 + PUNCT ; [joint] 0:Root[0000, 0]@36..37#ROOT2024 SUBTREE () 0:Root[0000, 0]@34..35#ROOT2024 0:Root[0000, 0]@34..35#ROOT2024 IDENT const 1:Root[0000, 0]@12..17#ROOT2024 SUBTREE {} 1:Root[0000, 0]@18..19#ROOT2024 1:Root[0000, 0]@22..23#ROOT2024 LITERAL Integer 1 1:Root[0000, 0]@20..21#ROOT2024 - PUNCH ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 + PUNCT ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 _; (const { @@ -261,7 +261,7 @@ fn expr_2021() { } SUBTREE $$ 1:Root[0000, 0]@0..8#ROOT2024 1:Root[0000, 0]@0..8#ROOT2024 - PUNCH ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 + PUNCT ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 ;"#]], ); @@ -285,7 +285,7 @@ fn expr_2021() { } SUBTREE $$ 1:Root[0000, 0]@0..18#ROOT2024 1:Root[0000, 0]@0..18#ROOT2024 - PUNCH ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 + PUNCT ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 ;"#]], ); @@ -307,24 +307,24 @@ fn expr_2021() { expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..76#ROOT2024 1:Root[0000, 0]@0..76#ROOT2024 LITERAL Integer 4 1:Root[0000, 0]@5..6#ROOT2024 - PUNCH ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 + PUNCT ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 LITERAL Str literal 1:Root[0000, 0]@12..21#ROOT2024 - PUNCH ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 + PUNCT ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 SUBTREE () 0:Root[0000, 0]@39..40#ROOT2024 0:Root[0000, 0]@39..40#ROOT2024 IDENT funcall 1:Root[0000, 0]@27..34#ROOT2024 SUBTREE () 1:Root[0000, 0]@34..35#ROOT2024 1:Root[0000, 0]@35..36#ROOT2024 - PUNCH ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 + PUNCT ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 SUBTREE () 0:Root[0000, 0]@39..40#ROOT2024 0:Root[0000, 0]@39..40#ROOT2024 IDENT future 1:Root[0000, 0]@42..48#ROOT2024 - PUNCH . [alone] 1:Root[0000, 0]@48..49#ROOT2024 + PUNCT . [alone] 1:Root[0000, 0]@48..49#ROOT2024 IDENT await 1:Root[0000, 0]@49..54#ROOT2024 - PUNCH ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 + PUNCT ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 SUBTREE () 0:Root[0000, 0]@39..40#ROOT2024 0:Root[0000, 0]@39..40#ROOT2024 IDENT break 1:Root[0000, 0]@60..65#ROOT2024 - PUNCH ' [joint] 1:Root[0000, 0]@66..67#ROOT2024 + PUNCT ' [joint] 1:Root[0000, 0]@66..67#ROOT2024 IDENT foo 1:Root[0000, 0]@67..70#ROOT2024 IDENT bar 1:Root[0000, 0]@71..74#ROOT2024 - PUNCH ; [alone] 0:Root[0000, 0]@44..45#ROOT2024 + PUNCT ; [alone] 0:Root[0000, 0]@44..45#ROOT2024 4; "literal"; @@ -352,7 +352,7 @@ fn expr_2021() { } SUBTREE $$ 1:Root[0000, 0]@0..8#ROOT2024 1:Root[0000, 0]@0..8#ROOT2024 - PUNCH ; [alone] 0:Root[0000, 0]@44..45#ROOT2024 + PUNCT ; [alone] 0:Root[0000, 0]@44..45#ROOT2024 ;"#]], ); @@ -371,7 +371,7 @@ fn minus_belongs_to_literal() { "-1", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..2#ROOT2024 1:Root[0000, 0]@0..2#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@10..11#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@10..11#ROOT2024 LITERAL Integer 1 0:Root[0000, 0]@11..12#ROOT2024 -1"#]], @@ -380,7 +380,7 @@ fn minus_belongs_to_literal() { "- 1", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..3#ROOT2024 1:Root[0000, 0]@0..3#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@10..11#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@10..11#ROOT2024 LITERAL Integer 1 0:Root[0000, 0]@11..12#ROOT2024 -1"#]], @@ -389,7 +389,7 @@ fn minus_belongs_to_literal() { "-2", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..2#ROOT2024 1:Root[0000, 0]@0..2#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@25..26#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@25..26#ROOT2024 LITERAL Integer 2 0:Root[0000, 0]@27..28#ROOT2024 -2"#]], @@ -398,7 +398,7 @@ fn minus_belongs_to_literal() { "- 2", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..3#ROOT2024 1:Root[0000, 0]@0..3#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@25..26#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@25..26#ROOT2024 LITERAL Integer 2 0:Root[0000, 0]@27..28#ROOT2024 -2"#]], @@ -407,7 +407,7 @@ fn minus_belongs_to_literal() { "-3.0", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..4#ROOT2024 1:Root[0000, 0]@0..4#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@43..44#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@43..44#ROOT2024 LITERAL Float 3.0 0:Root[0000, 0]@45..48#ROOT2024 -3.0"#]], @@ -416,7 +416,7 @@ fn minus_belongs_to_literal() { "- 3.0", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..5#ROOT2024 1:Root[0000, 0]@0..5#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@43..44#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@43..44#ROOT2024 LITERAL Float 3.0 0:Root[0000, 0]@45..48#ROOT2024 -3.0"#]], @@ -433,7 +433,7 @@ fn minus_belongs_to_literal() { "@-1", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..3#ROOT2024 1:Root[0000, 0]@0..3#ROOT2024 - PUNCH - [alone] 1:Root[0000, 0]@1..2#ROOT2024 + PUNCT - [alone] 1:Root[0000, 0]@1..2#ROOT2024 LITERAL Integer 1 1:Root[0000, 0]@2..3#ROOT2024 -1"#]], @@ -450,7 +450,7 @@ fn minus_belongs_to_literal() { "@-1.0", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..5#ROOT2024 1:Root[0000, 0]@0..5#ROOT2024 - PUNCH - [alone] 1:Root[0000, 0]@1..2#ROOT2024 + PUNCT - [alone] 1:Root[0000, 0]@1..2#ROOT2024 LITERAL Float 1.0 1:Root[0000, 0]@2..5#ROOT2024 -1.0"#]], diff --git a/crates/parser/Cargo.toml b/crates/parser/Cargo.toml index 2bdf8d76fbc6..8710f2c51228 100644 --- a/crates/parser/Cargo.toml +++ b/crates/parser/Cargo.toml @@ -19,6 +19,7 @@ rustc-literal-escaper.workspace = true tracing.workspace = true edition.workspace = true +stdx.workspace = true winnow = { version = "0.7.13", default-features = false } [dev-dependencies] @@ -32,3 +33,6 @@ in-rust-tree = [] [lints] workspace = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_lexer"] diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index 5900d7cfeda9..ba9b3d39a8f5 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -21,12 +21,12 @@ #![allow(rustdoc::private_intra_doc_links)] #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] -#[cfg(not(feature = "in-rust-tree"))] -extern crate ra_ap_rustc_lexer as rustc_lexer; #[cfg(feature = "in-rust-tree")] extern crate rustc_driver as _; -#[cfg(feature = "in-rust-tree")] -extern crate rustc_lexer; + +stdx::rustc_crates! { + extern crate rustc_lexer or ra_ap_rustc_lexer; +} mod event; mod frontmatter; diff --git a/crates/proc-macro-api/Cargo.toml b/crates/proc-macro-api/Cargo.toml index 7342e0ecdcf6..e88f43178868 100644 --- a/crates/proc-macro-api/Cargo.toml +++ b/crates/proc-macro-api/Cargo.toml @@ -22,20 +22,36 @@ indexmap.workspace = true # local deps paths = { workspace = true, features = ["serde1"] } -tt.workspace = true +tt = { path = "../tt", version = "0.0.0", default-features = false } stdx.workspace = true -proc-macro-srv = {workspace = true, optional = true} # span = {workspace = true, default-features = false} does not work -span = { path = "../span", version = "0.0.0", default-features = false} +span = { path = "../span", version = "0.0.0", default-features = false } + +ra-ap-rustc_lexer.workspace = true intern.workspace = true postcard.workspace = true semver.workspace = true rayon.workspace = true +[dev-dependencies] +# Enable both features for test. +proc-macro-api = { path = "../proc-macro-api", features = [ + "in-ra", + "in-proc-macro-srv", +] } + [features] -in-rust-tree = ["proc-macro-srv", "proc-macro-srv/in-rust-tree"] -default = [] +default = ["in-ra"] +in-rust-tree = ["tt/in-rust-tree"] +in-ra = ["tt/in-ra"] +in-proc-macro-srv = [] [lints] workspace = true + +[package.metadata.rust-analyzer] +rustc_private = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_lexer"] diff --git a/crates/proc-macro-api/src/bidirectional_protocol.rs b/crates/proc-macro-api/src/bidirectional_protocol.rs index f070b1c9a334..c475ebfbbe43 100644 --- a/crates/proc-macro-api/src/bidirectional_protocol.rs +++ b/crates/proc-macro-api/src/bidirectional_protocol.rs @@ -1,214 +1,8 @@ //! Bidirectional protocol methods -use std::{ - io::{self, BufRead, Write}, - panic::{AssertUnwindSafe, catch_unwind}, - sync::Arc, -}; - -use paths::AbsPath; -use span::Span; - -use crate::{ - ProcMacro, ProcMacroKind, ServerError, - bidirectional_protocol::msg::{ - ApiVersionCheck, BidirectionalMessage, ExpandMacro, ExpandMacroData, ExpnGlobals, - ListMacros, Request, Response, SubRequest, SubResponse, - }, - legacy_protocol::{ - SpanMode, - msg::{ - FlatTree, ServerConfig, SpanDataIndexMap, deserialize_span_data_index_map, - serialize_span_data_index_map, - }, - }, - process::ProcMacroServerProcess, - transport::postcard, -}; - pub mod msg; +#[cfg(feature = "in-ra")] +mod sender; -pub type SubCallback<'a> = &'a dyn Fn(SubRequest) -> Result; - -pub fn run_conversation( - writer: &mut dyn Write, - reader: &mut dyn BufRead, - buf: &mut Vec, - msg: BidirectionalMessage, - callback: SubCallback<'_>, -) -> Result { - let encoded = postcard::encode(&msg).map_err(wrap_encode)?; - postcard::write(writer, &encoded).map_err(wrap_io("failed to write initial request"))?; - - loop { - let maybe_buf = postcard::read(reader, buf).map_err(wrap_io("failed to read message"))?; - let Some(b) = maybe_buf else { - return Err(ServerError { - message: "proc-macro server closed the stream".into(), - io: Some(Arc::new(io::Error::new(io::ErrorKind::UnexpectedEof, "closed"))), - }); - }; - - let msg: BidirectionalMessage = postcard::decode(b).map_err(wrap_decode)?; - - match msg { - BidirectionalMessage::Response(response) => { - return Ok(BidirectionalMessage::Response(response)); - } - BidirectionalMessage::SubRequest(sr) => { - // TODO: Avoid `AssertUnwindSafe` by making the callback `UnwindSafe` once `SourceDatabase` - // becomes unwind-safe (currently blocked by `parking_lot::RwLock` in the VFS). - let resp = match catch_unwind(AssertUnwindSafe(|| callback(sr))) { - Ok(Ok(resp)) => BidirectionalMessage::SubResponse(resp), - Ok(Err(err)) => BidirectionalMessage::SubResponse(SubResponse::Cancel { - reason: err.to_string(), - }), - Err(_) => BidirectionalMessage::SubResponse(SubResponse::Cancel { - reason: "callback panicked or was cancelled".into(), - }), - }; - - let encoded = postcard::encode(&resp).map_err(wrap_encode)?; - postcard::write(writer, &encoded) - .map_err(wrap_io("failed to write sub-response"))?; - } - _ => { - return Err(ServerError { - message: format!("unexpected message {:?}", msg), - io: None, - }); - } - } - } -} - -fn wrap_io(msg: &'static str) -> impl Fn(io::Error) -> ServerError { - move |err| ServerError { message: msg.into(), io: Some(Arc::new(err)) } -} - -fn wrap_encode(err: io::Error) -> ServerError { - ServerError { message: "failed to encode message".into(), io: Some(Arc::new(err)) } -} - -fn wrap_decode(err: io::Error) -> ServerError { - ServerError { message: "failed to decode message".into(), io: Some(Arc::new(err)) } -} - -pub(crate) fn version_check( - srv: &ProcMacroServerProcess, - callback: SubCallback<'_>, -) -> Result { - let request = BidirectionalMessage::Request(Request::ApiVersionCheck(ApiVersionCheck {})); - - let response_payload = run_request(srv, request, callback)?; - - match response_payload { - BidirectionalMessage::Response(Response::ApiVersionCheck(version)) => Ok(version), - other => { - Err(ServerError { message: format!("unexpected response: {:?}", other), io: None }) - } - } -} - -/// Enable support for rust-analyzer span mode if the server supports it. -pub(crate) fn enable_rust_analyzer_spans( - srv: &ProcMacroServerProcess, - callback: SubCallback<'_>, -) -> Result { - let request = BidirectionalMessage::Request(Request::SetConfig(ServerConfig { - span_mode: SpanMode::RustAnalyzer, - })); - - let response_payload = run_request(srv, request, callback)?; - - match response_payload { - BidirectionalMessage::Response(Response::SetConfig(ServerConfig { span_mode })) => { - Ok(span_mode) - } - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -/// Finds proc-macros in a given dynamic library. -pub(crate) fn find_proc_macros( - srv: &ProcMacroServerProcess, - dylib_path: &AbsPath, - callback: SubCallback<'_>, -) -> Result, String>, ServerError> { - let request = BidirectionalMessage::Request(Request::ListMacros(ListMacros { - dylib_path: dylib_path.to_path_buf().into(), - })); - - let response_payload = run_request(srv, request, callback)?; - - match response_payload { - BidirectionalMessage::Response(Response::ListMacros(it)) => Ok(it), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -pub(crate) fn expand( - proc_macro: &ProcMacro, - process: &ProcMacroServerProcess, - subtree: tt::SubtreeView<'_>, - attr: Option>, - env: Vec<(String, String)>, - def_site: Span, - call_site: Span, - mixed_site: Span, - current_dir: String, - callback: SubCallback<'_>, -) -> Result, crate::ServerError> { - let version = process.version(); - let mut span_data_table = SpanDataIndexMap::default(); - let def_site = span_data_table.insert_full(def_site).0; - let call_site = span_data_table.insert_full(call_site).0; - let mixed_site = span_data_table.insert_full(mixed_site).0; - let task = BidirectionalMessage::Request(Request::ExpandMacro(Box::new(ExpandMacro { - data: ExpandMacroData { - macro_body: FlatTree::from_subtree(subtree, version, &mut span_data_table), - macro_name: proc_macro.name.to_string(), - attributes: attr - .map(|subtree| FlatTree::from_subtree(subtree, version, &mut span_data_table)), - has_global_spans: ExpnGlobals { def_site, call_site, mixed_site }, - span_data_table: if process.rust_analyzer_spans() { - serialize_span_data_index_map(&span_data_table) - } else { - Vec::new() - }, - }, - lib: proc_macro.dylib_path.to_path_buf().into(), - env, - current_dir: Some(current_dir), - }))); - - let response_payload = run_request(process, task, callback)?; - - match response_payload { - BidirectionalMessage::Response(Response::ExpandMacro(it)) => Ok(it - .map(|resp| { - FlatTree::to_subtree_resolved( - resp.tree, - version, - &deserialize_span_data_index_map(&resp.span_data_table), - ) - }) - .map_err(|msg| msg.0)), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -fn run_request( - srv: &ProcMacroServerProcess, - msg: BidirectionalMessage, - callback: SubCallback<'_>, -) -> Result { - if let Some(err) = srv.exited() { - return Err(err.clone()); - } - srv.run_bidirectional(msg, callback) -} - -pub fn reject_subrequests(req: SubRequest) -> Result { - Err(ServerError { message: format!("{req:?} sub-request not supported here"), io: None }) -} +#[cfg(feature = "in-ra")] +pub use self::sender::*; diff --git a/crates/proc-macro-api/src/bidirectional_protocol/msg.rs b/crates/proc-macro-api/src/bidirectional_protocol/msg.rs index e516297f0619..5094ba0b0453 100644 --- a/crates/proc-macro-api/src/bidirectional_protocol/msg.rs +++ b/crates/proc-macro-api/src/bidirectional_protocol/msg.rs @@ -12,7 +12,8 @@ use serde::{Deserialize, Serialize}; use crate::{ ProcMacroKind, - legacy_protocol::msg::{FlatTree, Message, PanicMessage, ServerConfig}, + flat::FlatTree, + legacy_protocol::msg::{Message, PanicMessage, ServerConfig}, transport::postcard, }; diff --git a/crates/proc-macro-api/src/bidirectional_protocol/sender.rs b/crates/proc-macro-api/src/bidirectional_protocol/sender.rs new file mode 100644 index 000000000000..0bf571b3bfb2 --- /dev/null +++ b/crates/proc-macro-api/src/bidirectional_protocol/sender.rs @@ -0,0 +1,210 @@ +//! Functions for the sender side, i.e. the rust-analyzer side. + +use std::{ + io::{self, BufRead, Write}, + panic::{AssertUnwindSafe, catch_unwind}, + sync::Arc, +}; + +use paths::AbsPath; +use span::Span; + +use crate::{ + ProcMacroKind, + bidirectional_protocol::msg::{ + ApiVersionCheck, BidirectionalMessage, ExpandMacro, ExpandMacroData, ExpnGlobals, + ListMacros, Request, Response, SubRequest, SubResponse, + }, + client::{ProcMacro, ServerError}, + flat::{ + FlatTree, SpanDataIndexMap, deserialize_span_data_index_map, serialize_span_data_index_map, + }, + legacy_protocol::msg::{ServerConfig, SpanMode}, + process::ProcMacroServerProcess, + transport::postcard, +}; + +pub type SubCallback<'a> = &'a dyn Fn(SubRequest) -> Result; + +pub fn run_conversation( + writer: &mut dyn Write, + reader: &mut dyn BufRead, + buf: &mut Vec, + msg: BidirectionalMessage, + callback: SubCallback<'_>, +) -> Result { + let encoded = postcard::encode(&msg).map_err(wrap_encode)?; + postcard::write(writer, &encoded).map_err(wrap_io("failed to write initial request"))?; + + loop { + let maybe_buf = postcard::read(reader, buf).map_err(wrap_io("failed to read message"))?; + let Some(b) = maybe_buf else { + return Err(ServerError { + message: "proc-macro server closed the stream".into(), + io: Some(Arc::new(io::Error::new(io::ErrorKind::UnexpectedEof, "closed"))), + }); + }; + + let msg: BidirectionalMessage = postcard::decode(b).map_err(wrap_decode)?; + + match msg { + BidirectionalMessage::Response(response) => { + return Ok(BidirectionalMessage::Response(response)); + } + BidirectionalMessage::SubRequest(sr) => { + // TODO: Avoid `AssertUnwindSafe` by making the callback `UnwindSafe` once `SourceDatabase` + // becomes unwind-safe (currently blocked by `parking_lot::RwLock` in the VFS). + let resp = match catch_unwind(AssertUnwindSafe(|| callback(sr))) { + Ok(Ok(resp)) => BidirectionalMessage::SubResponse(resp), + Ok(Err(err)) => BidirectionalMessage::SubResponse(SubResponse::Cancel { + reason: err.to_string(), + }), + Err(_) => BidirectionalMessage::SubResponse(SubResponse::Cancel { + reason: "callback panicked or was cancelled".into(), + }), + }; + + let encoded = postcard::encode(&resp).map_err(wrap_encode)?; + postcard::write(writer, &encoded) + .map_err(wrap_io("failed to write sub-response"))?; + } + _ => { + return Err(ServerError { + message: format!("unexpected message {:?}", msg), + io: None, + }); + } + } + } +} + +fn wrap_io(msg: &'static str) -> impl Fn(io::Error) -> ServerError { + move |err| ServerError { message: msg.into(), io: Some(Arc::new(err)) } +} + +fn wrap_encode(err: io::Error) -> ServerError { + ServerError { message: "failed to encode message".into(), io: Some(Arc::new(err)) } +} + +fn wrap_decode(err: io::Error) -> ServerError { + ServerError { message: "failed to decode message".into(), io: Some(Arc::new(err)) } +} + +pub(crate) fn version_check( + srv: &ProcMacroServerProcess, + callback: SubCallback<'_>, +) -> Result { + let request = BidirectionalMessage::Request(Request::ApiVersionCheck(ApiVersionCheck {})); + + let response_payload = run_request(srv, request, callback)?; + + match response_payload { + BidirectionalMessage::Response(Response::ApiVersionCheck(version)) => Ok(version), + other => { + Err(ServerError { message: format!("unexpected response: {:?}", other), io: None }) + } + } +} + +/// Enable support for rust-analyzer span mode if the server supports it. +pub(crate) fn enable_rust_analyzer_spans( + srv: &ProcMacroServerProcess, + callback: SubCallback<'_>, +) -> Result { + let request = BidirectionalMessage::Request(Request::SetConfig(ServerConfig { + span_mode: SpanMode::RustAnalyzer, + })); + + let response_payload = run_request(srv, request, callback)?; + + match response_payload { + BidirectionalMessage::Response(Response::SetConfig(ServerConfig { span_mode })) => { + Ok(span_mode) + } + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +/// Finds proc-macros in a given dynamic library. +pub(crate) fn find_proc_macros( + srv: &ProcMacroServerProcess, + dylib_path: &AbsPath, + callback: SubCallback<'_>, +) -> Result, String>, ServerError> { + let request = BidirectionalMessage::Request(Request::ListMacros(ListMacros { + dylib_path: dylib_path.to_path_buf().into(), + })); + + let response_payload = run_request(srv, request, callback)?; + + match response_payload { + BidirectionalMessage::Response(Response::ListMacros(it)) => Ok(it), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +pub(crate) fn expand( + proc_macro: &ProcMacro, + process: &ProcMacroServerProcess, + subtree: tt::SubtreeView<'_>, + attr: Option>, + env: Vec<(String, String)>, + def_site: Span, + call_site: Span, + mixed_site: Span, + current_dir: String, + callback: SubCallback<'_>, +) -> Result, ServerError> { + let version = process.version(); + let mut span_data_table = SpanDataIndexMap::default(); + let def_site = span_data_table.insert_full(def_site).0; + let call_site = span_data_table.insert_full(call_site).0; + let mixed_site = span_data_table.insert_full(mixed_site).0; + let task = BidirectionalMessage::Request(Request::ExpandMacro(Box::new(ExpandMacro { + data: ExpandMacroData { + macro_body: FlatTree::from_subtree(subtree, version, &mut span_data_table), + macro_name: proc_macro.name.to_string(), + attributes: attr + .map(|subtree| FlatTree::from_subtree(subtree, version, &mut span_data_table)), + has_global_spans: ExpnGlobals { def_site, call_site, mixed_site }, + span_data_table: if process.rust_analyzer_spans() { + serialize_span_data_index_map(&span_data_table) + } else { + Vec::new() + }, + }, + lib: proc_macro.dylib_path.to_path_buf().into(), + env, + current_dir: Some(current_dir), + }))); + + let response_payload = run_request(process, task, callback)?; + + match response_payload { + BidirectionalMessage::Response(Response::ExpandMacro(it)) => Ok(it + .map(|resp| { + FlatTree::to_subtree( + resp.tree, + version, + &deserialize_span_data_index_map(&resp.span_data_table), + ) + }) + .map_err(|msg| msg.0)), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +fn run_request( + srv: &ProcMacroServerProcess, + msg: BidirectionalMessage, + callback: SubCallback<'_>, +) -> Result { + if let Some(err) = srv.exited() { + return Err(err.clone()); + } + srv.run_bidirectional(msg, callback) +} + +pub fn reject_subrequests(req: SubRequest) -> Result { + Err(ServerError { message: format!("{req:?} sub-request not supported here"), io: None }) +} diff --git a/crates/proc-macro-api/src/client.rs b/crates/proc-macro-api/src/client.rs new file mode 100644 index 000000000000..23638850331f --- /dev/null +++ b/crates/proc-macro-api/src/client.rs @@ -0,0 +1,214 @@ +//! Definitions and operations for the proc macro client operated in rust-analyzer. + +use paths::{AbsPath, AbsPathBuf}; +use semver::Version; +use span::{ErasedFileAstId, FIXUP_ERASED_FILE_AST_ID_MARKER, Span}; +use std::{fmt, io, sync::Arc, time::SystemTime}; + +use crate::{ + ProcMacroKind, ProtocolFormat, + bidirectional_protocol::SubCallback, + pool::ProcMacroServerPool, + process::{self, ProcMacroServerProcess}, + version, +}; + +/// A handle to proc-macro server process pool which load dylibs with macros (.so or .dll) +/// and runs actual macro expansion functions. +#[derive(Debug, Clone)] +pub struct ProcMacroClient { + /// Currently, the proc macro process expands all procedural macros sequentially. + /// + /// That means that concurrent salsa requests may block each other when expanding proc macros, + /// which is unfortunate, but simple and good enough for the time being. + pool: Arc, + /// The path to the proc-macro server binary. + path: AbsPathBuf, +} + +/// Represents a dynamically loaded library containing procedural macros. +pub struct MacroDylib { + pub(crate) path: AbsPathBuf, +} + +impl MacroDylib { + /// Creates a new MacroDylib instance with the given path. + pub fn new(path: AbsPathBuf) -> MacroDylib { + MacroDylib { path } + } +} + +/// A handle to a specific proc-macro (a `#[proc_macro]` annotated function). +/// +/// It exists within the context of a specific proc-macro server -- currently +/// we share a single expander process for all macros within a workspace. +#[derive(Debug, Clone)] +pub struct ProcMacro { + pub(crate) pool: ProcMacroServerPool, + pub(crate) dylib_path: Arc, + pub(crate) name: Box, + pub(crate) kind: ProcMacroKind, + pub(crate) dylib_last_modified: Option, +} + +impl Eq for ProcMacro {} +impl PartialEq for ProcMacro { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + && self.kind == other.kind + && self.dylib_path == other.dylib_path + && self.dylib_last_modified == other.dylib_last_modified + } +} + +/// Represents errors encountered when communicating with the proc-macro server. +#[derive(Clone, Debug)] +pub struct ServerError { + pub message: String, + pub io: Option>, +} + +impl fmt::Display for ServerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.message.fmt(f)?; + if let Some(io) = &self.io { + f.write_str(": ")?; + io.fmt(f)?; + } + Ok(()) + } +} + +impl ProcMacroClient { + /// Spawns an external process as the proc macro server and returns a client connected to it. + pub fn spawn<'a>( + process_path: &AbsPath, + env: impl IntoIterator< + Item = (impl AsRef, &'a Option>), + > + Clone, + version: Option<&Version>, + num_process: usize, + ) -> io::Result { + let pool_size = num_process; + let mut workers = Vec::with_capacity(pool_size); + for _ in 0..pool_size { + let worker = ProcMacroServerProcess::spawn(process_path, env.clone(), version)?; + workers.push(worker); + } + + let pool = ProcMacroServerPool::new(workers); + Ok(ProcMacroClient { pool: Arc::new(pool), path: process_path.to_owned() }) + } + + /// Invokes `spawn` and returns a client connected to the resulting read and write handles. + /// + /// The `process_path` is used for `Self::server_path`. This function is mainly used for testing. + pub fn with_io_channels( + process_path: &AbsPath, + spawn: impl Fn( + Option, + ) -> io::Result<( + Box, + Box, + Box, + )> + Clone, + version: Option<&Version>, + num_process: usize, + ) -> io::Result { + let pool_size = num_process; + let mut workers = Vec::with_capacity(pool_size); + for _ in 0..pool_size { + let worker = + ProcMacroServerProcess::run(spawn.clone(), version, || "".to_owned())?; + workers.push(worker); + } + + let pool = ProcMacroServerPool::new(workers); + Ok(ProcMacroClient { pool: Arc::new(pool), path: process_path.to_owned() }) + } + + /// Returns the absolute path to the proc-macro server. + pub fn server_path(&self) -> &AbsPath { + &self.path + } + + /// Loads a proc-macro dylib into the server process returning a list of `ProcMacro`s loaded. + pub fn load_dylib(&self, dylib: MacroDylib) -> Result, ServerError> { + self.pool.load_dylib(&dylib) + } + + /// Checks if the proc-macro server has exited. + pub fn exited(&self) -> Option<&ServerError> { + self.pool.exited() + } +} + +impl ProcMacro { + /// Returns the name of the procedural macro. + pub fn name(&self) -> &str { + &self.name + } + + /// Returns the type of procedural macro. + pub fn kind(&self) -> ProcMacroKind { + self.kind + } + + pub(crate) fn needs_fixup_change(&self) -> bool { + let version = self.pool.version(); + (version::RUST_ANALYZER_SPAN_SUPPORT..version::HASHED_AST_ID).contains(&version) + } + + /// On some server versions, the fixup ast id is different than ours. So change it to match. + pub(crate) fn change_fixup_to_match_old_server(&self, tt: &mut tt::TopSubtree) { + const OLD_FIXUP_AST_ID: ErasedFileAstId = ErasedFileAstId::from_raw(!0 - 1); + tt.change_every_ast_id(|ast_id| { + if *ast_id == FIXUP_ERASED_FILE_AST_ID_MARKER { + *ast_id = OLD_FIXUP_AST_ID; + } else if *ast_id == OLD_FIXUP_AST_ID { + // Swap between them, that means no collision plus the change can be reversed by doing itself. + *ast_id = FIXUP_ERASED_FILE_AST_ID_MARKER; + } + }); + } + + /// Expands the procedural macro by sending an expansion request to the server. + /// This includes span information and environmental context. + pub fn expand( + &self, + subtree: tt::SubtreeView<'_>, + attr: Option>, + env: Vec<(String, String)>, + def_site: Span, + call_site: Span, + mixed_site: Span, + current_dir: String, + callback: Option>, + ) -> Result, ServerError> { + let (mut subtree, mut attr) = (subtree, attr); + let (mut subtree_changed, mut attr_changed); + if self.needs_fixup_change() { + subtree_changed = tt::TopSubtree::from_subtree(subtree); + self.change_fixup_to_match_old_server(&mut subtree_changed); + subtree = subtree_changed.view(); + + if let Some(attr) = &mut attr { + attr_changed = tt::TopSubtree::from_subtree(*attr); + self.change_fixup_to_match_old_server(&mut attr_changed); + *attr = attr_changed.view(); + } + } + + self.pool.pick_process()?.expand( + self, + subtree, + attr, + env, + def_site, + call_site, + mixed_site, + current_dir, + callback, + ) + } +} diff --git a/crates/proc-macro-api/src/flat.rs b/crates/proc-macro-api/src/flat.rs new file mode 100644 index 000000000000..b73792653641 --- /dev/null +++ b/crates/proc-macro-api/src/flat.rs @@ -0,0 +1,588 @@ +//! Serialization-friendly representation of `tt::TopSubtree`. +//! +//! It is possible to serialize `TopSubtree` recursively, as a tree, but using +//! arbitrary-nested trees in JSON is problematic, as they can cause the JSON +//! parser to overflow the stack. +//! +//! Additionally, such implementation would be pretty verbose, and we do care +//! about performance here a bit. +//! +//! So what this module does is dumping a `tt::TopSubtree` into a bunch of flat +//! array of numbers. +//! +//! ```json +//! { +//! // Array of subtrees, each subtree is represented by 4 numbers: +//! // id of delimiter, delimiter kind, index of first child in `token_tree`, +//! // index of last child in `token_tree` +//! "subtree":[4294967295,0,0,5,2,2,5,5], +//! // 2 ints per literal: [token id, index into `text`] +//! "literal":[4294967295,1], +//! // 3 ints per punct: [token id, char, spacing] +//! "punct":[4294967295,64,1], +//! // 2 ints per ident: [token id, index into `text`] +//! "ident": [0,0,1,1], +//! // children of all subtrees, concatenated. Each child is represented as `index << shift_indices_by | tag` +//! // where tag denotes one of subtree, literal, punct or ident. +//! "token_tree":[3,7,1,4], +//! // Strings shared by idents and literals +//! "text": ["struct","Foo"] +//! } +//! ``` +//! +//! We probably should replace most of the code here with bincode someday, but, +//! as we don't have bincode in Cargo.toml yet, let's stick with serde_json for +//! the time being. + +#[cfg(feature = "in-proc-macro-srv")] +mod proc_macro_srv_side; +#[cfg(feature = "in-ra")] +mod ra_side; + +use std::{borrow::Borrow, collections::VecDeque, marker::PhantomData}; + +use intern::Symbol; +use rustc_hash::FxHashMap; +use serde_derive::{Deserialize, Serialize}; +use span::{EditionedFileId, ErasedFileAstId, Span, SpanAnchor, SyntaxContext, TextRange}; + +use crate::{ + legacy_protocol::SpanId, + version::{ENCODE_CLOSE_SPAN_VERSION, EXTENDED_LEAF_DATA}, +}; + +pub type SpanDataIndexMap = + indexmap::IndexSet>; + +pub fn serialize_span_data_index_map(map: &SpanDataIndexMap) -> Vec { + map.iter() + .map(|span| { + [ + span.anchor.file_id.as_u32(), + span.anchor.ast_id.into_raw(), + span.range.start().into(), + span.range.end().into(), + span.ctx.into_u32(), + ] + }) + .collect::>() + .into_flattened() +} + +pub fn deserialize_span_data_index_map(map: &[u32]) -> SpanDataIndexMap { + let (chunks, remainder) = map.as_chunks(); + assert!(remainder.is_empty()); + chunks + .iter() + .map(|&[file_id, ast_id, start, end, e]| { + Span { + anchor: SpanAnchor { + file_id: EditionedFileId::from_raw(file_id), + ast_id: ErasedFileAstId::from_raw(ast_id), + }, + range: TextRange::new(start.into(), end.into()), + // SAFETY: We only receive spans from the server. If someone mess up the communication UB can happen, + // but that will be their problem. + ctx: unsafe { SyntaxContext::from_u32(e) }, + } + }) + .collect() +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct FlatTree { + subtree: Vec, + literal: Vec, + punct: Vec, + ident: Vec, + token_tree: Vec, + text: Vec, +} + +impl FlatTree { + fn deserialize<'a, ST: SpanTransformer, W: WriterTrait<'a, ST::Span>>( + top_subtree: W::Subtree, + version: u32, + span_data_table: &mut ST::Table, + ) -> FlatTree { + let mut w = Writer:: { + string_table: FxHashMap::default(), + work: VecDeque::new(), + span_data_table, + + subtree: Vec::new(), + literal: Vec::new(), + punct: Vec::new(), + ident: Vec::new(), + token_tree: Vec::new(), + text: Vec::new(), + version, + }; + w.write_subtree(top_subtree); + + FlatTree { + subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { + write_vec(w.subtree, SubtreeRepr::write_with_close_span) + } else { + write_vec(w.subtree, SubtreeRepr::write) + }, + literal: if version >= EXTENDED_LEAF_DATA { + write_vec(w.literal, LiteralRepr::write_with_kind) + } else { + write_vec(w.literal, LiteralRepr::write) + }, + punct: write_vec(w.punct, PunctRepr::write), + ident: if version >= EXTENDED_LEAF_DATA { + write_vec(w.ident, IdentRepr::write_with_rawness) + } else { + write_vec(w.ident, IdentRepr::write) + }, + token_tree: w.token_tree, + text: w.text, + } + } + + fn serialize>( + self, + version: u32, + span_data_table: &ST::Table, + ) -> (tt::Delimiter, Vec) { + Reader:: { + subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { + read_vec(self.subtree, SubtreeRepr::read_with_close_span) + } else { + read_vec(self.subtree, SubtreeRepr::read) + }, + literal: if version >= EXTENDED_LEAF_DATA { + read_vec(self.literal, LiteralRepr::read_with_kind) + } else { + read_vec(self.literal, LiteralRepr::read) + }, + punct: read_vec(self.punct, PunctRepr::read), + ident: if version >= EXTENDED_LEAF_DATA { + read_vec(self.ident, IdentRepr::read_with_rawness) + } else { + read_vec(self.ident, IdentRepr::read) + }, + token_tree: self.token_tree, + text: self.text, + span_data_table, + version, + _marker: PhantomData, + } + .read() + } +} + +#[derive(Debug)] +struct SubtreeRepr { + open: SpanId, + close: SpanId, + kind: tt::DelimiterKind, + tt: [u32; 2], +} + +#[derive(Debug)] +struct LiteralRepr { + id: SpanId, + text: u32, + suffix: u32, + kind: u16, +} + +#[derive(Debug)] +struct PunctRepr { + id: SpanId, + char: char, + spacing: tt::Spacing, +} + +#[derive(Debug)] +struct IdentRepr { + id: SpanId, + text: u32, + is_raw: bool, +} + +fn read_vec T, const N: usize>(xs: Vec, f: F) -> Vec { + let (chunks, remainder) = xs.as_chunks(); + assert!(remainder.is_empty()); + chunks.iter().map(|chunk| f(*chunk)).collect() +} + +fn write_vec [u32; N], const N: usize>(xs: Vec, f: F) -> Vec { + xs.into_iter().map(f).collect::>().into_flattened() +} + +impl SubtreeRepr { + fn write(self) -> [u32; 4] { + let kind = match self.kind { + tt::DelimiterKind::Invisible => 0, + tt::DelimiterKind::Parenthesis => 1, + tt::DelimiterKind::Brace => 2, + tt::DelimiterKind::Bracket => 3, + }; + [self.open.0, kind, self.tt[0], self.tt[1]] + } + fn read([open, kind, lo, len]: [u32; 4]) -> SubtreeRepr { + let kind = match kind { + 0 => tt::DelimiterKind::Invisible, + 1 => tt::DelimiterKind::Parenthesis, + 2 => tt::DelimiterKind::Brace, + 3 => tt::DelimiterKind::Bracket, + other => panic!("bad kind {other}"), + }; + SubtreeRepr { open: SpanId(open), close: SpanId(!0), kind, tt: [lo, len] } + } + fn write_with_close_span(self) -> [u32; 5] { + let kind = match self.kind { + tt::DelimiterKind::Invisible => 0, + tt::DelimiterKind::Parenthesis => 1, + tt::DelimiterKind::Brace => 2, + tt::DelimiterKind::Bracket => 3, + }; + [self.open.0, self.close.0, kind, self.tt[0], self.tt[1]] + } + fn read_with_close_span([open, close, kind, lo, len]: [u32; 5]) -> SubtreeRepr { + let kind = match kind { + 0 => tt::DelimiterKind::Invisible, + 1 => tt::DelimiterKind::Parenthesis, + 2 => tt::DelimiterKind::Brace, + 3 => tt::DelimiterKind::Bracket, + other => panic!("bad kind {other}"), + }; + SubtreeRepr { open: SpanId(open), close: SpanId(close), kind, tt: [lo, len] } + } +} + +impl LiteralRepr { + fn write(self) -> [u32; 2] { + [self.id.0, self.text] + } + fn read([id, text]: [u32; 2]) -> LiteralRepr { + LiteralRepr { id: SpanId(id), text, kind: 0, suffix: !0 } + } + fn write_with_kind(self) -> [u32; 4] { + [self.id.0, self.text, self.kind as u32, self.suffix] + } + fn read_with_kind([id, text, kind, suffix]: [u32; 4]) -> LiteralRepr { + LiteralRepr { id: SpanId(id), text, kind: kind as u16, suffix } + } +} + +impl PunctRepr { + fn write(self) -> [u32; 3] { + let spacing = match self.spacing { + tt::Spacing::Alone | tt::Spacing::JointHidden => 0, + tt::Spacing::Joint => 1, + }; + [self.id.0, self.char as u32, spacing] + } + fn read([id, char, spacing]: [u32; 3]) -> PunctRepr { + let spacing = match spacing { + 0 => tt::Spacing::Alone, + 1 => tt::Spacing::Joint, + other => panic!("bad spacing {other}"), + }; + PunctRepr { id: SpanId(id), char: char.try_into().unwrap(), spacing } + } +} + +impl IdentRepr { + fn write(self) -> [u32; 2] { + [self.id.0, self.text] + } + fn read(data: [u32; 2]) -> IdentRepr { + IdentRepr { id: SpanId(data[0]), text: data[1], is_raw: false } + } + fn write_with_rawness(self) -> [u32; 3] { + [self.id.0, self.text, self.is_raw as u32] + } + fn read_with_rawness([id, text, is_raw]: [u32; 3]) -> IdentRepr { + IdentRepr { id: SpanId(id), text, is_raw: is_raw == 1 } + } +} + +pub trait SpanTransformer { + type Table; + type Span: Copy + 'static; + fn token_id_of(table: &mut Self::Table, s: Self::Span) -> SpanId; + fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span; +} +impl SpanTransformer for SpanId { + type Table = (); + type Span = Self; + fn token_id_of((): &mut Self::Table, token_id: Self::Span) -> SpanId { + token_id + } + + fn span_for_token_id((): &Self::Table, id: SpanId) -> Self::Span { + id + } +} +impl SpanTransformer for Span { + type Table = SpanDataIndexMap; + type Span = Self; + fn token_id_of(table: &mut Self::Table, span: Self::Span) -> SpanId { + SpanId(table.insert_full(span).0 as u32) + } + fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span { + *table.get_index(id.0 as usize).unwrap_or_else(|| &table[0]) + } +} + +enum SubtreeOrLeafRef<'a, Span, W: WriterTrait<'a, Span>> { + Subtree(W::Subtree), + Leaf(W::Leaf), +} + +trait WriterTrait<'a, Span>: Sized { + type Subtree; + type Leaf: Borrow>; + + type SubtreeIter; + + fn subtree_data(subtree: Self::Subtree) -> (usize, tt::Delimiter, Self::SubtreeIter); + + fn subtree_iter_next(iter: &mut Self::SubtreeIter) -> Option>; +} + +struct Writer<'a, 'span, ST: SpanTransformer, W: WriterTrait<'a, ST::Span>> { + work: VecDeque<(usize, usize, W::SubtreeIter)>, + string_table: FxHashMap, u32>, + span_data_table: &'span mut ST::Table, + version: u32, + + subtree: Vec, + literal: Vec, + punct: Vec, + ident: Vec, + token_tree: Vec, + text: Vec, +} + +impl<'a, ST: SpanTransformer, W: WriterTrait<'a, ST::Span>> Writer<'a, '_, ST, W> { + fn write_subtree(&mut self, root: W::Subtree) { + self.enqueue(root); + while let Some((idx, len, subtree)) = self.work.pop_front() { + self.subtree(idx, len, subtree); + } + } + + fn subtree(&mut self, idx: usize, n_tt: usize, mut subtree: W::SubtreeIter) { + let mut first_tt = self.token_tree.len(); + self.token_tree.resize(first_tt + n_tt, !0); + + self.subtree[idx].tt = [first_tt as u32, (first_tt + n_tt) as u32]; + + while let Some(child) = W::subtree_iter_next(&mut subtree) { + let idx_tag = match child { + SubtreeOrLeafRef::Subtree(subtree) => { + let idx = self.enqueue(subtree); + idx << 2 + } + SubtreeOrLeafRef::Leaf(leaf) => match leaf.borrow() { + tt::Leaf::Literal(lit) => { + let idx = self.literal.len() as u32; + let id = self.token_id_of(lit.span); + let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA { + let (text, suffix) = lit.text_and_suffix(); + ( + self.intern_owned(text.to_owned()), + if suffix.is_empty() { + !0 + } else { + self.intern_owned(suffix.to_owned()) + }, + ) + } else { + (self.intern_owned(format!("{lit}")), !0) + }; + self.literal.push(LiteralRepr { + id, + text, + kind: u16::from_le_bytes(match lit.kind { + tt::LitKind::Err(_) => [0, 0], + tt::LitKind::Byte => [1, 0], + tt::LitKind::Char => [2, 0], + tt::LitKind::Integer => [3, 0], + tt::LitKind::Float => [4, 0], + tt::LitKind::Str => [5, 0], + tt::LitKind::StrRaw(r) => [6, r], + tt::LitKind::ByteStr => [7, 0], + tt::LitKind::ByteStrRaw(r) => [8, r], + tt::LitKind::CStr => [9, 0], + tt::LitKind::CStrRaw(r) => [10, r], + }), + suffix, + }); + (idx << 2) | 0b01 + } + tt::Leaf::Punct(punct) => { + let idx = self.punct.len() as u32; + let id = self.token_id_of(punct.span); + self.punct.push(PunctRepr { char: punct.char, spacing: punct.spacing, id }); + (idx << 2) | 0b10 + } + tt::Leaf::Ident(ident) => { + let idx = self.ident.len() as u32; + let id = self.token_id_of(ident.span); + let text = if self.version >= EXTENDED_LEAF_DATA { + self.intern_owned(ident.sym.as_str().to_owned()) + } else if ident.is_raw.yes() { + self.intern_owned(format!("r#{}", ident.sym.as_str(),)) + } else { + self.intern_owned(ident.sym.as_str().to_owned()) + }; + self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw.yes() }); + (idx << 2) | 0b11 + } + }, + }; + self.token_tree[first_tt] = idx_tag; + first_tt += 1; + } + } + + fn enqueue(&mut self, subtree: W::Subtree) -> u32 { + let idx = self.subtree.len(); + let (len, delimiter, contents) = W::subtree_data(subtree); + let open = self.token_id_of(delimiter.open); + let close = self.token_id_of(delimiter.close); + let delimiter_kind = delimiter.kind; + self.subtree.push(SubtreeRepr { open, close, kind: delimiter_kind, tt: [!0, !0] }); + self.work.push_back((idx, len, contents)); + idx as u32 + } +} + +impl<'a, ST: SpanTransformer, W: WriterTrait<'a, ST::Span>> Writer<'a, '_, ST, W> { + fn token_id_of(&mut self, span: ST::Span) -> SpanId { + ST::token_id_of(self.span_data_table, span) + } + + pub(crate) fn intern_owned(&mut self, text: String) -> u32 { + let table = &mut self.text; + *self.string_table.entry(text.clone().into()).or_insert_with(|| { + let idx = table.len(); + table.push(text); + idx as u32 + }) + } +} + +trait ReaderTrait { + type TokenTree; + + fn leaf(leaf: tt::Leaf) -> Self::TokenTree; + + fn append_subtree( + delimiter: tt::Delimiter, + children: Vec, + insert_into: &mut Vec, + ); +} + +struct Reader<'span, ST: SpanTransformer, R: ReaderTrait> { + version: u32, + subtree: Vec, + literal: Vec, + punct: Vec, + ident: Vec, + token_tree: Vec, + text: Vec, + span_data_table: &'span ST::Table, + _marker: PhantomData, +} + +impl> Reader<'_, ST, R> { + pub(crate) fn read(self) -> (tt::Delimiter, Vec) { + let mut res: Vec, Vec)>> = + (0..self.subtree.len()).map(|_| None).collect(); + let read_span = |id| ST::span_for_token_id(self.span_data_table, id); + for i in (0..self.subtree.len()).rev() { + let repr = &self.subtree[i]; + let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize]; + let delimiter = tt::Delimiter { + open: read_span(repr.open), + close: read_span(repr.close), + kind: repr.kind, + }; + let mut s = Vec::new(); + for &idx_tag in token_trees { + let tag = idx_tag & 0b11; + let idx = (idx_tag >> 2) as usize; + match tag { + // XXX: we iterate subtrees in reverse to guarantee + // that this unwrap doesn't fire. + 0b00 => { + let (delimiter, subtree) = res[idx].take().unwrap(); + R::append_subtree(delimiter, subtree, &mut s); + } + 0b01 => { + use tt::LitKind::*; + let repr = &self.literal[idx]; + let text = self.text[repr.text as usize].as_str(); + let span = read_span(repr.id); + s.push(R::leaf(tt::Leaf::Literal(if self.version >= EXTENDED_LEAF_DATA { + tt::Literal::new( + text, + span, + match u16::to_le_bytes(repr.kind) { + [0, _] => Err(()), + [1, _] => Byte, + [2, _] => Char, + [3, _] => Integer, + [4, _] => Float, + [5, _] => Str, + [6, r] => StrRaw(r), + [7, _] => ByteStr, + [8, r] => ByteStrRaw(r), + [9, _] => CStr, + [10, r] => CStrRaw(r), + _ => unreachable!(), + }, + if repr.suffix != !0 { + self.text[repr.suffix as usize].as_str() + } else { + "" + }, + ) + } else { + tt::literal_from_str_or_err(text, span) + }))) + } + 0b10 => { + let repr = &self.punct[idx]; + s.push(R::leaf(tt::Leaf::Punct(tt::Punct { + char: repr.char, + spacing: repr.spacing, + span: read_span(repr.id), + }))) + } + 0b11 => { + let repr = &self.ident[idx]; + let text = self.text[repr.text as usize].as_str(); + let (is_raw, text) = if self.version >= EXTENDED_LEAF_DATA { + ( + if repr.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No }, + text, + ) + } else { + tt::IdentIsRaw::split_from_symbol(text) + }; + s.push(R::leaf(tt::Leaf::Ident(tt::Ident { + sym: Symbol::intern(text), + span: read_span(repr.id), + is_raw, + }))) + } + other => panic!("bad tag: {other}"), + } + } + res[i] = Some((delimiter, s)); + } + + res[0].take().unwrap() + } +} diff --git a/crates/proc-macro-api/src/flat/proc_macro_srv_side.rs b/crates/proc-macro-api/src/flat/proc_macro_srv_side.rs new file mode 100644 index 000000000000..03701f3f2660 --- /dev/null +++ b/crates/proc-macro-api/src/flat/proc_macro_srv_side.rs @@ -0,0 +1,87 @@ +//! Conversion from/to the flat tree for the proc macro server. + +use crate::{ + flat::{FlatTree, ReaderTrait, SpanTransformer, SubtreeOrLeafRef, WriterTrait}, + token_stream::{Group, SpanLike, TokenStream, TokenTree}, +}; + +struct Writer; + +impl<'a, Span: Copy + 'a> WriterTrait<'a, Span> for Writer { + type Subtree = &'a Group; + type Leaf = &'a tt::Leaf; + + type SubtreeIter = Option>>; + + fn subtree_data(subtree: Self::Subtree) -> (usize, tt::Delimiter, Self::SubtreeIter) { + (subtree.stream_len(), subtree.delimiter, subtree.stream.as_ref().map(|it| it.iter())) + } + + fn subtree_iter_next(iter: &mut Self::SubtreeIter) -> Option> { + iter.as_mut()?.next().map(|item| match item { + TokenTree::Leaf(leaf) => SubtreeOrLeafRef::Leaf(leaf), + TokenTree::Group(group) => SubtreeOrLeafRef::Subtree(group), + }) + } +} + +struct Reader; + +impl ReaderTrait for Reader { + type TokenTree = TokenTree; + + fn leaf(leaf: tt::Leaf) -> Self::TokenTree { + TokenTree::Leaf(leaf) + } + + fn append_subtree( + delimiter: tt::Delimiter, + children: Vec, + insert_into: &mut Vec, + ) { + insert_into.push(TokenTree::Group(Group { + delimiter, + stream: TokenStream::new_or_empty(children), + })); + } +} + +impl FlatTree { + pub fn from_tokenstream( + tokenstream: TokenStream, + call_site: ST::Span, + version: u32, + span_data_table: &mut ST::Table, + ) -> FlatTree { + let root = if let Some(group) = tokenstream.as_single_group() { + group.clone() + } else { + Group { + delimiter: tt::Delimiter { + open: call_site, + close: call_site, + kind: tt::DelimiterKind::Invisible, + }, + stream: Some(tokenstream), + } + }; + FlatTree::deserialize::(&root, version, span_data_table) + } + + pub fn to_tokenstream>( + self, + version: u32, + span_data_table: &ST::Table, + ) -> TokenStream { + let (top_delimiter, top_children) = self.serialize::(version, span_data_table); + let result = if top_delimiter.kind == tt::DelimiterKind::Invisible { + top_children + } else { + vec![TokenTree::Group(Group { + delimiter: top_delimiter, + stream: TokenStream::new_or_empty(top_children), + })] + }; + TokenStream::new(result) + } +} diff --git a/crates/proc-macro-api/src/flat/ra_side.rs b/crates/proc-macro-api/src/flat/ra_side.rs new file mode 100644 index 000000000000..bd42f4461463 --- /dev/null +++ b/crates/proc-macro-api/src/flat/ra_side.rs @@ -0,0 +1,73 @@ +//! Conversion from/to the flat tree for rust-analyzer. + +use tt::Span; + +use crate::flat::{FlatTree, ReaderTrait, SpanDataIndexMap, SubtreeOrLeafRef, WriterTrait}; + +struct Writer; + +impl<'a> WriterTrait<'a, Span> for Writer { + type Subtree = (tt::Subtree, tt::TtIter<'a>); + type Leaf = tt::Leaf; + + type SubtreeIter = tt::TtIter<'a>; + + fn subtree_data((subtree, iter): Self::Subtree) -> (usize, tt::Delimiter, Self::SubtreeIter) { + // FIXME: `count()` walks over the iterator. + (iter.clone().count(), subtree.delimiter, iter) + } + + fn subtree_iter_next(iter: &mut Self::SubtreeIter) -> Option> { + iter.next().map(|item| match item { + tt::TtElement::Leaf(leaf) => SubtreeOrLeafRef::Leaf(leaf), + tt::TtElement::Subtree(subtree, iter) => SubtreeOrLeafRef::Subtree((subtree, iter)), + }) + } +} + +struct Reader; + +impl ReaderTrait for Reader { + type TokenTree = tt::TokenTree; + + fn leaf(leaf: tt::Leaf) -> Self::TokenTree { + tt::TokenTree::Leaf(leaf) + } + + fn append_subtree( + delimiter: tt::Delimiter, + children: Vec, + insert_into: &mut Vec, + ) { + insert_into + .push(tt::TokenTree::Subtree(tt::Subtree { delimiter, len: children.len() as u32 })); + insert_into.extend(children); + } +} + +impl FlatTree { + pub fn from_subtree( + subtree: tt::SubtreeView<'_>, + version: u32, + span_data_table: &mut SpanDataIndexMap, + ) -> FlatTree { + FlatTree::deserialize::( + (subtree.top_subtree(), subtree.iter()), + version, + span_data_table, + ) + } + + pub fn to_subtree(self, version: u32, span_data_table: &SpanDataIndexMap) -> tt::TopSubtree { + let (top_delimiter, mut top_children) = + self.serialize::(version, span_data_table); + top_children.insert( + 0, + tt::TokenTree::Subtree(tt::Subtree { + delimiter: top_delimiter, + len: top_children.len() as u32, + }), + ); + tt::TopSubtree::from_serialized(top_children) + } +} diff --git a/crates/proc-macro-api/src/legacy_protocol.rs b/crates/proc-macro-api/src/legacy_protocol.rs index ee1795d39c2e..929729e0492f 100644 --- a/crates/proc-macro-api/src/legacy_protocol.rs +++ b/crates/proc-macro-api/src/legacy_protocol.rs @@ -1,27 +1,11 @@ //! The initial proc-macro-srv protocol, soon to be deprecated. pub mod msg; +#[cfg(feature = "in-ra")] +mod sender; -use std::{ - io::{BufRead, Write}, - sync::Arc, -}; - -use paths::AbsPath; -use span::Span; - -use crate::{ - ProcMacro, ProcMacroKind, ServerError, - legacy_protocol::msg::{ - ExpandMacro, ExpandMacroData, ExpnGlobals, FlatTree, Message, Request, Response, - ServerConfig, SpanDataIndexMap, deserialize_span_data_index_map, - flat::serialize_span_data_index_map, - }, - process::ProcMacroServerProcess, - version, -}; - -pub(crate) use crate::legacy_protocol::msg::SpanMode; +#[cfg(feature = "in-ra")] +pub(crate) use self::sender::*; /// Legacy span type, only defined here as it is still used by the proc-macro server. /// While rust-analyzer doesn't use this anymore at all, RustRover relies on the legacy type for @@ -34,136 +18,3 @@ impl std::fmt::Debug for SpanId { self.0.fmt(f) } } - -pub(crate) fn version_check(srv: &ProcMacroServerProcess) -> Result { - let request = Request::ApiVersionCheck {}; - let response = send_task(srv, request)?; - - match response { - Response::ApiVersionCheck(version) => Ok(version), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -/// Enable support for rust-analyzer span mode if the server supports it. -pub(crate) fn enable_rust_analyzer_spans( - srv: &ProcMacroServerProcess, -) -> Result { - let request = Request::SetConfig(ServerConfig { span_mode: SpanMode::RustAnalyzer }); - let response = send_task(srv, request)?; - - match response { - Response::SetConfig(ServerConfig { span_mode }) => Ok(span_mode), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -/// Finds proc-macros in a given dynamic library. -pub(crate) fn find_proc_macros( - srv: &ProcMacroServerProcess, - dylib_path: &AbsPath, -) -> Result, String>, ServerError> { - let request = Request::ListMacros { dylib_path: dylib_path.to_path_buf().into() }; - - let response = send_task(srv, request)?; - - match response { - Response::ListMacros(it) => Ok(it), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -pub(crate) fn expand( - proc_macro: &ProcMacro, - process: &ProcMacroServerProcess, - subtree: tt::SubtreeView<'_>, - attr: Option>, - env: Vec<(String, String)>, - def_site: Span, - call_site: Span, - mixed_site: Span, - current_dir: String, -) -> Result, crate::ServerError> { - let version = process.version(); - let mut span_data_table = SpanDataIndexMap::default(); - let def_site = span_data_table.insert_full(def_site).0; - let call_site = span_data_table.insert_full(call_site).0; - let mixed_site = span_data_table.insert_full(mixed_site).0; - let task = ExpandMacro { - data: ExpandMacroData { - macro_body: FlatTree::from_subtree(subtree, version, &mut span_data_table), - macro_name: proc_macro.name.to_string(), - attributes: attr - .map(|subtree| FlatTree::from_subtree(subtree, version, &mut span_data_table)), - has_global_spans: ExpnGlobals { - serialize: version >= version::HAS_GLOBAL_SPANS, - def_site, - call_site, - mixed_site, - }, - span_data_table: if process.rust_analyzer_spans() { - serialize_span_data_index_map(&span_data_table) - } else { - Vec::new() - }, - }, - lib: proc_macro.dylib_path.to_path_buf().into(), - env, - current_dir: Some(current_dir), - }; - - let response = send_task(process, Request::ExpandMacro(Box::new(task)))?; - - match response { - Response::ExpandMacro(it) => Ok(it - .map(|tree| { - let mut expanded = FlatTree::to_subtree_resolved(tree, version, &span_data_table); - if proc_macro.needs_fixup_change() { - proc_macro.change_fixup_to_match_old_server(&mut expanded); - } - expanded - }) - .map_err(|msg| msg.0)), - Response::ExpandMacroExtended(it) => Ok(it - .map(|resp| { - let mut expanded = FlatTree::to_subtree_resolved( - resp.tree, - version, - &deserialize_span_data_index_map(&resp.span_data_table), - ); - if proc_macro.needs_fixup_change() { - proc_macro.change_fixup_to_match_old_server(&mut expanded); - } - expanded - }) - .map_err(|msg| msg.0)), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -/// Sends a request to the proc-macro server and waits for a response. -fn send_task(srv: &ProcMacroServerProcess, req: Request) -> Result { - if let Some(server_error) = srv.exited() { - return Err(server_error.clone()); - } - - srv.send_task_legacy::<_, _>(send_request, req) -} - -/// Sends a request to the server and reads the response. -fn send_request( - mut writer: &mut dyn Write, - mut reader: &mut dyn BufRead, - req: Request, - buf: &mut String, -) -> Result, ServerError> { - req.write(&mut writer).map_err(|err| ServerError { - message: "failed to write request".into(), - io: Some(Arc::new(err)), - })?; - let res = Response::read(&mut reader, buf).map_err(|err| ServerError { - message: "failed to read response".into(), - io: Some(Arc::new(err)), - })?; - Ok(res) -} diff --git a/crates/proc-macro-api/src/legacy_protocol/msg.rs b/crates/proc-macro-api/src/legacy_protocol/msg.rs index 9b71a8b70c8f..9babf6a3dbf5 100644 --- a/crates/proc-macro-api/src/legacy_protocol/msg.rs +++ b/crates/proc-macro-api/src/legacy_protocol/msg.rs @@ -1,6 +1,4 @@ //! Defines messages for cross-process message passing based on `ndjson` wire protocol -pub(crate) mod flat; -pub use self::flat::*; use std::io::{self, BufRead, Write}; @@ -8,7 +6,7 @@ use paths::Utf8PathBuf; use serde::de::DeserializeOwned; use serde_derive::{Deserialize, Serialize}; -use crate::{ProcMacroKind, transport::json}; +use crate::{ProcMacroKind, flat::FlatTree, transport::json}; /// Represents requests sent from the client to the proc-macro-srv. #[derive(Debug, Serialize, Deserialize)] @@ -376,7 +374,7 @@ mod tests { assert_eq!( tt, - back.data.macro_body.to_subtree_resolved(v, &span_data_table), + back.data.macro_body.to_subtree(v, &span_data_table), "version: {v}" ); } @@ -384,7 +382,6 @@ mod tests { } #[test] - #[cfg(feature = "in-rust-tree")] fn test_proc_macro_rpc_works_ts() { for tt in [ fixture_token_tree_top_many_none, @@ -395,18 +392,19 @@ mod tests { for v in version::RUST_ANALYZER_SPAN_SUPPORT..=version::CURRENT_API_VERSION { let mut span_data_table = Default::default(); let flat_tree = FlatTree::from_subtree(tt.view(), v, &mut span_data_table); - assert_eq!( - tt, - flat_tree.clone().to_subtree_resolved(v, &span_data_table), - "version: {v}" - ); - let ts = flat_tree.to_tokenstream_resolved(v, &span_data_table, |a, b| a.cover(b)); + assert_eq!(tt, flat_tree.clone().to_subtree(v, &span_data_table), "version: {v}"); + let ts = flat_tree.to_tokenstream::(v, &span_data_table); let call_site = *span_data_table.first().unwrap(); let mut span_data_table = Default::default(); assert_eq!( tt, - FlatTree::from_tokenstream(ts.clone(), v, call_site, &mut span_data_table) - .to_subtree_resolved(v, &span_data_table), + FlatTree::from_tokenstream::( + ts.clone(), + call_site, + v, + &mut span_data_table + ) + .to_subtree(v, &span_data_table), "version: {v}, ts:\n{ts:#?}" ); } diff --git a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs b/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs deleted file mode 100644 index b9b6247b54fa..000000000000 --- a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs +++ /dev/null @@ -1,980 +0,0 @@ -//! Serialization-friendly representation of `tt::TopSubtree`. -//! -//! It is possible to serialize `TopSubtree` recursively, as a tree, but using -//! arbitrary-nested trees in JSON is problematic, as they can cause the JSON -//! parser to overflow the stack. -//! -//! Additionally, such implementation would be pretty verbose, and we do care -//! about performance here a bit. -//! -//! So what this module does is dumping a `tt::TopSubtree` into a bunch of flat -//! array of numbers. -//! -//! ```json -//! { -//! // Array of subtrees, each subtree is represented by 4 numbers: -//! // id of delimiter, delimiter kind, index of first child in `token_tree`, -//! // index of last child in `token_tree` -//! "subtree":[4294967295,0,0,5,2,2,5,5], -//! // 2 ints per literal: [token id, index into `text`] -//! "literal":[4294967295,1], -//! // 3 ints per punct: [token id, char, spacing] -//! "punct":[4294967295,64,1], -//! // 2 ints per ident: [token id, index into `text`] -//! "ident": [0,0,1,1], -//! // children of all subtrees, concatenated. Each child is represented as `index << 2 | tag` -//! // where tag denotes one of subtree, literal, punct or ident. -//! "token_tree":[3,7,1,4], -//! // Strings shared by idents and literals -//! "text": ["struct","Foo"] -//! } -//! ``` -//! -//! We probably should replace most of the code here with bincode someday, but, -//! as we don't have bincode in Cargo.toml yet, let's stick with serde_json for -//! the time being. - -#[cfg(feature = "in-rust-tree")] -use proc_macro_srv::TokenStream; - -use std::collections::VecDeque; - -use intern::Symbol; -use rustc_hash::FxHashMap; -use serde_derive::{Deserialize, Serialize}; -use span::{EditionedFileId, ErasedFileAstId, Span, SpanAnchor, SyntaxContext, TextRange}; - -use crate::{ - legacy_protocol::SpanId, - version::{ENCODE_CLOSE_SPAN_VERSION, EXTENDED_LEAF_DATA}, -}; - -pub type SpanDataIndexMap = - indexmap::IndexSet>; - -pub fn serialize_span_data_index_map(map: &SpanDataIndexMap) -> Vec { - map.iter() - .map(|span| { - [ - span.anchor.file_id.as_u32(), - span.anchor.ast_id.into_raw(), - span.range.start().into(), - span.range.end().into(), - span.ctx.into_u32(), - ] - }) - .collect::>() - .into_flattened() -} - -pub fn deserialize_span_data_index_map(map: &[u32]) -> SpanDataIndexMap { - let (chunks, remainder) = map.as_chunks(); - assert!(remainder.is_empty()); - chunks - .iter() - .map(|&[file_id, ast_id, start, end, e]| { - Span { - anchor: SpanAnchor { - file_id: EditionedFileId::from_raw(file_id), - ast_id: ErasedFileAstId::from_raw(ast_id), - }, - range: TextRange::new(start.into(), end.into()), - // SAFETY: We only receive spans from the server. If someone mess up the communication UB can happen, - // but that will be their problem. - ctx: unsafe { SyntaxContext::from_u32(e) }, - } - }) - .collect() -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct FlatTree { - subtree: Vec, - literal: Vec, - punct: Vec, - ident: Vec, - token_tree: Vec, - text: Vec, -} - -struct SubtreeRepr { - open: SpanId, - close: SpanId, - kind: tt::DelimiterKind, - tt: [u32; 2], -} - -struct LiteralRepr { - id: SpanId, - text: u32, - suffix: u32, - kind: u16, -} - -struct PunctRepr { - id: SpanId, - char: char, - spacing: tt::Spacing, -} - -struct IdentRepr { - id: SpanId, - text: u32, - is_raw: bool, -} - -impl FlatTree { - pub fn from_subtree( - subtree: tt::SubtreeView<'_>, - version: u32, - span_data_table: &mut SpanDataIndexMap, - ) -> FlatTree { - let mut w = Writer:: { - string_table: FxHashMap::default(), - work: VecDeque::new(), - span_data_table, - - subtree: Vec::new(), - literal: Vec::new(), - punct: Vec::new(), - ident: Vec::new(), - token_tree: Vec::new(), - text: Vec::new(), - version, - }; - w.write_subtree(subtree); - - FlatTree { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - write_vec(w.subtree, SubtreeRepr::write_with_close_span) - } else { - write_vec(w.subtree, SubtreeRepr::write) - }, - literal: if version >= EXTENDED_LEAF_DATA { - write_vec(w.literal, LiteralRepr::write_with_kind) - } else { - write_vec(w.literal, LiteralRepr::write) - }, - punct: write_vec(w.punct, PunctRepr::write), - ident: if version >= EXTENDED_LEAF_DATA { - write_vec(w.ident, IdentRepr::write_with_rawness) - } else { - write_vec(w.ident, IdentRepr::write) - }, - token_tree: w.token_tree, - text: w.text, - } - } - - pub fn to_subtree_resolved( - self, - version: u32, - span_data_table: &SpanDataIndexMap, - ) -> tt::TopSubtree { - Reader:: { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - read_vec(self.subtree, SubtreeRepr::read_with_close_span) - } else { - read_vec(self.subtree, SubtreeRepr::read) - }, - literal: if version >= EXTENDED_LEAF_DATA { - read_vec(self.literal, LiteralRepr::read_with_kind) - } else { - read_vec(self.literal, LiteralRepr::read) - }, - punct: read_vec(self.punct, PunctRepr::read), - ident: if version >= EXTENDED_LEAF_DATA { - read_vec(self.ident, IdentRepr::read_with_rawness) - } else { - read_vec(self.ident, IdentRepr::read) - }, - token_tree: self.token_tree, - text: self.text, - span_data_table, - version, - } - .read_subtree() - } -} - -#[cfg(feature = "in-rust-tree")] -impl FlatTree { - pub fn from_tokenstream( - tokenstream: proc_macro_srv::TokenStream, - version: u32, - call_site: Span, - span_data_table: &mut SpanDataIndexMap, - ) -> FlatTree { - let mut w = Writer:: { - string_table: FxHashMap::default(), - work: VecDeque::new(), - span_data_table, - - subtree: Vec::new(), - literal: Vec::new(), - punct: Vec::new(), - ident: Vec::new(), - token_tree: Vec::new(), - text: Vec::new(), - version, - }; - w.write_tokenstream(call_site, &tokenstream); - - FlatTree { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - write_vec(w.subtree, SubtreeRepr::write_with_close_span) - } else { - write_vec(w.subtree, SubtreeRepr::write) - }, - literal: if version >= EXTENDED_LEAF_DATA { - write_vec(w.literal, LiteralRepr::write_with_kind) - } else { - write_vec(w.literal, LiteralRepr::write) - }, - punct: write_vec(w.punct, PunctRepr::write), - ident: if version >= EXTENDED_LEAF_DATA { - write_vec(w.ident, IdentRepr::write_with_rawness) - } else { - write_vec(w.ident, IdentRepr::write) - }, - token_tree: w.token_tree, - text: w.text, - } - } - - pub fn from_tokenstream_raw>( - tokenstream: proc_macro_srv::TokenStream, - call_site: T::Span, - version: u32, - ) -> FlatTree { - let mut w = Writer:: { - string_table: FxHashMap::default(), - work: VecDeque::new(), - span_data_table: &mut (), - - subtree: Vec::new(), - literal: Vec::new(), - punct: Vec::new(), - ident: Vec::new(), - token_tree: Vec::new(), - text: Vec::new(), - version, - }; - w.write_tokenstream(call_site, &tokenstream); - - FlatTree { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - write_vec(w.subtree, SubtreeRepr::write_with_close_span) - } else { - write_vec(w.subtree, SubtreeRepr::write) - }, - literal: if version >= EXTENDED_LEAF_DATA { - write_vec(w.literal, LiteralRepr::write_with_kind) - } else { - write_vec(w.literal, LiteralRepr::write) - }, - punct: write_vec(w.punct, PunctRepr::write), - ident: if version >= EXTENDED_LEAF_DATA { - write_vec(w.ident, IdentRepr::write_with_rawness) - } else { - write_vec(w.ident, IdentRepr::write) - }, - token_tree: w.token_tree, - text: w.text, - } - } - - pub fn to_tokenstream_unresolved>( - self, - version: u32, - span_join: impl Fn(T::Span, T::Span) -> T::Span, - ) -> proc_macro_srv::TokenStream { - Reader:: { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - read_vec(self.subtree, SubtreeRepr::read_with_close_span) - } else { - read_vec(self.subtree, SubtreeRepr::read) - }, - literal: if version >= EXTENDED_LEAF_DATA { - read_vec(self.literal, LiteralRepr::read_with_kind) - } else { - read_vec(self.literal, LiteralRepr::read) - }, - punct: read_vec(self.punct, PunctRepr::read), - ident: if version >= EXTENDED_LEAF_DATA { - read_vec(self.ident, IdentRepr::read_with_rawness) - } else { - read_vec(self.ident, IdentRepr::read) - }, - token_tree: self.token_tree, - text: self.text, - span_data_table: &(), - version, - } - .read_tokenstream(span_join) - } - - pub fn to_tokenstream_resolved( - self, - version: u32, - span_data_table: &SpanDataIndexMap, - span_join: impl Fn(Span, Span) -> Span, - ) -> proc_macro_srv::TokenStream { - Reader:: { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - read_vec(self.subtree, SubtreeRepr::read_with_close_span) - } else { - read_vec(self.subtree, SubtreeRepr::read) - }, - literal: if version >= EXTENDED_LEAF_DATA { - read_vec(self.literal, LiteralRepr::read_with_kind) - } else { - read_vec(self.literal, LiteralRepr::read) - }, - punct: read_vec(self.punct, PunctRepr::read), - ident: if version >= EXTENDED_LEAF_DATA { - read_vec(self.ident, IdentRepr::read_with_rawness) - } else { - read_vec(self.ident, IdentRepr::read) - }, - token_tree: self.token_tree, - text: self.text, - span_data_table, - version, - } - .read_tokenstream(span_join) - } -} - -fn read_vec T, const N: usize>(xs: Vec, f: F) -> Vec { - let (chunks, remainder) = xs.as_chunks(); - assert!(remainder.is_empty()); - chunks.iter().map(|chunk| f(*chunk)).collect() -} - -fn write_vec [u32; N], const N: usize>(xs: Vec, f: F) -> Vec { - xs.into_iter().map(f).collect::>().into_flattened() -} - -impl SubtreeRepr { - fn write(self) -> [u32; 4] { - let kind = match self.kind { - tt::DelimiterKind::Invisible => 0, - tt::DelimiterKind::Parenthesis => 1, - tt::DelimiterKind::Brace => 2, - tt::DelimiterKind::Bracket => 3, - }; - [self.open.0, kind, self.tt[0], self.tt[1]] - } - fn read([open, kind, lo, len]: [u32; 4]) -> SubtreeRepr { - let kind = match kind { - 0 => tt::DelimiterKind::Invisible, - 1 => tt::DelimiterKind::Parenthesis, - 2 => tt::DelimiterKind::Brace, - 3 => tt::DelimiterKind::Bracket, - other => panic!("bad kind {other}"), - }; - SubtreeRepr { open: SpanId(open), close: SpanId(!0), kind, tt: [lo, len] } - } - fn write_with_close_span(self) -> [u32; 5] { - let kind = match self.kind { - tt::DelimiterKind::Invisible => 0, - tt::DelimiterKind::Parenthesis => 1, - tt::DelimiterKind::Brace => 2, - tt::DelimiterKind::Bracket => 3, - }; - [self.open.0, self.close.0, kind, self.tt[0], self.tt[1]] - } - fn read_with_close_span([open, close, kind, lo, len]: [u32; 5]) -> SubtreeRepr { - let kind = match kind { - 0 => tt::DelimiterKind::Invisible, - 1 => tt::DelimiterKind::Parenthesis, - 2 => tt::DelimiterKind::Brace, - 3 => tt::DelimiterKind::Bracket, - other => panic!("bad kind {other}"), - }; - SubtreeRepr { open: SpanId(open), close: SpanId(close), kind, tt: [lo, len] } - } -} - -impl LiteralRepr { - fn write(self) -> [u32; 2] { - [self.id.0, self.text] - } - fn read([id, text]: [u32; 2]) -> LiteralRepr { - LiteralRepr { id: SpanId(id), text, kind: 0, suffix: !0 } - } - fn write_with_kind(self) -> [u32; 4] { - [self.id.0, self.text, self.kind as u32, self.suffix] - } - fn read_with_kind([id, text, kind, suffix]: [u32; 4]) -> LiteralRepr { - LiteralRepr { id: SpanId(id), text, kind: kind as u16, suffix } - } -} - -impl PunctRepr { - fn write(self) -> [u32; 3] { - let spacing = match self.spacing { - tt::Spacing::Alone | tt::Spacing::JointHidden => 0, - tt::Spacing::Joint => 1, - }; - [self.id.0, self.char as u32, spacing] - } - fn read([id, char, spacing]: [u32; 3]) -> PunctRepr { - let spacing = match spacing { - 0 => tt::Spacing::Alone, - 1 => tt::Spacing::Joint, - other => panic!("bad spacing {other}"), - }; - PunctRepr { id: SpanId(id), char: char.try_into().unwrap(), spacing } - } -} - -impl IdentRepr { - fn write(self) -> [u32; 2] { - [self.id.0, self.text] - } - fn read(data: [u32; 2]) -> IdentRepr { - IdentRepr { id: SpanId(data[0]), text: data[1], is_raw: false } - } - fn write_with_rawness(self) -> [u32; 3] { - [self.id.0, self.text, self.is_raw as u32] - } - fn read_with_rawness([id, text, is_raw]: [u32; 3]) -> IdentRepr { - IdentRepr { id: SpanId(id), text, is_raw: is_raw == 1 } - } -} - -pub trait SpanTransformer { - type Table; - type Span: Copy; - fn token_id_of(table: &mut Self::Table, s: Self::Span) -> SpanId; - fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span; -} -impl SpanTransformer for SpanId { - type Table = (); - type Span = Self; - fn token_id_of((): &mut Self::Table, token_id: Self::Span) -> SpanId { - token_id - } - - fn span_for_token_id((): &Self::Table, id: SpanId) -> Self::Span { - id - } -} -impl SpanTransformer for Span { - type Table = SpanDataIndexMap; - type Span = Self; - fn token_id_of(table: &mut Self::Table, span: Self::Span) -> SpanId { - SpanId(table.insert_full(span).0 as u32) - } - fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span { - *table.get_index(id.0 as usize).unwrap_or_else(|| &table[0]) - } -} - -struct Writer<'a, 'span, S: SpanTransformer, W> { - work: VecDeque<(usize, usize, W)>, - string_table: FxHashMap, u32>, - span_data_table: &'span mut S::Table, - version: u32, - - subtree: Vec, - literal: Vec, - punct: Vec, - ident: Vec, - token_tree: Vec, - text: Vec, -} - -impl<'a, T: SpanTransformer> Writer<'a, '_, T, tt::iter::TtIter<'a>> { - fn write_subtree(&mut self, root: tt::SubtreeView<'a>) { - let subtree = root.top_subtree(); - self.enqueue(&subtree, root.iter()); - while let Some((idx, len, subtree)) = self.work.pop_front() { - self.subtree(idx, len, subtree); - } - } - - #[expect( - clippy::explicit_counter_loop, - reason = "it looks better the current way since we use `first_tt` before the loop" - )] - fn subtree(&mut self, idx: usize, n_tt: usize, subtree: tt::iter::TtIter<'a>) { - let mut first_tt = self.token_tree.len(); - self.token_tree.resize(first_tt + n_tt, !0); - - self.subtree[idx].tt = [first_tt as u32, (first_tt + n_tt) as u32]; - - for child in subtree { - let idx_tag = match child { - tt::iter::TtElement::Subtree(subtree, subtree_iter) => { - let idx = self.enqueue(&subtree, subtree_iter); - idx << 2 - } - tt::iter::TtElement::Leaf(leaf) => match leaf { - tt::Leaf::Literal(lit) => { - let idx = self.literal.len() as u32; - let id = self.token_id_of(lit.span); - let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA { - let (text, suffix) = lit.text_and_suffix(); - ( - self.intern_owned(text.to_owned()), - if suffix.is_empty() { - !0 - } else { - self.intern_owned(suffix.to_owned()) - }, - ) - } else { - (self.intern_owned(format!("{lit}")), !0) - }; - self.literal.push(LiteralRepr { - id, - text, - kind: u16::from_le_bytes(match lit.kind { - tt::LitKind::Err(_) => [0, 0], - tt::LitKind::Byte => [1, 0], - tt::LitKind::Char => [2, 0], - tt::LitKind::Integer => [3, 0], - tt::LitKind::Float => [4, 0], - tt::LitKind::Str => [5, 0], - tt::LitKind::StrRaw(r) => [6, r], - tt::LitKind::ByteStr => [7, 0], - tt::LitKind::ByteStrRaw(r) => [8, r], - tt::LitKind::CStr => [9, 0], - tt::LitKind::CStrRaw(r) => [10, r], - }), - suffix, - }); - (idx << 2) | 0b01 - } - tt::Leaf::Punct(punct) => { - let idx = self.punct.len() as u32; - let id = self.token_id_of(punct.span); - self.punct.push(PunctRepr { char: punct.char, spacing: punct.spacing, id }); - (idx << 2) | 0b10 - } - tt::Leaf::Ident(ident) => { - let idx = self.ident.len() as u32; - let id = self.token_id_of(ident.span); - let text = if self.version >= EXTENDED_LEAF_DATA { - self.intern_owned(ident.sym.as_str().to_owned()) - } else if ident.is_raw.yes() { - self.intern_owned(format!("r#{}", ident.sym.as_str(),)) - } else { - self.intern_owned(ident.sym.as_str().to_owned()) - }; - self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw.yes() }); - (idx << 2) | 0b11 - } - }, - }; - self.token_tree[first_tt] = idx_tag; - first_tt += 1; - } - } - - fn enqueue(&mut self, subtree: &tt::Subtree, contents: tt::iter::TtIter<'a>) -> u32 { - let idx = self.subtree.len(); - let open = self.token_id_of(subtree.delimiter.open); - let close = self.token_id_of(subtree.delimiter.close); - let delimiter_kind = subtree.delimiter.kind; - self.subtree.push(SubtreeRepr { open, close, kind: delimiter_kind, tt: [!0, !0] }); - // FIXME: `count()` walks over the entire iterator. - self.work.push_back((idx, contents.clone().count(), contents)); - idx as u32 - } -} - -impl<'a, T: SpanTransformer, U> Writer<'a, '_, T, U> { - fn token_id_of(&mut self, span: T::Span) -> SpanId { - T::token_id_of(self.span_data_table, span) - } - - #[cfg(feature = "in-rust-tree")] - pub(crate) fn intern(&mut self, text: &'a str) -> u32 { - let table = &mut self.text; - *self.string_table.entry(text.into()).or_insert_with(|| { - let idx = table.len(); - table.push(text.to_owned()); - idx as u32 - }) - } - - pub(crate) fn intern_owned(&mut self, text: String) -> u32 { - let table = &mut self.text; - *self.string_table.entry(text.clone().into()).or_insert_with(|| { - let idx = table.len(); - table.push(text); - idx as u32 - }) - } -} - -#[cfg(feature = "in-rust-tree")] -impl<'a, T: SpanTransformer> - Writer<'a, '_, T, Option>> -{ - fn write_tokenstream( - &mut self, - call_site: T::Span, - root: &'a proc_macro_srv::TokenStream, - ) { - let call_site = self.token_id_of(call_site); - if let Some(group) = root.as_single_group() { - self.enqueue(group); - } else { - self.subtree.push(SubtreeRepr { - open: call_site, - close: call_site, - kind: tt::DelimiterKind::Invisible, - tt: [!0, !0], - }); - self.work.push_back((0, root.len(), Some(root.iter()))); - } - while let Some((idx, len, group)) = self.work.pop_front() { - self.group(idx, len, group); - } - } - - fn group( - &mut self, - idx: usize, - n_tt: usize, - group: Option>, - ) { - let mut first_tt = self.token_tree.len(); - self.token_tree.resize(first_tt + n_tt, !0); - - self.subtree[idx].tt = [first_tt as u32, (first_tt + n_tt) as u32]; - - for tt in group.into_iter().flatten() { - let idx_tag = match tt { - proc_macro_srv::TokenTree::Group(group) => { - let idx = self.enqueue(group); - idx << 2 - } - proc_macro_srv::TokenTree::Literal(lit) => { - let idx = self.literal.len() as u32; - let id = self.token_id_of(lit.span); - let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA { - ( - self.intern(lit.symbol.as_str()), - lit.suffix.as_ref().map(|s| self.intern(s.as_str())).unwrap_or(!0), - ) - } else { - (self.intern_owned(proc_macro_srv::literal_to_string(lit)), !0) - }; - self.literal.push(LiteralRepr { - id, - text, - kind: u16::from_le_bytes(match lit.kind { - proc_macro_srv::LitKind::ErrWithGuar => [0, 0], - proc_macro_srv::LitKind::Byte => [1, 0], - proc_macro_srv::LitKind::Char => [2, 0], - proc_macro_srv::LitKind::Integer => [3, 0], - proc_macro_srv::LitKind::Float => [4, 0], - proc_macro_srv::LitKind::Str => [5, 0], - proc_macro_srv::LitKind::StrRaw(r) => [6, r], - proc_macro_srv::LitKind::ByteStr => [7, 0], - proc_macro_srv::LitKind::ByteStrRaw(r) => [8, r], - proc_macro_srv::LitKind::CStr => [9, 0], - proc_macro_srv::LitKind::CStrRaw(r) => [10, r], - }), - suffix, - }); - (idx << 2) | 0b01 - } - proc_macro_srv::TokenTree::Punct(punct) => { - let idx = self.punct.len() as u32; - let id = self.token_id_of(punct.span); - self.punct.push(PunctRepr { - char: punct.ch as char, - spacing: if punct.joint { tt::Spacing::Joint } else { tt::Spacing::Alone }, - id, - }); - (idx << 2) | 0b10 - } - proc_macro_srv::TokenTree::Ident(ident) => { - let idx = self.ident.len() as u32; - let id = self.token_id_of(ident.span); - let text = if self.version >= EXTENDED_LEAF_DATA { - self.intern(ident.sym.as_str()) - } else if ident.is_raw { - self.intern_owned(format!("r#{}", ident.sym.as_str(),)) - } else { - self.intern(ident.sym.as_str()) - }; - self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw }); - (idx << 2) | 0b11 - } - }; - self.token_tree[first_tt] = idx_tag; - first_tt += 1; - } - } - - fn enqueue(&mut self, group: &'a proc_macro_srv::Group) -> u32 { - let idx = self.subtree.len(); - let open = self.token_id_of(group.span.open); - let close = self.token_id_of(group.span.close); - let delimiter_kind = match group.delimiter { - proc_macro_srv::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis, - proc_macro_srv::Delimiter::Brace => tt::DelimiterKind::Brace, - proc_macro_srv::Delimiter::Bracket => tt::DelimiterKind::Bracket, - proc_macro_srv::Delimiter::None => tt::DelimiterKind::Invisible, - }; - self.subtree.push(SubtreeRepr { open, close, kind: delimiter_kind, tt: [!0, !0] }); - self.work.push_back(( - idx, - group.stream.as_ref().map_or(0, |stream| stream.len()), - group.stream.as_ref().map(|ts| ts.iter()), - )); - idx as u32 - } -} - -struct Reader<'span, S: SpanTransformer> { - version: u32, - subtree: Vec, - literal: Vec, - punct: Vec, - ident: Vec, - token_tree: Vec, - text: Vec, - span_data_table: &'span S::Table, -} - -impl> Reader<'_, T> { - pub(crate) fn read_subtree(self) -> tt::TopSubtree { - let mut res: Vec)>> = - vec![None; self.subtree.len()]; - let read_span = |id| T::span_for_token_id(self.span_data_table, id); - for i in (0..self.subtree.len()).rev() { - let repr = &self.subtree[i]; - let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize]; - let delimiter = tt::Delimiter { - open: read_span(repr.open), - close: read_span(repr.close), - kind: repr.kind, - }; - let mut s = Vec::new(); - for &idx_tag in token_trees { - let tag = idx_tag & 0b11; - let idx = (idx_tag >> 2) as usize; - match tag { - // XXX: we iterate subtrees in reverse to guarantee - // that this unwrap doesn't fire. - 0b00 => { - let (delimiter, subtree) = res[idx].take().unwrap(); - s.push(tt::TokenTree::Subtree(tt::Subtree { - delimiter, - len: subtree.len() as u32, - })); - s.extend(subtree) - } - 0b01 => { - use tt::LitKind::*; - let repr = &self.literal[idx]; - let text = self.text[repr.text as usize].as_str(); - let span = read_span(repr.id); - s.push( - tt::Leaf::Literal(if self.version >= EXTENDED_LEAF_DATA { - tt::Literal::new( - text, - span, - match u16::to_le_bytes(repr.kind) { - [0, _] => Err(()), - [1, _] => Byte, - [2, _] => Char, - [3, _] => Integer, - [4, _] => Float, - [5, _] => Str, - [6, r] => StrRaw(r), - [7, _] => ByteStr, - [8, r] => ByteStrRaw(r), - [9, _] => CStr, - [10, r] => CStrRaw(r), - _ => unreachable!(), - }, - if repr.suffix != !0 { - self.text[repr.suffix as usize].as_str() - } else { - "" - }, - ) - } else { - tt::token_to_literal(text, span) - }) - .into(), - ) - } - 0b10 => { - let repr = &self.punct[idx]; - s.push( - tt::Leaf::Punct(tt::Punct { - char: repr.char, - spacing: repr.spacing, - span: read_span(repr.id), - }) - .into(), - ) - } - 0b11 => { - let repr = &self.ident[idx]; - let text = self.text[repr.text as usize].as_str(); - let (is_raw, text) = if self.version >= EXTENDED_LEAF_DATA { - ( - if repr.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No }, - text, - ) - } else { - tt::IdentIsRaw::split_from_symbol(text) - }; - s.push( - tt::Leaf::Ident(tt::Ident { - sym: Symbol::intern(text), - span: read_span(repr.id), - is_raw, - }) - .into(), - ) - } - other => panic!("bad tag: {other}"), - } - } - res[i] = Some((delimiter, s)); - } - - let (delimiter, mut res) = res[0].take().unwrap(); - res.insert(0, tt::TokenTree::Subtree(tt::Subtree { delimiter, len: res.len() as u32 })); - tt::TopSubtree::from_serialized(res) - } -} - -#[cfg(feature = "in-rust-tree")] -impl Reader<'_, T> { - pub(crate) fn read_tokenstream( - self, - span_join: impl Fn(T::Span, T::Span) -> T::Span, - ) -> proc_macro_srv::TokenStream { - let mut res: Vec>> = vec![None; self.subtree.len()]; - let read_span = |id| T::span_for_token_id(self.span_data_table, id); - for i in (0..self.subtree.len()).rev() { - let repr = &self.subtree[i]; - let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize]; - - let stream = token_trees - .iter() - .copied() - .map(|idx_tag| { - let tag = idx_tag & 0b11; - let idx = (idx_tag >> 2) as usize; - match tag { - // XXX: we iterate subtrees in reverse to guarantee - // that this unwrap doesn't fire. - 0b00 => proc_macro_srv::TokenTree::Group(res[idx].take().unwrap()), - 0b01 => { - let repr = &self.literal[idx]; - let text = self.text[repr.text as usize].as_str(); - let span = read_span(repr.id); - proc_macro_srv::TokenTree::Literal( - if self.version >= EXTENDED_LEAF_DATA { - proc_macro_srv::Literal { - symbol: Symbol::intern(text), - span, - kind: match u16::to_le_bytes(repr.kind) { - [0, _] => proc_macro_srv::LitKind::ErrWithGuar, - [1, _] => proc_macro_srv::LitKind::Byte, - [2, _] => proc_macro_srv::LitKind::Char, - [3, _] => proc_macro_srv::LitKind::Integer, - [4, _] => proc_macro_srv::LitKind::Float, - [5, _] => proc_macro_srv::LitKind::Str, - [6, r] => proc_macro_srv::LitKind::StrRaw(r), - [7, _] => proc_macro_srv::LitKind::ByteStr, - [8, r] => proc_macro_srv::LitKind::ByteStrRaw(r), - [9, _] => proc_macro_srv::LitKind::CStr, - [10, r] => proc_macro_srv::LitKind::CStrRaw(r), - _ => unreachable!(), - }, - suffix: if repr.suffix != !0 { - Some(Symbol::intern( - self.text[repr.suffix as usize].as_str(), - )) - } else { - None - }, - } - } else { - proc_macro_srv::literal_from_str(text, span).unwrap_or_else( - |_| proc_macro_srv::Literal { - symbol: Symbol::intern("internal error"), - span, - kind: proc_macro_srv::LitKind::ErrWithGuar, - suffix: None, - }, - ) - }, - ) - } - 0b10 => { - let repr = &self.punct[idx]; - proc_macro_srv::TokenTree::Punct(proc_macro_srv::Punct { - ch: repr.char as u8, - joint: repr.spacing == tt::Spacing::Joint, - span: read_span(repr.id), - }) - } - 0b11 => { - let repr = &self.ident[idx]; - let text = self.text[repr.text as usize].as_str(); - let (is_raw, text) = if self.version >= EXTENDED_LEAF_DATA { - ( - if repr.is_raw { - tt::IdentIsRaw::Yes - } else { - tt::IdentIsRaw::No - }, - text, - ) - } else { - tt::IdentIsRaw::split_from_symbol(text) - }; - proc_macro_srv::TokenTree::Ident(proc_macro_srv::Ident { - sym: Symbol::intern(text), - span: read_span(repr.id), - is_raw: is_raw.yes(), - }) - } - other => panic!("bad tag: {other}"), - } - }) - .collect::>(); - let open = read_span(repr.open); - let close = read_span(repr.close); - let g = proc_macro_srv::Group { - delimiter: match repr.kind { - tt::DelimiterKind::Parenthesis => proc_macro_srv::Delimiter::Parenthesis, - tt::DelimiterKind::Brace => proc_macro_srv::Delimiter::Brace, - tt::DelimiterKind::Bracket => proc_macro_srv::Delimiter::Bracket, - tt::DelimiterKind::Invisible => proc_macro_srv::Delimiter::None, - }, - stream: if stream.is_empty() { None } else { Some(TokenStream::new(stream)) }, - span: proc_macro_srv::DelimSpan { - open, - close, - // FIXME: The protocol does not yet encode entire spans ... - entire: span_join(open, close), - }, - }; - res[i] = Some(g); - } - let group = res[0].take().unwrap(); - if group.delimiter == proc_macro_srv::Delimiter::None { - group.stream.unwrap_or_default() - } else { - TokenStream::new(vec![proc_macro_srv::TokenTree::Group(group)]) - } - } -} diff --git a/crates/proc-macro-api/src/legacy_protocol/sender.rs b/crates/proc-macro-api/src/legacy_protocol/sender.rs new file mode 100644 index 000000000000..9b7a2f8f254b --- /dev/null +++ b/crates/proc-macro-api/src/legacy_protocol/sender.rs @@ -0,0 +1,156 @@ +//! Functions for the sender side, i.e. the rust-analyzer side. + +use std::{ + io::{BufRead, Write}, + sync::Arc, +}; + +use paths::AbsPath; +use span::Span; + +use crate::{ + ProcMacroKind, + client::{ProcMacro, ServerError}, + flat::{ + FlatTree, SpanDataIndexMap, deserialize_span_data_index_map, serialize_span_data_index_map, + }, + legacy_protocol::msg::{ + ExpandMacro, ExpandMacroData, ExpnGlobals, Message, Request, Response, ServerConfig, + SpanMode, + }, + process::ProcMacroServerProcess, + version, +}; + +pub(crate) fn version_check(srv: &ProcMacroServerProcess) -> Result { + let request = Request::ApiVersionCheck {}; + let response = send_task(srv, request)?; + + match response { + Response::ApiVersionCheck(version) => Ok(version), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +/// Enable support for rust-analyzer span mode if the server supports it. +pub(crate) fn enable_rust_analyzer_spans( + srv: &ProcMacroServerProcess, +) -> Result { + let request = Request::SetConfig(ServerConfig { span_mode: SpanMode::RustAnalyzer }); + let response = send_task(srv, request)?; + + match response { + Response::SetConfig(ServerConfig { span_mode }) => Ok(span_mode), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +/// Finds proc-macros in a given dynamic library. +pub(crate) fn find_proc_macros( + srv: &ProcMacroServerProcess, + dylib_path: &AbsPath, +) -> Result, String>, ServerError> { + let request = Request::ListMacros { dylib_path: dylib_path.to_path_buf().into() }; + + let response = send_task(srv, request)?; + + match response { + Response::ListMacros(it) => Ok(it), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +pub(crate) fn expand( + proc_macro: &ProcMacro, + process: &ProcMacroServerProcess, + subtree: tt::SubtreeView<'_>, + attr: Option>, + env: Vec<(String, String)>, + def_site: Span, + call_site: Span, + mixed_site: Span, + current_dir: String, +) -> Result, ServerError> { + let version = process.version(); + let mut span_data_table = SpanDataIndexMap::default(); + let def_site = span_data_table.insert_full(def_site).0; + let call_site = span_data_table.insert_full(call_site).0; + let mixed_site = span_data_table.insert_full(mixed_site).0; + let task = ExpandMacro { + data: ExpandMacroData { + macro_body: FlatTree::from_subtree(subtree, version, &mut span_data_table), + macro_name: proc_macro.name.to_string(), + attributes: attr + .map(|subtree| FlatTree::from_subtree(subtree, version, &mut span_data_table)), + has_global_spans: ExpnGlobals { + serialize: version >= version::HAS_GLOBAL_SPANS, + def_site, + call_site, + mixed_site, + }, + span_data_table: if process.rust_analyzer_spans() { + serialize_span_data_index_map(&span_data_table) + } else { + Vec::new() + }, + }, + lib: proc_macro.dylib_path.to_path_buf().into(), + env, + current_dir: Some(current_dir), + }; + + let response = send_task(process, Request::ExpandMacro(Box::new(task)))?; + + match response { + Response::ExpandMacro(it) => Ok(it + .map(|tree| { + let mut expanded = FlatTree::to_subtree(tree, version, &span_data_table); + if proc_macro.needs_fixup_change() { + proc_macro.change_fixup_to_match_old_server(&mut expanded); + } + expanded + }) + .map_err(|msg| msg.0)), + Response::ExpandMacroExtended(it) => Ok(it + .map(|resp| { + let mut expanded = FlatTree::to_subtree( + resp.tree, + version, + &deserialize_span_data_index_map(&resp.span_data_table), + ); + if proc_macro.needs_fixup_change() { + proc_macro.change_fixup_to_match_old_server(&mut expanded); + } + expanded + }) + .map_err(|msg| msg.0)), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +/// Sends a request to the proc-macro server and waits for a response. +fn send_task(srv: &ProcMacroServerProcess, req: Request) -> Result { + if let Some(server_error) = srv.exited() { + return Err(server_error.clone()); + } + + srv.send_task_legacy::<_, _>(send_request, req) +} + +/// Sends a request to the server and reads the response. +fn send_request( + mut writer: &mut dyn Write, + mut reader: &mut dyn BufRead, + req: Request, + buf: &mut String, +) -> Result, ServerError> { + req.write(&mut writer).map_err(|err| ServerError { + message: "failed to write request".into(), + io: Some(Arc::new(err)), + })?; + let res = Response::read(&mut reader, buf).map_err(|err| ServerError { + message: "failed to read response".into(), + io: Some(Arc::new(err)), + })?; + Ok(res) +} diff --git a/crates/proc-macro-api/src/lib.rs b/crates/proc-macro-api/src/lib.rs index 4b5e25e48801..1869a019df6d 100644 --- a/crates/proc-macro-api/src/lib.rs +++ b/crates/proc-macro-api/src/lib.rs @@ -5,30 +5,34 @@ //! is used to provide basic infrastructure for communication between two //! processes: Client (RA itself), Server (the external program) -#![cfg_attr(not(feature = "in-rust-tree"), allow(unused_crate_dependencies))] +#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] #![cfg_attr( - feature = "in-rust-tree", - feature(proc_macro_internals, proc_macro_diagnostic, proc_macro_span, rustc_private) + all(feature = "in-rust-tree", feature = "in-proc-macro-srv"), + feature(proc_macro_internals, proc_macro_diagnostic, proc_macro_span) )] #![allow(internal_features, unused_features)] #[cfg(feature = "in-rust-tree")] extern crate rustc_driver as _; +stdx::rustc_crates! { + extern crate rustc_lexer or ra_ap_rustc_lexer; +} + pub mod bidirectional_protocol; +#[cfg(feature = "in-ra")] +pub mod client; +pub mod flat; pub mod legacy_protocol; +#[cfg(feature = "in-ra")] pub mod pool; +#[cfg(feature = "in-ra")] pub mod process; +#[cfg(feature = "in-proc-macro-srv")] +pub mod token_stream; pub mod transport; -use paths::{AbsPath, AbsPathBuf}; -use semver::Version; -use span::{ErasedFileAstId, FIXUP_ERASED_FILE_AST_ID_MARKER, Span}; -use std::{fmt, io, sync::Arc, time::SystemTime}; - -use crate::{ - bidirectional_protocol::SubCallback, pool::ProcMacroServerPool, process::ProcMacroServerProcess, -}; +use std::fmt; /// The versions of the server protocol pub mod version { @@ -77,203 +81,3 @@ pub enum ProcMacroKind { #[serde(rename(serialize = "FuncLike", deserialize = "FuncLike"))] Bang, } - -/// A handle to proc-macro server process pool which load dylibs with macros (.so or .dll) -/// and runs actual macro expansion functions. -#[derive(Debug, Clone)] -pub struct ProcMacroClient { - /// Currently, the proc macro process expands all procedural macros sequentially. - /// - /// That means that concurrent salsa requests may block each other when expanding proc macros, - /// which is unfortunate, but simple and good enough for the time being. - pool: Arc, - /// The path to the proc-macro server binary. - path: AbsPathBuf, -} - -/// Represents a dynamically loaded library containing procedural macros. -pub struct MacroDylib { - path: AbsPathBuf, -} - -impl MacroDylib { - /// Creates a new MacroDylib instance with the given path. - pub fn new(path: AbsPathBuf) -> MacroDylib { - MacroDylib { path } - } -} - -/// A handle to a specific proc-macro (a `#[proc_macro]` annotated function). -/// -/// It exists within the context of a specific proc-macro server -- currently -/// we share a single expander process for all macros within a workspace. -#[derive(Debug, Clone)] -pub struct ProcMacro { - pool: ProcMacroServerPool, - dylib_path: Arc, - name: Box, - kind: ProcMacroKind, - dylib_last_modified: Option, -} - -impl Eq for ProcMacro {} -impl PartialEq for ProcMacro { - fn eq(&self, other: &Self) -> bool { - self.name == other.name - && self.kind == other.kind - && self.dylib_path == other.dylib_path - && self.dylib_last_modified == other.dylib_last_modified - } -} - -/// Represents errors encountered when communicating with the proc-macro server. -#[derive(Clone, Debug)] -pub struct ServerError { - pub message: String, - pub io: Option>, -} - -impl fmt::Display for ServerError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.message.fmt(f)?; - if let Some(io) = &self.io { - f.write_str(": ")?; - io.fmt(f)?; - } - Ok(()) - } -} - -impl ProcMacroClient { - /// Spawns an external process as the proc macro server and returns a client connected to it. - pub fn spawn<'a>( - process_path: &AbsPath, - env: impl IntoIterator< - Item = (impl AsRef, &'a Option>), - > + Clone, - version: Option<&Version>, - num_process: usize, - ) -> io::Result { - let pool_size = num_process; - let mut workers = Vec::with_capacity(pool_size); - for _ in 0..pool_size { - let worker = ProcMacroServerProcess::spawn(process_path, env.clone(), version)?; - workers.push(worker); - } - - let pool = ProcMacroServerPool::new(workers); - Ok(ProcMacroClient { pool: Arc::new(pool), path: process_path.to_owned() }) - } - - /// Invokes `spawn` and returns a client connected to the resulting read and write handles. - /// - /// The `process_path` is used for `Self::server_path`. This function is mainly used for testing. - pub fn with_io_channels( - process_path: &AbsPath, - spawn: impl Fn( - Option, - ) -> io::Result<( - Box, - Box, - Box, - )> + Clone, - version: Option<&Version>, - num_process: usize, - ) -> io::Result { - let pool_size = num_process; - let mut workers = Vec::with_capacity(pool_size); - for _ in 0..pool_size { - let worker = - ProcMacroServerProcess::run(spawn.clone(), version, || "".to_owned())?; - workers.push(worker); - } - - let pool = ProcMacroServerPool::new(workers); - Ok(ProcMacroClient { pool: Arc::new(pool), path: process_path.to_owned() }) - } - - /// Returns the absolute path to the proc-macro server. - pub fn server_path(&self) -> &AbsPath { - &self.path - } - - /// Loads a proc-macro dylib into the server process returning a list of `ProcMacro`s loaded. - pub fn load_dylib(&self, dylib: MacroDylib) -> Result, ServerError> { - self.pool.load_dylib(&dylib) - } - - /// Checks if the proc-macro server has exited. - pub fn exited(&self) -> Option<&ServerError> { - self.pool.exited() - } -} - -impl ProcMacro { - /// Returns the name of the procedural macro. - pub fn name(&self) -> &str { - &self.name - } - - /// Returns the type of procedural macro. - pub fn kind(&self) -> ProcMacroKind { - self.kind - } - - fn needs_fixup_change(&self) -> bool { - let version = self.pool.version(); - (version::RUST_ANALYZER_SPAN_SUPPORT..version::HASHED_AST_ID).contains(&version) - } - - /// On some server versions, the fixup ast id is different than ours. So change it to match. - fn change_fixup_to_match_old_server(&self, tt: &mut tt::TopSubtree) { - const OLD_FIXUP_AST_ID: ErasedFileAstId = ErasedFileAstId::from_raw(!0 - 1); - tt.change_every_ast_id(|ast_id| { - if *ast_id == FIXUP_ERASED_FILE_AST_ID_MARKER { - *ast_id = OLD_FIXUP_AST_ID; - } else if *ast_id == OLD_FIXUP_AST_ID { - // Swap between them, that means no collision plus the change can be reversed by doing itself. - *ast_id = FIXUP_ERASED_FILE_AST_ID_MARKER; - } - }); - } - - /// Expands the procedural macro by sending an expansion request to the server. - /// This includes span information and environmental context. - pub fn expand( - &self, - subtree: tt::SubtreeView<'_>, - attr: Option>, - env: Vec<(String, String)>, - def_site: Span, - call_site: Span, - mixed_site: Span, - current_dir: String, - callback: Option>, - ) -> Result, ServerError> { - let (mut subtree, mut attr) = (subtree, attr); - let (mut subtree_changed, mut attr_changed); - if self.needs_fixup_change() { - subtree_changed = tt::TopSubtree::from_subtree(subtree); - self.change_fixup_to_match_old_server(&mut subtree_changed); - subtree = subtree_changed.view(); - - if let Some(attr) = &mut attr { - attr_changed = tt::TopSubtree::from_subtree(*attr); - self.change_fixup_to_match_old_server(&mut attr_changed); - *attr = attr_changed.view(); - } - } - - self.pool.pick_process()?.expand( - self, - subtree, - attr, - env, - def_site, - call_site, - mixed_site, - current_dir, - callback, - ) - } -} diff --git a/crates/proc-macro-api/src/pool.rs b/crates/proc-macro-api/src/pool.rs index e6541823da58..1f449a12399e 100644 --- a/crates/proc-macro-api/src/pool.rs +++ b/crates/proc-macro-api/src/pool.rs @@ -3,7 +3,10 @@ use std::sync::Arc; use rayon::iter::{IntoParallelIterator, ParallelIterator}; -use crate::{MacroDylib, ProcMacro, ServerError, process::ProcMacroServerProcess}; +use crate::{ + client::{MacroDylib, ProcMacro, ServerError}, + process::ProcMacroServerProcess, +}; #[derive(Debug, Clone)] pub(crate) struct ProcMacroServerPool { diff --git a/crates/proc-macro-api/src/process.rs b/crates/proc-macro-api/src/process.rs index 035c12669c8f..2781d2b4db0b 100644 --- a/crates/proc-macro-api/src/process.rs +++ b/crates/proc-macro-api/src/process.rs @@ -17,13 +17,14 @@ use span::Span; use stdx::JodChild; use crate::{ - ProcMacro, ProcMacroKind, ProtocolFormat, ServerError, + ProcMacroKind, ProtocolFormat, bidirectional_protocol::{ self, SubCallback, msg::{BidirectionalMessage, SubResponse}, reject_subrequests, }, - legacy_protocol::{self, SpanMode}, + client::{ProcMacro, ServerError}, + legacy_protocol::{self, msg::SpanMode}, version, }; diff --git a/crates/proc-macro-api/src/token_stream.rs b/crates/proc-macro-api/src/token_stream.rs new file mode 100644 index 000000000000..d51662b4eef3 --- /dev/null +++ b/crates/proc-macro-api/src/token_stream.rs @@ -0,0 +1,691 @@ +//! The proc-macro server token stream implementation. + +use core::fmt; +use std::{mem, rc::Rc}; + +use intern::{Symbol, sym}; +use tt::{ + Delimiter, DelimiterKind, Ident, IdentIsRaw, Leaf, LitKind, Literal, Punct, Spacing, + literal_from_lexer, +}; + +/// Trait for allowing tests to parse tokenstreams with dynamic span ranges +pub trait SpanLike: Copy { + fn derive_ranged(&self, range: std::ops::Range) -> Self; + + fn cover(self, other: Self) -> Self; +} + +#[derive(Debug, Clone)] +pub struct Group { + pub delimiter: Delimiter, + pub stream: Option>, +} + +impl Group { + pub fn stream_len(&self) -> usize { + self.stream.as_ref().map_or(0, |it| it.len()) + } +} + +#[derive(Clone)] +pub enum TokenTree { + Leaf(Leaf), + Group(Group), +} + +#[derive(Clone)] +#[expect(clippy::rc_buffer, reason = "we commonly mutate this via `Rc::make_mut()`")] +pub struct TokenStream(Rc>>); + +impl Default for TokenStream { + fn default() -> Self { + Self(Default::default()) + } +} + +impl TokenStream { + #[inline] + pub fn new(tts: Vec>) -> TokenStream { + TokenStream(Rc::new(tts)) + } + + #[inline] + pub fn new_or_empty(tts: Vec>) -> Option> { + if tts.is_empty() { None } else { Some(TokenStream(Rc::new(tts))) } + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + #[inline] + pub fn len(&self) -> usize { + self.0.len() + } + + #[inline] + pub fn iter(&self) -> std::slice::Iter<'_, TokenTree> { + self.0.iter() + } + + #[inline] + pub fn as_slice(&self) -> &[TokenTree] { + &self.0 + } + + #[inline] + pub fn as_single_group(&self) -> Option<&Group> { + match &**self.0 { + [TokenTree::Group(group)] => Some(group), + _ => None, + } + } + + pub fn from_str(s: &str, span: S) -> Result + where + S: SpanLike + Copy, + { + let mut groups = Vec::new(); + groups.push((DelimiterKind::Invisible, 0..0, vec![])); + let mut offset = 0; + let mut tokens = rustc_lexer::tokenize(s, rustc_lexer::FrontmatterAllowed::No).peekable(); + while let Some(token) = tokens.next() { + let range = offset..offset + token.len as usize; + offset += token.len as usize; + + let mut spacing = || { + let is_joint = tokens.peek().is_some_and(|token| { + matches!( + token.kind, + rustc_lexer::TokenKind::RawLifetime + | rustc_lexer::TokenKind::GuardedStrPrefix + | rustc_lexer::TokenKind::Lifetime { .. } + | rustc_lexer::TokenKind::Semi + | rustc_lexer::TokenKind::Comma + | rustc_lexer::TokenKind::Dot + | rustc_lexer::TokenKind::OpenParen + | rustc_lexer::TokenKind::CloseParen + | rustc_lexer::TokenKind::OpenBrace + | rustc_lexer::TokenKind::CloseBrace + | rustc_lexer::TokenKind::OpenBracket + | rustc_lexer::TokenKind::CloseBracket + | rustc_lexer::TokenKind::At + | rustc_lexer::TokenKind::Pound + | rustc_lexer::TokenKind::Tilde + | rustc_lexer::TokenKind::Question + | rustc_lexer::TokenKind::Colon + | rustc_lexer::TokenKind::Dollar + | rustc_lexer::TokenKind::Eq + | rustc_lexer::TokenKind::Bang + | rustc_lexer::TokenKind::Lt + | rustc_lexer::TokenKind::Gt + | rustc_lexer::TokenKind::Minus + | rustc_lexer::TokenKind::And + | rustc_lexer::TokenKind::Or + | rustc_lexer::TokenKind::Plus + | rustc_lexer::TokenKind::Star + | rustc_lexer::TokenKind::Slash + | rustc_lexer::TokenKind::Percent + | rustc_lexer::TokenKind::Caret + ) + }); + if is_joint { Spacing::Joint } else { Spacing::Alone } + }; + + let Some((open_delim, _, tokenstream)) = groups.last_mut() else { + return Err("Unbalanced delimiters".to_owned()); + }; + match token.kind { + rustc_lexer::TokenKind::OpenParen => { + groups.push((DelimiterKind::Parenthesis, range, vec![])) + } + rustc_lexer::TokenKind::CloseParen if *open_delim != DelimiterKind::Parenthesis => { + return if *open_delim == DelimiterKind::Invisible { + Err("Unexpected ')'".to_owned()) + } else { + Err("Expected ')'".to_owned()) + }; + } + rustc_lexer::TokenKind::CloseParen => { + let (delimiter, open_range, stream) = groups.pop().unwrap(); + groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( + TokenTree::Group(Group { + delimiter: Delimiter { + open: span.derive_ranged(open_range), + close: span.derive_ranged(range), + kind: delimiter, + }, + stream: TokenStream::new_or_empty(stream), + }), + ); + } + rustc_lexer::TokenKind::OpenBrace => { + groups.push((DelimiterKind::Brace, range, vec![])) + } + rustc_lexer::TokenKind::CloseBrace if *open_delim != DelimiterKind::Brace => { + return if *open_delim == DelimiterKind::Invisible { + Err("Unexpected '}'".to_owned()) + } else { + Err("Expected '}'".to_owned()) + }; + } + rustc_lexer::TokenKind::CloseBrace => { + let (delimiter, open_range, stream) = groups.pop().unwrap(); + groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( + TokenTree::Group(Group { + delimiter: Delimiter { + open: span.derive_ranged(open_range), + close: span.derive_ranged(range), + kind: delimiter, + }, + stream: TokenStream::new_or_empty(stream), + }), + ); + } + rustc_lexer::TokenKind::OpenBracket => { + groups.push((DelimiterKind::Bracket, range, vec![])) + } + rustc_lexer::TokenKind::CloseBracket if *open_delim != DelimiterKind::Bracket => { + return if *open_delim == DelimiterKind::Invisible { + Err("Unexpected ']'".to_owned()) + } else { + Err("Expected ']'".to_owned()) + }; + } + rustc_lexer::TokenKind::CloseBracket => { + let (delimiter, open_range, stream) = groups.pop().unwrap(); + groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( + TokenTree::Group(Group { + delimiter: Delimiter { + open: span.derive_ranged(open_range), + close: span.derive_ranged(range), + kind: delimiter, + }, + stream: TokenStream::new_or_empty(stream), + }), + ); + } + rustc_lexer::TokenKind::LineComment { doc_style: None } + | rustc_lexer::TokenKind::BlockComment { doc_style: None, terminated: _ } => { + continue; + } + rustc_lexer::TokenKind::LineComment { doc_style: Some(doc_style) } => { + let text = &s[range.start + 3..range.end]; + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '#', + spacing: Spacing::Alone, + span, + }))); + if doc_style == rustc_lexer::DocStyle::Inner { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '!', + spacing: Spacing::Alone, + span, + }))); + } + let span = span.derive_ranged(range); + tokenstream.push(TokenTree::Group(Group { + delimiter: Delimiter { + open: span, + close: span, + kind: DelimiterKind::Bracket, + }, + stream: TokenStream::new_or_empty(vec![ + TokenTree::Leaf(Leaf::Ident(Ident { + sym: sym::doc, + is_raw: IdentIsRaw::No, + span, + })), + TokenTree::Leaf(Leaf::Punct(Punct { + char: '=', + spacing: Spacing::Alone, + span, + })), + TokenTree::Leaf(Leaf::Literal(Literal::new_no_suffix( + &text.escape_debug().to_string(), + span, + LitKind::Str, + ))), + ]), + })); + } + rustc_lexer::TokenKind::BlockComment { doc_style: Some(doc_style), terminated } => { + let text = + &s[range.start + 3..if terminated { range.end - 2 } else { range.end }]; + let span = span.derive_ranged(range); + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '#', + spacing: Spacing::Alone, + span, + }))); + if doc_style == rustc_lexer::DocStyle::Inner { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '!', + spacing: Spacing::Alone, + span, + }))); + } + tokenstream.push(TokenTree::Group(Group { + delimiter: Delimiter { + open: span, + close: span, + kind: DelimiterKind::Bracket, + }, + stream: TokenStream::new_or_empty(vec![ + TokenTree::Leaf(Leaf::Ident(Ident { + sym: sym::doc, + is_raw: IdentIsRaw::No, + span, + })), + TokenTree::Leaf(Leaf::Punct(Punct { + char: '=', + spacing: Spacing::Alone, + span, + })), + TokenTree::Leaf(Leaf::Literal(Literal::new_no_suffix( + &text.escape_debug().to_string(), + span, + LitKind::Str, + ))), + ]), + })); + } + rustc_lexer::TokenKind::Whitespace => continue, + rustc_lexer::TokenKind::Frontmatter { .. } => unreachable!(), + rustc_lexer::TokenKind::Unknown => { + return Err(format!("Unknown token: `{}`", &s[range])); + } + rustc_lexer::TokenKind::UnknownPrefix => { + return Err(format!("Unknown prefix: `{}`", &s[range])); + } + rustc_lexer::TokenKind::UnknownPrefixLifetime => { + return Err(format!("Unknown lifetime prefix: `{}`", &s[range])); + } + // FIXME: Error on edition >= 2024 ... I dont think the proc-macro server can fetch editions currently + // and whose edition is this? + rustc_lexer::TokenKind::GuardedStrPrefix => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: s.as_bytes()[range.start].into(), + spacing: Spacing::Joint, + span: span.derive_ranged(range.start..range.start + 1), + }))); + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: s.as_bytes()[range.start + 1].into(), + spacing: spacing(), + span: span.derive_ranged(range.start + 1..range.end), + }))) + } + rustc_lexer::TokenKind::Ident => { + tokenstream.push(TokenTree::Leaf(Leaf::Ident(Ident { + sym: Symbol::intern(&s[range.clone()]), + is_raw: IdentIsRaw::No, + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::InvalidIdent => { + return Err(format!("Invalid identifier: `{}`", &s[range])); + } + rustc_lexer::TokenKind::RawIdent => { + let range = range.start + 2..range.end; + tokenstream.push(TokenTree::Leaf(Leaf::Ident(Ident { + sym: Symbol::intern(&s[range.clone()]), + is_raw: IdentIsRaw::Yes, + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Literal { kind, suffix_start } => { + tokenstream.push(TokenTree::Leaf(Leaf::Literal(literal_from_lexer( + &s[range.clone()], + span.derive_ranged(range), + kind, + suffix_start, + )))) + } + rustc_lexer::TokenKind::RawLifetime => { + let range = range.start + 1 + 2..range.end; + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '\'', + spacing: Spacing::Joint, + span: span.derive_ranged(range.start..range.start + 1), + }))); + tokenstream.push(TokenTree::Leaf(Leaf::Ident(Ident { + sym: Symbol::intern(&s[range.clone()]), + is_raw: IdentIsRaw::Yes, + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Lifetime { starts_with_number } => { + if starts_with_number { + return Err("Lifetime cannot start with a number".to_owned()); + } + let range = range.start + 1..range.end; + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '\'', + spacing: Spacing::Joint, + span: span.derive_ranged(range.start..range.start + 1), + }))); + tokenstream.push(TokenTree::Leaf(Leaf::Ident(Ident { + sym: Symbol::intern(&s[range.clone()]), + is_raw: IdentIsRaw::No, + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Semi => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: ';', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Comma => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: ',', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Dot => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '.', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::At => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '@', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Pound => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '#', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Tilde => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '~', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Question => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '?', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Colon => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: ':', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Dollar => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '$', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Eq => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '=', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Bang => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '!', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Lt => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '<', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Gt => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '>', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Minus => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '-', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::And => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '&', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Or => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '|', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Plus => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '+', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Star => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '*', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Slash => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '/', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Caret => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '^', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Percent => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '%', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Eof => break, + } + } + if let Some((DelimiterKind::Invisible, _, tokentrees)) = groups.pop() + && groups.is_empty() + { + Ok(TokenStream::new(tokentrees)) + } else { + Err("Mismatched token groups".to_owned()) + } + } +} + +impl fmt::Display for TokenStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut emit_whitespace = false; + for tt in self.0.iter() { + display_token_tree(tt, &mut emit_whitespace, f)?; + } + Ok(()) + } +} + +fn display_token_tree( + tt: &TokenTree, + emit_whitespace: &mut bool, + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + if mem::take(emit_whitespace) { + write!(f, " ")?; + } + match tt { + TokenTree::Group(Group { delimiter, stream }) => { + let (open, close) = delimiter.kind.display_open_close(); + write!(f, "{open}")?; + if let Some(stream) = stream { + write!(f, "{stream}")?; + } + write!(f, "{close}")?; + } + TokenTree::Leaf(leaf) => { + fmt::Display::fmt(leaf, f)?; + *emit_whitespace = match leaf { + Leaf::Literal(literal) => !matches!( + literal.kind, + LitKind::Str + | LitKind::StrRaw(_) + | LitKind::ByteStr + | LitKind::ByteStrRaw(_) + | LitKind::CStr + | LitKind::CStrRaw(_) + ), + Leaf::Punct(punct) => punct.spacing == Spacing::Alone, + Leaf::Ident(_) => true, + }; + } + } + Ok(()) +} + +impl fmt::Debug for TokenStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + debug_token_stream(self, 0, f) + } +} + +fn debug_token_stream( + ts: &TokenStream, + depth: usize, + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + for tt in ts.0.iter() { + debug_token_tree(tt, depth, f)?; + } + Ok(()) +} + +fn debug_token_tree( + tt: &TokenTree, + depth: usize, + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + write!(f, "{:indent$}", "", indent = depth * 2)?; + + match tt { + TokenTree::Group(Group { delimiter, stream }) => { + writeln!( + f, + "GROUP {} {:#?} {:#?}", + delimiter.kind.debug_view(), + delimiter.open, + delimiter.close, + )?; + if let Some(stream) = stream { + debug_token_stream(stream, depth + 1, f)?; + } + return Ok(()); + } + TokenTree::Leaf(leaf) => leaf.print_debug(f)?, + } + writeln!(f) +} + +impl TokenStream { + pub fn extend_with_streams(&mut self, streams: std::vec::IntoIter>) { + let vec_mut = Rc::make_mut(&mut self.0); + + vec_mut.reserve(streams.as_slice().iter().map(|item| item.len()).sum()); + streams.into_iter().for_each(|item| vec_mut.extend(item.iter().cloned())); + } +} + +impl FromIterator> for TokenStream { + fn from_iter>>(iter: I) -> Self { + TokenStream::new(Vec::from_iter(iter)) + } +} + +impl Extend> for TokenStream { + fn extend>>(&mut self, iter: T) { + let vec_mut = Rc::make_mut(&mut self.0); + vec_mut.extend(iter); + } +} + +impl SpanLike for () { + fn derive_ranged(&self, _: std::ops::Range) -> Self { + *self + } + + fn cover(self, _other: Self) -> Self { + self + } +} + +impl SpanLike for span::Span { + fn derive_ranged(&self, range: std::ops::Range) -> Self { + span::Span { + range: span::TextRange::new( + span::TextSize::new(range.start as u32), + span::TextSize::new(range.end as u32), + ), + anchor: self.anchor, + ctx: self.ctx, + } + } + + fn cover(self, other: Self) -> Self { + self.cover(other) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ts_to_string() { + let token_stream = + TokenStream::from_str("{} () [] <> ;/., \"gfhdgfuiofghd\" 0f32 r#\"dff\"# 'r#lt", ()) + .unwrap(); + assert_eq!(token_stream.to_string(), "{}()[]<> ;/., \"gfhdgfuiofghd\"0f32 r#\"dff\"#'r#lt"); + } + + #[test] + fn doc_comment_from_str() { + let token_stream = TokenStream::from_str("/// foo", ()).unwrap(); + assert_eq!(token_stream.to_string(), r#"# [doc = " foo"]"#); + } +} diff --git a/crates/proc-macro-srv-cli/Cargo.toml b/crates/proc-macro-srv-cli/Cargo.toml index 44e19f2d3c3b..cfc157596cf7 100644 --- a/crates/proc-macro-srv-cli/Cargo.toml +++ b/crates/proc-macro-srv-cli/Cargo.toml @@ -15,26 +15,33 @@ doctest = false [dependencies] proc-macro-srv.workspace = true -proc-macro-api.workspace = true -clap = {version = "4.5.42", default-features = false, features = ["std"]} +proc-macro-api = { path = "../proc-macro-api", version = "0.0.0", default-features = false, features = [ + "in-proc-macro-srv", +] } +clap = { version = "4.5.42", default-features = false, features = ["std"] } +# span = { workspace = true, default-features = false } does not work +span = { path = "../span", version = "0.0.0", default-features = false } [dev-dependencies] expect-test.workspace = true paths.workspace = true -# span = {workspace = true, default-features = false} does not work -span = { path = "../span", default-features = false} -tt.workspace = true intern.workspace = true # used as proc macro test target proc-macro-test.path = "../proc-macro-srv/proc-macro-test" +# Enable the in-ra feature for tests. +proc-macro-api = { path = "../proc-macro-api", default-features = false, features = [ + "in-proc-macro-srv", + "in-ra", +] } +tt.workspace = true + [features] default = [] # default = ["in-rust-tree"] in-rust-tree = ["proc-macro-srv/in-rust-tree", "proc-macro-api/in-rust-tree"] - [[bin]] name = "rust-analyzer-proc-macro-srv" path = "src/main.rs" diff --git a/crates/proc-macro-srv-cli/src/lib.rs b/crates/proc-macro-srv-cli/src/lib.rs index c330928fbc9c..e51708a4a025 100644 --- a/crates/proc-macro-srv-cli/src/lib.rs +++ b/crates/proc-macro-srv-cli/src/lib.rs @@ -4,7 +4,88 @@ #![cfg(feature = "in-rust-tree")] #![feature(rustc_private)] +#![expect(clippy::print_stdout, clippy::print_stderr)] extern crate rustc_driver as _; pub mod main_loop; +mod version; + +use clap::{Command, ValueEnum}; +use proc_macro_api::ProtocolFormat; + +pub fn main() -> std::io::Result<()> { + let v = std::env::var("RUST_ANALYZER_INTERNALS_DO_NOT_USE"); + if v.is_err() { + eprintln!( + "This is an IDE implementation detail, you can use this tool by exporting RUST_ANALYZER_INTERNALS_DO_NOT_USE." + ); + eprintln!( + "Note that this tool's API is highly unstable and may break without prior notice" + ); + std::process::exit(122); + } + let matches = Command::new("proc-macro-srv") + .args(&[ + clap::Arg::new("format") + .long("format") + .action(clap::ArgAction::Set) + .default_value("json-legacy") + .value_parser(clap::builder::EnumValueParser::::new()), + clap::Arg::new("version") + .long("version") + .action(clap::ArgAction::SetTrue) + .help("Prints the version of the proc-macro-srv"), + ]) + .get_matches(); + if matches.get_flag("version") { + println!("rust-analyzer-proc-macro-srv {}", version::version()); + return Ok(()); + } + let &format = matches + .get_one::("format") + .expect("format value should always be present"); + + let mut stdin = std::io::BufReader::new(std::io::stdin()); + let mut stdout = std::io::stdout(); + + main_loop::run(&mut stdin, &mut stdout, format.into()) +} + +/// Wrapper for CLI argument parsing that implements `ValueEnum`. +#[derive(Copy, Clone)] +struct ProtocolFormatArg(ProtocolFormat); + +impl From for ProtocolFormat { + fn from(arg: ProtocolFormatArg) -> Self { + arg.0 + } +} + +impl ValueEnum for ProtocolFormatArg { + fn value_variants<'a>() -> &'a [Self] { + &[ + ProtocolFormatArg(ProtocolFormat::JsonLegacy), + ProtocolFormatArg(ProtocolFormat::BidirectionalPostcardPrototype), + ] + } + + fn to_possible_value(&self) -> Option { + match self.0 { + ProtocolFormat::JsonLegacy => Some(clap::builder::PossibleValue::new("json-legacy")), + ProtocolFormat::BidirectionalPostcardPrototype => { + Some(clap::builder::PossibleValue::new("bidirectional-postcard-prototype")) + } + } + } + + fn from_str(input: &str, _ignore_case: bool) -> Result { + match input { + "json-legacy" => Ok(ProtocolFormatArg(ProtocolFormat::JsonLegacy)), + "bidirectional-postcard-prototype" => { + Ok(ProtocolFormatArg(ProtocolFormat::BidirectionalPostcardPrototype)) + } + _ => Err(format!("unknown protocol format: {input}")), + } + } +} diff --git a/crates/proc-macro-srv-cli/src/main.rs b/crates/proc-macro-srv-cli/src/main.rs index 926633df628d..46181aceaa89 100644 --- a/crates/proc-macro-srv-cli/src/main.rs +++ b/crates/proc-macro-srv-cli/src/main.rs @@ -2,104 +2,13 @@ //! Driver for proc macro server #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] #![cfg_attr(not(feature = "in-rust-tree"), allow(unused_crate_dependencies))] -#![allow(clippy::print_stdout, clippy::print_stderr)] #[cfg(feature = "in-rust-tree")] extern crate rustc_driver as _; -mod version; - -use clap::{Command, ValueEnum}; -use proc_macro_api::ProtocolFormat; - -#[cfg(feature = "in-rust-tree")] -use proc_macro_srv_cli::main_loop::run; - fn main() -> std::io::Result<()> { - let v = std::env::var("RUST_ANALYZER_INTERNALS_DO_NOT_USE"); - if v.is_err() { - eprintln!( - "This is an IDE implementation detail, you can use this tool by exporting RUST_ANALYZER_INTERNALS_DO_NOT_USE." - ); - eprintln!( - "Note that this tool's API is highly unstable and may break without prior notice" - ); - std::process::exit(122); - } - let matches = Command::new("proc-macro-srv") - .args(&[ - clap::Arg::new("format") - .long("format") - .action(clap::ArgAction::Set) - .default_value("json-legacy") - .value_parser(clap::builder::EnumValueParser::::new()), - clap::Arg::new("version") - .long("version") - .action(clap::ArgAction::SetTrue) - .help("Prints the version of the proc-macro-srv"), - ]) - .get_matches(); - if matches.get_flag("version") { - println!("rust-analyzer-proc-macro-srv {}", version::version()); - return Ok(()); - } - let &format = matches - .get_one::("format") - .expect("format value should always be present"); - - let mut stdin = std::io::BufReader::new(std::io::stdin()); - let mut stdout = std::io::stdout(); - - run(&mut stdin, &mut stdout, format.into()) -} - -/// Wrapper for CLI argument parsing that implements `ValueEnum`. -#[derive(Copy, Clone)] -struct ProtocolFormatArg(ProtocolFormat); - -impl From for ProtocolFormat { - fn from(arg: ProtocolFormatArg) -> Self { - arg.0 - } -} - -impl ValueEnum for ProtocolFormatArg { - fn value_variants<'a>() -> &'a [Self] { - &[ - ProtocolFormatArg(ProtocolFormat::JsonLegacy), - ProtocolFormatArg(ProtocolFormat::BidirectionalPostcardPrototype), - ] - } - - fn to_possible_value(&self) -> Option { - match self.0 { - ProtocolFormat::JsonLegacy => Some(clap::builder::PossibleValue::new("json-legacy")), - ProtocolFormat::BidirectionalPostcardPrototype => { - Some(clap::builder::PossibleValue::new("bidirectional-postcard-prototype")) - } - } - } - - fn from_str(input: &str, _ignore_case: bool) -> Result { - match input { - "json-legacy" => Ok(ProtocolFormatArg(ProtocolFormat::JsonLegacy)), - "bidirectional-postcard-prototype" => { - Ok(ProtocolFormatArg(ProtocolFormat::BidirectionalPostcardPrototype)) - } - _ => Err(format!("unknown protocol format: {input}")), - } + cfg_select! { + feature = "in-rust-tree" => proc_macro_srv_cli::main(), + _ => Ok(()), } } - -#[cfg(not(feature = "in-rust-tree"))] -fn run( - _: &mut std::io::BufReader, - _: &mut std::io::Stdout, - _: ProtocolFormat, -) -> std::io::Result<()> { - Err(std::io::Error::new( - std::io::ErrorKind::Unsupported, - "proc-macro-srv-cli needs to be compiled with the `in-rust-tree` feature to function" - .to_owned(), - )) -} diff --git a/crates/proc-macro-srv-cli/src/main_loop.rs b/crates/proc-macro-srv-cli/src/main_loop.rs index 6697b6380dd2..4b92fccc0922 100644 --- a/crates/proc-macro-srv-cli/src/main_loop.rs +++ b/crates/proc-macro-srv-cli/src/main_loop.rs @@ -1,22 +1,24 @@ //! The main loop of the proc-macro server. -use proc_macro_api::bidirectional_protocol::msg::{ApiVersionCheck, ListMacros}; -use proc_macro_api::{ - ProtocolFormat, bidirectional_protocol::msg as bidirectional, legacy_protocol::msg as legacy, - version::CURRENT_API_VERSION, -}; + use std::panic::{panic_any, resume_unwind}; use std::{ io::{self, BufRead, Write}, ops::Range, }; -use legacy::Message; - +use proc_macro_api::{ + ProtocolFormat, + bidirectional_protocol::msg::{self as bidirectional, ApiVersionCheck, ListMacros}, + flat::{self, SpanTransformer}, + legacy_protocol::msg::{self as legacy, Message}, + version::CURRENT_API_VERSION, +}; use proc_macro_srv::{EnvSnapshot, ProcMacroClientError, ProcMacroPanicMarker, SpanId}; +use span::Span; struct SpanTrans; -impl legacy::SpanTransformer for SpanTrans { +impl SpanTransformer for SpanTrans { type Table = (); type Span = SpanId; fn token_id_of( @@ -138,14 +140,9 @@ fn handle_expand_id( let call_site = SpanId(call_site as u32); let mixed_site = SpanId(mixed_site as u32); - let macro_body = - macro_body.to_tokenstream_unresolved::(CURRENT_API_VERSION, |_, b| b); - let attributes = attributes - .map(|it| it.to_tokenstream_unresolved::(CURRENT_API_VERSION, |_, b| b)); - let mut tracked_env = Default::default(); let res = srv - .expand( + .expand::( lib, &env, current_dir, @@ -156,11 +153,9 @@ fn handle_expand_id( call_site, mixed_site, &mut tracked_env, + &mut (), None, ) - .map(|it| { - legacy::FlatTree::from_tokenstream_raw::(it, call_site, CURRENT_API_VERSION) - }) .map(|tree| bidirectional::ExpandMacroResponse { tree, span_data_table: vec![], @@ -415,27 +410,16 @@ fn handle_expand_ra( }, } = task; - let mut span_data_table = legacy::deserialize_span_data_index_map(&span_data_table); + let mut span_data_table = flat::deserialize_span_data_index_map(&span_data_table); let def_site = span_data_table[def_site]; let call_site = span_data_table[call_site]; let mixed_site = span_data_table[mixed_site]; - let macro_body = - macro_body.to_tokenstream_resolved(CURRENT_API_VERSION, &span_data_table, |a, b| { - srv.join_spans(a, b).unwrap_or(b) - }); - - let attributes = attributes.map(|it| { - it.to_tokenstream_resolved(CURRENT_API_VERSION, &span_data_table, |a, b| { - srv.join_spans(a, b).unwrap_or(b) - }) - }); - let mut tracked_env = Default::default(); let res = srv - .expand( + .expand::( lib, &env, current_dir, @@ -446,19 +430,10 @@ fn handle_expand_ra( call_site, mixed_site, &mut tracked_env, + &mut span_data_table, Some(&mut ProcMacroClientHandle { stdin, stdout, buf }), ) - .map(|it| { - ( - legacy::FlatTree::from_tokenstream( - it, - CURRENT_API_VERSION, - call_site, - &mut span_data_table, - ), - legacy::serialize_span_data_index_map(&span_data_table), - ) - }) + .map(|it| (it, flat::serialize_span_data_index_map(&span_data_table))) .map(|(tree, span_data_table)| bidirectional::ExpandMacroResponse { tree, span_data_table, @@ -521,13 +496,7 @@ fn run_old( let call_site = SpanId(call_site as u32); let mixed_site = SpanId(mixed_site as u32); - let macro_body = macro_body - .to_tokenstream_unresolved::(CURRENT_API_VERSION, |_, b| b); - let attributes = attributes.map(|it| { - it.to_tokenstream_unresolved::(CURRENT_API_VERSION, |_, b| b) - }); - - srv.expand( + srv.expand::( lib, &env, current_dir, @@ -538,39 +507,21 @@ fn run_old( call_site, mixed_site, &mut Default::default(), + &mut (), None, ) - .map(|it| { - legacy::FlatTree::from_tokenstream_raw::( - it, - call_site, - CURRENT_API_VERSION, - ) - }) .map_err(|e| e.into_string().unwrap_or_default()) .map_err(legacy::PanicMessage) }), legacy::SpanMode::RustAnalyzer => legacy::Response::ExpandMacroExtended({ let mut span_data_table = - legacy::deserialize_span_data_index_map(&span_data_table); + flat::deserialize_span_data_index_map(&span_data_table); let def_site = span_data_table[def_site]; let call_site = span_data_table[call_site]; let mixed_site = span_data_table[mixed_site]; - let macro_body = macro_body.to_tokenstream_resolved( - CURRENT_API_VERSION, - &span_data_table, - |a, b| srv.join_spans(a, b).unwrap_or(b), - ); - let attributes = attributes.map(|it| { - it.to_tokenstream_resolved( - CURRENT_API_VERSION, - &span_data_table, - |a, b| srv.join_spans(a, b).unwrap_or(b), - ) - }); - srv.expand( + srv.expand::( lib, &env, current_dir, @@ -581,19 +532,10 @@ fn run_old( call_site, mixed_site, &mut Default::default(), + &mut span_data_table, None, ) - .map(|it| { - ( - legacy::FlatTree::from_tokenstream( - it, - CURRENT_API_VERSION, - call_site, - &mut span_data_table, - ), - legacy::serialize_span_data_index_map(&span_data_table), - ) - }) + .map(|it| (it, flat::serialize_span_data_index_map(&span_data_table))) .map(|(tree, span_data_table)| legacy::ExpandMacroExtended { tree, span_data_table, diff --git a/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs b/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs index 9c55eed08e52..9f4b62daa6ac 100644 --- a/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs +++ b/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs @@ -20,7 +20,8 @@ use proc_macro_api::{ }, reject_subrequests, }, - legacy_protocol::msg::{PanicMessage, ServerConfig, SpanDataIndexMap, SpanMode}, + flat::SpanDataIndexMap, + legacy_protocol::msg::{PanicMessage, ServerConfig, SpanMode}, version::CURRENT_API_VERSION, }; diff --git a/crates/proc-macro-srv-cli/tests/common/utils.rs b/crates/proc-macro-srv-cli/tests/common/utils.rs index b78e10745274..7c7dae39b157 100644 --- a/crates/proc-macro-srv-cli/tests/common/utils.rs +++ b/crates/proc-macro-srv-cli/tests/common/utils.rs @@ -7,11 +7,12 @@ use std::{ use paths::Utf8PathBuf; use proc_macro_api::{ - ServerError, bidirectional_protocol::msg::{ BidirectionalMessage, Request as BiRequest, Response as BiResponse, SubRequest, SubResponse, }, - legacy_protocol::msg::{FlatTree, Message, Request, Response, SpanDataIndexMap}, + client::ServerError, + flat::{FlatTree, SpanDataIndexMap}, + legacy_protocol::msg::{Message, Request, Response}, }; use span::{Edition, EditionedFileId, FileId, Span, SpanAnchor, SyntaxContext, TextRange}; use tt::{Delimiter, DelimiterKind, TopSubtreeBuilder}; diff --git a/crates/proc-macro-srv-cli/tests/legacy_json.rs b/crates/proc-macro-srv-cli/tests/legacy_json.rs index f5cbaa7421eb..3dbe67c595c3 100644 --- a/crates/proc-macro-srv-cli/tests/legacy_json.rs +++ b/crates/proc-macro-srv-cli/tests/legacy_json.rs @@ -18,9 +18,10 @@ use common::utils::{ use expect_test::expect; use proc_macro_api::{ ProtocolFormat::JsonLegacy, + flat::SpanDataIndexMap, legacy_protocol::msg::{ ExpandMacro, ExpandMacroData, ExpnGlobals, PanicMessage, Request, Response, ServerConfig, - SpanDataIndexMap, SpanMode, + SpanMode, }, version::CURRENT_API_VERSION, }; diff --git a/crates/proc-macro-srv/Cargo.toml b/crates/proc-macro-srv/Cargo.toml index 05e0012586d5..68ae00760207 100644 --- a/crates/proc-macro-srv/Cargo.toml +++ b/crates/proc-macro-srv/Cargo.toml @@ -15,8 +15,12 @@ doctest = false [dependencies] paths.workspace = true # span = {workspace = true, default-features = false} does not work -span = { path = "../span", version = "0.0.0", default-features = false} +span = { path = "../span", version = "0.0.0", default-features = false } intern.workspace = true +tt = { path = "../tt", version = "0.0.0", default-features = false } +proc-macro-api = { path = "../proc-macro-api", version = "0.0.0", default-features = false, features = [ + "in-proc-macro-srv", +] } stdx.workspace = true [dev-dependencies] @@ -28,10 +32,10 @@ proc-macro-test.path = "./proc-macro-test" [features] default = [] -in-rust-tree = [] +in-rust-tree = ["tt/in-rust-tree", "proc-macro-api/in-rust-tree"] [lints] workspace = true [package.metadata.rust-analyzer] -rustc_private=true +rustc_private = true diff --git a/crates/proc-macro-srv/src/bridge.rs b/crates/proc-macro-srv/src/bridge.rs deleted file mode 100644 index fc62f9413a34..000000000000 --- a/crates/proc-macro-srv/src/bridge.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! `proc_macro::bridge` newtypes. - -use rustc_proc_macro::bridge as pm_bridge; - -pub use pm_bridge::{DelimSpan, Diagnostic, ExpnGlobals, LitKind}; - -pub type TokenTree = - pm_bridge::TokenTree, S, intern::Symbol>; -pub type Literal = pm_bridge::Literal; -pub type Group = pm_bridge::Group, S>; -pub type Punct = pm_bridge::Punct; -pub type Ident = pm_bridge::Ident; diff --git a/crates/proc-macro-srv/src/dylib.rs b/crates/proc-macro-srv/src/dylib.rs index 2a5e79d9e5cc..8da34f63688b 100644 --- a/crates/proc-macro-srv/src/dylib.rs +++ b/crates/proc-macro-srv/src/dylib.rs @@ -3,6 +3,7 @@ mod proc_macros; use paths::{Utf8Path, Utf8PathBuf}; +use proc_macro_api::token_stream::TokenStream; use rustc_codegen_ssa::back::metadata::DefaultMetadataLoader; use rustc_interface::util::rustc_version_str; use rustc_proc_macro::bridge; @@ -11,7 +12,7 @@ use stdx::tempfile::NamedTempFile; use crate::{ PanicMessage, ProcMacroClientHandle, ProcMacroKind, ProcMacroSrvSpan, TrackedEnv, - dylib::proc_macros::ProcMacros, token_stream::TokenStream, + dylib::proc_macros::ProcMacros, }; pub(crate) struct Expander { diff --git a/crates/proc-macro-srv/src/dylib/proc_macros.rs b/crates/proc-macro-srv/src/dylib/proc_macros.rs index 4976298d5fcd..5324a02a54ac 100644 --- a/crates/proc-macro-srv/src/dylib/proc_macros.rs +++ b/crates/proc-macro-srv/src/dylib/proc_macros.rs @@ -1,9 +1,9 @@ //! Proc macro ABI -use crate::{ - ProcMacroClientHandle, ProcMacroKind, ProcMacroSrvSpan, TrackedEnv, token_stream::TokenStream, -}; +use proc_macro_api::token_stream::TokenStream; use rustc_proc_macro::bridge; +use crate::{ProcMacroClientHandle, ProcMacroKind, ProcMacroSrvSpan, TrackedEnv}; + impl From for crate::PanicMessage { fn from(p: bridge::PanicMessage) -> Self { Self { message: p.into_string() } diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs index 7fc04a05155f..6fb4e618e8f8 100644 --- a/crates/proc-macro-srv/src/lib.rs +++ b/crates/proc-macro-srv/src/lib.rs @@ -18,15 +18,12 @@ extern crate rustc_codegen_ssa; extern crate rustc_driver as _; extern crate rustc_interface; -extern crate rustc_lexer; extern crate rustc_metadata; extern crate rustc_proc_macro; extern crate rustc_span; -mod bridge; mod dylib; mod server_impl; -mod token_stream; use std::{ collections::{HashMap, HashSet, hash_map::Entry}, @@ -40,16 +37,19 @@ use std::{ }; use paths::{Utf8Path, Utf8PathBuf}; +use proc_macro_api::{ + flat::{FlatTree, SpanTransformer}, + token_stream::SpanLike, + version::CURRENT_API_VERSION, +}; use span::{FIXUP_ERASED_FILE_AST_ID_MARKER, Span}; -pub use crate::server_impl::token_id::SpanId; - pub use rustc_proc_macro::Delimiter; pub use span; -pub use crate::bridge::*; -pub use crate::server_impl::literal_from_str; -pub use crate::token_stream::{TokenStream, TokenStreamIter, literal_to_string}; +pub use tt::literal_from_str; + +pub use crate::server_impl::token_id::SpanId; #[derive(Copy, Clone, Eq, PartialEq, Debug)] pub enum ProcMacroKind { @@ -127,20 +127,26 @@ impl ExpandError { } impl ProcMacroSrv<'_> { - pub fn expand<'a, S: ProcMacroSrvSpan + 'a>( + pub fn expand<'a, ST>( &self, lib: impl AsRef, env: &[(String, String)], current_dir: Option>, macro_name: &str, - macro_body: token_stream::TokenStream, - attribute: Option>, - def_site: S, - call_site: S, - mixed_site: S, + macro_body: FlatTree, + attribute: Option, + def_site: ST::Span, + call_site: ST::Span, + mixed_site: ST::Span, tracked_env: &'a mut TrackedEnv, + span_data_table: &mut ST::Table, callback: Option>, - ) -> Result, ExpandError> { + ) -> Result + where + ST: SpanTransformer, + ST::Span: ProcMacroSrvSpan + SpanLike + 'a, + ST::Table: Send, + { let snapped_env = self.env; let expander = self.expander(lib.as_ref()).map_err(|err| ExpandError::Internal { reason: Some(format!("failed to load macro: {err}")), @@ -155,16 +161,29 @@ impl ProcMacroSrv<'_> { .stack_size(EXPANDER_STACK_SIZE) .name(macro_name.to_owned()) .spawn_scoped(s, move || { - expander.expand( - macro_name, - macro_body, - attribute, - def_site, - call_site, - mixed_site, - tracked_env, - callback, - ) + let macro_body = + macro_body.to_tokenstream::(CURRENT_API_VERSION, span_data_table); + let attribute = attribute + .map(|it| it.to_tokenstream::(CURRENT_API_VERSION, span_data_table)); + expander + .expand( + macro_name, + macro_body, + attribute, + def_site, + call_site, + mixed_site, + tracked_env, + callback, + ) + .map(|result| { + FlatTree::from_tokenstream::( + result, + call_site, + CURRENT_API_VERSION, + span_data_table, + ) + }) }); match thread.unwrap().join() { Ok(res) => res.map_err(ExpandError::Panic), @@ -232,7 +251,7 @@ pub struct TrackedEnv { pub trait ProcMacroSrvSpan: Copy + Send + Sync { type Server<'a>: rustc_proc_macro::bridge::server::Server< - TokenStream = crate::token_stream::TokenStream, + TokenStream = proc_macro_api::token_stream::TokenStream, >; fn make_server<'a>( call_site: Self, diff --git a/crates/proc-macro-srv/src/server_impl.rs b/crates/proc-macro-srv/src/server_impl.rs index bacead1a88da..aaf8f540d9f4 100644 --- a/crates/proc-macro-srv/src/server_impl.rs +++ b/crates/proc-macro-srv/src/server_impl.rs @@ -6,34 +6,6 @@ //! The original idea from fedochet is using proc-macro2 as backend, //! we use tt instead for better integration with RA. +mod bridge; pub(crate) mod rust_analyzer_span; pub(crate) mod token_id; - -pub fn literal_from_str( - s: &str, - span: Span, -) -> Result, ()> { - use rustc_lexer::{LiteralKind, Token, TokenKind}; - let mut tokens = rustc_lexer::tokenize(s, rustc_lexer::FrontmatterAllowed::No); - let minus_or_lit = tokens.next().unwrap_or(Token { kind: TokenKind::Eof, len: 0 }); - - let lit = if minus_or_lit.kind == TokenKind::Minus { - let lit = tokens.next().ok_or(())?; - if !matches!( - lit.kind, - TokenKind::Literal { kind: LiteralKind::Int { .. } | LiteralKind::Float { .. }, .. } - ) { - return Err(()); - } - lit - } else { - minus_or_lit - }; - - if tokens.next().is_some() { - return Err(()); - } - - let TokenKind::Literal { kind, suffix_start } = lit.kind else { return Err(()) }; - Ok(crate::token_stream::literal_from_lexer(s, span, kind, suffix_start)) -} diff --git a/crates/proc-macro-srv/src/server_impl/bridge.rs b/crates/proc-macro-srv/src/server_impl/bridge.rs new file mode 100644 index 000000000000..34b0a8547911 --- /dev/null +++ b/crates/proc-macro-srv/src/server_impl/bridge.rs @@ -0,0 +1,160 @@ +//! Conversions between proc_macro bridge types and tt types. + +use proc_macro_api::token_stream::SpanLike; + +pub(super) mod ours { + pub(crate) type Literal = tt::Literal; + pub(crate) type Punct = tt::Punct; + pub(crate) type Ident = tt::Ident; + pub(crate) type Leaf = tt::Leaf; + pub(crate) type Group = proc_macro_api::token_stream::Group; + pub(crate) type TokenTree = proc_macro_api::token_stream::TokenTree; + pub(crate) type TokenStream = proc_macro_api::token_stream::TokenStream; +} + +#[expect(clippy::module_inception, reason = "this is not a mistake")] +pub(super) mod bridge { + use rustc_proc_macro::bridge as pm_bridge; + + use super::ours; + + pub(crate) use pm_bridge::*; + + pub(crate) type TokenTree = + pm_bridge::TokenTree, Span, intern::Symbol>; + pub(crate) type Literal = pm_bridge::Literal; + pub(crate) type Group = pm_bridge::Group, Span>; + pub(crate) type Punct = pm_bridge::Punct; + pub(crate) type Ident = pm_bridge::Ident; +} + +pub(super) fn literal_into_bridge(literal: ours::Literal) -> bridge::Literal { + let kind = match literal.kind { + tt::LitKind::Byte => bridge::LitKind::Byte, + tt::LitKind::Char => bridge::LitKind::Char, + tt::LitKind::Integer => bridge::LitKind::Integer, + tt::LitKind::Float => bridge::LitKind::Float, + tt::LitKind::Str => bridge::LitKind::Str, + tt::LitKind::StrRaw(count) => bridge::LitKind::StrRaw(count), + tt::LitKind::ByteStr => bridge::LitKind::ByteStr, + tt::LitKind::ByteStrRaw(count) => bridge::LitKind::ByteStrRaw(count), + tt::LitKind::CStr => bridge::LitKind::CStr, + tt::LitKind::CStrRaw(count) => bridge::LitKind::CStrRaw(count), + tt::LitKind::Err(()) => bridge::LitKind::ErrWithGuar, + }; + let (symbol, suffix) = literal.text_and_suffix_symbols(); + bridge::Literal { kind, symbol, suffix, span: literal.span } +} + +pub(super) fn punct_into_bridge(punct: ours::Punct) -> bridge::Punct { + bridge::Punct { + // FIXME: Is `as u8` correct here? + ch: punct.char as u8, + joint: punct.spacing != tt::Spacing::Alone, + span: punct.span, + } +} + +pub(super) fn ident_into_bridge(ident: ours::Ident) -> bridge::Ident { + bridge::Ident { sym: ident.sym, is_raw: ident.is_raw.yes(), span: ident.span } +} + +pub(super) fn group_into_bridge(group: ours::Group) -> bridge::Group { + let delimiter = match group.delimiter.kind { + tt::DelimiterKind::Parenthesis => rustc_proc_macro::Delimiter::Parenthesis, + tt::DelimiterKind::Brace => rustc_proc_macro::Delimiter::Brace, + tt::DelimiterKind::Bracket => rustc_proc_macro::Delimiter::Bracket, + tt::DelimiterKind::Invisible => rustc_proc_macro::Delimiter::None, + }; + let span = bridge::DelimSpan { + open: group.delimiter.open, + close: group.delimiter.close, + entire: group.delimiter.open.cover(group.delimiter.close), + }; + bridge::Group { delimiter, stream: group.stream, span } +} + +pub(super) fn token_tree_into_bridge( + token_tree: ours::TokenTree, +) -> bridge::TokenTree { + match token_tree { + ours::TokenTree::Leaf(ours::Leaf::Literal(literal)) => { + bridge::TokenTree::Literal(literal_into_bridge(literal)) + } + ours::TokenTree::Leaf(ours::Leaf::Ident(ident)) => { + bridge::TokenTree::Ident(ident_into_bridge(ident)) + } + ours::TokenTree::Leaf(ours::Leaf::Punct(punct)) => { + bridge::TokenTree::Punct(punct_into_bridge(punct)) + } + ours::TokenTree::Group(group) => bridge::TokenTree::Group(group_into_bridge(group)), + } +} + +pub(super) fn literal_from_bridge(literal: bridge::Literal) -> ours::Literal { + let kind = match literal.kind { + bridge::LitKind::Byte => tt::LitKind::Byte, + bridge::LitKind::Char => tt::LitKind::Char, + bridge::LitKind::Integer => tt::LitKind::Integer, + bridge::LitKind::Float => tt::LitKind::Float, + bridge::LitKind::Str => tt::LitKind::Str, + bridge::LitKind::StrRaw(count) => tt::LitKind::StrRaw(count), + bridge::LitKind::ByteStr => tt::LitKind::ByteStr, + bridge::LitKind::ByteStrRaw(count) => tt::LitKind::ByteStrRaw(count), + bridge::LitKind::CStr => tt::LitKind::CStr, + bridge::LitKind::CStrRaw(count) => tt::LitKind::CStrRaw(count), + bridge::LitKind::ErrWithGuar => tt::LitKind::Err(()), + }; + match literal.suffix { + Some(suffix) => { + tt::Literal::new(literal.symbol.as_str(), literal.span, kind, suffix.as_str()) + } + None => { + tt::Literal { text_and_suffix: literal.symbol, span: literal.span, kind, suffix_len: 0 } + } + } +} + +pub(super) fn punct_from_bridge(punct: bridge::Punct) -> ours::Punct { + ours::Punct { + char: char::from(punct.ch), + spacing: if punct.joint { tt::Spacing::Joint } else { tt::Spacing::Alone }, + span: punct.span, + } +} + +pub(super) fn ident_from_bridge(ident: bridge::Ident) -> ours::Ident { + ours::Ident { + sym: ident.sym, + is_raw: if ident.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No }, + span: ident.span, + } +} + +pub(super) fn group_from_bridge(group: bridge::Group) -> ours::Group { + let kind = match group.delimiter { + rustc_proc_macro::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis, + rustc_proc_macro::Delimiter::Brace => tt::DelimiterKind::Brace, + rustc_proc_macro::Delimiter::Bracket => tt::DelimiterKind::Bracket, + rustc_proc_macro::Delimiter::None => tt::DelimiterKind::Invisible, + }; + let delimiter = tt::Delimiter { open: group.span.open, close: group.span.close, kind }; + ours::Group { delimiter, stream: group.stream } +} + +pub(super) fn token_tree_from_bridge( + token_tree: bridge::TokenTree, +) -> ours::TokenTree { + match token_tree { + bridge::TokenTree::Literal(literal) => { + ours::TokenTree::Leaf(ours::Leaf::Literal(literal_from_bridge(literal))) + } + bridge::TokenTree::Ident(ident) => { + ours::TokenTree::Leaf(ours::Leaf::Ident(ident_from_bridge(ident))) + } + bridge::TokenTree::Punct(punct) => { + ours::TokenTree::Leaf(ours::Leaf::Punct(punct_from_bridge(punct))) + } + bridge::TokenTree::Group(group) => ours::TokenTree::Group(group_from_bridge(group)), + } +} diff --git a/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs b/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs index bdac9c920c43..ec8882e5b2b1 100644 --- a/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs +++ b/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs @@ -9,13 +9,28 @@ use std::ops::{Bound, Range}; use intern::Symbol; use rustc_proc_macro::bridge::server; use span::{ErasedFileAstId, Span, TextRange, TextSize}; +use tt::literal_from_str; use crate::{ ProcMacroClientHandle, TrackedEnv, - bridge::{Diagnostic, ExpnGlobals, Literal, TokenTree}, - server_impl::literal_from_str, + server_impl::bridge::{literal_into_bridge, token_tree_from_bridge, token_tree_into_bridge}, }; +mod ours { + use span::Span; + + pub(super) type TokenStream = crate::server_impl::bridge::ours::TokenStream; +} + +mod bridge { + use span::Span; + + pub(super) use crate::server_impl::bridge::bridge::*; + + pub(super) type TokenTree = crate::server_impl::bridge::bridge::TokenTree; + pub(super) type Literal = crate::server_impl::bridge::bridge::Literal; +} + pub struct RaSpanServer<'a> { pub tracked_env: &'a mut TrackedEnv, pub call_site: Span, @@ -26,12 +41,12 @@ pub struct RaSpanServer<'a> { } impl server::Server for RaSpanServer<'_> { - type TokenStream = crate::token_stream::TokenStream; + type TokenStream = ours::TokenStream; type Span = Span; type Symbol = Symbol; - fn globals(&mut self) -> ExpnGlobals { - ExpnGlobals { + fn globals(&mut self) -> bridge::ExpnGlobals { + bridge::ExpnGlobals { def_site: self.def_site, call_site: self.call_site, mixed_site: self.mixed_site, @@ -57,12 +72,13 @@ impl server::Server for RaSpanServer<'_> { self.tracked_env.paths.insert(path.into()); } - fn literal_from_str(&mut self, s: &str) -> Result, String> { + fn literal_from_str(&mut self, s: &str) -> Result { literal_from_str(s, self.call_site) - .map_err(|()| "cannot parse string into literal".to_string()) + .map(literal_into_bridge) + .map_err(|()| "cannot parse string into literal".to_owned()) } - fn emit_diagnostic(&mut self, _: Diagnostic) { + fn emit_diagnostic(&mut self, _: bridge::Diagnostic) { // FIXME handle diagnostic } @@ -85,8 +101,8 @@ impl server::Server for RaSpanServer<'_> { stream.to_string() } - fn ts_from_token_tree(&mut self, tree: TokenTree) -> Self::TokenStream { - Self::TokenStream::new(vec![tree]) + fn ts_from_token_tree(&mut self, tree: bridge::TokenTree) -> Self::TokenStream { + ours::TokenStream::new(vec![token_tree_from_bridge(tree)]) } fn ts_expand_expr(&mut self, self_: &Self::TokenStream) -> Result { @@ -100,17 +116,16 @@ impl server::Server for RaSpanServer<'_> { fn ts_concat_trees( &mut self, - base: Option, - trees: Vec>, + base: Option, + trees: Vec, ) -> Self::TokenStream { + let trees = trees.into_iter().map(token_tree_from_bridge); match base { Some(mut base) => { - for tt in trees { - base.push_tree(tt); - } + base.extend(trees); base } - None => Self::TokenStream::new(trees), + None => trees.collect(), } } @@ -119,15 +134,14 @@ impl server::Server for RaSpanServer<'_> { base: Option, streams: Vec, ) -> Self::TokenStream { - let mut stream = base.unwrap_or_default(); - for s in streams { - stream.push_stream(s); - } + let mut streams = streams.into_iter(); + let mut stream = base.or_else(|| streams.next()).unwrap_or_default(); + stream.extend_with_streams(streams); stream } - fn ts_into_trees(&mut self, stream: Self::TokenStream) -> Vec> { - (*stream.0).clone() + fn ts_into_trees(&mut self, stream: ours::TokenStream) -> Vec { + stream.iter().cloned().map(token_tree_into_bridge).collect() } fn span_debug(&mut self, span: Self::Span) -> String { diff --git a/crates/proc-macro-srv/src/server_impl/token_id.rs b/crates/proc-macro-srv/src/server_impl/token_id.rs index 6c393b8befb1..266809be985a 100644 --- a/crates/proc-macro-srv/src/server_impl/token_id.rs +++ b/crates/proc-macro-srv/src/server_impl/token_id.rs @@ -4,23 +4,48 @@ use std::ops::{Bound, Range}; use intern::Symbol; use rustc_proc_macro::bridge::server; +use tt::literal_from_str; use crate::{ ProcMacroClientHandle, - bridge::{Diagnostic, ExpnGlobals, Literal, TokenTree}, - server_impl::literal_from_str, + server_impl::bridge::{literal_into_bridge, token_tree_from_bridge, token_tree_into_bridge}, }; #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct SpanId(pub u32); +impl proc_macro_api::token_stream::SpanLike for crate::SpanId { + fn derive_ranged(&self, _: std::ops::Range) -> Self { + *self + } + + fn cover(self, _other: Self) -> Self { + self + } +} + impl std::fmt::Debug for SpanId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.0.fmt(f) } } -type Span = SpanId; +use SpanId as Span; + +mod ours { + use super::Span; + + pub(super) type TokenStream = crate::server_impl::bridge::ours::TokenStream; +} + +mod bridge { + use super::Span; + + pub(super) use crate::server_impl::bridge::bridge::*; + + pub(super) type TokenTree = crate::server_impl::bridge::bridge::TokenTree; + pub(super) type Literal = crate::server_impl::bridge::bridge::Literal; +} pub struct SpanIdServer<'a> { pub call_site: Span, @@ -30,12 +55,12 @@ pub struct SpanIdServer<'a> { } impl server::Server for SpanIdServer<'_> { - type TokenStream = crate::token_stream::TokenStream; + type TokenStream = ours::TokenStream; type Span = Span; type Symbol = Symbol; - fn globals(&mut self) -> ExpnGlobals { - ExpnGlobals { + fn globals(&mut self) -> bridge::ExpnGlobals { + bridge::ExpnGlobals { def_site: self.def_site, call_site: self.call_site, mixed_site: self.mixed_site, @@ -57,12 +82,13 @@ impl server::Server for SpanIdServer<'_> { fn track_path(&mut self, _: &str) {} - fn literal_from_str(&mut self, s: &str) -> Result, String> { + fn literal_from_str(&mut self, s: &str) -> Result { literal_from_str(s, self.call_site) - .map_err(|()| "cannot parse string into literal".to_string()) + .map(literal_into_bridge) + .map_err(|()| "cannot parse string into literal".to_owned()) } - fn emit_diagnostic(&mut self, _: Diagnostic) {} + fn emit_diagnostic(&mut self, _: bridge::Diagnostic) {} fn ts_drop(&mut self, stream: Self::TokenStream) { drop(stream); @@ -82,8 +108,8 @@ impl server::Server for SpanIdServer<'_> { fn ts_to_string(&mut self, stream: &Self::TokenStream) -> String { stream.to_string() } - fn ts_from_token_tree(&mut self, tree: TokenTree) -> Self::TokenStream { - Self::TokenStream::new(vec![tree]) + fn ts_from_token_tree(&mut self, tree: bridge::TokenTree) -> Self::TokenStream { + Self::TokenStream::new(vec![token_tree_from_bridge(tree)]) } fn ts_expand_expr(&mut self, self_: &Self::TokenStream) -> Result { @@ -93,16 +119,15 @@ impl server::Server for SpanIdServer<'_> { fn ts_concat_trees( &mut self, base: Option, - trees: Vec>, + trees: Vec, ) -> Self::TokenStream { + let trees = trees.into_iter().map(token_tree_from_bridge); match base { Some(mut base) => { - for tt in trees { - base.push_tree(tt); - } + base.extend(trees); base } - None => Self::TokenStream::new(trees), + None => trees.collect(), } } @@ -111,15 +136,14 @@ impl server::Server for SpanIdServer<'_> { base: Option, streams: Vec, ) -> Self::TokenStream { - let mut stream = base.unwrap_or_default(); - for s in streams { - stream.push_stream(s); - } + let mut streams = streams.into_iter(); + let mut stream = base.or_else(|| streams.next()).unwrap_or_default(); + stream.extend_with_streams(streams); stream } - fn ts_into_trees(&mut self, stream: Self::TokenStream) -> Vec> { - (*stream.0).clone() + fn ts_into_trees(&mut self, stream: ours::TokenStream) -> Vec { + stream.iter().cloned().map(token_tree_into_bridge).collect() } fn span_debug(&mut self, span: Self::Span) -> String { diff --git a/crates/proc-macro-srv/src/tests/mod.rs b/crates/proc-macro-srv/src/tests/mod.rs index 2cb6817fb916..d3f3cdd57bbb 100644 --- a/crates/proc-macro-srv/src/tests/mod.rs +++ b/crates/proc-macro-srv/src/tests/mod.rs @@ -12,42 +12,42 @@ fn test_derive_empty() { "DeriveEmpty", r#"struct S { field: &'r#lt fn(u32) -> &'a r#u32 }"#, expect![[r#" - IDENT 1 struct - IDENT 1 S - GROUP {} 1 1 1 - IDENT 1 field - PUNCT 1 : [alone] - PUNCT 1 & [joint] - PUNCT 1 ' [joint] - IDENT 1 r#lt - IDENT 1 fn - GROUP () 1 1 1 - IDENT 1 u32 - PUNCT 1 - [joint] - PUNCT 1 > [alone] - PUNCT 1 & [joint] - PUNCT 1 ' [joint] - IDENT 1 a - IDENT 1 r#u32 + IDENT struct 1 + IDENT S 1 + GROUP {} 1 1 + IDENT field 1 + PUNCT : [alone] 1 + PUNCT & [joint] 1 + PUNCT ' [joint] 1 + IDENT r#lt 1 + IDENT fn 1 + GROUP () 1 1 + IDENT u32 1 + PUNCT - [joint] 1 + PUNCT > [alone] 1 + PUNCT & [joint] 1 + PUNCT ' [joint] 1 + IDENT a 1 + IDENT r#u32 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..6#0 struct - IDENT 42:Root[0000, 0]@7..8#0 S - GROUP {} 42:Root[0000, 0]@9..10#0 42:Root[0000, 0]@46..47#0 42:Root[0000, 0]@9..47#0 - IDENT 42:Root[0000, 0]@11..16#0 field - PUNCT 42:Root[0000, 0]@16..17#0 : [alone] - PUNCT 42:Root[0000, 0]@18..19#0 & [joint] - PUNCT 42:Root[0000, 0]@22..23#0 ' [joint] - IDENT 42:Root[0000, 0]@22..24#0 r#lt - IDENT 42:Root[0000, 0]@25..27#0 fn - GROUP () 42:Root[0000, 0]@27..28#0 42:Root[0000, 0]@31..32#0 42:Root[0000, 0]@27..32#0 - IDENT 42:Root[0000, 0]@28..31#0 u32 - PUNCT 42:Root[0000, 0]@33..34#0 - [joint] - PUNCT 42:Root[0000, 0]@34..35#0 > [alone] - PUNCT 42:Root[0000, 0]@36..37#0 & [joint] - PUNCT 42:Root[0000, 0]@38..39#0 ' [joint] - IDENT 42:Root[0000, 0]@38..39#0 a - IDENT 42:Root[0000, 0]@42..45#0 r#u32 + IDENT struct 42:Root[0000, 0]@0..6#0 + IDENT S 42:Root[0000, 0]@7..8#0 + GROUP {} 42:Root[0000, 0]@9..10#0 42:Root[0000, 0]@46..47#0 + IDENT field 42:Root[0000, 0]@11..16#0 + PUNCT : [alone] 42:Root[0000, 0]@16..17#0 + PUNCT & [joint] 42:Root[0000, 0]@18..19#0 + PUNCT ' [joint] 42:Root[0000, 0]@22..23#0 + IDENT r#lt 42:Root[0000, 0]@22..24#0 + IDENT fn 42:Root[0000, 0]@25..27#0 + GROUP () 42:Root[0000, 0]@27..28#0 42:Root[0000, 0]@31..32#0 + IDENT u32 42:Root[0000, 0]@28..31#0 + PUNCT - [joint] 42:Root[0000, 0]@33..34#0 + PUNCT > [alone] 42:Root[0000, 0]@34..35#0 + PUNCT & [joint] 42:Root[0000, 0]@36..37#0 + PUNCT ' [joint] 42:Root[0000, 0]@38..39#0 + IDENT a 42:Root[0000, 0]@38..39#0 + IDENT r#u32 42:Root[0000, 0]@42..45#0 "#]], ); } @@ -65,148 +65,148 @@ pub struct Foo { } "#, expect![[r#" - PUNCT 1 # [joint] - GROUP [] 1 1 1 - IDENT 1 helper - GROUP () 1 1 1 - IDENT 1 build_fn - GROUP () 1 1 1 - IDENT 1 private - PUNCT 1 , [alone] - IDENT 1 name - PUNCT 1 = [alone] - LITER 1 Str partial_build - IDENT 1 pub - IDENT 1 struct - IDENT 1 Foo - GROUP {} 1 1 1 - PUNCT 1 # [alone] - GROUP [] 1 1 1 - IDENT 1 doc - PUNCT 1 = [alone] - LITER 1 Str The domain where this federated instance is running - PUNCT 1 # [joint] - GROUP [] 1 1 1 - IDENT 1 helper - GROUP () 1 1 1 - IDENT 1 setter - GROUP () 1 1 1 - IDENT 1 into - IDENT 1 pub - GROUP () 1 1 1 - IDENT 1 crate - IDENT 1 domain - PUNCT 1 : [alone] - IDENT 1 String - PUNCT 1 , [alone] - - - PUNCT 1 # [joint] - GROUP [] 1 1 1 - IDENT 1 helper - GROUP () 1 1 1 - IDENT 1 build_fn - GROUP () 1 1 1 - IDENT 1 private - PUNCT 1 , [alone] - IDENT 1 name - PUNCT 1 = [alone] - LITER 1 Str partial_build - IDENT 1 pub - IDENT 1 struct - IDENT 1 Foo - GROUP {} 1 1 1 - PUNCT 1 # [alone] - GROUP [] 1 1 1 - IDENT 1 doc - PUNCT 1 = [alone] - LITER 1 Str The domain where this federated instance is running - PUNCT 1 # [joint] - GROUP [] 1 1 1 - IDENT 1 helper - GROUP () 1 1 1 - IDENT 1 setter - GROUP () 1 1 1 - IDENT 1 into - IDENT 1 pub - GROUP () 1 1 1 - IDENT 1 crate - IDENT 1 domain - PUNCT 1 : [alone] - IDENT 1 String - PUNCT 1 , [alone] + PUNCT # [joint] 1 + GROUP [] 1 1 + IDENT helper 1 + GROUP () 1 1 + IDENT build_fn 1 + GROUP () 1 1 + IDENT private 1 + PUNCT , [alone] 1 + IDENT name 1 + PUNCT = [alone] 1 + LITERAL Str partial_build 1 + IDENT pub 1 + IDENT struct 1 + IDENT Foo 1 + GROUP {} 1 1 + PUNCT # [alone] 1 + GROUP [] 1 1 + IDENT doc 1 + PUNCT = [alone] 1 + LITERAL Str The domain where this federated instance is running 1 + PUNCT # [joint] 1 + GROUP [] 1 1 + IDENT helper 1 + GROUP () 1 1 + IDENT setter 1 + GROUP () 1 1 + IDENT into 1 + IDENT pub 1 + GROUP () 1 1 + IDENT crate 1 + IDENT domain 1 + PUNCT : [alone] 1 + IDENT String 1 + PUNCT , [alone] 1 + + + PUNCT # [joint] 1 + GROUP [] 1 1 + IDENT helper 1 + GROUP () 1 1 + IDENT build_fn 1 + GROUP () 1 1 + IDENT private 1 + PUNCT , [alone] 1 + IDENT name 1 + PUNCT = [alone] 1 + LITERAL Str partial_build 1 + IDENT pub 1 + IDENT struct 1 + IDENT Foo 1 + GROUP {} 1 1 + PUNCT # [alone] 1 + GROUP [] 1 1 + IDENT doc 1 + PUNCT = [alone] 1 + LITERAL Str The domain where this federated instance is running 1 + PUNCT # [joint] 1 + GROUP [] 1 1 + IDENT helper 1 + GROUP () 1 1 + IDENT setter 1 + GROUP () 1 1 + IDENT into 1 + IDENT pub 1 + GROUP () 1 1 + IDENT crate 1 + IDENT domain 1 + PUNCT : [alone] 1 + IDENT String 1 + PUNCT , [alone] 1 "#]], expect![[r#" - PUNCT 42:Root[0000, 0]@1..2#0 # [joint] - GROUP [] 42:Root[0000, 0]@2..3#0 42:Root[0000, 0]@52..53#0 42:Root[0000, 0]@2..53#0 - IDENT 42:Root[0000, 0]@3..9#0 helper - GROUP () 42:Root[0000, 0]@9..10#0 42:Root[0000, 0]@51..52#0 42:Root[0000, 0]@9..52#0 - IDENT 42:Root[0000, 0]@10..18#0 build_fn - GROUP () 42:Root[0000, 0]@18..19#0 42:Root[0000, 0]@50..51#0 42:Root[0000, 0]@18..51#0 - IDENT 42:Root[0000, 0]@19..26#0 private - PUNCT 42:Root[0000, 0]@26..27#0 , [alone] - IDENT 42:Root[0000, 0]@28..32#0 name - PUNCT 42:Root[0000, 0]@33..34#0 = [alone] - LITER 42:Root[0000, 0]@35..50#0 Str partial_build - IDENT 42:Root[0000, 0]@54..57#0 pub - IDENT 42:Root[0000, 0]@58..64#0 struct - IDENT 42:Root[0000, 0]@65..68#0 Foo - GROUP {} 42:Root[0000, 0]@69..70#0 42:Root[0000, 0]@190..191#0 42:Root[0000, 0]@69..191#0 - PUNCT 42:Root[0000, 0]@0..0#0 # [alone] - GROUP [] 42:Root[0000, 0]@0..0#0 42:Root[0000, 0]@0..0#0 42:Root[0000, 0]@0..0#0 - IDENT 42:Root[0000, 0]@0..0#0 doc - PUNCT 42:Root[0000, 0]@0..0#0 = [alone] - LITER 42:Root[0000, 0]@75..130#0 Str The domain where this federated instance is running - PUNCT 42:Root[0000, 0]@135..136#0 # [joint] - GROUP [] 42:Root[0000, 0]@136..137#0 42:Root[0000, 0]@157..158#0 42:Root[0000, 0]@136..158#0 - IDENT 42:Root[0000, 0]@137..143#0 helper - GROUP () 42:Root[0000, 0]@143..144#0 42:Root[0000, 0]@156..157#0 42:Root[0000, 0]@143..157#0 - IDENT 42:Root[0000, 0]@144..150#0 setter - GROUP () 42:Root[0000, 0]@150..151#0 42:Root[0000, 0]@155..156#0 42:Root[0000, 0]@150..156#0 - IDENT 42:Root[0000, 0]@151..155#0 into - IDENT 42:Root[0000, 0]@163..166#0 pub - GROUP () 42:Root[0000, 0]@166..167#0 42:Root[0000, 0]@172..173#0 42:Root[0000, 0]@166..173#0 - IDENT 42:Root[0000, 0]@167..172#0 crate - IDENT 42:Root[0000, 0]@174..180#0 domain - PUNCT 42:Root[0000, 0]@180..181#0 : [alone] - IDENT 42:Root[0000, 0]@182..188#0 String - PUNCT 42:Root[0000, 0]@188..189#0 , [alone] - - - PUNCT 42:Root[0000, 0]@1..2#0 # [joint] - GROUP [] 42:Root[0000, 0]@2..3#0 42:Root[0000, 0]@52..53#0 42:Root[0000, 0]@2..53#0 - IDENT 42:Root[0000, 0]@3..9#0 helper - GROUP () 42:Root[0000, 0]@9..10#0 42:Root[0000, 0]@51..52#0 42:Root[0000, 0]@9..52#0 - IDENT 42:Root[0000, 0]@10..18#0 build_fn - GROUP () 42:Root[0000, 0]@18..19#0 42:Root[0000, 0]@50..51#0 42:Root[0000, 0]@18..51#0 - IDENT 42:Root[0000, 0]@19..26#0 private - PUNCT 42:Root[0000, 0]@26..27#0 , [alone] - IDENT 42:Root[0000, 0]@28..32#0 name - PUNCT 42:Root[0000, 0]@33..34#0 = [alone] - LITER 42:Root[0000, 0]@35..50#0 Str partial_build - IDENT 42:Root[0000, 0]@54..57#0 pub - IDENT 42:Root[0000, 0]@58..64#0 struct - IDENT 42:Root[0000, 0]@65..68#0 Foo - GROUP {} 42:Root[0000, 0]@69..70#0 42:Root[0000, 0]@190..191#0 42:Root[0000, 0]@69..191#0 - PUNCT 42:Root[0000, 0]@0..0#0 # [alone] - GROUP [] 42:Root[0000, 0]@0..0#0 42:Root[0000, 0]@0..0#0 42:Root[0000, 0]@0..0#0 - IDENT 42:Root[0000, 0]@0..0#0 doc - PUNCT 42:Root[0000, 0]@0..0#0 = [alone] - LITER 42:Root[0000, 0]@75..130#0 Str The domain where this federated instance is running - PUNCT 42:Root[0000, 0]@135..136#0 # [joint] - GROUP [] 42:Root[0000, 0]@136..137#0 42:Root[0000, 0]@157..158#0 42:Root[0000, 0]@136..158#0 - IDENT 42:Root[0000, 0]@137..143#0 helper - GROUP () 42:Root[0000, 0]@143..144#0 42:Root[0000, 0]@156..157#0 42:Root[0000, 0]@143..157#0 - IDENT 42:Root[0000, 0]@144..150#0 setter - GROUP () 42:Root[0000, 0]@150..151#0 42:Root[0000, 0]@155..156#0 42:Root[0000, 0]@150..156#0 - IDENT 42:Root[0000, 0]@151..155#0 into - IDENT 42:Root[0000, 0]@163..166#0 pub - GROUP () 42:Root[0000, 0]@166..167#0 42:Root[0000, 0]@172..173#0 42:Root[0000, 0]@166..173#0 - IDENT 42:Root[0000, 0]@167..172#0 crate - IDENT 42:Root[0000, 0]@174..180#0 domain - PUNCT 42:Root[0000, 0]@180..181#0 : [alone] - IDENT 42:Root[0000, 0]@182..188#0 String - PUNCT 42:Root[0000, 0]@188..189#0 , [alone] + PUNCT # [joint] 42:Root[0000, 0]@1..2#0 + GROUP [] 42:Root[0000, 0]@2..3#0 42:Root[0000, 0]@52..53#0 + IDENT helper 42:Root[0000, 0]@3..9#0 + GROUP () 42:Root[0000, 0]@9..10#0 42:Root[0000, 0]@51..52#0 + IDENT build_fn 42:Root[0000, 0]@10..18#0 + GROUP () 42:Root[0000, 0]@18..19#0 42:Root[0000, 0]@50..51#0 + IDENT private 42:Root[0000, 0]@19..26#0 + PUNCT , [alone] 42:Root[0000, 0]@26..27#0 + IDENT name 42:Root[0000, 0]@28..32#0 + PUNCT = [alone] 42:Root[0000, 0]@33..34#0 + LITERAL Str partial_build 42:Root[0000, 0]@35..50#0 + IDENT pub 42:Root[0000, 0]@54..57#0 + IDENT struct 42:Root[0000, 0]@58..64#0 + IDENT Foo 42:Root[0000, 0]@65..68#0 + GROUP {} 42:Root[0000, 0]@69..70#0 42:Root[0000, 0]@190..191#0 + PUNCT # [alone] 42:Root[0000, 0]@0..0#0 + GROUP [] 42:Root[0000, 0]@75..130#0 42:Root[0000, 0]@75..130#0 + IDENT doc 42:Root[0000, 0]@75..130#0 + PUNCT = [alone] 42:Root[0000, 0]@75..130#0 + LITERAL Str The domain where this federated instance is running 42:Root[0000, 0]@75..130#0 + PUNCT # [joint] 42:Root[0000, 0]@135..136#0 + GROUP [] 42:Root[0000, 0]@136..137#0 42:Root[0000, 0]@157..158#0 + IDENT helper 42:Root[0000, 0]@137..143#0 + GROUP () 42:Root[0000, 0]@143..144#0 42:Root[0000, 0]@156..157#0 + IDENT setter 42:Root[0000, 0]@144..150#0 + GROUP () 42:Root[0000, 0]@150..151#0 42:Root[0000, 0]@155..156#0 + IDENT into 42:Root[0000, 0]@151..155#0 + IDENT pub 42:Root[0000, 0]@163..166#0 + GROUP () 42:Root[0000, 0]@166..167#0 42:Root[0000, 0]@172..173#0 + IDENT crate 42:Root[0000, 0]@167..172#0 + IDENT domain 42:Root[0000, 0]@174..180#0 + PUNCT : [alone] 42:Root[0000, 0]@180..181#0 + IDENT String 42:Root[0000, 0]@182..188#0 + PUNCT , [alone] 42:Root[0000, 0]@188..189#0 + + + PUNCT # [joint] 42:Root[0000, 0]@1..2#0 + GROUP [] 42:Root[0000, 0]@2..3#0 42:Root[0000, 0]@52..53#0 + IDENT helper 42:Root[0000, 0]@3..9#0 + GROUP () 42:Root[0000, 0]@9..10#0 42:Root[0000, 0]@51..52#0 + IDENT build_fn 42:Root[0000, 0]@10..18#0 + GROUP () 42:Root[0000, 0]@18..19#0 42:Root[0000, 0]@50..51#0 + IDENT private 42:Root[0000, 0]@19..26#0 + PUNCT , [alone] 42:Root[0000, 0]@26..27#0 + IDENT name 42:Root[0000, 0]@28..32#0 + PUNCT = [alone] 42:Root[0000, 0]@33..34#0 + LITERAL Str partial_build 42:Root[0000, 0]@35..50#0 + IDENT pub 42:Root[0000, 0]@54..57#0 + IDENT struct 42:Root[0000, 0]@58..64#0 + IDENT Foo 42:Root[0000, 0]@65..68#0 + GROUP {} 42:Root[0000, 0]@69..70#0 42:Root[0000, 0]@190..191#0 + PUNCT # [alone] 42:Root[0000, 0]@0..0#0 + GROUP [] 42:Root[0000, 0]@75..130#0 42:Root[0000, 0]@75..130#0 + IDENT doc 42:Root[0000, 0]@75..130#0 + PUNCT = [alone] 42:Root[0000, 0]@75..130#0 + LITERAL Str The domain where this federated instance is running 42:Root[0000, 0]@75..130#0 + PUNCT # [joint] 42:Root[0000, 0]@135..136#0 + GROUP [] 42:Root[0000, 0]@136..137#0 42:Root[0000, 0]@157..158#0 + IDENT helper 42:Root[0000, 0]@137..143#0 + GROUP () 42:Root[0000, 0]@143..144#0 42:Root[0000, 0]@156..157#0 + IDENT setter 42:Root[0000, 0]@144..150#0 + GROUP () 42:Root[0000, 0]@150..151#0 42:Root[0000, 0]@155..156#0 + IDENT into 42:Root[0000, 0]@151..155#0 + IDENT pub 42:Root[0000, 0]@163..166#0 + GROUP () 42:Root[0000, 0]@166..167#0 42:Root[0000, 0]@172..173#0 + IDENT crate 42:Root[0000, 0]@167..172#0 + IDENT domain 42:Root[0000, 0]@174..180#0 + PUNCT : [alone] 42:Root[0000, 0]@180..181#0 + IDENT String 42:Root[0000, 0]@182..188#0 + PUNCT , [alone] 42:Root[0000, 0]@188..189#0 "#]], ); } @@ -217,34 +217,34 @@ fn test_derive_error() { "DeriveError", r#"struct S { field: u32 }"#, expect![[r#" - IDENT 1 struct - IDENT 1 S - GROUP {} 1 1 1 - IDENT 1 field - PUNCT 1 : [alone] - IDENT 1 u32 - - - IDENT 1 compile_error - PUNCT 1 ! [joint] - GROUP () 1 1 1 - LITER 1 Str #[derive(DeriveError)] struct S {field : u32} - PUNCT 1 ; [alone] + IDENT struct 1 + IDENT S 1 + GROUP {} 1 1 + IDENT field 1 + PUNCT : [alone] 1 + IDENT u32 1 + + + IDENT compile_error 1 + PUNCT ! [joint] 1 + GROUP () 1 1 + LITERAL Str #[derive(DeriveError)] struct S {field : u32} 1 + PUNCT ; [alone] 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..6#0 struct - IDENT 42:Root[0000, 0]@7..8#0 S - GROUP {} 42:Root[0000, 0]@9..10#0 42:Root[0000, 0]@22..23#0 42:Root[0000, 0]@9..23#0 - IDENT 42:Root[0000, 0]@11..16#0 field - PUNCT 42:Root[0000, 0]@16..17#0 : [alone] - IDENT 42:Root[0000, 0]@18..21#0 u32 - - - IDENT 42:Root[0000, 0]@0..13#0 compile_error - PUNCT 42:Root[0000, 0]@13..14#0 ! [joint] - GROUP () 42:Root[0000, 0]@14..15#0 42:Root[0000, 0]@62..63#0 42:Root[0000, 0]@14..63#0 - LITER 42:Root[0000, 0]@15..62#0 Str #[derive(DeriveError)] struct S {field : u32} - PUNCT 42:Root[0000, 0]@63..64#0 ; [alone] + IDENT struct 42:Root[0000, 0]@0..6#0 + IDENT S 42:Root[0000, 0]@7..8#0 + GROUP {} 42:Root[0000, 0]@9..10#0 42:Root[0000, 0]@22..23#0 + IDENT field 42:Root[0000, 0]@11..16#0 + PUNCT : [alone] 42:Root[0000, 0]@16..17#0 + IDENT u32 42:Root[0000, 0]@18..21#0 + + + IDENT compile_error 42:Root[0000, 0]@0..13#0 + PUNCT ! [joint] 42:Root[0000, 0]@13..14#0 + GROUP () 42:Root[0000, 0]@14..15#0 42:Root[0000, 0]@62..63#0 + LITERAL Str #[derive(DeriveError)] struct S {field : u32} 42:Root[0000, 0]@15..62#0 + PUNCT ; [alone] 42:Root[0000, 0]@63..64#0 "#]], ); } @@ -255,40 +255,40 @@ fn test_fn_like_macro_noop() { "fn_like_noop", r#"ident, 0, 1, []"#, expect![[r#" - IDENT 1 ident - PUNCT 1 , [alone] - LITER 1 Integer 0 - PUNCT 1 , [alone] - LITER 1 Integer 1 - PUNCT 1 , [alone] - GROUP [] 1 1 1 - - - IDENT 1 ident - PUNCT 1 , [alone] - LITER 1 Integer 0 - PUNCT 1 , [alone] - LITER 1 Integer 1 - PUNCT 1 , [alone] - GROUP [] 1 1 1 + IDENT ident 1 + PUNCT , [alone] 1 + LITERAL Integer 0 1 + PUNCT , [alone] 1 + LITERAL Integer 1 1 + PUNCT , [alone] 1 + GROUP [] 1 1 + + + IDENT ident 1 + PUNCT , [alone] 1 + LITERAL Integer 0 1 + PUNCT , [alone] 1 + LITERAL Integer 1 1 + PUNCT , [alone] 1 + GROUP [] 1 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..5#0 ident - PUNCT 42:Root[0000, 0]@5..6#0 , [alone] - LITER 42:Root[0000, 0]@7..8#0 Integer 0 - PUNCT 42:Root[0000, 0]@8..9#0 , [alone] - LITER 42:Root[0000, 0]@10..11#0 Integer 1 - PUNCT 42:Root[0000, 0]@11..12#0 , [alone] - GROUP [] 42:Root[0000, 0]@13..14#0 42:Root[0000, 0]@14..15#0 42:Root[0000, 0]@13..15#0 - - - IDENT 42:Root[0000, 0]@0..5#0 ident - PUNCT 42:Root[0000, 0]@5..6#0 , [alone] - LITER 42:Root[0000, 0]@7..8#0 Integer 0 - PUNCT 42:Root[0000, 0]@8..9#0 , [alone] - LITER 42:Root[0000, 0]@10..11#0 Integer 1 - PUNCT 42:Root[0000, 0]@11..12#0 , [alone] - GROUP [] 42:Root[0000, 0]@13..14#0 42:Root[0000, 0]@14..15#0 42:Root[0000, 0]@13..15#0 + IDENT ident 42:Root[0000, 0]@0..5#0 + PUNCT , [alone] 42:Root[0000, 0]@5..6#0 + LITERAL Integer 0 42:Root[0000, 0]@7..8#0 + PUNCT , [alone] 42:Root[0000, 0]@8..9#0 + LITERAL Integer 1 42:Root[0000, 0]@10..11#0 + PUNCT , [alone] 42:Root[0000, 0]@11..12#0 + GROUP [] 42:Root[0000, 0]@13..14#0 42:Root[0000, 0]@14..15#0 + + + IDENT ident 42:Root[0000, 0]@0..5#0 + PUNCT , [alone] 42:Root[0000, 0]@5..6#0 + LITERAL Integer 0 42:Root[0000, 0]@7..8#0 + PUNCT , [alone] 42:Root[0000, 0]@8..9#0 + LITERAL Integer 1 42:Root[0000, 0]@10..11#0 + PUNCT , [alone] 42:Root[0000, 0]@11..12#0 + GROUP [] 42:Root[0000, 0]@13..14#0 42:Root[0000, 0]@14..15#0 "#]], ); } @@ -299,36 +299,36 @@ fn test_fn_like_macro_clone_ident_subtree() { "fn_like_clone_tokens", r#"ident, [ident2, ident3]"#, expect![[r#" - IDENT 1 ident - PUNCT 1 , [alone] - GROUP [] 1 1 1 - IDENT 1 ident2 - PUNCT 1 , [alone] - IDENT 1 ident3 - - - IDENT 1 ident - PUNCT 1 , [alone] - GROUP [] 1 1 1 - IDENT 1 ident2 - PUNCT 1 , [alone] - IDENT 1 ident3 + IDENT ident 1 + PUNCT , [alone] 1 + GROUP [] 1 1 + IDENT ident2 1 + PUNCT , [alone] 1 + IDENT ident3 1 + + + IDENT ident 1 + PUNCT , [alone] 1 + GROUP [] 1 1 + IDENT ident2 1 + PUNCT , [alone] 1 + IDENT ident3 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..5#0 ident - PUNCT 42:Root[0000, 0]@5..6#0 , [alone] - GROUP [] 42:Root[0000, 0]@7..8#0 42:Root[0000, 0]@22..23#0 42:Root[0000, 0]@7..23#0 - IDENT 42:Root[0000, 0]@8..14#0 ident2 - PUNCT 42:Root[0000, 0]@14..15#0 , [alone] - IDENT 42:Root[0000, 0]@16..22#0 ident3 - - - IDENT 42:Root[0000, 0]@0..5#0 ident - PUNCT 42:Root[0000, 0]@5..6#0 , [alone] - GROUP [] 42:Root[0000, 0]@7..23#0 42:Root[0000, 0]@7..23#0 42:Root[0000, 0]@7..23#0 - IDENT 42:Root[0000, 0]@8..14#0 ident2 - PUNCT 42:Root[0000, 0]@14..15#0 , [alone] - IDENT 42:Root[0000, 0]@16..22#0 ident3 + IDENT ident 42:Root[0000, 0]@0..5#0 + PUNCT , [alone] 42:Root[0000, 0]@5..6#0 + GROUP [] 42:Root[0000, 0]@7..8#0 42:Root[0000, 0]@22..23#0 + IDENT ident2 42:Root[0000, 0]@8..14#0 + PUNCT , [alone] 42:Root[0000, 0]@14..15#0 + IDENT ident3 42:Root[0000, 0]@16..22#0 + + + IDENT ident 42:Root[0000, 0]@0..5#0 + PUNCT , [alone] 42:Root[0000, 0]@5..6#0 + GROUP [] 42:Root[0000, 0]@7..23#0 42:Root[0000, 0]@7..23#0 + IDENT ident2 42:Root[0000, 0]@8..14#0 + PUNCT , [alone] 42:Root[0000, 0]@14..15#0 + IDENT ident3 42:Root[0000, 0]@16..22#0 "#]], ); } @@ -339,16 +339,16 @@ fn test_fn_like_macro_clone_raw_ident() { "fn_like_clone_tokens", "r#async", expect![[r#" - IDENT 1 r#async + IDENT r#async 1 - IDENT 1 r#async + IDENT r#async 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@2..7#0 r#async + IDENT r#async 42:Root[0000, 0]@2..7#0 - IDENT 42:Root[0000, 0]@2..7#0 r#async + IDENT r#async 42:Root[0000, 0]@2..7#0 "#]], ); } @@ -359,18 +359,18 @@ fn test_fn_like_fn_like_span_join() { "fn_like_span_join", "foo bar", expect![[r#" - IDENT 1 foo - IDENT 1 bar + IDENT foo 1 + IDENT bar 1 - IDENT 1 r#joined + IDENT r#joined 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..3#0 foo - IDENT 42:Root[0000, 0]@8..11#0 bar + IDENT foo 42:Root[0000, 0]@0..3#0 + IDENT bar 42:Root[0000, 0]@8..11#0 - IDENT 42:Root[0000, 0]@0..11#0 r#joined + IDENT r#joined 42:Root[0000, 0]@0..11#0 "#]], ); } @@ -381,24 +381,24 @@ fn test_fn_like_fn_like_span_ops() { "fn_like_span_ops", "set_def_site resolved_at_def_site start_span", expect![[r#" - IDENT 1 set_def_site - IDENT 1 resolved_at_def_site - IDENT 1 start_span + IDENT set_def_site 1 + IDENT resolved_at_def_site 1 + IDENT start_span 1 - IDENT 0 set_def_site - IDENT 1 resolved_at_def_site - IDENT 1 start_span + IDENT set_def_site 0 + IDENT resolved_at_def_site 1 + IDENT start_span 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..12#0 set_def_site - IDENT 42:Root[0000, 0]@13..33#0 resolved_at_def_site - IDENT 42:Root[0000, 0]@34..44#0 start_span + IDENT set_def_site 42:Root[0000, 0]@0..12#0 + IDENT resolved_at_def_site 42:Root[0000, 0]@13..33#0 + IDENT start_span 42:Root[0000, 0]@34..44#0 - IDENT 41:Root[0000, 0]@0..150#0 set_def_site - IDENT 42:Root[0000, 0]@13..33#0 resolved_at_def_site - IDENT 42:Root[0000, 0]@34..34#0 start_span + IDENT set_def_site 41:Root[0000, 0]@0..150#0 + IDENT resolved_at_def_site 42:Root[0000, 0]@13..33#0 + IDENT start_span 42:Root[0000, 0]@34..34#0 "#]], ); } @@ -411,36 +411,36 @@ fn test_fn_like_mk_literals() { expect![[r#" - LITER 1 ByteStr byte_string - LITER 1 Char c - LITER 1 Str string - LITER 1 Str -string - LITER 1 CStr cstring - LITER 1 Float 3.14f64 - LITER 1 Float -3.14f64 - LITER 1 Float 3.14 - LITER 1 Float -3.14 - LITER 1 Integer 123i64 - LITER 1 Integer -123i64 - LITER 1 Integer 123 - LITER 1 Integer -123 + LITERAL ByteStr byte_string 1 + LITERAL Char c 1 + LITERAL Str string 1 + LITERAL Str -string 1 + LITERAL CStr cstring 1 + LITERAL Float 3.14f64 1 + LITERAL Float -3.14f64 1 + LITERAL Float 3.14 1 + LITERAL Float -3.14 1 + LITERAL Integer 123i64 1 + LITERAL Integer -123i64 1 + LITERAL Integer 123 1 + LITERAL Integer -123 1 "#]], expect![[r#" - LITER 42:Root[0000, 0]@0..100#0 ByteStr byte_string - LITER 42:Root[0000, 0]@0..100#0 Char c - LITER 42:Root[0000, 0]@0..100#0 Str string - LITER 42:Root[0000, 0]@0..100#0 Str -string - LITER 42:Root[0000, 0]@0..100#0 CStr cstring - LITER 42:Root[0000, 0]@0..100#0 Float 3.14f64 - LITER 42:Root[0000, 0]@0..100#0 Float -3.14f64 - LITER 42:Root[0000, 0]@0..100#0 Float 3.14 - LITER 42:Root[0000, 0]@0..100#0 Float -3.14 - LITER 42:Root[0000, 0]@0..100#0 Integer 123i64 - LITER 42:Root[0000, 0]@0..100#0 Integer -123i64 - LITER 42:Root[0000, 0]@0..100#0 Integer 123 - LITER 42:Root[0000, 0]@0..100#0 Integer -123 + LITERAL ByteStr byte_string 42:Root[0000, 0]@0..100#0 + LITERAL Char c 42:Root[0000, 0]@0..100#0 + LITERAL Str string 42:Root[0000, 0]@0..100#0 + LITERAL Str -string 42:Root[0000, 0]@0..100#0 + LITERAL CStr cstring 42:Root[0000, 0]@0..100#0 + LITERAL Float 3.14f64 42:Root[0000, 0]@0..100#0 + LITERAL Float -3.14f64 42:Root[0000, 0]@0..100#0 + LITERAL Float 3.14 42:Root[0000, 0]@0..100#0 + LITERAL Float -3.14 42:Root[0000, 0]@0..100#0 + LITERAL Integer 123i64 42:Root[0000, 0]@0..100#0 + LITERAL Integer -123i64 42:Root[0000, 0]@0..100#0 + LITERAL Integer 123 42:Root[0000, 0]@0..100#0 + LITERAL Integer -123 42:Root[0000, 0]@0..100#0 "#]], ); } @@ -453,14 +453,14 @@ fn test_fn_like_mk_idents() { expect![[r#" - IDENT 1 standard - IDENT 1 r#raw + IDENT standard 1 + IDENT r#raw 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..100#0 standard - IDENT 42:Root[0000, 0]@0..100#0 r#raw + IDENT standard 42:Root[0000, 0]@0..100#0 + IDENT r#raw 42:Root[0000, 0]@0..100#0 "#]], ); } @@ -471,92 +471,92 @@ fn test_fn_like_macro_clone_literals() { "fn_like_clone_tokens", r###"1u16, 2_u32, -4i64, 3.14f32, "hello bridge", "suffixed"suffix, r##"raw"##, 'a', b'b', c"null""###, expect![[r#" - LITER 1 Integer 1u16 - PUNCT 1 , [alone] - LITER 1 Integer 2_u32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Integer 4i64 - PUNCT 1 , [alone] - LITER 1 Float 3.14f32 - PUNCT 1 , [alone] - LITER 1 Str hello bridge - PUNCT 1 , [alone] - LITER 1 Str suffixedsuffix - PUNCT 1 , [alone] - LITER 1 StrRaw(2) raw - PUNCT 1 , [alone] - LITER 1 Char a - PUNCT 1 , [alone] - LITER 1 Byte b - PUNCT 1 , [alone] - LITER 1 CStr null - - - LITER 1 Integer 1u16 - PUNCT 1 , [alone] - LITER 1 Integer 2_u32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Integer 4i64 - PUNCT 1 , [alone] - LITER 1 Float 3.14f32 - PUNCT 1 , [alone] - LITER 1 Str hello bridge - PUNCT 1 , [alone] - LITER 1 Str suffixedsuffix - PUNCT 1 , [alone] - LITER 1 StrRaw(2) raw - PUNCT 1 , [alone] - LITER 1 Char a - PUNCT 1 , [alone] - LITER 1 Byte b - PUNCT 1 , [alone] - LITER 1 CStr null + LITERAL Integer 1u16 1 + PUNCT , [alone] 1 + LITERAL Integer 2_u32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Integer 4i64 1 + PUNCT , [alone] 1 + LITERAL Float 3.14f32 1 + PUNCT , [alone] 1 + LITERAL Str hello bridge 1 + PUNCT , [alone] 1 + LITERAL Err(()) "suffixed"suffix 1 + PUNCT , [alone] 1 + LITERAL StrRaw(2) raw 1 + PUNCT , [alone] 1 + LITERAL Char a 1 + PUNCT , [alone] 1 + LITERAL Byte b 1 + PUNCT , [alone] 1 + LITERAL CStr null 1 + + + LITERAL Integer 1u16 1 + PUNCT , [alone] 1 + LITERAL Integer 2_u32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Integer 4i64 1 + PUNCT , [alone] 1 + LITERAL Float 3.14f32 1 + PUNCT , [alone] 1 + LITERAL Str hello bridge 1 + PUNCT , [alone] 1 + LITERAL Err(()) "suffixed"suffix 1 + PUNCT , [alone] 1 + LITERAL StrRaw(2) raw 1 + PUNCT , [alone] 1 + LITERAL Char a 1 + PUNCT , [alone] 1 + LITERAL Byte b 1 + PUNCT , [alone] 1 + LITERAL CStr null 1 "#]], expect![[r#" - LITER 42:Root[0000, 0]@0..4#0 Integer 1u16 - PUNCT 42:Root[0000, 0]@4..5#0 , [alone] - LITER 42:Root[0000, 0]@6..11#0 Integer 2_u32 - PUNCT 42:Root[0000, 0]@11..12#0 , [alone] - PUNCT 42:Root[0000, 0]@13..14#0 - [alone] - LITER 42:Root[0000, 0]@14..18#0 Integer 4i64 - PUNCT 42:Root[0000, 0]@18..19#0 , [alone] - LITER 42:Root[0000, 0]@20..27#0 Float 3.14f32 - PUNCT 42:Root[0000, 0]@27..28#0 , [alone] - LITER 42:Root[0000, 0]@29..43#0 Str hello bridge - PUNCT 42:Root[0000, 0]@43..44#0 , [alone] - LITER 42:Root[0000, 0]@45..61#0 Str suffixedsuffix - PUNCT 42:Root[0000, 0]@61..62#0 , [alone] - LITER 42:Root[0000, 0]@63..73#0 StrRaw(2) raw - PUNCT 42:Root[0000, 0]@73..74#0 , [alone] - LITER 42:Root[0000, 0]@75..78#0 Char a - PUNCT 42:Root[0000, 0]@78..79#0 , [alone] - LITER 42:Root[0000, 0]@80..84#0 Byte b - PUNCT 42:Root[0000, 0]@84..85#0 , [alone] - LITER 42:Root[0000, 0]@86..93#0 CStr null - - - LITER 42:Root[0000, 0]@0..4#0 Integer 1u16 - PUNCT 42:Root[0000, 0]@4..5#0 , [alone] - LITER 42:Root[0000, 0]@6..11#0 Integer 2_u32 - PUNCT 42:Root[0000, 0]@11..12#0 , [alone] - PUNCT 42:Root[0000, 0]@13..14#0 - [alone] - LITER 42:Root[0000, 0]@14..18#0 Integer 4i64 - PUNCT 42:Root[0000, 0]@18..19#0 , [alone] - LITER 42:Root[0000, 0]@20..27#0 Float 3.14f32 - PUNCT 42:Root[0000, 0]@27..28#0 , [alone] - LITER 42:Root[0000, 0]@29..43#0 Str hello bridge - PUNCT 42:Root[0000, 0]@43..44#0 , [alone] - LITER 42:Root[0000, 0]@45..61#0 Str suffixedsuffix - PUNCT 42:Root[0000, 0]@61..62#0 , [alone] - LITER 42:Root[0000, 0]@63..73#0 StrRaw(2) raw - PUNCT 42:Root[0000, 0]@73..74#0 , [alone] - LITER 42:Root[0000, 0]@75..78#0 Char a - PUNCT 42:Root[0000, 0]@78..79#0 , [alone] - LITER 42:Root[0000, 0]@80..84#0 Byte b - PUNCT 42:Root[0000, 0]@84..85#0 , [alone] - LITER 42:Root[0000, 0]@86..93#0 CStr null + LITERAL Integer 1u16 42:Root[0000, 0]@0..4#0 + PUNCT , [alone] 42:Root[0000, 0]@4..5#0 + LITERAL Integer 2_u32 42:Root[0000, 0]@6..11#0 + PUNCT , [alone] 42:Root[0000, 0]@11..12#0 + PUNCT - [alone] 42:Root[0000, 0]@13..14#0 + LITERAL Integer 4i64 42:Root[0000, 0]@14..18#0 + PUNCT , [alone] 42:Root[0000, 0]@18..19#0 + LITERAL Float 3.14f32 42:Root[0000, 0]@20..27#0 + PUNCT , [alone] 42:Root[0000, 0]@27..28#0 + LITERAL Str hello bridge 42:Root[0000, 0]@29..43#0 + PUNCT , [alone] 42:Root[0000, 0]@43..44#0 + LITERAL Err(()) "suffixed"suffix 42:Root[0000, 0]@45..61#0 + PUNCT , [alone] 42:Root[0000, 0]@61..62#0 + LITERAL StrRaw(2) raw 42:Root[0000, 0]@63..73#0 + PUNCT , [alone] 42:Root[0000, 0]@73..74#0 + LITERAL Char a 42:Root[0000, 0]@75..78#0 + PUNCT , [alone] 42:Root[0000, 0]@78..79#0 + LITERAL Byte b 42:Root[0000, 0]@80..84#0 + PUNCT , [alone] 42:Root[0000, 0]@84..85#0 + LITERAL CStr null 42:Root[0000, 0]@86..93#0 + + + LITERAL Integer 1u16 42:Root[0000, 0]@0..4#0 + PUNCT , [alone] 42:Root[0000, 0]@4..5#0 + LITERAL Integer 2_u32 42:Root[0000, 0]@6..11#0 + PUNCT , [alone] 42:Root[0000, 0]@11..12#0 + PUNCT - [alone] 42:Root[0000, 0]@13..14#0 + LITERAL Integer 4i64 42:Root[0000, 0]@14..18#0 + PUNCT , [alone] 42:Root[0000, 0]@18..19#0 + LITERAL Float 3.14f32 42:Root[0000, 0]@20..27#0 + PUNCT , [alone] 42:Root[0000, 0]@27..28#0 + LITERAL Str hello bridge 42:Root[0000, 0]@29..43#0 + PUNCT , [alone] 42:Root[0000, 0]@43..44#0 + LITERAL Err(()) "suffixed"suffix 42:Root[0000, 0]@45..61#0 + PUNCT , [alone] 42:Root[0000, 0]@61..62#0 + LITERAL StrRaw(2) raw 42:Root[0000, 0]@63..73#0 + PUNCT , [alone] 42:Root[0000, 0]@73..74#0 + LITERAL Char a 42:Root[0000, 0]@75..78#0 + PUNCT , [alone] 42:Root[0000, 0]@78..79#0 + LITERAL Byte b 42:Root[0000, 0]@80..84#0 + PUNCT , [alone] 42:Root[0000, 0]@84..85#0 + LITERAL CStr null 42:Root[0000, 0]@86..93#0 "#]], ); } @@ -567,56 +567,56 @@ fn test_fn_like_macro_negative_literals() { "fn_like_clone_tokens", r###"-1u16, - 2_u32, -3.14f32, - 2.7"###, expect![[r#" - PUNCT 1 - [alone] - LITER 1 Integer 1u16 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Integer 2_u32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Float 3.14f32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Float 2.7 - - - PUNCT 1 - [alone] - LITER 1 Integer 1u16 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Integer 2_u32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Float 3.14f32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Float 2.7 + PUNCT - [alone] 1 + LITERAL Integer 1u16 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Integer 2_u32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Float 3.14f32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Float 2.7 1 + + + PUNCT - [alone] 1 + LITERAL Integer 1u16 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Integer 2_u32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Float 3.14f32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Float 2.7 1 "#]], expect![[r#" - PUNCT 42:Root[0000, 0]@0..1#0 - [alone] - LITER 42:Root[0000, 0]@1..5#0 Integer 1u16 - PUNCT 42:Root[0000, 0]@5..6#0 , [alone] - PUNCT 42:Root[0000, 0]@7..8#0 - [alone] - LITER 42:Root[0000, 0]@9..14#0 Integer 2_u32 - PUNCT 42:Root[0000, 0]@14..15#0 , [alone] - PUNCT 42:Root[0000, 0]@16..17#0 - [alone] - LITER 42:Root[0000, 0]@17..24#0 Float 3.14f32 - PUNCT 42:Root[0000, 0]@24..25#0 , [alone] - PUNCT 42:Root[0000, 0]@26..27#0 - [alone] - LITER 42:Root[0000, 0]@28..31#0 Float 2.7 - - - PUNCT 42:Root[0000, 0]@0..1#0 - [alone] - LITER 42:Root[0000, 0]@1..5#0 Integer 1u16 - PUNCT 42:Root[0000, 0]@5..6#0 , [alone] - PUNCT 42:Root[0000, 0]@7..8#0 - [alone] - LITER 42:Root[0000, 0]@9..14#0 Integer 2_u32 - PUNCT 42:Root[0000, 0]@14..15#0 , [alone] - PUNCT 42:Root[0000, 0]@16..17#0 - [alone] - LITER 42:Root[0000, 0]@17..24#0 Float 3.14f32 - PUNCT 42:Root[0000, 0]@24..25#0 , [alone] - PUNCT 42:Root[0000, 0]@26..27#0 - [alone] - LITER 42:Root[0000, 0]@28..31#0 Float 2.7 + PUNCT - [alone] 42:Root[0000, 0]@0..1#0 + LITERAL Integer 1u16 42:Root[0000, 0]@1..5#0 + PUNCT , [alone] 42:Root[0000, 0]@5..6#0 + PUNCT - [alone] 42:Root[0000, 0]@7..8#0 + LITERAL Integer 2_u32 42:Root[0000, 0]@9..14#0 + PUNCT , [alone] 42:Root[0000, 0]@14..15#0 + PUNCT - [alone] 42:Root[0000, 0]@16..17#0 + LITERAL Float 3.14f32 42:Root[0000, 0]@17..24#0 + PUNCT , [alone] 42:Root[0000, 0]@24..25#0 + PUNCT - [alone] 42:Root[0000, 0]@26..27#0 + LITERAL Float 2.7 42:Root[0000, 0]@28..31#0 + + + PUNCT - [alone] 42:Root[0000, 0]@0..1#0 + LITERAL Integer 1u16 42:Root[0000, 0]@1..5#0 + PUNCT , [alone] 42:Root[0000, 0]@5..6#0 + PUNCT - [alone] 42:Root[0000, 0]@7..8#0 + LITERAL Integer 2_u32 42:Root[0000, 0]@9..14#0 + PUNCT , [alone] 42:Root[0000, 0]@14..15#0 + PUNCT - [alone] 42:Root[0000, 0]@16..17#0 + LITERAL Float 3.14f32 42:Root[0000, 0]@17..24#0 + PUNCT , [alone] 42:Root[0000, 0]@24..25#0 + PUNCT - [alone] 42:Root[0000, 0]@26..27#0 + LITERAL Float 2.7 42:Root[0000, 0]@28..31#0 "#]], ); } @@ -631,36 +631,36 @@ fn test_attr_macro() { r#"mod m {}"#, r#"some arguments"#, expect![[r#" - IDENT 1 mod - IDENT 1 m - GROUP {} 1 1 1 + IDENT mod 1 + IDENT m 1 + GROUP {} 1 1 - IDENT 1 some - IDENT 1 arguments + IDENT some 1 + IDENT arguments 1 - IDENT 1 compile_error - PUNCT 1 ! [joint] - GROUP () 1 1 1 - LITER 1 Str #[attr_error(some arguments)] mod m {} - PUNCT 1 ; [alone] + IDENT compile_error 1 + PUNCT ! [joint] 1 + GROUP () 1 1 + LITERAL Str #[attr_error(some arguments)] mod m {} 1 + PUNCT ; [alone] 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..3#0 mod - IDENT 42:Root[0000, 0]@4..5#0 m - GROUP {} 42:Root[0000, 0]@6..7#0 42:Root[0000, 0]@7..8#0 42:Root[0000, 0]@6..8#0 + IDENT mod 42:Root[0000, 0]@0..3#0 + IDENT m 42:Root[0000, 0]@4..5#0 + GROUP {} 42:Root[0000, 0]@6..7#0 42:Root[0000, 0]@7..8#0 - IDENT 42:Root[0000, 0]@0..4#0 some - IDENT 42:Root[0000, 0]@5..14#0 arguments + IDENT some 42:Root[0000, 0]@0..4#0 + IDENT arguments 42:Root[0000, 0]@5..14#0 - IDENT 42:Root[0000, 0]@0..13#0 compile_error - PUNCT 42:Root[0000, 0]@13..14#0 ! [joint] - GROUP () 42:Root[0000, 0]@14..15#0 42:Root[0000, 0]@55..56#0 42:Root[0000, 0]@14..56#0 - LITER 42:Root[0000, 0]@15..55#0 Str #[attr_error(some arguments)] mod m {} - PUNCT 42:Root[0000, 0]@56..57#0 ; [alone] + IDENT compile_error 42:Root[0000, 0]@0..13#0 + PUNCT ! [joint] 42:Root[0000, 0]@13..14#0 + GROUP () 42:Root[0000, 0]@14..15#0 42:Root[0000, 0]@55..56#0 + LITERAL Str #[attr_error(some arguments)] mod m {} 42:Root[0000, 0]@15..55#0 + PUNCT ; [alone] 42:Root[0000, 0]@56..57#0 "#]], ); } @@ -722,8 +722,8 @@ fn test_fn_like_span_line_column() { " hello", expect![[r#" - LITER 42:Root[0000, 0]@0..100#0 Integer 2 - LITER 42:Root[0000, 0]@0..100#0 Integer 1 + LITERAL Integer 2 42:Root[0000, 0]@0..100#0 + LITERAL Integer 1 42:Root[0000, 0]@0..100#0 "#]], ); } diff --git a/crates/proc-macro-srv/src/tests/utils.rs b/crates/proc-macro-srv/src/tests/utils.rs index 7f92c66fb69d..b514e8e3d8f9 100644 --- a/crates/proc-macro-srv/src/tests/utils.rs +++ b/crates/proc-macro-srv/src/tests/utils.rs @@ -1,6 +1,7 @@ //! utils used in proc-macro tests use expect_test::Expect; +use proc_macro_api::token_stream::TokenStream; use span::{ EditionedFileId, FileId, ROOT_ERASED_FILE_AST_ID, Span, SpanAnchor, SyntaxContext, TextRange, }; @@ -8,7 +9,6 @@ use std::ops::Range; use crate::{ EnvSnapshot, ProcMacroClientInterface, ProcMacroSrv, SpanId, dylib, proc_macro_test_dylib_path, - token_stream::TokenStream, }; fn make_ctx() -> SyntaxContext { diff --git a/crates/proc-macro-srv/src/token_stream.rs b/crates/proc-macro-srv/src/token_stream.rs deleted file mode 100644 index 5201bb6aeb86..000000000000 --- a/crates/proc-macro-srv/src/token_stream.rs +++ /dev/null @@ -1,767 +0,0 @@ -//! The proc-macro server token stream implementation. - -use core::fmt; -use std::{mem, sync::Arc}; - -use intern::Symbol; -use rustc_lexer::{DocStyle, LiteralKind}; -use rustc_proc_macro::Delimiter; - -use crate::bridge::{DelimSpan, Group, Ident, LitKind, Literal, Punct, TokenTree}; - -/// Trait for allowing tests to parse tokenstreams with dynamic span ranges -pub(crate) trait SpanLike { - fn derive_ranged(&self, range: std::ops::Range) -> Self; -} - -#[derive(Clone)] -pub struct TokenStream(pub(crate) Arc>>); - -impl Default for TokenStream { - fn default() -> Self { - Self(Default::default()) - } -} - -impl TokenStream { - pub fn new(tts: Vec>) -> TokenStream { - TokenStream(Arc::new(tts)) - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - pub fn len(&self) -> usize { - self.0.len() - } - - pub fn iter(&self) -> TokenStreamIter<'_, S> { - TokenStreamIter::new(self) - } - - pub fn as_single_group(&self) -> Option<&Group> { - match &**self.0 { - [TokenTree::Group(group)] => Some(group), - _ => None, - } - } - - pub(crate) fn from_str(s: &str, span: S) -> Result - where - S: SpanLike + Copy, - { - let mut groups = Vec::new(); - groups.push((rustc_proc_macro::Delimiter::None, 0..0, vec![])); - let mut offset = 0; - let mut tokens = rustc_lexer::tokenize(s, rustc_lexer::FrontmatterAllowed::No).peekable(); - while let Some(token) = tokens.next() { - let range = offset..offset + token.len as usize; - offset += token.len as usize; - - let mut is_joint = || { - tokens.peek().is_some_and(|token| { - matches!( - token.kind, - rustc_lexer::TokenKind::RawLifetime - | rustc_lexer::TokenKind::GuardedStrPrefix - | rustc_lexer::TokenKind::Lifetime { .. } - | rustc_lexer::TokenKind::Semi - | rustc_lexer::TokenKind::Comma - | rustc_lexer::TokenKind::Dot - | rustc_lexer::TokenKind::OpenParen - | rustc_lexer::TokenKind::CloseParen - | rustc_lexer::TokenKind::OpenBrace - | rustc_lexer::TokenKind::CloseBrace - | rustc_lexer::TokenKind::OpenBracket - | rustc_lexer::TokenKind::CloseBracket - | rustc_lexer::TokenKind::At - | rustc_lexer::TokenKind::Pound - | rustc_lexer::TokenKind::Tilde - | rustc_lexer::TokenKind::Question - | rustc_lexer::TokenKind::Colon - | rustc_lexer::TokenKind::Dollar - | rustc_lexer::TokenKind::Eq - | rustc_lexer::TokenKind::Bang - | rustc_lexer::TokenKind::Lt - | rustc_lexer::TokenKind::Gt - | rustc_lexer::TokenKind::Minus - | rustc_lexer::TokenKind::And - | rustc_lexer::TokenKind::Or - | rustc_lexer::TokenKind::Plus - | rustc_lexer::TokenKind::Star - | rustc_lexer::TokenKind::Slash - | rustc_lexer::TokenKind::Percent - | rustc_lexer::TokenKind::Caret - ) - }) - }; - - let Some((open_delim, _, tokenstream)) = groups.last_mut() else { - return Err("Unbalanced delimiters".to_owned()); - }; - match token.kind { - rustc_lexer::TokenKind::OpenParen => { - groups.push((rustc_proc_macro::Delimiter::Parenthesis, range, vec![])) - } - rustc_lexer::TokenKind::CloseParen if *open_delim != Delimiter::Parenthesis => { - return if *open_delim == Delimiter::None { - Err("Unexpected ')'".to_owned()) - } else { - Err("Expected ')'".to_owned()) - }; - } - rustc_lexer::TokenKind::CloseParen => { - let (delimiter, open_range, stream) = groups.pop().unwrap(); - groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( - TokenTree::Group(Group { - delimiter, - stream: if stream.is_empty() { - None - } else { - Some(TokenStream::new(stream)) - }, - span: DelimSpan { - entire: span.derive_ranged(open_range.start..range.end), - open: span.derive_ranged(open_range), - close: span.derive_ranged(range), - }, - }), - ); - } - rustc_lexer::TokenKind::OpenBrace => { - groups.push((rustc_proc_macro::Delimiter::Brace, range, vec![])) - } - rustc_lexer::TokenKind::CloseBrace if *open_delim != Delimiter::Brace => { - return if *open_delim == Delimiter::None { - Err("Unexpected '}'".to_owned()) - } else { - Err("Expected '}'".to_owned()) - }; - } - rustc_lexer::TokenKind::CloseBrace => { - let (delimiter, open_range, stream) = groups.pop().unwrap(); - groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( - TokenTree::Group(Group { - delimiter, - stream: if stream.is_empty() { - None - } else { - Some(TokenStream::new(stream)) - }, - span: DelimSpan { - entire: span.derive_ranged(open_range.start..range.end), - open: span.derive_ranged(open_range), - close: span.derive_ranged(range), - }, - }), - ); - } - rustc_lexer::TokenKind::OpenBracket => { - groups.push((rustc_proc_macro::Delimiter::Bracket, range, vec![])) - } - rustc_lexer::TokenKind::CloseBracket if *open_delim != Delimiter::Bracket => { - return if *open_delim == Delimiter::None { - Err("Unexpected ']'".to_owned()) - } else { - Err("Expected ']'".to_owned()) - }; - } - rustc_lexer::TokenKind::CloseBracket => { - let (delimiter, open_range, stream) = groups.pop().unwrap(); - groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( - TokenTree::Group(Group { - delimiter, - stream: if stream.is_empty() { - None - } else { - Some(TokenStream::new(stream)) - }, - span: DelimSpan { - entire: span.derive_ranged(open_range.start..range.end), - open: span.derive_ranged(open_range), - close: span.derive_ranged(range), - }, - }), - ); - } - rustc_lexer::TokenKind::LineComment { doc_style: None } - | rustc_lexer::TokenKind::BlockComment { doc_style: None, terminated: _ } => { - continue; - } - rustc_lexer::TokenKind::LineComment { doc_style: Some(doc_style) } => { - let text = &s[range.start + 3..range.end]; - tokenstream.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span })); - if doc_style == DocStyle::Inner { - tokenstream.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span })); - } - tokenstream.push(TokenTree::Group(Group { - delimiter: Delimiter::Bracket, - stream: Some(TokenStream::new(vec![ - TokenTree::Ident(Ident { - sym: Symbol::intern("doc"), - is_raw: false, - span, - }), - TokenTree::Punct(Punct { ch: b'=', joint: false, span }), - TokenTree::Literal(Literal { - kind: LitKind::Str, - symbol: Symbol::intern(&text.escape_debug().to_string()), - suffix: None, - span: span.derive_ranged(range), - }), - ])), - span: DelimSpan { open: span, close: span, entire: span }, - })); - } - rustc_lexer::TokenKind::BlockComment { doc_style: Some(doc_style), terminated } => { - let text = - &s[range.start + 3..if terminated { range.end - 2 } else { range.end }]; - tokenstream.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span })); - if doc_style == DocStyle::Inner { - tokenstream.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span })); - } - tokenstream.push(TokenTree::Group(Group { - delimiter: Delimiter::Bracket, - stream: Some(TokenStream::new(vec![ - TokenTree::Ident(Ident { - sym: Symbol::intern("doc"), - is_raw: false, - span, - }), - TokenTree::Punct(Punct { ch: b'=', joint: false, span }), - TokenTree::Literal(Literal { - kind: LitKind::Str, - symbol: Symbol::intern(&text.escape_debug().to_string()), - suffix: None, - span: span.derive_ranged(range), - }), - ])), - span: DelimSpan { open: span, close: span, entire: span }, - })); - } - rustc_lexer::TokenKind::Whitespace => continue, - rustc_lexer::TokenKind::Frontmatter { .. } => unreachable!(), - rustc_lexer::TokenKind::Unknown => { - return Err(format!("Unknown token: `{}`", &s[range])); - } - rustc_lexer::TokenKind::UnknownPrefix => { - return Err(format!("Unknown prefix: `{}`", &s[range])); - } - rustc_lexer::TokenKind::UnknownPrefixLifetime => { - return Err(format!("Unknown lifetime prefix: `{}`", &s[range])); - } - // FIXME: Error on edition >= 2024 ... I dont think the proc-macro server can fetch editions currently - // and whose edition is this? - rustc_lexer::TokenKind::GuardedStrPrefix => { - tokenstream.push(TokenTree::Punct(Punct { - ch: s.as_bytes()[range.start], - joint: true, - span: span.derive_ranged(range.start..range.start + 1), - })); - tokenstream.push(TokenTree::Punct(Punct { - ch: s.as_bytes()[range.start + 1], - joint: is_joint(), - span: span.derive_ranged(range.start + 1..range.end), - })) - } - rustc_lexer::TokenKind::Ident => tokenstream.push(TokenTree::Ident(Ident { - sym: Symbol::intern(&s[range.clone()]), - is_raw: false, - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::InvalidIdent => { - return Err(format!("Invalid identifier: `{}`", &s[range])); - } - rustc_lexer::TokenKind::RawIdent => { - let range = range.start + 2..range.end; - tokenstream.push(TokenTree::Ident(Ident { - sym: Symbol::intern(&s[range.clone()]), - is_raw: true, - span: span.derive_ranged(range), - })) - } - rustc_lexer::TokenKind::Literal { kind, suffix_start } => { - tokenstream.push(TokenTree::Literal(literal_from_lexer( - &s[range.clone()], - span.derive_ranged(range), - kind, - suffix_start, - ))) - } - rustc_lexer::TokenKind::RawLifetime => { - let range = range.start + 1 + 2..range.end; - tokenstream.push(TokenTree::Punct(Punct { - ch: b'\'', - joint: true, - span: span.derive_ranged(range.start..range.start + 1), - })); - tokenstream.push(TokenTree::Ident(Ident { - sym: Symbol::intern(&s[range.clone()]), - is_raw: true, - span: span.derive_ranged(range), - })) - } - rustc_lexer::TokenKind::Lifetime { starts_with_number } => { - if starts_with_number { - return Err("Lifetime cannot start with a number".to_owned()); - } - let range = range.start + 1..range.end; - tokenstream.push(TokenTree::Punct(Punct { - ch: b'\'', - joint: true, - span: span.derive_ranged(range.start..range.start + 1), - })); - tokenstream.push(TokenTree::Ident(Ident { - sym: Symbol::intern(&s[range.clone()]), - is_raw: false, - span: span.derive_ranged(range), - })) - } - rustc_lexer::TokenKind::Semi => tokenstream.push(TokenTree::Punct(Punct { - ch: b';', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Comma => tokenstream.push(TokenTree::Punct(Punct { - ch: b',', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Dot => tokenstream.push(TokenTree::Punct(Punct { - ch: b'.', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::At => tokenstream.push(TokenTree::Punct(Punct { - ch: b'@', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Pound => tokenstream.push(TokenTree::Punct(Punct { - ch: b'#', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Tilde => tokenstream.push(TokenTree::Punct(Punct { - ch: b'~', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Question => tokenstream.push(TokenTree::Punct(Punct { - ch: b'?', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Colon => tokenstream.push(TokenTree::Punct(Punct { - ch: b':', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Dollar => tokenstream.push(TokenTree::Punct(Punct { - ch: b'$', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Eq => tokenstream.push(TokenTree::Punct(Punct { - ch: b'=', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Bang => tokenstream.push(TokenTree::Punct(Punct { - ch: b'!', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Lt => tokenstream.push(TokenTree::Punct(Punct { - ch: b'<', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Gt => tokenstream.push(TokenTree::Punct(Punct { - ch: b'>', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Minus => tokenstream.push(TokenTree::Punct(Punct { - ch: b'-', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::And => tokenstream.push(TokenTree::Punct(Punct { - ch: b'&', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Or => tokenstream.push(TokenTree::Punct(Punct { - ch: b'|', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Plus => tokenstream.push(TokenTree::Punct(Punct { - ch: b'+', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Star => tokenstream.push(TokenTree::Punct(Punct { - ch: b'*', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Slash => tokenstream.push(TokenTree::Punct(Punct { - ch: b'/', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Caret => tokenstream.push(TokenTree::Punct(Punct { - ch: b'^', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Percent => tokenstream.push(TokenTree::Punct(Punct { - ch: b'%', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Eof => break, - } - } - if let Some((Delimiter::None, _, tokentrees)) = groups.pop() - && groups.is_empty() - { - Ok(TokenStream::new(tokentrees)) - } else { - Err("Mismatched token groups".to_owned()) - } - } -} - -impl fmt::Display for TokenStream { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut emit_whitespace = false; - for tt in self.0.iter() { - display_token_tree(tt, &mut emit_whitespace, f)?; - } - Ok(()) - } -} - -fn display_token_tree( - tt: &TokenTree, - emit_whitespace: &mut bool, - f: &mut std::fmt::Formatter<'_>, -) -> std::fmt::Result { - if mem::take(emit_whitespace) { - write!(f, " ")?; - } - match tt { - TokenTree::Group(Group { delimiter, stream, span: _ }) => { - write!( - f, - "{}", - match delimiter { - rustc_proc_macro::Delimiter::Parenthesis => "(", - rustc_proc_macro::Delimiter::Brace => "{", - rustc_proc_macro::Delimiter::Bracket => "[", - rustc_proc_macro::Delimiter::None => "", - } - )?; - if let Some(stream) = stream { - write!(f, "{stream}")?; - } - write!( - f, - "{}", - match delimiter { - rustc_proc_macro::Delimiter::Parenthesis => ")", - rustc_proc_macro::Delimiter::Brace => "}", - rustc_proc_macro::Delimiter::Bracket => "]", - rustc_proc_macro::Delimiter::None => "", - } - )?; - } - TokenTree::Punct(Punct { ch, joint, span: _ }) => { - *emit_whitespace = !*joint; - write!(f, "{}", *ch as char)?; - } - TokenTree::Ident(Ident { sym, is_raw, span: _ }) => { - if *is_raw { - write!(f, "r#")?; - } - write!(f, "{sym}")?; - *emit_whitespace = true; - } - TokenTree::Literal(lit) => { - display_fmt_literal(lit, f)?; - let joint = match lit.kind { - LitKind::Str - | LitKind::StrRaw(_) - | LitKind::ByteStr - | LitKind::ByteStrRaw(_) - | LitKind::CStr - | LitKind::CStrRaw(_) => true, - _ => false, - }; - *emit_whitespace = !joint; - } - } - Ok(()) -} - -pub fn literal_to_string(literal: &Literal) -> String { - let mut buf = String::new(); - display_fmt_literal(literal, &mut buf).unwrap(); - buf -} - -fn display_fmt_literal(literal: &Literal, f: &mut impl std::fmt::Write) -> fmt::Result { - match literal.kind { - LitKind::Byte => write!(f, "b'{}'", literal.symbol), - LitKind::Char => write!(f, "'{}'", literal.symbol), - LitKind::Integer | LitKind::Float | LitKind::ErrWithGuar => { - write!(f, "{}", literal.symbol) - } - LitKind::Str => write!(f, "\"{}\"", literal.symbol), - LitKind::ByteStr => write!(f, "b\"{}\"", literal.symbol), - LitKind::CStr => write!(f, "c\"{}\"", literal.symbol), - LitKind::StrRaw(num_of_hashes) => { - let num_of_hashes = num_of_hashes as usize; - write!( - f, - r#"r{0:# { - let num_of_hashes = num_of_hashes as usize; - write!( - f, - r#"br{0:# { - let num_of_hashes = num_of_hashes as usize; - write!( - f, - r#"cr{0:# fmt::Debug for TokenStream { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - debug_token_stream(self, 0, f) - } -} - -fn debug_token_stream( - ts: &TokenStream, - depth: usize, - f: &mut std::fmt::Formatter<'_>, -) -> std::fmt::Result { - for tt in ts.0.iter() { - debug_token_tree(tt, depth, f)?; - } - Ok(()) -} - -fn debug_token_tree( - tt: &TokenTree, - depth: usize, - f: &mut std::fmt::Formatter<'_>, -) -> std::fmt::Result { - write!(f, "{:indent$}", "", indent = depth * 2)?; - match tt { - TokenTree::Group(Group { delimiter, stream, span }) => { - writeln!( - f, - "GROUP {}{} {:#?} {:#?} {:#?}", - match delimiter { - rustc_proc_macro::Delimiter::Parenthesis => "(", - rustc_proc_macro::Delimiter::Brace => "{", - rustc_proc_macro::Delimiter::Bracket => "[", - rustc_proc_macro::Delimiter::None => "$", - }, - match delimiter { - rustc_proc_macro::Delimiter::Parenthesis => ")", - rustc_proc_macro::Delimiter::Brace => "}", - rustc_proc_macro::Delimiter::Bracket => "]", - rustc_proc_macro::Delimiter::None => "$", - }, - span.open, - span.close, - span.entire, - )?; - if let Some(stream) = stream { - debug_token_stream(stream, depth + 1, f)?; - } - return Ok(()); - } - TokenTree::Punct(Punct { ch, joint, span }) => write!( - f, - "PUNCT {span:#?} {} {}", - *ch as char, - if *joint { "[joint]" } else { "[alone]" } - )?, - TokenTree::Ident(Ident { sym, is_raw, span }) => { - write!(f, "IDENT {span:#?} ")?; - if *is_raw { - write!(f, "r#")?; - } - write!(f, "{sym}")?; - } - TokenTree::Literal(Literal { kind, symbol, suffix, span }) => write!( - f, - "LITER {span:#?} {kind:?} {symbol}{}", - match suffix { - Some(suffix) => suffix.clone(), - None => Symbol::intern(""), - } - )?, - } - writeln!(f) -} - -impl TokenStream { - /// Push `tt` onto the end of the stream, possibly gluing it to the last - /// token. Uses `make_mut` to maximize efficiency. - pub(crate) fn push_tree(&mut self, tt: TokenTree) { - let vec_mut = Arc::make_mut(&mut self.0); - vec_mut.push(tt); - } - - /// Push `stream` onto the end of the stream, possibly gluing the first - /// token tree to the last token. (No other token trees will be glued.) - /// Uses `make_mut` to maximize efficiency. - pub(crate) fn push_stream(&mut self, stream: TokenStream) { - let vec_mut = Arc::make_mut(&mut self.0); - - let stream_iter = stream.0.iter().cloned(); - - vec_mut.extend(stream_iter); - } -} - -impl FromIterator> for TokenStream { - fn from_iter>>(iter: I) -> Self { - TokenStream::new(iter.into_iter().collect::>>()) - } -} - -#[derive(Clone)] -pub struct TokenStreamIter<'t, S> { - stream: &'t TokenStream, - index: usize, -} - -impl<'t, S> TokenStreamIter<'t, S> { - fn new(stream: &'t TokenStream) -> Self { - TokenStreamIter { stream, index: 0 } - } -} - -impl<'t, S> Iterator for TokenStreamIter<'t, S> { - type Item = &'t TokenTree; - - fn next(&mut self) -> Option<&'t TokenTree> { - self.stream.0.get(self.index).map(|tree| { - self.index += 1; - tree - }) - } -} - -pub(super) fn literal_from_lexer( - s: &str, - span: Span, - kind: rustc_lexer::LiteralKind, - suffix_start: u32, -) -> Literal { - let (kind, start_offset, end_offset) = match kind { - LiteralKind::Int { .. } => (LitKind::Integer, 0, 0), - LiteralKind::Float { .. } => (LitKind::Float, 0, 0), - LiteralKind::Char { terminated } => (LitKind::Char, 1, terminated as usize), - LiteralKind::Byte { terminated } => (LitKind::Byte, 2, terminated as usize), - LiteralKind::Str { terminated } => (LitKind::Str, 1, terminated as usize), - LiteralKind::ByteStr { terminated } => (LitKind::ByteStr, 2, terminated as usize), - LiteralKind::CStr { terminated } => (LitKind::CStr, 2, terminated as usize), - LiteralKind::RawStr { n_hashes } => ( - LitKind::StrRaw(n_hashes.unwrap_or_default()), - 2 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - LiteralKind::RawByteStr { n_hashes } => ( - LitKind::ByteStrRaw(n_hashes.unwrap_or_default()), - 3 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - LiteralKind::RawCStr { n_hashes } => ( - LitKind::CStrRaw(n_hashes.unwrap_or_default()), - 3 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - }; - - let (lit, suffix) = s.split_at(suffix_start as usize); - let lit = &lit[start_offset..lit.len() - end_offset]; - let suffix = match suffix { - "" | "_" => None, - suffix => Some(Symbol::intern(suffix)), - }; - - Literal { kind, symbol: Symbol::intern(lit), suffix, span } -} - -impl SpanLike for crate::SpanId { - fn derive_ranged(&self, _: std::ops::Range) -> Self { - *self - } -} - -impl SpanLike for () { - fn derive_ranged(&self, _: std::ops::Range) -> Self { - *self - } -} - -impl SpanLike for crate::Span { - fn derive_ranged(&self, range: std::ops::Range) -> Self { - crate::Span { - range: span::TextRange::new( - span::TextSize::new(range.start as u32), - span::TextSize::new(range.end as u32), - ), - anchor: self.anchor, - ctx: self.ctx, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn ts_to_string() { - let token_stream = - TokenStream::from_str("{} () [] <> ;/., \"gfhdgfuiofghd\" 0f32 r#\"dff\"# 'r#lt", ()) - .unwrap(); - assert_eq!(token_stream.to_string(), "{}()[]<> ;/., \"gfhdgfuiofghd\"0f32 r#\"dff\"#'r#lt"); - } - - #[test] - fn doc_comment_from_str() { - let token_stream = TokenStream::from_str("/// foo", ()).unwrap(); - assert_eq!(token_stream.to_string(), r#"# [doc = " foo"]"#); - } -} diff --git a/crates/rust-analyzer/src/global_state.rs b/crates/rust-analyzer/src/global_state.rs index c0d87104fbe1..3659c718b177 100644 --- a/crates/rust-analyzer/src/global_state.rs +++ b/crates/rust-analyzer/src/global_state.rs @@ -23,7 +23,7 @@ use parking_lot::{ MappedRwLockReadGuard, Mutex, RwLock, RwLockReadGuard, RwLockUpgradableReadGuard, RwLockWriteGuard, }; -use proc_macro_api::ProcMacroClient; +use proc_macro_api::client::ProcMacroClient; use project_model::{ ManifestPath, ProjectWorkspace, ProjectWorkspaceKind, TargetKind, WorkspaceBuildScripts, }; diff --git a/crates/rust-analyzer/src/reload.rs b/crates/rust-analyzer/src/reload.rs index 039fbeff828e..91014f0f7198 100644 --- a/crates/rust-analyzer/src/reload.rs +++ b/crates/rust-analyzer/src/reload.rs @@ -24,7 +24,7 @@ use itertools::Itertools; use load_cargo::{ProjectFolders, load_proc_macro}; use lsp_types::FileSystemWatcher; use paths::Utf8Path; -use proc_macro_api::ProcMacroClient; +use proc_macro_api::client::ProcMacroClient; use project_model::{ ManifestPath, ProjectWorkspace, ProjectWorkspaceKind, WorkspaceBuildScripts, project_json, }; diff --git a/crates/stdx/src/lib.rs b/crates/stdx/src/lib.rs index dcba06415b5f..7d844c041b0a 100644 --- a/crates/stdx/src/lib.rs +++ b/crates/stdx/src/lib.rs @@ -363,6 +363,32 @@ pub fn slice_tails(this: &[T]) -> impl Iterator { (0..this.len()).map(|i| &this[i..]) } +/// Imports a sysroot crate from the sysroot or from crates.io, depending on whether the `in-rust-tree` +/// feature is active. +/// +/// Syntax: +/// ``` +/// extern crate sysroot_crate or crates_io_crate; +/// ``` +// FIXME: Should this really be in `stdx`? +#[macro_export] +macro_rules! rustc_crates { + ( + $( + extern crate $sysroot_crate:ident or $crates_io_crate:ident; + )* + ) => { + ::std::cfg_select! { + feature = "in-rust-tree" => { + $( extern crate $sysroot_crate; )* + } + _ => { + $( extern crate $crates_io_crate as $sysroot_crate; )* + } + } + }; +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/syntax-bridge/src/lib.rs b/crates/syntax-bridge/src/lib.rs index 181f9a14e764..f0e1c5cf27ee 100644 --- a/crates/syntax-bridge/src/lib.rs +++ b/crates/syntax-bridge/src/lib.rs @@ -18,7 +18,7 @@ use syntax::{ ast::{self, make::tokens::doc_comment}, format_smolstr, }; -use tt::{Punct, buffer::Cursor, token_to_literal}; +use tt::{Punct, buffer::Cursor}; pub mod prettify_macro_expansion; mod to_parser_input; @@ -318,7 +318,7 @@ where k if k.is_literal() => { let text = token.to_text(conv); let span = conv.span_for(abs_range); - token_to_literal(&text, span).into() + tt::literal_from_str_or_err(&text, span).into() } LIFETIME_IDENT => { let apostrophe = tt::Leaf::from(tt::Punct { diff --git a/crates/syntax/Cargo.toml b/crates/syntax/Cargo.toml index a9df1acdae9a..8bf7bb582942 100644 --- a/crates/syntax/Cargo.toml +++ b/crates/syntax/Cargo.toml @@ -40,3 +40,6 @@ in-rust-tree = [] [lints] workspace = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_lexer"] diff --git a/crates/syntax/src/lib.rs b/crates/syntax/src/lib.rs index dc7d7af7a2b9..8f46683da748 100644 --- a/crates/syntax/src/lib.rs +++ b/crates/syntax/src/lib.rs @@ -21,12 +21,12 @@ #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] -#[cfg(not(feature = "in-rust-tree"))] -extern crate ra_ap_rustc_lexer as rustc_lexer; #[cfg(feature = "in-rust-tree")] extern crate rustc_driver as _; -#[cfg(feature = "in-rust-tree")] -extern crate rustc_lexer; + +stdx::rustc_crates! { + extern crate rustc_lexer or ra_ap_rustc_lexer; +} mod parsing; mod ptr; diff --git a/crates/tt/Cargo.toml b/crates/tt/Cargo.toml index bd8f740b2f50..d7465d09c129 100644 --- a/crates/tt/Cargo.toml +++ b/crates/tt/Cargo.toml @@ -14,16 +14,30 @@ doctest = false [dependencies] arrayvec.workspace = true -text-size.workspace = true -rustc-hash.workspace = true +text-size = { workspace = true, optional = true } +rustc-hash = { workspace = true, optional = true } -span = { path = "../span", version = "0.0", default-features = false } -stdx.workspace = true +span = { path = "../span", version = "0.0", default-features = false, optional = true } +stdx = { workspace = true, optional = true } intern.workspace = true ra-ap-rustc_lexer.workspace = true [features] +default = ["in-ra"] in-rust-tree = [] +# Inside rust-analyzer and not the proc macro server. Being in the proc macro server disables everything but the TokenTree types. +in-ra = [ + "dep:text-size", + "dep:rustc-hash", + "dep:span", + "dep:stdx", +] [lints] workspace = true + +[package.metadata.rust-analyzer] +rustc_private = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_lexer"] diff --git a/crates/tt/src/leaf_types.rs b/crates/tt/src/leaf_types.rs new file mode 100644 index 000000000000..a8d48d097e6b --- /dev/null +++ b/crates/tt/src/leaf_types.rs @@ -0,0 +1,398 @@ +//! Types that are shared between rust-analyzer and the proc macro server. + +use std::fmt; + +use arrayvec::ArrayString; +use intern::Symbol; + +#[cfg(feature = "in-ra")] +type DefaultSpan = span::Span; +#[cfg(not(feature = "in-ra"))] +pub enum DefaultSpan {} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +#[repr(u8)] +// The discriminants are important for `storage.rs` decoding. +pub enum IdentIsRaw { + No = 0, + Yes = 1, +} + +impl IdentIsRaw { + pub fn yes(self) -> bool { + matches!(self, IdentIsRaw::Yes) + } + pub fn no(&self) -> bool { + matches!(self, IdentIsRaw::No) + } + pub fn as_str(self) -> &'static str { + match self { + IdentIsRaw::No => "", + IdentIsRaw::Yes => "r#", + } + } + pub fn split_from_symbol(sym: &str) -> (Self, &str) { + if let Some(sym) = sym.strip_prefix("r#") { + (IdentIsRaw::Yes, sym) + } else { + (IdentIsRaw::No, sym) + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +pub enum LitKind { + Byte, + Char, + Integer, // e.g. `1`, `1u8`, `1f32` + Float, // e.g. `1.`, `1.0`, `1e3f32` + Str, + StrRaw(u8), // raw string delimited by `n` hash symbols + ByteStr, + ByteStrRaw(u8), // raw byte string delimited by `n` hash symbols + CStr, + CStrRaw(u8), + Err(()), +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Leaf { + Literal(Literal), + Punct(Punct), + Ident(Ident), +} + +#[derive(Debug, Copy, Clone, PartialEq)] +pub struct DelimSpan { + pub open: Span, + pub close: Span, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Delimiter { + pub open: Span, + pub close: Span, + pub kind: DelimiterKind, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[repr(u8)] +// The discriminants are important for decoding for `storage.rs`. +pub enum DelimiterKind { + Parenthesis = 0, + Brace = 1, + Bracket = 2, + Invisible = 3, +} + +impl DelimiterKind { + pub fn display_open_close(self) -> (&'static str, &'static str) { + match self { + DelimiterKind::Brace => ("{", "}"), + DelimiterKind::Bracket => ("[", "]"), + DelimiterKind::Parenthesis => ("(", ")"), + DelimiterKind::Invisible => ("", ""), + } + } + + pub fn debug_view(self) -> &'static str { + match self { + DelimiterKind::Invisible => "$$", + DelimiterKind::Parenthesis => "()", + DelimiterKind::Brace => "{}", + DelimiterKind::Bracket => "[]", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Literal { + /// Escaped, text then suffix concatenated. + pub text_and_suffix: Symbol, + pub span: Span, + pub kind: LitKind, + pub suffix_len: u8, +} + +impl Literal { + #[inline] + pub fn text_and_suffix(&self) -> (&str, &str) { + let text_and_suffix = self.text_and_suffix.as_str(); + text_and_suffix.split_at(text_and_suffix.len() - usize::from(self.suffix_len)) + } + + pub fn text_and_suffix_symbols(&self) -> (Symbol, Option) { + if self.suffix_len == 0 { + (self.text_and_suffix.clone(), None) + } else { + let (text, suffix) = self.text_and_suffix(); + (Symbol::intern(text), Some(Symbol::intern(suffix))) + } + } + + #[inline] + pub fn text(&self) -> &str { + self.text_and_suffix().0 + } + + #[inline] + pub fn suffix(&self) -> &str { + self.text_and_suffix().1 + } + + pub fn new(text: &str, span: Span, kind: LitKind, suffix: &str) -> Self { + const MAX_INLINE_CAPACITY: usize = 30; + let text_and_suffix = if suffix.is_empty() { + Symbol::intern(text) + } else if (text.len() + suffix.len()) < MAX_INLINE_CAPACITY { + let mut text_and_suffix = ArrayString::::new(); + text_and_suffix.push_str(text); + text_and_suffix.push_str(suffix); + Symbol::intern(&text_and_suffix) + } else { + let mut text_and_suffix = String::with_capacity(text.len() + suffix.len()); + text_and_suffix.push_str(text); + text_and_suffix.push_str(suffix); + Symbol::intern(&text_and_suffix) + }; + + Self { text_and_suffix, span, kind, suffix_len: suffix.len().try_into().unwrap() } + } + + #[inline] + pub fn new_no_suffix(text: &str, span: Span, kind: LitKind) -> Self { + Self { text_and_suffix: Symbol::intern(text), span, kind, suffix_len: 0 } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Punct { + pub char: char, + pub spacing: Spacing, + pub span: Span, +} + +/// Indicates whether a token can join with the following token to form a +/// compound token. Used for conversions to `proc_macro::Spacing`. Also used to +/// guide pretty-printing, which is where the `JointHidden` value (which isn't +/// part of `proc_macro::Spacing`) comes in useful. +// The discriminants are important for decoding for `storage.rs`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(u8)] +pub enum Spacing { + /// The token cannot join with the following token to form a compound + /// token. + /// + /// In token streams parsed from source code, the compiler will use `Alone` + /// for any token immediately followed by whitespace, a non-doc comment, or + /// EOF. + /// + /// When constructing token streams within the compiler, use this for each + /// token that (a) should be pretty-printed with a space after it, or (b) + /// is the last token in the stream. (In the latter case the choice of + /// spacing doesn't matter because it is never used for the last token. We + /// arbitrarily use `Alone`.) + /// + /// Converts to `proc_macro::Spacing::Alone`, and + /// `proc_macro::Spacing::Alone` converts back to this. + Alone = 0, + + /// The token can join with the following token to form a compound token. + /// + /// In token streams parsed from source code, the compiler will use `Joint` + /// for any token immediately followed by punctuation (as determined by + /// `Token::is_punct`). + /// + /// When constructing token streams within the compiler, use this for each + /// token that (a) should be pretty-printed without a space after it, and + /// (b) is followed by a punctuation token. + /// + /// Converts to `proc_macro::Spacing::Joint`, and + /// `proc_macro::Spacing::Joint` converts back to this. + Joint = 1, + + /// The token can join with the following token to form a compound token, + /// but this will not be visible at the proc macro level. (This is what the + /// `Hidden` means; see below.) + /// + /// In token streams parsed from source code, the compiler will use + /// `JointHidden` for any token immediately followed by anything not + /// covered by the `Alone` and `Joint` cases: an identifier, lifetime, + /// literal, delimiter, doc comment. + /// + /// When constructing token streams, use this for each token that (a) + /// should be pretty-printed without a space after it, and (b) is followed + /// by a non-punctuation token. + /// + /// Converts to `proc_macro::Spacing::Alone`, but + /// `proc_macro::Spacing::Alone` converts back to `token::Spacing::Alone`. + /// Because of that, pretty-printing of `TokenStream`s produced by proc + /// macros is unavoidably uglier (with more whitespace between tokens) than + /// pretty-printing of `TokenStream`'s produced by other means (i.e. parsed + /// source code, internally constructed token streams, and token streams + /// produced by declarative macros). + JointHidden = 2, +} + +/// Identifier or keyword. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Ident { + pub sym: Symbol, + pub span: Span, + pub is_raw: IdentIsRaw, +} + +impl fmt::Display for Leaf { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Leaf::Ident(it) => fmt::Display::fmt(it, f), + Leaf::Literal(it) => fmt::Display::fmt(it, f), + Leaf::Punct(it) => fmt::Display::fmt(it, f), + } + } +} + +impl fmt::Display for Ident { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.is_raw.as_str(), f)?; + fmt::Display::fmt(&self.sym, f) + } +} + +impl fmt::Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (text, suffix) = self.text_and_suffix(); + match self.kind { + LitKind::Byte => write!(f, "b'{}'", text), + LitKind::Char => write!(f, "'{}'", text), + LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", text), + LitKind::Str => write!(f, "\"{}\"", text), + LitKind::ByteStr => write!(f, "b\"{}\"", text), + LitKind::CStr => write!(f, "c\"{}\"", text), + LitKind::StrRaw(num_of_hashes) => { + let num_of_hashes = num_of_hashes as usize; + write!(f, r#"r{0:# { + let num_of_hashes = num_of_hashes as usize; + write!(f, r#"br{0:# { + let num_of_hashes = num_of_hashes as usize; + write!(f, r#"cr{0:# fmt::Display for Punct { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.char, f) + } +} + +impl Leaf { + pub fn print_debug(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Leaf::Literal(lit) => { + let (text, suffix) = lit.text_and_suffix(); + write!(f, "LITERAL {:?} {}{} {:#?}", lit.kind, text, suffix, lit.span)?; + } + Leaf::Punct(punct) => { + write!( + f, + "PUNCT {} [{}] {:#?}", + punct.char, + if punct.spacing == Spacing::Alone { "alone" } else { "joint" }, + punct.span + )?; + } + Leaf::Ident(ident) => { + write!(f, "IDENT {}{} {:#?}", ident.is_raw.as_str(), ident.sym, ident.span)?; + } + } + + Ok(()) + } +} + +pub fn literal_from_lexer( + text: &str, + span: Span, + kind: rustc_lexer::LiteralKind, + suffix_start: u32, +) -> Literal { + use rustc_lexer::LiteralKind; + + let (kind, start_offset, end_offset) = match kind { + LiteralKind::Int { .. } => (LitKind::Integer, 0, 0), + LiteralKind::Float { .. } => (LitKind::Float, 0, 0), + LiteralKind::Char { terminated } => (LitKind::Char, 1, terminated as usize), + LiteralKind::Byte { terminated } => (LitKind::Byte, 2, terminated as usize), + LiteralKind::Str { terminated } => (LitKind::Str, 1, terminated as usize), + LiteralKind::ByteStr { terminated } => (LitKind::ByteStr, 2, terminated as usize), + LiteralKind::CStr { terminated } => (LitKind::CStr, 2, terminated as usize), + LiteralKind::RawStr { n_hashes } => ( + LitKind::StrRaw(n_hashes.unwrap_or_default()), + 2 + n_hashes.unwrap_or_default() as usize, + 1 + n_hashes.unwrap_or_default() as usize, + ), + LiteralKind::RawByteStr { n_hashes } => ( + LitKind::ByteStrRaw(n_hashes.unwrap_or_default()), + 3 + n_hashes.unwrap_or_default() as usize, + 1 + n_hashes.unwrap_or_default() as usize, + ), + LiteralKind::RawCStr { n_hashes } => ( + LitKind::CStrRaw(n_hashes.unwrap_or_default()), + 3 + n_hashes.unwrap_or_default() as usize, + 1 + n_hashes.unwrap_or_default() as usize, + ), + }; + + let (lit, suffix) = text.split_at(suffix_start as usize); + let lit = &lit[start_offset..lit.len() - end_offset]; + let suffix = match suffix { + "" | "_" => "", + // ill-suffixed literals + _ if !matches!(kind, LitKind::Integer | LitKind::Float | LitKind::Err(_)) => { + return Literal::new_no_suffix(text, span, LitKind::Err(())); + } + suffix => suffix, + }; + + Literal::new(lit, span, kind, suffix) +} + +pub fn literal_from_str(text: &str, span: Span) -> Result, ()> { + use rustc_lexer::{LiteralKind, Token, TokenKind}; + + let mut tokens = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No); + let minus_or_lit = tokens.next().unwrap_or(Token { kind: TokenKind::Eof, len: 0 }); + + let lit = if minus_or_lit.kind == TokenKind::Minus { + let lit = tokens.next().ok_or(())?; + if !matches!( + lit.kind, + TokenKind::Literal { kind: LiteralKind::Int { .. } | LiteralKind::Float { .. }, .. } + ) { + return Err(()); + } + lit + } else { + minus_or_lit + }; + + if tokens.next().is_some() { + return Err(()); + } + + let TokenKind::Literal { kind, suffix_start } = lit.kind else { return Err(()) }; + Ok(literal_from_lexer(text, span, kind, suffix_start)) +} + +pub fn literal_from_str_or_err(text: &str, span: Span) -> Literal { + literal_from_str(text, span) + .unwrap_or_else(|_| Literal::new_no_suffix(text, span, LitKind::Err(()))) +} diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs index 2bc2b64cd4fd..d41f9a9c58f7 100644 --- a/crates/tt/src/lib.rs +++ b/crates/tt/src/lib.rs @@ -8,812 +8,481 @@ #[cfg(feature = "in-rust-tree")] extern crate rustc_driver as _; -#[cfg(not(feature = "in-rust-tree"))] -extern crate ra_ap_rustc_lexer as rustc_lexer; -#[cfg(feature = "in-rust-tree")] -extern crate rustc_lexer; +stdx::rustc_crates! { + extern crate rustc_lexer or ra_ap_rustc_lexer; +} +#[cfg(feature = "in-ra")] pub mod buffer; +#[cfg(feature = "in-ra")] pub mod iter; +mod leaf_types; +#[cfg(feature = "in-ra")] mod storage; -use std::fmt; +#[cfg(feature = "in-ra")] +pub use self::in_ra::*; +pub use self::leaf_types::*; -use arrayvec::ArrayString; -use buffer::Cursor; -use intern::Symbol; -use stdx::{impl_from, itertools::Itertools as _}; +#[cfg(feature = "in-ra")] +mod in_ra { + use std::fmt; -pub use span::Span; -pub use text_size::{TextRange, TextSize}; + use intern::Symbol; + use stdx::impl_from; -use crate::storage::TokenTreesSlice; + pub use span::Span; + pub use text_size::{TextRange, TextSize}; -pub use self::iter::{TtElement, TtIter}; -pub use self::storage::{TopSubtree, TopSubtreeBuilder}; + use crate::{leaf_types::*, storage::TokenTreesSlice}; -pub const MAX_GLUED_PUNCT_LEN: usize = 3; + pub use crate::{ + buffer::Cursor, + iter::{TtElement, TtIter}, + storage::{TopSubtree, TopSubtreeBuilder}, + }; -#[derive(Clone, PartialEq, Debug)] -pub struct Lit { - pub kind: LitKind, - pub symbol: Symbol, - pub suffix: Option, -} + pub const MAX_GLUED_PUNCT_LEN: usize = 3; -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -#[repr(u8)] -// The discriminants are important for `storage.rs` decoding. -pub enum IdentIsRaw { - No = 0, - Yes = 1, -} -impl IdentIsRaw { - pub fn yes(self) -> bool { - matches!(self, IdentIsRaw::Yes) - } - pub fn no(&self) -> bool { - matches!(self, IdentIsRaw::No) + #[derive(Debug, Clone, PartialEq, Eq, Hash)] + pub enum TokenTree { + Leaf(Leaf), + Subtree(Subtree), } - pub fn as_str(self) -> &'static str { - match self { - IdentIsRaw::No => "", - IdentIsRaw::Yes => "r#", - } - } - pub fn split_from_symbol(sym: &str) -> (Self, &str) { - if let Some(sym) = sym.strip_prefix("r#") { - (IdentIsRaw::Yes, sym) - } else { - (IdentIsRaw::No, sym) + impl_from!(Leaf, Subtree for TokenTree); + impl TokenTree { + pub fn first_span(&self) -> Span { + match self { + TokenTree::Leaf(l) => *l.span(), + TokenTree::Subtree(s) => s.delimiter.open, + } } } -} -#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] -pub enum LitKind { - Byte, - Char, - Integer, // e.g. `1`, `1u8`, `1f32` - Float, // e.g. `1.`, `1.0`, `1e3f32` - Str, - StrRaw(u8), // raw string delimited by `n` hash symbols - ByteStr, - ByteStrRaw(u8), // raw byte string delimited by `n` hash symbols - CStr, - CStrRaw(u8), - Err(()), -} + impl Leaf { + pub fn span(&self) -> &Span { + match self { + Leaf::Literal(it) => &it.span, + Leaf::Punct(it) => &it.span, + Leaf::Ident(it) => &it.span, + } + } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum TokenTree { - Leaf(Leaf), - Subtree(Subtree), -} -impl_from!(Leaf, Subtree for TokenTree); -impl TokenTree { - pub fn first_span(&self) -> Span { - match self { - TokenTree::Leaf(l) => *l.span(), - TokenTree::Subtree(s) => s.delimiter.open, + pub(crate) fn symbol(&self) -> Option<&Symbol> { + match self { + Leaf::Literal(Literal { text_and_suffix: symbol, .. }) + | Leaf::Ident(Ident { sym: symbol, .. }) => Some(symbol), + Leaf::Punct(_) => None, + } } } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum Leaf { - Literal(Literal), - Punct(Punct), - Ident(Ident), -} + impl_from!(Literal, Punct, Ident for Leaf); -impl Leaf { - pub fn span(&self) -> &Span { - match self { - Leaf::Literal(it) => &it.span, - Leaf::Punct(it) => &it.span, - Leaf::Ident(it) => &it.span, - } + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub struct Subtree { + pub delimiter: Delimiter, + /// Number of following token trees that belong to this subtree, excluding this subtree. + pub len: u32, } - fn symbol(&self) -> Option<&Symbol> { - match self { - Leaf::Literal(Literal { text_and_suffix: symbol, .. }) - | Leaf::Ident(Ident { sym: symbol, .. }) => Some(symbol), - Leaf::Punct(_) => None, + impl Subtree { + pub fn usize_len(&self) -> usize { + self.len as usize } } -} -impl_from!(Literal, Punct, Ident for Leaf); -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct Subtree { - pub delimiter: Delimiter, - /// Number of following token trees that belong to this subtree, excluding this subtree. - pub len: u32, -} - -impl Subtree { - pub fn usize_len(&self) -> usize { - self.len as usize + #[derive(Clone, Copy)] + pub struct TokenTreesView<'a> { + pub(crate) slice: TokenTreesSlice<'a>, + pub(crate) len: usize, } -} -#[derive(Clone, Copy)] -pub struct TokenTreesView<'a> { - slice: TokenTreesSlice<'a>, - len: usize, -} - -impl<'a> TokenTreesView<'a> { - #[inline] - pub fn empty() -> Self { - Self { slice: TokenTreesSlice::empty(), len: 0 } - } + impl<'a> TokenTreesView<'a> { + #[inline] + pub fn empty() -> Self { + Self { slice: TokenTreesSlice::empty(), len: 0 } + } - pub fn iter(&self) -> TtIter<'a> { - TtIter::new(*self) - } + pub fn iter(&self) -> TtIter<'a> { + TtIter::new(*self) + } - pub fn cursor(&self) -> Cursor<'a> { - Cursor::new(*self) - } + pub fn cursor(&self) -> Cursor<'a> { + Cursor::new(*self) + } - pub fn len(&self) -> usize { - self.len - } + pub fn len(&self) -> usize { + self.len + } - pub fn is_empty(&self) -> bool { - self.len() == 0 - } + pub fn is_empty(&self) -> bool { + self.len() == 0 + } - pub fn try_into_subtree(self) -> Option> { - let is_subtree = self.iter_flat_tokens().next().is_some_and( + pub fn try_into_subtree(self) -> Option> { + let is_subtree = self.iter_flat_tokens().next().is_some_and( |it| matches!(it, TokenTree::Subtree(subtree) if subtree.usize_len() == self.len - 1), ); - if is_subtree { Some(SubtreeView(self)) } else { None } - } + if is_subtree { Some(SubtreeView(self)) } else { None } + } - pub fn strip_invisible(self) -> TokenTreesView<'a> { - self.try_into_subtree().map(|subtree| subtree.strip_invisible()).unwrap_or(self) - } + pub fn strip_invisible(self) -> TokenTreesView<'a> { + self.try_into_subtree().map(|subtree| subtree.strip_invisible()).unwrap_or(self) + } - pub fn split( - self, - mut split_fn: impl FnMut(TtElement<'a>) -> bool, - ) -> impl Iterator> { - let mut subtree_iter = self.iter(); - let mut need_to_yield_even_if_empty = true; + pub fn split( + self, + mut split_fn: impl FnMut(TtElement<'a>) -> bool, + ) -> impl Iterator> { + let mut subtree_iter = self.iter(); + let mut need_to_yield_even_if_empty = true; - std::iter::from_fn(move || { - if subtree_iter.is_empty() && !need_to_yield_even_if_empty { - return None; - }; + std::iter::from_fn(move || { + if subtree_iter.is_empty() && !need_to_yield_even_if_empty { + return None; + }; - need_to_yield_even_if_empty = false; - let savepoint = subtree_iter.savepoint(); - let mut result = subtree_iter.from_savepoint(savepoint); - while let Some(tt) = subtree_iter.next() { - if split_fn(tt) { - need_to_yield_even_if_empty = true; - break; + need_to_yield_even_if_empty = false; + let savepoint = subtree_iter.savepoint(); + let mut result = subtree_iter.from_savepoint(savepoint); + while let Some(tt) = subtree_iter.next() { + if split_fn(tt) { + need_to_yield_even_if_empty = true; + break; + } + result = subtree_iter.from_savepoint(savepoint); } - result = subtree_iter.from_savepoint(savepoint); - } - Some(result) - }) - } + Some(result) + }) + } - pub fn first_span(&self) -> Option { - self.iter_flat_tokens().next().map(|it| it.first_span()) - } + pub fn first_span(&self) -> Option { + self.iter_flat_tokens().next().map(|it| it.first_span()) + } - /// Note: this is quite expensive, this needs to decode the whole view, - /// although it "tricks" by skipping subtrees (since we know their byte length). - pub fn last_span(&self) -> Option { - let mut iter = self.iter(); - loop { - match iter.last()? { - TtElement::Leaf(leaf) => return Some(*leaf.span()), - TtElement::Subtree(subtree, tt_iter) => { - if subtree.len == 0 { - return Some(subtree.delimiter.close); - } else { - iter = tt_iter; + /// Note: this is quite expensive, this needs to decode the whole view, + /// although it "tricks" by skipping subtrees (since we know their byte length). + pub fn last_span(&self) -> Option { + let mut iter = self.iter(); + loop { + match iter.last()? { + TtElement::Leaf(leaf) => return Some(*leaf.span()), + TtElement::Subtree(subtree, tt_iter) => { + if subtree.len == 0 { + return Some(subtree.delimiter.close); + } else { + iter = tt_iter; + } } } } } - } - pub fn iter_flat_tokens(&self) -> impl Iterator + use<'a> { - self.slice.iter().take(self.len) + pub fn iter_flat_tokens(&self) -> impl Iterator + use<'a> { + self.slice.iter().take(self.len) + } } -} -impl fmt::Debug for TokenTreesView<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut iter = self.iter(); - while let Some(tt) = iter.next() { - print_debug_token(f, 0, tt)?; - if !iter.is_empty() { - writeln!(f)?; + impl fmt::Debug for TokenTreesView<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut iter = self.iter(); + while let Some(tt) = iter.next() { + print_debug_token(f, 0, tt)?; + if !iter.is_empty() { + writeln!(f)?; + } } + Ok(()) } - Ok(()) } -} -impl fmt::Display for TokenTreesView<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - return token_trees_display(f, self.iter()); - - fn subtree_display( - subtree: &Subtree, - f: &mut fmt::Formatter<'_>, - iter: TtIter<'_>, - ) -> fmt::Result { - let (l, r) = match subtree.delimiter.kind { - DelimiterKind::Parenthesis => ("(", ")"), - DelimiterKind::Brace => ("{", "}"), - DelimiterKind::Bracket => ("[", "]"), - DelimiterKind::Invisible => ("", ""), - }; - f.write_str(l)?; - token_trees_display(f, iter)?; - f.write_str(r)?; - Ok(()) - } + impl fmt::Display for TokenTreesView<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + return token_trees_display(f, self.iter()); - fn token_trees_display(f: &mut fmt::Formatter<'_>, iter: TtIter<'_>) -> fmt::Result { - let mut needs_space = false; - for child in iter { - if needs_space { - f.write_str(" ")?; - } - needs_space = true; + fn subtree_display( + subtree: &Subtree, + f: &mut fmt::Formatter<'_>, + iter: TtIter<'_>, + ) -> fmt::Result { + let (l, r) = match subtree.delimiter.kind { + DelimiterKind::Parenthesis => ("(", ")"), + DelimiterKind::Brace => ("{", "}"), + DelimiterKind::Bracket => ("[", "]"), + DelimiterKind::Invisible => ("", ""), + }; + f.write_str(l)?; + token_trees_display(f, iter)?; + f.write_str(r)?; + Ok(()) + } - match child { - TtElement::Leaf(Leaf::Punct(p)) => { - needs_space = p.spacing == Spacing::Alone; - fmt::Display::fmt(&p, f)?; + fn token_trees_display(f: &mut fmt::Formatter<'_>, iter: TtIter<'_>) -> fmt::Result { + let mut needs_space = false; + for child in iter { + if needs_space { + f.write_str(" ")?; } - TtElement::Leaf(leaf) => fmt::Display::fmt(&leaf, f)?, - TtElement::Subtree(subtree, subtree_iter) => { - subtree_display(&subtree, f, subtree_iter)? + needs_space = true; + + match child { + TtElement::Leaf(Leaf::Punct(p)) => { + needs_space = p.spacing == Spacing::Alone; + fmt::Display::fmt(&p, f)?; + } + TtElement::Leaf(leaf) => fmt::Display::fmt(&leaf, f)?, + TtElement::Subtree(subtree, subtree_iter) => { + subtree_display(&subtree, f, subtree_iter)? + } } } + Ok(()) } - Ok(()) } } -} - -#[derive(Clone, Copy)] -// Invariant: always starts with `Subtree` that covers the entire thing. -pub struct SubtreeView<'a>(TokenTreesView<'a>); - -impl<'a> SubtreeView<'a> { - pub fn as_token_trees(self) -> TokenTreesView<'a> { - self.0 - } - pub fn iter(&self) -> TtIter<'a> { - self.token_trees().iter() - } - - pub fn top_subtree(&self) -> Subtree { - let Some(TokenTree::Subtree(subtree)) = self.0.iter_flat_tokens().next() else { - unreachable!("the first token tree is always the top subtree"); - }; - subtree - } + #[derive(Clone, Copy)] + // Invariant: always starts with `Subtree` that covers the entire thing. + pub struct SubtreeView<'a>(pub(crate) TokenTreesView<'a>); - pub fn strip_invisible(&self) -> TokenTreesView<'a> { - if self.top_subtree().delimiter.kind == DelimiterKind::Invisible { - self.token_trees() - } else { + impl<'a> SubtreeView<'a> { + pub fn as_token_trees(self) -> TokenTreesView<'a> { self.0 } - } - - pub fn token_trees(&self) -> TokenTreesView<'a> { - let mut result = self.0; - result.slice.advance(); - result.len -= 1; - result - } -} - -impl fmt::Debug for SubtreeView<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&self.0, f) - } -} -impl fmt::Display for SubtreeView<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.0, f) - } -} - -#[derive(Debug, Copy, Clone, PartialEq)] -pub struct DelimSpan { - pub open: Span, - pub close: Span, -} + pub fn iter(&self) -> TtIter<'a> { + self.token_trees().iter() + } -impl DelimSpan { - pub fn from_single(sp: Span) -> Self { - DelimSpan { open: sp, close: sp } - } + pub fn top_subtree(&self) -> Subtree { + let Some(TokenTree::Subtree(subtree)) = self.0.iter_flat_tokens().next() else { + unreachable!("the first token tree is always the top subtree"); + }; + subtree + } - pub fn from_pair(open: Span, close: Span) -> Self { - DelimSpan { open, close } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct Delimiter { - pub open: Span, - pub close: Span, - pub kind: DelimiterKind, -} + pub fn strip_invisible(&self) -> TokenTreesView<'a> { + if self.top_subtree().delimiter.kind == DelimiterKind::Invisible { + self.token_trees() + } else { + self.0 + } + } -impl Delimiter { - pub const fn invisible_spanned(span: Span) -> Self { - Delimiter { open: span, close: span, kind: DelimiterKind::Invisible } + pub fn token_trees(&self) -> TokenTreesView<'a> { + let mut result = self.0; + result.slice.advance(); + result.len -= 1; + result + } } - pub const fn invisible_delim_spanned(span: DelimSpan) -> Self { - Delimiter { open: span.open, close: span.close, kind: DelimiterKind::Invisible } + impl fmt::Debug for SubtreeView<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.0, f) + } } - pub fn delim_span(&self) -> DelimSpan { - DelimSpan { open: self.open, close: self.close } + impl fmt::Display for SubtreeView<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -#[repr(u8)] -// The discriminants are important for decoding for `storage.rs`. -pub enum DelimiterKind { - Parenthesis = 0, - Brace = 1, - Bracket = 2, - Invisible = 3, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Literal { - /// Escaped, text then suffix concatenated. - pub text_and_suffix: Symbol, - pub span: Span, - pub kind: LitKind, - pub suffix_len: u8, -} -impl Literal { - #[inline] - pub fn text_and_suffix(&self) -> (&str, &str) { - let text_and_suffix = self.text_and_suffix.as_str(); - text_and_suffix.split_at(text_and_suffix.len() - usize::from(self.suffix_len)) - } + impl DelimSpan { + pub fn from_single(sp: Span) -> Self { + DelimSpan { open: sp, close: sp } + } - #[inline] - pub fn text(&self) -> &str { - self.text_and_suffix().0 + pub fn from_pair(open: Span, close: Span) -> Self { + DelimSpan { open, close } + } } - #[inline] - pub fn suffix(&self) -> &str { - self.text_and_suffix().1 - } + impl Delimiter { + pub const fn invisible_spanned(span: Span) -> Self { + Delimiter { open: span, close: span, kind: DelimiterKind::Invisible } + } - pub fn new(text: &str, span: Span, kind: LitKind, suffix: &str) -> Self { - const MAX_INLINE_CAPACITY: usize = 30; - let text_and_suffix = if suffix.is_empty() { - Symbol::intern(text) - } else if (text.len() + suffix.len()) < MAX_INLINE_CAPACITY { - let mut text_and_suffix = ArrayString::::new(); - text_and_suffix.push_str(text); - text_and_suffix.push_str(suffix); - Symbol::intern(&text_and_suffix) - } else { - let mut text_and_suffix = String::with_capacity(text.len() + suffix.len()); - text_and_suffix.push_str(text); - text_and_suffix.push_str(suffix); - Symbol::intern(&text_and_suffix) - }; - - Self { text_and_suffix, span, kind, suffix_len: suffix.len().try_into().unwrap() } - } + pub const fn invisible_delim_spanned(span: DelimSpan) -> Self { + Delimiter { open: span.open, close: span.close, kind: DelimiterKind::Invisible } + } - #[inline] - pub fn new_no_suffix(text: &str, span: Span, kind: LitKind) -> Self { - Self { text_and_suffix: Symbol::intern(text), span, kind, suffix_len: 0 } + pub fn delim_span(&self) -> DelimSpan { + DelimSpan { open: self.open, close: self.close } + } } -} - -pub fn token_to_literal(text: &str, span: Span) -> Literal { - use rustc_lexer::LiteralKind; - - let token = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No).next_tuple(); - let Some((rustc_lexer::Token { - kind: rustc_lexer::TokenKind::Literal { kind, suffix_start }, - .. - },)) = token - else { - return Literal::new_no_suffix(text, span, LitKind::Err(())); - }; - - let (kind, start_offset, end_offset) = match kind { - LiteralKind::Int { .. } => (LitKind::Integer, 0, 0), - LiteralKind::Float { .. } => (LitKind::Float, 0, 0), - LiteralKind::Char { terminated } => (LitKind::Char, 1, terminated as usize), - LiteralKind::Byte { terminated } => (LitKind::Byte, 2, terminated as usize), - LiteralKind::Str { terminated } => (LitKind::Str, 1, terminated as usize), - LiteralKind::ByteStr { terminated } => (LitKind::ByteStr, 2, terminated as usize), - LiteralKind::CStr { terminated } => (LitKind::CStr, 2, terminated as usize), - LiteralKind::RawStr { n_hashes } => ( - LitKind::StrRaw(n_hashes.unwrap_or_default()), - 2 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - LiteralKind::RawByteStr { n_hashes } => ( - LitKind::ByteStrRaw(n_hashes.unwrap_or_default()), - 3 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - LiteralKind::RawCStr { n_hashes } => ( - LitKind::CStrRaw(n_hashes.unwrap_or_default()), - 3 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - }; - let (lit, suffix) = text.split_at(suffix_start as usize); - let lit = &lit[start_offset..lit.len() - end_offset]; - let suffix = match suffix { - "" | "_" => "", - // ill-suffixed literals - _ if !matches!(kind, LitKind::Integer | LitKind::Float | LitKind::Err(_)) => { - return Literal::new_no_suffix(text, span, LitKind::Err(())); + impl Ident { + pub fn new(text: &str, span: Span) -> Self { + // let raw_stripped = IdentIsRaw::split_from_symbol(text.as_ref()); + let (is_raw, text) = IdentIsRaw::split_from_symbol(text); + Ident { sym: Symbol::intern(text), span, is_raw } } - suffix => suffix, - }; - - Literal::new(lit, span, kind, suffix) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct Punct { - pub char: char, - pub spacing: Spacing, - pub span: Span, -} - -/// Indicates whether a token can join with the following token to form a -/// compound token. Used for conversions to `proc_macro::Spacing`. Also used to -/// guide pretty-printing, which is where the `JointHidden` value (which isn't -/// part of `proc_macro::Spacing`) comes in useful. -// The discriminants are important for decoding for `storage.rs`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -pub enum Spacing { - /// The token cannot join with the following token to form a compound - /// token. - /// - /// In token streams parsed from source code, the compiler will use `Alone` - /// for any token immediately followed by whitespace, a non-doc comment, or - /// EOF. - /// - /// When constructing token streams within the compiler, use this for each - /// token that (a) should be pretty-printed with a space after it, or (b) - /// is the last token in the stream. (In the latter case the choice of - /// spacing doesn't matter because it is never used for the last token. We - /// arbitrarily use `Alone`.) - /// - /// Converts to `proc_macro::Spacing::Alone`, and - /// `proc_macro::Spacing::Alone` converts back to this. - Alone = 0, - - /// The token can join with the following token to form a compound token. - /// - /// In token streams parsed from source code, the compiler will use `Joint` - /// for any token immediately followed by punctuation (as determined by - /// `Token::is_punct`). - /// - /// When constructing token streams within the compiler, use this for each - /// token that (a) should be pretty-printed without a space after it, and - /// (b) is followed by a punctuation token. - /// - /// Converts to `proc_macro::Spacing::Joint`, and - /// `proc_macro::Spacing::Joint` converts back to this. - Joint = 1, - - /// The token can join with the following token to form a compound token, - /// but this will not be visible at the proc macro level. (This is what the - /// `Hidden` means; see below.) - /// - /// In token streams parsed from source code, the compiler will use - /// `JointHidden` for any token immediately followed by anything not - /// covered by the `Alone` and `Joint` cases: an identifier, lifetime, - /// literal, delimiter, doc comment. - /// - /// When constructing token streams, use this for each token that (a) - /// should be pretty-printed without a space after it, and (b) is followed - /// by a non-punctuation token. - /// - /// Converts to `proc_macro::Spacing::Alone`, but - /// `proc_macro::Spacing::Alone` converts back to `token::Spacing::Alone`. - /// Because of that, pretty-printing of `TokenStream`s produced by proc - /// macros is unavoidably uglier (with more whitespace between tokens) than - /// pretty-printing of `TokenStream`'s produced by other means (i.e. parsed - /// source code, internally constructed token streams, and token streams - /// produced by declarative macros). - JointHidden = 2, -} - -/// Identifier or keyword. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Ident { - pub sym: Symbol, - pub span: Span, - pub is_raw: IdentIsRaw, -} - -impl Ident { - pub fn new(text: &str, span: Span) -> Self { - // let raw_stripped = IdentIsRaw::split_from_symbol(text.as_ref()); - let (is_raw, text) = IdentIsRaw::split_from_symbol(text); - Ident { sym: Symbol::intern(text), span, is_raw } } -} -fn print_debug_subtree( - f: &mut fmt::Formatter<'_>, - subtree: &Subtree, - level: usize, - iter: TtIter<'_>, -) -> fmt::Result { - let align = " ".repeat(level); - - let Delimiter { kind, open, close } = &subtree.delimiter; - let delim = match kind { - DelimiterKind::Invisible => "$$", - DelimiterKind::Parenthesis => "()", - DelimiterKind::Brace => "{}", - DelimiterKind::Bracket => "[]", - }; + fn print_debug_subtree( + f: &mut fmt::Formatter<'_>, + subtree: &Subtree, + level: usize, + iter: TtIter<'_>, + ) -> fmt::Result { + let Delimiter { kind, open, close } = &subtree.delimiter; + let delim = kind.debug_view(); + + write!(f, "SUBTREE {delim} ",)?; + write!(f, "{open:#?}")?; + write!(f, " ")?; + write!(f, "{close:#?}")?; + for child in iter { + writeln!(f)?; + print_debug_token(f, level + 1, child)?; + } - write!(f, "{align}SUBTREE {delim} ",)?; - write!(f, "{open:#?}")?; - write!(f, " ")?; - write!(f, "{close:#?}")?; - for child in iter { - writeln!(f)?; - print_debug_token(f, level + 1, child)?; + Ok(()) } - Ok(()) -} - -fn print_debug_token(f: &mut fmt::Formatter<'_>, level: usize, tt: TtElement<'_>) -> fmt::Result { - let align = " ".repeat(level); + fn print_debug_token( + f: &mut fmt::Formatter<'_>, + level: usize, + tt: TtElement<'_>, + ) -> fmt::Result { + write!(f, "{:indent$}", "", indent = level * 2)?; - match tt { - TtElement::Leaf(leaf) => match leaf { - Leaf::Literal(lit) => { - let (text, suffix) = lit.text_and_suffix(); - write!(f, "{}LITERAL {:?} {}{} {:#?}", align, lit.kind, text, suffix, lit.span)?; - } - Leaf::Punct(punct) => { - write!( - f, - "{}PUNCH {} [{}] {:#?}", - align, - punct.char, - if punct.spacing == Spacing::Alone { "alone" } else { "joint" }, - punct.span - )?; - } - Leaf::Ident(ident) => { - write!( - f, - "{}IDENT {}{} {:#?}", - align, - ident.is_raw.as_str(), - ident.sym, - ident.span - )?; + match tt { + TtElement::Leaf(leaf) => leaf.print_debug(f), + TtElement::Subtree(subtree, subtree_iter) => { + print_debug_subtree(f, &subtree, level, subtree_iter) } - }, - TtElement::Subtree(subtree, subtree_iter) => { - print_debug_subtree(f, &subtree, level, subtree_iter)?; } } - Ok(()) -} - -impl fmt::Debug for TopSubtree { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&self.view(), f) - } -} - -impl fmt::Display for TopSubtree { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.view(), f) - } -} - -impl fmt::Display for Leaf { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Leaf::Ident(it) => fmt::Display::fmt(it, f), - Leaf::Literal(it) => fmt::Display::fmt(it, f), - Leaf::Punct(it) => fmt::Display::fmt(it, f), + impl fmt::Debug for TopSubtree { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.view(), f) } } -} - -impl fmt::Display for Ident { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.is_raw.as_str(), f)?; - fmt::Display::fmt(&self.sym, f) - } -} -impl fmt::Display for Literal { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let (text, suffix) = self.text_and_suffix(); - match self.kind { - LitKind::Byte => write!(f, "b'{}'", text), - LitKind::Char => write!(f, "'{}'", text), - LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", text), - LitKind::Str => write!(f, "\"{}\"", text), - LitKind::ByteStr => write!(f, "b\"{}\"", text), - LitKind::CStr => write!(f, "c\"{}\"", text), - LitKind::StrRaw(num_of_hashes) => { - let num_of_hashes = num_of_hashes as usize; - write!(f, r#"r{0:# { - let num_of_hashes = num_of_hashes as usize; - write!(f, r#"br{0:# { - let num_of_hashes = num_of_hashes as usize; - write!(f, r#"cr{0:#) -> fmt::Result { - fmt::Display::fmt(&self.char, f) + impl fmt::Display for TopSubtree { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.view(), f) + } } -} -impl Subtree { - /// Count the number of tokens recursively - pub fn count(&self) -> usize { - self.usize_len() + impl Subtree { + /// Count the number of tokens recursively + pub fn count(&self) -> usize { + self.usize_len() + } } -} -pub fn pretty(tkns: TokenTreesView<'_>) -> String { - return pretty_impl(tkns.iter()); + pub fn pretty(tkns: TokenTreesView<'_>) -> String { + return pretty_impl(tkns.iter()); - fn tokentree_to_text(tkn: TtElement<'_>) -> String { - match tkn { - TtElement::Leaf(leaf) => { - format!("{}", leaf) - } - TtElement::Subtree(Subtree { delimiter, .. }, subtree_content) => { - let content = pretty_impl(subtree_content); - let (open, close) = match delimiter.kind { - DelimiterKind::Brace => ("{", "}"), - DelimiterKind::Bracket => ("[", "]"), - DelimiterKind::Parenthesis => ("(", ")"), - DelimiterKind::Invisible => ("", ""), - }; - format!("{open}{content}{close}") + fn tokentree_to_text(tkn: TtElement<'_>) -> String { + match tkn { + TtElement::Leaf(leaf) => { + format!("{}", leaf) + } + TtElement::Subtree(Subtree { delimiter, .. }, subtree_content) => { + let content = pretty_impl(subtree_content); + let (open, close) = delimiter.kind.display_open_close(); + format!("{open}{content}{close}") + } } } - } - fn pretty_impl(tkns: TtIter<'_>) -> String { - let mut last = String::new(); - let mut last_to_joint = true; - - for tkn in tkns { - last = - [last, tokentree_to_text(tkn.clone())].join(if last_to_joint { "" } else { " " }); - last_to_joint = false; - if let TtElement::Leaf(Leaf::Punct(Punct { spacing, .. })) = tkn - && spacing == Spacing::Joint - { - last_to_joint = true; + fn pretty_impl(tkns: TtIter<'_>) -> String { + let mut last = String::new(); + let mut last_to_joint = true; + + for tkn in tkns { + last = [last, tokentree_to_text(tkn.clone())].join(if last_to_joint { + "" + } else { + " " + }); + last_to_joint = false; + if let TtElement::Leaf(Leaf::Punct(Punct { spacing, .. })) = tkn + && spacing == Spacing::Joint + { + last_to_joint = true; + } } + last } - last } -} -#[derive(Debug)] -pub enum TransformTtAction<'a> { - Keep, - ReplaceWith(TokenTreesView<'a>), -} - -impl TransformTtAction<'_> { - #[inline] - pub fn remove() -> Self { - Self::ReplaceWith(TokenTreesView::empty()) + #[derive(Debug)] + pub enum TransformTtAction<'a> { + Keep, + ReplaceWith(TokenTreesView<'a>), } -} -/// This function takes a token tree, and calls `callback` with each token tree in it. -/// Then it does what the callback says: keeps the tt or replaces it with a (possibly empty) -/// tts view. -pub fn transform_tt<'b>( - tt: &mut TopSubtree, - mut callback: impl FnMut(&TokenTree) -> TransformTtAction<'b>, -) { - let mut tt_vec = tt.as_token_trees().iter_flat_tokens().collect::>(); - - // We need to keep a stack of the currently open subtrees, because we need to update - // them if we change the number of items in them. - let mut subtrees_stack = Vec::new(); - let mut i = 0; - while i < tt_vec.len() { - 'pop_finished_subtrees: while let Some(&subtree_idx) = subtrees_stack.last() { - let TokenTree::Subtree(subtree) = &tt_vec[subtree_idx] else { - unreachable!("non-subtree on subtrees stack"); - }; - if i >= subtree_idx + 1 + subtree.usize_len() { - subtrees_stack.pop(); - } else { - break 'pop_finished_subtrees; - } + impl TransformTtAction<'_> { + #[inline] + pub fn remove() -> Self { + Self::ReplaceWith(TokenTreesView::empty()) } + } - let current = &tt_vec[i]; - let action = callback(current); - match action { - TransformTtAction::Keep => { - // This cannot be shared with the replaced case, because then we may push the same subtree - // twice, and will update it twice which will lead to errors. - if let TokenTree::Subtree(_) = current { - subtrees_stack.push(i); + /// This function takes a token tree, and calls `callback` with each token tree in it. + /// Then it does what the callback says: keeps the tt or replaces it with a (possibly empty) + /// tts view. + pub fn transform_tt<'b>( + tt: &mut TopSubtree, + mut callback: impl FnMut(&TokenTree) -> TransformTtAction<'b>, + ) { + let mut tt_vec = tt.as_token_trees().iter_flat_tokens().collect::>(); + + // We need to keep a stack of the currently open subtrees, because we need to update + // them if we change the number of items in them. + let mut subtrees_stack = Vec::new(); + let mut i = 0; + while i < tt_vec.len() { + 'pop_finished_subtrees: while let Some(&subtree_idx) = subtrees_stack.last() { + let TokenTree::Subtree(subtree) = &tt_vec[subtree_idx] else { + unreachable!("non-subtree on subtrees stack"); + }; + if i >= subtree_idx + 1 + subtree.usize_len() { + subtrees_stack.pop(); + } else { + break 'pop_finished_subtrees; } - - i += 1; } - TransformTtAction::ReplaceWith(replacement) => { - let old_len = 1 + match current { - TokenTree::Leaf(_) => 0, - TokenTree::Subtree(subtree) => subtree.usize_len(), - }; - let len_diff = replacement.len() as i64 - old_len as i64; - tt_vec.splice(i..i + old_len, replacement.iter_flat_tokens()); - // Skip the newly inserted replacement, we don't want to visit it. - i += replacement.len(); - - for &subtree_idx in &subtrees_stack { - let TokenTree::Subtree(subtree) = &mut tt_vec[subtree_idx] else { - unreachable!("non-subtree on subtrees stack"); + + let current = &tt_vec[i]; + let action = callback(current); + match action { + TransformTtAction::Keep => { + // This cannot be shared with the replaced case, because then we may push the same subtree + // twice, and will update it twice which will lead to errors. + if let TokenTree::Subtree(_) = current { + subtrees_stack.push(i); + } + + i += 1; + } + TransformTtAction::ReplaceWith(replacement) => { + let old_len = 1 + match current { + TokenTree::Leaf(_) => 0, + TokenTree::Subtree(subtree) => subtree.usize_len(), }; - subtree.len = (i64::from(subtree.len) + len_diff).try_into().unwrap(); + let len_diff = replacement.len() as i64 - old_len as i64; + tt_vec.splice(i..i + old_len, replacement.iter_flat_tokens()); + // Skip the newly inserted replacement, we don't want to visit it. + i += replacement.len(); + + for &subtree_idx in &subtrees_stack { + let TokenTree::Subtree(subtree) = &mut tt_vec[subtree_idx] else { + unreachable!("non-subtree on subtrees stack"); + }; + subtree.len = (i64::from(subtree.len) + len_diff).try_into().unwrap(); + } } } } - } - *tt = TopSubtree::from_serialized(tt_vec); + *tt = TopSubtree::from_serialized(tt_vec); + } }