From 539d21db9e5daab8c18a46a823c05a22bc7fb54e Mon Sep 17 00:00:00 2001 From: cyrgani Date: Thu, 29 Jan 2026 15:03:29 +0000 Subject: [PATCH 1/8] remove `TokenStream` from the bridge --- compiler/rustc_expand/src/proc_macro.rs | 46 ++++--- .../rustc_expand/src/proc_macro_server.rs | 124 +++++++----------- library/proc_macro/src/bridge/client.rs | 48 +------ library/proc_macro/src/bridge/handle.rs | 4 - library/proc_macro/src/bridge/mod.rs | 65 ++++----- library/proc_macro/src/bridge/server.rs | 47 ++----- library/proc_macro/src/lib.rs | 94 ++++++------- 7 files changed, 156 insertions(+), 272 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index ea63ff7bfc46a..a0381317eeee0 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -57,15 +57,19 @@ impl base::BangProcMacro for BangProcMacro { let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace; let strategy = exec_strategy(ecx.sess); - let server = proc_macro_server::Rustc::new(ecx); - self.client.run(&strategy, server, input, proc_macro_backtrace).map_err(|e| { - ecx.dcx().emit_err(errors::ProcMacroPanicked { + let mut server = proc_macro_server::Rustc::new(ecx); + let input = server.ts_rustc_to_pm(input); + let output = self.client.run(&strategy, &mut server, input, proc_macro_backtrace); + + match output { + Ok(stream) => Ok(server.ts_pm_to_rustc(stream)), + Err(e) => Err(ecx.dcx().emit_err(errors::ProcMacroPanicked { span, message: e .as_str() .map(|message| errors::ProcMacroPanickedHelp { message: message.into() }), - }) - }) + })), + } } } @@ -88,17 +92,20 @@ impl base::AttrProcMacro for AttrProcMacro { let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace; let strategy = exec_strategy(ecx.sess); - let server = proc_macro_server::Rustc::new(ecx); - self.client.run(&strategy, server, annotation, annotated, proc_macro_backtrace).map_err( - |e| { - ecx.dcx().emit_err(errors::CustomAttributePanicked { - span, - message: e.as_str().map(|message| errors::CustomAttributePanickedHelp { - message: message.into(), - }), - }) - }, - ) + let mut server = proc_macro_server::Rustc::new(ecx); + let annotation = server.ts_rustc_to_pm(annotation); + let annotated = server.ts_rustc_to_pm(annotated); + let output = + self.client.run(&strategy, &mut server, annotation, annotated, proc_macro_backtrace); + match output { + Ok(stream) => Ok(server.ts_pm_to_rustc(stream)), + Err(e) => Err(ecx.dcx().emit_err(errors::CustomAttributePanicked { + span, + message: e + .as_str() + .map(|message| errors::CustomAttributePanickedHelp { message: message.into() }), + })), + } } } @@ -221,10 +228,11 @@ fn expand_derive_macro( let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace; let strategy = exec_strategy(ecx.sess); - let server = proc_macro_server::Rustc::new(ecx); + let mut server = proc_macro_server::Rustc::new(ecx); + let input = server.ts_rustc_to_pm(input); - match client.run(&strategy, server, input, proc_macro_backtrace) { - Ok(stream) => Ok(stream), + match client.run(&strategy, &mut server, input, proc_macro_backtrace) { + Ok(stream) => Ok(server.ts_pm_to_rustc(stream)), Err(e) => { let invoc_expn_data = invoc_id.expn_data(); let span = invoc_expn_data.call_site; diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 947b8a6e3e5ee..e4477fc7148d5 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -103,7 +103,7 @@ impl ToInternal for LitKind { } } -impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec> { +impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec> { fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self { use rustc_ast::token::*; @@ -150,7 +150,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec)> for Vec)> for Vec)> for Vec> - for (TokenTree, &mut Rustc<'_, '_>) + for (TokenTree, &mut Rustc<'_, '_>) { fn to_internal(self) -> SmallVec<[tokenstream::TokenTree; 2]> { use rustc_ast::token::*; @@ -378,7 +378,7 @@ impl ToInternal> tokenstream::DelimSpan { open, close }, DelimSpacing::new(Spacing::Alone, Spacing::Alone), delimiter.to_internal(), - stream.unwrap_or_default(), + rustc.ts_pm_to_rustc(stream.unwrap_or_default()), )] } TokenTree::Ident(self::Ident { sym, is_raw, span }) => { @@ -431,6 +431,8 @@ impl ToInternal for Level { } } +type BridgeTokenStream = rustc_proc_macro::bridge::TokenStream; + pub(crate) struct Rustc<'a, 'b> { ecx: &'a mut ExtCtxt<'b>, def_site: Span, @@ -456,10 +458,27 @@ impl<'a, 'b> Rustc<'a, 'b> { fn psess(&self) -> &ParseSess { self.ecx.psess() } + + pub(crate) fn ts_pm_to_rustc(&mut self, ts: BridgeTokenStream) -> tokenstream::TokenStream { + let mut t = tokenstream::TokenStream::new(vec![]); + for tree in ts.trees { + let internal: SmallVec<[tokenstream::TokenTree; 2]> = + ToInternal::to_internal((tree, &mut *self)); + for tree in internal { + t.push_tree(tree); + } + } + + t + } + + pub(crate) fn ts_rustc_to_pm(&mut self, ts: tokenstream::TokenStream) -> BridgeTokenStream { + let trees: Vec> = FromInternal::from_internal((ts, self)); + BridgeTokenStream { trees } + } } -impl server::Server for Rustc<'_, '_> { - type TokenStream = TokenStream; +impl server::Server for &mut Rustc<'_, '_> { type Span = Span; type Symbol = Symbol; @@ -564,35 +583,24 @@ impl server::Server for Rustc<'_, '_> { diag.emit(); } - fn ts_drop(&mut self, stream: Self::TokenStream) { - drop(stream); - } - - fn ts_clone(&mut self, stream: &Self::TokenStream) -> Self::TokenStream { - stream.clone() - } - - fn ts_is_empty(&mut self, stream: &Self::TokenStream) -> bool { - stream.is_empty() - } - - fn ts_from_str(&mut self, src: &str) -> Self::TokenStream { - unwrap_or_emit_fatal(source_str_to_stream( + fn ts_from_str(&mut self, src: &str) -> BridgeTokenStream { + self.ts_rustc_to_pm(unwrap_or_emit_fatal(source_str_to_stream( self.psess(), FileName::proc_macro_source_code(src), src.to_string(), Some(self.call_site), - )) + ))) } - fn ts_to_string(&mut self, stream: &Self::TokenStream) -> String { - pprust::tts_to_string(stream) + fn ts_to_string(&mut self, stream: BridgeTokenStream) -> String { + pprust::tts_to_string(&self.ts_pm_to_rustc(stream)) } - fn ts_expand_expr(&mut self, stream: &Self::TokenStream) -> Result { + fn ts_expand_expr(&mut self, stream: BridgeTokenStream) -> Result { // Parse the expression from our tokenstream. let expr = try { - let mut p = Parser::new(self.psess(), stream.clone(), Some("proc_macro expand expr")); + let stream = self.ts_pm_to_rustc(stream); + let mut p = Parser::new(self.psess(), stream, Some("proc_macro expand expr")); let expr = p.parse_expr()?; if p.token != token::Eof { p.unexpected()?; @@ -615,32 +623,34 @@ impl server::Server for Rustc<'_, '_> { // We don't use `TokenStream::from_ast` as the tokenstream currently cannot // be recovered in the general case. match &expr.kind { - ast::ExprKind::Lit(token_lit) if token_lit.kind == token::Bool => { - Ok(tokenstream::TokenStream::token_alone( + ast::ExprKind::Lit(token_lit) if token_lit.kind == token::Bool => Ok(self + .ts_rustc_to_pm(tokenstream::TokenStream::token_alone( token::Ident(token_lit.symbol, IdentIsRaw::No), expr.span, - )) - } - ast::ExprKind::Lit(token_lit) => { - Ok(tokenstream::TokenStream::token_alone(token::Literal(*token_lit), expr.span)) - } + ))), + ast::ExprKind::Lit(token_lit) => Ok(self.ts_rustc_to_pm( + tokenstream::TokenStream::token_alone(token::Literal(*token_lit), expr.span), + )), ast::ExprKind::IncludedBytes(byte_sym) => { let lit = token::Lit::new( token::ByteStr, escape_byte_str_symbol(byte_sym.as_byte_str()), None, ); - Ok(tokenstream::TokenStream::token_alone(token::TokenKind::Literal(lit), expr.span)) + Ok(self.ts_rustc_to_pm(tokenstream::TokenStream::token_alone( + token::TokenKind::Literal(lit), + expr.span, + ))) } ast::ExprKind::Unary(ast::UnOp::Neg, e) => match &e.kind { ast::ExprKind::Lit(token_lit) => match token_lit { token::Lit { kind: token::Integer | token::Float, .. } => { - Ok(Self::TokenStream::from_iter([ + Ok(self.ts_rustc_to_pm(tokenstream::TokenStream::from_iter([ // FIXME: The span of the `-` token is lost when // parsing, so we cannot faithfully recover it here. tokenstream::TokenTree::token_joint_hidden(token::Minus, e.span), tokenstream::TokenTree::token_alone(token::Literal(*token_lit), e.span), - ])) + ]))) } _ => Err(()), }, @@ -650,46 +660,6 @@ impl server::Server for Rustc<'_, '_> { } } - fn ts_from_token_tree( - &mut self, - tree: TokenTree, - ) -> Self::TokenStream { - Self::TokenStream::new((tree, &mut *self).to_internal().into_iter().collect::>()) - } - - fn ts_concat_trees( - &mut self, - base: Option, - trees: Vec>, - ) -> Self::TokenStream { - let mut stream = base.unwrap_or_default(); - for tree in trees { - for tt in (tree, &mut *self).to_internal() { - stream.push_tree(tt); - } - } - stream - } - - fn ts_concat_streams( - &mut self, - base: Option, - streams: Vec, - ) -> Self::TokenStream { - let mut stream = base.unwrap_or_default(); - for s in streams { - stream.push_stream(s); - } - stream - } - - fn ts_into_trees( - &mut self, - stream: Self::TokenStream, - ) -> Vec> { - FromInternal::from_internal((stream, self)) - } - fn span_debug(&mut self, span: Self::Span) -> String { if self.ecx.ecfg.span_debug { format!("{span:?}") diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index 02a408802b6fa..f9b20eb20ed82 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -8,44 +8,10 @@ use super::*; #[repr(C)] pub(super) struct HandleCounters { - pub(super) token_stream: AtomicU32, pub(super) span: AtomicU32, } -static COUNTERS: HandleCounters = - HandleCounters { token_stream: AtomicU32::new(1), span: AtomicU32::new(1) }; - -pub(crate) struct TokenStream { - handle: handle::Handle, -} - -impl !Send for TokenStream {} -impl !Sync for TokenStream {} - -// Forward `Drop::drop` to the inherent `drop` method. -impl Drop for TokenStream { - fn drop(&mut self) { - Methods::ts_drop(TokenStream { handle: self.handle }); - } -} - -impl Encode for TokenStream { - fn encode(self, w: &mut Buffer, s: &mut S) { - mem::ManuallyDrop::new(self).handle.encode(w, s); - } -} - -impl Encode for &TokenStream { - fn encode(self, w: &mut Buffer, s: &mut S) { - self.handle.encode(w, s); - } -} - -impl Decode<'_, '_, S> for TokenStream { - fn decode(r: &mut &[u8], s: &mut S) -> Self { - TokenStream { handle: handle::Handle::decode(r, s) } - } -} +static COUNTERS: HandleCounters = HandleCounters { span: AtomicU32::new(1) }; #[derive(Copy, Clone, PartialEq, Eq, Hash)] pub(crate) struct Span { @@ -67,12 +33,6 @@ impl Decode<'_, '_, S> for Span { } } -impl Clone for TokenStream { - fn clone(&self) -> Self { - Methods::ts_clone(self) - } -} - impl Span { pub(crate) fn def_site() -> Span { Bridge::with(|bridge| bridge.globals.def_site) @@ -121,7 +81,7 @@ macro_rules! define_client_side { } } } -with_api!(define_client_side, TokenStream, Span, Symbol); +with_api!(define_client_side, Span, Symbol); struct Bridge<'a> { /// Reusable buffer (only `clear`-ed, never shrunk), primarily @@ -296,7 +256,7 @@ impl Client { Client { handle_counters: &COUNTERS, run: super::selfless_reify::reify_to_extern_c_fn_hrt_bridge(move |bridge| { - run_client(bridge, |input| f(crate::TokenStream(Some(input))).0) + run_client(bridge, |input| f(crate::TokenStream(input)).0) }), _marker: PhantomData, } @@ -311,7 +271,7 @@ impl Client<(crate::TokenStream, crate::TokenStream), crate::TokenStream> { handle_counters: &COUNTERS, run: super::selfless_reify::reify_to_extern_c_fn_hrt_bridge(move |bridge| { run_client(bridge, |(input, input2)| { - f(crate::TokenStream(Some(input)), crate::TokenStream(Some(input2))).0 + f(crate::TokenStream(input), crate::TokenStream(input2)).0 }) }), _marker: PhantomData, diff --git a/library/proc_macro/src/bridge/handle.rs b/library/proc_macro/src/bridge/handle.rs index f0c01e39de32d..961227022147b 100644 --- a/library/proc_macro/src/bridge/handle.rs +++ b/library/proc_macro/src/bridge/handle.rs @@ -34,10 +34,6 @@ impl OwnedStore { assert!(self.data.insert(handle, x).is_none()); handle } - - pub(super) fn take(&mut self, h: Handle) -> T { - self.data.remove(&h).expect("use-after-free in `proc_macro` handle") - } } impl Index for OwnedStore { diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index 6a9027046af00..8bca55af4baa4 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -11,7 +11,7 @@ use std::hash::Hash; use std::ops::{Bound, Range}; use std::sync::Once; -use std::{fmt, marker, mem, panic, thread}; +use std::{fmt, marker, panic, thread}; use crate::{Delimiter, Level}; @@ -32,7 +32,7 @@ use crate::{Delimiter, Level}; /// special handling, to enable several different representations of /// these types. macro_rules! with_api { - ($m:ident, $TokenStream: path, $Span: path, $Symbol: path) => { + ($m:ident, $Span: path, $Symbol: path) => { $m! { fn injected_env_var(var: &str) -> Option; fn track_env_var(var: &str, value: Option<&str>); @@ -40,26 +40,9 @@ macro_rules! with_api { fn literal_from_str(s: &str) -> Result, ()>; fn emit_diagnostic(diagnostic: Diagnostic<$Span>); - fn ts_drop(stream: $TokenStream); - fn ts_clone(stream: &$TokenStream) -> $TokenStream; - fn ts_is_empty(stream: &$TokenStream) -> bool; - fn ts_expand_expr(stream: &$TokenStream) -> Result<$TokenStream, ()>; - fn ts_from_str(src: &str) -> $TokenStream; - fn ts_to_string(stream: &$TokenStream) -> String; - fn ts_from_token_tree( - tree: TokenTree<$TokenStream, $Span, $Symbol>, - ) -> $TokenStream; - fn ts_concat_trees( - base: Option<$TokenStream>, - trees: Vec>, - ) -> $TokenStream; - fn ts_concat_streams( - base: Option<$TokenStream>, - streams: Vec<$TokenStream>, - ) -> $TokenStream; - fn ts_into_trees( - stream: $TokenStream - ) -> Vec>; + fn ts_expand_expr(stream: TokenStream<$Span, $Symbol>) -> Result, ()>; + fn ts_from_str(src: &str) -> TokenStream<$Span, $Symbol>; + fn ts_to_string(stream: TokenStream<$Span, $Symbol>) -> String; fn span_debug(span: $Span) -> String; fn span_parent(span: $Span) -> Option<$Span>; @@ -142,7 +125,7 @@ macro_rules! declare_tags { rpc_encode_decode!(enum ApiTags { $($method),* }); } } -with_api!(declare_tags, __, __, __); +with_api!(declare_tags, __, __); /// Helper to wrap associated types to allow trait impl dispatch. /// That is, normally a pair of impls for `T::Foo` and `T::Bar` @@ -169,15 +152,6 @@ impl Mark for Marked { self.value } } -impl<'a, T> Mark for &'a Marked { - type Unmarked = &'a T; - fn mark(_: Self::Unmarked) -> Self { - unreachable!() - } - fn unmark(self) -> Self::Unmarked { - &self.value - } -} impl Mark for Vec { type Unmarked = Vec; @@ -354,13 +328,13 @@ impl DelimSpan { compound_traits!(struct DelimSpan { open, close, entire }); #[derive(Clone)] -pub struct Group { +pub struct Group { pub delimiter: Delimiter, - pub stream: Option, + pub stream: Option>, pub span: DelimSpan, } -compound_traits!(struct Group { delimiter, stream, span }); +compound_traits!(struct Group { delimiter, stream, span }); #[derive(Clone)] pub struct Punct { @@ -391,15 +365,15 @@ pub struct Literal { compound_traits!(struct Literal { kind, symbol, suffix, span }); #[derive(Clone)] -pub enum TokenTree { - Group(Group), +pub enum TokenTree { + Group(Group), Punct(Punct), Ident(Ident), Literal(Literal), } compound_traits!( - enum TokenTree { + enum TokenTree { Group(tt), Punct(tt), Ident(tt), @@ -407,6 +381,21 @@ compound_traits!( } ); +#[derive(Clone)] +pub struct TokenStream { + pub trees: Vec>, +} + +impl Default for TokenStream { + fn default() -> Self { + Self { trees: Vec::new() } + } +} + +compound_traits!( + struct TokenStream { trees } +); + #[derive(Clone, Debug)] pub struct Diagnostic { pub level: Level, diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index a3c6a232264e0..9f67e222bdfdb 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -4,43 +4,23 @@ use std::cell::Cell; use std::marker::PhantomData; use super::*; +use crate::bridge; pub(super) struct HandleStore { - token_stream: handle::OwnedStore>, span: handle::InternedStore>, } impl HandleStore { fn new(handle_counters: &'static client::HandleCounters) -> Self { - HandleStore { - token_stream: handle::OwnedStore::new(&handle_counters.token_stream), - span: handle::InternedStore::new(&handle_counters.span), - } + HandleStore { span: handle::InternedStore::new(&handle_counters.span) } } } -pub(super) type MarkedTokenStream = Marked<::TokenStream, client::TokenStream>; +pub(super) type MarkedTokenStream = bridge::TokenStream, MarkedSymbol>; +//bridge::TokenStream; pub(super) type MarkedSpan = Marked<::Span, client::Span>; pub(super) type MarkedSymbol = Marked<::Symbol, client::Symbol>; -impl Encode> for MarkedTokenStream { - fn encode(self, w: &mut Buffer, s: &mut HandleStore) { - s.token_stream.alloc(self).encode(w, s); - } -} - -impl Decode<'_, '_, HandleStore> for MarkedTokenStream { - fn decode(r: &mut &[u8], s: &mut HandleStore) -> Self { - s.token_stream.take(handle::Handle::decode(r, &mut ())) - } -} - -impl<'s, S: Server> Decode<'_, 's, HandleStore> for &'s MarkedTokenStream { - fn decode(r: &mut &[u8], s: &'s mut HandleStore) -> Self { - &s.token_stream[handle::Handle::decode(r, &mut ())] - } -} - impl Encode> for MarkedSpan { fn encode(self, w: &mut Buffer, s: &mut HandleStore) { s.span.alloc(self).encode(w, s); @@ -63,7 +43,6 @@ macro_rules! define_server { $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)*;)* ) => { pub trait Server { - type TokenStream: 'static + Clone + Default; type Span: 'static + Copy + Eq + Hash; type Symbol: 'static; @@ -79,7 +58,7 @@ macro_rules! define_server { } } } -with_api!(define_server, Self::TokenStream, Self::Span, Self::Symbol); +with_api!(define_server, Self::Span, Self::Symbol); macro_rules! define_dispatcher { ( @@ -125,7 +104,7 @@ macro_rules! define_dispatcher { } } } -with_api!(define_dispatcher, MarkedTokenStream, MarkedSpan, MarkedSymbol); +with_api!(define_dispatcher, MarkedSpan, MarkedSymbol); pub trait ExecutionStrategy { fn run_bridge_and_client( @@ -305,9 +284,9 @@ impl client::Client { &self, strategy: &impl ExecutionStrategy, server: S, - input: S::TokenStream, + input: TokenStream, force_show_panics: bool, - ) -> Result + ) -> Result, PanicMessage> where S: Server, { @@ -320,7 +299,7 @@ impl client::Client { run, force_show_panics, ) - .map(|s| >>::unmark(s).unwrap_or_default()) + .map(|s| >::unmark(s)) } } @@ -329,10 +308,10 @@ impl client::Client<(crate::TokenStream, crate::TokenStream), crate::TokenStream &self, strategy: &impl ExecutionStrategy, server: S, - input: S::TokenStream, - input2: S::TokenStream, + input: TokenStream, + input2: TokenStream, force_show_panics: bool, - ) -> Result + ) -> Result, PanicMessage> where S: Server, { @@ -345,6 +324,6 @@ impl client::Client<(crate::TokenStream, crate::TokenStream), crate::TokenStream run, force_show_panics, ) - .map(|s| >>::unmark(s).unwrap_or_default()) + .map(|s| >::unmark(s)) } } diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index e2f39c015bdd7..66f5798d4c70d 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -102,7 +102,7 @@ pub fn is_available() -> bool { #[cfg_attr(feature = "rustc-dep-of-std", rustc_diagnostic_item = "TokenStream")] #[stable(feature = "proc_macro_lib", since = "1.15.0")] #[derive(Clone)] -pub struct TokenStream(Option); +pub struct TokenStream(Vec>); #[stable(feature = "proc_macro_lib", since = "1.15.0")] impl !Send for TokenStream {} @@ -156,13 +156,13 @@ impl TokenStream { /// Returns an empty `TokenStream` containing no token trees. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new() -> TokenStream { - TokenStream(None) + TokenStream(Vec::new()) } /// Checks if this `TokenStream` is empty. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn is_empty(&self) -> bool { - self.0.as_ref().map(|h| BridgeMethods::ts_is_empty(h)).unwrap_or(true) + self.0.is_empty() } /// Parses this `TokenStream` as an expression and attempts to expand any @@ -177,9 +177,8 @@ impl TokenStream { /// considered errors, is unspecified and may change in the future. #[unstable(feature = "proc_macro_expand", issue = "90765")] pub fn expand_expr(&self) -> Result { - let stream = self.0.as_ref().ok_or(ExpandError)?; - match BridgeMethods::ts_expand_expr(stream) { - Ok(stream) => Ok(TokenStream(Some(stream))), + match BridgeMethods::ts_expand_expr(stream_to_bridge_stream(self.clone())) { + Ok(stream) => Ok(TokenStream(stream.trees)), Err(_) => Err(ExpandError), } } @@ -197,7 +196,7 @@ impl FromStr for TokenStream { type Err = LexError; fn from_str(src: &str) -> Result { - Ok(TokenStream(Some(BridgeMethods::ts_from_str(src)))) + Ok(TokenStream(BridgeMethods::ts_from_str(src).trees)) } } @@ -215,10 +214,7 @@ impl FromStr for TokenStream { #[stable(feature = "proc_macro_lib", since = "1.15.0")] impl fmt::Display for TokenStream { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match &self.0 { - Some(ts) => write!(f, "{}", BridgeMethods::ts_to_string(ts)), - None => Ok(()), - } + write!(f, "{}", BridgeMethods::ts_to_string(stream_to_bridge_stream(self.clone()))) } } @@ -241,9 +237,15 @@ impl Default for TokenStream { #[unstable(feature = "proc_macro_quote", issue = "54722")] pub use quote::{HasIterator, RepInterp, ThereIsNoIteratorInRepetition, ext, quote, quote_span}; +fn stream_to_bridge_stream( + stream: TokenStream, +) -> bridge::TokenStream { + bridge::TokenStream { trees: stream.0 } +} + fn tree_to_bridge_tree( tree: TokenTree, -) -> bridge::TokenTree { +) -> bridge::TokenTree { match tree { TokenTree::Group(tt) => bridge::TokenTree::Group(tt.0), TokenTree::Punct(tt) => bridge::TokenTree::Punct(tt.0), @@ -256,20 +258,14 @@ fn tree_to_bridge_tree( #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl From for TokenStream { fn from(tree: TokenTree) -> TokenStream { - TokenStream(Some(BridgeMethods::ts_from_token_tree(tree_to_bridge_tree(tree)))) + TokenStream(vec![tree_to_bridge_tree(tree)]) } } /// Non-generic helper for implementing `FromIterator` and /// `Extend` with less monomorphization in calling crates. struct ConcatTreesHelper { - trees: Vec< - bridge::TokenTree< - bridge::client::TokenStream, - bridge::client::Span, - bridge::client::Symbol, - >, - >, + trees: Vec>, } impl ConcatTreesHelper { @@ -282,25 +278,21 @@ impl ConcatTreesHelper { } fn build(self) -> TokenStream { - if self.trees.is_empty() { - TokenStream(None) - } else { - TokenStream(Some(BridgeMethods::ts_concat_trees(None, self.trees))) - } + TokenStream(self.trees) } - fn append_to(self, stream: &mut TokenStream) { + fn append_to(mut self, stream: &mut TokenStream) { if self.trees.is_empty() { return; } - stream.0 = Some(BridgeMethods::ts_concat_trees(stream.0.take(), self.trees)) + stream.0.append(&mut self.trees); } } /// Non-generic helper for implementing `FromIterator` and /// `Extend` with less monomorphization in calling crates. struct ConcatStreamsHelper { - streams: Vec, + streams: Vec, } impl ConcatStreamsHelper { @@ -309,28 +301,23 @@ impl ConcatStreamsHelper { } fn push(&mut self, stream: TokenStream) { - if let Some(stream) = stream.0 { - self.streams.push(stream); - } + self.streams.push(stream); } - fn build(mut self) -> TokenStream { - if self.streams.len() <= 1 { - TokenStream(self.streams.pop()) - } else { - TokenStream(Some(BridgeMethods::ts_concat_streams(None, self.streams))) + fn build(self) -> TokenStream { + let mut stream = TokenStream::new(); + for s in self.streams { + stream.extend(s); } + stream } - fn append_to(mut self, stream: &mut TokenStream) { + fn append_to(self, stream: &mut TokenStream) { if self.streams.is_empty() { return; } - let base = stream.0.take(); - if base.is_none() && self.streams.len() == 1 { - stream.0 = self.streams.pop(); - } else { - stream.0 = Some(BridgeMethods::ts_concat_streams(base, self.streams)); + for mut s in self.streams { + stream.0.append(&mut s.0); } } } @@ -396,7 +383,7 @@ extend_items!(Group Literal Punct Ident); /// Public implementation details for the `TokenStream` type, such as iterators. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub mod token_stream { - use crate::{BridgeMethods, Group, Ident, Literal, Punct, TokenStream, TokenTree, bridge}; + use crate::{Group, Ident, Literal, Punct, TokenStream, TokenTree, bridge}; /// An iterator over `TokenStream`'s `TokenTree`s. /// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups, @@ -404,13 +391,7 @@ pub mod token_stream { #[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub struct IntoIter( - std::vec::IntoIter< - bridge::TokenTree< - bridge::client::TokenStream, - bridge::client::Span, - bridge::client::Symbol, - >, - >, + std::vec::IntoIter>, ); #[stable(feature = "proc_macro_lib2", since = "1.29.0")] @@ -441,9 +422,7 @@ pub mod token_stream { type IntoIter = IntoIter; fn into_iter(self) -> IntoIter { - IntoIter( - self.0.map(|v| BridgeMethods::ts_into_trees(v)).unwrap_or_default().into_iter(), - ) + IntoIter(self.0.into_iter()) } } } @@ -770,7 +749,7 @@ impl fmt::Display for TokenTree { /// A `Group` internally contains a `TokenStream` which is surrounded by `Delimiter`s. #[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] -pub struct Group(bridge::Group); +pub struct Group(bridge::Group); #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl !Send for Group {} @@ -821,7 +800,7 @@ impl Group { pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group { Group(bridge::Group { delimiter, - stream: stream.0, + stream: Some(stream_to_bridge_stream(stream)), span: bridge::DelimSpan::from_single(Span::call_site().0), }) } @@ -838,7 +817,10 @@ impl Group { /// returned above. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn stream(&self) -> TokenStream { - TokenStream(self.0.stream.clone()) + match &self.0.stream { + Some(stream) => TokenStream(stream.trees.clone()), + None => TokenStream(vec![]), + } } /// Returns the span for the delimiters of this token stream, spanning the From 2ac0b71f724a43e1b2f9238284155ce04c38bc8d Mon Sep 17 00:00:00 2001 From: cyrgani Date: Fri, 30 Jan 2026 22:08:35 +0000 Subject: [PATCH 2/8] use rc, take 3 --- library/proc_macro/src/bridge/client.rs | 10 +++++++-- library/proc_macro/src/lib.rs | 28 ++++++++++++++----------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index f9b20eb20ed82..fa8479cbede36 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -2,6 +2,7 @@ use std::cell::RefCell; use std::marker::PhantomData; +use std::rc::Rc; use std::sync::atomic::AtomicU32; use super::*; @@ -256,7 +257,9 @@ impl Client { Client { handle_counters: &COUNTERS, run: super::selfless_reify::reify_to_extern_c_fn_hrt_bridge(move |bridge| { - run_client(bridge, |input| f(crate::TokenStream(input)).0) + run_client(bridge, |input| { + Rc::unwrap_or_clone(f(crate::TokenStream(Rc::new(input))).0) + }) }), _marker: PhantomData, } @@ -271,7 +274,10 @@ impl Client<(crate::TokenStream, crate::TokenStream), crate::TokenStream> { handle_counters: &COUNTERS, run: super::selfless_reify::reify_to_extern_c_fn_hrt_bridge(move |bridge| { run_client(bridge, |(input, input2)| { - f(crate::TokenStream(input), crate::TokenStream(input2)).0 + Rc::unwrap_or_clone( + f(crate::TokenStream(Rc::new(input)), crate::TokenStream(Rc::new(input2))) + .0, + ) }) }), _marker: PhantomData, diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 66f5798d4c70d..d63c0c5f3ddbf 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -48,6 +48,7 @@ use core::ops::BitOr; use std::ffi::CStr; use std::ops::{Range, RangeBounds}; use std::path::PathBuf; +use std::rc::Rc; use std::str::FromStr; use std::{error, fmt}; @@ -102,7 +103,7 @@ pub fn is_available() -> bool { #[cfg_attr(feature = "rustc-dep-of-std", rustc_diagnostic_item = "TokenStream")] #[stable(feature = "proc_macro_lib", since = "1.15.0")] #[derive(Clone)] -pub struct TokenStream(Vec>); +pub struct TokenStream(Rc>>); #[stable(feature = "proc_macro_lib", since = "1.15.0")] impl !Send for TokenStream {} @@ -156,7 +157,7 @@ impl TokenStream { /// Returns an empty `TokenStream` containing no token trees. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new() -> TokenStream { - TokenStream(Vec::new()) + TokenStream(Rc::new(Vec::new())) } /// Checks if this `TokenStream` is empty. @@ -178,7 +179,7 @@ impl TokenStream { #[unstable(feature = "proc_macro_expand", issue = "90765")] pub fn expand_expr(&self) -> Result { match BridgeMethods::ts_expand_expr(stream_to_bridge_stream(self.clone())) { - Ok(stream) => Ok(TokenStream(stream.trees)), + Ok(stream) => Ok(TokenStream(Rc::new(stream.trees))), Err(_) => Err(ExpandError), } } @@ -196,7 +197,7 @@ impl FromStr for TokenStream { type Err = LexError; fn from_str(src: &str) -> Result { - Ok(TokenStream(BridgeMethods::ts_from_str(src).trees)) + Ok(TokenStream(Rc::new(BridgeMethods::ts_from_str(src).trees))) } } @@ -240,7 +241,7 @@ pub use quote::{HasIterator, RepInterp, ThereIsNoIteratorInRepetition, ext, quot fn stream_to_bridge_stream( stream: TokenStream, ) -> bridge::TokenStream { - bridge::TokenStream { trees: stream.0 } + bridge::TokenStream { trees: stream.0.to_vec() } } fn tree_to_bridge_tree( @@ -258,7 +259,7 @@ fn tree_to_bridge_tree( #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl From for TokenStream { fn from(tree: TokenTree) -> TokenStream { - TokenStream(vec![tree_to_bridge_tree(tree)]) + TokenStream(Rc::new(vec![tree_to_bridge_tree(tree)])) } } @@ -278,14 +279,14 @@ impl ConcatTreesHelper { } fn build(self) -> TokenStream { - TokenStream(self.trees) + TokenStream(Rc::new(self.trees)) } fn append_to(mut self, stream: &mut TokenStream) { if self.trees.is_empty() { return; } - stream.0.append(&mut self.trees); + Rc::make_mut(&mut stream.0).append(&mut self.trees); } } @@ -316,8 +317,9 @@ impl ConcatStreamsHelper { if self.streams.is_empty() { return; } + let this = Rc::make_mut(&mut stream.0); for mut s in self.streams { - stream.0.append(&mut s.0); + this.append(Rc::make_mut(&mut s.0)); } } } @@ -383,6 +385,8 @@ extend_items!(Group Literal Punct Ident); /// Public implementation details for the `TokenStream` type, such as iterators. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub mod token_stream { + use std::rc::Rc; + use crate::{Group, Ident, Literal, Punct, TokenStream, TokenTree, bridge}; /// An iterator over `TokenStream`'s `TokenTree`s. @@ -422,7 +426,7 @@ pub mod token_stream { type IntoIter = IntoIter; fn into_iter(self) -> IntoIter { - IntoIter(self.0.into_iter()) + IntoIter(Rc::unwrap_or_clone(self.0).into_iter()) } } } @@ -818,8 +822,8 @@ impl Group { #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn stream(&self) -> TokenStream { match &self.0.stream { - Some(stream) => TokenStream(stream.trees.clone()), - None => TokenStream(vec![]), + Some(stream) => TokenStream(Rc::new(stream.trees.clone())), + None => TokenStream(Rc::new(vec![])), } } From 2866d0668118ddddd3130b00c239dc8899ed90ff Mon Sep 17 00:00:00 2001 From: cyrgani Date: Fri, 30 Jan 2026 23:03:07 +0000 Subject: [PATCH 3/8] use references for encoding --- library/proc_macro/src/bridge/client.rs | 2 +- library/proc_macro/src/bridge/rpc.rs | 32 ++++++++++++------------- library/proc_macro/src/bridge/server.rs | 4 ++-- library/proc_macro/src/bridge/symbol.rs | 6 ++--- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index fa8479cbede36..736a690d1e8ee 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -23,7 +23,7 @@ impl !Send for Span {} impl !Sync for Span {} impl Encode for Span { - fn encode(self, w: &mut Buffer, s: &mut S) { + fn encode(&self, w: &mut Buffer, s: &mut S) { self.handle.encode(w, s); } } diff --git a/library/proc_macro/src/bridge/rpc.rs b/library/proc_macro/src/bridge/rpc.rs index 7fee8654bc788..ef6d3a0073708 100644 --- a/library/proc_macro/src/bridge/rpc.rs +++ b/library/proc_macro/src/bridge/rpc.rs @@ -7,7 +7,7 @@ use std::num::NonZero; use super::buffer::Buffer; pub(super) trait Encode: Sized { - fn encode(self, w: &mut Buffer, s: &mut S); + fn encode(&self, w: &mut Buffer, s: &mut S); } pub(super) trait Decode<'a, 's, S>: Sized { @@ -17,7 +17,7 @@ pub(super) trait Decode<'a, 's, S>: Sized { macro_rules! rpc_encode_decode { (le $ty:ty) => { impl Encode for $ty { - fn encode(self, w: &mut Buffer, _: &mut S) { + fn encode(&self, w: &mut Buffer, _: &mut S) { w.extend_from_array(&self.to_le_bytes()); } } @@ -36,7 +36,7 @@ macro_rules! rpc_encode_decode { }; (struct $name:ident $(<$($T:ident),+>)? { $($field:ident),* $(,)? }) => { impl),+)?> Encode for $name $(<$($T),+>)? { - fn encode(self, w: &mut Buffer, s: &mut S) { + fn encode(&self, w: &mut Buffer, s: &mut S) { $(self.$field.encode(w, s);)* } } @@ -59,7 +59,7 @@ macro_rules! rpc_encode_decode { $(const $variant: u8 = Tag::$variant as u8;)* impl),+)?> Encode for $name $(<$($T),+>)? { - fn encode(self, w: &mut Buffer, s: &mut S) { + fn encode(&self, w: &mut Buffer, s: &mut S) { match self { $($name::$variant $(($field))* => { $variant.encode(w, s); @@ -87,7 +87,7 @@ macro_rules! rpc_encode_decode { } impl Encode for () { - fn encode(self, _: &mut Buffer, _: &mut S) {} + fn encode(&self, _: &mut Buffer, _: &mut S) {} } impl Decode<'_, '_, S> for () { @@ -95,8 +95,8 @@ impl Decode<'_, '_, S> for () { } impl Encode for u8 { - fn encode(self, w: &mut Buffer, _: &mut S) { - w.push(self); + fn encode(&self, w: &mut Buffer, _: &mut S) { + w.push(*self); } } @@ -112,8 +112,8 @@ rpc_encode_decode!(le u32); rpc_encode_decode!(le usize); impl Encode for bool { - fn encode(self, w: &mut Buffer, s: &mut S) { - (self as u8).encode(w, s); + fn encode(&self, w: &mut Buffer, s: &mut S) { + (*self as u8).encode(w, s); } } @@ -128,7 +128,7 @@ impl Decode<'_, '_, S> for bool { } impl Encode for NonZero { - fn encode(self, w: &mut Buffer, s: &mut S) { + fn encode(&self, w: &mut Buffer, s: &mut S) { self.get().encode(w, s); } } @@ -140,7 +140,7 @@ impl Decode<'_, '_, S> for NonZero { } impl, B: Encode> Encode for (A, B) { - fn encode(self, w: &mut Buffer, s: &mut S) { + fn encode(&self, w: &mut Buffer, s: &mut S) { self.0.encode(w, s); self.1.encode(w, s); } @@ -155,7 +155,7 @@ impl<'a, S, A: for<'s> Decode<'a, 's, S>, B: for<'s> Decode<'a, 's, S>> Decode<' } impl Encode for &str { - fn encode(self, w: &mut Buffer, s: &mut S) { + fn encode(&self, w: &mut Buffer, s: &mut S) { let bytes = self.as_bytes(); bytes.len().encode(w, s); w.write_all(bytes).unwrap(); @@ -172,8 +172,8 @@ impl<'a, S> Decode<'a, '_, S> for &'a str { } impl Encode for String { - fn encode(self, w: &mut Buffer, s: &mut S) { - self[..].encode(w, s); + fn encode(&self, w: &mut Buffer, s: &mut S) { + (&self[..]).encode(w, s); } } @@ -184,7 +184,7 @@ impl Decode<'_, '_, S> for String { } impl> Encode for Vec { - fn encode(self, w: &mut Buffer, s: &mut S) { + fn encode(&self, w: &mut Buffer, s: &mut S) { self.len().encode(w, s); for x in self { x.encode(w, s); @@ -247,7 +247,7 @@ impl PanicMessage { } impl Encode for PanicMessage { - fn encode(self, w: &mut Buffer, s: &mut S) { + fn encode(&self, w: &mut Buffer, s: &mut S) { self.as_str().encode(w, s); } } diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index 9f67e222bdfdb..78d62c08bcdab 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -22,8 +22,8 @@ pub(super) type MarkedSpan = Marked<::Span, client::Span>; pub(super) type MarkedSymbol = Marked<::Symbol, client::Symbol>; impl Encode> for MarkedSpan { - fn encode(self, w: &mut Buffer, s: &mut HandleStore) { - s.span.alloc(self).encode(w, s); + fn encode(&self, w: &mut Buffer, s: &mut HandleStore) { + s.span.alloc(*self).encode(w, s); } } diff --git a/library/proc_macro/src/bridge/symbol.rs b/library/proc_macro/src/bridge/symbol.rs index 2a04f7d808bd5..71b5f243aaff2 100644 --- a/library/proc_macro/src/bridge/symbol.rs +++ b/library/proc_macro/src/bridge/symbol.rs @@ -94,7 +94,7 @@ impl fmt::Display for Symbol { } impl Encode for Symbol { - fn encode(self, w: &mut Buffer, s: &mut S) { + fn encode(&self, w: &mut Buffer, s: &mut S) { self.with(|sym| sym.encode(w, s)) } } @@ -106,8 +106,8 @@ impl Decode<'_, '_, server::HandleStore> for server::Marke } impl Encode> for server::MarkedSymbol { - fn encode(self, w: &mut Buffer, s: &mut server::HandleStore) { - S::with_symbol_string(&self.unmark(), |sym| sym.encode(w, s)) + fn encode(&self, w: &mut Buffer, s: &mut server::HandleStore) { + S::with_symbol_string(&self.value, |sym| sym.encode(w, s)) } } From b019a81d98b1a2fb7e92567901013d7a00c701b5 Mon Sep 17 00:00:00 2001 From: cyrgani Date: Fri, 30 Jan 2026 23:04:14 +0000 Subject: [PATCH 4/8] inline smallvec use --- .../rustc_expand/src/proc_macro_server.rs | 196 +++++++++--------- 1 file changed, 94 insertions(+), 102 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index e4477fc7148d5..6dd0e43478e9b 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -18,7 +18,6 @@ use rustc_proc_macro::{Delimiter, Level}; use rustc_session::parse::ParseSess; use rustc_span::def_id::CrateNum; use rustc_span::{BytePos, FileName, Pos, Span, Symbol, sym}; -use smallvec::{SmallVec, smallvec}; use crate::base::ExtCtxt; @@ -322,103 +321,6 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec> - for (TokenTree, &mut Rustc<'_, '_>) -{ - fn to_internal(self) -> SmallVec<[tokenstream::TokenTree; 2]> { - use rustc_ast::token::*; - - // The code below is conservative, using `token_alone`/`Spacing::Alone` - // in most places. It's hard in general to do better when working at - // the token level. When the resulting code is pretty-printed by - // `print_tts` the `space_between` function helps avoid a lot of - // unnecessary whitespace, so the results aren't too bad. - let (tree, rustc) = self; - match tree { - TokenTree::Punct(Punct { ch, joint, span }) => { - let kind = match ch { - b'=' => Eq, - b'<' => Lt, - b'>' => Gt, - b'!' => Bang, - b'~' => Tilde, - b'+' => Plus, - b'-' => Minus, - b'*' => Star, - b'/' => Slash, - b'%' => Percent, - b'^' => Caret, - b'&' => And, - b'|' => Or, - b'@' => At, - b'.' => Dot, - b',' => Comma, - b';' => Semi, - b':' => Colon, - b'#' => Pound, - b'$' => Dollar, - b'?' => Question, - b'\'' => SingleQuote, - _ => unreachable!(), - }; - // We never produce `token::Spacing::JointHidden` here, which - // means the pretty-printing of code produced by proc macros is - // ugly, with lots of whitespace between tokens. This is - // unavoidable because `proc_macro::Spacing` only applies to - // `Punct` token trees. - smallvec![if joint { - tokenstream::TokenTree::token_joint(kind, span) - } else { - tokenstream::TokenTree::token_alone(kind, span) - }] - } - TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => { - smallvec![tokenstream::TokenTree::Delimited( - tokenstream::DelimSpan { open, close }, - DelimSpacing::new(Spacing::Alone, Spacing::Alone), - delimiter.to_internal(), - rustc.ts_pm_to_rustc(stream.unwrap_or_default()), - )] - } - TokenTree::Ident(self::Ident { sym, is_raw, span }) => { - rustc.psess().symbol_gallery.insert(sym, span); - smallvec![tokenstream::TokenTree::token_alone(Ident(sym, is_raw.into()), span)] - } - TokenTree::Literal(self::Literal { - kind: self::LitKind::Integer, - symbol, - suffix, - span, - }) if let Some(symbol) = symbol.as_str().strip_prefix('-') => { - let symbol = Symbol::intern(symbol); - let integer = TokenKind::lit(token::Integer, symbol, suffix); - let a = tokenstream::TokenTree::token_joint_hidden(Minus, span); - let b = tokenstream::TokenTree::token_alone(integer, span); - smallvec![a, b] - } - TokenTree::Literal(self::Literal { - kind: self::LitKind::Float, - symbol, - suffix, - span, - }) if let Some(symbol) = symbol.as_str().strip_prefix('-') => { - let symbol = Symbol::intern(symbol); - let float = TokenKind::lit(token::Float, symbol, suffix); - let a = tokenstream::TokenTree::token_joint_hidden(Minus, span); - let b = tokenstream::TokenTree::token_alone(float, span); - smallvec![a, b] - } - TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => { - smallvec![tokenstream::TokenTree::token_alone( - TokenKind::lit(kind.to_internal(), symbol, suffix), - span, - )] - } - } - } -} - impl ToInternal for Level { fn to_internal(self) -> rustc_errors::Level { match self { @@ -462,10 +364,100 @@ impl<'a, 'b> Rustc<'a, 'b> { pub(crate) fn ts_pm_to_rustc(&mut self, ts: BridgeTokenStream) -> tokenstream::TokenStream { let mut t = tokenstream::TokenStream::new(vec![]); for tree in ts.trees { - let internal: SmallVec<[tokenstream::TokenTree; 2]> = - ToInternal::to_internal((tree, &mut *self)); - for tree in internal { - t.push_tree(tree); + use rustc_ast::token::*; + + // The code below is conservative, using `token_alone`/`Spacing::Alone` + // in most places. It's hard in general to do better when working at + // the token level. When the resulting code is pretty-printed by + // `print_tts` the `space_between` function helps avoid a lot of + // unnecessary whitespace, so the results aren't too bad. + match tree { + TokenTree::Punct(Punct { ch, joint, span }) => { + let kind = match ch { + b'=' => Eq, + b'<' => Lt, + b'>' => Gt, + b'!' => Bang, + b'~' => Tilde, + b'+' => Plus, + b'-' => Minus, + b'*' => Star, + b'/' => Slash, + b'%' => Percent, + b'^' => Caret, + b'&' => And, + b'|' => Or, + b'@' => At, + b'.' => Dot, + b',' => Comma, + b';' => Semi, + b':' => Colon, + b'#' => Pound, + b'$' => Dollar, + b'?' => Question, + b'\'' => SingleQuote, + _ => unreachable!(), + }; + // We never produce `token::Spacing::JointHidden` here, which + // means the pretty-printing of code produced by proc macros is + // ugly, with lots of whitespace between tokens. This is + // unavoidable because `proc_macro::Spacing` only applies to + // `Punct` token trees. + t.push_tree(if joint { + tokenstream::TokenTree::token_joint(kind, span) + } else { + tokenstream::TokenTree::token_alone(kind, span) + }) + } + TokenTree::Group(Group { + delimiter, + stream, + span: DelimSpan { open, close, .. }, + }) => t.push_tree(tokenstream::TokenTree::Delimited( + tokenstream::DelimSpan { open, close }, + DelimSpacing::new(Spacing::Alone, Spacing::Alone), + delimiter.to_internal(), + self.ts_pm_to_rustc(stream.unwrap_or_default()), + )), + TokenTree::Ident(self::Ident { sym, is_raw, span }) => { + self.psess().symbol_gallery.insert(sym, span); + t.push_tree(tokenstream::TokenTree::token_alone( + Ident(sym, is_raw.into()), + span, + )) + } + TokenTree::Literal(self::Literal { + kind: self::LitKind::Integer, + symbol, + suffix, + span, + }) if let Some(symbol) = symbol.as_str().strip_prefix('-') => { + let symbol = Symbol::intern(symbol); + let integer = TokenKind::lit(token::Integer, symbol, suffix); + let a = tokenstream::TokenTree::token_joint_hidden(Minus, span); + let b = tokenstream::TokenTree::token_alone(integer, span); + t.push_tree(a); + t.push_tree(b); + } + TokenTree::Literal(self::Literal { + kind: self::LitKind::Float, + symbol, + suffix, + span, + }) if let Some(symbol) = symbol.as_str().strip_prefix('-') => { + let symbol = Symbol::intern(symbol); + let float = TokenKind::lit(token::Float, symbol, suffix); + let a = tokenstream::TokenTree::token_joint_hidden(Minus, span); + let b = tokenstream::TokenTree::token_alone(float, span); + t.push_tree(a); + t.push_tree(b); + } + TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => { + t.push_tree(tokenstream::TokenTree::token_alone( + TokenKind::lit(kind.to_internal(), symbol, suffix), + span, + )) + } } } From ae312ab1ede322e8f61f7db6941c1ff333e409ba Mon Sep 17 00:00:00 2001 From: cyrgani Date: Sat, 31 Jan 2026 20:17:10 +0000 Subject: [PATCH 5/8] rc everywhere --- .../rustc_expand/src/proc_macro_server.rs | 6 ++- library/proc_macro/src/bridge/mod.rs | 39 ++++++++++++++++--- library/proc_macro/src/bridge/rpc.rs | 13 +++++++ library/proc_macro/src/bridge/server.rs | 2 +- library/proc_macro/src/lib.rs | 8 ++-- 5 files changed, 56 insertions(+), 12 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 6dd0e43478e9b..6fb629988aef4 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -1,4 +1,5 @@ use std::ops::{Bound, Range}; +use std::rc::Rc; use ast::token::IdentIsRaw; use rustc_ast as ast; @@ -363,7 +364,8 @@ impl<'a, 'b> Rustc<'a, 'b> { pub(crate) fn ts_pm_to_rustc(&mut self, ts: BridgeTokenStream) -> tokenstream::TokenStream { let mut t = tokenstream::TokenStream::new(vec![]); - for tree in ts.trees { + + for tree in Rc::into_inner(ts.trees).expect("just decoded this `Rc`") { use rustc_ast::token::*; // The code below is conservative, using `token_alone`/`Spacing::Alone` @@ -466,7 +468,7 @@ impl<'a, 'b> Rustc<'a, 'b> { pub(crate) fn ts_rustc_to_pm(&mut self, ts: tokenstream::TokenStream) -> BridgeTokenStream { let trees: Vec> = FromInternal::from_internal((ts, self)); - BridgeTokenStream { trees } + BridgeTokenStream { trees: Rc::new(trees) } } } diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index 8bca55af4baa4..916e1499b5135 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -10,6 +10,7 @@ use std::hash::Hash; use std::ops::{Bound, Range}; +use std::rc::Rc; use std::sync::Once; use std::{fmt, marker, panic, thread}; @@ -131,8 +132,8 @@ with_api!(declare_tags, __, __); /// That is, normally a pair of impls for `T::Foo` and `T::Bar` /// can overlap, but if the impls are, instead, on types like /// `Marked` and `Marked`, they can't. -trait Mark { - type Unmarked; +trait Mark: Clone { + type Unmarked: Clone; fn mark(unmarked: Self::Unmarked) -> Self; fn unmark(self) -> Self::Unmarked; } @@ -143,7 +144,7 @@ struct Marked { _marker: marker::PhantomData, } -impl Mark for Marked { +impl Mark for Marked { type Unmarked = T; fn mark(unmarked: Self::Unmarked) -> Self { Marked { value: unmarked, _marker: marker::PhantomData } @@ -165,6 +166,19 @@ impl Mark for Vec { } } +impl Mark for Rc +where + T::Unmarked: Clone, +{ + type Unmarked = Rc; + fn mark(unmarked: Self::Unmarked) -> Self { + Rc::new(Mark::mark(Rc::unwrap_or_clone(unmarked))) + } + fn unmark(self) -> Self::Unmarked { + Rc::new(Mark::unmark(Rc::unwrap_or_clone(self))) + } +} + macro_rules! mark_noop { ($($ty:ty),* $(,)?) => { $( @@ -383,18 +397,33 @@ compound_traits!( #[derive(Clone)] pub struct TokenStream { - pub trees: Vec>, + pub trees: Rc>>, } impl Default for TokenStream { fn default() -> Self { - Self { trees: Vec::new() } + Self { trees: Rc::new(Vec::new()) } } } compound_traits!( struct TokenStream { trees } ); +/* +#[derive(Clone)] +pub struct RcTokenStream { + pub trees: Rc>>, +} + +impl Default for RcTokenStream { + fn default() -> Self { + Self { trees: Rc::new(Vec::new()) } + } +} + +compound_traits!( + struct RcTokenStream { trees } +);*/ #[derive(Clone, Debug)] pub struct Diagnostic { diff --git a/library/proc_macro/src/bridge/rpc.rs b/library/proc_macro/src/bridge/rpc.rs index ef6d3a0073708..9e94d44c5e0c0 100644 --- a/library/proc_macro/src/bridge/rpc.rs +++ b/library/proc_macro/src/bridge/rpc.rs @@ -3,6 +3,7 @@ use std::any::Any; use std::io::Write; use std::num::NonZero; +use std::rc::Rc; use super::buffer::Buffer; @@ -203,6 +204,18 @@ impl<'a, S, T: for<'s> Decode<'a, 's, S>> Decode<'a, '_, S> for Vec { } } +impl> Encode for Rc { + fn encode(&self, w: &mut Buffer, s: &mut S) { + self.as_ref().encode(w, s); + } +} + +impl<'a, S, T: for<'s> Decode<'a, 's, S>> Decode<'a, '_, S> for Rc { + fn decode(r: &mut &'a [u8], s: &mut S) -> Self { + Rc::new(T::decode(r, s)) + } +} + /// Simplified version of panic payloads, ignoring /// types other than `&'static str` and `String`. pub enum PanicMessage { diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index 78d62c08bcdab..a2dfd32f7c572 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -44,7 +44,7 @@ macro_rules! define_server { ) => { pub trait Server { type Span: 'static + Copy + Eq + Hash; - type Symbol: 'static; + type Symbol: 'static + Clone; fn globals(&mut self) -> ExpnGlobals; diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index d63c0c5f3ddbf..1af961252d75c 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -179,7 +179,7 @@ impl TokenStream { #[unstable(feature = "proc_macro_expand", issue = "90765")] pub fn expand_expr(&self) -> Result { match BridgeMethods::ts_expand_expr(stream_to_bridge_stream(self.clone())) { - Ok(stream) => Ok(TokenStream(Rc::new(stream.trees))), + Ok(stream) => Ok(TokenStream(Rc::clone(&stream.trees))), Err(_) => Err(ExpandError), } } @@ -197,7 +197,7 @@ impl FromStr for TokenStream { type Err = LexError; fn from_str(src: &str) -> Result { - Ok(TokenStream(Rc::new(BridgeMethods::ts_from_str(src).trees))) + Ok(TokenStream(Rc::clone(&BridgeMethods::ts_from_str(src).trees))) } } @@ -241,7 +241,7 @@ pub use quote::{HasIterator, RepInterp, ThereIsNoIteratorInRepetition, ext, quot fn stream_to_bridge_stream( stream: TokenStream, ) -> bridge::TokenStream { - bridge::TokenStream { trees: stream.0.to_vec() } + bridge::TokenStream { trees: stream.0.clone() } } fn tree_to_bridge_tree( @@ -822,7 +822,7 @@ impl Group { #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn stream(&self) -> TokenStream { match &self.0.stream { - Some(stream) => TokenStream(Rc::new(stream.trees.clone())), + Some(stream) => TokenStream(Rc::clone(&stream.trees)), None => TokenStream(Rc::new(vec![])), } } From e2b0eff089ee02f617f32dbe490a6bb237738512 Mon Sep 17 00:00:00 2001 From: cyrgani Date: Sun, 1 Feb 2026 16:01:39 +0000 Subject: [PATCH 6/8] clone reductions --- .../rustc_expand/src/proc_macro_server.rs | 3 +- library/proc_macro/src/bridge/client.rs | 12 ++++-- library/proc_macro/src/bridge/mod.rs | 24 +++--------- library/proc_macro/src/lib.rs | 38 ++++++++----------- 4 files changed, 32 insertions(+), 45 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 6fb629988aef4..dcbaa6e538c79 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -467,8 +467,7 @@ impl<'a, 'b> Rustc<'a, 'b> { } pub(crate) fn ts_rustc_to_pm(&mut self, ts: tokenstream::TokenStream) -> BridgeTokenStream { - let trees: Vec> = FromInternal::from_internal((ts, self)); - BridgeTokenStream { trees: Rc::new(trees) } + BridgeTokenStream::new(>>::from_internal((ts, self))) } } diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index 736a690d1e8ee..27a59fc958206 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -258,7 +258,9 @@ impl Client { handle_counters: &COUNTERS, run: super::selfless_reify::reify_to_extern_c_fn_hrt_bridge(move |bridge| { run_client(bridge, |input| { - Rc::unwrap_or_clone(f(crate::TokenStream(Rc::new(input))).0) + Rc::unwrap_or_clone( + f(crate::TokenStream(crate::bridge::TokenStream::new(input))).0.trees, + ) }) }), _marker: PhantomData, @@ -275,8 +277,12 @@ impl Client<(crate::TokenStream, crate::TokenStream), crate::TokenStream> { run: super::selfless_reify::reify_to_extern_c_fn_hrt_bridge(move |bridge| { run_client(bridge, |(input, input2)| { Rc::unwrap_or_clone( - f(crate::TokenStream(Rc::new(input)), crate::TokenStream(Rc::new(input2))) - .0, + f( + crate::TokenStream(crate::bridge::TokenStream::new(input)), + crate::TokenStream(crate::bridge::TokenStream::new(input2)), + ) + .0 + .trees, ) }) }), diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index 916e1499b5135..b1655d6cf4cf0 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -166,10 +166,7 @@ impl Mark for Vec { } } -impl Mark for Rc -where - T::Unmarked: Clone, -{ +impl Mark for Rc { type Unmarked = Rc; fn mark(unmarked: Self::Unmarked) -> Self { Rc::new(Mark::mark(Rc::unwrap_or_clone(unmarked))) @@ -406,24 +403,15 @@ impl Default for TokenStream { } } -compound_traits!( - struct TokenStream { trees } -); -/* -#[derive(Clone)] -pub struct RcTokenStream { - pub trees: Rc>>, -} - -impl Default for RcTokenStream { - fn default() -> Self { - Self { trees: Rc::new(Vec::new()) } +impl TokenStream { + pub fn new(tts: Vec>) -> Self { + Self { trees: Rc::new(tts) } } } compound_traits!( - struct RcTokenStream { trees } -);*/ + struct TokenStream { trees } +); #[derive(Clone, Debug)] pub struct Diagnostic { diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 1af961252d75c..bb0d427b4725f 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -103,7 +103,7 @@ pub fn is_available() -> bool { #[cfg_attr(feature = "rustc-dep-of-std", rustc_diagnostic_item = "TokenStream")] #[stable(feature = "proc_macro_lib", since = "1.15.0")] #[derive(Clone)] -pub struct TokenStream(Rc>>); +pub struct TokenStream(bridge::TokenStream); #[stable(feature = "proc_macro_lib", since = "1.15.0")] impl !Send for TokenStream {} @@ -157,13 +157,13 @@ impl TokenStream { /// Returns an empty `TokenStream` containing no token trees. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new() -> TokenStream { - TokenStream(Rc::new(Vec::new())) + TokenStream(bridge::TokenStream::new(Vec::new())) } /// Checks if this `TokenStream` is empty. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn is_empty(&self) -> bool { - self.0.is_empty() + self.0.trees.is_empty() } /// Parses this `TokenStream` as an expression and attempts to expand any @@ -178,8 +178,8 @@ impl TokenStream { /// considered errors, is unspecified and may change in the future. #[unstable(feature = "proc_macro_expand", issue = "90765")] pub fn expand_expr(&self) -> Result { - match BridgeMethods::ts_expand_expr(stream_to_bridge_stream(self.clone())) { - Ok(stream) => Ok(TokenStream(Rc::clone(&stream.trees))), + match BridgeMethods::ts_expand_expr(self.0.clone()) { + Ok(stream) => Ok(TokenStream(stream)), Err(_) => Err(ExpandError), } } @@ -197,7 +197,7 @@ impl FromStr for TokenStream { type Err = LexError; fn from_str(src: &str) -> Result { - Ok(TokenStream(Rc::clone(&BridgeMethods::ts_from_str(src).trees))) + Ok(TokenStream(BridgeMethods::ts_from_str(src))) } } @@ -215,7 +215,7 @@ impl FromStr for TokenStream { #[stable(feature = "proc_macro_lib", since = "1.15.0")] impl fmt::Display for TokenStream { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", BridgeMethods::ts_to_string(stream_to_bridge_stream(self.clone()))) + write!(f, "{}", BridgeMethods::ts_to_string(self.0.clone())) } } @@ -238,12 +238,6 @@ impl Default for TokenStream { #[unstable(feature = "proc_macro_quote", issue = "54722")] pub use quote::{HasIterator, RepInterp, ThereIsNoIteratorInRepetition, ext, quote, quote_span}; -fn stream_to_bridge_stream( - stream: TokenStream, -) -> bridge::TokenStream { - bridge::TokenStream { trees: stream.0.clone() } -} - fn tree_to_bridge_tree( tree: TokenTree, ) -> bridge::TokenTree { @@ -259,7 +253,7 @@ fn tree_to_bridge_tree( #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl From for TokenStream { fn from(tree: TokenTree) -> TokenStream { - TokenStream(Rc::new(vec![tree_to_bridge_tree(tree)])) + TokenStream(bridge::TokenStream::new(vec![tree_to_bridge_tree(tree)])) } } @@ -279,14 +273,14 @@ impl ConcatTreesHelper { } fn build(self) -> TokenStream { - TokenStream(Rc::new(self.trees)) + TokenStream(bridge::TokenStream::new(self.trees)) } fn append_to(mut self, stream: &mut TokenStream) { if self.trees.is_empty() { return; } - Rc::make_mut(&mut stream.0).append(&mut self.trees); + Rc::make_mut(&mut stream.0.trees).append(&mut self.trees); } } @@ -317,9 +311,9 @@ impl ConcatStreamsHelper { if self.streams.is_empty() { return; } - let this = Rc::make_mut(&mut stream.0); + let this = Rc::make_mut(&mut stream.0.trees); for mut s in self.streams { - this.append(Rc::make_mut(&mut s.0)); + this.append(Rc::make_mut(&mut s.0.trees)); } } } @@ -426,7 +420,7 @@ pub mod token_stream { type IntoIter = IntoIter; fn into_iter(self) -> IntoIter { - IntoIter(Rc::unwrap_or_clone(self.0).into_iter()) + IntoIter(Rc::unwrap_or_clone(self.0.trees).into_iter()) } } } @@ -804,7 +798,7 @@ impl Group { pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group { Group(bridge::Group { delimiter, - stream: Some(stream_to_bridge_stream(stream)), + stream: Some(stream.0), span: bridge::DelimSpan::from_single(Span::call_site().0), }) } @@ -822,8 +816,8 @@ impl Group { #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn stream(&self) -> TokenStream { match &self.0.stream { - Some(stream) => TokenStream(Rc::clone(&stream.trees)), - None => TokenStream(Rc::new(vec![])), + Some(stream) => TokenStream(stream.clone()), + None => TokenStream(bridge::TokenStream::new(vec![])), } } From a888f394ec70d273dced5b3d0d28d94a24e56b4a Mon Sep 17 00:00:00 2001 From: cyrgani Date: Mon, 2 Feb 2026 10:58:46 +0000 Subject: [PATCH 7/8] cleanup, remove `OwnedStore` --- library/proc_macro/src/bridge/client.rs | 11 ++---- library/proc_macro/src/bridge/handle.rs | 48 +++++++------------------ 2 files changed, 14 insertions(+), 45 deletions(-) diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index 27a59fc958206..49434f0f6946b 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -258,9 +258,7 @@ impl Client { handle_counters: &COUNTERS, run: super::selfless_reify::reify_to_extern_c_fn_hrt_bridge(move |bridge| { run_client(bridge, |input| { - Rc::unwrap_or_clone( - f(crate::TokenStream(crate::bridge::TokenStream::new(input))).0.trees, - ) + Rc::unwrap_or_clone(f(crate::TokenStream(input)).0.trees) }) }), _marker: PhantomData, @@ -277,12 +275,7 @@ impl Client<(crate::TokenStream, crate::TokenStream), crate::TokenStream> { run: super::selfless_reify::reify_to_extern_c_fn_hrt_bridge(move |bridge| { run_client(bridge, |(input, input2)| { Rc::unwrap_or_clone( - f( - crate::TokenStream(crate::bridge::TokenStream::new(input)), - crate::TokenStream(crate::bridge::TokenStream::new(input2)), - ) - .0 - .trees, + f(crate::TokenStream(input), crate::TokenStream(input2)).0.trees, ) }) }), diff --git a/library/proc_macro/src/bridge/handle.rs b/library/proc_macro/src/bridge/handle.rs index 961227022147b..da6efa54d5187 100644 --- a/library/proc_macro/src/bridge/handle.rs +++ b/library/proc_macro/src/bridge/handle.rs @@ -3,7 +3,6 @@ use std::collections::BTreeMap; use std::hash::Hash; use std::num::NonZero; -use std::ops::Index; use std::sync::atomic::{AtomicU32, Ordering}; use super::fxhash::FxHashMap; @@ -11,55 +10,32 @@ use super::fxhash::FxHashMap; pub(super) type Handle = NonZero; /// A store that associates values of type `T` with numeric handles. A value can -/// be looked up using its handle. -pub(super) struct OwnedStore { +/// be looked up using its handle. Avoids storing any value more than once. +pub(super) struct InternedStore { counter: &'static AtomicU32, data: BTreeMap, + interner: FxHashMap, } -impl OwnedStore { +impl InternedStore { pub(super) fn new(counter: &'static AtomicU32) -> Self { // Ensure the handle counter isn't 0, which would panic later, // when `NonZero::new` (aka `Handle::new`) is called in `alloc`. assert_ne!(counter.load(Ordering::Relaxed), 0); - OwnedStore { counter, data: BTreeMap::new() } - } -} - -impl OwnedStore { - pub(super) fn alloc(&mut self, x: T) -> Handle { - let counter = self.counter.fetch_add(1, Ordering::Relaxed); - let handle = Handle::new(counter).expect("`proc_macro` handle counter overflowed"); - assert!(self.data.insert(handle, x).is_none()); - handle - } -} - -impl Index for OwnedStore { - type Output = T; - fn index(&self, h: Handle) -> &T { - self.data.get(&h).expect("use-after-free in `proc_macro` handle") - } -} - -/// Like `OwnedStore`, but avoids storing any value more than once. -pub(super) struct InternedStore { - owned: OwnedStore, - interner: FxHashMap, -} - -impl InternedStore { - pub(super) fn new(counter: &'static AtomicU32) -> Self { - InternedStore { owned: OwnedStore::new(counter), interner: FxHashMap::default() } + InternedStore { counter, data: BTreeMap::new(), interner: FxHashMap::default() } } pub(super) fn alloc(&mut self, x: T) -> Handle { - let owned = &mut self.owned; - *self.interner.entry(x).or_insert_with(|| owned.alloc(x)) + *self.interner.entry(x).or_insert_with(|| { + let counter = self.counter.fetch_add(1, Ordering::Relaxed); + let handle = Handle::new(counter).expect("`proc_macro` handle counter overflowed"); + assert!(self.data.insert(handle, x).is_none()); + handle + }) } pub(super) fn copy(&mut self, h: Handle) -> T { - self.owned[h] + *self.data.get(&h).expect("use-after-free in `proc_macro` handle") } } From 7c750ba7b9181f041d8e65b07a929eb3de075324 Mon Sep 17 00:00:00 2001 From: cyrgani Date: Tue, 10 Feb 2026 21:45:05 +0000 Subject: [PATCH 8/8] fix r-a --- library/proc_macro/src/bridge/mod.rs | 6 +- library/proc_macro/src/bridge/server.rs | 1 - .../crates/proc-macro-srv/src/bridge.rs | 5 +- .../crates/proc-macro-srv/src/dylib.rs | 6 +- .../proc-macro-srv/src/dylib/proc_macros.rs | 14 +-- .../crates/proc-macro-srv/src/lib.rs | 6 +- .../src/server_impl/rust_analyzer_span.rs | 69 +++---------- .../src/server_impl/token_id.rs | 68 +++---------- .../crates/proc-macro-srv/src/token_stream.rs | 98 ++++++++++++++----- 9 files changed, 116 insertions(+), 157 deletions(-) diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index b1655d6cf4cf0..1aa38745c7abd 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -19,16 +19,16 @@ use crate::{Delimiter, Level}; /// Higher-order macro describing the server RPC API, allowing automatic /// generation of type-safe Rust APIs, both client-side and server-side. /// -/// `with_api!(my_macro, MyTokenStream, MySpan, MySymbol)` expands to: +/// `with_api!(my_macro, MySpan, MySymbol)` expands to: /// ```rust,ignore (pseudo-code) /// my_macro! { -/// fn ts_clone(stream: &MyTokenStream) -> MyTokenStream; +/// fn normalize(string: &str) -> MySymbol; /// fn span_debug(span: &MySpan) -> String; /// // ... /// } /// ``` /// -/// The second (`TokenStream`), third (`Span`) and fourth (`Symbol`) +/// The second (`Span`) and third (`Symbol`) /// argument serve to customize the argument/return types that need /// special handling, to enable several different representations of /// these types. diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index a2dfd32f7c572..596497742323c 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -17,7 +17,6 @@ impl HandleStore { } pub(super) type MarkedTokenStream = bridge::TokenStream, MarkedSymbol>; -//bridge::TokenStream; pub(super) type MarkedSpan = Marked<::Span, client::Span>; pub(super) type MarkedSymbol = Marked<::Symbol, client::Symbol>; diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/bridge.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/bridge.rs index fc62f9413a34e..5b2112650cfca 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/bridge.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/bridge.rs @@ -4,9 +4,10 @@ use rustc_proc_macro::bridge as pm_bridge; pub use pm_bridge::{DelimSpan, Diagnostic, ExpnGlobals, LitKind}; +pub type TokenStream = pm_bridge::TokenStream; pub type TokenTree = - pm_bridge::TokenTree, S, intern::Symbol>; + pm_bridge::TokenTree; pub type Literal = pm_bridge::Literal; -pub type Group = pm_bridge::Group, S>; +pub type Group = pm_bridge::Group; pub type Punct = pm_bridge::Punct; pub type Ident = pm_bridge::Ident; diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/dylib.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/dylib.rs index 9a65538675fe9..5787aecedfe59 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/dylib.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/dylib.rs @@ -3,7 +3,6 @@ mod proc_macros; mod version; -use rustc_proc_macro::bridge; use std::{fmt, fs, io, time::SystemTime}; use temp_dir::TempDir; @@ -46,10 +45,7 @@ impl Expander { call_site: S, mixed_site: S, callback: Option>, - ) -> Result, PanicMessage> - where - as bridge::server::Server>::TokenStream: Default, - { + ) -> Result, PanicMessage> { self.inner .proc_macros .expand(macro_name, macro_body, attribute, def_site, call_site, mixed_site, callback) diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/dylib/proc_macros.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/dylib/proc_macros.rs index 76c5097101c70..f7b0fb80510d9 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/dylib/proc_macros.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/dylib/proc_macros.rs @@ -32,29 +32,29 @@ impl ProcMacros { let res = client.run( &bridge::server::SameThread, S::make_server(call_site, def_site, mixed_site, callback), - macro_body, + macro_body.into_bridge(), cfg!(debug_assertions), ); - return res.map_err(crate::PanicMessage::from); + return res.map(TokenStream::from_bridge).map_err(crate::PanicMessage::from); } bridge::client::ProcMacro::Bang { name, client } if *name == macro_name => { let res = client.run( &bridge::server::SameThread, S::make_server(call_site, def_site, mixed_site, callback), - macro_body, + macro_body.into_bridge(), cfg!(debug_assertions), ); - return res.map_err(crate::PanicMessage::from); + return res.map(TokenStream::from_bridge).map_err(crate::PanicMessage::from); } bridge::client::ProcMacro::Attr { name, client } if *name == macro_name => { let res = client.run( &bridge::server::SameThread, S::make_server(call_site, def_site, mixed_site, callback), - parsed_attributes, - macro_body, + parsed_attributes.into_bridge(), + macro_body.into_bridge(), cfg!(debug_assertions), ); - return res.map_err(crate::PanicMessage::from); + return res.map(TokenStream::from_bridge).map_err(crate::PanicMessage::from); } _ => continue, } diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs index c548dc620ad13..0ab34d42117df 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs @@ -61,7 +61,7 @@ pub use span; pub use crate::bridge::*; pub use crate::server_impl::literal_from_str; -pub use crate::token_stream::{TokenStream, TokenStreamIter, literal_to_string}; +pub use crate::token_stream::{TokenStream, TokenTree, Group, TokenStreamIter, literal_to_string}; #[derive(Copy, Clone, Eq, PartialEq, Debug)] pub enum ProcMacroKind { @@ -233,9 +233,7 @@ impl ProcMacroSrv<'_> { } pub trait ProcMacroSrvSpan: Copy + Send + Sync { - type Server<'a>: rustc_proc_macro::bridge::server::Server< - TokenStream = crate::token_stream::TokenStream, - >; + type Server<'a>: rustc_proc_macro::bridge::server::Server; fn make_server<'a>( call_site: Self, def_site: Self, diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs index c114d52ec33c6..285b9c5c85680 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs @@ -15,7 +15,7 @@ use span::{FIXUP_ERASED_FILE_AST_ID_MARKER, Span, TextRange, TextSize}; use crate::{ ProcMacroClientHandle, - bridge::{Diagnostic, ExpnGlobals, Literal, TokenTree}, + bridge::{Diagnostic, ExpnGlobals, Literal}, server_impl::literal_from_str, }; @@ -30,8 +30,10 @@ pub struct RaSpanServer<'a> { pub callback: Option>, } +type TokenStream = crate::token_stream::TokenStream; +type BridgeTokenStream = crate::bridge::TokenStream; + impl server::Server for RaSpanServer<'_> { - type TokenStream = crate::token_stream::TokenStream; type Span = Span; type Symbol = Symbol; @@ -70,73 +72,26 @@ impl server::Server for RaSpanServer<'_> { // FIXME handle diagnostic } - fn ts_drop(&mut self, stream: Self::TokenStream) { - drop(stream); - } - - fn ts_clone(&mut self, stream: &Self::TokenStream) -> Self::TokenStream { - stream.clone() - } - - fn ts_is_empty(&mut self, stream: &Self::TokenStream) -> bool { - stream.is_empty() - } - fn ts_from_str(&mut self, src: &str) -> Self::TokenStream { - Self::TokenStream::from_str(src, self.call_site).unwrap_or_else(|e| { - Self::TokenStream::from_str( + fn ts_from_str(&mut self, src: &str) -> BridgeTokenStream { + TokenStream::from_str(src, self.call_site).unwrap_or_else(|e| { + TokenStream::from_str( &format!("compile_error!(\"failed to parse str to token stream: {e}\")"), self.call_site, ) .unwrap() - }) + }).into_bridge() } - fn ts_to_string(&mut self, stream: &Self::TokenStream) -> String { - stream.to_string() + fn ts_to_string(&mut self, stream: BridgeTokenStream) -> String { + TokenStream::from_bridge(stream).to_string() } - fn ts_from_token_tree(&mut self, tree: TokenTree) -> Self::TokenStream { - Self::TokenStream::new(vec![tree]) - } - - fn ts_expand_expr(&mut self, self_: &Self::TokenStream) -> Result { + fn ts_expand_expr(&mut self, self_: BridgeTokenStream) -> Result { // FIXME: requires db, more importantly this requires name resolution so we would need to // eagerly expand this proc-macro, but we can't know that this proc-macro is eager until we // expand it ... // This calls for some kind of marker that a proc-macro wants to access this eager API, // otherwise we need to treat every proc-macro eagerly / or not support this. - Ok(self_.clone()) - } - - fn ts_concat_trees( - &mut self, - base: Option, - trees: Vec>, - ) -> Self::TokenStream { - match base { - Some(mut base) => { - for tt in trees { - base.push_tree(tt); - } - base - } - None => Self::TokenStream::new(trees), - } - } - - fn ts_concat_streams( - &mut self, - base: Option, - streams: Vec, - ) -> Self::TokenStream { - let mut stream = base.unwrap_or_default(); - for s in streams { - stream.push_stream(s); - } - stream - } - - fn ts_into_trees(&mut self, stream: Self::TokenStream) -> Vec> { - (*stream.0).clone() + Ok(self_) } fn span_debug(&mut self, span: Self::Span) -> String { diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/token_id.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/token_id.rs index 70484c4dc28fe..bf732ddca5330 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/token_id.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/token_id.rs @@ -10,7 +10,7 @@ use rustc_proc_macro::bridge::server; use crate::{ ProcMacroClientHandle, - bridge::{Diagnostic, ExpnGlobals, Literal, TokenTree}, + bridge::{Diagnostic, ExpnGlobals, Literal}, server_impl::literal_from_str, }; @@ -36,8 +36,10 @@ pub struct SpanIdServer<'a> { pub callback: Option>, } +type TokenStream = crate::token_stream::TokenStream; +type BridgeTokenStream = crate::bridge::TokenStream; + impl server::Server for SpanIdServer<'_> { - type TokenStream = crate::token_stream::TokenStream; type Span = Span; type Symbol = Symbol; @@ -73,67 +75,21 @@ impl server::Server for SpanIdServer<'_> { fn emit_diagnostic(&mut self, _: Diagnostic) {} - fn ts_drop(&mut self, stream: Self::TokenStream) { - drop(stream); - } - - fn ts_clone(&mut self, stream: &Self::TokenStream) -> Self::TokenStream { - stream.clone() - } - - fn ts_is_empty(&mut self, stream: &Self::TokenStream) -> bool { - stream.is_empty() - } - fn ts_from_str(&mut self, src: &str) -> Self::TokenStream { - Self::TokenStream::from_str(src, self.call_site).unwrap_or_else(|e| { - Self::TokenStream::from_str( + fn ts_from_str(&mut self, src: &str) -> BridgeTokenStream { + TokenStream::from_str(src, self.call_site).unwrap_or_else(|e| { + TokenStream::from_str( &format!("compile_error!(\"failed to parse str to token stream: {e}\")"), self.call_site, ) .unwrap() - }) - } - fn ts_to_string(&mut self, stream: &Self::TokenStream) -> String { - stream.to_string() + }).into_bridge() } - fn ts_from_token_tree(&mut self, tree: TokenTree) -> Self::TokenStream { - Self::TokenStream::new(vec![tree]) - } - - fn ts_expand_expr(&mut self, self_: &Self::TokenStream) -> Result { - Ok(self_.clone()) - } - - fn ts_concat_trees( - &mut self, - base: Option, - trees: Vec>, - ) -> Self::TokenStream { - match base { - Some(mut base) => { - for tt in trees { - base.push_tree(tt); - } - base - } - None => Self::TokenStream::new(trees), - } - } - - fn ts_concat_streams( - &mut self, - base: Option, - streams: Vec, - ) -> Self::TokenStream { - let mut stream = base.unwrap_or_default(); - for s in streams { - stream.push_stream(s); - } - stream + fn ts_to_string(&mut self, stream: BridgeTokenStream) -> String { + TokenStream::from_bridge(stream).to_string() } - fn ts_into_trees(&mut self, stream: Self::TokenStream) -> Vec> { - (*stream.0).clone() + fn ts_expand_expr(&mut self, self_: BridgeTokenStream) -> Result { + Ok(self_) } fn span_debug(&mut self, span: Self::Span) -> String { diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/token_stream.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/token_stream.rs index 2358f6963c79e..19607d3673d19 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/token_stream.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/token_stream.rs @@ -1,19 +1,75 @@ //! The proc-macro server token stream implementation. use core::fmt; -use std::{mem, sync::Arc}; +use std::{mem, rc::Rc, sync::Arc}; use intern::Symbol; use rustc_lexer::{DocStyle, LiteralKind}; use rustc_proc_macro::Delimiter; -use crate::bridge::{DelimSpan, Group, Ident, LitKind, Literal, Punct, TokenTree}; +use crate::bridge::{ + DelimSpan, Group as BridgeGroup, Ident, LitKind, Literal, Punct, + TokenStream as BridgeTokenStream, TokenTree as BridgeTokenTree, +}; /// Trait for allowing tests to parse tokenstreams with dynamic span ranges pub(crate) trait SpanLike { fn derive_ranged(&self, range: std::ops::Range) -> Self; } +#[derive(Clone)] +pub struct Group { + pub delimiter: Delimiter, + pub stream: Option>, + pub span: DelimSpan, +} + +impl Group { + pub fn from_bridge(g: BridgeGroup) -> Self { + Self { + delimiter: g.delimiter, + stream: g.stream.map(TokenStream::from_bridge), + span: g.span, + } + } + + pub fn into_bridge(self) -> BridgeGroup { + BridgeGroup { + delimiter: self.delimiter, + stream: self.stream.map(TokenStream::into_bridge), + span: self.span, + } + } +} + +#[derive(Clone)] +pub enum TokenTree { + Group(Group), + Punct(Punct), + Ident(Ident), + Literal(Literal), +} + +impl TokenTree { + pub fn from_bridge(t: BridgeTokenTree) -> Self { + match t { + BridgeTokenTree::Group(x) => Self::Group(Group::from_bridge(x)), + BridgeTokenTree::Punct(x) => Self::Punct(x), + BridgeTokenTree::Ident(x) => Self::Ident(x), + BridgeTokenTree::Literal(x) => Self::Literal(x), + } + } + + pub fn into_bridge(self) -> BridgeTokenTree { + match self { + Self::Group(x) => BridgeTokenTree::Group(Group::into_bridge(x)), + Self::Punct(x) => BridgeTokenTree::Punct(x), + Self::Ident(x) => BridgeTokenTree::Ident(x), + Self::Literal(x) => BridgeTokenTree::Literal(x), + } + } +} + #[derive(Clone)] pub struct TokenStream(pub(crate) Arc>>); @@ -28,6 +84,24 @@ impl TokenStream { TokenStream(Arc::new(tts)) } + pub fn from_bridge(ts: BridgeTokenStream) -> TokenStream + where + S: Clone, + { + TokenStream::new( + Rc::unwrap_or_clone(ts.trees).into_iter().map(TokenTree::from_bridge).collect(), + ) + } + + pub fn into_bridge(self) -> BridgeTokenStream + where + S: Clone, + { + BridgeTokenStream::new( + Arc::unwrap_or_clone(self.0).into_iter().map(TokenTree::into_bridge).collect(), + ) + } + pub fn is_empty(&self) -> bool { self.0.is_empty() } @@ -632,26 +706,6 @@ fn debug_token_tree( writeln!(f) } -impl TokenStream { - /// Push `tt` onto the end of the stream, possibly gluing it to the last - /// token. Uses `make_mut` to maximize efficiency. - pub(crate) fn push_tree(&mut self, tt: TokenTree) { - let vec_mut = Arc::make_mut(&mut self.0); - vec_mut.push(tt); - } - - /// Push `stream` onto the end of the stream, possibly gluing the first - /// token tree to the last token. (No other token trees will be glued.) - /// Uses `make_mut` to maximize efficiency. - pub(crate) fn push_stream(&mut self, stream: TokenStream) { - let vec_mut = Arc::make_mut(&mut self.0); - - let stream_iter = stream.0.iter().cloned(); - - vec_mut.extend(stream_iter); - } -} - impl FromIterator> for TokenStream { fn from_iter>>(iter: I) -> Self { TokenStream::new(iter.into_iter().collect::>>())