From 35c84d223db5fd1993b4fa78103688f25eee8de5 Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 24 Sep 2026 10:52:46 +0000 Subject: [PATCH 1/2] yeast: Remove explicit `ctx` from tree builders We never used the ability to specify a different context from `ctx` in these builders anyway, so this commit makes it so that we always use the ambient `ctx` that is provided inside of rule bodies. Note that if we end up needing a particular context anyway, then we can simply reassign `ctx` before calling the tree macros. This has the same effect as passing in a custom context would have now. --- shared/yeast-macros/src/lib.rs | 46 ++++++++-- shared/yeast-macros/src/parse.rs | 90 ++++++++++++++----- shared/yeast/doc/yeast.md | 19 ++-- shared/yeast/src/lib.rs | 36 +------- shared/yeast/tests/test.rs | 15 ++-- .../extractor/src/languages/swift/swift.rs | 15 +--- 6 files changed, 125 insertions(+), 96 deletions(-) diff --git a/shared/yeast-macros/src/lib.rs b/shared/yeast-macros/src/lib.rs index f5637542dc8c..5a227c5fb7dc 100644 --- a/shared/yeast-macros/src/lib.rs +++ b/shared/yeast-macros/src/lib.rs @@ -75,12 +75,10 @@ pub fn query(input: TokenStream) -> TokenStream { /// error, so the choice between "unset the field" and "unwrap it" stays /// explicit. /// -/// Can be called with an explicit context or using the implicit context -/// from an enclosing `rule!`: +/// Uses the `BuildCtx` binding named `ctx` from the surrounding scope: /// /// ```text -/// tree!(ctx, (kind ...)) // explicit BuildCtx -/// tree!((kind ...)) // implicit context from rule! +/// tree!((kind ...)) /// ``` #[proc_macro] pub fn tree(input: TokenStream) -> TokenStream { @@ -96,12 +94,10 @@ pub fn tree(input: TokenStream) -> TokenStream { /// Like `tree!` but returns `Vec` and supports multiple top-level /// elements. All syntax from `tree!` is available. /// -/// Can be called with an explicit context or using the implicit context -/// from an enclosing `rule!`: +/// Uses the `BuildCtx` binding named `ctx` from the surrounding scope: /// /// ```text -/// trees!(ctx, (node1 ...) (node2 ...)) // explicit BuildCtx -/// trees!((node1 ...) (node2 ...)) // implicit context from rule! +/// trees!((node1 ...) (node2 ...)) /// ``` #[proc_macro] pub fn trees(input: TokenStream) -> TokenStream { @@ -112,6 +108,38 @@ pub fn trees(input: TokenStream) -> TokenStream { } } +/// Build one AST node whose root uses another node's source range. +/// +/// Uses the `BuildCtx` binding named `ctx` from the surrounding scope: +/// +/// ```text +/// tree_at!(source, (kind ...)) +/// ``` +#[proc_macro] +pub fn tree_at(input: TokenStream) -> TokenStream { + let input2: TokenStream2 = input.into(); + match parse::parse_tree_at_top(input2) { + Ok(output) => output.into(), + Err(err) => err.to_compile_error().into(), + } +} + +/// Build one AST node whose root spans a collection of nodes. +/// +/// Uses the `BuildCtx` binding named `ctx` from the surrounding scope: +/// +/// ```text +/// tree_spanning!(sources, (kind ...)) +/// ``` +#[proc_macro] +pub fn tree_spanning(input: TokenStream) -> TokenStream { + let input2: TokenStream2 = input.into(); + match parse::parse_tree_spanning_top(input2) { + Ok(output) => output.into(), + Err(err) => err.to_compile_error().into(), + } +} + /// Define a desugaring rule with query and transform in one declaration. /// /// ```text @@ -146,7 +174,7 @@ pub fn trees(input: TokenStream) -> TokenStream { /// Mutations to `ctx` are visible to the transform when the guard succeeds. /// Omitting the guard is equivalent to writing `where true`. /// -/// `tree!` and `trees!` can be used without explicit context inside `{...}`. +/// `tree!` and `trees!` use the rule transform's `ctx` binding inside `{...}`. #[proc_macro] pub fn rule(input: TokenStream) -> TokenStream { let input2: TokenStream2 = input.into(); diff --git a/shared/yeast-macros/src/parse.rs b/shared/yeast-macros/src/parse.rs index 80cca632d76c..a4e7e1b11908 100644 --- a/shared/yeast-macros/src/parse.rs +++ b/shared/yeast-macros/src/parse.rs @@ -332,28 +332,10 @@ fn parse_query_list(tokens: &mut Tokens) -> Result> { const IMPLICIT_CTX: &str = "ctx"; -/// Determine the context identifier: either explicit `ctx,` or the implicit -/// `ctx` from an enclosing `rule!`. -fn parse_ctx_or_implicit(tokens: &mut Tokens) -> Ident { - // Check if first token is an ident followed by a comma - let mut lookahead = tokens.clone(); - let is_explicit = matches!(lookahead.next(), Some(TokenTree::Ident(_))) - && matches!(lookahead.next(), Some(TokenTree::Punct(p)) if p.as_char() == ','); - - if is_explicit { - let ctx = expect_ident(tokens, "unreachable: ident was just peeked") - .expect("unreachable: ident was just peeked"); - let _ = tokens.next(); // consume comma - ctx - } else { - Ident::new(IMPLICIT_CTX, Span::call_site()) - } -} - -/// Parse `tree!(ctx, (template))` or `tree!((template))` — returns single `Id`. +/// Parse `tree!((template))` — returns single `Id`. pub fn parse_tree_top(input: TokenStream) -> Result { let mut tokens = input.into_iter().peekable(); - let ctx = parse_ctx_or_implicit(&mut tokens); + let ctx = Ident::new(IMPLICIT_CTX, Span::call_site()); let first = parse_direct_node(&mut tokens, &ctx, None)?; @@ -368,10 +350,10 @@ pub fn parse_tree_top(input: TokenStream) -> Result { Ok(quote! { { #first } }) } -/// Parse `trees!(ctx, ...)` or `trees!(...)` — returns `Vec`. +/// Parse `trees!(...)` — returns `Vec`. pub fn parse_trees_top(input: TokenStream) -> Result { let mut tokens = input.into_iter().peekable(); - let ctx = parse_ctx_or_implicit(&mut tokens); + let ctx = Ident::new(IMPLICIT_CTX, Span::call_site()); let items = parse_direct_list(&mut tokens, &ctx)?; if let Some(tok) = tokens.next() { return Err(syn::Error::new_spanned( @@ -388,6 +370,70 @@ pub fn parse_trees_top(input: TokenStream) -> Result { }) } +pub fn parse_tree_at_top(input: TokenStream) -> Result { + let mut tokens = input.into_iter().peekable(); + let source = parse_argument(&mut tokens, "expected `,` after source node")?; + let ctx = Ident::new(IMPLICIT_CTX, Span::call_site()); + let node = parse_direct_node(&mut tokens, &ctx, None)?; + if let Some(tok) = tokens.next() { + return Err(syn::Error::new_spanned( + tok, + "unexpected token after tree_at! template", + )); + } + + Ok(quote! { + { + let __yeast_source: yeast::Id = { #source }; + let __yeast_source_range = #ctx + .ast + .get_node(__yeast_source) + .and_then(|node| node.source_range()); + let __yeast_node: yeast::Id = #node; + #ctx.set_node_source_range(__yeast_node, __yeast_source_range) + } + }) +} + +pub fn parse_tree_spanning_top(input: TokenStream) -> Result { + let mut tokens = input.into_iter().peekable(); + let sources = parse_argument(&mut tokens, "expected `,` after source nodes")?; + let ctx = Ident::new(IMPLICIT_CTX, Span::call_site()); + let node = parse_direct_node(&mut tokens, &ctx, None)?; + if let Some(tok) = tokens.next() { + return Err(syn::Error::new_spanned( + tok, + "unexpected token after tree_spanning! template", + )); + } + + Ok(quote! { + { + let __yeast_source_range = ::std::iter::IntoIterator::into_iter({ #sources }) + .filter_map(|source: yeast::Id| { + #ctx.ast.get_node(source).and_then(|node| node.source_range()) + }) + .reduce(yeast::Range::union); + let __yeast_node: yeast::Id = #node; + #ctx.set_node_source_range(__yeast_node, __yeast_source_range) + } + }) +} + +fn parse_argument(tokens: &mut Tokens, missing_comma: &str) -> Result { + let mut argument = TokenStream::new(); + while let Some(token) = tokens.next() { + if matches!(&token, TokenTree::Punct(p) if p.as_char() == ',') { + if argument.is_empty() { + return Err(syn::Error::new_spanned(token, "expected expression")); + } + return Ok(argument); + } + argument.extend([token]); + } + Err(syn::Error::new(Span::call_site(), missing_comma)) +} + /// Parse a single node template and generate code that returns an `Id`. /// Handles: `(kind fields... children...)` and `{expr}`. /// diff --git a/shared/yeast/doc/yeast.md b/shared/yeast/doc/yeast.md index 9b7626f45fb4..03699a75051f 100644 --- a/shared/yeast/doc/yeast.md +++ b/shared/yeast/doc/yeast.md @@ -160,22 +160,21 @@ Templates construct new AST nodes using the `tree!` and `trees!` macros. All children in a template must be in named fields — output AST nodes are always fully fielded. -When used inside a `rule!` macro, the context is implicit — no explicit -`BuildCtx` argument is needed. When used standalone, they take a `BuildCtx` -as the first argument: +The macros use a `BuildCtx` binding named `ctx` from the surrounding scope. +`rule!` provides this binding automatically; standalone uses must create it: ```rust -// Inside rule! — implicit context, captures are Rust variables +// Inside rule! — ctx is provided automatically, captures are Rust variables yeast::rule!( (assignment left: (_) @left right: (_) @right) => (assignment left: {right} right: {left}) ); -// Standalone — explicit context +// Standalone — create a binding named ctx let mut user_ctx = (); let mut ctx = BuildCtx::new(ast, &captures, &mut user_ctx); -let id = yeast::tree!(ctx, +let id = yeast::tree!( (assignment left: {ctx.capture("lhs")} right: {ctx.capture("rhs")} @@ -188,7 +187,7 @@ let id = yeast::tree!(ctx, `tree!(...)` returns a single node `Id`: ```rust -yeast::tree!(ctx, +yeast::tree!( (assignment left: {ctx.capture("lhs")} right: {ctx.capture("rhs")} @@ -201,7 +200,7 @@ yeast::tree!(ctx, `trees!(...)` returns `Vec`: ```rust -yeast::trees!(ctx, +yeast::trees!( (assignment left: {tmp} right: {right}) {body} ) @@ -262,7 +261,6 @@ rule!( => synthetic_node { tree_at!( - ctx, source_node, (synthetic_node child: (nested value: {child})) ) @@ -282,7 +280,6 @@ rule!( => synthetic_node { tree_spanning!( - ctx, [first, second], (synthetic_node child: {child}) ) @@ -382,7 +379,7 @@ options uniformly: right: {rhs} // a captured value (inside rule!) ) -yeast::trees!(ctx, +yeast::trees!( (assignment left: {tmp} right: {right}) {extra_nodes} // splices a Vec ) diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index 2e8af818d7f8..9bbb8b5343eb 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -15,41 +15,7 @@ pub mod schema; mod visitor; pub use range::{Point, Range}; -pub use yeast_macros::{query, rule, rules, tree, trees}; - -/// Build a single AST node whose root uses another node's source range. -/// -/// Nested nodes in the template are built normally and derive their locations -/// from their own children. -#[macro_export] -macro_rules! tree_at { - ($ctx:ident, $source:expr, ($($tree:tt)*)) => {{ - let __yeast_source: $crate::Id = $source; - let __yeast_source_range = $ctx - .ast - .get_node(__yeast_source) - .and_then(|node| node.source_range()); - let __yeast_node: $crate::Id = $crate::tree!($ctx, ($($tree)*)); - $ctx.set_node_source_range(__yeast_node, __yeast_source_range) - }}; -} - -/// Build a single AST node whose root spans a collection of nodes. -/// -/// Nested nodes in the template are built normally and derive their locations -/// from their own children. -#[macro_export] -macro_rules! tree_spanning { - ($ctx:ident, $sources:expr, ($($tree:tt)*)) => {{ - let __yeast_source_range = ::std::iter::IntoIterator::into_iter($sources) - .filter_map(|source: $crate::Id| { - $ctx.ast.get_node(source).and_then(|node| node.source_range()) - }) - .reduce($crate::Range::union); - let __yeast_node: $crate::Id = $crate::tree!($ctx, ($($tree)*)); - $ctx.set_node_source_range(__yeast_node, __yeast_source_range) - }}; -} +pub use yeast_macros::{query, rule, rules, tree, tree_at, tree_spanning, trees}; use captures::Captures; use query::QueryNode; diff --git a/shared/yeast/tests/test.rs b/shared/yeast/tests/test.rs index a8f4144f9fce..099f5ebe7eb9 100644 --- a/shared/yeast/tests/test.rs +++ b/shared/yeast/tests/test.rs @@ -695,7 +695,7 @@ fn test_tree_builder() { // Swap left and right let mut user_ctx = (); let mut ctx = yeast::build::BuildCtx::new(&mut ast, &captures, &mut user_ctx); - let new_id = yeast::tree!(ctx, + let new_id = yeast::tree!( (program child: (assignment left: {ctx.capture("right")} @@ -723,8 +723,8 @@ fn build_optional_right(ast: &mut Ast, value: Option) -> (yeast::Id, let captures = yeast::captures::Captures::new(); let mut user_ctx = (); let mut ctx = yeast::build::BuildCtx::new(ast, &captures, &mut user_ctx); - let left = yeast::tree!(ctx, (identifier "x")); - let root = yeast::tree!(ctx, + let left = yeast::tree!((identifier "x")); + let root = yeast::tree!( (assignment left: {left} right: (integer #{value})? @@ -781,8 +781,8 @@ fn test_optional_field_propagates_through_nested_nodes() { // The absent value sits two levels below the `?`, so the whole // `left_assignment_list` subtree is abandoned along with it. let absent: Option = None; - let right = yeast::tree!(ctx, (integer "1")); - let root = yeast::tree!(ctx, + let right = yeast::tree!((integer "1")); + let root = yeast::tree!( (assignment left: (left_assignment_list child: (identifier #{absent}))? right: {right} @@ -809,7 +809,7 @@ fn test_innermost_optional_field_catches_first() { // The inner `?` catches, so only `child` is dropped; `left` survives. let absent: Option = None; - let root = yeast::tree!(ctx, + let root = yeast::tree!( (assignment left: (left_assignment_list child: (identifier #{absent})?)? ) @@ -1782,7 +1782,7 @@ fn test_tree_at_assigns_capture_range_to_root_only() { ) @@source => call { - let arguments = tree_at!(ctx, source, (argument_list argument: (integer "0"))); + let arguments = tree_at!(source, (argument_list argument: (integer "0"))); tree!((call method: {name} receiver: {recv} arguments: {arguments})) } ); @@ -1817,7 +1817,6 @@ fn test_tree_spanning_assigns_union_to_root_only() { => call { let arguments = tree_spanning!( - ctx, [recv, name], (argument_list argument: (integer "0")) ); diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index d7026b19092d..5b961e7d75a9 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -393,7 +393,6 @@ fn translation_rules() -> Vec> { None => None, }; tree_spanning!( - ctx, std::iter::once(spec).chain(body), (accessor_declaration modifier: {binding} @@ -473,7 +472,6 @@ fn translation_rules() -> Vec> { => class_like_declaration { let constructor = tree_spanning!( - ctx, [name, clause], (constructor_declaration parameter: {params} body: (block)) ); @@ -648,7 +646,6 @@ fn translation_rules() -> Vec> { => call_expr { let callee = tree_at!( - ctx, array, (generic_type_expr base: (identifier "Array") @@ -668,7 +665,6 @@ fn translation_rules() -> Vec> { => call_expr { let callee = tree_at!( - ctx, array, (generic_type_expr base: (identifier "Array") @@ -719,7 +715,6 @@ fn translation_rules() -> Vec> { => member_access_expr { let base = tree_at!( - ctx, array, (generic_type_expr base: (identifier "Array") @@ -985,7 +980,6 @@ fn translation_rules() -> Vec> { expr { let op = format!("try{}", m.map(|m| ctx.source_text(m)).unwrap_or_default()); let operator = tree_spanning!( - ctx, std::iter::once(keyword).chain(m), (prefix_operator #{op}) ); @@ -1028,7 +1022,6 @@ fn translation_rules() -> Vec> { rule!((asExpr expression: @val asKeyword: @@keyword questionOrExclamationMark: _? @@mark type: @ty) => type_cast_expr { let op = format!("as{}", mark.map(|m| ctx.source_text(m)).unwrap_or_default()); let operator = tree_spanning!( - ctx, std::iter::once(keyword).chain(mark), (infix_operator #{op}) ); @@ -1037,16 +1030,16 @@ fn translation_rules() -> Vec> { // Check expression (`x is T`) → type_test_expr rule!((isExpr expression: @val isKeyword: @@keyword type: @ty) => (type_test_expr expr: {val} - operator: {tree_at!(ctx, keyword, (infix_operator "is"))} + operator: {tree_at!(keyword, (infix_operator "is"))} type: {ty})), // Await expression → unary_expr with operator "await" rule!((awaitExpr awaitKeyword: @@keyword expression: @val) => (unary_expr - operator: {tree_at!(ctx, keyword, (prefix_operator "await"))} + operator: {tree_at!(keyword, (prefix_operator "await"))} operand: {val})), // Force-unwrap (`x!`) → postfix unary_expr, via swift-syntax's dedicated // `forceUnwrapExpr` node. rule!((forceUnwrapExpr expression: @e exclamationMark: @@mark) => (unary_expr - operator: {tree_at!(ctx, mark, (postfix_operator "!"))} + operator: {tree_at!(mark, (postfix_operator "!"))} operand: {e})), // ---- Imports ---- // An import declaration. The dotted path (a list of @@ -1068,7 +1061,7 @@ fn translation_rules() -> Vec> { let last = *parts.last().ok_or("import has no path")?; let pattern = match kind { None => { - let bulk = tree_at!(ctx, decl, (bulk_importing_pattern)); + let bulk = tree_at!(decl, (bulk_importing_pattern)); tree!((named_pattern name_node: (identifier #{last}) sub_pattern: {bulk})) From 2ee0621f0cfcfdb3fa3e7135fefc1840f8d0d389 Mon Sep 17 00:00:00 2001 From: Taus Date: Fri, 25 Sep 2026 14:06:07 +0000 Subject: [PATCH 2/2] yeast: Address review comments Does the parsing of arguments in a slightly more principled way. --- shared/yeast-macros/src/parse.rs | 41 +++++++++++++++++++------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/shared/yeast-macros/src/parse.rs b/shared/yeast-macros/src/parse.rs index a4e7e1b11908..37ecff08bd20 100644 --- a/shared/yeast-macros/src/parse.rs +++ b/shared/yeast-macros/src/parse.rs @@ -2,7 +2,10 @@ use proc_macro2::{Delimiter, Ident, Literal, Span, TokenStream, TokenTree}; use quote::quote; use std::iter::Peekable; use std::sync::atomic::{AtomicUsize, Ordering}; -use syn::Lifetime; +use syn::{ + Expr, Lifetime, Token, + parse::{Parse, ParseStream}, +}; type Tokens = Peekable; type Result = std::result::Result; @@ -371,8 +374,11 @@ pub fn parse_trees_top(input: TokenStream) -> Result { } pub fn parse_tree_at_top(input: TokenStream) -> Result { - let mut tokens = input.into_iter().peekable(); - let source = parse_argument(&mut tokens, "expected `,` after source node")?; + let LocatedTreeInput { + source, + template, + } = syn::parse2(input)?; + let mut tokens = template.into_iter().peekable(); let ctx = Ident::new(IMPLICIT_CTX, Span::call_site()); let node = parse_direct_node(&mut tokens, &ctx, None)?; if let Some(tok) = tokens.next() { @@ -396,8 +402,11 @@ pub fn parse_tree_at_top(input: TokenStream) -> Result { } pub fn parse_tree_spanning_top(input: TokenStream) -> Result { - let mut tokens = input.into_iter().peekable(); - let sources = parse_argument(&mut tokens, "expected `,` after source nodes")?; + let LocatedTreeInput { + source: sources, + template, + } = syn::parse2(input)?; + let mut tokens = template.into_iter().peekable(); let ctx = Ident::new(IMPLICIT_CTX, Span::call_site()); let node = parse_direct_node(&mut tokens, &ctx, None)?; if let Some(tok) = tokens.next() { @@ -420,18 +429,18 @@ pub fn parse_tree_spanning_top(input: TokenStream) -> Result { }) } -fn parse_argument(tokens: &mut Tokens, missing_comma: &str) -> Result { - let mut argument = TokenStream::new(); - while let Some(token) = tokens.next() { - if matches!(&token, TokenTree::Punct(p) if p.as_char() == ',') { - if argument.is_empty() { - return Err(syn::Error::new_spanned(token, "expected expression")); - } - return Ok(argument); - } - argument.extend([token]); +struct LocatedTreeInput { + source: Expr, + template: TokenStream, +} + +impl Parse for LocatedTreeInput { + fn parse(input: ParseStream<'_>) -> syn::Result { + let source = input.parse()?; + input.parse::()?; + let template = input.parse()?; + Ok(Self { source, template }) } - Err(syn::Error::new(Span::call_site(), missing_comma)) } /// Parse a single node template and generate code that returns an `Id`.