Merge pull request #22016 from asgerf/commonast-rebased5

Unified/swift: new AST spec and Swift mappings

Author: asgerf

shared/yeast-macros/src/lib.rs

@@ -44,8 +44,19 @@ pub fn query(input: TokenStream) -> TokenStream {
 /// {expr}                       - embed a Rust expression returning Id
 /// {..expr}                     - splice an iterable of Id (in child/field position)
 /// field: {..expr}              - splice into a named field
+/// {expr}.map(p -> tpl)         - apply tpl to each element; splice result
+/// {expr}.reduce_left(f -> init, acc, e -> fold)
+///                              - fold with per-element init; splice 0 or 1 result
 /// ```
 ///
+/// Chain syntax after `{expr}` or `{..expr}`:
+/// - `.map(param -> template)` — one output node per input element.
+/// - `.reduce_left(first -> init, acc, elem -> fold)` — fold left; the first
+///   element is converted by `init`, subsequent elements are folded by `fold`
+///   with the accumulator bound to `acc`. An empty iterable yields nothing.
+/// - Chains always splice (the result is iterable).
+/// - Multiple chains can be chained, e.g. `.map(...).reduce_left(...)`.
+///
 /// Can be called with an explicit context or using the implicit context
 /// from an enclosing `rule!`:
 ///

shared/yeast-macros/src/parse.rs

@@ -141,7 +141,12 @@ fn parse_query_fields(tokens: &mut Tokens) -> Result<Vec<TokenStream>> {
             // Parse the field's pattern. To support repetition like
             // `field: (kind)* @cap`, parse the atom first, then check for
             // a quantifier, and lastly handle a trailing `@capture`.
-            let atom = parse_query_atom(tokens)?;
+            // `field: @cap` is sugar for `field: _ @cap`.
+            let atom = if peek_is_at(tokens) {
+                quote! { yeast::query::QueryNode::Any { match_unnamed: true } }
+            } else {
+                parse_query_atom(tokens)?
+            };
             if peek_is_repetition(tokens) {
                 let rep = expect_repetition(tokens)?;
                 let elem = quote! {
@@ -259,6 +264,7 @@ fn parse_query_list(tokens: &mut Tokens) -> Result<Vec<TokenStream>> {
                     yeast::query::QueryListElem::SingleNode(#node)
                 },
             )?;
+            let elem = maybe_wrap_list_capture(tokens, elem)?;
             elems.push(elem);
             continue;
         }
@@ -276,6 +282,7 @@ fn parse_query_list(tokens: &mut Tokens) -> Result<Vec<TokenStream>> {
                     yeast::query::QueryListElem::SingleNode(#node)
                 },
             )?;
+            let elem = maybe_wrap_list_capture(tokens, elem)?;
             elems.push(elem);
             continue;
         }
@@ -389,8 +396,10 @@ fn parse_direct_node_inner(tokens: &mut Tokens, ctx: &Ident) -> Result<TokenStre
         let expr = group.stream();
         return Ok(quote! {
             {
-                let __value = yeast::YeastDisplay::yeast_to_string(&(#expr), &*#ctx.ast);
-                #ctx.literal(#kind_str, &__value)
+                let __expr = (#expr);
+                let __value = yeast::YeastDisplay::yeast_to_string(&__expr, &*#ctx.ast);
+                let __source_range = yeast::YeastSourceRange::yeast_source_range(&__expr, &*#ctx.ast);
+                #ctx.literal_with_source_range(#kind_str, &__value, __source_range)
             }
         });
     }
@@ -419,23 +428,35 @@ fn parse_direct_node_inner(tokens: &mut Tokens, ctx: &Ident) -> Result<TokenStre
         );
         field_counter += 1;
 
-        // Check for field: {..expr} — splice a Vec<Id> into the field
+        // Check for field: {..expr}.chain or field: {expr}.chain — splice a Vec<Id> into the field
         if peek_is_group(tokens, Delimiter::Brace) {
             let group_clone = tokens.clone().next().unwrap();
             if let TokenTree::Group(g) = &group_clone {
                 let mut inner_check = g.stream().into_iter();
                 let is_splice = matches!(inner_check.next(), Some(TokenTree::Punct(p)) if p.as_char() == '.')
                     && matches!(inner_check.next(), Some(TokenTree::Punct(p)) if p.as_char() == '.');
-                if is_splice {
+                // Determine if a chain (.map(..)) follows the `{}` group.
+                let mut after = tokens.clone();
+                after.next(); // skip the brace group
+                let has_chain = matches!(after.peek(), Some(TokenTree::Punct(p)) if p.as_char() == '.');
+
+                if is_splice || has_chain {
                     let group = expect_group(tokens, Delimiter::Brace)?;
-                    let mut inner = group.stream().into_iter().peekable();
-                    inner.next(); // consume first .
-                    inner.next(); // consume second .
-                    let expr: proc_macro2::TokenStream = inner.collect();
+                    let base: TokenStream = if is_splice {
+                        let mut inner = group.stream().into_iter().peekable();
+                        inner.next(); // consume first .
+                        inner.next(); // consume second .
+                        let expr: TokenStream = inner.collect();
+                        quote! {
+                            (#expr).into_iter().map(::std::convert::Into::<usize>::into)
+                        }
+                    } else {
+                        let expr = group.stream();
+                        quote! { (#expr).into_iter() }
+                    };
+                    let chained = parse_chain_suffix(tokens, ctx, base)?;
                     stmts.push(quote! {
-                        let #temp: Vec<usize> = (#expr).into_iter()
-                            .map(::std::convert::Into::<usize>::into)
-                            .collect();
+                        let #temp: Vec<usize> = #chained.collect();
                     });
                     // An empty splice means the field is absent — skip it
                     // entirely rather than emitting an empty named field.
@@ -472,6 +493,98 @@ fn parse_direct_node_inner(tokens: &mut Tokens, ctx: &Ident) -> Result<TokenStre
     })
 }
 
+/// Parse a chain of `.method(args)` suffixes after a `{expr}` or `{..expr}`
+/// placeholder in tree templates. Currently supports:
+///
+/// ```text
+/// .map(param -> template)   -- iterator map: produces Vec<usize>
+/// ```
+///
+/// The chain may be empty (returns `base` unchanged). Multiple chained calls
+/// are supported, e.g. `.map(p -> ...).map(q -> ...)`.
+///
+/// Each call expects the receiver to be an iterator. The `base` argument
+/// should therefore already be an iterator (use `.into_iter()` on it before
+/// calling this function).
+fn parse_chain_suffix(
+    tokens: &mut Tokens,
+    ctx: &Ident,
+    base: TokenStream,
+) -> Result<TokenStream> {
+    let mut current = base;
+    while matches!(tokens.peek(), Some(TokenTree::Punct(p)) if p.as_char() == '.') {
+        tokens.next(); // consume .
+        let method = expect_ident(tokens, "expected method name after `.`")?;
+        let method_str = method.to_string();
+        let args_group = expect_group(tokens, Delimiter::Parenthesis)?;
+        match method_str.as_str() {
+            "map" => {
+                let mut inner = args_group.stream().into_iter().peekable();
+                let param = expect_ident(&mut inner, "expected lambda parameter name")?;
+                expect_punct(&mut inner, '-', "expected `->` after lambda parameter")?;
+                expect_punct(&mut inner, '>', "expected `->` after lambda parameter")?;
+                let body = parse_direct_node(&mut inner, ctx)?;
+                if let Some(tok) = inner.next() {
+                    return Err(syn::Error::new_spanned(
+                        tok,
+                        "unexpected token after lambda body",
+                    ));
+                }
+                current = quote! {
+                    #current.map(|#param| #body)
+                };
+            }
+            "reduce_left" => {
+                // Syntax: reduce_left(first -> init_tpl, acc, elem -> fold_tpl)
+                // - first -> init_tpl : converts the first element to the initial accumulator
+                // - acc, elem -> fold_tpl : fold step (acc = current accumulator, elem = next element)
+                // Empty iterator produces an empty iterator; non-empty produces a single-element iterator.
+                let mut inner = args_group.stream().into_iter().peekable();
+                let init_param = expect_ident(&mut inner, "expected initial lambda parameter")?;
+                expect_punct(&mut inner, '-', "expected `->` after init parameter")?;
+                expect_punct(&mut inner, '>', "expected `->` after init parameter")?;
+                let init_body = parse_direct_node(&mut inner, ctx)?;
+                expect_punct(&mut inner, ',', "expected `,` after init template")?;
+                let acc_param = expect_ident(&mut inner, "expected accumulator parameter")?;
+                expect_punct(&mut inner, ',', "expected `,` after accumulator parameter")?;
+                let elem_param = expect_ident(&mut inner, "expected element parameter")?;
+                expect_punct(&mut inner, '-', "expected `->` after element parameter")?;
+                expect_punct(&mut inner, '>', "expected `->` after element parameter")?;
+                let fold_body = parse_direct_node(&mut inner, ctx)?;
+                if let Some(tok) = inner.next() {
+                    return Err(syn::Error::new_spanned(
+                        tok,
+                        "unexpected token after fold template",
+                    ));
+                }
+                current = quote! {
+                    {
+                        let mut __iter = #current;
+                        let __result: Option<usize> = if let Some(#init_param) = __iter.next() {
+                            let mut __acc: usize = #init_body;
+                            for #elem_param in __iter {
+                                let #acc_param: usize = __acc;
+                                __acc = #fold_body;
+                            }
+                            Some(__acc)
+                        } else {
+                            None
+                        };
+                        __result.into_iter()
+                    }
+                };
+            }
+            _ => {
+                return Err(syn::Error::new_spanned(
+                    method,
+                    format!("unknown builtin method `.{method_str}()`"),
+                ));
+            }
+        }
+    }
+    Ok(current)
+}
+
 /// Parse the top-level list of a `trees!` template.
 /// Each item is a node template or `{expr}` splice.
 fn parse_direct_list(tokens: &mut Tokens, ctx: &Ident) -> Result<Vec<TokenStream>> {
@@ -492,18 +605,27 @@ fn parse_direct_list(tokens: &mut Tokens, ctx: &Ident) -> Result<Vec<TokenStream
             continue;
         }
 
-        // {expr} or {..expr} — single node or splice
+        // {expr} or {..expr} (with optional .chain) — single node or splice
         if peek_is_group(tokens, Delimiter::Brace) {
             let group = expect_group(tokens, Delimiter::Brace)?;
+            let has_chain = matches!(tokens.peek(), Some(TokenTree::Punct(p)) if p.as_char() == '.');
             let mut inner = group.stream().into_iter().peekable();
-            if peek_is_dotdot(&inner) {
-                inner.next(); // consume first .
-                inner.next(); // consume second .
-                let expr: TokenStream = inner.collect();
-                items.push(quote! {
-                    __nodes.extend(
+            let is_splice = peek_is_dotdot(&inner);
+            if is_splice || has_chain {
+                let base: TokenStream = if is_splice {
+                    inner.next(); // consume first .
+                    inner.next(); // consume second .
+                    let expr: TokenStream = inner.collect();
+                    quote! {
                         (#expr).into_iter().map(::std::convert::Into::<usize>::into)
-                    );
+                    }
+                } else {
+                    let expr = group.stream();
+                    quote! { (#expr).into_iter() }
+                };
+                let chained = parse_chain_suffix(tokens, ctx, base)?;
+                items.push(quote! {
+                    __nodes.extend(#chained);
                 });
             } else {
                 let expr = group.stream();
@@ -604,8 +726,11 @@ fn extract_captures_inner(
                 }
                 last_mult = CaptureMultiplicity::Single;
             }
-            TokenTree::Punct(p) if matches!(p.as_char(), '*' | '+' | '?') => {
-                // Keep last_mult — the @capture follows
+            TokenTree::Punct(p) if p.as_char() == '*' || p.as_char() == '+' => {
+                last_mult = CaptureMultiplicity::Repeated;
+            }
+            TokenTree::Punct(p) if p.as_char() == '?' => {
+                last_mult = CaptureMultiplicity::Optional;
             }
             _ => {
                 last_mult = CaptureMultiplicity::Single;

shared/yeast/src/build.rs

@@ -82,10 +82,34 @@ impl<'a> BuildCtx<'a> {
             .create_named_token_with_range(kind, value.to_string(), self.source_range)
     }
 
+    /// Create a leaf node with fixed content and an optional preferred source range.
+    /// If `source_range` is `None`, falls back to this context's inherited range.
+    pub fn literal_with_source_range(
+        &mut self,
+        kind: &'static str,
+        value: &str,
+        source_range: Option<tree_sitter::Range>,
+    ) -> Id {
+        self.ast.create_named_token_with_range(
+            kind,
+            value.to_string(),
+            source_range.or(self.source_range),
+        )
+    }
+
     /// Create a leaf node with an auto-generated unique name.
     pub fn fresh(&mut self, kind: &'static str, name: &str) -> Id {
         let generated = self.fresh.resolve(name);
         self.ast
             .create_named_token_with_range(kind, generated, self.source_range)
     }
+
+    /// Prepend a value to a field of an existing node.
+    pub fn prepend_field(&mut self, node_id: Id, field_name: &str, value_id: Id) {
+        let field_id = self
+            .ast
+            .field_id_for_name(field_name)
+            .unwrap_or_else(|| panic!("build: field '{field_name}' not found"));
+        self.ast.prepend_field_child(node_id, field_id, value_id);
+    }
 }