Rollup merge of #153308 - aytey:macro_meta_hygiene, r=jdonszelmann

Add hygiene annotations for tokens in `macro_rules!` bodies

`-Zunpretty=expanded,hygiene` was not printing syntax context annotations for identifiers and lifetimes inside `macro_rules!` bodies. These tokens are printed via `print_tt()` → `token_to_string_ext()`, which converts tokens to strings without calling `ann_post()`. This meant that macro-generated `macro_rules!` definitions with hygienic metavar parameters (e.g. multiple `$marg` distinguished only by hygiene) were printed with no way to tell them apart.

This was fixed by adding a match on `token.kind` in `print_tt()` to call `ann_post()` for `Ident`, `NtIdent`, `Lifetime`, and `NtLifetime` tokens, matching how `print_ident()` and `print_lifetime()` already handle AST-level identifiers and lifetimes.

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -737,6 +737,23 @@ fn print_tt(&mut self, tt: &TokenTree, convert_dollar_crate: bool) -> Spacing {
             TokenTree::Token(token, spacing) => {
                 let token_str = self.token_to_string_ext(token, convert_dollar_crate);
                 self.word(token_str);
+                // Emit hygiene annotations for identity-bearing tokens,
+                // matching how print_ident() and print_lifetime() call ann_post().
+                match token.kind {
+                    token::Ident(name, _) => {
+                        self.ann_post(Ident::new(name, token.span));
+                    }
+                    token::NtIdent(ident, _) => {
+                        self.ann_post(ident);
+                    }
+                    token::Lifetime(name, _) => {
+                        self.ann_post(Ident::new(name, token.span));
+                    }
+                    token::NtLifetime(ident, _) => {
+                        self.ann_post(ident);
+                    }
+                    _ => {}
+                }
                 if let token::DocComment(..) = token.kind {
                     self.hardbreak()
                 }

tests/ui/hygiene/unpretty-debug-lifetimes.stdout

@@ -7,15 +7,17 @@
 // Don't break whenever Symbol numbering changes
 //@ normalize-stdout: "\d+#" -> "0#"
 
-#![feature /* 0#0 */(decl_macro)]
-#![feature /* 0#0 */(no_core)]
+#![feature /* 0#0 */(decl_macro /* 0#0 */)]
+#![feature /* 0#0 */(no_core /* 0#0 */)]
 #![no_core /* 0#0 */]
 
 macro lifetime_hygiene
     /*
     0#0
     */ {
-    ($f:ident<$a:lifetime>) => { fn $f<$a, 'a>() {} }
+    ($f /* 0#0 */:ident /* 0#0 */<$a /* 0#0 */:lifetime /* 0#0 */>)
+    =>
+    { fn /* 0#0 */ $f /* 0#0 */<$a /* 0#0 */, 'a /* 0#0 */>() {} }
 }
 fn f /* 0#0 */<'a /* 0#0 */, 'a /* 0#1 */>() {}
 

tests/ui/hygiene/unpretty-debug-metavars.rs

@@ -0,0 +1,26 @@
+//@ check-pass
+//@ compile-flags: -Zunpretty=expanded,hygiene
+
+// Regression test for token hygiene annotations in -Zunpretty=expanded,hygiene
+// Previously, metavar parameters in macro-generated macro_rules! definitions
+// were missing hygiene annotations, making identical `$marg` bindings
+// indistinguishable.
+
+// Don't break whenever Symbol numbering changes
+//@ normalize-stdout: "\d+#" -> "0#"
+
+#![feature(no_core)]
+#![no_core]
+
+macro_rules! make_macro {
+    (@inner $name:ident ($dol:tt) $a:ident) => {
+        macro_rules! $name {
+            ($dol $a : expr, $dol marg : expr) => {}
+        }
+    };
+    ($name:ident) => {
+        make_macro!{@inner $name ($) marg}
+    };
+}
+
+make_macro!(add2);

tests/ui/hygiene/unpretty-debug-metavars.stdout

@@ -0,0 +1,53 @@
+//@ check-pass
+//@ compile-flags: -Zunpretty=expanded,hygiene
+
+// Regression test for token hygiene annotations in -Zunpretty=expanded,hygiene
+// Previously, metavar parameters in macro-generated macro_rules! definitions
+// were missing hygiene annotations, making identical `$marg` bindings
+// indistinguishable.
+
+// Don't break whenever Symbol numbering changes
+//@ normalize-stdout: "\d+#" -> "0#"
+
+#![feature /* 0#0 */(no_core /* 0#0 */)]
+#![no_core /* 0#0 */]
+
+macro_rules! make_macro
+    /*
+    0#0
+    */ {
+    (@inner /* 0#0 */ $name /* 0#0 */:ident /* 0#0
+    */($dol /* 0#0 */:tt /* 0#0 */) $a /* 0#0 */:ident /* 0#0 */)
+    =>
+    {
+        macro_rules /* 0#0 */! $name /* 0#0 */
+        {
+            ($dol /* 0#0 */ $a /* 0#0 */ : expr /* 0#0 */, $dol /*
+            0#0 */ marg /* 0#0 */ : expr /* 0#0 */) => {}
+        }
+    }; ($name /* 0#0 */:ident /* 0#0 */) =>
+    {
+        make_macro /* 0#0
+        */!{@inner /* 0#0 */ $name /* 0#0 */($) marg /* 0#0 */}
+    };
+}
+macro_rules! add2
+    /*
+    0#0
+    */ {
+    ($ marg /* 0#1 */ : expr /* 0#2 */, $marg /* 0#2 */ : expr /*
+    0#2 */) => {}
+}
+
+
+/*
+Expansions:
+crate0::{{expn0}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: Root
+crate0::{{expn1}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: Macro(Bang, "make_macro")
+crate0::{{expn2}}: parent: crate0::{{expn1}}, call_site_ctxt: #1, def_site_ctxt: #0, kind: Macro(Bang, "make_macro")
+
+SyntaxContexts:
+#0: parent: #0, outer_mark: (crate0::{{expn0}}, Opaque)
+#1: parent: #0, outer_mark: (crate0::{{expn1}}, SemiOpaque)
+#2: parent: #0, outer_mark: (crate0::{{expn2}}, SemiOpaque)
+*/

tests/ui/hygiene/unpretty-debug-shadow.rs

@@ -0,0 +1,20 @@
+//@ check-pass
+//@ compile-flags: -Zunpretty=expanded,hygiene
+
+// Regression test for token hygiene annotations in -Zunpretty=expanded,hygiene
+// Previously, tokens in macro_rules! bodies were missing hygiene annotations,
+// making it impossible to see how a macro's reference to a shadowed variable
+// is distinguished from the shadowing binding.
+
+// Don't break whenever Symbol numbering changes
+//@ normalize-stdout: "\d+#" -> "0#"
+
+#![feature(no_core)]
+#![no_core]
+
+fn f() {
+    let x = 0;
+    macro_rules! use_x { () => { x }; }
+    let x = 1;
+    use_x!();
+}

tests/ui/hygiene/unpretty-debug-shadow.stdout

@@ -0,0 +1,30 @@
+//@ check-pass
+//@ compile-flags: -Zunpretty=expanded,hygiene
+
+// Regression test for token hygiene annotations in -Zunpretty=expanded,hygiene
+// Previously, tokens in macro_rules! bodies were missing hygiene annotations,
+// making it impossible to see how a macro's reference to a shadowed variable
+// is distinguished from the shadowing binding.
+
+// Don't break whenever Symbol numbering changes
+//@ normalize-stdout: "\d+#" -> "0#"
+
+#![feature /* 0#0 */(no_core /* 0#0 */)]
+#![no_core /* 0#0 */]
+
+fn f /* 0#0 */() {
+    let x /* 0#0 */ = 0;
+    macro_rules! use_x /* 0#0 */ { () => { x /* 0#0 */ }; }
+    let x /* 0#0 */ = 1;
+    x /* 0#1 */;
+}
+
+/*
+Expansions:
+crate0::{{expn0}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: Root
+crate0::{{expn1}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: Macro(Bang, "use_x")
+
+SyntaxContexts:
+#0: parent: #0, outer_mark: (crate0::{{expn0}}, Opaque)
+#1: parent: #0, outer_mark: (crate0::{{expn1}}, SemiOpaque)
+*/

tests/ui/hygiene/unpretty-debug.stdout

@@ -5,10 +5,16 @@
 //@ normalize-stdout: "\d+#" -> "0#"
 
 // minimal junk
-#![feature /* 0#0 */(no_core)]
+#![feature /* 0#0 */(no_core /* 0#0 */)]
 #![no_core /* 0#0 */]
 
-macro_rules! foo /* 0#0 */ { ($x: ident) => { y + $x } }
+macro_rules! foo
+    /*
+    0#0
+    */ {
+    ($x /* 0#0 */: ident /* 0#0 */) =>
+    { y /* 0#0 */ + $x /* 0#0 */ }
+}
 
 fn bar /* 0#0 */() {
     let x /* 0#0 */ = 1;

tests/ui/proc-macro/meta-macro-hygiene.stdout

@@ -1,7 +1,7 @@
 Def site: $DIR/auxiliary/make-macro.rs:7:9: 7:56 (#4)
 Input: TokenStream [Ident { ident: "$crate", span: $DIR/meta-macro-hygiene.rs:26:37: 26:43 (#3) }, Punct { ch: ':', spacing: Joint, span: $DIR/meta-macro-hygiene.rs:26:43: 26:44 (#3) }, Punct { ch: ':', spacing: Alone, span: $DIR/meta-macro-hygiene.rs:26:44: 26:45 (#3) }, Ident { ident: "dummy", span: $DIR/meta-macro-hygiene.rs:26:45: 26:50 (#3) }, Punct { ch: '!', spacing: Alone, span: $DIR/meta-macro-hygiene.rs:26:50: 26:51 (#3) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: $DIR/meta-macro-hygiene.rs:26:51: 26:53 (#3) }]
 Respanned: TokenStream [Ident { ident: "$crate", span: $DIR/auxiliary/make-macro.rs:7:9: 7:56 (#4) }, Punct { ch: ':', spacing: Joint, span: $DIR/auxiliary/make-macro.rs:7:9: 7:56 (#4) }, Punct { ch: ':', spacing: Alone, span: $DIR/auxiliary/make-macro.rs:7:9: 7:56 (#4) }, Ident { ident: "dummy", span: $DIR/auxiliary/make-macro.rs:7:9: 7:56 (#4) }, Punct { ch: '!', spacing: Alone, span: $DIR/auxiliary/make-macro.rs:7:9: 7:56 (#4) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: $DIR/auxiliary/make-macro.rs:7:9: 7:56 (#4) }]
-#![feature /* 0#0 */(prelude_import)]
+#![feature /* 0#0 */(prelude_import /* 0#0 */)]
 //@ aux-build:make-macro.rs
 //@ proc-macro: meta-macro.rs
 //@ edition:2018
@@ -30,7 +30,8 @@ macro_rules! produce_it
     */ {
     () =>
     {
-        meta_macro::print_def_site!($crate::dummy!());
+        meta_macro /* 0#0 */::print_def_site /* 0#0
+        */!($crate /* 0#0 */::dummy /* 0#0 */!());
         // `print_def_site!` will respan the `$crate` identifier
         // with `Span::def_site()`. This should cause it to resolve
         // relative to `meta_macro`, *not* `make_macro` (despite

tests/ui/proc-macro/nonterminal-token-hygiene.stdout

@@ -20,7 +20,7 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
         span: $DIR/nonterminal-token-hygiene.rs:23:27: 23:32 (#4),
     },
 ]
-#![feature /* 0#0 */(prelude_import)]
+#![feature /* 0#0 */(prelude_import /* 0#0 */)]
 #![no_std /* 0#0 */]
 // Make sure that marks from declarative macros are applied to tokens in nonterminal.
 
@@ -34,7 +34,7 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
 //@ proc-macro: test-macros.rs
 //@ edition: 2015
 
-#![feature /* 0#0 */(decl_macro)]
+#![feature /* 0#0 */(decl_macro /* 0#0 */)]
 #![no_std /* 0#0 */]
 extern crate core /* 0#2 */;
 #[prelude_import /* 0#1 */]
@@ -49,15 +49,22 @@ macro_rules! outer
     /*
     0#0
     */ {
-    ($item:item) =>
+    ($item /* 0#0 */:item /* 0#0 */) =>
     {
-        macro inner() { print_bang! { $item } } inner!();
+        macro /* 0#0 */ inner /* 0#0 */()
+        { print_bang /* 0#0 */! { $item /* 0#0 */ } } inner /* 0#0
+        */!();
 
     };
 }
 
 struct S /* 0#0 */;
-macro inner /* 0#3 */ { () => { print_bang! { struct S; } } }
+macro inner
+    /*
+    0#3
+    */ {
+    () => { print_bang /* 0#3 */! { struct /* 0#0 */ S /* 0#0 */; } }
+}
 
 struct S /* 0#5 */;
 // OK, not a duplicate definition of `S`