Mirrors/rust
1
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2026-06-01 14:10:03 +03:00
Code Issues Packages Projects Releases Wiki Activity

Rollup merge of #137902 - nnethercote:ast-lexer-TokenKind, r=compiler-errors

Browse Source 
Make `ast::TokenKind` more like `lexer::TokenKind`

This is step 2 of https://github.com/rust-lang/compiler-team/issues/831.

r? `@spastorino`
This commit is contained in:
Matthias Krüger
2025-03-03 20:47:13 +01:00
committed by GitHub
parent 70b9968d1e 53167c0b7f
commit decb0c9bad
24 changed files with 396 additions and 353 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/rustc_ast/src/token.rs
+170 -126
View File
@@ -2,7 +2,6 @@
use std::fmt;
use std::sync::Arc;
pub use BinOpToken::*;
pub use LitKind::*;
pub use Nonterminal::*;
pub use NtExprKind::*;
@@ -26,21 +25,6 @@ pub enum CommentKind {
Block,
}
#[derive(Clone, PartialEq, Encodable, Decodable, Hash, Debug, Copy)]
#[derive(HashStable_Generic)]
pub enum BinOpToken {
Plus,
Minus,
Star,
Slash,
Percent,
Caret,
And,
Or,
Shl,
Shr,
}
// This type must not implement `Hash` due to the unusual `PartialEq` impl below.
#[derive(Copy, Clone, Debug, Encodable, Decodable, HashStable_Generic)]
pub enum InvisibleOrigin {
@@ -376,11 +360,49 @@ pub enum TokenKind {
/// `||`
OrOr,
/// `!`
Not,
Bang,
/// `~`
Tilde,
BinOp(BinOpToken),
BinOpEq(BinOpToken),
// `+`
Plus,
// `-`
Minus,
// `*`
Star,
// `/`
Slash,
// `%`
Percent,
// `^`
Caret,
// `&`
And,
// `|`
Or,
// `<<`
Shl,
// `>>`
Shr,
// `+=`
PlusEq,
// `-=`
MinusEq,
// `*=`
StarEq,
// `/=`
SlashEq,
// `%=`
PercentEq,
// `^=`
CaretEq,
// `&=`
AndEq,
// `|=`
OrEq,
// `<<=`
ShlEq,
// `>>=`
ShrEq,
/* Structural symbols */
/// `@`
@@ -500,31 +522,31 @@ pub fn break_two_token_op(&self, n: u32) -> Option<(TokenKind, TokenKind)> {
Some(match (self, n) {
(Le, 1) => (Lt, Eq),
(EqEq, 1) => (Eq, Eq),
(Ne, 1) => (Not, Eq),
(Ne, 1) => (Bang, Eq),
(Ge, 1) => (Gt, Eq),
(AndAnd, 1) => (BinOp(And), BinOp(And)),
(OrOr, 1) => (BinOp(Or), BinOp(Or)),
(BinOp(Shl), 1) => (Lt, Lt),
(BinOp(Shr), 1) => (Gt, Gt),
(BinOpEq(Plus), 1) => (BinOp(Plus), Eq),
(BinOpEq(Minus), 1) => (BinOp(Minus), Eq),
(BinOpEq(Star), 1) => (BinOp(Star), Eq),
(BinOpEq(Slash), 1) => (BinOp(Slash), Eq),
(BinOpEq(Percent), 1) => (BinOp(Percent), Eq),
(BinOpEq(Caret), 1) => (BinOp(Caret), Eq),
(BinOpEq(And), 1) => (BinOp(And), Eq),
(BinOpEq(Or), 1) => (BinOp(Or), Eq),
(BinOpEq(Shl), 1) => (Lt, Le), // `<` + `<=`
(BinOpEq(Shl), 2) => (BinOp(Shl), Eq), // `<<` + `=`
(BinOpEq(Shr), 1) => (Gt, Ge), // `>` + `>=`
(BinOpEq(Shr), 2) => (BinOp(Shr), Eq), // `>>` + `=`
(AndAnd, 1) => (And, And),
(OrOr, 1) => (Or, Or),
(Shl, 1) => (Lt, Lt),
(Shr, 1) => (Gt, Gt),
(PlusEq, 1) => (Plus, Eq),
(MinusEq, 1) => (Minus, Eq),
(StarEq, 1) => (Star, Eq),
(SlashEq, 1) => (Slash, Eq),
(PercentEq, 1) => (Percent, Eq),
(CaretEq, 1) => (Caret, Eq),
(AndEq, 1) => (And, Eq),
(OrEq, 1) => (Or, Eq),
(ShlEq, 1) => (Lt, Le), // `<` + `<=`
(ShlEq, 2) => (Shl, Eq), // `<<` + `=`
(ShrEq, 1) => (Gt, Ge), // `>` + `>=`
(ShrEq, 2) => (Shr, Eq), // `>>` + `=`
(DotDot, 1) => (Dot, Dot),
(DotDotDot, 1) => (Dot, DotDot), // `.` + `..`
(DotDotDot, 2) => (DotDot, Dot), // `..` + `.`
(DotDotEq, 2) => (DotDot, Eq),
(PathSep, 1) => (Colon, Colon),
(RArrow, 1) => (BinOp(Minus), Gt),
(LArrow, 1) => (Lt, BinOp(Minus)),
(RArrow, 1) => (Minus, Gt),
(LArrow, 1) => (Lt, Minus),
(FatArrow, 1) => (Eq, Gt),
_ => return None,
})
@@ -543,7 +565,7 @@ pub fn similar_tokens(&self) -> &[TokenKind] {
}
pub fn should_end_const_arg(&self) -> bool {
matches!(self, Gt | Ge | BinOp(Shr) | BinOpEq(Shr))
matches!(self, Gt | Ge | Shr | ShrEq)
}
}
@@ -582,11 +604,11 @@ pub fn is_range_separator(&self) -> bool {
pub fn is_punct(&self) -> bool {
match self.kind {
Eq | Lt | Le | EqEq | Ne | Ge | Gt | AndAnd | OrOr | Not | Tilde | BinOp(_)
| BinOpEq(_) | At | Dot | DotDot | DotDotDot | DotDotEq | Comma | Semi | Colon
| PathSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question | SingleQuote => {
true
}
Eq | Lt | Le | EqEq | Ne | Ge | Gt | AndAnd | OrOr | Bang | Tilde | Plus | Minus
| Star | Slash | Percent | Caret | And | Or | Shl | Shr | PlusEq | MinusEq | StarEq
| SlashEq | PercentEq | CaretEq | AndEq | OrEq | ShlEq | ShrEq | At | Dot | DotDot
| DotDotDot | DotDotEq | Comma | Semi | Colon | PathSep | RArrow | LArrow
| FatArrow | Pound | Dollar | Question | SingleQuote => true,
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
| NtIdent(..) | Lifetime(..) | NtLifetime(..) | Interpolated(..) | Eof => false,
@@ -594,7 +616,7 @@ pub fn is_punct(&self) -> bool {
}
pub fn is_like_plus(&self) -> bool {
matches!(self.kind, BinOp(Plus) | BinOpEq(Plus))
matches!(self.kind, Plus | PlusEq)
}
/// Returns `true` if the token can appear at the start of an expression.
@@ -608,15 +630,15 @@ pub fn can_begin_expr(&self) -> bool {
ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
OpenDelim(Parenthesis | Brace | Bracket) | // tuple, array or block
Literal(..) | // literal
Not | // operator not
BinOp(Minus) | // unary minus
BinOp(Star) | // dereference
BinOp(Or) | OrOr | // closure
BinOp(And) | // reference
Bang | // operator not
Minus | // unary minus
Star | // dereference
Or | OrOr | // closure
And | // reference
AndAnd | // double reference
// DotDotDot is no longer supported, but we need some way to display the error
DotDot | DotDotDot | DotDotEq | // range notation
Lt | BinOp(Shl) | // associated path
Lt | Shl | // associated path
PathSep | // global path
Lifetime(..) | // labeled loop
Pound => true, // expression attributes
@@ -645,17 +667,16 @@ pub fn can_begin_pattern(&self, pat_kind: NtPatKind) -> bool {
Ident(..) | NtIdent(..) |
OpenDelim(Delimiter::Parenthesis) | // tuple pattern
OpenDelim(Delimiter::Bracket) | // slice pattern
BinOp(And) | // reference
BinOp(Minus) | // negative literal
AndAnd | // double reference
Literal(_) | // literal
DotDot | // range pattern (future compat)
DotDotDot | // range pattern (future compat)
PathSep | // path
Lt | // path (UFCS constant)
BinOp(Shl) => true, // path (double UFCS)
// leading vert `|` or-pattern
BinOp(Or) => matches!(pat_kind, PatWithOr),
And | // reference
Minus | // negative literal
AndAnd | // double reference
Literal(_) | // literal
DotDot | // range pattern (future compat)
DotDotDot | // range pattern (future compat)
PathSep | // path
Lt | // path (UFCS constant)
Shl => true, // path (double UFCS)
Or => matches!(pat_kind, PatWithOr), // leading vert `|` or-pattern
Interpolated(nt) =>
matches!(&**nt,
| NtExpr(..)
@@ -676,18 +697,18 @@ pub fn can_begin_pattern(&self, pat_kind: NtPatKind) -> bool {
/// Returns `true` if the token can appear at the start of a type.
pub fn can_begin_type(&self) -> bool {
match self.uninterpolate().kind {
Ident(name, is_raw) =>
Ident(name, is_raw) =>
ident_can_begin_type(name, self.span, is_raw), // type name or keyword
OpenDelim(Delimiter::Parenthesis) | // tuple
OpenDelim(Delimiter::Bracket) | // array
Not | // never
BinOp(Star) | // raw pointer
BinOp(And) | // reference
AndAnd | // double reference
Question | // maybe bound in trait object
Lifetime(..) | // lifetime bound in trait object
Lt | BinOp(Shl) | // associated path
PathSep => true, // global path
Bang | // never
Star | // raw pointer
And | // reference
AndAnd | // double reference
Question | // maybe bound in trait object
Lifetime(..) | // lifetime bound in trait object
Lt | Shl | // associated path
PathSep => true, // global path
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
MetaVarKind::Ty { .. } |
MetaVarKind::Path
@@ -701,7 +722,7 @@ pub fn can_begin_type(&self) -> bool {
/// Returns `true` if the token can appear at the start of a const param.
pub fn can_begin_const_arg(&self) -> bool {
match self.kind {
OpenDelim(Delimiter::Brace) | Literal(..) | BinOp(Minus) => true,
OpenDelim(Delimiter::Brace) | Literal(..) | Minus => true,
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
Interpolated(ref nt) => matches!(&**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
@@ -750,7 +771,7 @@ pub fn is_lit(&self) -> bool {
/// Keep this in sync with and `Lit::from_token`, excluding unary negation.
pub fn can_begin_literal_maybe_minus(&self) -> bool {
match self.uninterpolate().kind {
Literal(..) | BinOp(Minus) => true,
Literal(..) | Minus => true,
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
Interpolated(ref nt) => match &**nt {
NtLiteral(_) => true,
@@ -875,7 +896,7 @@ pub fn is_mutability(&self) -> bool {
}
pub fn is_qpath_start(&self) -> bool {
self == &Lt || self == &BinOp(Shl)
self == &Lt || self == &Shl
}
pub fn is_path_start(&self) -> bool {
@@ -967,59 +988,82 @@ pub fn is_metavar_seq(&self) -> Option<MetaVarKind> {
}
pub fn glue(&self, joint: &Token) -> Option<Token> {
let kind = match self.kind {
Eq => match joint.kind {
Eq => EqEq,
Gt => FatArrow,
_ => return None,
},
Lt => match joint.kind {
Eq => Le,
Lt => BinOp(Shl),
Le => BinOpEq(Shl),
BinOp(Minus) => LArrow,
_ => return None,
},
Gt => match joint.kind {
Eq => Ge,
Gt => BinOp(Shr),
Ge => BinOpEq(Shr),
_ => return None,
},
Not => match joint.kind {
Eq => Ne,
_ => return None,
},
BinOp(op) => match joint.kind {
Eq => BinOpEq(op),
BinOp(And) if op == And => AndAnd,
BinOp(Or) if op == Or => OrOr,
Gt if op == Minus => RArrow,
_ => return None,
},
Dot => match joint.kind {
Dot => DotDot,
DotDot => DotDotDot,
_ => return None,
},
DotDot => match joint.kind {
Dot => DotDotDot,
Eq => DotDotEq,
_ => return None,
},
Colon => match joint.kind {
Colon => PathSep,
_ => return None,
},
SingleQuote => match joint.kind {
Ident(name, is_raw) => Lifetime(Symbol::intern(&format!("'{name}")), is_raw),
_ => return None,
},
let kind = match (&self.kind, &joint.kind) {
(Eq, Eq) => EqEq,
(Eq, Gt) => FatArrow,
(Eq, _) => return None,
Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
| DotDotEq | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar
| Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | NtIdent(..)
| Lifetime(..) | NtLifetime(..) | Interpolated(..) | DocComment(..) | Eof => {
(Lt, Eq) => Le,
(Lt, Lt) => Shl,
(Lt, Le) => ShlEq,
(Lt, Minus) => LArrow,
(Lt, _) => return None,
(Gt, Eq) => Ge,
(Gt, Gt) => Shr,
(Gt, Ge) => ShrEq,
(Gt, _) => return None,
(Bang, Eq) => Ne,
(Bang, _) => return None,
(Plus, Eq) => PlusEq,
(Plus, _) => return None,
(Minus, Eq) => MinusEq,
(Minus, Gt) => RArrow,
(Minus, _) => return None,
(Star, Eq) => StarEq,
(Star, _) => return None,
(Slash, Eq) => SlashEq,
(Slash, _) => return None,
(Percent, Eq) => PercentEq,
(Percent, _) => return None,
(Caret, Eq) => CaretEq,
(Caret, _) => return None,
(And, Eq) => AndEq,
(And, And) => AndAnd,
(And, _) => return None,
(Or, Eq) => OrEq,
(Or, Or) => OrOr,
(Or, _) => return None,
(Shl, Eq) => ShlEq,
(Shl, _) => return None,
(Shr, Eq) => ShrEq,
(Shr, _) => return None,
(Dot, Dot) => DotDot,
(Dot, DotDot) => DotDotDot,
(Dot, _) => return None,
(DotDot, Dot) => DotDotDot,
(DotDot, Eq) => DotDotEq,
(DotDot, _) => return None,
(Colon, Colon) => PathSep,
(Colon, _) => return None,
(SingleQuote, Ident(name, is_raw)) => {
Lifetime(Symbol::intern(&format!("'{name}")), *is_raw)
}
(SingleQuote, _) => return None,
(
Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | PlusEq | MinusEq | StarEq | SlashEq
| PercentEq | CaretEq | AndEq | OrEq | ShlEq | ShrEq | At | DotDotDot | DotDotEq
| Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question
| OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | NtIdent(..)
| Lifetime(..) | NtLifetime(..) | Interpolated(..) | DocComment(..) | Eof,
_,
) => {
return None;
}
};
compiler/rustc_ast/src/tokenstream.rs
+1 -1
View File
@@ -651,7 +651,7 @@ fn desugared_tts(attr_style: AttrStyle, data: Symbol, span: Span) -> Vec<TokenTr
if attr_style == AttrStyle::Inner {
vec![
TokenTree::token_joint(token::Pound, span),
TokenTree::token_joint_hidden(token::Not, span),
TokenTree::token_joint_hidden(token::Bang, span),
body,
]
} else {
compiler/rustc_ast/src/util/parser.rs
+21 -21
View File
@@ -1,7 +1,7 @@
use rustc_span::kw;
use crate::ast::{self, BinOpKind, RangeLimits};
use crate::token::{self, BinOpToken, Token};
use crate::token::{self, Token};
/// Associative operator.
#[derive(Copy, Clone, PartialEq, Debug)]
@@ -34,26 +34,26 @@ pub fn from_token(t: &Token) -> Option<AssocOp> {
use AssocOp::*;
match t.kind {
token::Eq => Some(Assign),
token::BinOp(BinOpToken::Plus) => Some(Binary(BinOpKind::Add)),
token::BinOp(BinOpToken::Minus) => Some(Binary(BinOpKind::Sub)),
token::BinOp(BinOpToken::Star) => Some(Binary(BinOpKind::Mul)),
token::BinOp(BinOpToken::Slash) => Some(Binary(BinOpKind::Div)),
token::BinOp(BinOpToken::Percent) => Some(Binary(BinOpKind::Rem)),
token::BinOp(BinOpToken::Caret) => Some(Binary(BinOpKind::BitXor)),
token::BinOp(BinOpToken::And) => Some(Binary(BinOpKind::BitAnd)),
token::BinOp(BinOpToken::Or) => Some(Binary(BinOpKind::BitOr)),
token::BinOp(BinOpToken::Shl) => Some(Binary(BinOpKind::Shl)),
token::BinOp(BinOpToken::Shr) => Some(Binary(BinOpKind::Shr)),
token::BinOpEq(BinOpToken::Plus) => Some(AssignOp(BinOpKind::Add)),
token::BinOpEq(BinOpToken::Minus) => Some(AssignOp(BinOpKind::Sub)),
token::BinOpEq(BinOpToken::Star) => Some(AssignOp(BinOpKind::Mul)),
token::BinOpEq(BinOpToken::Slash) => Some(AssignOp(BinOpKind::Div)),
token::BinOpEq(BinOpToken::Percent) => Some(AssignOp(BinOpKind::Rem)),
token::BinOpEq(BinOpToken::Caret) => Some(AssignOp(BinOpKind::BitXor)),
token::BinOpEq(BinOpToken::And) => Some(AssignOp(BinOpKind::BitAnd)),
token::BinOpEq(BinOpToken::Or) => Some(AssignOp(BinOpKind::BitOr)),
token::BinOpEq(BinOpToken::Shl) => Some(AssignOp(BinOpKind::Shl)),
token::BinOpEq(BinOpToken::Shr) => Some(AssignOp(BinOpKind::Shr)),
token::Plus => Some(Binary(BinOpKind::Add)),
token::Minus => Some(Binary(BinOpKind::Sub)),
token::Star => Some(Binary(BinOpKind::Mul)),
token::Slash => Some(Binary(BinOpKind::Div)),
token::Percent => Some(Binary(BinOpKind::Rem)),
token::Caret => Some(Binary(BinOpKind::BitXor)),
token::And => Some(Binary(BinOpKind::BitAnd)),
token::Or => Some(Binary(BinOpKind::BitOr)),
token::Shl => Some(Binary(BinOpKind::Shl)),
token::Shr => Some(Binary(BinOpKind::Shr)),
token::PlusEq => Some(AssignOp(BinOpKind::Add)),
token::MinusEq => Some(AssignOp(BinOpKind::Sub)),
token::StarEq => Some(AssignOp(BinOpKind::Mul)),
token::SlashEq => Some(AssignOp(BinOpKind::Div)),
token::PercentEq => Some(AssignOp(BinOpKind::Rem)),
token::CaretEq => Some(AssignOp(BinOpKind::BitXor)),
token::AndEq => Some(AssignOp(BinOpKind::BitAnd)),
token::OrEq => Some(AssignOp(BinOpKind::BitOr)),
token::ShlEq => Some(AssignOp(BinOpKind::Shl)),
token::ShrEq => Some(AssignOp(BinOpKind::Shr)),
token::Lt => Some(Binary(BinOpKind::Lt)),
token::Le => Some(Binary(BinOpKind::Le)),
token::Ge => Some(Binary(BinOpKind::Ge)),
compiler/rustc_ast_pretty/src/pprust/state.rs
+23 -22
View File
@@ -11,9 +11,7 @@
use rustc_ast::attr::AttrIdGenerator;
use rustc_ast::ptr::P;
use rustc_ast::token::{
self, BinOpToken, CommentKind, Delimiter, IdentIsRaw, Nonterminal, Token, TokenKind,
};
use rustc_ast::token::{self, CommentKind, Delimiter, IdentIsRaw, Nonterminal, Token, TokenKind};
use rustc_ast::tokenstream::{Spacing, TokenStream, TokenTree};
use rustc_ast::util::classify;
use rustc_ast::util::comments::{Comment, CommentStyle};
@@ -319,7 +317,7 @@ fn is_punct(tt: &TokenTree) -> bool {
(tt1, Tok(Token { kind: Comma | Semi | Dot, .. }, _)) if !is_punct(tt1) => false,
// IDENT + `!`: `println!()`, but `if !x { ... }` needs a space after the `if`
(Tok(Token { kind: Ident(sym, is_raw), span }, _), Tok(Token { kind: Not, .. }, _))
(Tok(Token { kind: Ident(sym, is_raw), span }, _), Tok(Token { kind: Bang, .. }, _))
if !Ident::new(*sym, *span).is_reserved() || matches!(is_raw, IdentIsRaw::Yes) =>
{
false
@@ -344,21 +342,6 @@ fn is_punct(tt: &TokenTree) -> bool {
}
}
fn binop_to_string(op: BinOpToken) -> &'static str {
match op {
token::Plus => "+",
token::Minus => "-",
token::Star => "*",
token::Slash => "/",
token::Percent => "%",
token::Caret => "^",
token::And => "&",
token::Or => "|",
token::Shl => "<<",
token::Shr => ">>",
}
}
pub fn doc_comment_to_string(
comment_kind: CommentKind,
attr_style: ast::AttrStyle,
@@ -913,12 +896,30 @@ fn token_kind_to_string_ext(
token::Ne => "!=".into(),
token::Ge => ">=".into(),
token::Gt => ">".into(),
token::Not => "!".into(),
token::Bang => "!".into(),
token::Tilde => "~".into(),
token::OrOr => "||".into(),
token::AndAnd => "&&".into(),
token::BinOp(op) => binop_to_string(op).into(),
token::BinOpEq(op) => format!("{}=", binop_to_string(op)).into(),
token::Plus => "+".into(),
token::Minus => "-".into(),
token::Star => "*".into(),
token::Slash => "/".into(),
token::Percent => "%".into(),
token::Caret => "^".into(),
token::And => "&".into(),
token::Or => "|".into(),
token::Shl => "<<".into(),
token::Shr => ">>".into(),
token::PlusEq => "+=".into(),
token::MinusEq => "-=".into(),
token::StarEq => "*=".into(),
token::SlashEq => "/=".into(),
token::PercentEq => "%=".into(),
token::CaretEq => "^=".into(),
token::AndEq => "&=".into(),
token::OrEq => "|=".into(),
token::ShlEq => "<<=".into(),
token::ShrEq => ">>=".into(),
/* Structural symbols */
token::At => "@".into(),
compiler/rustc_expand/src/config.rs
+1 -1
View File
@@ -328,7 +328,7 @@ fn expand_cfg_attr_item(
// For inner attributes, we do the same thing for the `!` in `#![attr]`.
let mut trees = if cfg_attr.style == AttrStyle::Inner {
let Some(TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _)) =
let Some(TokenTree::Token(bang_token @ Token { kind: TokenKind::Bang, .. }, _)) =
orig_trees.next()
else {
panic!("Bad tokens for attribute {cfg_attr:?}");
compiler/rustc_expand/src/mbe/macro_check.rs
+1 -1
View File
@@ -432,7 +432,7 @@ fn check_nested_occurrences(
}
(
NestedMacroState::MacroRules,
&TokenTree::Token(Token { kind: TokenKind::Not, .. }),
&TokenTree::Token(Token { kind: TokenKind::Bang, .. }),
) => {
state = NestedMacroState::MacroRulesNot;
}
compiler/rustc_expand/src/mbe/macro_rules.rs
+6 -6
View File
@@ -690,7 +690,7 @@ fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {
&& let TokenKind::Ident(ident, _) = ident.kind
&& ident == sym::compile_error
&& let mbe::TokenTree::Token(bang) = bang
&& let TokenKind::Not = bang.kind
&& let TokenKind::Bang = bang.kind
&& let mbe::TokenTree::Delimited(.., del) = args
&& !del.delim.skip()
{
@@ -1135,7 +1135,7 @@ fn check_matcher_core<'tt>(
&& matches!(kind, NonterminalKind::Pat(PatParam { inferred: true }))
&& matches!(
next_token,
TokenTree::Token(token) if *token == BinOp(token::BinOpToken::Or)
TokenTree::Token(token) if *token == token::Or
)
{
// It is suggestion to use pat_param, for example: $x:pat -> $x:pat_param.
@@ -1177,7 +1177,7 @@ fn check_matcher_core<'tt>(
if kind == NonterminalKind::Pat(PatWithOr)
&& sess.psess.edition.at_least_rust_2021()
&& next_token.is_token(&BinOp(token::BinOpToken::Or))
&& next_token.is_token(&token::Or)
{
let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl(
span,
@@ -1296,7 +1296,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"];
match tok {
TokenTree::Token(token) => match token.kind {
FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes,
FatArrow | Comma | Eq | Or => IsInFollow::Yes,
Ident(name, IdentIsRaw::No) if name == kw::If || name == kw::In => {
IsInFollow::Yes
}
@@ -1332,9 +1332,9 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
| Colon
| Eq
| Gt
| BinOp(token::Shr)
| Shr
| Semi
| BinOp(token::Or) => IsInFollow::Yes,
| Or => IsInFollow::Yes,
Ident(name, IdentIsRaw::No) if name == kw::As || name == kw::Where => {
IsInFollow::Yes
}
compiler/rustc_expand/src/mbe/quoted.rs
+2 -2
View File
@@ -302,8 +302,8 @@ fn parse_tree<'a>(
/// `None`.
fn kleene_op(token: &Token) -> Option<KleeneOp> {
match token.kind {
token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore),
token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore),
token::Star => Some(KleeneOp::ZeroOrMore),
token::Plus => Some(KleeneOp::OneOrMore),
token::Question => Some(KleeneOp::ZeroOrOne),
_ => None,
}
compiler/rustc_expand/src/proc_macro_server.rs
+33 -38
View File
@@ -180,28 +180,28 @@ fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
Gt => op(">"),
AndAnd => op("&&"),
OrOr => op("||"),
Not => op("!"),
Bang => op("!"),
Tilde => op("~"),
BinOp(Plus) => op("+"),
BinOp(Minus) => op("-"),
BinOp(Star) => op("*"),
BinOp(Slash) => op("/"),
BinOp(Percent) => op("%"),
BinOp(Caret) => op("^"),
BinOp(And) => op("&"),
BinOp(Or) => op("|"),
BinOp(Shl) => op("<<"),
BinOp(Shr) => op(">>"),
BinOpEq(Plus) => op("+="),
BinOpEq(Minus) => op("-="),
BinOpEq(Star) => op("*="),
BinOpEq(Slash) => op("/="),
BinOpEq(Percent) => op("%="),
BinOpEq(Caret) => op("^="),
BinOpEq(And) => op("&="),
BinOpEq(Or) => op("|="),
BinOpEq(Shl) => op("<<="),
BinOpEq(Shr) => op(">>="),
Plus => op("+"),
Minus => op("-"),
Star => op("*"),
Slash => op("/"),
Percent => op("%"),
Caret => op("^"),
And => op("&"),
Or => op("|"),
Shl => op("<<"),
Shr => op(">>"),
PlusEq => op("+="),
MinusEq => op("-="),
StarEq => op("*="),
SlashEq => op("/="),
PercentEq => op("%="),
CaretEq => op("^="),
AndEq => op("&="),
OrEq => op("|="),
ShlEq => op("<<="),
ShrEq => op(">>="),
At => op("@"),
Dot => op("."),
DotDot => op(".."),
@@ -322,16 +322,16 @@ fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
b'=' => Eq,
b'<' => Lt,
b'>' => Gt,
b'!' => Not,
b'!' => Bang,
b'~' => Tilde,
b'+' => BinOp(Plus),
b'-' => BinOp(Minus),
b'*' => BinOp(Star),
b'/' => BinOp(Slash),
b'%' => BinOp(Percent),
b'^' => BinOp(Caret),
b'&' => BinOp(And),
b'|' => BinOp(Or),
b'+' => Plus,
b'-' => Minus,
b'*' => Star,
b'/' => Slash,
b'%' => Percent,
b'^' => Caret,
b'&' => And,
b'|' => Or,
b'@' => At,
b'.' => Dot,
b',' => Comma,
@@ -372,10 +372,9 @@ fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
suffix,
span,
}) if symbol.as_str().starts_with('-') => {
let minus = BinOp(BinOpToken::Minus);
let symbol = Symbol::intern(&symbol.as_str()[1..]);
let integer = TokenKind::lit(token::Integer, symbol, suffix);
let a = tokenstream::TokenTree::token_joint_hidden(minus, span);
let a = tokenstream::TokenTree::token_joint_hidden(Minus, span);
let b = tokenstream::TokenTree::token_alone(integer, span);
smallvec![a, b]
}
@@ -385,10 +384,9 @@ fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
suffix,
span,
}) if symbol.as_str().starts_with('-') => {
let minus = BinOp(BinOpToken::Minus);
let symbol = Symbol::intern(&symbol.as_str()[1..]);
let float = TokenKind::lit(token::Float, symbol, suffix);
let a = tokenstream::TokenTree::token_joint_hidden(minus, span);
let a = tokenstream::TokenTree::token_joint_hidden(Minus, span);
let b = tokenstream::TokenTree::token_alone(float, span);
smallvec![a, b]
}
@@ -599,10 +597,7 @@ fn expand_expr(&mut self, stream: &Self::TokenStream) -> Result<Self::TokenStrea
Ok(Self::TokenStream::from_iter([
// FIXME: The span of the `-` token is lost when
// parsing, so we cannot faithfully recover it here.
tokenstream::TokenTree::token_joint_hidden(
token::BinOp(token::Minus),
e.span,
),
tokenstream::TokenTree::token_joint_hidden(token::Minus, e.span),
tokenstream::TokenTree::token_alone(token::Literal(*token_lit), e.span),
]))
}
compiler/rustc_parse/src/lexer/mod.rs
+9 -9
View File
@@ -384,17 +384,17 @@ fn next_token_from_cursor(&mut self) -> (Token, bool) {
rustc_lexer::TokenKind::Colon => token::Colon,
rustc_lexer::TokenKind::Dollar => token::Dollar,
rustc_lexer::TokenKind::Eq => token::Eq,
rustc_lexer::TokenKind::Bang => token::Not,
rustc_lexer::TokenKind::Bang => token::Bang,
rustc_lexer::TokenKind::Lt => token::Lt,
rustc_lexer::TokenKind::Gt => token::Gt,
rustc_lexer::TokenKind::Minus => token::BinOp(token::Minus),
rustc_lexer::TokenKind::And => token::BinOp(token::And),
rustc_lexer::TokenKind::Or => token::BinOp(token::Or),
rustc_lexer::TokenKind::Plus => token::BinOp(token::Plus),
rustc_lexer::TokenKind::Star => token::BinOp(token::Star),
rustc_lexer::TokenKind::Slash => token::BinOp(token::Slash),
rustc_lexer::TokenKind::Caret => token::BinOp(token::Caret),
rustc_lexer::TokenKind::Percent => token::BinOp(token::Percent),
rustc_lexer::TokenKind::Minus => token::Minus,
rustc_lexer::TokenKind::And => token::And,
rustc_lexer::TokenKind::Or => token::Or,
rustc_lexer::TokenKind::Plus => token::Plus,
rustc_lexer::TokenKind::Star => token::Star,
rustc_lexer::TokenKind::Slash => token::Slash,
rustc_lexer::TokenKind::Caret => token::Caret,
rustc_lexer::TokenKind::Percent => token::Percent,
rustc_lexer::TokenKind::Unknown | rustc_lexer::TokenKind::InvalidIdent => {
// Don't emit diagnostics for sequences of the same invalid token
compiler/rustc_parse/src/lexer/unicode_chars.rs
+6 -6
View File
@@ -308,11 +308,11 @@
const ASCII_ARRAY: &[(&str, &str, Option<token::TokenKind>)] = &[
(" ", "Space", None),
("_", "Underscore", Some(token::Ident(kw::Underscore, token::IdentIsRaw::No))),
("-", "Minus/Hyphen", Some(token::BinOp(token::Minus))),
("-", "Minus/Hyphen", Some(token::Minus)),
(",", "Comma", Some(token::Comma)),
(";", "Semicolon", Some(token::Semi)),
(":", "Colon", Some(token::Colon)),
("!", "Exclamation Mark", Some(token::Not)),
("!", "Exclamation Mark", Some(token::Bang)),
("?", "Question Mark", Some(token::Question)),
(".", "Period", Some(token::Dot)),
("(", "Left Parenthesis", Some(token::OpenDelim(Delimiter::Parenthesis))),
@@ -321,11 +321,11 @@
("]", "Right Square Bracket", Some(token::CloseDelim(Delimiter::Bracket))),
("{", "Left Curly Brace", Some(token::OpenDelim(Delimiter::Brace))),
("}", "Right Curly Brace", Some(token::CloseDelim(Delimiter::Brace))),
("*", "Asterisk", Some(token::BinOp(token::Star))),
("/", "Slash", Some(token::BinOp(token::Slash))),
("*", "Asterisk", Some(token::Star)),
("/", "Slash", Some(token::Slash)),
("\\", "Backslash", None),
("&", "Ampersand", Some(token::BinOp(token::And))),
("+", "Plus Sign", Some(token::BinOp(token::Plus))),
("&", "Ampersand", Some(token::And)),
("+", "Plus Sign", Some(token::Plus)),
("<", "Less-Than Sign", Some(token::Lt)),
("=", "Equals Sign", Some(token::Eq)),
("==", "Double Equals Sign", Some(token::EqEq)),
compiler/rustc_parse/src/parser/attr.rs
+2 -2
View File
@@ -130,7 +130,7 @@ pub fn parse_attribute(
assert!(this.eat(exp!(Pound)), "parse_attribute called in non-attribute position");
let style =
if this.eat(exp!(Not)) { ast::AttrStyle::Inner } else { ast::AttrStyle::Outer };
if this.eat(exp!(Bang)) { ast::AttrStyle::Inner } else { ast::AttrStyle::Outer };
this.expect(exp!(OpenBracket))?;
let item = this.parse_attr_item(ForceCollect::No)?;
@@ -312,7 +312,7 @@ pub fn parse_inner_attributes(&mut self) -> PResult<'a, ast::AttrVec> {
loop {
let start_pos = self.num_bump_calls;
// Only try to parse if it is an inner attribute (has `!`).
let attr = if self.check(exp!(Pound)) && self.look_ahead(1, |t| t == &token::Not) {
let attr = if self.check(exp!(Pound)) && self.look_ahead(1, |t| t == &token::Bang) {
Some(self.parse_attribute(InnerAttrPolicy::Permitted)?)
} else if let token::DocComment(comment_kind, attr_style, data) = self.token.kind {
if attr_style == ast::AttrStyle::Inner {
compiler/rustc_parse/src/parser/diagnostics.rs
+10 -15
View File
@@ -1167,7 +1167,7 @@ pub(super) fn check_trailing_angle_brackets(
let mut number_of_gt = 0;
while self.look_ahead(position, |t| {
trace!("check_trailing_angle_brackets: t={:?}", t);
if *t == token::BinOp(token::BinOpToken::Shr) {
if *t == token::Shr {
number_of_shr += 1;
true
} else if *t == token::Gt {
@@ -1222,7 +1222,7 @@ pub(super) fn check_turbofish_missing_angle_brackets(&mut self, segment: &mut Pa
let span = lo.to(self.prev_token.span);
// Detect trailing `>` like in `x.collect::Vec<_>>()`.
let mut trailing_span = self.prev_token.span.shrink_to_hi();
while self.token == token::BinOp(token::Shr) || self.token == token::Gt {
while self.token == token::Shr || self.token == token::Gt {
trailing_span = trailing_span.to(self.token.span);
self.bump();
}
@@ -1468,8 +1468,7 @@ pub(super) fn check_no_chained_comparison(
let snapshot = self.create_snapshot_for_diagnostic();
self.bump();
// So far we have parsed `foo<bar<`, consume the rest of the type args.
let modifiers =
[(token::Lt, 1), (token::Gt, -1), (token::BinOp(token::Shr), -2)];
let modifiers = [(token::Lt, 1), (token::Gt, -1), (token::Shr, -2)];
self.consume_tts(1, &modifiers);
if !&[token::OpenDelim(Delimiter::Parenthesis), token::PathSep]
@@ -1962,7 +1961,7 @@ pub(super) fn recover_incorrect_await_syntax(
&mut self,
await_sp: Span,
) -> PResult<'a, P<Expr>> {
let (hi, expr, is_question) = if self.token == token::Not {
let (hi, expr, is_question) = if self.token == token::Bang {
// Handle `await!(<expr>)`.
self.recover_await_macro()?
} else {
@@ -1974,7 +1973,7 @@ pub(super) fn recover_incorrect_await_syntax(
}
fn recover_await_macro(&mut self) -> PResult<'a, (Span, P<Expr>, bool)> {
self.expect(exp!(Not))?;
self.expect(exp!(Bang))?;
self.expect(exp!(OpenParen))?;
let expr = self.parse_expr()?;
self.expect(exp!(CloseParen))?;
@@ -2034,7 +2033,7 @@ pub(super) fn recover_from_await_method_call(&mut self) {
pub(super) fn try_macro_suggestion(&mut self) -> PResult<'a, P<Expr>> {
let is_try = self.token.is_keyword(kw::Try);
let is_questionmark = self.look_ahead(1, |t| t == &token::Not); //check for !
let is_questionmark = self.look_ahead(1, |t| t == &token::Bang); //check for !
let is_open = self.look_ahead(2, |t| t == &token::OpenDelim(Delimiter::Parenthesis)); //check for (
if is_try && is_questionmark && is_open {
@@ -2613,8 +2612,7 @@ pub(super) fn recover_const_arg(
|| self.token == TokenKind::Dot;
// This will be true when a trait object type `Foo +` or a path which was a `const fn` with
// type params has been parsed.
let was_op =
matches!(self.prev_token.kind, token::BinOp(token::Plus | token::Shr) | token::Gt);
let was_op = matches!(self.prev_token.kind, token::Plus | token::Shr | token::Gt);
if !is_op_or_dot && !was_op {
// We perform these checks and early return to avoid taking a snapshot unnecessarily.
return Err(err);
@@ -2992,8 +2990,7 @@ fn conflict_marker(&mut self, long_kind: &TokenKind, short_kind: &TokenKind) ->
pub(super) fn recover_vcs_conflict_marker(&mut self) {
// <<<<<<<
let Some(start) = self.conflict_marker(&TokenKind::BinOp(token::Shl), &TokenKind::Lt)
else {
let Some(start) = self.conflict_marker(&TokenKind::Shl, &TokenKind::Lt) else {
return;
};
let mut spans = Vec::with_capacity(3);
@@ -3008,15 +3005,13 @@ pub(super) fn recover_vcs_conflict_marker(&mut self) {
if self.token == TokenKind::Eof {
break;
}
if let Some(span) = self.conflict_marker(&TokenKind::OrOr, &TokenKind::BinOp(token::Or))
{
if let Some(span) = self.conflict_marker(&TokenKind::OrOr, &TokenKind::Or) {
middlediff3 = Some(span);
}
if let Some(span) = self.conflict_marker(&TokenKind::EqEq, &TokenKind::Eq) {
middle = Some(span);
}
if let Some(span) = self.conflict_marker(&TokenKind::BinOp(token::Shr), &TokenKind::Gt)
{
if let Some(span) = self.conflict_marker(&TokenKind::Shr, &TokenKind::Gt) {
spans.push(span);
end = Some(span);
break;
compiler/rustc_parse/src/parser/expr.rs
+18 -22
View File
@@ -239,8 +239,8 @@ pub(super) fn parse_expr_assoc_rest_with(
self.bump();
}
if self.prev_token == token::BinOp(token::Plus)
&& self.token == token::BinOp(token::Plus)
if self.prev_token == token::Plus
&& self.token == token::Plus
&& self.prev_token.span.between(self.token.span).is_empty()
{
let op_span = self.prev_token.span.to(self.token.span);
@@ -250,8 +250,8 @@ pub(super) fn parse_expr_assoc_rest_with(
continue;
}
if self.prev_token == token::BinOp(token::Minus)
&& self.token == token::BinOp(token::Minus)
if self.prev_token == token::Minus
&& self.token == token::Minus
&& self.prev_token.span.between(self.token.span).is_empty()
&& !self.look_ahead(1, |tok| tok.can_begin_expr())
{
@@ -505,23 +505,23 @@ macro_rules! make_it {
// Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr()
match this.token.uninterpolate().kind {
// `!expr`
token::Not => make_it!(this, attrs, |this, _| this.parse_expr_unary(lo, UnOp::Not)),
token::Bang => make_it!(this, attrs, |this, _| this.parse_expr_unary(lo, UnOp::Not)),
// `~expr`
token::Tilde => make_it!(this, attrs, |this, _| this.recover_tilde_expr(lo)),
// `-expr`
token::BinOp(token::Minus) => {
token::Minus => {
make_it!(this, attrs, |this, _| this.parse_expr_unary(lo, UnOp::Neg))
}
// `*expr`
token::BinOp(token::Star) => {
token::Star => {
make_it!(this, attrs, |this, _| this.parse_expr_unary(lo, UnOp::Deref))
}
// `&expr` and `&&expr`
token::BinOp(token::And) | token::AndAnd => {
token::And | token::AndAnd => {
make_it!(this, attrs, |this, _| this.parse_expr_borrow(lo))
}
// `+lit`
token::BinOp(token::Plus) if this.look_ahead(1, |tok| tok.is_numeric_lit()) => {
token::Plus if this.look_ahead(1, |tok| tok.is_numeric_lit()) => {
let mut err = errors::LeadingPlusNotSupported {
span: lo,
remove_plus: None,
@@ -541,9 +541,7 @@ macro_rules! make_it {
this.parse_expr_prefix(attrs)
}
// Recover from `++x`:
token::BinOp(token::Plus)
if this.look_ahead(1, |t| *t == token::BinOp(token::Plus)) =>
{
token::Plus if this.look_ahead(1, |t| *t == token::Plus) => {
let starts_stmt = this.prev_token == token::Semi
|| this.prev_token == token::CloseDelim(Delimiter::Brace);
let pre_span = this.token.span.to(this.look_ahead(1, |t| t.span));
@@ -723,14 +721,12 @@ fn parse_assoc_op_cast(
suggestion,
})
}
token::BinOp(token::Shl) => {
self.dcx().emit_err(errors::ShiftInterpretedAsGeneric {
shift: self.token.span,
r#type: path,
args: args_span,
suggestion,
})
}
token::Shl => self.dcx().emit_err(errors::ShiftInterpretedAsGeneric {
shift: self.token.span,
r#type: path,
args: args_span,
suggestion,
}),
_ => {
// We can end up here even without `<` being the next token, for
// example because `parse_ty_no_plus` returns `Err` on keywords,
@@ -1574,7 +1570,7 @@ fn parse_expr_path_start(&mut self) -> PResult<'a, P<Expr>> {
};
// `!`, as an operator, is prefix, so we know this isn't that.
let (span, kind) = if self.eat(exp!(Not)) {
let (span, kind) = if self.eat(exp!(Bang)) {
// MACRO INVOCATION expression
if qself.is_some() {
self.dcx().emit_err(errors::MacroInvocationWithQualifiedPath(path.span));
@@ -2595,7 +2591,7 @@ fn parse_expr_let(&mut self, restrictions: Restrictions) -> PResult<'a, P<Expr>>
missing_let: None,
comparison: None,
};
if self.prev_token == token::BinOp(token::Or) {
if self.prev_token == token::Or {
// This was part of a closure, the that part of the parser recover.
return Err(self.dcx().create_err(err));
} else {
compiler/rustc_parse/src/parser/item.rs
+14 -14
View File
@@ -382,7 +382,7 @@ fn is_reuse_path_item(&mut self) -> bool {
/// Are we sure this could not possibly be a macro invocation?
fn isnt_macro_invocation(&mut self) -> bool {
self.check_ident() && self.look_ahead(1, |t| *t != token::Not && *t != token::PathSep)
self.check_ident() && self.look_ahead(1, |t| *t != token::Bang && *t != token::PathSep)
}
/// Recover on encountering a struct, enum, or method definition where the user
@@ -480,7 +480,7 @@ fn parse_item_builtin(&mut self) -> PResult<'a, Option<ItemInfo>> {
/// Parses an item macro, e.g., `item!();`.
fn parse_item_macro(&mut self, vis: &Visibility) -> PResult<'a, MacCall> {
let path = self.parse_path(PathStyle::Mod)?; // `foo::bar`
self.expect(exp!(Not))?; // `!`
self.expect(exp!(Bang))?; // `!`
match self.parse_delim_args() {
// `( .. )` or `[ .. ]` (followed by `;`), or `{ .. }`.
Ok(args) => {
@@ -540,7 +540,7 @@ fn is_async_fn(&self) -> bool {
fn parse_polarity(&mut self) -> ast::ImplPolarity {
// Disambiguate `impl !Trait for Type { ... }` and `impl ! { ... }` for the never type.
if self.check(exp!(Not)) && self.look_ahead(1, |t| t.can_begin_type()) {
if self.check(exp!(Bang)) && self.look_ahead(1, |t| t.can_begin_type()) {
self.bump(); // `!`
ast::ImplPolarity::Negative(self.prev_token.span)
} else {
@@ -1293,7 +1293,7 @@ fn is_static_global(&mut self) -> bool {
if token.is_keyword(kw::Move) {
return true;
}
matches!(token.kind, token::BinOp(token::Or) | token::OrOr)
matches!(token.kind, token::Or | token::OrOr)
})
} else {
// `$qual static`
@@ -1579,7 +1579,7 @@ fn parse_enum_variant(&mut self, span: Span) -> PResult<'a, Option<Variant>> {
}
let ident = this.parse_field_ident("enum", vlo)?;
if this.token == token::Not {
if this.token == token::Bang {
if let Err(err) = this.unexpected() {
err.with_note(fluent::parse_macro_expands_to_enum_variant).emit();
}
@@ -1814,7 +1814,7 @@ pub(super) fn parse_tuple_struct_body(&mut self) -> PResult<'a, ThinVec<FieldDef
let attrs = p.parse_outer_attributes()?;
p.collect_tokens(None, attrs, ForceCollect::No, |p, attrs| {
let mut snapshot = None;
if p.is_vcs_conflict_marker(&TokenKind::BinOp(token::Shl), &TokenKind::Lt) {
if p.is_vcs_conflict_marker(&TokenKind::Shl, &TokenKind::Lt) {
// Account for `<<<<<<<` diff markers. We can't proactively error here because
// that can be a valid type start, so we snapshot and reparse only we've
// encountered another parse error.
@@ -2034,7 +2034,7 @@ fn parse_name_and_ty(
attrs: AttrVec,
) -> PResult<'a, FieldDef> {
let name = self.parse_field_ident(adt_ty, lo)?;
if self.token == token::Not {
if self.token == token::Bang {
if let Err(mut err) = self.unexpected() {
// Encounter the macro invocation
err.subdiagnostic(MacroExpandsToAdtField { adt_ty });
@@ -2184,7 +2184,7 @@ fn is_macro_rules_item(&mut self) -> IsMacroRulesItem {
if self.check_keyword(exp!(MacroRules)) {
let macro_rules_span = self.token.span;
if self.look_ahead(1, |t| *t == token::Not) && self.look_ahead(2, |t| t.is_ident()) {
if self.look_ahead(1, |t| *t == token::Bang) && self.look_ahead(2, |t| t.is_ident()) {
return IsMacroRulesItem::Yes { has_bang: true };
} else if self.look_ahead(1, |t| (t.is_ident())) {
// macro_rules foo
@@ -2209,11 +2209,11 @@ fn parse_item_macro_rules(
self.expect_keyword(exp!(MacroRules))?; // `macro_rules`
if has_bang {
self.expect(exp!(Not))?; // `!`
self.expect(exp!(Bang))?; // `!`
}
let ident = self.parse_ident()?;
if self.eat(exp!(Not)) {
if self.eat(exp!(Bang)) {
// Handle macro_rules! foo!
let span = self.prev_token.span;
self.dcx().emit_err(errors::MacroNameRemoveBang { span });
@@ -3011,7 +3011,7 @@ fn parse_self_param(&mut self) -> PResult<'a, Option<Param>> {
// else is parsed as a normal function parameter list, so some lookahead is required.
let eself_lo = self.token.span;
let (eself, eself_ident, eself_hi) = match self.token.uninterpolate().kind {
token::BinOp(token::And) => {
token::And => {
let eself = if is_isolated_self(self, 1) {
// `&self`
self.bump();
@@ -3041,12 +3041,12 @@ fn parse_self_param(&mut self) -> PResult<'a, Option<Param>> {
(eself, self_ident, hi)
}
// `*self`
token::BinOp(token::Star) if is_isolated_self(self, 1) => {
token::Star if is_isolated_self(self, 1) => {
self.bump();
recover_self_ptr(self)?
}
// `*mut self` and `*const self`
token::BinOp(token::Star)
token::Star
if self.look_ahead(1, |t| t.is_mutability()) && is_isolated_self(self, 2) =>
{
self.bump();
@@ -3077,7 +3077,7 @@ fn is_named_param(&self) -> bool {
}
_ => 0,
},
token::BinOp(token::And) | token::AndAnd => 1,
token::And | token::AndAnd => 1,
_ if self.token.is_keyword(kw::Mut) => 1,
_ => 0,
};
compiler/rustc_parse/src/parser/mod.rs
+4 -4
View File
@@ -813,9 +813,9 @@ fn check_const_closure(&self) -> bool {
self.is_keyword_ahead(0, &[kw::Const])
&& self.look_ahead(1, |t| match &t.kind {
// async closures do not work with const closures, so we do not parse that here.
token::Ident(kw::Move | kw::Static, IdentIsRaw::No)
| token::OrOr
| token::BinOp(token::Or) => true,
token::Ident(kw::Move | kw::Static, IdentIsRaw::No) | token::OrOr | token::Or => {
true
}
_ => false,
})
}
@@ -1651,7 +1651,7 @@ fn check_path_sep_and_look_ahead(&mut self, looker: impl Fn(&Token) -> bool) ->
/// `::{` or `::*`
fn is_import_coupler(&mut self) -> bool {
self.check_path_sep_and_look_ahead(|t| {
matches!(t.kind, token::OpenDelim(Delimiter::Brace) | token::BinOp(token::Star))
matches!(t.kind, token::OpenDelim(Delimiter::Brace) | token::Star)
})
}
compiler/rustc_parse/src/parser/pat.rs
+11 -7
View File
@@ -3,7 +3,7 @@
use rustc_ast::mut_visit::{self, MutVisitor};
use rustc_ast::ptr::P;
use rustc_ast::token::NtPatKind::*;
use rustc_ast::token::{self, BinOpToken, Delimiter, IdentIsRaw, MetaVarKind, Token};
use rustc_ast::token::{self, Delimiter, IdentIsRaw, MetaVarKind, Token};
use rustc_ast::util::parser::ExprPrecedence;
use rustc_ast::visit::{self, Visitor};
use rustc_ast::{
@@ -358,7 +358,7 @@ fn recover_trailing_vert(&mut self, lo: Option<Span>) -> bool {
)
});
match (is_end_ahead, &self.token.kind) {
(true, token::BinOp(token::Or) | token::OrOr) => {
(true, token::Or | token::OrOr) => {
// A `|` or possibly `||` token shouldn't be here. Ban it.
self.dcx().emit_err(TrailingVertNotAllowed {
span: self.token.span,
@@ -432,7 +432,11 @@ fn maybe_recover_trailing_expr(
// `[` is included for indexing operations,
// `[]` is excluded as `a[]` isn't an expression and should be recovered as `a, []` (cf. `tests/ui/parser/pat-lt-bracket-7.rs`),
// `as` is included for type casts
let has_trailing_operator = matches!(self.token.kind, token::BinOp(op) if op != BinOpToken::Or)
let has_trailing_operator = matches!(
self.token.kind,
token::Plus | token::Minus | token::Star | token::Slash | token::Percent
| token::Caret | token::And | token::Shl | token::Shr // excludes `Or`
)
|| self.token == token::Question
|| (self.token == token::OpenDelim(Delimiter::Bracket)
&& self.look_ahead(1, |t| *t != token::CloseDelim(Delimiter::Bracket))) // excludes `[]`
@@ -763,7 +767,7 @@ fn parse_pat_with_range_pat(
self.recover_dotdotdot_rest_pat(lo)
} else if let Some(form) = self.parse_range_end() {
self.parse_pat_range_to(form)? // `..=X`, `...X`, or `..X`.
} else if self.eat(exp!(Not)) {
} else if self.eat(exp!(Bang)) {
// Parse `!`
self.psess.gated_spans.gate(sym::never_patterns, self.prev_token.span);
PatKind::Never
@@ -819,7 +823,7 @@ fn parse_pat_with_range_pat(
};
let span = lo.to(self.prev_token.span);
if qself.is_none() && self.check(exp!(Not)) {
if qself.is_none() && self.check(exp!(Bang)) {
self.parse_pat_mac_invoc(path)?
} else if let Some(form) = self.parse_range_end() {
let begin = self.mk_expr(span, ExprKind::Path(qself, path));
@@ -1255,7 +1259,7 @@ fn is_pat_range_end_start(&self, dist: usize) -> bool {
|| self.look_ahead(dist, |t| {
t.is_path_start() // e.g. `MY_CONST`;
|| *t == token::Dot // e.g. `.5` for recovery;
|| matches!(t.kind, token::Literal(..) | token::BinOp(token::Minus))
|| matches!(t.kind, token::Literal(..) | token::Minus)
|| t.is_bool_lit()
|| t.is_whole_expr()
|| t.is_lifetime() // recover `'a` instead of `'a'`
@@ -1331,7 +1335,7 @@ fn can_be_ident_pat(&mut self) -> bool {
| token::OpenDelim(Delimiter::Brace) // A struct pattern.
| token::DotDotDot | token::DotDotEq | token::DotDot // A range pattern.
| token::PathSep // A tuple / struct variant pattern.
| token::Not)) // A macro expanding to a pattern.
| token::Bang)) // A macro expanding to a pattern.
}
/// Parses `ident` or `ident @ pat`.
compiler/rustc_parse/src/parser/path.rs
+1 -4
View File
@@ -305,10 +305,7 @@ pub(super) fn parse_path_segment(
let is_args_start = |token: &Token| {
matches!(
token.kind,
token::Lt
| token::BinOp(token::Shl)
| token::OpenDelim(Delimiter::Parenthesis)
| token::LArrow
token::Lt | token::Shl | token::OpenDelim(Delimiter::Parenthesis) | token::LArrow
)
};
let check_args_start = |this: &mut Self| {
compiler/rustc_parse/src/parser/stmt.rs
+12 -3
View File
@@ -176,7 +176,7 @@ fn parse_stmt_path_start(&mut self, lo: Span, attrs: AttrWrapper) -> PResult<'a,
let stmt = self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| {
let path = this.parse_path(PathStyle::Expr)?;
if this.eat(exp!(Not)) {
if this.eat(exp!(Bang)) {
let stmt_mac = this.parse_stmt_mac(lo, attrs, path)?;
return Ok((
stmt_mac,
@@ -442,7 +442,16 @@ fn check_let_else_init_trailing_brace(&self, init: &ast::Expr) {
/// Parses the RHS of a local variable declaration (e.g., `= 14;`).
fn parse_initializer(&mut self, eq_optional: bool) -> PResult<'a, Option<P<Expr>>> {
let eq_consumed = match self.token.kind {
token::BinOpEq(..) => {
token::PlusEq
| token::MinusEq
| token::StarEq
| token::SlashEq
| token::PercentEq
| token::CaretEq
| token::AndEq
| token::OrEq
| token::ShlEq
| token::ShrEq => {
// Recover `let x <op>= 1` as `let x = 1` We must not use `+ BytePos(1)` here
// because `<op>` can be a multi-byte lookalike that was recovered, e.g. `➖=` (the
// `➖` is a U+2796 Heavy Minus Sign Unicode Character) that was recovered as a
@@ -688,7 +697,7 @@ pub(crate) fn parse_block_tail(
if self.token == token::Eof {
break;
}
if self.is_vcs_conflict_marker(&TokenKind::BinOp(token::Shl), &TokenKind::Lt) {
if self.is_vcs_conflict_marker(&TokenKind::Shl, &TokenKind::Lt) {
// Account for `<<<<<<<` diff markers. We can't proactively error here because
// that can be a valid path start, so we snapshot and reparse only we've
// encountered another parse error.
compiler/rustc_parse/src/parser/tests.rs
+1 -1
View File
@@ -2291,7 +2291,7 @@ fn string_to_tts_macro() {
Token { kind: token::Ident(name_macro_rules, IdentIsRaw::No), .. },
_,
),
TokenTree::Token(Token { kind: token::Not, .. }, _),
TokenTree::Token(Token { kind: token::Bang, .. }, _),
TokenTree::Token(Token { kind: token::Ident(name_zip, IdentIsRaw::No), .. }, _),
TokenTree::Delimited(.., macro_delim, macro_tts),
] if name_macro_rules == &kw::MacroRules && name_zip.as_str() == "zip" => {
compiler/rustc_parse/src/parser/token_type.rs
+9 -16
View File
@@ -25,7 +25,7 @@ pub enum TokenType {
Gt,
AndAnd,
OrOr,
Not,
Bang,
Tilde,
// BinOps
@@ -172,7 +172,7 @@ fn from_u32(val: u32) -> TokenType {
Gt,
AndAnd,
OrOr,
Not,
Bang,
Tilde,
Plus,
@@ -366,7 +366,7 @@ pub(super) fn to_string(&self) -> String {
TokenType::Gt => "`>`",
TokenType::AndAnd => "`&&`",
TokenType::OrOr => "`||`",
TokenType::Not => "`!`",
TokenType::Bang => "`!`",
TokenType::Tilde => "`~`",
TokenType::Plus => "`+`",
@@ -445,12 +445,6 @@ macro_rules! exp {
token_type: $crate::parser::token_type::TokenType::$tok
}
};
(@binop, $op:ident) => {
$crate::parser::token_type::ExpTokenPair {
tok: &rustc_ast::token::BinOp(rustc_ast::token::BinOpToken::$op),
token_type: $crate::parser::token_type::TokenType::$op,
}
};
(@open, $delim:ident, $token_type:ident) => {
$crate::parser::token_type::ExpTokenPair {
tok: &rustc_ast::token::OpenDelim(rustc_ast::token::Delimiter::$delim),
@@ -485,8 +479,13 @@ macro_rules! exp {
(Gt) => { exp!(@tok, Gt) };
(AndAnd) => { exp!(@tok, AndAnd) };
(OrOr) => { exp!(@tok, OrOr) };
(Not) => { exp!(@tok, Not) };
(Bang) => { exp!(@tok, Bang) };
(Tilde) => { exp!(@tok, Tilde) };
(Plus) => { exp!(@tok, Plus) };
(Minus) => { exp!(@tok, Minus) };
(Star) => { exp!(@tok, Star) };
(And) => { exp!(@tok, And) };
(Or) => { exp!(@tok, Or) };
(At) => { exp!(@tok, At) };
(Dot) => { exp!(@tok, Dot) };
(DotDot) => { exp!(@tok, DotDot) };
@@ -502,12 +501,6 @@ macro_rules! exp {
(Question) => { exp!(@tok, Question) };
(Eof) => { exp!(@tok, Eof) };
(Plus) => { exp!(@binop, Plus) };
(Minus) => { exp!(@binop, Minus) };
(Star) => { exp!(@binop, Star) };
(And) => { exp!(@binop, And) };
(Or) => { exp!(@binop, Or) };
(OpenParen) => { exp!(@open, Parenthesis, OpenParen) };
(OpenBrace) => { exp!(@open, Brace, OpenBrace) };
(OpenBracket) => { exp!(@open, Bracket, OpenBracket) };
compiler/rustc_parse/src/parser/ty.rs
+11 -15
View File
@@ -1,5 +1,5 @@
use rustc_ast::ptr::P;
use rustc_ast::token::{self, BinOpToken, Delimiter, IdentIsRaw, MetaVarKind, Token, TokenKind};
use rustc_ast::token::{self, Delimiter, IdentIsRaw, MetaVarKind, Token, TokenKind};
use rustc_ast::util::case::Case;
use rustc_ast::{
self as ast, BareFnTy, BoundAsyncness, BoundConstness, BoundPolarity, DUMMY_NODE_ID, FnRetTy,
@@ -86,7 +86,7 @@ enum AllowCVariadic {
/// Types can also be of the form `IDENT(u8, u8) -> u8`, however this assumes
/// that `IDENT` is not the ident of a fn trait.
fn can_continue_type_after_non_fn_ident(t: &Token) -> bool {
t == &token::PathSep || t == &token::Lt || t == &token::BinOp(token::Shl)
t == &token::PathSep || t == &token::Lt || t == &token::Shl
}
fn can_begin_dyn_bound_in_edition_2015(t: &Token) -> bool {
@@ -260,7 +260,7 @@ fn parse_ty_common(
let mut impl_dyn_multi = false;
let kind = if self.check(exp!(OpenParen)) {
self.parse_ty_tuple_or_parens(lo, allow_plus)?
} else if self.eat(exp!(Not)) {
} else if self.eat(exp!(Bang)) {
// Never type `!`
TyKind::Never
} else if self.eat(exp!(Star)) {
@@ -399,7 +399,7 @@ fn parse_ty_tuple_or_parens(&mut self, lo: Span, allow_plus: AllowPlus) -> PResu
let mut trailing_plus = false;
let (ts, trailing) = self.parse_paren_comma_seq(|p| {
let ty = p.parse_ty()?;
trailing_plus = p.prev_token == TokenKind::BinOp(token::Plus);
trailing_plus = p.prev_token == TokenKind::Plus;
Ok(ty)
})?;
@@ -735,7 +735,7 @@ fn parse_impl_ty(&mut self, impl_dyn_multi: &mut bool) -> PResult<'a, TyKind> {
// Always parse bounds greedily for better error recovery.
let bounds = self.parse_generic_bounds()?;
*impl_dyn_multi = bounds.len() > 1 || self.prev_token == TokenKind::BinOp(token::Plus);
*impl_dyn_multi = bounds.len() > 1 || self.prev_token == TokenKind::Plus;
Ok(TyKind::ImplTrait(ast::DUMMY_NODE_ID, bounds))
}
@@ -747,11 +747,7 @@ fn parse_precise_capturing_args(
self.expect_lt()?;
let (args, _, _) = self.parse_seq_to_before_tokens(
&[exp!(Gt)],
&[
&TokenKind::Ge,
&TokenKind::BinOp(BinOpToken::Shr),
&TokenKind::BinOpEq(BinOpToken::Shr),
],
&[&TokenKind::Ge, &TokenKind::Shr, &TokenKind::Shr],
SeqSep::trailing_allowed(exp!(Comma)),
|self_| {
if self_.check_keyword(exp!(SelfUpper)) {
@@ -781,7 +777,7 @@ fn is_explicit_dyn_type(&mut self) -> bool {
self.check_keyword(exp!(Dyn))
&& (self.token.uninterpolated_span().at_least_rust_2018()
|| self.look_ahead(1, |t| {
(can_begin_dyn_bound_in_edition_2015(t) || *t == TokenKind::BinOp(token::Star))
(can_begin_dyn_bound_in_edition_2015(t) || *t == TokenKind::Star)
&& !can_continue_type_after_non_fn_ident(t)
}))
}
@@ -803,7 +799,7 @@ fn parse_dyn_ty(&mut self, impl_dyn_multi: &mut bool) -> PResult<'a, TyKind> {
// Always parse bounds greedily for better error recovery.
let bounds = self.parse_generic_bounds()?;
*impl_dyn_multi = bounds.len() > 1 || self.prev_token == TokenKind::BinOp(token::Plus);
*impl_dyn_multi = bounds.len() > 1 || self.prev_token == TokenKind::Plus;
Ok(TyKind::TraitObject(bounds, syntax))
}
@@ -821,7 +817,7 @@ fn parse_path_start_ty(
) -> PResult<'a, TyKind> {
// Simple path
let path = self.parse_path_inner(PathStyle::Type, ty_generics)?;
if self.eat(exp!(Not)) {
if self.eat(exp!(Bang)) {
// Macro invocation in type position
Ok(TyKind::MacCall(P(MacCall { path, args: self.parse_delim_args()? })))
} else if allow_plus == AllowPlus::Yes && self.check_plus() {
@@ -874,7 +870,7 @@ fn parse_generic_bounds_common(&mut self, allow_plus: AllowPlus) -> PResult<'a,
fn can_begin_bound(&mut self) -> bool {
self.check_path()
|| self.check_lifetime()
|| self.check(exp!(Not))
|| self.check(exp!(Bang))
|| self.check(exp!(Question))
|| self.check(exp!(Tilde))
|| self.check_keyword(exp!(For))
@@ -1025,7 +1021,7 @@ fn parse_trait_bound_modifiers(&mut self) -> PResult<'a, TraitBoundModifiers> {
let polarity = if self.eat(exp!(Question)) {
BoundPolarity::Maybe(self.prev_token.span)
} else if self.eat(exp!(Not)) {
} else if self.eat(exp!(Bang)) {
self.psess.gated_spans.gate(sym::negative_bounds, self.prev_token.span);
BoundPolarity::Negative(self.prev_token.span)
} else {
src/librustdoc/clean/render_macro_matchers.rs
+4 -9
View File
@@ -1,4 +1,4 @@
use rustc_ast::token::{self, BinOpToken, Delimiter, IdentIsRaw};
use rustc_ast::token::{self, Delimiter, IdentIsRaw};
use rustc_ast::tokenstream::{TokenStream, TokenTree};
use rustc_ast_pretty::pprust::PrintState;
use rustc_ast_pretty::pprust::state::State as Printer;
@@ -137,15 +137,10 @@ enum State {
(Dollar, token::Ident(..)) => (false, DollarIdent),
(DollarIdent, token::Colon) => (false, DollarIdentColon),
(DollarIdentColon, token::Ident(..)) => (false, Other),
(
DollarParen,
token::BinOp(BinOpToken::Plus | BinOpToken::Star) | token::Question,
) => (false, Other),
(DollarParen, token::Plus | token::Star | token::Question) => (false, Other),
(DollarParen, _) => (false, DollarParenSep),
(DollarParenSep, token::BinOp(BinOpToken::Plus | BinOpToken::Star)) => {
(false, Other)
}
(Pound, token::Not) => (false, PoundBang),
(DollarParenSep, token::Plus | token::Star) => (false, Other),
(Pound, token::Bang) => (false, PoundBang),
(_, token::Ident(symbol, IdentIsRaw::No))
if !usually_needs_space_between_keyword_and_open_delim(*symbol, tt.span) =>
{
src/tools/rustfmt/src/macros.rs
+26 -8
View File
@@ -12,7 +12,7 @@
use std::collections::HashMap;
use std::panic::{AssertUnwindSafe, catch_unwind};
use rustc_ast::token::{BinOpToken, Delimiter, Token, TokenKind};
use rustc_ast::token::{Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::{TokenStream, TokenStreamIter, TokenTree};
use rustc_ast::{ast, ptr};
use rustc_ast_pretty::pprust;
@@ -841,7 +841,7 @@ fn add_repeat(
match tok {
TokenTree::Token(
Token {
kind: TokenKind::BinOp(BinOpToken::Plus),
kind: TokenKind::Plus,
..
},
_,
@@ -855,7 +855,7 @@ fn add_repeat(
)
| TokenTree::Token(
Token {
kind: TokenKind::BinOp(BinOpToken::Star),
kind: TokenKind::Star,
..
},
_,
@@ -1088,14 +1088,32 @@ fn force_space_before(tok: &TokenKind) -> bool {
| TokenKind::Gt
| TokenKind::AndAnd
| TokenKind::OrOr
| TokenKind::Not
| TokenKind::Bang
| TokenKind::Tilde
| TokenKind::BinOpEq(_)
| TokenKind::PlusEq
| TokenKind::MinusEq
| TokenKind::StarEq
| TokenKind::SlashEq
| TokenKind::PercentEq
| TokenKind::CaretEq
| TokenKind::AndEq
| TokenKind::OrEq
| TokenKind::ShlEq
| TokenKind::ShrEq
| TokenKind::At
| TokenKind::RArrow
| TokenKind::LArrow
| TokenKind::FatArrow
| TokenKind::BinOp(_)
| TokenKind::Plus
| TokenKind::Minus
| TokenKind::Star
| TokenKind::Slash
| TokenKind::Percent
| TokenKind::Caret
| TokenKind::And
| TokenKind::Or
| TokenKind::Shl
| TokenKind::Shr
| TokenKind::Pound
| TokenKind::Dollar => true,
_ => false,
@@ -1113,8 +1131,8 @@ fn next_space(tok: &TokenKind) -> SpaceState {
debug!("next_space: {:?}", tok);
match tok {
TokenKind::Not
| TokenKind::BinOp(BinOpToken::And)
TokenKind::Bang
| TokenKind::And
| TokenKind::Tilde
| TokenKind::At
| TokenKind::Comma