diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index f104400a572b..f165c4ddcdd4 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -416,10 +416,7 @@ fn from_tokens(iter: &mut TokenStreamIter<'_>) -> Option { // This path is currently unreachable in the test suite. unreachable!() } - Some(TokenTree::Token( - Token { kind: token::OpenDelim(_) | token::CloseDelim(_), .. }, - _, - )) => { + Some(TokenTree::Token(Token { kind, .. }, _)) if kind.is_delim() => { panic!("Should be `AttrTokenTree::Delimited`, not delim tokens: {:?}", tt); } _ => return None, diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index 055481f5d878..54781e8235e2 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -32,6 +32,18 @@ pub enum InvisibleOrigin { ProcMacro, } +impl InvisibleOrigin { + // Should the parser skip these invisible delimiters? Ideally this function + // will eventually disappear and no invisible delimiters will be skipped. + #[inline] + pub fn skip(&self) -> bool { + match self { + InvisibleOrigin::MetaVar(_) => false, + InvisibleOrigin::ProcMacro => true, + } + } +} + impl PartialEq for InvisibleOrigin { #[inline] fn eq(&self, _other: &InvisibleOrigin) -> bool { @@ -125,8 +137,7 @@ impl Delimiter { pub fn skip(&self) -> bool { match self { Delimiter::Parenthesis | Delimiter::Bracket | Delimiter::Brace => false, - Delimiter::Invisible(InvisibleOrigin::MetaVar(_)) => false, - Delimiter::Invisible(InvisibleOrigin::ProcMacro) => true, + Delimiter::Invisible(origin) => origin.skip(), } } @@ -140,6 +151,24 @@ pub fn eq_ignoring_invisible_origin(&self, other: &Delimiter) -> bool { _ => false, } } + + pub fn as_open_token_kind(&self) -> TokenKind { + match *self { + Delimiter::Parenthesis => OpenParen, + Delimiter::Brace => OpenBrace, + Delimiter::Bracket => OpenBracket, + Delimiter::Invisible(origin) => OpenInvisible(origin), + } + } + + pub fn as_close_token_kind(&self) -> TokenKind { + match *self { + Delimiter::Parenthesis => CloseParen, + Delimiter::Brace => CloseBrace, + Delimiter::Bracket => CloseBracket, + Delimiter::Invisible(origin) => CloseInvisible(origin), + } + } } // Note that the suffix is *not* considered when deciding the `LitKind` in this @@ -194,9 +223,9 @@ pub fn from_token(token: &Token) -> Option { match token.uninterpolate().kind { Ident(name, IdentIsRaw::No) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)), Literal(token_lit) => Some(token_lit), - OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar( + OpenInvisible(InvisibleOrigin::MetaVar( MetaVarKind::Literal | MetaVarKind::Expr { .. }, - ))) => { + )) => { // Unreachable with the current test suite. panic!("from_token metavar"); } @@ -426,10 +455,22 @@ pub enum TokenKind { Question, /// Used by proc macros for representing lifetimes, not generated by lexer right now. SingleQuote, - /// An opening delimiter (e.g., `{`). - OpenDelim(Delimiter), - /// A closing delimiter (e.g., `}`). - CloseDelim(Delimiter), + /// `(` + OpenParen, + /// `)` + CloseParen, + /// `{` + OpenBrace, + /// `}` + CloseBrace, + /// `[` + OpenBracket, + /// `]` + CloseBracket, + /// Invisible opening delimiter, produced by a macro. + OpenInvisible(InvisibleOrigin), + /// Invisible closing delimiter, produced by a macro. + CloseInvisible(InvisibleOrigin), /* Literals */ Literal(Lit), @@ -530,6 +571,37 @@ pub fn similar_tokens(&self) -> &[TokenKind] { pub fn should_end_const_arg(&self) -> bool { matches!(self, Gt | Ge | Shr | ShrEq) } + + pub fn is_delim(&self) -> bool { + self.open_delim().is_some() || self.close_delim().is_some() + } + + pub fn open_delim(&self) -> Option { + match *self { + OpenParen => Some(Delimiter::Parenthesis), + OpenBrace => Some(Delimiter::Brace), + OpenBracket => Some(Delimiter::Bracket), + OpenInvisible(origin) => Some(Delimiter::Invisible(origin)), + _ => None, + } + } + + pub fn close_delim(&self) -> Option { + match *self { + CloseParen => Some(Delimiter::Parenthesis), + CloseBrace => Some(Delimiter::Brace), + CloseBracket => Some(Delimiter::Bracket), + CloseInvisible(origin) => Some(Delimiter::Invisible(origin)), + _ => None, + } + } + + pub fn is_close_delim_or_eof(&self) -> bool { + match self { + CloseParen | CloseBrace | CloseBracket | CloseInvisible(_) | Eof => true, + _ => false, + } + } } impl Token { @@ -559,7 +631,8 @@ pub fn is_punct(&self) -> bool { | DotDotDot | DotDotEq | Comma | Semi | Colon | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question | SingleQuote => true, - OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..) + OpenParen | CloseParen | OpenBrace | CloseBrace | OpenBracket | CloseBracket + | OpenInvisible(_) | CloseInvisible(_) | Literal(..) | DocComment(..) | Ident(..) | NtIdent(..) | Lifetime(..) | NtLifetime(..) | Eof => false, } } @@ -573,11 +646,12 @@ pub fn is_like_plus(&self) -> bool { /// **NB**: Take care when modifying this function, since it will change /// the stable set of tokens that are allowed to match an expr nonterminal. pub fn can_begin_expr(&self) -> bool { - use Delimiter::*; match self.uninterpolate().kind { Ident(name, is_raw) => ident_can_begin_expr(name, self.span, is_raw), // value name or keyword - OpenDelim(Parenthesis | Brace | Bracket) | // tuple, array or block + OpenParen | // tuple + OpenBrace | // block + OpenBracket | // array Literal(..) | // literal Bang | // operator not Minus | // unary minus @@ -591,12 +665,12 @@ pub fn can_begin_expr(&self) -> bool { PathSep | // global path Lifetime(..) | // labeled loop Pound => true, // expression attributes - OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar( + OpenInvisible(InvisibleOrigin::MetaVar( MetaVarKind::Block | MetaVarKind::Expr { .. } | MetaVarKind::Literal | MetaVarKind::Path - ))) => true, + )) => true, _ => false, } } @@ -608,8 +682,8 @@ pub fn can_begin_pattern(&self, pat_kind: NtPatKind) -> bool { match &self.uninterpolate().kind { // box, ref, mut, and other identifiers (can stricten) Ident(..) | NtIdent(..) | - OpenDelim(Delimiter::Parenthesis) | // tuple pattern - OpenDelim(Delimiter::Bracket) | // slice pattern + OpenParen | // tuple pattern + OpenBracket | // slice pattern And | // reference Minus | // negative literal AndAnd | // double reference @@ -620,14 +694,14 @@ pub fn can_begin_pattern(&self, pat_kind: NtPatKind) -> bool { Lt | // path (UFCS constant) Shl => true, // path (double UFCS) Or => matches!(pat_kind, PatWithOr), // leading vert `|` or-pattern - OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar( + OpenInvisible(InvisibleOrigin::MetaVar( MetaVarKind::Expr { .. } | MetaVarKind::Literal | MetaVarKind::Meta { .. } | MetaVarKind::Pat(_) | MetaVarKind::Path | MetaVarKind::Ty { .. } - ))) => true, + )) => true, _ => false, } } @@ -637,8 +711,8 @@ pub fn can_begin_type(&self) -> bool { match self.uninterpolate().kind { Ident(name, is_raw) => ident_can_begin_type(name, self.span, is_raw), // type name or keyword - OpenDelim(Delimiter::Parenthesis) | // tuple - OpenDelim(Delimiter::Bracket) | // array + OpenParen | // tuple + OpenBracket | // array Bang | // never Star | // raw pointer And | // reference @@ -647,10 +721,10 @@ pub fn can_begin_type(&self) -> bool { Lifetime(..) | // lifetime bound in trait object Lt | Shl | // associated path PathSep => true, // global path - OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar( + OpenInvisible(InvisibleOrigin::MetaVar( MetaVarKind::Ty { .. } | MetaVarKind::Path - ))) => true, + )) => true, // For anonymous structs or unions, which only appear in specific positions // (type of struct fields or union fields), we don't consider them as regular types _ => false, @@ -660,11 +734,11 @@ pub fn can_begin_type(&self) -> bool { /// Returns `true` if the token can appear at the start of a const param. pub fn can_begin_const_arg(&self) -> bool { match self.kind { - OpenDelim(Delimiter::Brace) | Literal(..) | Minus => true, + OpenBrace | Literal(..) | Minus => true, Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true, - OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar( + OpenInvisible(InvisibleOrigin::MetaVar( MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal, - ))) => true, + )) => true, _ => false, } } @@ -711,7 +785,7 @@ pub fn can_begin_literal_maybe_minus(&self) -> bool { match self.uninterpolate().kind { Literal(..) | Minus => true, Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true, - OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind { + OpenInvisible(InvisibleOrigin::MetaVar(mv_kind)) => match mv_kind { MetaVarKind::Literal => true, MetaVarKind::Expr { can_begin_literal_maybe_minus, .. } => { can_begin_literal_maybe_minus @@ -725,7 +799,7 @@ pub fn can_begin_literal_maybe_minus(&self) -> bool { pub fn can_begin_string_literal(&self) -> bool { match self.uninterpolate().kind { Literal(..) => true, - OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind { + OpenInvisible(InvisibleOrigin::MetaVar(mv_kind)) => match mv_kind { MetaVarKind::Literal => true, MetaVarKind::Expr { can_begin_string_literal, .. } => can_begin_string_literal, _ => false, @@ -892,7 +966,7 @@ pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool { /// from an expanded metavar? pub fn is_metavar_seq(&self) -> Option { match self.kind { - OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => Some(kind), + OpenInvisible(InvisibleOrigin::MetaVar(kind)) => Some(kind), _ => None, } } @@ -970,7 +1044,8 @@ pub fn glue(&self, joint: &Token) -> Option { Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | PlusEq | MinusEq | StarEq | SlashEq | PercentEq | CaretEq | AndEq | OrEq | ShlEq | ShrEq | At | DotDotDot | DotDotEq | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question - | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | NtIdent(..) + | OpenParen | CloseParen | OpenBrace | CloseBrace | OpenBracket | CloseBracket + | OpenInvisible(_) | CloseInvisible(_) | Literal(..) | Ident(..) | NtIdent(..) | Lifetime(..) | NtLifetime(..) | DocComment(..) | Eof, _, ) => { diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index 0985ebf945bb..6959cbd87f1f 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -770,12 +770,12 @@ fn print_mac_common( self.bclose(span, empty); } delim => { - let token_str = self.token_kind_to_string(&token::OpenDelim(delim)); + let token_str = self.token_kind_to_string(&delim.as_open_token_kind()); self.word(token_str); self.ibox(0); self.print_tts(tts, convert_dollar_crate); self.end(); - let token_str = self.token_kind_to_string(&token::CloseDelim(delim)); + let token_str = self.token_kind_to_string(&delim.as_close_token_kind()); self.word(token_str); } } @@ -932,14 +932,13 @@ fn token_kind_to_string_ext( token::RArrow => "->".into(), token::LArrow => "<-".into(), token::FatArrow => "=>".into(), - token::OpenDelim(Delimiter::Parenthesis) => "(".into(), - token::CloseDelim(Delimiter::Parenthesis) => ")".into(), - token::OpenDelim(Delimiter::Bracket) => "[".into(), - token::CloseDelim(Delimiter::Bracket) => "]".into(), - token::OpenDelim(Delimiter::Brace) => "{".into(), - token::CloseDelim(Delimiter::Brace) => "}".into(), - token::OpenDelim(Delimiter::Invisible(_)) - | token::CloseDelim(Delimiter::Invisible(_)) => "".into(), + token::OpenParen => "(".into(), + token::CloseParen => ")".into(), + token::OpenBracket => "[".into(), + token::CloseBracket => "]".into(), + token::OpenBrace => "{".into(), + token::CloseBrace => "}".into(), + token::OpenInvisible(_) | token::CloseInvisible(_) => "".into(), token::Pound => "#".into(), token::Dollar => "$".into(), token::Question => "?".into(), diff --git a/compiler/rustc_attr_parsing/src/parser.rs b/compiler/rustc_attr_parsing/src/parser.rs index 384fae59873f..40aa39711d3e 100644 --- a/compiler/rustc_attr_parsing/src/parser.rs +++ b/compiler/rustc_attr_parsing/src/parser.rs @@ -430,9 +430,7 @@ fn next_path(&mut self) -> Option { let span = span.with_hi(segments.last().unwrap().span.hi()); Some(AttrPath { segments: segments.into_boxed_slice(), span }) } - TokenTree::Token(Token { kind: token::OpenDelim(_) | token::CloseDelim(_), .. }, _) => { - None - } + TokenTree::Token(Token { kind, .. }, _) if kind.is_delim() => None, _ => { // malformed attributes can get here. We can't crash, but somewhere else should've // already warned for this. diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index c70e259b2cd8..d2e45d717d90 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -237,10 +237,7 @@ fn can_skip(stream: &AttrTokenStream) -> bool { inner = self.configure_tokens(&inner); Some(AttrTokenTree::Delimited(sp, spacing, delim, inner)) } - AttrTokenTree::Token( - Token { kind: TokenKind::OpenDelim(_) | TokenKind::CloseDelim(_), .. }, - _, - ) => { + AttrTokenTree::Token(Token { kind, .. }, _) if kind.is_delim() => { panic!("Should be `AttrTokenTree::Delimited`, not delim tokens: {:?}", tree); } AttrTokenTree::Token(token, spacing) => Some(AttrTokenTree::Token(token, spacing)), diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs index 1e26d6681943..1f430b0018f5 100644 --- a/compiler/rustc_expand/src/expand.rs +++ b/compiler/rustc_expand/src/expand.rs @@ -7,13 +7,12 @@ use rustc_ast as ast; use rustc_ast::mut_visit::*; use rustc_ast::ptr::P; -use rustc_ast::token::{self, Delimiter}; use rustc_ast::tokenstream::TokenStream; use rustc_ast::visit::{self, AssocCtxt, Visitor, VisitorResult, try_visit, walk_list}; use rustc_ast::{ AssocItemKind, AstNodeWrapper, AttrArgs, AttrStyle, AttrVec, ExprKind, ForeignItemKind, HasAttrs, HasNodeId, Inline, ItemKind, MacStmtStyle, MetaItemInner, MetaItemKind, ModKind, - NodeId, PatKind, StmtKind, TyKind, + NodeId, PatKind, StmtKind, TyKind, token, }; use rustc_ast_pretty::pprust; use rustc_data_structures::flat_map_in_place::FlatMapInPlace; @@ -1004,7 +1003,7 @@ pub fn parse_ast_fragment<'a>( AstFragmentKind::Stmts => { let mut stmts = SmallVec::new(); // Won't make progress on a `}`. - while this.token != token::Eof && this.token != token::CloseDelim(Delimiter::Brace) { + while this.token != token::Eof && this.token != token::CloseBrace { if let Some(stmt) = this.parse_full_stmt(AttemptLocalParseRecovery::Yes)? { stmts.push(stmt); } diff --git a/compiler/rustc_expand/src/mbe/diagnostics.rs b/compiler/rustc_expand/src/mbe/diagnostics.rs index b663e959744f..698492f42e28 100644 --- a/compiler/rustc_expand/src/mbe/diagnostics.rs +++ b/compiler/rustc_expand/src/mbe/diagnostics.rs @@ -1,6 +1,6 @@ use std::borrow::Cow; -use rustc_ast::token::{self, Delimiter, Token, TokenKind}; +use rustc_ast::token::{self, Token}; use rustc_ast::tokenstream::TokenStream; use rustc_errors::{Applicability, Diag, DiagCtxtHandle, DiagMessage}; use rustc_macros::Subdiagnostic; @@ -66,8 +66,8 @@ pub(super) fn failed_to_match_macro( } if let MatcherLoc::Token { token: expected_token } = &remaining_matcher - && (matches!(expected_token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_))) - || matches!(token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_)))) + && (matches!(expected_token.kind, token::OpenInvisible(_)) + || matches!(token.kind, token::OpenInvisible(_))) { err.note("captured metavariables except for `:tt`, `:ident` and `:lifetime` cannot be compared to other tokens"); err.note("see for more information"); diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs index 0065f83eb4ee..c78beb40688f 100644 --- a/compiler/rustc_expand/src/mbe/macro_parser.rs +++ b/compiler/rustc_expand/src/mbe/macro_parser.rs @@ -182,8 +182,8 @@ fn inner( locs.push(MatcherLoc::Token { token: *token }); } TokenTree::Delimited(span, _, delimited) => { - let open_token = Token::new(token::OpenDelim(delimited.delim), span.open); - let close_token = Token::new(token::CloseDelim(delimited.delim), span.close); + let open_token = Token::new(delimited.delim.as_open_token_kind(), span.open); + let close_token = Token::new(delimited.delim.as_close_token_kind(), span.close); locs.push(MatcherLoc::Delimited); locs.push(MatcherLoc::Token { token: open_token }); diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index c138b0908772..93604a149f1d 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -6,7 +6,7 @@ use ast::token::IdentIsRaw; use rustc_ast::token::NtPatKind::*; use rustc_ast::token::TokenKind::*; -use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind}; +use rustc_ast::token::{self, NonterminalKind, Token, TokenKind}; use rustc_ast::tokenstream::{DelimSpan, TokenStream}; use rustc_ast::{self as ast, DUMMY_NODE_ID, NodeId}; use rustc_ast_pretty::pprust; @@ -784,7 +784,7 @@ fn build_recur<'tt>(sets: &mut FirstSets<'tt>, tts: &'tt [TokenTree]) -> TokenSe TokenTree::Delimited(span, _, delimited) => { build_recur(sets, &delimited.tts); first.replace_with(TtHandle::from_token_kind( - token::OpenDelim(delimited.delim), + delimited.delim.as_open_token_kind(), span.open, )); } @@ -852,7 +852,7 @@ fn first(&self, tts: &'tt [mbe::TokenTree]) -> TokenSet<'tt> { } TokenTree::Delimited(span, _, delimited) => { first.add_one(TtHandle::from_token_kind( - token::OpenDelim(delimited.delim), + delimited.delim.as_open_token_kind(), span.open, )); return first; @@ -1099,7 +1099,7 @@ fn check_matcher_core<'tt>( } TokenTree::Delimited(span, _, d) => { let my_suffix = TokenSet::singleton(TtHandle::from_token_kind( - token::CloseDelim(d.delim), + d.delim.as_close_token_kind(), span.close, )); check_matcher_core(sess, node_id, first_sets, &d.tts, &my_suffix)?; @@ -1299,7 +1299,9 @@ enum IsInFollow { fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow { use mbe::TokenTree; - if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }) = *tok { + if let TokenTree::Token(Token { kind, .. }) = tok + && kind.close_delim().is_some() + { // closing a token tree can never be matched by any fragment; // iow, we always require that `(` and `)` match, etc. IsInFollow::Yes @@ -1358,16 +1360,8 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow { ]; match tok { TokenTree::Token(token) => match token.kind { - OpenDelim(Delimiter::Brace) - | OpenDelim(Delimiter::Bracket) - | Comma - | FatArrow - | Colon - | Eq - | Gt - | Shr - | Semi - | Or => IsInFollow::Yes, + OpenBrace | OpenBracket | Comma | FatArrow | Colon | Eq | Gt | Shr + | Semi | Or => IsInFollow::Yes, Ident(name, IdentIsRaw::No) if name == kw::As || name == kw::Where => { IsInFollow::Yes } diff --git a/compiler/rustc_expand/src/mbe/quoted.rs b/compiler/rustc_expand/src/mbe/quoted.rs index 3f0372599565..0c2362f23bcf 100644 --- a/compiler/rustc_expand/src/mbe/quoted.rs +++ b/compiler/rustc_expand/src/mbe/quoted.rs @@ -181,7 +181,10 @@ fn parse_tree<'a>( if delim != Delimiter::Parenthesis { span_dollar_dollar_or_metavar_in_the_lhs_err( sess, - &Token { kind: token::OpenDelim(delim), span: delim_span.entire() }, + &Token { + kind: delim.as_open_token_kind(), + span: delim_span.entire(), + }, ); } } else { @@ -217,7 +220,8 @@ fn parse_tree<'a>( } Delimiter::Parenthesis => {} _ => { - let token = pprust::token_kind_to_string(&token::OpenDelim(delim)); + let token = + pprust::token_kind_to_string(&delim.as_open_token_kind()); sess.dcx().emit_err(errors::ExpectedParenOrBrace { span: delim_span.entire(), token, diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 4edaf68c89ae..f00201ad202a 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -308,8 +308,8 @@ fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self { })); } - OpenDelim(..) | CloseDelim(..) => unreachable!(), - Eof => unreachable!(), + OpenParen | CloseParen | OpenBrace | CloseBrace | OpenBracket | CloseBracket + | OpenInvisible(_) | CloseInvisible(_) | Eof => unreachable!(), } } trees diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index 4935fc03256f..a8ec9a1e952e 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -371,12 +371,12 @@ fn next_token_from_cursor(&mut self) -> (Token, bool) { rustc_lexer::TokenKind::Semi => token::Semi, rustc_lexer::TokenKind::Comma => token::Comma, rustc_lexer::TokenKind::Dot => token::Dot, - rustc_lexer::TokenKind::OpenParen => token::OpenDelim(Delimiter::Parenthesis), - rustc_lexer::TokenKind::CloseParen => token::CloseDelim(Delimiter::Parenthesis), - rustc_lexer::TokenKind::OpenBrace => token::OpenDelim(Delimiter::Brace), - rustc_lexer::TokenKind::CloseBrace => token::CloseDelim(Delimiter::Brace), - rustc_lexer::TokenKind::OpenBracket => token::OpenDelim(Delimiter::Bracket), - rustc_lexer::TokenKind::CloseBracket => token::CloseDelim(Delimiter::Bracket), + rustc_lexer::TokenKind::OpenParen => token::OpenParen, + rustc_lexer::TokenKind::CloseParen => token::CloseParen, + rustc_lexer::TokenKind::OpenBrace => token::OpenBrace, + rustc_lexer::TokenKind::CloseBrace => token::CloseBrace, + rustc_lexer::TokenKind::OpenBracket => token::OpenBracket, + rustc_lexer::TokenKind::CloseBracket => token::CloseBracket, rustc_lexer::TokenKind::At => token::At, rustc_lexer::TokenKind::Pound => token::Pound, rustc_lexer::TokenKind::Tilde => token::Tilde, diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs index b3f83a320241..0ddd9a85df86 100644 --- a/compiler/rustc_parse/src/lexer/tokentrees.rs +++ b/compiler/rustc_parse/src/lexer/tokentrees.rs @@ -18,38 +18,33 @@ pub(super) fn lex_token_trees( let mut buf = Vec::new(); loop { - match self.token.kind { - token::OpenDelim(delim) => { - // Invisible delimiters cannot occur here because `TokenTreesReader` parses - // code directly from strings, with no macro expansion involved. - debug_assert!(!matches!(delim, Delimiter::Invisible(_))); - buf.push(match self.lex_token_tree_open_delim(delim) { - Ok(val) => val, - Err(errs) => return Err(errs), - }) - } - token::CloseDelim(delim) => { - // Invisible delimiters cannot occur here because `TokenTreesReader` parses - // code directly from strings, with no macro expansion involved. - debug_assert!(!matches!(delim, Delimiter::Invisible(_))); - return if is_delimited { - Ok((open_spacing, TokenStream::new(buf))) - } else { - Err(vec![self.close_delim_err(delim)]) - }; - } - token::Eof => { - return if is_delimited { - Err(vec![self.eof_err()]) - } else { - Ok((open_spacing, TokenStream::new(buf))) - }; - } - _ => { - // Get the next normal token. - let (this_tok, this_spacing) = self.bump(); - buf.push(TokenTree::Token(this_tok, this_spacing)); - } + if let Some(delim) = self.token.kind.open_delim() { + // Invisible delimiters cannot occur here because `TokenTreesReader` parses + // code directly from strings, with no macro expansion involved. + debug_assert!(!matches!(delim, Delimiter::Invisible(_))); + buf.push(match self.lex_token_tree_open_delim(delim) { + Ok(val) => val, + Err(errs) => return Err(errs), + }) + } else if let Some(delim) = self.token.kind.close_delim() { + // Invisible delimiters cannot occur here because `TokenTreesReader` parses + // code directly from strings, with no macro expansion involved. + debug_assert!(!matches!(delim, Delimiter::Invisible(_))); + return if is_delimited { + Ok((open_spacing, TokenStream::new(buf))) + } else { + Err(vec![self.close_delim_err(delim)]) + }; + } else if self.token.kind == token::Eof { + return if is_delimited { + Err(vec![self.eof_err()]) + } else { + Ok((open_spacing, TokenStream::new(buf))) + }; + } else { + // Get the next normal token. + let (this_tok, this_spacing) = self.bump(); + buf.push(TokenTree::Token(this_tok, this_spacing)); } } } @@ -111,9 +106,9 @@ fn lex_token_tree_open_delim( let delim_span = DelimSpan::from_pair(pre_span, self.token.span); let sm = self.psess.source_map(); - let close_spacing = match self.token.kind { - // Correct delimiter. - token::CloseDelim(close_delim) if close_delim == open_delim => { + let close_spacing = if let Some(close_delim) = self.token.kind.close_delim() { + if close_delim == open_delim { + // Correct delimiter. let (open_brace, open_brace_span) = self.diag_info.open_braces.pop().unwrap(); let close_brace_span = self.token.span; @@ -134,9 +129,8 @@ fn lex_token_tree_open_delim( // Move past the closing delimiter. self.bump_minimal() - } - // Incorrect delimiter. - token::CloseDelim(close_delim) => { + } else { + // Incorrect delimiter. let mut unclosed_delimiter = None; let mut candidate = None; @@ -182,14 +176,13 @@ fn lex_token_tree_open_delim( Spacing::Alone } } - token::Eof => { - // Silently recover, the EOF token will be seen again - // and an error emitted then. Thus we don't pop from - // self.open_braces here. The choice of spacing value here - // doesn't matter. - Spacing::Alone - } - _ => unreachable!(), + } else { + assert_eq!(self.token.kind, token::Eof); + // Silently recover, the EOF token will be seen again + // and an error emitted then. Thus we don't pop from + // self.open_braces here. The choice of spacing value here + // doesn't matter. + Spacing::Alone }; let spacing = DelimSpacing::new(open_spacing, close_spacing); diff --git a/compiler/rustc_parse/src/lexer/unicode_chars.rs b/compiler/rustc_parse/src/lexer/unicode_chars.rs index 2bfa1ea4e058..751d13af4331 100644 --- a/compiler/rustc_parse/src/lexer/unicode_chars.rs +++ b/compiler/rustc_parse/src/lexer/unicode_chars.rs @@ -5,7 +5,7 @@ use super::Lexer; use crate::errors::TokenSubstitution; -use crate::token::{self, Delimiter}; +use crate::token; #[rustfmt::skip] // for line breaks pub(super) static UNICODE_ARRAY: &[(char, &str, &str)] = &[ @@ -315,12 +315,12 @@ ("!", "Exclamation Mark", Some(token::Bang)), ("?", "Question Mark", Some(token::Question)), (".", "Period", Some(token::Dot)), - ("(", "Left Parenthesis", Some(token::OpenDelim(Delimiter::Parenthesis))), - (")", "Right Parenthesis", Some(token::CloseDelim(Delimiter::Parenthesis))), - ("[", "Left Square Bracket", Some(token::OpenDelim(Delimiter::Bracket))), - ("]", "Right Square Bracket", Some(token::CloseDelim(Delimiter::Bracket))), - ("{", "Left Curly Brace", Some(token::OpenDelim(Delimiter::Brace))), - ("}", "Right Curly Brace", Some(token::CloseDelim(Delimiter::Brace))), + ("(", "Left Parenthesis", Some(token::OpenParen)), + (")", "Right Parenthesis", Some(token::CloseParen)), + ("[", "Left Square Bracket", Some(token::OpenBracket)), + ("]", "Right Square Bracket", Some(token::CloseBracket)), + ("{", "Left Curly Brace", Some(token::OpenBrace)), + ("}", "Right Curly Brace", Some(token::CloseBrace)), ("*", "Asterisk", Some(token::Star)), ("/", "Slash", Some(token::Slash)), ("\\", "Backslash", None), diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index f1bd6a227309..6061c9cb4856 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -1,7 +1,7 @@ use std::borrow::Cow; use std::{iter, mem}; -use rustc_ast::token::{Delimiter, Token, TokenKind}; +use rustc_ast::token::{Delimiter, Token}; use rustc_ast::tokenstream::{ AttrTokenStream, AttrTokenTree, AttrsTarget, DelimSpacing, DelimSpan, LazyAttrTokenStream, Spacing, ToAttrTokenStream, @@ -501,27 +501,27 @@ struct FrameData { let mut stack_rest = vec![]; for flat_token in iter { match flat_token { - FlatToken::Token((Token { kind: TokenKind::OpenDelim(delim), span }, spacing)) => { - stack_rest.push(mem::replace( - &mut stack_top, - FrameData { open_delim_sp: Some((delim, span, spacing)), inner: vec![] }, - )); - } - FlatToken::Token((Token { kind: TokenKind::CloseDelim(delim), span }, spacing)) => { - let frame_data = mem::replace(&mut stack_top, stack_rest.pop().unwrap()); - let (open_delim, open_sp, open_spacing) = frame_data.open_delim_sp.unwrap(); - assert!( - open_delim.eq_ignoring_invisible_origin(&delim), - "Mismatched open/close delims: open={open_delim:?} close={span:?}" - ); - let dspan = DelimSpan::from_pair(open_sp, span); - let dspacing = DelimSpacing::new(open_spacing, spacing); - let stream = AttrTokenStream::new(frame_data.inner); - let delimited = AttrTokenTree::Delimited(dspan, dspacing, delim, stream); - stack_top.inner.push(delimited); - } - FlatToken::Token((token, spacing)) => { - stack_top.inner.push(AttrTokenTree::Token(token, spacing)) + FlatToken::Token((token @ Token { kind, span }, spacing)) => { + if let Some(delim) = kind.open_delim() { + stack_rest.push(mem::replace( + &mut stack_top, + FrameData { open_delim_sp: Some((delim, span, spacing)), inner: vec![] }, + )); + } else if let Some(delim) = kind.close_delim() { + let frame_data = mem::replace(&mut stack_top, stack_rest.pop().unwrap()); + let (open_delim, open_sp, open_spacing) = frame_data.open_delim_sp.unwrap(); + assert!( + open_delim.eq_ignoring_invisible_origin(&delim), + "Mismatched open/close delims: open={open_delim:?} close={span:?}" + ); + let dspan = DelimSpan::from_pair(open_sp, span); + let dspacing = DelimSpacing::new(open_spacing, spacing); + let stream = AttrTokenStream::new(frame_data.inner); + let delimited = AttrTokenTree::Delimited(dspan, dspacing, delim, stream); + stack_top.inner.push(delimited); + } else { + stack_top.inner.push(AttrTokenTree::Token(token, spacing)) + } } FlatToken::AttrsTarget(target) => { stack_top.inner.push(AttrTokenTree::AttrsTarget(target)) diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index 7c8e0146c3d2..23c8db7bca78 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -4,7 +4,7 @@ use ast::token::IdentIsRaw; use rustc_ast as ast; use rustc_ast::ptr::P; -use rustc_ast::token::{self, Delimiter, Lit, LitKind, Token, TokenKind}; +use rustc_ast::token::{self, Lit, LitKind, Token, TokenKind}; use rustc_ast::util::parser::AssocOp; use rustc_ast::{ AngleBracketedArg, AngleBracketedArgs, AnonConst, AttrVec, BinOpKind, BindingMode, Block, @@ -304,10 +304,10 @@ pub(super) fn expected_ident_found( TokenKind::Comma, TokenKind::Semi, TokenKind::PathSep, - TokenKind::OpenDelim(Delimiter::Brace), - TokenKind::OpenDelim(Delimiter::Parenthesis), - TokenKind::CloseDelim(Delimiter::Brace), - TokenKind::CloseDelim(Delimiter::Parenthesis), + TokenKind::OpenBrace, + TokenKind::OpenParen, + TokenKind::CloseBrace, + TokenKind::CloseParen, ]; if let TokenKind::DocComment(..) = self.prev_token.kind && valid_follow.contains(&self.token.kind) @@ -507,7 +507,7 @@ fn tokens_to_string(tokens: &[TokenType]) -> String { } else if !sm.is_multiline(self.prev_token.span.until(self.token.span)) { // The current token is in the same line as the prior token, not recoverable. } else if [token::Comma, token::Colon].contains(&self.token.kind) - && self.prev_token == token::CloseDelim(Delimiter::Parenthesis) + && self.prev_token == token::CloseParen { // Likely typo: The current token is on a new line and is expected to be // `.`, `;`, `?`, or an operator after a close delimiter token. @@ -518,8 +518,7 @@ fn tokens_to_string(tokens: &[TokenType]) -> String { // ^ // https://github.com/rust-lang/rust/issues/72253 } else if self.look_ahead(1, |t| { - t == &token::CloseDelim(Delimiter::Brace) - || t.can_begin_expr() && *t != token::Colon + t == &token::CloseBrace || t.can_begin_expr() && *t != token::Colon }) && [token::Comma, token::Colon].contains(&self.token.kind) { // Likely typo: `,` → `;` or `:` → `;`. This is triggered if the current token is @@ -537,7 +536,7 @@ fn tokens_to_string(tokens: &[TokenType]) -> String { self.bump(); return Ok(guar); } else if self.look_ahead(0, |t| { - t == &token::CloseDelim(Delimiter::Brace) + t == &token::CloseBrace || ((t.can_begin_expr() || t.can_begin_item()) && t != &token::Semi && t != &token::Pound) @@ -675,8 +674,7 @@ fn tokens_to_string(tokens: &[TokenType]) -> String { // `pub` may be used for an item or `pub(crate)` if self.prev_token.is_ident_named(sym::public) - && (self.token.can_begin_item() - || self.token == TokenKind::OpenDelim(Delimiter::Parenthesis)) + && (self.token.can_begin_item() || self.token == TokenKind::OpenParen) { err.span_suggestion_short( self.prev_token.span, @@ -843,9 +841,7 @@ pub(super) fn attr_on_non_tail_expr(&self, expr: &Expr) -> ErrorGuaranteed { if expr.attrs.len() == 1 { "this attribute" } else { "these attributes" }, ), ); - if self.token == token::Pound - && self.look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Bracket)) - { + if self.token == token::Pound && self.look_ahead(1, |t| *t == token::OpenBracket) { // We have // #[attr] // expr @@ -1037,9 +1033,7 @@ pub(super) fn recover_closure_body( ) -> PResult<'a, P> { err.span_label(lo.to(decl_hi), "while parsing the body of this closure"); let guar = match before.kind { - token::OpenDelim(Delimiter::Brace) - if !matches!(token.kind, token::OpenDelim(Delimiter::Brace)) => - { + token::OpenBrace if token.kind != token::OpenBrace => { // `{ || () }` should have been `|| { () }` err.multipart_suggestion( "you might have meant to open the body of the closure, instead of enclosing \ @@ -1054,9 +1048,7 @@ pub(super) fn recover_closure_body( self.eat_to_tokens(&[exp!(CloseBrace)]); guar } - token::OpenDelim(Delimiter::Parenthesis) - if !matches!(token.kind, token::OpenDelim(Delimiter::Brace)) => - { + token::OpenParen if token.kind != token::OpenBrace => { // We are within a function call or tuple, we can emit the error // and recover. self.eat_to_tokens(&[exp!(CloseParen), exp!(Comma)]); @@ -1071,7 +1063,7 @@ pub(super) fn recover_closure_body( ); err.emit() } - _ if !matches!(token.kind, token::OpenDelim(Delimiter::Brace)) => { + _ if token.kind != token::OpenBrace => { // We don't have a heuristic to correctly identify where the block // should be closed. err.multipart_suggestion_verbose( @@ -1225,7 +1217,7 @@ pub(super) fn check_turbofish_missing_angle_brackets(&mut self, segment: &mut Pa trailing_span = trailing_span.to(self.token.span); self.bump(); } - if self.token == token::OpenDelim(Delimiter::Parenthesis) { + if self.token == token::OpenParen { // Recover from bad turbofish: `foo.collect::Vec<_>()`. segment.args = Some(AngleBracketedArgs { args, span }.into()); @@ -1470,9 +1462,7 @@ pub(super) fn check_no_chained_comparison( let modifiers = [(token::Lt, 1), (token::Gt, -1), (token::Shr, -2)]; self.consume_tts(1, &modifiers); - if !&[token::OpenDelim(Delimiter::Parenthesis), token::PathSep] - .contains(&self.token.kind) - { + if !matches!(self.token.kind, token::OpenParen | token::PathSep) { // We don't have `foo< bar >(` or `foo< bar >::`, so we rewind the // parser and bail out. self.restore_snapshot(snapshot); @@ -1510,7 +1500,7 @@ pub(super) fn check_no_chained_comparison( Err(self.dcx().create_err(err)) } } - } else if self.token == token::OpenDelim(Delimiter::Parenthesis) { + } else if self.token == token::OpenParen { // We have high certainty that this was a bad turbofish at this point. // `foo< bar >(` if let ExprKind::Binary(o, ..) = inner_op.kind @@ -1570,10 +1560,7 @@ fn consume_fn_args(&mut self) -> Result<(), ()> { self.bump(); // `(` // Consume the fn call arguments. - let modifiers = [ - (token::OpenDelim(Delimiter::Parenthesis), 1), - (token::CloseDelim(Delimiter::Parenthesis), -1), - ]; + let modifiers = [(token::OpenParen, 1), (token::CloseParen, -1)]; self.consume_tts(1, &modifiers); if self.token == token::Eof { @@ -1978,7 +1965,7 @@ fn recover_await_macro(&mut self) -> PResult<'a, (Span, P, bool)> { fn recover_await_prefix(&mut self, await_sp: Span) -> PResult<'a, (Span, P, bool)> { let is_question = self.eat(exp!(Question)); // Handle `await? `. - let expr = if self.token == token::OpenDelim(Delimiter::Brace) { + let expr = if self.token == token::OpenBrace { // Handle `await { }`. // This needs to be handled separately from the next arm to avoid // interpreting `await { }?` as `?.await`. @@ -2014,9 +2001,7 @@ fn error_on_incorrect_await( /// If encountering `future.await()`, consumes and emits an error. pub(super) fn recover_from_await_method_call(&mut self) { - if self.token == token::OpenDelim(Delimiter::Parenthesis) - && self.look_ahead(1, |t| t == &token::CloseDelim(Delimiter::Parenthesis)) - { + if self.token == token::OpenParen && self.look_ahead(1, |t| t == &token::CloseParen) { // future.await() let lo = self.token.span; self.bump(); // ( @@ -2029,9 +2014,7 @@ pub(super) fn recover_from_await_method_call(&mut self) { /// /// If encountering `x.use()`, consumes and emits an error. pub(super) fn recover_from_use(&mut self) { - if self.token == token::OpenDelim(Delimiter::Parenthesis) - && self.look_ahead(1, |t| t == &token::CloseDelim(Delimiter::Parenthesis)) - { + if self.token == token::OpenParen && self.look_ahead(1, |t| t == &token::CloseParen) { // var.use() let lo = self.token.span; self.bump(); // ( @@ -2045,7 +2028,7 @@ pub(super) fn recover_from_use(&mut self) { pub(super) fn try_macro_suggestion(&mut self) -> PResult<'a, P> { let is_try = self.token.is_keyword(kw::Try); let is_questionmark = self.look_ahead(1, |t| t == &token::Bang); //check for ! - let is_open = self.look_ahead(2, |t| t == &token::OpenDelim(Delimiter::Parenthesis)); //check for ( + let is_open = self.look_ahead(2, |t| t == &token::OpenParen); //check for ( if is_try && is_questionmark && is_open { let lo = self.token.span; @@ -2053,7 +2036,7 @@ pub(super) fn try_macro_suggestion(&mut self) -> PResult<'a, P> { self.bump(); //remove ! let try_span = lo.to(self.token.span); //we take the try!( span self.bump(); //remove ( - let is_empty = self.token == token::CloseDelim(Delimiter::Parenthesis); //check if the block is empty + let is_empty = self.token == token::CloseParen; //check if the block is empty self.consume_block(exp!(OpenParen), exp!(CloseParen), ConsumeClosingDelim::No); //eat the block let hi = self.token.span; self.bump(); //remove ) @@ -2148,7 +2131,7 @@ pub(super) fn recover_stmt_( loop { debug!("recover_stmt_ loop {:?}", self.token); match self.token.kind { - token::OpenDelim(Delimiter::Brace) => { + token::OpenBrace => { brace_depth += 1; self.bump(); if break_on_block == BlockMode::Break && brace_depth == 1 && bracket_depth == 0 @@ -2156,11 +2139,11 @@ pub(super) fn recover_stmt_( in_block = true; } } - token::OpenDelim(Delimiter::Bracket) => { + token::OpenBracket => { bracket_depth += 1; self.bump(); } - token::CloseDelim(Delimiter::Brace) => { + token::CloseBrace => { if brace_depth == 0 { debug!("recover_stmt_ return - close delim {:?}", self.token); break; @@ -2172,7 +2155,7 @@ pub(super) fn recover_stmt_( break; } } - token::CloseDelim(Delimiter::Bracket) => { + token::CloseBracket => { bracket_depth -= 1; if bracket_depth < 0 { bracket_depth = 0; @@ -2219,12 +2202,10 @@ pub(super) fn eat_incorrect_doc_comment_for_param_type(&mut self) { if let token::DocComment(..) = self.token.kind { self.dcx().emit_err(DocCommentOnParamType { span: self.token.span }); self.bump(); - } else if self.token == token::Pound - && self.look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Bracket)) - { + } else if self.token == token::Pound && self.look_ahead(1, |t| *t == token::OpenBracket) { let lo = self.token.span; // Skip every token until next possible arg. - while self.token != token::CloseDelim(Delimiter::Bracket) { + while self.token != token::CloseBracket { self.bump(); } let sp = lo.to(self.token.span); @@ -2243,9 +2224,7 @@ pub(super) fn parameter_without_type( // If we find a pattern followed by an identifier, it could be an (incorrect) // C-style parameter declaration. if self.check_ident() - && self.look_ahead(1, |t| { - *t == token::Comma || *t == token::CloseDelim(Delimiter::Parenthesis) - }) + && self.look_ahead(1, |t| *t == token::Comma || *t == token::CloseParen) { // `fn foo(String s) {}` let ident = self.parse_ident().unwrap(); @@ -2261,7 +2240,7 @@ pub(super) fn parameter_without_type( } else if require_name && (self.token == token::Comma || self.token == token::Lt - || self.token == token::CloseDelim(Delimiter::Parenthesis)) + || self.token == token::CloseParen) { let rfc_note = "anonymous parameters are removed in the 2018 edition (see RFC 1685)"; @@ -2872,7 +2851,7 @@ pub(crate) fn maybe_recover_unexpected_block_label( // Check for `'a : {` if !(self.check_lifetime() && self.look_ahead(1, |t| *t == token::Colon) - && self.look_ahead(2, |t| *t == token::OpenDelim(Delimiter::Brace))) + && self.look_ahead(2, |t| *t == token::OpenBrace)) { return false; } diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 71cc814cb500..11e60175a1f4 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -436,7 +436,7 @@ fn parse_expr_range( fn is_at_start_of_range_notation_rhs(&self) -> bool { if self.token.can_begin_expr() { // Parse `for i in 1.. { }` as infinite loop, not as `for i in (1..{})`. - if self.token == token::OpenDelim(Delimiter::Brace) { + if self.token == token::OpenBrace { return !self.restrictions.contains(Restrictions::NO_STRUCT_LITERAL); } true @@ -542,8 +542,8 @@ macro_rules! make_it { } // Recover from `++x`: token::Plus if this.look_ahead(1, |t| *t == token::Plus) => { - let starts_stmt = this.prev_token == token::Semi - || this.prev_token == token::CloseDelim(Delimiter::Brace); + let starts_stmt = + this.prev_token == token::Semi || this.prev_token == token::CloseBrace; let pre_span = this.token.span.to(this.look_ahead(1, |t| t.span)); // Eat both `+`s. this.bump(); @@ -637,8 +637,8 @@ fn recover_not_expr(&mut self, lo: Span) -> PResult<'a, (Span, ExprKind)> { /// Returns the span of expr if it was not interpolated, or the span of the interpolated token. fn interpolated_or_expr_span(&self, expr: &Expr) -> Span { match self.prev_token.kind { - TokenKind::NtIdent(..) | TokenKind::NtLifetime(..) => self.prev_token.span, - TokenKind::CloseDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => { + token::NtIdent(..) | token::NtLifetime(..) => self.prev_token.span, + token::CloseInvisible(InvisibleOrigin::MetaVar(_)) => { // `expr.span` is the interpolated span, because invisible open // and close delims both get marked with the same span, one // that covers the entire thing between them. (See @@ -912,8 +912,8 @@ pub(super) fn parse_expr_dot_or_call_with( return Ok(e); } e = match self.token.kind { - token::OpenDelim(Delimiter::Parenthesis) => self.parse_expr_fn_call(lo, e), - token::OpenDelim(Delimiter::Bracket) => self.parse_expr_index(lo, e)?, + token::OpenParen => self.parse_expr_fn_call(lo, e), + token::OpenBracket => self.parse_expr_index(lo, e)?, _ => return Ok(e), } } @@ -1002,7 +1002,7 @@ fn error_unexpected_after_dot(&self) { (token::Eof, Some(_)) if let Ok(snippet) = sm.span_to_snippet(sm.next_point(span)) => { (span.shrink_to_hi(), format!("`{}`", snippet)) } - (token::CloseDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))), _) => { + (token::CloseInvisible(InvisibleOrigin::MetaVar(_)), _) => { // No need to report an error. This case will only occur when parsing a pasted // metavariable, and we should have emitted an error when parsing the macro call in // the first place. E.g. in this code: @@ -1202,7 +1202,7 @@ fn parse_floating_field_access(&mut self) -> PResult<'a, P<[Ident]>> { } } - if matches!(self.token.kind, token::CloseDelim(..) | token::Comma) { + if self.token.kind.close_delim().is_some() || self.token.kind == token::Comma { break; } else if trailing_dot.is_none() { // This loop should only repeat if there is a trailing dot. @@ -1232,7 +1232,7 @@ fn mk_expr_tuple_field_access( /// Parse a function call expression, `expr(...)`. fn parse_expr_fn_call(&mut self, lo: Span, fun: P) -> P { - let snapshot = if self.token == token::OpenDelim(Delimiter::Parenthesis) { + let snapshot = if self.token == token::OpenParen { Some((self.create_snapshot_for_diagnostic(), fun.kind.clone())) } else { None @@ -1676,14 +1676,11 @@ pub(super) fn parse_expr_labeled( self.parse_expr_for(label, lo) } else if self.eat_keyword(exp!(Loop)) { self.parse_expr_loop(label, lo) - } else if self.check_noexpect(&token::OpenDelim(Delimiter::Brace)) - || self.token.is_metavar_block() - { + } else if self.check_noexpect(&token::OpenBrace) || self.token.is_metavar_block() { self.parse_expr_block(label, lo, BlockCheckMode::Default) } else if !ate_colon && self.may_recover() - && (matches!(self.token.kind, token::CloseDelim(_) | token::Comma) - || self.token.is_punct()) + && (self.token.kind.close_delim().is_some() || self.token.is_punct()) && could_be_unclosed_char_literal(label_.ident) { let (lit, _) = @@ -1878,7 +1875,7 @@ fn parse_expr_break(&mut self) -> PResult<'a, P> { }, }); Some(lexpr) - } else if self.token != token::OpenDelim(Delimiter::Brace) + } else if self.token != token::OpenBrace || !self.restrictions.contains(Restrictions::NO_STRUCT_LITERAL) { let mut expr = self.parse_expr_opt()?; @@ -2016,7 +2013,7 @@ pub(crate) fn parse_expr_offset_of(&mut self, lo: Span) -> PResult<'a, P> // Eat tokens until the macro call ends. if self.may_recover() { - while !matches!(self.token.kind, token::CloseDelim(..) | token::Eof) { + while !self.token.kind.is_close_delim_or_eof() { self.bump(); } } @@ -2157,9 +2154,7 @@ fn eat_token_lit(&mut self) -> Option { self.bump(); Some(token_lit) } - token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar( - MetaVarKind::Literal, - ))) => { + token::OpenInvisible(InvisibleOrigin::MetaVar(MetaVarKind::Literal)) => { let lit = self .eat_metavar_seq(MetaVarKind::Literal, |this| this.parse_literal_maybe_minus()) .expect("metavar seq literal"); @@ -2168,9 +2163,9 @@ fn eat_token_lit(&mut self) -> Option { }; Some(token_lit) } - token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar( + token::OpenInvisible(InvisibleOrigin::MetaVar( mv_kind @ MetaVarKind::Expr { can_begin_literal_maybe_minus: true, .. }, - ))) => { + )) => { let expr = self .eat_metavar_seq(mv_kind, |this| this.parse_expr()) .expect("metavar seq expr"); @@ -2275,7 +2270,7 @@ pub fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P> { } fn is_array_like_block(&mut self) -> bool { - matches!(self.token.kind, TokenKind::OpenDelim(Delimiter::Brace)) + self.token.kind == TokenKind::OpenBrace && self .look_ahead(1, |t| matches!(t.kind, TokenKind::Ident(..) | TokenKind::Literal(_))) && self.look_ahead(2, |t| t == &token::Comma) @@ -2328,8 +2323,8 @@ fn suggest_missing_semicolon_before_array( |p| p.parse_expr(), ) { Ok(_) - // When the close delim is `)`, `token.kind` is expected to be `token::CloseDelim(Delimiter::Parenthesis)`, - // but the actual `token.kind` is `token::CloseDelim(Delimiter::Bracket)`. + // When the close delim is `)`, `token.kind` is expected to be `token::CloseParen`, + // but the actual `token.kind` is `token::CloseBracket`. // This is because the `token.kind` of the close delim is treated as the same as // that of the open delim in `TokenTreesReader::parse_token_tree`, even if the delimiters of them are different. // Therefore, `token.kind` should not be compared here. @@ -2484,7 +2479,7 @@ fn parse_expr_closure(&mut self) -> PResult<'a, P> { fn parse_closure_block_body(&mut self, ret_span: Span) -> PResult<'a, P> { if self.may_recover() && self.token.can_begin_expr() - && !matches!(self.token.kind, TokenKind::OpenDelim(Delimiter::Brace)) + && self.token.kind != TokenKind::OpenBrace && !self.token.is_metavar_block() { let snapshot = self.create_snapshot_for_diagnostic(); @@ -2887,7 +2882,7 @@ fn error_on_extra_if(&mut self, cond: &P) -> PResult<'a, ()> { } fn parse_for_head(&mut self) -> PResult<'a, (P, P)> { - let begin_paren = if self.token == token::OpenDelim(Delimiter::Parenthesis) { + let begin_paren = if self.token == token::OpenParen { // Record whether we are about to parse `for (`. // This is used below for recovery in case of `for ( $stuff ) $block` // in which case we will suggest `for $stuff $block`. @@ -2921,7 +2916,7 @@ fn parse_for_head(&mut self) -> PResult<'a, (P, P)> { return Err(err); } }; - return if self.token == token::CloseDelim(Delimiter::Parenthesis) { + return if self.token == token::CloseParen { // We know for sure we have seen `for ($SOMETHING in $EXPR)`, so we recover the // parser state and emit a targeted suggestion. let span = vec![start_span, self.token.span]; @@ -2965,7 +2960,7 @@ fn parse_expr_for(&mut self, opt_label: Option