Rollup merge of #68845 - dwrensha:fix-68783, r=estebank

stop using BytePos for computing spans in librustc_parse/parser/mod.rs

Computing spans using logic such as `self.token.span.lo() + BytePos(1)` can cause internal compiler errors like #68730 when non-ascii characters are given as input.

#68735 partially addressed this problem, but only for one case. Moreover, its usage of `next_point()` does not actually align with what `bump_with()` expects. For example, given the token `>>=`, we should pass the span consisting of the final two characters `>=`, but `next_point()` advances the span beyond the end of the `=`.

This pull request instead computes the start of the new span by doing `start_point(self.token.span).hi()`. This matches `self.token.span.lo() + BytePos(1)` in the common case where the characters are ascii, and it gracefully handles multibyte characters.

Fixes #68783.

src/librustc_parse/parser/mod.rs

@@ -21,7 +21,7 @@
 use rustc_session::parse::ParseSess;
 use rustc_span::source_map::respan;
 use rustc_span::symbol::{kw, sym, Symbol};
-use rustc_span::{BytePos, FileName, Span, DUMMY_SP};
+use rustc_span::{FileName, Span, DUMMY_SP};
 use syntax::ast::{self, AttrStyle, AttrVec, CrateSugar, Extern, Ident, Unsafety, DUMMY_NODE_ID};
 use syntax::ast::{IsAsync, MacArgs, MacDelimiter, Mutability, StrLit, Visibility, VisibilityKind};
 use syntax::ptr::P;
@@ -615,8 +615,8 @@ fn eat_plus(&mut self) -> bool {
                 true
             }
             token::BinOpEq(token::Plus) => {
-                let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1));
-                self.bump_with(token::Eq, span);
+                let start_point = self.sess.source_map().start_point(self.token.span);
+                self.bump_with(token::Eq, self.token.span.with_lo(start_point.hi()));
                 true
             }
             _ => false,
@@ -633,8 +633,9 @@ fn expect_and(&mut self) -> PResult<'a, ()> {
                 Ok(())
             }
             token::AndAnd => {
-                let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1));
-                Ok(self.bump_with(token::BinOp(token::And), span))
+                let start_point = self.sess.source_map().start_point(self.token.span);
+                Ok(self
+                    .bump_with(token::BinOp(token::And), self.token.span.with_lo(start_point.hi())))
             }
             _ => self.unexpected(),
         }
@@ -650,8 +651,9 @@ fn expect_or(&mut self) -> PResult<'a, ()> {
                 Ok(())
             }
             token::OrOr => {
-                let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1));
-                Ok(self.bump_with(token::BinOp(token::Or), span))
+                let start_point = self.sess.source_map().start_point(self.token.span);
+                Ok(self
+                    .bump_with(token::BinOp(token::Or), self.token.span.with_lo(start_point.hi())))
             }
             _ => self.unexpected(),
         }
@@ -671,13 +673,16 @@ fn eat_lt(&mut self) -> bool {
                 true
             }
             token::BinOp(token::Shl) => {
-                let span = self.sess.source_map().next_point(self.token.span);
-                self.bump_with(token::Lt, span);
+                let start_point = self.sess.source_map().start_point(self.token.span);
+                self.bump_with(token::Lt, self.token.span.with_lo(start_point.hi()));
                 true
             }
             token::LArrow => {
-                let span = self.sess.source_map().next_point(self.token.span);
-                self.bump_with(token::BinOp(token::Minus), span);
+                let start_point = self.sess.source_map().start_point(self.token.span);
+                self.bump_with(
+                    token::BinOp(token::Minus),
+                    self.token.span.with_lo(start_point.hi()),
+                );
                 true
             }
             _ => false,
@@ -707,16 +712,16 @@ fn expect_gt(&mut self) -> PResult<'a, ()> {
                 Some(())
             }
             token::BinOp(token::Shr) => {
-                let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1));
-                Some(self.bump_with(token::Gt, span))
+                let start_point = self.sess.source_map().start_point(self.token.span);
+                Some(self.bump_with(token::Gt, self.token.span.with_lo(start_point.hi())))
             }
             token::BinOpEq(token::Shr) => {
-                let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1));
-                Some(self.bump_with(token::Ge, span))
+                let start_point = self.sess.source_map().start_point(self.token.span);
+                Some(self.bump_with(token::Ge, self.token.span.with_lo(start_point.hi())))
             }
             token::Ge => {
-                let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1));
-                Some(self.bump_with(token::Eq, span))
+                let start_point = self.sess.source_map().start_point(self.token.span);
+                Some(self.bump_with(token::Eq, self.token.span.with_lo(start_point.hi())))
             }
             _ => None,
         };