mirror of
https://github.com/rust-lang/rust.git
synced 2026-05-22 18:15:07 +03:00
Auto merge of #34575 - cgswords:tstream, r=nrc
Introducing TokenStreams and TokenSlices for procedural macros This pull request introduces TokenStreams and TokenSlices into the compiler in preparation for usage as part of RFC 1566 (procedural macros). r? @nrc
This commit is contained in:
bors
+1121
-39
@@ -8,18 +8,34 @@
|
||||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
//! # Token Trees
|
||||
//! TokenTrees are syntactic forms for dealing with tokens. The description below is
|
||||
//! more complete; in short a TokenTree is a single token, a delimited sequence of token
|
||||
//! trees, or a sequence with repetition for list splicing as part of macro expansion.
|
||||
//! # Token Streams
|
||||
//!
|
||||
//! TokenStreams represent syntactic objects before they are converted into ASTs.
|
||||
//! A `TokenStream` is, roughly speaking, a sequence (eg stream) of `TokenTree`s,
|
||||
//! which are themselves either a single Token, a Delimited subsequence of tokens,
|
||||
//! or a SequenceRepetition specifier (for the purpose of sequence generation during macro
|
||||
//! expansion).
|
||||
//!
|
||||
//! A TokenStream also has a slice view, `TokenSlice`, that is analogous to `str` for
|
||||
//! `String`: it allows the programmer to divvy up, explore, and otherwise partition a
|
||||
//! TokenStream as borrowed subsequences.
|
||||
|
||||
use ast::{AttrStyle};
|
||||
use codemap::{Span};
|
||||
use ast::{self, AttrStyle, LitKind};
|
||||
use syntax_pos::{Span, DUMMY_SP, NO_EXPANSION};
|
||||
use codemap::Spanned;
|
||||
use ext::base;
|
||||
use ext::tt::macro_parser;
|
||||
use parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
|
||||
use parse::lexer;
|
||||
use parse::token;
|
||||
use parse;
|
||||
use parse::token::{self, Token, Lit, InternedString, Nonterminal};
|
||||
use parse::token::Lit as TokLit;
|
||||
|
||||
use std::fmt;
|
||||
use std::mem;
|
||||
use std::ops::Index;
|
||||
use std::ops;
|
||||
use std::iter::*;
|
||||
|
||||
use std::rc::Rc;
|
||||
|
||||
@@ -56,6 +72,11 @@ pub fn open_tt(&self) -> TokenTree {
|
||||
pub fn close_tt(&self) -> TokenTree {
|
||||
TokenTree::Token(self.close_span, self.close_token())
|
||||
}
|
||||
|
||||
/// Returns the token trees inside the delimiters.
|
||||
pub fn subtrees(&self) -> &[TokenTree] {
|
||||
&self.tts
|
||||
}
|
||||
}
|
||||
|
||||
/// A sequence of token trees
|
||||
@@ -91,7 +112,7 @@ pub enum KleeneOp {
|
||||
///
|
||||
/// The RHS of an MBE macro is the only place `SubstNt`s are substituted.
|
||||
/// Nothing special happens to misnamed or misplaced `SubstNt`s.
|
||||
#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash)]
|
||||
pub enum TokenTree {
|
||||
/// A single token
|
||||
Token(Span, token::Token),
|
||||
@@ -99,9 +120,7 @@ pub enum TokenTree {
|
||||
Delimited(Span, Rc<Delimited>),
|
||||
|
||||
// This only makes sense in MBE macros.
|
||||
|
||||
/// A kleene-style repetition sequence with a span
|
||||
// FIXME(eddyb) #12938 Use DST.
|
||||
Sequence(Span, Rc<SequenceRepetition>),
|
||||
}
|
||||
|
||||
@@ -111,28 +130,22 @@ pub fn len(&self) -> usize {
|
||||
TokenTree::Token(_, token::DocComment(name)) => {
|
||||
match doc_comment_style(&name.as_str()) {
|
||||
AttrStyle::Outer => 2,
|
||||
AttrStyle::Inner => 3
|
||||
AttrStyle::Inner => 3,
|
||||
}
|
||||
}
|
||||
TokenTree::Token(_, token::SpecialVarNt(..)) => 2,
|
||||
TokenTree::Token(_, token::MatchNt(..)) => 3,
|
||||
TokenTree::Delimited(_, ref delimed) => {
|
||||
delimed.tts.len() + 2
|
||||
}
|
||||
TokenTree::Sequence(_, ref seq) => {
|
||||
seq.tts.len()
|
||||
}
|
||||
TokenTree::Token(..) => 0
|
||||
TokenTree::Delimited(_, ref delimed) => delimed.tts.len() + 2,
|
||||
TokenTree::Sequence(_, ref seq) => seq.tts.len(),
|
||||
TokenTree::Token(..) => 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_tt(&self, index: usize) -> TokenTree {
|
||||
match (self, index) {
|
||||
(&TokenTree::Token(sp, token::DocComment(_)), 0) => {
|
||||
TokenTree::Token(sp, token::Pound)
|
||||
}
|
||||
(&TokenTree::Token(sp, token::DocComment(_)), 0) => TokenTree::Token(sp, token::Pound),
|
||||
(&TokenTree::Token(sp, token::DocComment(name)), 1)
|
||||
if doc_comment_style(&name.as_str()) == AttrStyle::Inner => {
|
||||
if doc_comment_style(&name.as_str()) == AttrStyle::Inner => {
|
||||
TokenTree::Token(sp, token::Not)
|
||||
}
|
||||
(&TokenTree::Token(sp, token::DocComment(name)), _) => {
|
||||
@@ -140,16 +153,19 @@ pub fn get_tt(&self, index: usize) -> TokenTree {
|
||||
|
||||
// Searches for the occurrences of `"#*` and returns the minimum number of `#`s
|
||||
// required to wrap the text.
|
||||
let num_of_hashes = stripped.chars().scan(0, |cnt, x| {
|
||||
*cnt = if x == '"' {
|
||||
1
|
||||
} else if *cnt != 0 && x == '#' {
|
||||
*cnt + 1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
Some(*cnt)
|
||||
}).max().unwrap_or(0);
|
||||
let num_of_hashes = stripped.chars()
|
||||
.scan(0, |cnt, x| {
|
||||
*cnt = if x == '"' {
|
||||
1
|
||||
} else if *cnt != 0 && x == '#' {
|
||||
*cnt + 1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
Some(*cnt)
|
||||
})
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
|
||||
TokenTree::Delimited(sp, Rc::new(Delimited {
|
||||
delim: token::Bracket,
|
||||
@@ -181,24 +197,24 @@ pub fn get_tt(&self, index: usize) -> TokenTree {
|
||||
TokenTree::Token(sp, token::Ident(kind))];
|
||||
v[index].clone()
|
||||
}
|
||||
(&TokenTree::Sequence(_, ref seq), _) => {
|
||||
seq.tts[index].clone()
|
||||
}
|
||||
_ => panic!("Cannot expand a token tree")
|
||||
(&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(),
|
||||
_ => panic!("Cannot expand a token tree"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the `Span` corresponding to this token tree.
|
||||
pub fn get_span(&self) -> Span {
|
||||
match *self {
|
||||
TokenTree::Token(span, _) => span,
|
||||
TokenTree::Token(span, _) => span,
|
||||
TokenTree::Delimited(span, _) => span,
|
||||
TokenTree::Sequence(span, _) => span,
|
||||
TokenTree::Sequence(span, _) => span,
|
||||
}
|
||||
}
|
||||
|
||||
/// Use this token tree as a matcher to parse given tts.
|
||||
pub fn parse(cx: &base::ExtCtxt, mtch: &[TokenTree], tts: &[TokenTree])
|
||||
pub fn parse(cx: &base::ExtCtxt,
|
||||
mtch: &[TokenTree],
|
||||
tts: &[TokenTree])
|
||||
-> macro_parser::NamedParseResult {
|
||||
// `None` is because we're not interpolating
|
||||
let arg_rdr = lexer::new_tt_reader_with_doc_flag(&cx.parse_sess().span_diagnostic,
|
||||
@@ -208,5 +224,1071 @@ pub fn parse(cx: &base::ExtCtxt, mtch: &[TokenTree], tts: &[TokenTree])
|
||||
true);
|
||||
macro_parser::parse(cx.parse_sess(), cx.cfg(), arg_rdr, mtch)
|
||||
}
|
||||
|
||||
/// Check if this TokenTree is equal to the other, regardless of span information.
|
||||
pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
|
||||
match (self, other) {
|
||||
(&TokenTree::Token(_, ref tk), &TokenTree::Token(_, ref tk2)) => tk == tk2,
|
||||
(&TokenTree::Delimited(_, ref dl), &TokenTree::Delimited(_, ref dl2)) => {
|
||||
(*dl).delim == (*dl2).delim && dl.tts.len() == dl2.tts.len() &&
|
||||
{
|
||||
for (tt1, tt2) in dl.tts.iter().zip(dl2.tts.iter()) {
|
||||
if !tt1.eq_unspanned(tt2) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
(_, _) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Retrieve the TokenTree's span.
|
||||
pub fn span(&self) -> Span {
|
||||
match *self {
|
||||
TokenTree::Token(sp, _) |
|
||||
TokenTree::Delimited(sp, _) |
|
||||
TokenTree::Sequence(sp, _) => sp,
|
||||
}
|
||||
}
|
||||
|
||||
/// Indicates if the stream is a token that is equal to the provided token.
|
||||
pub fn eq_token(&self, t: Token) -> bool {
|
||||
match *self {
|
||||
TokenTree::Token(_, ref tk) => *tk == t,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Indicates if the token is an identifier.
|
||||
pub fn is_ident(&self) -> bool {
|
||||
self.maybe_ident().is_some()
|
||||
}
|
||||
|
||||
/// Returns an identifier.
|
||||
pub fn maybe_ident(&self) -> Option<ast::Ident> {
|
||||
match *self {
|
||||
TokenTree::Token(_, Token::Ident(t)) => Some(t.clone()),
|
||||
TokenTree::Delimited(_, ref dl) => {
|
||||
let tts = dl.subtrees();
|
||||
if tts.len() != 1 {
|
||||
return None;
|
||||
}
|
||||
tts[0].maybe_ident()
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a Token literal.
|
||||
pub fn maybe_lit(&self) -> Option<token::Lit> {
|
||||
match *self {
|
||||
TokenTree::Token(_, Token::Literal(l, _)) => Some(l.clone()),
|
||||
TokenTree::Delimited(_, ref dl) => {
|
||||
let tts = dl.subtrees();
|
||||
if tts.len() != 1 {
|
||||
return None;
|
||||
}
|
||||
tts[0].maybe_lit()
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an AST string literal.
|
||||
pub fn maybe_str(&self) -> Option<ast::Lit> {
|
||||
match *self {
|
||||
TokenTree::Token(sp, Token::Literal(Lit::Str_(s), _)) => {
|
||||
let l = LitKind::Str(token::intern_and_get_ident(&parse::str_lit(&s.as_str())),
|
||||
ast::StrStyle::Cooked);
|
||||
Some(Spanned {
|
||||
node: l,
|
||||
span: sp,
|
||||
})
|
||||
}
|
||||
TokenTree::Token(sp, Token::Literal(Lit::StrRaw(s, n), _)) => {
|
||||
let l = LitKind::Str(token::intern_and_get_ident(&parse::raw_str_lit(&s.as_str())),
|
||||
ast::StrStyle::Raw(n));
|
||||
Some(Spanned {
|
||||
node: l,
|
||||
span: sp,
|
||||
})
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// #Token Streams
|
||||
///
|
||||
/// TokenStreams are a syntactic abstraction over TokenTrees. The goal is for procedural
|
||||
/// macros to work over TokenStreams instead of arbitrary syntax. For now, however, we
|
||||
/// are going to cut a few corners (i.e., use some of the AST structure) when we need to
|
||||
/// for backwards compatibility.
|
||||
|
||||
/// TokenStreams are collections of TokenTrees that represent a syntactic structure. The
|
||||
/// struct itself shouldn't be directly manipulated; the internal structure is not stable,
|
||||
/// and may be changed at any time in the future. The operators will not, however (except
|
||||
/// for signatures, later on).
|
||||
#[derive(Eq,Clone,Hash,RustcEncodable,RustcDecodable)]
|
||||
pub struct TokenStream {
|
||||
pub span: Span,
|
||||
pub tts: Vec<TokenTree>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for TokenStream {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
if self.tts.len() == 0 {
|
||||
write!(f, "([empty")?;
|
||||
} else {
|
||||
write!(f, "([")?;
|
||||
write!(f, "{:?}", self.tts[0])?;
|
||||
|
||||
for tt in self.tts.iter().skip(1) {
|
||||
write!(f, ",{:?}", tt)?;
|
||||
}
|
||||
}
|
||||
write!(f, "|")?;
|
||||
self.span.fmt(f)?;
|
||||
write!(f, "])")
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks if two TokenStreams are equivalent (including spans). For unspanned
|
||||
/// equality, see `eq_unspanned`.
|
||||
impl PartialEq<TokenStream> for TokenStream {
|
||||
fn eq(&self, other: &TokenStream) -> bool {
|
||||
self.tts == other.tts
|
||||
}
|
||||
}
|
||||
|
||||
// NB this will disregard gaps. if we have [a|{2,5} , b|{11,13}], the resultant span
|
||||
// will be at {2,13}. Without finer-grained span structures, however, this seems to be
|
||||
// our only recourse.
|
||||
// FIXME Do something smarter to compute the expansion id.
|
||||
fn covering_span(trees: &[TokenTree]) -> Span {
|
||||
// disregard any dummy spans we have
|
||||
let trees = trees.iter().filter(|t| t.span() != DUMMY_SP).collect::<Vec<&TokenTree>>();
|
||||
|
||||
// if we're out of spans, stop
|
||||
if trees.len() < 1 {
|
||||
return DUMMY_SP;
|
||||
}
|
||||
|
||||
// set up the initial values
|
||||
let fst_span = trees[0].span();
|
||||
|
||||
let mut lo_span = fst_span.lo;
|
||||
let mut hi_span = fst_span.hi;
|
||||
let mut expn_id = fst_span.expn_id;
|
||||
|
||||
// compute the spans iteratively
|
||||
for t in trees.iter().skip(1) {
|
||||
let sp = t.span();
|
||||
if sp.lo < lo_span {
|
||||
lo_span = sp.lo;
|
||||
}
|
||||
if hi_span < sp.hi {
|
||||
hi_span = sp.hi;
|
||||
}
|
||||
if expn_id != sp.expn_id {
|
||||
expn_id = NO_EXPANSION;
|
||||
}
|
||||
}
|
||||
|
||||
Span {
|
||||
lo: lo_span,
|
||||
hi: hi_span,
|
||||
expn_id: expn_id,
|
||||
}
|
||||
}
|
||||
|
||||
/// TokenStream operators include basic destructuring, boolean operations, `maybe_...`
|
||||
/// operations, and `maybe_..._prefix` operations. Boolean operations are straightforward,
|
||||
/// indicating information about the structure of the stream. The `maybe_...` operations
|
||||
/// return `Some<...>` if the tokenstream contains the appropriate item.
|
||||
///
|
||||
/// Similarly, the `maybe_..._prefix` operations potentially return a
|
||||
/// partially-destructured stream as a pair where the first element is the expected item
|
||||
/// and the second is the remainder of the stream. As anb example,
|
||||
///
|
||||
/// `maybe_path_prefix("a::b::c(a,b,c).foo()") -> (a::b::c, "(a,b,c).foo()")`
|
||||
impl TokenStream {
|
||||
/// Convert a vector of `TokenTree`s into a `TokenStream`.
|
||||
pub fn from_tts(trees: Vec<TokenTree>) -> TokenStream {
|
||||
let span = covering_span(&trees);
|
||||
TokenStream {
|
||||
tts: trees,
|
||||
span: span,
|
||||
}
|
||||
}
|
||||
|
||||
/// Copies all of the TokenTrees from the TokenSlice, appending them to the stream.
|
||||
pub fn append_stream(mut self, ts2: &TokenSlice) {
|
||||
for tt in ts2.iter() {
|
||||
self.tts.push(tt.clone());
|
||||
}
|
||||
self.span = covering_span(&self.tts[..]);
|
||||
}
|
||||
|
||||
/// Manually change a TokenStream's span.
|
||||
pub fn respan(self, span: Span) -> TokenStream {
|
||||
TokenStream {
|
||||
tts: self.tts,
|
||||
span: span,
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct a TokenStream from an ast literal.
|
||||
pub fn from_ast_lit_str(lit: ast::Lit) -> Option<TokenStream> {
|
||||
match lit.node {
|
||||
LitKind::Str(val, _) => {
|
||||
let val = TokLit::Str_(token::intern(&val));
|
||||
Some(TokenStream::from_tts(vec![TokenTree::Token(lit.span,
|
||||
Token::Literal(val, None))]))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/// Convert a vector of TokenTrees into a parentheses-delimited TokenStream.
|
||||
pub fn as_paren_delimited_stream(tts: Vec<TokenTree>) -> TokenStream {
|
||||
let new_sp = covering_span(&tts);
|
||||
|
||||
let new_delim = Rc::new(Delimited {
|
||||
delim: token::DelimToken::Paren,
|
||||
open_span: DUMMY_SP,
|
||||
tts: tts,
|
||||
close_span: DUMMY_SP,
|
||||
});
|
||||
|
||||
TokenStream::from_tts(vec![TokenTree::Delimited(new_sp, new_delim)])
|
||||
}
|
||||
|
||||
/// Convert an interned string into a one-element TokenStream.
|
||||
pub fn from_interned_string_as_ident(s: InternedString) -> TokenStream {
|
||||
TokenStream::from_tts(vec![TokenTree::Token(DUMMY_SP,
|
||||
Token::Ident(token::str_to_ident(&s[..])))])
|
||||
}
|
||||
}
|
||||
|
||||
/// TokenSlices are 'views' of `TokenStream's; they fit the same role as `str`s do for
|
||||
/// `String`s. In general, most TokenStream manipulations will be refocusing their internal
|
||||
/// contents by taking a TokenSlice and then using indexing and the provided operators.
|
||||
#[derive(PartialEq, Eq, Debug)]
|
||||
pub struct TokenSlice([TokenTree]);
|
||||
|
||||
impl ops::Deref for TokenStream {
|
||||
type Target = TokenSlice;
|
||||
|
||||
fn deref(&self) -> &TokenSlice {
|
||||
let tts: &[TokenTree] = &*self.tts;
|
||||
unsafe { mem::transmute(tts) }
|
||||
}
|
||||
}
|
||||
|
||||
impl TokenSlice {
|
||||
/// Convert a borrowed TokenTree slice into a borrowed TokenSlice.
|
||||
fn from_tts(tts: &[TokenTree]) -> &TokenSlice {
|
||||
unsafe { mem::transmute(tts) }
|
||||
}
|
||||
|
||||
/// Indicates whether the `TokenStream` is empty.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
|
||||
/// Return the `TokenSlice`'s length.
|
||||
pub fn len(&self) -> usize {
|
||||
self.0.len()
|
||||
}
|
||||
|
||||
/// Check equality versus another TokenStream, ignoring span information.
|
||||
pub fn eq_unspanned(&self, other: &TokenSlice) -> bool {
|
||||
if self.len() != other.len() {
|
||||
return false;
|
||||
}
|
||||
for (tt1, tt2) in self.iter().zip(other.iter()) {
|
||||
if !tt1.eq_unspanned(tt2) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Compute a span that covers the entire TokenSlice (eg, one wide enough to include
|
||||
/// the entire slice). If the inputs share expansion identification, it is preserved.
|
||||
/// If they do not, it is discarded.
|
||||
pub fn covering_span(&self) -> Span {
|
||||
covering_span(&self.0)
|
||||
}
|
||||
|
||||
/// Indicates where the stream is of the form `= <ts>`, where `<ts>` is a continued
|
||||
/// `TokenStream`.
|
||||
pub fn is_assignment(&self) -> bool {
|
||||
self.maybe_assignment().is_some()
|
||||
}
|
||||
|
||||
/// Returns the RHS of an assigment.
|
||||
pub fn maybe_assignment(&self) -> Option<&TokenSlice> {
|
||||
if !(self.len() > 1) {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(&self[1..])
|
||||
}
|
||||
|
||||
/// Indicates where the stream is a single, delimited expression (e.g., `(a,b,c)` or
|
||||
/// `{a,b,c}`).
|
||||
pub fn is_delimited(&self) -> bool {
|
||||
self.maybe_delimited().is_some()
|
||||
}
|
||||
|
||||
/// Returns the inside of the delimited term as a new TokenStream.
|
||||
pub fn maybe_delimited(&self) -> Option<&TokenSlice> {
|
||||
if !(self.len() == 1) {
|
||||
return None;
|
||||
}
|
||||
|
||||
match self[0] {
|
||||
TokenTree::Delimited(_, ref rc) => Some(TokenSlice::from_tts(&*rc.tts)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a list of `TokenSlice`s if the stream is a delimited list, breaking the
|
||||
/// stream on commas.
|
||||
pub fn maybe_comma_list(&self) -> Option<Vec<&TokenSlice>> {
|
||||
let maybe_tts = self.maybe_delimited();
|
||||
|
||||
let ts: &TokenSlice;
|
||||
match maybe_tts {
|
||||
Some(t) => {
|
||||
ts = t;
|
||||
}
|
||||
None => {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
||||
let splits: Vec<&TokenSlice> = ts.split(|x| match *x {
|
||||
TokenTree::Token(_, Token::Comma) => true,
|
||||
_ => false,
|
||||
})
|
||||
.filter(|x| x.len() > 0)
|
||||
.collect();
|
||||
|
||||
Some(splits)
|
||||
}
|
||||
|
||||
/// Returns a Nonterminal if it is Interpolated.
|
||||
pub fn maybe_interpolated_nonterminal(&self) -> Option<Nonterminal> {
|
||||
if !(self.len() == 1) {
|
||||
return None;
|
||||
}
|
||||
|
||||
match self[0] {
|
||||
TokenTree::Token(_, Token::Interpolated(ref nt)) => Some(nt.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Indicates if the stream is exactly one identifier.
|
||||
pub fn is_ident(&self) -> bool {
|
||||
self.maybe_ident().is_some()
|
||||
}
|
||||
|
||||
/// Returns an identifier
|
||||
pub fn maybe_ident(&self) -> Option<ast::Ident> {
|
||||
if !(self.len() == 1) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let tok = if let Some(tts) = self.maybe_delimited() {
|
||||
if tts.len() != 1 {
|
||||
return None;
|
||||
}
|
||||
&tts[0]
|
||||
} else {
|
||||
&self[0]
|
||||
};
|
||||
|
||||
match *tok {
|
||||
TokenTree::Token(_, Token::Ident(t)) => Some(t),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Indicates if the stream is exactly one literal
|
||||
pub fn is_lit(&self) -> bool {
|
||||
self.maybe_lit().is_some()
|
||||
}
|
||||
|
||||
/// Returns a literal
|
||||
pub fn maybe_lit(&self) -> Option<token::Lit> {
|
||||
if !(self.len() == 1) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let tok = if let Some(tts) = self.maybe_delimited() {
|
||||
if tts.len() != 1 {
|
||||
return None;
|
||||
}
|
||||
&tts[0]
|
||||
} else {
|
||||
&self[0]
|
||||
};
|
||||
|
||||
match *tok {
|
||||
TokenTree::Token(_, Token::Literal(l, _)) => Some(l),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an AST string literal if the TokenStream is either a normal ('cooked') or
|
||||
/// raw string literal.
|
||||
pub fn maybe_str(&self) -> Option<ast::Lit> {
|
||||
if !(self.len() == 1) {
|
||||
return None;
|
||||
}
|
||||
|
||||
match self[0] {
|
||||
TokenTree::Token(sp, Token::Literal(Lit::Str_(s), _)) => {
|
||||
let l = LitKind::Str(token::intern_and_get_ident(&parse::str_lit(&s.as_str())),
|
||||
ast::StrStyle::Cooked);
|
||||
Some(Spanned {
|
||||
node: l,
|
||||
span: sp,
|
||||
})
|
||||
}
|
||||
TokenTree::Token(sp, Token::Literal(Lit::StrRaw(s, n), _)) => {
|
||||
let l = LitKind::Str(token::intern_and_get_ident(&parse::raw_str_lit(&s.as_str())),
|
||||
ast::StrStyle::Raw(n));
|
||||
Some(Spanned {
|
||||
node: l,
|
||||
span: sp,
|
||||
})
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// This operation extracts the path prefix , returning an AST path struct and the remainder
|
||||
/// of the stream (if it finds one). To be more specific, a tokenstream that has a valid,
|
||||
/// non-global path as a prefix (eg `foo(bar, baz)`, `foo::bar(bar)`, but *not*
|
||||
/// `::foo::bar(baz)`) will yield the path and the remaining tokens (as a slice). The previous
|
||||
/// examples will yield
|
||||
/// `Some((Path { segments = vec![foo], ... }, [(bar, baz)]))`,
|
||||
/// `Some((Path { segments = vec![foo, bar] }, [(baz)]))`,
|
||||
/// and `None`, respectively.
|
||||
pub fn maybe_path_prefix(&self) -> Option<(ast::Path, &TokenSlice)> {
|
||||
let mut segments: Vec<ast::PathSegment> = Vec::new();
|
||||
|
||||
let path: Vec<&TokenTree> = self.iter()
|
||||
.take_while(|x| x.is_ident() || x.eq_token(Token::ModSep))
|
||||
.collect::<Vec<&TokenTree>>();
|
||||
|
||||
let path_size = path.len();
|
||||
if path_size == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let cov_span = self[..path_size].covering_span();
|
||||
let rst = &self[path_size..];
|
||||
|
||||
let fst_id = path[0];
|
||||
|
||||
if let Some(id) = fst_id.maybe_ident() {
|
||||
segments.push(ast::PathSegment {
|
||||
identifier: id,
|
||||
parameters: ast::PathParameters::none(),
|
||||
});
|
||||
} else {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Let's use a state machine to parse out the rest.
|
||||
enum State {
|
||||
Mod, // Expect a `::`, or return None otherwise.
|
||||
Ident, // Expect an ident, or return None otherwise.
|
||||
}
|
||||
let mut state = State::Mod;
|
||||
|
||||
for p in &path[1..] {
|
||||
match state {
|
||||
State::Mod => {
|
||||
// State 0: ['::' -> state 1, else return None]
|
||||
if p.eq_token(Token::ModSep) {
|
||||
state = State::Ident;
|
||||
} else {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
State::Ident => {
|
||||
// State 1: [ident -> state 0, else return None]
|
||||
if let Some(id) = p.maybe_ident() {
|
||||
segments.push(ast::PathSegment {
|
||||
identifier: id,
|
||||
parameters: ast::PathParameters::none(),
|
||||
});
|
||||
state = State::Mod;
|
||||
} else {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let path = ast::Path {
|
||||
span: cov_span,
|
||||
global: false,
|
||||
segments: segments,
|
||||
};
|
||||
Some((path, rst))
|
||||
}
|
||||
|
||||
/// Returns an iterator over a TokenSlice (as a sequence of TokenStreams).
|
||||
fn iter(&self) -> Iter {
|
||||
Iter { vs: self }
|
||||
}
|
||||
|
||||
/// Splits a TokenSlice based on the provided `&TokenTree -> bool` predicate.
|
||||
fn split<P>(&self, pred: P) -> Split<P>
|
||||
where P: FnMut(&TokenTree) -> bool
|
||||
{
|
||||
Split {
|
||||
vs: self,
|
||||
pred: pred,
|
||||
finished: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Iter<'a> {
|
||||
vs: &'a TokenSlice,
|
||||
}
|
||||
|
||||
impl<'a> Iterator for Iter<'a> {
|
||||
type Item = &'a TokenTree;
|
||||
|
||||
fn next(&mut self) -> Option<&'a TokenTree> {
|
||||
if self.vs.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let ret = Some(&self.vs[0]);
|
||||
self.vs = &self.vs[1..];
|
||||
ret
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Split<'a, P>
|
||||
where P: FnMut(&TokenTree) -> bool
|
||||
{
|
||||
vs: &'a TokenSlice,
|
||||
pred: P,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
impl<'a, P> Iterator for Split<'a, P>
|
||||
where P: FnMut(&TokenTree) -> bool
|
||||
{
|
||||
type Item = &'a TokenSlice;
|
||||
|
||||
fn next(&mut self) -> Option<&'a TokenSlice> {
|
||||
if self.finished {
|
||||
return None;
|
||||
}
|
||||
|
||||
match self.vs.iter().position(|x| (self.pred)(x)) {
|
||||
None => {
|
||||
self.finished = true;
|
||||
Some(&self.vs[..])
|
||||
}
|
||||
Some(idx) => {
|
||||
let ret = Some(&self.vs[..idx]);
|
||||
self.vs = &self.vs[idx + 1..];
|
||||
ret
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Index<usize> for TokenStream {
|
||||
type Output = TokenTree;
|
||||
|
||||
fn index(&self, index: usize) -> &TokenTree {
|
||||
Index::index(&**self, index)
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::Index<ops::Range<usize>> for TokenStream {
|
||||
type Output = TokenSlice;
|
||||
|
||||
fn index(&self, index: ops::Range<usize>) -> &TokenSlice {
|
||||
Index::index(&**self, index)
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::Index<ops::RangeTo<usize>> for TokenStream {
|
||||
type Output = TokenSlice;
|
||||
|
||||
fn index(&self, index: ops::RangeTo<usize>) -> &TokenSlice {
|
||||
Index::index(&**self, index)
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::Index<ops::RangeFrom<usize>> for TokenStream {
|
||||
type Output = TokenSlice;
|
||||
|
||||
fn index(&self, index: ops::RangeFrom<usize>) -> &TokenSlice {
|
||||
Index::index(&**self, index)
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::Index<ops::RangeFull> for TokenStream {
|
||||
type Output = TokenSlice;
|
||||
|
||||
fn index(&self, _index: ops::RangeFull) -> &TokenSlice {
|
||||
Index::index(&**self, _index)
|
||||
}
|
||||
}
|
||||
|
||||
impl Index<usize> for TokenSlice {
|
||||
type Output = TokenTree;
|
||||
|
||||
fn index(&self, index: usize) -> &TokenTree {
|
||||
&self.0[index]
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::Index<ops::Range<usize>> for TokenSlice {
|
||||
type Output = TokenSlice;
|
||||
|
||||
fn index(&self, index: ops::Range<usize>) -> &TokenSlice {
|
||||
TokenSlice::from_tts(&self.0[index])
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::Index<ops::RangeTo<usize>> for TokenSlice {
|
||||
type Output = TokenSlice;
|
||||
|
||||
fn index(&self, index: ops::RangeTo<usize>) -> &TokenSlice {
|
||||
TokenSlice::from_tts(&self.0[index])
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::Index<ops::RangeFrom<usize>> for TokenSlice {
|
||||
type Output = TokenSlice;
|
||||
|
||||
fn index(&self, index: ops::RangeFrom<usize>) -> &TokenSlice {
|
||||
TokenSlice::from_tts(&self.0[index])
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::Index<ops::RangeFull> for TokenSlice {
|
||||
type Output = TokenSlice;
|
||||
|
||||
fn index(&self, _index: ops::RangeFull) -> &TokenSlice {
|
||||
TokenSlice::from_tts(&self.0[_index])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use ast;
|
||||
use syntax_pos::{Span, BytePos, NO_EXPANSION, DUMMY_SP};
|
||||
use parse::token::{self, str_to_ident, Token, Lit};
|
||||
use util::parser_testing::string_to_tts;
|
||||
use std::rc::Rc;
|
||||
|
||||
fn sp(a: u32, b: u32) -> Span {
|
||||
Span {
|
||||
lo: BytePos(a),
|
||||
hi: BytePos(b),
|
||||
expn_id: NO_EXPANSION,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_empty() {
|
||||
let test0 = TokenStream::from_tts(Vec::new());
|
||||
let test1 = TokenStream::from_tts(vec![TokenTree::Token(sp(0, 1),
|
||||
Token::Ident(str_to_ident("a")))]);
|
||||
let test2 = TokenStream::from_tts(string_to_tts("foo(bar::baz)".to_string()));
|
||||
|
||||
assert_eq!(test0.is_empty(), true);
|
||||
assert_eq!(test1.is_empty(), false);
|
||||
assert_eq!(test2.is_empty(), false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_delimited() {
|
||||
let test0 = TokenStream::from_tts(string_to_tts("foo(bar::baz)".to_string()));
|
||||
let test1 = TokenStream::from_tts(string_to_tts("(bar::baz)".to_string()));
|
||||
let test2 = TokenStream::from_tts(string_to_tts("(foo,bar,baz)".to_string()));
|
||||
let test3 = TokenStream::from_tts(string_to_tts("(foo,bar,baz)(zab,rab,oof)".to_string()));
|
||||
let test4 = TokenStream::from_tts(string_to_tts("(foo,bar,baz)foo".to_string()));
|
||||
let test5 = TokenStream::from_tts(string_to_tts("".to_string()));
|
||||
|
||||
assert_eq!(test0.is_delimited(), false);
|
||||
assert_eq!(test1.is_delimited(), true);
|
||||
assert_eq!(test2.is_delimited(), true);
|
||||
assert_eq!(test3.is_delimited(), false);
|
||||
assert_eq!(test4.is_delimited(), false);
|
||||
assert_eq!(test5.is_delimited(), false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_assign() {
|
||||
let test0 = TokenStream::from_tts(string_to_tts("= bar::baz".to_string()));
|
||||
let test1 = TokenStream::from_tts(string_to_tts("= \"5\"".to_string()));
|
||||
let test2 = TokenStream::from_tts(string_to_tts("= 5".to_string()));
|
||||
let test3 = TokenStream::from_tts(string_to_tts("(foo = 10)".to_string()));
|
||||
let test4 = TokenStream::from_tts(string_to_tts("= (foo,bar,baz)".to_string()));
|
||||
let test5 = TokenStream::from_tts(string_to_tts("".to_string()));
|
||||
|
||||
assert_eq!(test0.is_assignment(), true);
|
||||
assert_eq!(test1.is_assignment(), true);
|
||||
assert_eq!(test2.is_assignment(), true);
|
||||
assert_eq!(test3.is_assignment(), false);
|
||||
assert_eq!(test4.is_assignment(), true);
|
||||
assert_eq!(test5.is_assignment(), false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_lit() {
|
||||
let test0 = TokenStream::from_tts(string_to_tts("\"foo\"".to_string()));
|
||||
let test1 = TokenStream::from_tts(string_to_tts("5".to_string()));
|
||||
let test2 = TokenStream::from_tts(string_to_tts("foo".to_string()));
|
||||
let test3 = TokenStream::from_tts(string_to_tts("foo::bar".to_string()));
|
||||
let test4 = TokenStream::from_tts(string_to_tts("foo(bar)".to_string()));
|
||||
|
||||
assert_eq!(test0.is_lit(), true);
|
||||
assert_eq!(test1.is_lit(), true);
|
||||
assert_eq!(test2.is_lit(), false);
|
||||
assert_eq!(test3.is_lit(), false);
|
||||
assert_eq!(test4.is_lit(), false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_ident() {
|
||||
let test0 = TokenStream::from_tts(string_to_tts("\"foo\"".to_string()));
|
||||
let test1 = TokenStream::from_tts(string_to_tts("5".to_string()));
|
||||
let test2 = TokenStream::from_tts(string_to_tts("foo".to_string()));
|
||||
let test3 = TokenStream::from_tts(string_to_tts("foo::bar".to_string()));
|
||||
let test4 = TokenStream::from_tts(string_to_tts("foo(bar)".to_string()));
|
||||
|
||||
assert_eq!(test0.is_ident(), false);
|
||||
assert_eq!(test1.is_ident(), false);
|
||||
assert_eq!(test2.is_ident(), true);
|
||||
assert_eq!(test3.is_ident(), false);
|
||||
assert_eq!(test4.is_ident(), false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_maybe_assignment() {
|
||||
let test0_input = TokenStream::from_tts(string_to_tts("= bar::baz".to_string()));
|
||||
let test1_input = TokenStream::from_tts(string_to_tts("= \"5\"".to_string()));
|
||||
let test2_input = TokenStream::from_tts(string_to_tts("= 5".to_string()));
|
||||
let test3_input = TokenStream::from_tts(string_to_tts("(foo = 10)".to_string()));
|
||||
let test4_input = TokenStream::from_tts(string_to_tts("= (foo,bar,baz)".to_string()));
|
||||
let test5_input = TokenStream::from_tts(string_to_tts("".to_string()));
|
||||
|
||||
let test0 = test0_input.maybe_assignment();
|
||||
let test1 = test1_input.maybe_assignment();
|
||||
let test2 = test2_input.maybe_assignment();
|
||||
let test3 = test3_input.maybe_assignment();
|
||||
let test4 = test4_input.maybe_assignment();
|
||||
let test5 = test5_input.maybe_assignment();
|
||||
|
||||
let test0_expected = TokenStream::from_tts(vec![TokenTree::Token(sp(2, 5),
|
||||
token::Ident(str_to_ident("bar"))),
|
||||
TokenTree::Token(sp(5, 7), token::ModSep),
|
||||
TokenTree::Token(sp(7, 10),
|
||||
token::Ident(str_to_ident("baz")))]);
|
||||
assert_eq!(test0, Some(&test0_expected[..]));
|
||||
|
||||
let test1_expected = TokenStream::from_tts(vec![TokenTree::Token(sp(2, 5),
|
||||
token::Literal(Lit::Str_(token::intern("5")), None))]);
|
||||
assert_eq!(test1, Some(&test1_expected[..]));
|
||||
|
||||
let test2_expected = TokenStream::from_tts(vec![TokenTree::Token( sp(2,3)
|
||||
, token::Literal(
|
||||
Lit::Integer(
|
||||
token::intern(&(5.to_string()))),
|
||||
None))]);
|
||||
assert_eq!(test2, Some(&test2_expected[..]));
|
||||
|
||||
assert_eq!(test3, None);
|
||||
|
||||
|
||||
let test4_tts = vec![TokenTree::Token(sp(3, 6), token::Ident(str_to_ident("foo"))),
|
||||
TokenTree::Token(sp(6, 7), token::Comma),
|
||||
TokenTree::Token(sp(7, 10), token::Ident(str_to_ident("bar"))),
|
||||
TokenTree::Token(sp(10, 11), token::Comma),
|
||||
TokenTree::Token(sp(11, 14), token::Ident(str_to_ident("baz")))];
|
||||
|
||||
let test4_expected = TokenStream::from_tts(vec![TokenTree::Delimited(sp(2, 15),
|
||||
Rc::new(Delimited {
|
||||
delim: token::DelimToken::Paren,
|
||||
open_span: sp(2, 3),
|
||||
tts: test4_tts,
|
||||
close_span: sp(14, 15),
|
||||
}))]);
|
||||
assert_eq!(test4, Some(&test4_expected[..]));
|
||||
|
||||
assert_eq!(test5, None);
|
||||
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_maybe_delimited() {
|
||||
let test0_input = TokenStream::from_tts(string_to_tts("foo(bar::baz)".to_string()));
|
||||
let test1_input = TokenStream::from_tts(string_to_tts("(bar::baz)".to_string()));
|
||||
let test2_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)".to_string()));
|
||||
let test3_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)(zab,rab)"
|
||||
.to_string()));
|
||||
let test4_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)foo".to_string()));
|
||||
let test5_input = TokenStream::from_tts(string_to_tts("".to_string()));
|
||||
|
||||
let test0 = test0_input.maybe_delimited();
|
||||
let test1 = test1_input.maybe_delimited();
|
||||
let test2 = test2_input.maybe_delimited();
|
||||
let test3 = test3_input.maybe_delimited();
|
||||
let test4 = test4_input.maybe_delimited();
|
||||
let test5 = test5_input.maybe_delimited();
|
||||
|
||||
assert_eq!(test0, None);
|
||||
|
||||
let test1_expected = TokenStream::from_tts(vec![TokenTree::Token(sp(1, 4),
|
||||
token::Ident(str_to_ident("bar"))),
|
||||
TokenTree::Token(sp(4, 6), token::ModSep),
|
||||
TokenTree::Token(sp(6, 9),
|
||||
token::Ident(str_to_ident("baz")))]);
|
||||
assert_eq!(test1, Some(&test1_expected[..]));
|
||||
|
||||
let test2_expected = TokenStream::from_tts(vec![TokenTree::Token(sp(1, 4),
|
||||
token::Ident(str_to_ident("foo"))),
|
||||
TokenTree::Token(sp(4, 5), token::Comma),
|
||||
TokenTree::Token(sp(5, 8),
|
||||
token::Ident(str_to_ident("bar"))),
|
||||
TokenTree::Token(sp(8, 9), token::Comma),
|
||||
TokenTree::Token(sp(9, 12),
|
||||
token::Ident(str_to_ident("baz")))]);
|
||||
assert_eq!(test2, Some(&test2_expected[..]));
|
||||
|
||||
assert_eq!(test3, None);
|
||||
|
||||
assert_eq!(test4, None);
|
||||
|
||||
assert_eq!(test5, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_maybe_comma_list() {
|
||||
let test0_input = TokenStream::from_tts(string_to_tts("foo(bar::baz)".to_string()));
|
||||
let test1_input = TokenStream::from_tts(string_to_tts("(bar::baz)".to_string()));
|
||||
let test2_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)".to_string()));
|
||||
let test3_input = TokenStream::from_tts(string_to_tts("(foo::bar,bar,baz)".to_string()));
|
||||
let test4_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)(zab,rab)"
|
||||
.to_string()));
|
||||
let test5_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)foo".to_string()));
|
||||
let test6_input = TokenStream::from_tts(string_to_tts("".to_string()));
|
||||
// The following is supported behavior!
|
||||
let test7_input = TokenStream::from_tts(string_to_tts("(foo,bar,)".to_string()));
|
||||
|
||||
let test0 = test0_input.maybe_comma_list();
|
||||
let test1 = test1_input.maybe_comma_list();
|
||||
let test2 = test2_input.maybe_comma_list();
|
||||
let test3 = test3_input.maybe_comma_list();
|
||||
let test4 = test4_input.maybe_comma_list();
|
||||
let test5 = test5_input.maybe_comma_list();
|
||||
let test6 = test6_input.maybe_comma_list();
|
||||
let test7 = test7_input.maybe_comma_list();
|
||||
|
||||
assert_eq!(test0, None);
|
||||
|
||||
let test1_stream = TokenStream::from_tts(vec![TokenTree::Token(sp(1, 4),
|
||||
token::Ident(str_to_ident("bar"))),
|
||||
TokenTree::Token(sp(4, 6), token::ModSep),
|
||||
TokenTree::Token(sp(6, 9),
|
||||
token::Ident(str_to_ident("baz")))]);
|
||||
|
||||
let test1_expected: Vec<&TokenSlice> = vec![&test1_stream[..]];
|
||||
assert_eq!(test1, Some(test1_expected));
|
||||
|
||||
let test2_foo = TokenStream::from_tts(vec![TokenTree::Token(sp(1, 4),
|
||||
token::Ident(str_to_ident("foo")))]);
|
||||
let test2_bar = TokenStream::from_tts(vec![TokenTree::Token(sp(5, 8),
|
||||
token::Ident(str_to_ident("bar")))]);
|
||||
let test2_baz = TokenStream::from_tts(vec![TokenTree::Token(sp(9, 12),
|
||||
token::Ident(str_to_ident("baz")))]);
|
||||
let test2_expected: Vec<&TokenSlice> = vec![&test2_foo[..], &test2_bar[..], &test2_baz[..]];
|
||||
assert_eq!(test2, Some(test2_expected));
|
||||
|
||||
let test3_path = TokenStream::from_tts(vec![TokenTree::Token(sp(1, 4),
|
||||
token::Ident(str_to_ident("foo"))),
|
||||
TokenTree::Token(sp(4, 6), token::ModSep),
|
||||
TokenTree::Token(sp(6, 9),
|
||||
token::Ident(str_to_ident("bar")))]);
|
||||
let test3_bar = TokenStream::from_tts(vec![TokenTree::Token(sp(10, 13),
|
||||
token::Ident(str_to_ident("bar")))]);
|
||||
let test3_baz = TokenStream::from_tts(vec![TokenTree::Token(sp(14, 17),
|
||||
token::Ident(str_to_ident("baz")))]);
|
||||
let test3_expected: Vec<&TokenSlice> =
|
||||
vec![&test3_path[..], &test3_bar[..], &test3_baz[..]];
|
||||
assert_eq!(test3, Some(test3_expected));
|
||||
|
||||
assert_eq!(test4, None);
|
||||
|
||||
assert_eq!(test5, None);
|
||||
|
||||
assert_eq!(test6, None);
|
||||
|
||||
|
||||
let test7_expected: Vec<&TokenSlice> = vec![&test2_foo[..], &test2_bar[..]];
|
||||
assert_eq!(test7, Some(test7_expected));
|
||||
}
|
||||
|
||||
// pub fn maybe_ident(&self) -> Option<ast::Ident>
|
||||
#[test]
|
||||
fn test_maybe_ident() {
|
||||
let test0 = TokenStream::from_tts(string_to_tts("\"foo\"".to_string())).maybe_ident();
|
||||
let test1 = TokenStream::from_tts(string_to_tts("5".to_string())).maybe_ident();
|
||||
let test2 = TokenStream::from_tts(string_to_tts("foo".to_string())).maybe_ident();
|
||||
let test3 = TokenStream::from_tts(string_to_tts("foo::bar".to_string())).maybe_ident();
|
||||
let test4 = TokenStream::from_tts(string_to_tts("foo(bar)".to_string())).maybe_ident();
|
||||
|
||||
assert_eq!(test0, None);
|
||||
assert_eq!(test1, None);
|
||||
assert_eq!(test2, Some(str_to_ident("foo")));
|
||||
assert_eq!(test3, None);
|
||||
assert_eq!(test4, None);
|
||||
}
|
||||
|
||||
// pub fn maybe_lit(&self) -> Option<token::Lit>
|
||||
#[test]
|
||||
fn test_maybe_lit() {
|
||||
let test0 = TokenStream::from_tts(string_to_tts("\"foo\"".to_string())).maybe_lit();
|
||||
let test1 = TokenStream::from_tts(string_to_tts("5".to_string())).maybe_lit();
|
||||
let test2 = TokenStream::from_tts(string_to_tts("foo".to_string())).maybe_lit();
|
||||
let test3 = TokenStream::from_tts(string_to_tts("foo::bar".to_string())).maybe_lit();
|
||||
let test4 = TokenStream::from_tts(string_to_tts("foo(bar)".to_string())).maybe_lit();
|
||||
|
||||
assert_eq!(test0, Some(Lit::Str_(token::intern("foo"))));
|
||||
assert_eq!(test1, Some(Lit::Integer(token::intern(&(5.to_string())))));
|
||||
assert_eq!(test2, None);
|
||||
assert_eq!(test3, None);
|
||||
assert_eq!(test4, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_maybe_path_prefix() {
|
||||
let test0_input = TokenStream::from_tts(string_to_tts("foo(bar::baz)".to_string()));
|
||||
let test1_input = TokenStream::from_tts(string_to_tts("(bar::baz)".to_string()));
|
||||
let test2_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)".to_string()));
|
||||
let test3_input = TokenStream::from_tts(string_to_tts("foo::bar(bar,baz)".to_string()));
|
||||
|
||||
let test0 = test0_input.maybe_path_prefix();
|
||||
let test1 = test1_input.maybe_path_prefix();
|
||||
let test2 = test2_input.maybe_path_prefix();
|
||||
let test3 = test3_input.maybe_path_prefix();
|
||||
|
||||
let test0_tts = vec![TokenTree::Token(sp(4, 7), token::Ident(str_to_ident("bar"))),
|
||||
TokenTree::Token(sp(7, 9), token::ModSep),
|
||||
TokenTree::Token(sp(9, 12), token::Ident(str_to_ident("baz")))];
|
||||
|
||||
let test0_stream = TokenStream::from_tts(vec![TokenTree::Delimited(sp(3, 13),
|
||||
Rc::new(Delimited {
|
||||
delim: token::DelimToken::Paren,
|
||||
open_span: sp(3, 4),
|
||||
tts: test0_tts,
|
||||
close_span: sp(12, 13),
|
||||
}))]);
|
||||
|
||||
let test0_expected = Some((ast::Path::from_ident(sp(0, 3), str_to_ident("foo")),
|
||||
&test0_stream[..]));
|
||||
assert_eq!(test0, test0_expected);
|
||||
|
||||
assert_eq!(test1, None);
|
||||
assert_eq!(test2, None);
|
||||
|
||||
let test3_path = ast::Path {
|
||||
span: sp(0, 8),
|
||||
global: false,
|
||||
segments: vec![ast::PathSegment {
|
||||
identifier: str_to_ident("foo"),
|
||||
parameters: ast::PathParameters::none(),
|
||||
},
|
||||
ast::PathSegment {
|
||||
identifier: str_to_ident("bar"),
|
||||
parameters: ast::PathParameters::none(),
|
||||
}],
|
||||
};
|
||||
|
||||
let test3_tts = vec![TokenTree::Token(sp(9, 12), token::Ident(str_to_ident("bar"))),
|
||||
TokenTree::Token(sp(12, 13), token::Comma),
|
||||
TokenTree::Token(sp(13, 16), token::Ident(str_to_ident("baz")))];
|
||||
|
||||
let test3_stream = TokenStream::from_tts(vec![TokenTree::Delimited(sp(8, 17),
|
||||
Rc::new(Delimited {
|
||||
delim: token::DelimToken::Paren,
|
||||
open_span: sp(8, 9),
|
||||
tts: test3_tts,
|
||||
close_span: sp(16, 17),
|
||||
}))]);
|
||||
let test3_expected = Some((test3_path, &test3_stream[..]));
|
||||
assert_eq!(test3, test3_expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_as_paren_delimited_stream() {
|
||||
let test0 = TokenStream::as_paren_delimited_stream(string_to_tts("foo,bar,".to_string()));
|
||||
let test1 = TokenStream::as_paren_delimited_stream(string_to_tts("baz(foo,bar)"
|
||||
.to_string()));
|
||||
|
||||
let test0_tts = vec![TokenTree::Token(sp(0, 3), token::Ident(str_to_ident("foo"))),
|
||||
TokenTree::Token(sp(3, 4), token::Comma),
|
||||
TokenTree::Token(sp(4, 7), token::Ident(str_to_ident("bar"))),
|
||||
TokenTree::Token(sp(7, 8), token::Comma)];
|
||||
let test0_stream = TokenStream::from_tts(vec![TokenTree::Delimited(sp(0, 8),
|
||||
Rc::new(Delimited {
|
||||
delim: token::DelimToken::Paren,
|
||||
open_span: DUMMY_SP,
|
||||
tts: test0_tts,
|
||||
close_span: DUMMY_SP,
|
||||
}))]);
|
||||
|
||||
assert_eq!(test0, test0_stream);
|
||||
|
||||
|
||||
let test1_tts = vec![TokenTree::Token(sp(4, 7), token::Ident(str_to_ident("foo"))),
|
||||
TokenTree::Token(sp(7, 8), token::Comma),
|
||||
TokenTree::Token(sp(8, 11), token::Ident(str_to_ident("bar")))];
|
||||
|
||||
let test1_parse = vec![TokenTree::Token(sp(0, 3), token::Ident(str_to_ident("baz"))),
|
||||
TokenTree::Delimited(sp(3, 12),
|
||||
Rc::new(Delimited {
|
||||
delim: token::DelimToken::Paren,
|
||||
open_span: sp(3, 4),
|
||||
tts: test1_tts,
|
||||
close_span: sp(11, 12),
|
||||
}))];
|
||||
|
||||
let test1_stream = TokenStream::from_tts(vec![TokenTree::Delimited(sp(0, 12),
|
||||
Rc::new(Delimited {
|
||||
delim: token::DelimToken::Paren,
|
||||
open_span: DUMMY_SP,
|
||||
tts: test1_parse,
|
||||
close_span: DUMMY_SP,
|
||||
}))]);
|
||||
|
||||
assert_eq!(test1, test1_stream);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user