Track distinct spans for open and close delimiter

src/libproc_macro/lib.rs

@@ -63,8 +63,8 @@ use std::str::FromStr;
 use syntax::errors::DiagnosticBuilder;
 use syntax::parse::{self, token};
 use syntax::symbol::Symbol;
-use syntax::tokenstream;
-use syntax_pos::{BytePos, Pos, FileName};
+use syntax::tokenstream::{self, DelimSpan};
+use syntax_pos::{Pos, FileName};
 
 /// The main type provided by this crate, representing an abstract stream of
 /// tokens, or, more specifically, a sequence of token trees.
@@ -609,7 +609,7 @@ impl fmt::Display for TokenTree {
 pub struct Group {
  delimiter: Delimiter,
  stream: TokenStream,
- span: Span,
+ span: DelimSpan,
 }
 
 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
@@ -650,7 +650,7 @@ impl Group {
  Group {
  delimiter: delimiter,
  stream: stream,
- span: Span::call_site(),
+ span: DelimSpan::from_single(Span::call_site().0),
  }
  }
 
@@ -678,43 +678,29 @@ impl Group {
  /// ```
  #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
  pub fn span(&self) -> Span {
- self.span
+ Span(self.span.entire())
  }
 
- /// Returns the span pointing to the opening delimiter of this group, or the
- /// span of the entire group if this is a None-delimited group.
+ /// Returns the span pointing to the opening delimiter of this group.
  ///
  /// ```text
  /// pub fn span_open(&self) -> Span {
  /// ^
  /// ```
  #[unstable(feature = "proc_macro_span", issue = "38356")]
  pub fn span_open(&self) -> Span {
- if self.delimiter == Delimiter::None {
- self.span
- } else {
- let lo = self.span.0.lo();
- let new_hi = BytePos::from_usize(lo.to_usize() + 1);
- Span(self.span.0.with_hi(new_hi))
- }
+ Span(self.span.open)
  }
 
- /// Returns the span pointing to the closing delimiter of this group, or the
- /// span of the entire group if this is a None-delimited group.
+ /// Returns the span pointing to the closing delimiter of this group.
  ///
  /// ```text
  /// pub fn span_close(&self) -> Span {
  /// ^
  /// ```
  #[unstable(feature = "proc_macro_span", issue = "38356")]
  pub fn span_close(&self) -> Span {
- let hi = self.span.0.hi();
- if self.delimiter == Delimiter::None || hi.to_usize() == 0 {
- self.span
- } else {
- let new_lo = BytePos::from_usize(hi.to_usize() - 1);
- Span(self.span.0.with_lo(new_lo))
- }
+ Span(self.span.close)
  }
 
  /// Configures the span for this `Group`'s delimiters, but not its internal
@@ -725,7 +711,7 @@ impl Group {
  /// tokens at the level of the `Group`.
  #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
  pub fn set_span(&mut self, span: Span) {
- self.span = span;
+ self.span = DelimSpan::from_single(span.0);
  }
 }
 

src/libproc_macro/rustc.rs

@@ -64,7 +64,7 @@ impl TokenTree {
  tokenstream::TokenTree::Delimited(span, delimed) => {
  let delimiter = Delimiter::from_internal(delimed.delim);
  let mut g = Group::new(delimiter, ::TokenStream(delimed.tts.into()));
- g.set_span(Span(span));
+ g.span = span;
  return g.into();
  }
  };
@@ -192,7 +192,7 @@ impl TokenTree {
  self::TokenTree::Punct(tt) => (tt.as_char(), tt.spacing(), tt.span()),
  self::TokenTree::Group(tt) => {
  return TokenTree::Delimited(
- tt.span.0,
+ tt.span,
  Delimited {
  delim: tt.delimiter.to_internal(),
  tts: tt.stream.0.into(),

src/librustc/ich/hcx.rs

@@ -28,6 +28,7 @@ use syntax::ast;
 use syntax::source_map::SourceMap;
 use syntax::ext::hygiene::SyntaxContext;
 use syntax::symbol::Symbol;
+use syntax::tokenstream::DelimSpan;
 use syntax_pos::{Span, DUMMY_SP};
 use syntax_pos::hygiene;
 
@@ -396,6 +397,17 @@ impl<'a> HashStable<StableHashingContext<'a>> for Span {
  }
 }
 
+impl<'a> HashStable<StableHashingContext<'a>> for DelimSpan {
+ fn hash_stable<W: StableHasherResult>(
+ &self,
+ hcx: &mut StableHashingContext<'a>,
+ hasher: &mut StableHasher<W>,
+ ) {
+ self.open.hash_stable(hcx, hasher);
+ self.close.hash_stable(hcx, hasher);
+ }
+}
+
 pub fn hash_stable_trait_impls<'a, 'gcx, W, R>(
  hcx: &mut StableHashingContext<'a>,
  hasher: &mut StableHasher<W>,

src/librustc_resolve/macros.rs

@@ -35,7 +35,7 @@ use syntax::parse::parser::PathStyle;
 use syntax::parse::token::{self, Token};
 use syntax::ptr::P;
 use syntax::symbol::{Symbol, keywords};
-use syntax::tokenstream::{TokenStream, TokenTree, Delimited};
+use syntax::tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan};
 use syntax::util::lev_distance::find_best_match_for_name;
 use syntax_pos::{Span, DUMMY_SP};
 use errors::Applicability;
@@ -279,7 +279,8 @@ impl<'a, 'crateloader: 'a> base::Resolver for Resolver<'a, 'crateloader> {
  tokens.push(TokenTree::Token(path.span, tok).into());
  }
  }
- attrs[i].tokens = TokenTree::Delimited(attrs[i].span, Delimited {
+ let delim_span = DelimSpan::from_single(attrs[i].span);
+ attrs[i].tokens = TokenTree::Delimited(delim_span, Delimited {
  delim: token::Paren,
  tts: TokenStream::concat(tokens).into(),
  }).into();

src/libsyntax/attr/mod.rs

@@ -34,7 +34,7 @@ use parse::token::{self, Token};
 use ptr::P;
 use symbol::Symbol;
 use ThinVec;
-use tokenstream::{TokenStream, TokenTree, Delimited};
+use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan};
 use GLOBALS;
 
 use std::iter;
@@ -535,7 +535,7 @@ impl MetaItemKind {
  }
  tokens.push(item.node.tokens());
  }
- TokenTree::Delimited(span, Delimited {
+ TokenTree::Delimited(DelimSpan::from_single(span), Delimited {
  delim: token::Paren,
  tts: TokenStream::concat(tokens).into(),
  }).into()

src/libsyntax/ext/quote.rs

@@ -10,14 +10,14 @@
 
 use ast::{self, Arg, Arm, Block, Expr, Item, Pat, Stmt, Ty};
 use source_map::respan;
-use syntax_pos::Span;
+use syntax_pos::{Span, DUMMY_SP};
 use ext::base::ExtCtxt;
 use ext::base;
 use ext::build::AstBuilder;
 use parse::parser::{Parser, PathStyle};
 use parse::token;
 use ptr::P;
-use tokenstream::{TokenStream, TokenTree};
+use tokenstream::{DelimSpan, TokenStream, TokenTree};
 
 /// Quasiquoting works via token trees.
 ///
@@ -36,7 +36,7 @@ pub mod rt {
  use symbol::Symbol;
  use ThinVec;
 
- use tokenstream::{self, TokenTree, TokenStream};
+ use tokenstream::{self, DelimSpan, TokenTree, TokenStream};
 
  pub use parse::new_parser_from_tts;
  pub use syntax_pos::{BytePos, Span, DUMMY_SP, FileName};
@@ -245,7 +245,8 @@ pub mod rt {
  }
  inner.push(self.tokens.clone());
 
- r.push(TokenTree::Delimited(self.span, tokenstream::Delimited {
+ let delim_span = DelimSpan::from_single(self.span);
+ r.push(TokenTree::Delimited(delim_span, tokenstream::Delimited {
  delim: token::Bracket, tts: TokenStream::concat(inner).into()
  }));
  r
@@ -261,7 +262,7 @@ pub mod rt {
 
  impl ToTokens for () {
  fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
- vec![TokenTree::Delimited(DUMMY_SP, tokenstream::Delimited {
+ vec![TokenTree::Delimited(DelimSpan::dummy(), tokenstream::Delimited {
  delim: token::Paren,
  tts: TokenStream::empty().into(),
  })]
@@ -385,13 +386,16 @@ pub fn unflatten(tts: Vec<TokenTree>) -> Vec<TokenTree> {
 
  let mut results = Vec::new();
  let mut result = Vec::new();
+ let mut open_span = DUMMY_SP;
  for tree in tts {
  match tree {
- TokenTree::Token(_, token::OpenDelim(..)) => {
+ TokenTree::Token(span, token::OpenDelim(..)) => {
+ open_span = span;
  results.push(::std::mem::replace(&mut result, Vec::new()));
  }
  TokenTree::Token(span, token::CloseDelim(delim)) => {
- let tree = TokenTree::Delimited(span, Delimited {
+ let delim_span = DelimSpan::from_pair(open_span, span);
+ let tree = TokenTree::Delimited(delim_span, Delimited {
  delim,
  tts: result.into_iter().map(TokenStream::from).collect::<TokenStream>().into(),
  });
@@ -756,9 +760,9 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, quoted: bool) -> Vec<ast::Stmt
  vec![cx.stmt_expr(e_push)]
  },
  TokenTree::Delimited(span, ref delimed) => {
- let mut stmts = statements_mk_tt(cx, &delimed.open_tt(span), false);
+ let mut stmts = statements_mk_tt(cx, &delimed.open_tt(span.open), false);
  stmts.extend(statements_mk_tts(cx, delimed.stream()));
- stmts.extend(statements_mk_tt(cx, &delimed.close_tt(span), false));
+ stmts.extend(statements_mk_tt(cx, &delimed.close_tt(span.close), false));
  stmts
  }
  }

src/libsyntax/ext/tt/macro_parser.rs

@@ -85,7 +85,7 @@ pub use self::ParseResult::*;
 use self::TokenTreeOrTokenTreeSlice::*;
 
 use ast::Ident;
-use syntax_pos::{self, BytePos, Span};
+use syntax_pos::{self, Span};
 use errors::FatalError;
 use ext::tt::quoted::{self, TokenTree};
 use parse::{Directory, ParseSess};
@@ -94,7 +94,7 @@ use parse::token::{self, DocComment, Nonterminal, Token};
 use print::pprust;
 use OneVector;
 use symbol::keywords;
-use tokenstream::TokenStream;
+use tokenstream::{DelimSpan, TokenStream};
 
 use rustc_data_structures::fx::FxHashMap;
 use std::collections::hash_map::Entry::{Occupied, Vacant};
@@ -154,7 +154,7 @@ struct MatcherPos<'a> {
  /// The beginning position in the source that the beginning of this matcher corresponds to. In
  /// other words, the token in the source at `sp_lo` is matched against the first token of the
  /// matcher.
- sp_lo: BytePos,
+ sp_lo: Span,
 
  /// For each named metavar in the matcher, we keep track of token trees matched against the
  /// metavar by the black box parser. In particular, there may be more than one match per
@@ -285,7 +285,7 @@ fn create_matches(len: usize) -> Vec<Rc<Vec<NamedMatch>>> {
 
 /// Generate the top-level matcher position in which the "dot" is before the first token of the
 /// matcher `ms` and we are going to start matching at position `lo` in the source.
-fn initial_matcher_pos(ms: &[TokenTree], lo: BytePos) -> MatcherPos {
+fn initial_matcher_pos(ms: &[TokenTree], lo: Span) -> MatcherPos {
  let match_idx_hi = count_names(ms);
  let matches = create_matches(match_idx_hi);
  MatcherPos {
@@ -332,7 +332,7 @@ fn initial_matcher_pos(ms: &[TokenTree], lo: BytePos) -> MatcherPos {
 /// token tree it was derived from.
 #[derive(Debug, Clone)]
 pub enum NamedMatch {
- MatchedSeq(Rc<Vec<NamedMatch>>, syntax_pos::Span),
+ MatchedSeq(Rc<Vec<NamedMatch>>, DelimSpan),
  MatchedNonterminal(Rc<Nonterminal>),
 }
 
@@ -488,7 +488,7 @@ fn inner_parse_loop<'a>(
  // Add matches from this repetition to the `matches` of `up`
  for idx in item.match_lo..item.match_hi {
  let sub = item.matches[idx].clone();
- let span = span.with_lo(item.sp_lo);
+ let span = DelimSpan::from_pair(item.sp_lo, span);
  new_pos.push_match(idx, MatchedSeq(sub, span));
  }
 
@@ -556,7 +556,7 @@ fn inner_parse_loop<'a>(
  match_cur: item.match_cur,
  match_hi: item.match_cur + seq.num_captures,
  up: Some(item),
- sp_lo: sp.lo(),
+ sp_lo: sp.open,
  top_elts: Tt(TokenTree::Sequence(sp, seq)),
  })));
  }
@@ -643,7 +643,7 @@ pub fn parse(
  //
  // This MatcherPos instance is allocated on the stack. All others -- and
  // there are frequently *no* others! -- are allocated on the heap.
- let mut initial = initial_matcher_pos(ms, parser.span.lo());
+ let mut initial = initial_matcher_pos(ms, parser.span);
  let mut cur_items = smallvec![MatcherPosHandle::Ref(&mut initial)];
  let mut next_items = Vec::new();