tokenizer.cc

#include "tokenizer.h"
#include "token.h"
#include <string>
#include <cctype>
#include <stdexcept>
#include <vector>
#include <sstream>
#include <cstdarg>

namespace {
  static const char EndOfFileChar = std::char_traits<char>::to_char_type(std::char_traits<char>::eof());
}

//--------------------------------------------------------------------------------------------------
Tokenizer::Tokenizer() :
  input_(nullptr),
  inputLength_(0),
  cursorPos_(0),
  cursorLine_(0)
{

}

//--------------------------------------------------------------------------------------------------
Tokenizer::~Tokenizer()
{

}

//--------------------------------------------------------------------------------------------------
void Tokenizer::Reset(const char* input, std::size_t startingLine)
{
  input_ = input;
  inputLength_ = std::char_traits<char>::length(input);
  cursorPos_ = 0;
  cursorLine_ = startingLine;
}

//--------------------------------------------------------------------------------------------------
char Tokenizer::GetChar()
{ 
	prevCursorPos_ = cursorPos_;
  prevCursorLine_ = cursorLine_;

  if(is_eof())
	{
		++cursorPos_;	// Do continue so UngetChar does what you think it does
    return EndOfFileChar;
	}
	
  char c = input_[cursorPos_];

  // New line moves the cursor to the new line
  if(c == '\n')
    cursorLine_++;

  cursorPos_++;
  return c;
}

//--------------------------------------------------------------------------------------------------
void Tokenizer::UngetChar()
{
  cursorLine_ = prevCursorLine_;
  cursorPos_ = prevCursorPos_;
}

//--------------------------------------------------------------------------------------------------
char Tokenizer::peek() const
{
  return !is_eof() ?
            input_[cursorPos_] :
            EndOfFileChar;
}

//--------------------------------------------------------------------------------------------------
char Tokenizer::GetLeadingChar()
{
  if (!comment_.text.empty())
    lastComment_ = comment_;

  comment_.text = "";
  comment_.startLine = cursorLine_;
  comment_.endLine = cursorLine_;

  char c;
  for(c = GetChar(); c != EndOfFileChar; c = GetChar())
  {
    // If this is a whitespace character skip it
    std::char_traits<char>::int_type intc = std::char_traits<char>::to_int_type(c);

    // In case of a new line
    if (c == '\n')
    {
      if (!comment_.text.empty())
        comment_.text += "\n";
      continue;
    }

    if(std::isspace(intc) || std::iscntrl(intc))
      continue;

    // If this is a single line comment
    char next = peek();
    if(c == '/' && next == '/')
    {
      std::vector<std::string> lines;

      size_t indentationLastLine = 0;
      while (!is_eof() && c == '/' && next == '/')
      {
        // Search for the end of the line
        std::string line;
        for (c = GetChar();
          c != EndOfFileChar && c != '\n';
          c = GetChar())
        {
          line += c;
        }
        
        // Store the line
        size_t lastSlashIndex = line.find_first_not_of("/");
        if (lastSlashIndex == std::string::npos)
          line = "";
        else
          line = line.substr(lastSlashIndex);

        size_t firstCharIndex = line.find_first_not_of(" \t");
        if (firstCharIndex == std::string::npos)
          line = "";
        else
          line = line.substr(firstCharIndex);

        if (firstCharIndex > indentationLastLine && !lines.empty())
          lines.back() += std::string(" ") + line;
        else
        {
          lines.emplace_back(std::move(line));
          indentationLastLine = firstCharIndex;
        }

        // Check the next line
        while (!is_eof() && std::isspace(c = GetChar()));

        if (!is_eof())
          next = peek();
      }

      // Unget previously get char
      if (!is_eof())
        UngetChar();

      // Build comment string
      std::stringstream ss;
      for (size_t i = 0; i < lines.size(); ++i)
      {
        if (i > 0)
          ss << "\n";
        ss << lines[i];
      }

      comment_.text = ss.str();
      comment_.endLine = cursorLine_;

      // Go to the next
      continue;
    }

    // If this is a block comment
    if(c == '/' && next == '*')
    {
      // Search for the end of the block comment
      std::vector<std::string> lines;
      std::string line;
      for (c = GetChar(), next = peek();
        c != EndOfFileChar && (c != '*' || next != '/');
        c = GetChar(), next = peek())
      {
        if (c == '\n')
        {
          if (!lines.empty() || !line.empty())
            lines.emplace_back(line);
          line.clear();
        }
        else
        {
          if (!line.empty() || !(std::isspace(c) || c == '*'))
            line += c;
        }
      }

      // Skip past the slash
      if(c != EndOfFileChar)
        GetChar();

      // Skip past new lines and spaces
      while (!is_eof() && std::isspace(c = GetChar()));
      if (!is_eof())
        UngetChar();

      // Remove empty lines from the back
      while (!lines.empty() && lines.back().empty())
        lines.pop_back();

      // Build comment string
      std::stringstream ss;
      for (size_t i = 0; i < lines.size(); ++i)
      {
        if (i > 0)
          ss << "\n";
        ss << lines[i]; 
      }

      comment_.text = ss.str();
      comment_.endLine = cursorLine_;

      // Move to the next character
      continue;
    }

    break;
  }
  
  return c;
}

//--------------------------------------------------------------------------------------------------
bool Tokenizer::GetToken(Token &token, bool angleBracketsForStrings, bool seperateBraces)
{
  // Get the next character
  char c = GetLeadingChar();
  char p = peek();
  std::char_traits<char>::int_type intc = std::char_traits<char>::to_int_type(c);
  std::char_traits<char>::int_type intp = std::char_traits<char>::to_int_type(p);

  if(c == EndOfFileChar)
  {
    UngetChar();
    return false;
  }

  // Record the start of the token position
  token.startPos = prevCursorPos_;
  token.startLine = prevCursorLine_;
  token.token.clear();
  token.tokenType = TokenType::kNone;

  // Alphanumeric token
  if(std::isalpha(intc) || c == '_')
  {
    // Read the rest of the alphanumeric characters
    do
    {
      token.token.push_back(c);
      c = GetChar();
      intc = std::char_traits<char>::to_int_type(c);
    } while(std::isalnum(intc) || c == '_');

    // Put back the last read character since it's not part of the identifier
    UngetChar();

    // Set the type of the token
    token.tokenType = TokenType::kIdentifier;

    if(token.token == "true")
    {
      token.tokenType = TokenType::kConst;
      token.constType = ConstType::kBoolean;
      token.boolConst = true;
    }
    else if(token.token == "false")
    {
      token.tokenType = TokenType::kConst;
      token.constType = ConstType::kBoolean;
      token.boolConst = false;
    }

    return true;
  }
  // Constant
  else if(std::isdigit(intc) || ((c == '-' || c == '+') && std::isdigit(intp)))
  {
    bool isFloat = false;
    bool isHex = false;
    bool isNegated = c == '-';
    do
    {
      if(c == '.')
        isFloat = true;

      if(c == 'x' || c == 'X')
        isHex = true;

      token.token.push_back(c);
      c = GetChar();
      intc = std::char_traits<char>::to_int_type(c);

    } while(std::isdigit(intc) ||
        (!isFloat && c == '.') ||
        (!isHex && (c == 'X' || c == 'x')) ||
        (isHex && std::isxdigit(intc)));

    if(!isFloat || (c != 'f' && c != 'F'))
      UngetChar();

    token.tokenType = TokenType::kConst;
    if(!isFloat)
    {
      try
      {
        if(isNegated)
        {
          token.int32Const = std::stoi(token.token, 0, 0);
          token.constType = ConstType::kInt32;
        }
        else
        {
          token.uint32Const = std::stoul(token.token, 0, 0);
          token.constType = ConstType::kUInt32;
        }
      }
      catch(std::out_of_range)
      {
        if(isNegated)
        {
          token.int64Const = std::stoll(token.token, 0, 0);
          token.constType = ConstType::kInt64;
        }
        else
        {
          token.uint64Const = std::stoull(token.token, 0, 0);
          token.constType = ConstType::kUInt64;
        }
      }
    }
    else
    {
      token.realConst = std::stod(token.token);
      token.constType = ConstType::kReal;
    }

    return true;
  }
  else if (c == '"' || (angleBracketsForStrings && c == '<'))
  {
    const char closingElement = c == '"' ? '"' : '>';

    c = GetChar();
    while (c != closingElement && std::char_traits<char>::not_eof(std::char_traits<char>::to_int_type(c)))
    {
      if(c == '\\')
      {
        c = GetChar();
        if(!std::char_traits<char>::not_eof(std::char_traits<char>::to_int_type(c)))
          break;
        else if(c == 'n')
          c = '\n';
        else if(c == 't')
          c = '\t';
        else if(c == 'r')
          c = '\r';
        else if(c == '"')
          c = '"';
      }

      token.token.push_back(c);
      c = GetChar();
    }

    if (c != closingElement)
      UngetChar();

    token.tokenType = TokenType::kConst;
    token.constType = ConstType::kString;
    token.stringConst = token.token;

    return true;
  }
  // Symbol
  else
  {
    // Push back the symbol
    token.token.push_back(c);

    #define PAIR(cc,dd) (c==cc&&d==dd) /* Comparison macro for two characters */
    const char d = GetChar();
    if(PAIR('<', '<') ||
       PAIR('-', '>') ||
       (!seperateBraces && PAIR('>', '>')) ||
       PAIR('!', '=') ||
       PAIR('<', '=') ||
       PAIR('>', '=') ||
       PAIR('+', '+') ||
       PAIR('-', '-') ||
       PAIR('+', '=') ||
       PAIR('-', '=') ||
       PAIR('*', '=') ||
       PAIR('/', '=') ||
       PAIR('^', '=') ||
       PAIR('|', '=') ||
       PAIR('&', '=') ||
       PAIR('~', '=') ||
       PAIR('%', '=') ||
       PAIR('&', '&') ||
       PAIR('|', '|') ||
       PAIR('=', '=') ||
       PAIR(':', ':')
      )
    #undef PAIR
    {
      token.token.push_back(d);
    }
    else
      UngetChar();

    token.tokenType = TokenType::kSymbol;

    return true;
  }

  return false;
}

//--------------------------------------------------------------------------------------------------
bool Tokenizer::is_eof() const
{
  return cursorPos_ >= inputLength_;
}

//--------------------------------------------------------------------------------------------------
bool Tokenizer::GetConst(Token &token)
{
	if (!GetToken(token))
		return false;

	if (token.tokenType == TokenType::kConst)
		return true;

	UngetToken(token);
	return false;
}

//--------------------------------------------------------------------------------------------------
bool Tokenizer::GetIdentifier(Token &token)
{
  if(!GetToken(token))
    return false;

  if(token.tokenType == TokenType::kIdentifier)
    return true;

  UngetToken(token);
  return false;
}

//--------------------------------------------------------------------------------------------------
void Tokenizer::UngetToken(const Token &token)
{
  cursorLine_ = token.startLine;
  cursorPos_ = token.startPos;
}

//--------------------------------------------------------------------------------------------------
bool Tokenizer::MatchIdentifier(const char *identifier)
{
  Token token;
  if(GetToken(token))
  {
    if(token.tokenType == TokenType::kIdentifier && token.token == identifier)
      return true;

    UngetToken(token);
  }

  return false;
}

//--------------------------------------------------------------------------------------------------
bool Tokenizer::MatchSymbol(const char *symbol)
{
  Token token;
  if(GetToken(token, false, std::char_traits<char>::length(symbol) == 1 && symbol[0] == '>'))
  {
    if(token.tokenType == TokenType::kSymbol && token.token == symbol)
      return true;

    UngetToken(token);
  }

  return false;
}

//--------------------------------------------------------------------------------------------------
bool Tokenizer::RequireIdentifier(const char *identifier)
{
  if(!MatchIdentifier(identifier))
    return Error("Missing identifier %s", identifier);
  return true;
}

//--------------------------------------------------------------------------------------------------
bool Tokenizer::RequireSymbol(const char *symbol)
{
  if (!MatchSymbol(symbol))
    return Error("Missing symbol %s", symbol);
  return true;
}

//-------------------------------------------------------------------------------------------------
bool Tokenizer::Error(const char* fmt, ...)
{
  char buffer[512];
  va_list args;
  va_start(args, fmt);
  vsnprintf(buffer, 512, fmt, args);
  va_end(args);
  printf("ERROR: %d:%d: %s", static_cast<int>(cursorLine_), 0, buffer);
  hasError_ = true;
  return false;
}