tokenize method

Implementation

List<Token> tokenize() {
  final out = <Token>[];
  while (_pos < src.length) {
    final c = src[_pos];
    if (_isWhitespace(c)) {
      _pos++;
      continue;
    }
    if (c == '-' && _peek(1) == '-') {
      _skipLineComment();
      continue;
    }
    if (c == '/' && _peek(1) == '*') {
      _skipBlockComment();
      continue;
    }
    if (c == "'" || c == '"') {
      out.add(_readString(c));
      continue;
    }
    // MySQL-style backtick-quoted identifier.
    if (c == '`') {
      out.add(_readBacktickIdent());
      continue;
    }
    // X'...' BLOB literal (hex). Must be a standalone X immediately
    // followed by a single-quoted run of hex digits.
    if ((c == 'X' || c == 'x') && _peek(1) == "'") {
      out.add(_readBlobLiteral());
      continue;
    }
    if (_isDigit(c) || (c == '.' && _isDigit(_peek(1) ?? ''))) {
      out.add(_readNumber());
      continue;
    }
    if (_isIdentStart(c)) {
      out.add(_readIdentOrKeyword());
      continue;
    }
    out.add(_readOperatorOrPunct());
  }
  out.add(Token(TokType.eof, '', _pos));
  return out;
}