unit dtokenizer_c;

{$MODE FPC}
{$MODESWITCH DEFAULTPARAMETERS}
{$MODESWITCH OUT}
{$MODESWITCH RESULT}

interface

type
  TCppTokenKind = PtrInt;

const
  CPP_TOKEN_EOS         = TCppTokenKind(0);

  CPP_TOKEN_UNKNOWN     = TCppTokenKind(1);

  CPP_TOKEN_IDENTIFIER  = TCppTokenKind(2);
  //
  //  Both keywords and identifiers are classified as CPP_TOKEN_IDENTIFIER in
  //  the library.
  //
  //  All `new`, `delete`, `and`, `or` etc are classified as
  //  CPP_TOKEN_IDENTIFIER.
  //

  CPP_TOKEN_LITERAL     = TCppTokenKind(3);
  CPP_TOKEN_OPERATOR    = TCppTokenKind(4);
  CPP_TOKEN_PUNCTUATOR  = TCppTokenKind(5);
  CPP_TOKEN_COMMENT     = TCppTokenKind(6);
  CPP_TOKEN_UNCLOSED_COMMENT = TCppTokenKind(7);
  CPP_TOKEN_KEYWORD = TCppTokenKind($0800);

  CPP_TOKEN_COMMENT_SINGLE_LINE = CPP_TOKEN_COMMENT;
  CPP_TOKEN_COMMENT_MULTI_LINE = CPP_TOKEN_COMMENT;

  // All keywords
  CPP_TOKEN_KEYWORD_DO = CPP_TOKEN_KEYWORD + 0;
  CPP_TOKEN_KEYWORD_IF = CPP_TOKEN_KEYWORD + 1;
  CPP_TOKEN_KEYWORD_ASM = CPP_TOKEN_KEYWORD + 2;
  CPP_TOKEN_KEYWORD_FOR = CPP_TOKEN_KEYWORD + 3;
  CPP_TOKEN_KEYWORD_INT = CPP_TOKEN_KEYWORD + 4;
  CPP_TOKEN_KEYWORD_NEW = CPP_TOKEN_KEYWORD + 5;
  CPP_TOKEN_KEYWORD_TRY = CPP_TOKEN_KEYWORD + 6;
  CPP_TOKEN_KEYWORD_AUTO = CPP_TOKEN_KEYWORD + 7;
  CPP_TOKEN_KEYWORD_BOOL = CPP_TOKEN_KEYWORD + 8;
  CPP_TOKEN_KEYWORD_CASE = CPP_TOKEN_KEYWORD + 9;
  CPP_TOKEN_KEYWORD_CHAR = CPP_TOKEN_KEYWORD + 10;
  CPP_TOKEN_KEYWORD_ELSE = CPP_TOKEN_KEYWORD + 11;
  CPP_TOKEN_KEYWORD_ENUM = CPP_TOKEN_KEYWORD + 12;
  CPP_TOKEN_KEYWORD_GOTO = CPP_TOKEN_KEYWORD + 13;
  CPP_TOKEN_KEYWORD_LONG = CPP_TOKEN_KEYWORD + 14;
  CPP_TOKEN_KEYWORD_THIS = CPP_TOKEN_KEYWORD + 15;
  CPP_TOKEN_KEYWORD_TRUE = CPP_TOKEN_KEYWORD + 16;
  CPP_TOKEN_KEYWORD_VOID = CPP_TOKEN_KEYWORD + 17;
  CPP_TOKEN_KEYWORD_BREAK = CPP_TOKEN_KEYWORD + 18;
  CPP_TOKEN_KEYWORD_CATCH = CPP_TOKEN_KEYWORD + 19;
  CPP_TOKEN_KEYWORD_CLASS = CPP_TOKEN_KEYWORD + 20;
  CPP_TOKEN_KEYWORD_CONST = CPP_TOKEN_KEYWORD + 21;
  CPP_TOKEN_KEYWORD_FALSE = CPP_TOKEN_KEYWORD + 22;
  CPP_TOKEN_KEYWORD_FLOAT = CPP_TOKEN_KEYWORD + 23;
  CPP_TOKEN_KEYWORD_SHORT = CPP_TOKEN_KEYWORD + 24;
  CPP_TOKEN_KEYWORD_THROW = CPP_TOKEN_KEYWORD + 25;
  CPP_TOKEN_KEYWORD_UNION = CPP_TOKEN_KEYWORD + 26;
  CPP_TOKEN_KEYWORD_USING = CPP_TOKEN_KEYWORD + 27;
  CPP_TOKEN_KEYWORD_WHILE = CPP_TOKEN_KEYWORD + 28;
  CPP_TOKEN_KEYWORD_DELETE = CPP_TOKEN_KEYWORD + 29;
  CPP_TOKEN_KEYWORD_DOUBLE = CPP_TOKEN_KEYWORD + 30;
  CPP_TOKEN_KEYWORD_EXPORT = CPP_TOKEN_KEYWORD + 31;
  CPP_TOKEN_KEYWORD_EXTERN = CPP_TOKEN_KEYWORD + 32;
  CPP_TOKEN_KEYWORD_FRIEND = CPP_TOKEN_KEYWORD + 33;
  CPP_TOKEN_KEYWORD_INLINE = CPP_TOKEN_KEYWORD + 34;
  CPP_TOKEN_KEYWORD_PUBLIC = CPP_TOKEN_KEYWORD + 35;
  CPP_TOKEN_KEYWORD_RETURN = CPP_TOKEN_KEYWORD + 36;
  CPP_TOKEN_KEYWORD_SIGNED = CPP_TOKEN_KEYWORD + 37;
  CPP_TOKEN_KEYWORD_SIZEOF = CPP_TOKEN_KEYWORD + 38;
  CPP_TOKEN_KEYWORD_STATIC = CPP_TOKEN_KEYWORD + 39;
  CPP_TOKEN_KEYWORD_STRUCT = CPP_TOKEN_KEYWORD + 40;
  CPP_TOKEN_KEYWORD_SWITCH = CPP_TOKEN_KEYWORD + 41;
  CPP_TOKEN_KEYWORD_TYPEID = CPP_TOKEN_KEYWORD + 42;
  CPP_TOKEN_KEYWORD_ALIGNAS = CPP_TOKEN_KEYWORD + 43;
  CPP_TOKEN_KEYWORD_ALIGNOF = CPP_TOKEN_KEYWORD + 44;
  CPP_TOKEN_KEYWORD_CONCEPT = CPP_TOKEN_KEYWORD + 45;
  CPP_TOKEN_KEYWORD_DEFAULT = CPP_TOKEN_KEYWORD + 46;
  CPP_TOKEN_KEYWORD_MUTABLE = CPP_TOKEN_KEYWORD + 47;
  CPP_TOKEN_KEYWORD_NULLPTR = CPP_TOKEN_KEYWORD + 48;
  CPP_TOKEN_KEYWORD_PRIVATE = CPP_TOKEN_KEYWORD + 49;
  CPP_TOKEN_KEYWORD_TYPEDEF = CPP_TOKEN_KEYWORD + 50;
  CPP_TOKEN_KEYWORD_VIRTUAL = CPP_TOKEN_KEYWORD + 51;
  CPP_TOKEN_KEYWORD_WCHAR_T = CPP_TOKEN_KEYWORD + 52;
  CPP_TOKEN_KEYWORD_CHAR16_T = CPP_TOKEN_KEYWORD + 53;
  CPP_TOKEN_KEYWORD_CHAR32_T = CPP_TOKEN_KEYWORD + 54;
  CPP_TOKEN_KEYWORD_CONTINUE = CPP_TOKEN_KEYWORD + 55;
  CPP_TOKEN_KEYWORD_DECLTYPE = CPP_TOKEN_KEYWORD + 56;
  CPP_TOKEN_KEYWORD_EXPLICIT = CPP_TOKEN_KEYWORD + 57;
  CPP_TOKEN_KEYWORD_NOEXCEPT = CPP_TOKEN_KEYWORD + 56;
  CPP_TOKEN_KEYWORD_OPERATOR = CPP_TOKEN_KEYWORD + 57;
  CPP_TOKEN_KEYWORD_REGISTER = CPP_TOKEN_KEYWORD + 58;
  CPP_TOKEN_KEYWORD_REQUIRES = CPP_TOKEN_KEYWORD + 59;
  CPP_TOKEN_KEYWORD_TEMPLATE = CPP_TOKEN_KEYWORD + 60;
  CPP_TOKEN_KEYWORD_TYPENAME = CPP_TOKEN_KEYWORD + 61;
  CPP_TOKEN_KEYWORD_UNSIGNED = CPP_TOKEN_KEYWORD + 62;
  CPP_TOKEN_KEYWORD_VOLATILE = CPP_TOKEN_KEYWORD + 63;
  CPP_TOKEN_KEYWORD_CONSTEXPR = CPP_TOKEN_KEYWORD + 64;
  CPP_TOKEN_KEYWORD_NAMESPACE = CPP_TOKEN_KEYWORD + 65;
  CPP_TOKEN_KEYWORD_PROTECTED = CPP_TOKEN_KEYWORD + 66;
  CPP_TOKEN_KEYWORD_CONST_CAST = CPP_TOKEN_KEYWORD + 67;
  CPP_TOKEN_KEYWORD_STATIC_CAST = CPP_TOKEN_KEYWORD + 68;
  CPP_TOKEN_KEYWORD_SYNAMIC_CAST = CPP_TOKEN_KEYWORD + 69;
  CPP_TOKEN_KEYWORD_THREAD_LOCAL = CPP_TOKEN_KEYWORD + 70;
  CPP_TOKEN_KEYWORD_STATIC_ASSERT = CPP_TOKEN_KEYWORD + 71;
  CPP_TOKEN_KEYWORD_REINTERPRET_CAST = CPP_TOKEN_KEYWORD + 72;

type
TCppToken = record
  Kind: TCppTokenKind;
  // Kind of the token. See comment for TCppTokenKind for details
  S: PAnsiChar;
  // Beginning of the token in the source
  SEnd: PAnsiChar;
  // End of the token in the source
end;

function NextCppToken(S, SEnd: PAnsiChar;
                      out Token: TCppToken;
                      out NextS: PAnsiChar): TCppTokenKind;

implementation

//
// List of C++ keywords:
//
//    alignas
//    alignof
//    asm
//    auto
//    bool
//    break
//    case
//    catch
//    char
//    char16_t
//    char32_t
//    class
//    concept
//    const
//    const_cast
//    constexpr
//    continue
//    decltype
//    default
//    delete
//    do
//    double
//    dynamic_cast
//    else
//    enum
//    explicit
//    export
//    extern
//    false
//    float
//    for
//    friend
//    goto
//    if
//    inline
//    int
//    long
//    mutable
//    namespace
//    new
//    noexcept
//    nullptr
//    operator
//    private
//    protected
//    public
//    register
//    reinterpret_cast
//    requires
//    return
//    short
//    signed
//    sizeof
//    static
//    static_assert
//    static_cast
//    struct
//    switch
//    template
//    this
//    thread_local
//    throw
//    true
//    try
//    typedef
//    typeid
//    typename
//    union
//    unsigned
//    using
//    virtual
//    void
//    volatile
//    wchar_t
//    while
//

//
// List of C++ operators:
//    !
//    !=
//    #
//    ##
//    %
//    %:
//    %:%:
//    %=
//    %>
//    &
//    &&
//    &=
//    (
//    )
//    *
//    *=
//    +
//    ++
//    +=
//    ,
//    -
//    --
//    -=
//    ->
//    ->*
//    .
//    .*
//    ...
//    /
//    /=
//    :
//    ::
//    :>
//    ;
//    <
//    <%
//    <:
//    <<
//    <<=
//    <=
//    <=>
//    =
//    ==
//    >
//    >=
//    >>
//    >>=
//    ?
//    [
//    ]
//    ^
//    ^=
//    and
//    and_eq
//    bitand
//    bitor
//    compl
//    delete
//    new
//    not
//    not_eq
//    or
//    or_eq
//    xor
//    xor_eq
//    {
//    |
//    |=
//    ||
//    }
//    ~
//

function SkipWhitespaces(S, SEnd: PAnsiChar): PAnsiChar; inline;
const
  TAB = #9;
  LF = #10;
  VERTICAL_TAB = #11;
  CR = #13;
begin
  while (S < SEnd) and (S^ in [' ', TAB, LF, VERTICAL_TAB, CR]) do
    Inc(S);
  Exit(S);
end;

function OneCharacterOperator(
                S: PAnsiChar;
                Character: AnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  Token.Kind := TCppTokenKind(Character);
  Token.S := S;
  Token.SEnd := S + 1;
  NextS := S + 1;
  Exit(TCppTokenKind(Character));
end;

function LongOp(
                S: PAnsiChar;
                OperatorEnd: PAnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  Token.Kind := CPP_TOKEN_OPERATOR;
  Token.S := S;
  Token.SEnd := OperatorEnd;
  NextS := OperatorEnd;
  Exit(CPP_TOKEN_OPERATOR);
end;

function Unknown(S, SEnd: PAnsiChar;
                 out Token: TCppToken;
                 out NextS: PAnsiChar): TCppTokenKind;
begin
  Token.Kind := CPP_TOKEN_UNKNOWN;
  Token.S := S;
  Token.SEnd := S + 1;
  NextS := S + 1;
  Exit(CPP_TOKEN_UNKNOWN);
end;

const
  W0 = {$IF Defined(ENDIAN_LITTLE)}  8 {$ELSE}  0 {$ENDIF};
  W1 = {$IF Defined(ENDIAN_LITTLE)}  0 {$ELSE}  8 {$ENDIF};
  U0 = {$IF Defined(ENDIAN_LITTLE)}  0 {$ELSE} 24 {$ENDIF};
  U1 = {$IF Defined(ENDIAN_LITTLE)}  8 {$ELSE} 16 {$ENDIF};
  U2 = {$IF Defined(ENDIAN_LITTLE)} 16 {$ELSE}  8 {$ENDIF};
  U3 = {$IF Defined(ENDIAN_LITTLE)} 24 {$ELSE}  0 {$ENDIF};
  FIRST_W = {$IF Defined(ENDIAN_LITTLE)} 16 {$ELSE}  0 {$ENDIF};

function Identifier(
        FirstBytes: UInt32;
        SecondBytes: UInt32;
        var Token: TCppToken;
        out NextS: PAnsiChar): TCppTokenKind;
begin
  Token.Kind := CPP_TOKEN_IDENTIFIER;
  Exit(CPP_TOKEN_IDENTIFIER);
end;

function Keyword(
        KeywordKind: TCppTokenKind;
        NameEnd: PAnsiChar;
        var Token: TCppToken;
        out NextS: PAnsiChar): TCppTokenKind;
begin
  Token.Kind := KeywordKind;
  Exit(KeywordKind);
end;

function ClassifyName2(
                FirstBytes: UInt32;
                NameEnd: PAnsiChar;
                var Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind;
begin
  Token.SEnd := NameEnd;
  NextS := NameEnd;
  case FirstBytes and (($ff shl U0) or ($ff shl U1)) of
    (Ord('d') shl U0) or (Ord('o') shl U1):
      Exit(Keyword(CPP_TOKEN_KEYWORD_DO, NameEnd, Token, NextS));
    (Ord('i') shl U0) or (Ord('f') shl U1):
      Exit(Keyword(CPP_TOKEN_KEYWORD_IF, NameEnd, Token, NextS));
  end;
  Exit(Identifier(FirstBytes, 0, Token, NextS));
end;

function ClassifyName3(
                FirstBytes: UInt32;
                NameEnd: PAnsiChar;
                var Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind;
begin
  Token.SEnd := NameEnd;
  NextS := NameEnd;
  case FirstBytes and (($ff shl U0) or ($ff shl U1) or ($ff shl U2)) of
    (Ord('a') shl U0) or (Ord('s') shl U1) or (Ord('m') shl U1):
      Exit(Keyword(CPP_TOKEN_KEYWORD_ASM, NameEnd, Token, NextS));
    (Ord('f') shl U0) or (Ord('o') shl U1) or (Ord('r') shl U1):
      Exit(Keyword(CPP_TOKEN_KEYWORD_FOR, NameEnd, Token, NextS));
    (Ord('i') shl U0) or (Ord('n') shl U1) or (Ord('t') shl U1):
      Exit(Keyword(CPP_TOKEN_KEYWORD_INT, NameEnd, Token, NextS));
    (Ord('n') shl U0) or (Ord('e') shl U1) or (Ord('w') shl U1):
      Exit(Keyword(CPP_TOKEN_KEYWORD_NEW, NameEnd, Token, NextS));
    (Ord('t') shl U0) or (Ord('r') shl U1) or (Ord('y') shl U1):
      Exit(Keyword(CPP_TOKEN_KEYWORD_TRY, NameEnd, Token, NextS));
  end;
  Exit(Identifier(FirstBytes, 0, Token, NextS));
end;

function ClassifyName4(
                FirstBytes: UInt32;
                NameEnd: PAnsiChar;
                var Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind;
begin
  Token.SEnd := NameEnd;
  NextS := NameEnd;
  case FirstBytes of
    (Ord('a') shl U0) or (Ord('u') shl U1) or (Ord('t') shl U2) or (Ord('o') shl U3):
      Exit(Keyword(CPP_TOKEN_KEYWORD_AUTO, NameEnd, Token, NextS));
    (Ord('b') shl U0) or (Ord('o') shl U1) or (Ord('o') shl U2) or (Ord('l') shl U3):
      Exit(Keyword(CPP_TOKEN_KEYWORD_BOOL, NameEnd, Token, NextS));
    (Ord('c') shl U0) or (Ord('a') shl U1) or (Ord('s') shl U2) or (Ord('e') shl U3):
      Exit(Keyword(CPP_TOKEN_KEYWORD_CASE, NameEnd, Token, NextS));
    (Ord('c') shl U0) or (Ord('h') shl U1) or (Ord('a') shl U2) or (Ord('r') shl U3):
      Exit(Keyword(CPP_TOKEN_KEYWORD_CHAR, NameEnd, Token, NextS));
    (Ord('e') shl U0) or (Ord('l') shl U1) or (Ord('s') shl U2) or (Ord('e') shl U3):
      Exit(Keyword(CPP_TOKEN_KEYWORD_ELSE, NameEnd, Token, NextS));
    (Ord('e') shl U0) or (Ord('n') shl U1) or (Ord('u') shl U2) or (Ord('m') shl U3):
      Exit(Keyword(CPP_TOKEN_KEYWORD_ENUM, NameEnd, Token, NextS));
    (Ord('g') shl U0) or (Ord('o') shl U1) or (Ord('t') shl U2) or (Ord('o') shl U3):
      Exit(Keyword(CPP_TOKEN_KEYWORD_GOTO, NameEnd, Token, NextS));
    (Ord('l') shl U0) or (Ord('o') shl U1) or (Ord('n') shl U2) or (Ord('g') shl U3):
      Exit(Keyword(CPP_TOKEN_KEYWORD_LONG, NameEnd, Token, NextS));
    (Ord('t') shl U0) or (Ord('h') shl U1) or (Ord('i') shl U2) or (Ord('s') shl U3):
      Exit(Keyword(CPP_TOKEN_KEYWORD_THIS, NameEnd, Token, NextS));
    (Ord('t') shl U0) or (Ord('r') shl U1) or (Ord('u') shl U2) or (Ord('e') shl U3):
      Exit(Keyword(CPP_TOKEN_KEYWORD_TRUE, NameEnd, Token, NextS));
    (Ord('v') shl U0) or (Ord('o') shl U1) or (Ord('i') shl U2) or (Ord('d') shl U3):
      Exit(Keyword(CPP_TOKEN_KEYWORD_VOID, NameEnd, Token, NextS));
  end;
  Exit(Identifier(FirstBytes, 0, Token, NextS));
end;

function ClassifyName5(
                FirstBytes: UInt32;
                SecondBytes: UInt32;
                var Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind;
begin
  NextS := Token.S + 5;
  Token.SEnd := NextS;
  case FirstBytes of
    (Ord('b') shl U0) or (Ord('r') shl U1) or (Ord('e') shl U2) or (Ord('a') shl U3):
      if (SecondBytes and ($ff shl U0)) = (Ord('k') shl U0) then begin
        Token.Kind := CPP_TOKEN_KEYWORD_BREAK;
        Exit(CPP_TOKEN_KEYWORD_BREAK);
      end;
    (Ord('c') shl U0) or (Ord('a') shl U1) or (Ord('t') shl U2) or (Ord('c') shl U3):
      if (SecondBytes and ($ff shl U0)) = (Ord('h') shl U0) then begin
        Token.Kind := CPP_TOKEN_KEYWORD_CATCH;
        Exit(CPP_TOKEN_KEYWORD_CATCH);
      end;
    (Ord('c') shl U0) or (Ord('l') shl U1) or (Ord('a') shl U2) or (Ord('s') shl U3):
      if (SecondBytes and ($ff shl U0)) = (Ord('s') shl U0) then begin
        Token.Kind := CPP_TOKEN_KEYWORD_CLASS;
        Exit(CPP_TOKEN_KEYWORD_CLASS);
      end;
    (Ord('c') shl U0) or (Ord('o') shl U1) or (Ord('n') shl U2) or (Ord('s') shl U3):
      if (SecondBytes and ($ff shl U0)) = (Ord('t') shl U0) then begin
        Token.Kind := CPP_TOKEN_KEYWORD_CONST;
        Exit(CPP_TOKEN_KEYWORD_CONST);
      end;
    (Ord('f') shl U0) or (Ord('a') shl U1) or (Ord('l') shl U2) or (Ord('s') shl U3):
      if (SecondBytes and ($ff shl U0)) = (Ord('e') shl U0) then begin
        Token.Kind := CPP_TOKEN_KEYWORD_FALSE;
        Exit(CPP_TOKEN_KEYWORD_FALSE);
      end;
    (Ord('f') shl U0) or (Ord('l') shl U1) or (Ord('o') shl U2) or (Ord('a') shl U3):
      if (SecondBytes and ($ff shl U0)) = (Ord('t') shl U0) then begin
        Token.Kind := CPP_TOKEN_KEYWORD_FLOAT;
        Exit(CPP_TOKEN_KEYWORD_FLOAT);
      end;
    (Ord('s') shl U0) or (Ord('h') shl U1) or (Ord('o') shl U2) or (Ord('r') shl U3):
      if (SecondBytes and ($ff shl U0)) = (Ord('t') shl U0) then begin
        Token.Kind := CPP_TOKEN_KEYWORD_SHORT;
        Exit(CPP_TOKEN_KEYWORD_SHORT);
      end;
    (Ord('t') shl U0) or (Ord('h') shl U1) or (Ord('r') shl U2) or (Ord('o') shl U3):
      if (SecondBytes and ($ff shl U0)) = (Ord('w') shl U0) then begin
        Token.Kind := CPP_TOKEN_KEYWORD_THROW;
        Exit(CPP_TOKEN_KEYWORD_THROW);
      end;
    (Ord('u') shl U0) or (Ord('n') shl U1) or (Ord('i') shl U2) or (Ord('o') shl U3):
      if (SecondBytes and ($ff shl U0)) = (Ord('n') shl U0) then begin
        Token.Kind := CPP_TOKEN_KEYWORD_UNION;
        Exit(CPP_TOKEN_KEYWORD_UNION);
      end;
    (Ord('u') shl U0) or (Ord('s') shl U1) or (Ord('i') shl U2) or (Ord('n') shl U3):
      if (SecondBytes and ($ff shl U0)) = (Ord('g') shl U0) then begin
        Token.Kind := CPP_TOKEN_KEYWORD_USING;
        Exit(CPP_TOKEN_KEYWORD_USING);
      end;
    (Ord('w') shl U0) or (Ord('h') shl U1) or (Ord('i') shl U2) or (Ord('l') shl U3):
      if (SecondBytes and ($ff shl U0)) = (Ord('e') shl U0) then begin
        Token.Kind := CPP_TOKEN_KEYWORD_WHILE;
        Exit(CPP_TOKEN_KEYWORD_WHILE);
      end;
  end;
  Exit(Identifier(FirstBytes, SecondBytes, Token, NextS));
end;

function SkipIdentifier(
                S, SEnd: PAnsiChar;
                var Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind;
begin
  while (S < SEnd) and (S^ in ['a'..'z', 'A'..'Z', '0'..'9', '_']) do
    Inc(S);
  Token.SEnd := S;
  NextS := S;
  Token.Kind := CPP_TOKEN_IDENTIFIER;
  Exit(CPP_TOKEN_IDENTIFIER);
end;

function ClassifyNameByFirstBytes(FirstBytes: UInt32;
                                  SecondBytes: UInt32;
                                  SEnd: PAnsiChar;
                                  var Token: TCppToken;
                                  out NextS: PAnsiChar): TCppTokenKind;
begin
  // We assume first byte was checked (no empty names allowed)
  //if not (AnsiChar((FirstBytes  shr U0) and $ff) in ['a'..'z', 'A'..'Z', '_']) then
  //  Exit(Identifier(Token.S, Token.S, Token, NextS);

  if not (AnsiChar((FirstBytes  shr U1) and $ff) in ['a'..'z', 'A'..'Z', '0'..'9', '_']) then begin
    Token.SEnd := Token.S + 1;
    NextS := Token.S + 1;
    Exit(Identifier(FirstBytes, SecondBytes, Token, NextS));
  end;
  if not (AnsiChar((FirstBytes  shr U2) and $ff) in ['a'..'z', 'A'..'Z', '0'..'9', '_']) then
    Exit(ClassifyName2(FirstBytes, Token.S + 2, Token, NextS));
  if not (AnsiChar((FirstBytes  shr U3) and $ff) in ['a'..'z', 'A'..'Z', '0'..'9', '_']) then
    Exit(ClassifyName3(FirstBytes, Token.S + 3, Token, NextS));

  if not (AnsiChar((SecondBytes shr U0) and $ff) in ['a'..'z', 'A'..'Z', '0'..'9', '_']) then
    Exit(ClassifyName4(FirstBytes, Token.S + 4, Token, NextS));
  if not (AnsiChar((SecondBytes shr U1) and $ff) in ['a'..'z', 'A'..'Z', '0'..'9', '_']) then
    Exit(ClassifyName5(FirstBytes, SecondBytes, Token, NextS));
  // if not (AnsiChar((SecondBytes shr U2) and $ff) in ['a'..'z', 'A'..'Z', '0'..'9', '_']) then
  //   Exit(ClassifyName6(FirstBytes, SecondBytes, Token, NextS));
  // if not (AnsiChar((SecondBytes shr U3) and $ff) in ['a'..'z', 'A'..'Z', '0'..'9', '_']) then
  //   Exit(ClassifyName7(FirstBytes, SecondBytes, Token, NextS));

  // TODO 8 characters and longer

  Exit(SkipIdentifier(Token.S, SEnd, Token, NextS));
end;

function NextName(S, SEnd: PAnsiChar;
                  out Token: TCppToken;
                  out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  Token.S := S;

  if S + 8 <= SEnd then begin
    Exit(ClassifyNameByFirstBytes(PUInt32(S)[0], PUInt32(S)[1], SEnd, Token, NextS));
  end else begin
    case SEnd - S of
      0: Exit(Unknown(S, S + 1, Token, NextS)); // Internal error?
      1: begin
           Token.SEnd := S + 1;
           NextS := S + 1;
           Exit(Identifier(0, 0, Token, NextS));
         end;
      2: Exit(ClassifyNameByFirstBytes(PUInt16(S)[0] shl U2, 0, SEnd, Token, NextS));
      3: Exit(ClassifyNameByFirstBytes((PUInt16(S)[0] shl U2) or (PByte(S)[3] shl U3), 0, SEnd, Token, NextS));
      4: Exit(ClassifyNameByFirstBytes(PUInt32(S)[0], 0, SEnd, Token, NextS));
      5: Exit(ClassifyNameByFirstBytes(PUInt32(S)[0], PByte(S + 4)^ shl U0, SEnd, Token, NextS));
      6: Exit(ClassifyNameByFirstBytes(PUInt32(S)[0], PUInt16(S + 4)^ shl U2, SEnd, Token, NextS));
      7: Exit(ClassifyNameByFirstBytes(PUInt32(S)[0], (PUInt16(S + 4)^ shl U2) or (PUInt8(S + 4)[3] shl U3), SEnd, Token, NextS));
    else
      Exit(Unknown(S, S + 1, Token, NextS)); // Internal error?
    end;
  end;
end;

function NumberLiteral(
                S, SEnd: PAnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  Token.Kind := CPP_TOKEN_LITERAL;
  Token.S := S;
  Inc(S);
  while (S < SEnd) and (S^ in ['0'..'9']) do
    Inc(S);
  Token.SEnd := S;
  NextS := S;
  Exit(CPP_TOKEN_LITERAL);
end;

function SkipLine(S, SEnd: PAnsiChar; out LineEnd: PAnsiChar): PAnsiChar; inline;
begin
  while S < SEnd do begin
    case S^ of
      #10: begin
             LineEnd := S;
             Exit(S + 1);
           end;
      #13: begin
             LineEnd := S;
             if (S + 1 < SEnd) and (S[1] = #10) then
               Exit(S + 2);
             Exit(S + 1);
           end;
    end;
    Inc(S);
  end;
  LineEnd := SEnd;
  Exit(SEnd);
end;

function MultilineComment(
                S, SEnd: PAnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
label
  LEndOfSource;
begin
  Token.S := S;
  while S < SEnd do begin
    while S^ = '*' do begin
      Inc(S);
      if S >= SEnd then
        goto LEndOfSource;
      if S^ = '/' then begin
        Token.Kind := CPP_TOKEN_COMMENT;
        Token.SEnd := S - 2;
        NextS := S + 1;
        Exit(CPP_TOKEN_UNCLOSED_COMMENT);
      end;
    end;
    Inc(S);
  end;

LEndOfSource:
  Token.Kind := CPP_TOKEN_UNCLOSED_COMMENT;
  Dec(Token.S, 2);
  Token.SEnd := SEnd;
  NextS := SEnd;
  Exit(CPP_TOKEN_UNCLOSED_COMMENT);
end;

function NextTokenStartedWithLessThan(
                S, SEnd: PAnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  if S < SEnd then case S^ of
    ':': Exit(LongOp(S - 1, S + 1, Token, NextS));      //  `<:` operator
    '%': Exit(LongOp(S - 1, S + 1, Token, NextS));      //  `<%` operator
    '=': if (S + 1 < SEnd) and ((S + 1)^ <> '>') then begin
           Exit(LongOp(S - 1, S + 1, Token, NextS));    //  `<=` operator
         end else
           Exit(LongOp(S - 1, S + 2, Token, NextS));    // `<=>` operator
    '<': if (S + 1 < SEnd) and ((S + 1)^ <> '=') then begin
           Exit(LongOp(S - 1, S + 1, Token, NextS));    //  `<<` operator
         end else
           Exit(LongOp(S - 1, S + 2, Token, NextS));    // `<<=` operator
  end;
  Exit(OneCharacterOperator(S - 1, '<', Token, NextS)); //   `<` operator
end;

function NextTokenStartedWithGreaterThan(
                S, SEnd: PAnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  if S < SEnd then case S^ of
    '=': Exit(LongOp(S - 1, S + 1, Token, NextS));    //  `>=` operator
    '>': if (S + 1 < SEnd) and ((S + 1)^ <> '=') then begin
           Exit(LongOp(S - 1, S + 1, Token, NextS));  //  `>>` operator
         end else
           Exit(LongOp(S - 1, S + 2, Token, NextS));  // `>>=` operator
  end;
  Exit(OneCharacterOperator(S - 1, '>', Token, NextS)); //   `>` operator
end;

function NextTokenStartedWithPercent(
                S, SEnd: PAnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  if S < SEnd then case S^ of
    '=': Exit(LongOp(S - 1, S + 1, Token, NextS));    //  `%=` operator
    '>': Exit(LongOp(S - 1, S + 1, Token, NextS));    //  `%>` operator
    ':': if (S + 2 >= SEnd) or (S[1] <> '%') or (S[2] <> ':') then begin
           Exit(LongOp(S - 1, S + 1, Token, NextS));  //  `%:` operator
         end else
           Exit(LongOp(S - 1, S + 3, Token, NextS));  //  `%:%:` operator
  end;
  Exit(OneCharacterOperator(S - 1, '%', Token, NextS)); //   `%` operator
end;

function NextTokenStartedWithColon(
                S, SEnd: PAnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  if S < SEnd then case S^ of
    ':': Exit(LongOp(S - 1, S + 1, Token, NextS));    //  `::` operator
    '>': Exit(LongOp(S - 1, S + 1, Token, NextS));    //  `:>` operator
  end;
  Exit(OneCharacterOperator(S - 1, ':', Token, NextS)); //   `:` operator
end;

function NextTokenStartedWithDot(
                S, SEnd: PAnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  if S < SEnd then case S^ of
    '*': Exit(LongOp(S - 1, S + 1, Token, NextS));    //  `.*` operator
    '.': if (S + 2 >= SEnd) or (S[1] <> '.') or (S[2] <> '.') then begin
           Exit(Unknown(S - 1, SEnd, Token, NextS));
         end else
           Exit(LongOp(S - 1, S + 2, Token, NextS));  // `...` operator
  end;
  Exit(OneCharacterOperator(S - 1, '.', Token, NextS)); //   `.` operator
end;

function NextTokenStartedWithMinus(
                S, SEnd: PAnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  if S < SEnd then case S^ of
    '-': Exit(LongOp(S - 1, S + 1, Token, NextS));    //  `--` operator
    '=': Exit(LongOp(S - 1, S + 1, Token, NextS));    //  `-=` operator
    '>': if (S + 1 >= SEnd) or (S[1] <> '*') then begin
           Exit(LongOp(S - 1, S + 1, Token, NextS));  //  `->` operator
         end else
           Exit(LongOp(S - 1, S + 2, Token, NextS));  // `->*` operator
  end;
  Exit(OneCharacterOperator(S - 1, '-', Token, NextS)); //   `-` operator
end;

function NextTokenStartedWithPlus(
                S, SEnd: PAnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  if S < SEnd then case S^ of
    '+': Exit(LongOp(S - 1, S + 1, Token, NextS));    // `++` operator
    '=': Exit(LongOp(S - 1, S + 1, Token, NextS));    // `+=` operator
  end;
  Exit(OneCharacterOperator(S - 1, '+', Token, NextS)); //   `+` operator
end;

function NextTokenStartedWithStar(
                S, SEnd: PAnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  if (S < SEnd) and (S^ = '=') then begin
    Exit(LongOp(S - 1, S + 1, Token, NextS));         // `*=` operator
  end;
  Exit(OneCharacterOperator(S - 1, '*', Token, NextS)); //  `*` operator
end;

function NextTokenStartedWithCaret(
                S, SEnd: PAnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  if (S < SEnd) and (S^ = '=') then begin
    Exit(LongOp(S - 1, S + 1, Token, NextS));         // `^=` operator
  end;
  Exit(OneCharacterOperator(S - 1, '^', Token, NextS)); //  `^` operator
end;

function NextTokenStartedWithAmpersand(
                S, SEnd: PAnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  if S < SEnd then case S^ of
    '&': Exit(LongOp(S - 1, S + 1, Token, NextS));    // `&&` operator
    '=': Exit(LongOp(S - 1, S + 1, Token, NextS));    // `&=` operator
  end;
  Exit(OneCharacterOperator(S - 1, '&', Token, NextS)); //  `&` operator
end;

function NextTokenStartedWithVerticalBar(
                S, SEnd: PAnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  if S < SEnd then case S^ of
    '|': Exit(LongOp(S - 1, S + 1, Token, NextS));    // `||` operator
    '=': Exit(LongOp(S - 1, S + 1, Token, NextS));    // `|=` operator
  end;
  Exit(OneCharacterOperator(S - 1, '|', Token, NextS)); //  `|` operator
end;

function NextTokenStartedWithEqualSign(
                S, SEnd: PAnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  if (S < SEnd) and (S^ = '=') then begin
    Exit(LongOp(S - 1, S + 1, Token, NextS));         // `==` operator
  end;
  Exit(OneCharacterOperator(S - 1, '=', Token, NextS)); //  `=` operator
end;

function NextTokenStartedWithExclamationMark(
                S, SEnd: PAnsiChar;
                out Token: TCppToken;
                out NextS: PAnsiChar): TCppTokenKind; inline;
begin
  if (S < SEnd) and (S^ = '=') then begin
    Exit(LongOp(S - 1, S + 1, Token, NextS));         // `!=` operator
  end;
  Exit(OneCharacterOperator(S - 1, '!', Token, NextS)); //  `!` operator
end;

function NextCppToken(S, SEnd: PAnsiChar;
                      out Token: TCppToken;
                      out NextS: PAnsiChar): TCppTokenKind;
begin
  S := SkipWhitespaces(S, SEnd);

  if S >= SEnd then begin
    Token.Kind := CPP_TOKEN_EOS;
    Token.S := S;
    Token.SEnd := SEnd;
    NextS := SEnd;
    Exit(CPP_TOKEN_EOS);
  end;

  case S^ of
    'a'..'z', 'A'..'Z', '_': Exit(NextName(S, SEnd, Token, NextS));
    '0'..'9': Exit(NumberLiteral(S, SEnd, Token, NextS));
    // '/': Exit(NextTokenStartedWithSlash(S + 1, SEnd, Token, NextS));
    '<': Exit(NextTokenStartedWithLessThan(S + 1, SEnd, Token, NextS));
    '>': Exit(NextTokenStartedWithGreaterThan(S + 1, SEnd, Token, NextS));
    '%': Exit(NextTokenStartedWithPercent(S + 1, SEnd, Token, NextS));
    ':': Exit(NextTokenStartedWithColon(S + 1, SEnd, Token, NextS));
    '.': Exit(NextTokenStartedWithDot(S + 1, SEnd, Token, NextS));
    '-': Exit(NextTokenStartedWithMinus(S + 1, SEnd, Token, NextS));
    '+': Exit(NextTokenStartedWithPlus(S + 1, SEnd, Token, NextS));
    '*': Exit(NextTokenStartedWithStar(S + 1, SEnd, Token, NextS));
    '^': Exit(NextTokenStartedWithCaret(S + 1, SEnd, Token, NextS));
    '&': Exit(NextTokenStartedWithAmpersand(S + 1, SEnd, Token, NextS));
    '|': Exit(NextTokenStartedWithVerticalBar(S + 1, SEnd, Token, NextS));
    '=': Exit(NextTokenStartedWithEqualSign(S + 1, SEnd, Token, NextS));
    '!': Exit(NextTokenStartedWithExclamationMark(S + 1, SEnd, Token, NextS));
    '{': Exit(OneCharacterOperator(S, '{', Token, NextS)); // `{` operator
    '}': Exit(OneCharacterOperator(S, '}', Token, NextS)); // `}` operator
    '[': Exit(OneCharacterOperator(S, '[', Token, NextS)); // `[` operator
    ']': Exit(OneCharacterOperator(S, ']', Token, NextS)); // `]` operator
    '#': Exit(OneCharacterOperator(S, '#', Token, NextS)); // `#` operator
    '(': Exit(OneCharacterOperator(S, '(', Token, NextS)); // `(` operator
    ')': Exit(OneCharacterOperator(S, ')', Token, NextS)); // `)` operator
    '?': Exit(OneCharacterOperator(S, '?', Token, NextS)); // `?` operator
    ',': Exit(OneCharacterOperator(S, ',', Token, NextS)); // `,` operator
    ';': Exit(OneCharacterOperator(S, ';', Token, NextS)); // `;` operator
  else
    Exit(Unknown(S, SEnd, Token, NextS));
  end;
end;

end.
