unit dlexer_pascal;

{$MODE FPC}
{$MODESWITCH DEFAULTPARAMETERS}
{$MODESWITCH OUT}
{$MODESWITCH RESULT}

interface

type
  TPascalTokenKind = PtrInt;

const
  // Major token kinds
  PASCAL_TOKEN_EOS              = TPascalTokenKind(0);
  PASCAL_TOKEN_UNKNOWN          = TPascalTokenKind(1);
  PASCAL_TOKEN_IDENTIFIER       = TPascalTokenKind(2);
  PASCAL_TOKEN_RESERVED_WORD    = TPascalTokenKind($100);
  PASCAL_TOKEN_SYMBOL           = TPascalTokenKind($200);
  PASCAL_TOKEN_COMMENT          = TPascalTokenKind($300);
  PASCAL_TOKEN_STRING           = TPascalTokenKind($400);

  PASCAL_COMMENT_SINGLE_LINE            = PASCAL_TOKEN_COMMENT + 0;
  PASCAL_COMMENT_CURLY                  = PASCAL_TOKEN_COMMENT + 1; // { ... }
  PASCAL_COMMENT_ROUND_STAR             = PASCAL_TOKEN_COMMENT + 2; // (* ... *)

  // Not all special symbols may be returned as a token. E.g. {, (* and // will
  // be parsed immediatly as a part of a comment.
  PASCAL_SPECIAL_SYMBOL_SINGLE_QUOTE            = PASCAL_TOKEN_SYMBOL +  0; // '
  PASCAL_SPECIAL_SYMBOL_PLUS                    = PASCAL_TOKEN_SYMBOL +  1; // +
  PASCAL_SPECIAL_SYMBOL_MINUS                   = PASCAL_TOKEN_SYMBOL +  2; // -
  PASCAL_SPECIAL_SYMBOL_STAR                    = PASCAL_TOKEN_SYMBOL +  3; // *
  PASCAL_SPECIAL_SYMBOL_SLASH                   = PASCAL_TOKEN_SYMBOL +  4; // /
  PASCAL_SPECIAL_SYMBOL_EQUAL                   = PASCAL_TOKEN_SYMBOL +  5; // =
  PASCAL_SPECIAL_SYMBOL_LESS_THAN               = PASCAL_TOKEN_SYMBOL +  6; // <
  PASCAL_SPECIAL_SYMBOL_GREATER_THAN            = PASCAL_TOKEN_SYMBOL +  7; // >
  PASCAL_SPECIAL_SYMBOL_SQUARE_BRACKET_OPENING  = PASCAL_TOKEN_SYMBOL +  8; // [
  PASCAL_SPECIAL_SYMBOL_SQUARE_BRACKET_CLOSING  = PASCAL_TOKEN_SYMBOL +  9; // ]
  PASCAL_SPECIAL_SYMBOL_DOT                     = PASCAL_TOKEN_SYMBOL + 10; // .
  PASCAL_SPECIAL_SYMBOL_COMMA                   = PASCAL_TOKEN_SYMBOL + 11; // ,
  PASCAL_SPECIAL_SYMBOL_ROUND_BRACKET_OPENING   = PASCAL_TOKEN_SYMBOL + 12; // (
  PASCAL_SPECIAL_SYMBOL_ROUND_BRACKET_CLOSING   = PASCAL_TOKEN_SYMBOL + 13; // )
  PASCAL_SPECIAL_SYMBOL_COLON                   = PASCAL_TOKEN_SYMBOL + 14; // :
  PASCAL_SPECIAL_SYMBOL_CARET                   = PASCAL_TOKEN_SYMBOL + 15; // ^
  PASCAL_SPECIAL_SYMBOL_AT_SIGN                 = PASCAL_TOKEN_SYMBOL + 16; // @
  PASCAL_SPECIAL_SYMBOL_CURLY_BRACKET_OPENING   = PASCAL_TOKEN_SYMBOL + 17; // {
  PASCAL_SPECIAL_SYMBOL_CURLY_BRACKET_CLOSING   = PASCAL_TOKEN_SYMBOL + 18; // }
  PASCAL_SPECIAL_SYMBOL_DOLLAR_SIGN             = PASCAL_TOKEN_SYMBOL + 19; // $
  PASCAL_SPECIAL_SYMBOL_HASH_SIGN               = PASCAL_TOKEN_SYMBOL + 20; // #
  PASCAL_SPECIAL_SYMBOL_AMPERSAND               = PASCAL_TOKEN_SYMBOL + 21; // &
  PASCAL_SPECIAL_SYMBOL_PERCENT                 = PASCAL_TOKEN_SYMBOL + 22; // %
  PASCAL_SPECIAL_PAIR_SHL                       = PASCAL_TOKEN_SYMBOL + 23; // <<
  PASCAL_SPECIAL_PAIR_SHR                       = PASCAL_TOKEN_SYMBOL + 24; // >>
  PASCAL_SPECIAL_PAIR_EXP                       = PASCAL_TOKEN_SYMBOL + 25; // **
  PASCAL_SPECIAL_PAIR_NEQ                       = PASCAL_TOKEN_SYMBOL + 26; // <>
  PASCAL_SPECIAL_PAIR_INTERSECTS                = PASCAL_TOKEN_SYMBOL + 27; // ><
  PASCAL_SPECIAL_PAIR_LEQ                       = PASCAL_TOKEN_SYMBOL + 28; // <=
  PASCAL_SPECIAL_PAIR_GEQ                       = PASCAL_TOKEN_SYMBOL + 29; // >=
  PASCAL_SPECIAL_PAIR_ASSIGNMENT                = PASCAL_TOKEN_SYMBOL + 30; // :=
  PASCAL_SPECIAL_PAIR_PLUS_ASSIGNMENT           = PASCAL_TOKEN_SYMBOL + 31; // +=
  PASCAL_SPECIAL_PAIR_MINUS_ASSIGNMENT          = PASCAL_TOKEN_SYMBOL + 32; // -=
  PASCAL_SPECIAL_PAIR_STAR_ASSIGNMENT           = PASCAL_TOKEN_SYMBOL + 33; // *=
  PASCAL_SPECIAL_PAIR_SLASH_ASSIGNMENT          = PASCAL_TOKEN_SYMBOL + 34; // /=
  PASCAL_SPECIAL_PAIR_ROUND_STAR_OPENING        = PASCAL_TOKEN_SYMBOL + 35; // (*
  PASCAL_SPECIAL_PAIR_ROUND_STAR_CLOSING        = PASCAL_TOKEN_SYMBOL + 36; // *)
  PASCAL_SPECIAL_PAIR_ROUND_DOT_OPENING         = PASCAL_TOKEN_SYMBOL + 37; // (.
  PASCAL_SPECIAL_PAIR_ROUND_DOT_CLOSING         = PASCAL_TOKEN_SYMBOL + 38; // .)
  PASCAL_SPECIAL_PAIR_ROUND_SLASH_SLAHSH        = PASCAL_TOKEN_SYMBOL + 39; // //

  PASCAL_RESERVED_WORD_DO                   = PASCAL_TOKEN_RESERVED_WORD + 1;
  PASCAL_RESERVED_WORD_IF                   = PASCAL_TOKEN_RESERVED_WORD + 2;
  PASCAL_RESERVED_WORD_IN                   = PASCAL_TOKEN_RESERVED_WORD + 3;
  PASCAL_RESERVED_WORD_OF                   = PASCAL_TOKEN_RESERVED_WORD + 4;
  PASCAL_RESERVED_WORD_OR                   = PASCAL_TOKEN_RESERVED_WORD + 5;
  PASCAL_RESERVED_WORD_TO                   = PASCAL_TOKEN_RESERVED_WORD + 6;
  PASCAL_RESERVED_WORD_AND                  = PASCAL_TOKEN_RESERVED_WORD + 7;
  PASCAL_RESERVED_WORD_ASM                  = PASCAL_TOKEN_RESERVED_WORD + 8;
  PASCAL_RESERVED_WORD_DIV                  = PASCAL_TOKEN_RESERVED_WORD + 9;
  PASCAL_RESERVED_WORD_END                  = PASCAL_TOKEN_RESERVED_WORD + 10;
  PASCAL_RESERVED_WORD_FOR                  = PASCAL_TOKEN_RESERVED_WORD + 11;
  PASCAL_RESERVED_WORD_MOD                  = PASCAL_TOKEN_RESERVED_WORD + 12;
  PASCAL_RESERVED_WORD_NIL                  = PASCAL_TOKEN_RESERVED_WORD + 13;
  PASCAL_RESERVED_WORD_NOT                  = PASCAL_TOKEN_RESERVED_WORD + 14;
  PASCAL_RESERVED_WORD_SET                  = PASCAL_TOKEN_RESERVED_WORD + 15;
  PASCAL_RESERVED_WORD_SHL                  = PASCAL_TOKEN_RESERVED_WORD + 16;
  PASCAL_RESERVED_WORD_SHR                  = PASCAL_TOKEN_RESERVED_WORD + 17;
  PASCAL_RESERVED_WORD_VAR                  = PASCAL_TOKEN_RESERVED_WORD + 18;
  PASCAL_RESERVED_WORD_XOR                  = PASCAL_TOKEN_RESERVED_WORD + 19;
  PASCAL_RESERVED_WORD_CASE                 = PASCAL_TOKEN_RESERVED_WORD + 20;
  PASCAL_RESERVED_WORD_ELSE                 = PASCAL_TOKEN_RESERVED_WORD + 21;
  PASCAL_RESERVED_WORD_FILE                 = PASCAL_TOKEN_RESERVED_WORD + 22;
  PASCAL_RESERVED_WORD_GOTO                 = PASCAL_TOKEN_RESERVED_WORD + 23;
  PASCAL_RESERVED_WORD_SELF                 = PASCAL_TOKEN_RESERVED_WORD + 23;
  PASCAL_RESERVED_WORD_THEN                 = PASCAL_TOKEN_RESERVED_WORD + 24;
  PASCAL_RESERVED_WORD_TYPE                 = PASCAL_TOKEN_RESERVED_WORD + 25;
  PASCAL_RESERVED_WORD_UNIT                 = PASCAL_TOKEN_RESERVED_WORD + 26;
  PASCAL_RESERVED_WORD_USES                 = PASCAL_TOKEN_RESERVED_WORD + 27;
  PASCAL_RESERVED_WORD_WITH                 = PASCAL_TOKEN_RESERVED_WORD + 28;
  PASCAL_RESERVED_WORD_CVAR                 = PASCAL_TOKEN_RESERVED_WORD + 29;
  PASCAL_RESERVED_WORD_NAME                 = PASCAL_TOKEN_RESERVED_WORD + 30;
  PASCAL_RESERVED_WORD_NEAR                 = PASCAL_TOKEN_RESERVED_WORD + 31;
  PASCAL_RESERVED_WORD_READ                 = PASCAL_TOKEN_RESERVED_WORD + 32;
  PASCAL_RESERVED_WORD_ARRAY                = PASCAL_TOKEN_RESERVED_WORD + 33;
  PASCAL_RESERVED_WORD_BEGIN                = PASCAL_TOKEN_RESERVED_WORD + 34;
  PASCAL_RESERVED_WORD_CONST                = PASCAL_TOKEN_RESERVED_WORD + 35;
  PASCAL_RESERVED_WORD_LABEL                = PASCAL_TOKEN_RESERVED_WORD + 36;
  PASCAL_RESERVED_WORD_UNTIL                = PASCAL_TOKEN_RESERVED_WORD + 37;
  PASCAL_RESERVED_WORD_WHILE                = PASCAL_TOKEN_RESERVED_WORD + 38;
  PASCAL_RESERVED_WORD_CLASS                = PASCAL_TOKEN_RESERVED_WORD + 39;
  PASCAL_RESERVED_WORD_RAISE                = PASCAL_TOKEN_RESERVED_WORD + 40;
  PASCAL_RESERVED_WORD_ALIAS                = PASCAL_TOKEN_RESERVED_WORD + 41;
  PASCAL_RESERVED_WORD_BREAK                = PASCAL_TOKEN_RESERVED_WORD + 42;
  PASCAL_RESERVED_WORD_CDECL                = PASCAL_TOKEN_RESERVED_WORD + 43;
  PASCAL_RESERVED_WORD_INDEX                = PASCAL_TOKEN_RESERVED_WORD + 44;
  PASCAL_RESERVED_WORD_LOCAL                = PASCAL_TOKEN_RESERVED_WORD + 45;
  PASCAL_RESERVED_WORD_WRITE                = PASCAL_TOKEN_RESERVED_WORD + 46;
  PASCAL_RESERVED_WORD_DOWNTO               = PASCAL_TOKEN_RESERVED_WORD + 47;
  PASCAL_RESERVED_WORD_INLINE               = PASCAL_TOKEN_RESERVED_WORD + 48;
  PASCAL_RESERVED_WORD_OBJECT               = PASCAL_TOKEN_RESERVED_WORD + 49;
  PASCAL_RESERVED_WORD_PACKED               = PASCAL_TOKEN_RESERVED_WORD + 50;
  PASCAL_RESERVED_WORD_RECORD               = PASCAL_TOKEN_RESERVED_WORD + 51;
  PASCAL_RESERVED_WORD_REPEAT               = PASCAL_TOKEN_RESERVED_WORD + 52;
  PASCAL_RESERVED_WORD_STRING               = PASCAL_TOKEN_RESERVED_WORD + 53;
  PASCAL_RESERVED_WORD_EXCEPT               = PASCAL_TOKEN_RESERVED_WORD + 54;
  PASCAL_RESERVED_WORD_EXPORT               = PASCAL_TOKEN_RESERVED_WORD + 57;
  PASCAL_RESERVED_WORD_HELPER               = PASCAL_TOKEN_RESERVED_WORD + 58;
  PASCAL_RESERVED_WORD_PASCAL               = PASCAL_TOKEN_RESERVED_WORD + 59;
  PASCAL_RESERVED_WORD_PUBLIC               = PASCAL_TOKEN_RESERVED_WORD + 60;
  PASCAL_RESERVED_WORD_RESULT               = PASCAL_TOKEN_RESERVED_WORD + 61;
  PASCAL_RESERVED_WORD_STATIC               = PASCAL_TOKEN_RESERVED_WORD + 62;
  PASCAL_RESERVED_WORD_STORED               = PASCAL_TOKEN_RESERVED_WORD + 63;
  PASCAL_RESERVED_WORD_STRICT               = PASCAL_TOKEN_RESERVED_WORD + 64;
  PASCAL_RESERVED_WORD_WINAPI               = PASCAL_TOKEN_RESERVED_WORD + 65;
  PASCAL_RESERVED_WORD_PROGRAM              = PASCAL_TOKEN_RESERVED_WORD + 66;
  PASCAL_RESERVED_WORD_EXPORTS              = PASCAL_TOKEN_RESERVED_WORD + 67;
  PASCAL_RESERVED_WORD_FINALLY              = PASCAL_TOKEN_RESERVED_WORD + 68;
  PASCAL_RESERVED_WORD_LIBRARY              = PASCAL_TOKEN_RESERVED_WORD + 69;
  PASCAL_RESERVED_WORD_CPPDECL              = PASCAL_TOKEN_RESERVED_WORD + 70;
  PASCAL_RESERVED_WORD_DEFAULT              = PASCAL_TOKEN_RESERVED_WORD + 71;
  PASCAL_RESERVED_WORD_DYNAMIC              = PASCAL_TOKEN_RESERVED_WORD + 72;
  PASCAL_RESERVED_WORD_FORWARD              = PASCAL_TOKEN_RESERVED_WORD + 73;
  PASCAL_RESERVED_WORD_GENERIC              = PASCAL_TOKEN_RESERVED_WORD + 74;
  PASCAL_RESERVED_WORD_IOCHECK              = PASCAL_TOKEN_RESERVED_WORD + 75;
  PASCAL_RESERVED_WORD_MESSAGE              = PASCAL_TOKEN_RESERVED_WORD + 76;
  PASCAL_RESERVED_WORD_PRIVATE              = PASCAL_TOKEN_RESERVED_WORD + 77;
  PASCAL_RESERVED_WORD_STDCALL              = PASCAL_TOKEN_RESERVED_WORD + 78;
  PASCAL_RESERVED_WORD_VARARGS              = PASCAL_TOKEN_RESERVED_WORD + 79;
  PASCAL_RESERVED_WORD_VIRTUAL              = PASCAL_TOKEN_RESERVED_WORD + 80;
  PASCAL_RESERVED_WORD_ABSOLUTE             = PASCAL_TOKEN_RESERVED_WORD + 81;
  PASCAL_RESERVED_WORD_FUNCTION             = PASCAL_TOKEN_RESERVED_WORD + 82;
  PASCAL_RESERVED_WORD_OPERATOR             = PASCAL_TOKEN_RESERVED_WORD + 83;
  PASCAL_RESERVED_WORD_PROPERTY             = PASCAL_TOKEN_RESERVED_WORD + 84;
  PASCAL_RESERVED_WORD_INHERITED            = PASCAL_TOKEN_RESERVED_WORD + 85;
  PASCAL_RESERVED_WORD_INTERFACE            = PASCAL_TOKEN_RESERVED_WORD + 86;
  PASCAL_RESERVED_WORD_PROCEDURE            = PASCAL_TOKEN_RESERVED_WORD + 87;
  PASCAL_RESERVED_WORD_THREADVAR            = PASCAL_TOKEN_RESERVED_WORD + 88;
  PASCAL_RESERVED_WORD_DESTRUCTOR           = PASCAL_TOKEN_RESERVED_WORD + 89;
  PASCAL_RESERVED_WORD_CONSTRUCTOR          = PASCAL_TOKEN_RESERVED_WORD + 90;
  PASCAL_RESERVED_WORD_REINTRODUCE          = PASCAL_TOKEN_RESERVED_WORD + 91;
  PASCAL_RESERVED_WORD_FINALIZATION         = PASCAL_TOKEN_RESERVED_WORD + 92;
  PASCAL_RESERVED_WORD_DISPINTERFACE        = PASCAL_TOKEN_RESERVED_WORD + 93;
  PASCAL_RESERVED_WORD_IMPLEMENTATION       = PASCAL_TOKEN_RESERVED_WORD + 94;
  PASCAL_RESERVED_WORD_INITIALIZATION       = PASCAL_TOKEN_RESERVED_WORD + 95;
  PASCAL_RESERVED_WORD_RESOURCESTRING       = PASCAL_TOKEN_RESERVED_WORD + 96;
  PASCAL_RESERVED_WORD_ABSTRACT             = PASCAL_TOKEN_RESERVED_WORD + 98;
  PASCAL_RESERVED_WORD_CONTINUE             = PASCAL_TOKEN_RESERVED_WORD + 99;
  PASCAL_RESERVED_WORD_ASSEMBLER            = PASCAL_TOKEN_RESERVED_WORD + 100;
  PASCAL_RESERVED_WORD_BITPACKED            = PASCAL_TOKEN_RESERVED_WORD + 101;
  PASCAL_RESERVED_WORD_DEPRECATED           = PASCAL_TOKEN_RESERVED_WORD + 102;
  PASCAL_RESERVED_WORD_ENUMERATOR           = PASCAL_TOKEN_RESERVED_WORD + 103;
  PASCAL_RESERVED_WORD_EXPERIMENTAL         = PASCAL_TOKEN_RESERVED_WORD + 104;
  PASCAL_RESERVED_WORD_EXTERNAL             = PASCAL_TOKEN_RESERVED_WORD + 105;
  PASCAL_RESERVED_WORD_FAR16                = PASCAL_TOKEN_RESERVED_WORD + 106;
  PASCAL_RESERVED_WORD_IMPLEMENTS           = PASCAL_TOKEN_RESERVED_WORD + 107;
  PASCAL_RESERVED_WORD_INTERRUPT            = PASCAL_TOKEN_RESERVED_WORD + 108;
  PASCAL_RESERVED_WORD_NODEFAULT            = PASCAL_TOKEN_RESERVED_WORD + 109;
  PASCAL_RESERVED_WORD_NORETURN             = PASCAL_TOKEN_RESERVED_WORD + 110;
  PASCAL_RESERVED_WORD_NOSTACKFRAME         = PASCAL_TOKEN_RESERVED_WORD + 111;
  PASCAL_RESERVED_WORD_OLDFPCCALL           = PASCAL_TOKEN_RESERVED_WORD + 112;
  PASCAL_RESERVED_WORD_OTHERWISE            = PASCAL_TOKEN_RESERVED_WORD + 113;
  PASCAL_RESERVED_WORD_OVERLOAD             = PASCAL_TOKEN_RESERVED_WORD + 114;
  PASCAL_RESERVED_WORD_OVERRIDE             = PASCAL_TOKEN_RESERVED_WORD + 115;
  PASCAL_RESERVED_WORD_PLATFORM             = PASCAL_TOKEN_RESERVED_WORD + 116;
  PASCAL_RESERVED_WORD_PROTECTED            = PASCAL_TOKEN_RESERVED_WORD + 117;
  PASCAL_RESERVED_WORD_PUBLISHED            = PASCAL_TOKEN_RESERVED_WORD + 118;
  PASCAL_RESERVED_WORD_REGISTER             = PASCAL_TOKEN_RESERVED_WORD + 119;
  PASCAL_RESERVED_WORD_SAFECALL             = PASCAL_TOKEN_RESERVED_WORD + 121;
  PASCAL_RESERVED_WORD_SAVEREGISTERS        = PASCAL_TOKEN_RESERVED_WORD + 122;
  PASCAL_RESERVED_WORD_SOFTFLOAT            = PASCAL_TOKEN_RESERVED_WORD + 123;
  PASCAL_RESERVED_WORD_SPECIALIZE           = PASCAL_TOKEN_RESERVED_WORD + 124;
  PASCAL_RESERVED_WORD_UNALIGNED            = PASCAL_TOKEN_RESERVED_WORD + 125;
  PASCAL_RESERVED_WORD_UNIMPLEMENTED        = PASCAL_TOKEN_RESERVED_WORD + 126;
  PASCAL_RESERVED_WORD_AS                   = PASCAL_TOKEN_RESERVED_WORD + 127;
  PASCAL_RESERVED_WORD_IS                   = PASCAL_TOKEN_RESERVED_WORD + 128;
  PASCAL_RESERVED_WORD_ON                   = PASCAL_TOKEN_RESERVED_WORD + 129;
  PASCAL_RESERVED_WORD_OUT                  = PASCAL_TOKEN_RESERVED_WORD + 130;
  PASCAL_RESERVED_WORD_TRY                  = PASCAL_TOKEN_RESERVED_WORD + 131;
  PASCAL_RESERVED_WORD_FAR                  = PASCAL_TOKEN_RESERVED_WORD + 132;

type
TPascalToken = record
  Kind: TPascalTokenKind;
  S: PAnsiChar;
  SEnd: PAnsiChar;
end;

function NextPascalToken(Cursor, SourceEnd: PAnsiChar;
                         out Token: TPascalToken;
                         out NextCursor: PAnsiChar): Boolean;

implementation

const
  U0 = {$IF Defined(ENDIAN_LITTLE)}  0 {$ELSE} 24 {$ENDIF};
  U1 = {$IF Defined(ENDIAN_LITTLE)}  8 {$ELSE} 16 {$ENDIF};
  U2 = {$IF Defined(ENDIAN_LITTLE)} 16 {$ELSE}  8 {$ENDIF};
  U3 = {$IF Defined(ENDIAN_LITTLE)} 24 {$ELSE}  0 {$ENDIF};
  W0 = {$IF Defined(ENDIAN_LITTLE)}  0 {$ELSE}  8 {$ENDIF};
  W1 = {$IF Defined(ENDIAN_LITTLE)}  8 {$ELSE}  0 {$ENDIF};
  MASK_UPPER_1BYTE  =  %11011111 shl U0;
  MASK_UPPER_2BYTES = (%11011111 shl U0) or (%11011111 shl U1);
  MASK_UPPER_3BYTES = (%11011111 shl U0) or (%11011111 shl U1) or
                      (%11011111 shl U2);
  MASK_UPPER_4BYTES = (%11011111 shl U0) or (%11011111 shl U1) or
                      (%11011111 shl U2) or (%11011111 shl U3);

function _Read4Bytes(S, SEnd: PAnsiChar): UInt32; inline;
begin
  if S + 4 <= SEnd then begin
    Exit(PUInt32(S)^);
  end else case SEnd - S of
    0: Exit(0);
    1: Exit(PByte(S)[0] shl U0);
    2: Exit((PByte(S)[0] shl U0) or (PByte(S)[1] shl U1));
    3: Exit((PByte(S)[0] shl U0) or (PByte(S)[1] shl U1) or
            (PByte(S)[2] shl U2));
  else
    Exit(0);
  end;
end;

function SkipIdentifier(Cursor, SourceEnd: PAnsiChar; FirstBytes: UInt32 = 0): PAnsiChar; inline;
begin
  while Cursor + 4 <= SourceEnd do begin
    FirstBytes := PUInt32(Cursor)^;
    if not (AnsiChar((FirstBytes shr U0) and $ff) in ['a'..'z', 'A'..'Z', '_', '0'..'9']) then
      Exit(Cursor);
    if not (AnsiChar((FirstBytes shr U1) and $ff) in ['a'..'z', 'A'..'Z', '_', '0'..'9']) then
      Exit(Cursor + 1);
    if not (AnsiChar((FirstBytes shr U2) and $ff) in ['a'..'z', 'A'..'Z', '_', '0'..'9']) then
      Exit(Cursor + 2);
    if not (AnsiChar((FirstBytes shr U3) and $ff) in ['a'..'z', 'A'..'Z', '_', '0'..'9']) then
      Exit(Cursor + 3);
    Inc(Cursor, 4);
  end;
  while Cursor < SourceEnd do begin
    if not (Cursor^ in ['a'..'z', 'A'..'Z', '_', '0'..'9']) then
      Exit(Cursor);
    Inc(Cursor);
  end;
  Exit(SourceEnd);
end;

function IsInsensitiveCaseStringMatch(S, SEnd: PAnsiChar;
                                      const T: AnsiString): Boolean; inline;
var
  I: PtrInt;
begin
  if SEnd - S <> Length(T) then
    Exit(False);
  for I := 1 to Length(T) do begin
    if (Ord(S^) and %11011111) <> Ord(T[I]) then
      Exit(False);
    Inc(S);
  end;
  Exit(True);
end;

function ClassifyReservedWord(
                FirstBytes: UInt32;
                S: PAnsiChar;
                SEnd: PAnsiChar): TPascalTokenKind;
begin
  case FirstBytes and MASK_UPPER_4BYTES of
  (Ord('C') shl U0) or (Ord('A') shl U1) or (Ord('S') shl U2) or (Ord('E') shl U3):
    if SEnd - S = 4 then
      Exit(PASCAL_RESERVED_WORD_CASE);
  (Ord('E') shl U0) or (Ord('L') shl U1) or (Ord('S') shl U2) or (Ord('E') shl U3):
    if SEnd - S = 4 then
      Exit(PASCAL_RESERVED_WORD_ELSE);
  (Ord('F') shl U0) or (Ord('I') shl U1) or (Ord('L') shl U2) or (Ord('E') shl U3):
    if SEnd - S = 4 then
      Exit(PASCAL_RESERVED_WORD_FILE);
  (Ord('G') shl U0) or (Ord('O') shl U1) or (Ord('T') shl U2) or (Ord('O') shl U3):
    if SEnd - S = 4 then
      Exit(PASCAL_RESERVED_WORD_GOTO);
  (Ord('S') shl U0) or (Ord('E') shl U1) or (Ord('L') shl U2) or (Ord('F') shl U3):
    if SEnd - S = 4 then
      Exit(PASCAL_RESERVED_WORD_SELF);
  (Ord('T') shl U0) or (Ord('H') shl U1) or (Ord('E') shl U2) or (Ord('N') shl U3):
    if SEnd - S = 4 then
      Exit(PASCAL_RESERVED_WORD_THEN);
  (Ord('T') shl U0) or (Ord('Y') shl U1) or (Ord('P') shl U2) or (Ord('E') shl U3):
    if SEnd - S = 4 then
      Exit(PASCAL_RESERVED_WORD_TYPE);
  (Ord('U') shl U0) or (Ord('N') shl U1) or (Ord('I') shl U2) or (Ord('T') shl U3):
    if SEnd - S = 4 then
      Exit(PASCAL_RESERVED_WORD_UNIT);
  (Ord('U') shl U0) or (Ord('S') shl U1) or (Ord('E') shl U2) or (Ord('S') shl U3):
    if SEnd - S = 4 then
      Exit(PASCAL_RESERVED_WORD_USES);
  (Ord('W') shl U0) or (Ord('I') shl U1) or (Ord('T') shl U2) or (Ord('H') shl U3):
    if SEnd - S = 4 then
      Exit(PASCAL_RESERVED_WORD_WITH);
  (Ord('C') shl U0) or (Ord('V') shl U1) or (Ord('A') shl U2) or (Ord('R') shl U3):
    if SEnd - S = 4 then
      Exit(PASCAL_RESERVED_WORD_CVAR);
  (Ord('N') shl U0) or (Ord('A') shl U1) or (Ord('M') shl U2) or (Ord('E') shl U3):
    if SEnd - S = 4 then
      Exit(PASCAL_RESERVED_WORD_NAME);
  (Ord('N') shl U0) or (Ord('E') shl U1) or (Ord('A') shl U2) or (Ord('R') shl U3):
    if SEnd - S = 4 then
      Exit(PASCAL_RESERVED_WORD_NEAR);
  (Ord('R') shl U0) or (Ord('E') shl U1) or (Ord('A') shl U2) or (Ord('D') shl U3):
    if SEnd - S = 4 then
      Exit(PASCAL_RESERVED_WORD_READ);
  (Ord('A') shl U0) or (Ord('R') shl U1) or (Ord('R') shl U2) or (Ord('A') shl U3):
    if (SEnd - S = 5) and ((Ord(S[4]) and %11011111) = Ord('Y')) then
      Exit(PASCAL_RESERVED_WORD_ARRAY);
  (Ord('B') shl U0) or (Ord('E') shl U1) or (Ord('G') shl U2) or (Ord('I') shl U3):
    if (SEnd - S = 5) and ((Ord(S[4]) and %11011111) = Ord('N')) then
      Exit(PASCAL_RESERVED_WORD_BEGIN);
  (Ord('C') shl U0) or (Ord('O') shl U1) or (Ord('N') shl U2) or (Ord('S') shl U3):
    if (SEnd - S = 5) and ((Ord(S[4]) and %11011111) = Ord('T')) then begin
      Exit(PASCAL_RESERVED_WORD_CONST);
    end else if SEnd - S = 11 then begin
      if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'tructor') then
        Exit(PASCAL_RESERVED_WORD_CONSTRUCTOR);
    end;
  (Ord('L') shl U0) or (Ord('A') shl U1) or (Ord('B') shl U2) or (Ord('E') shl U3):
    if (SEnd - S = 5) and ((Ord(S[4]) and %11011111) = Ord('L')) then
      Exit(PASCAL_RESERVED_WORD_LABEL);
  (Ord('U') shl U0) or (Ord('N') shl U1) or (Ord('T') shl U2) or (Ord('I') shl U3):
    if (SEnd - S = 5) and ((Ord(S[4]) and %11011111) = Ord('L')) then
      Exit(PASCAL_RESERVED_WORD_UNTIL);
  (Ord('W') shl U0) or (Ord('H') shl U1) or (Ord('I') shl U2) or (Ord('L') shl U3):
    if (SEnd - S = 5) and ((Ord(S[4]) and %11011111) = Ord('E')) then
      Exit(PASCAL_RESERVED_WORD_WHILE);
  (Ord('C') shl U0) or (Ord('L') shl U1) or (Ord('A') shl U2) or (Ord('S') shl U3):
    if (SEnd - S = 5) and ((Ord(S[4]) and %11011111) = Ord('S')) then
      Exit(PASCAL_RESERVED_WORD_CLASS);
  (Ord('R') shl U0) or (Ord('A') shl U1) or (Ord('I') shl U2) or (Ord('S') shl U3):
    if (SEnd - S = 5) and ((Ord(S[4]) and %11011111) = Ord('E')) then
      Exit(PASCAL_RESERVED_WORD_RAISE);
  (Ord('A') shl U0) or (Ord('L') shl U1) or (Ord('I') shl U2) or (Ord('A') shl U3):
    if (SEnd - S = 5) and ((Ord(S[4]) and %11011111) = Ord('S')) then
      Exit(PASCAL_RESERVED_WORD_ALIAS);
  (Ord('B') shl U0) or (Ord('R') shl U1) or (Ord('E') shl U2) or (Ord('A') shl U3):
    if (SEnd - S = 5) and ((Ord(S[4]) and %11011111) = Ord('K')) then
      Exit(PASCAL_RESERVED_WORD_BREAK);
  (Ord('C') shl U0) or (Ord('D') shl U1) or (Ord('E') shl U2) or (Ord('C') shl U3):
    if (SEnd - S = 5) and ((Ord(S[4]) and %11011111) = Ord('L')) then
      Exit(PASCAL_RESERVED_WORD_CDECL);
  (Ord('I') shl U0) or (Ord('N') shl U1) or (Ord('D') shl U2) or (Ord('E') shl U3):
    if (SEnd - S = 5) and ((Ord(S[4]) and %11011111) = Ord('X')) then
      Exit(PASCAL_RESERVED_WORD_INDEX);
  (Ord('L') shl U0) or (Ord('O') shl U1) or (Ord('C') shl U2) or (Ord('A') shl U3):
    if (SEnd - S = 5) and ((Ord(S[4]) and %11011111) = Ord('L')) then
      Exit(PASCAL_RESERVED_WORD_LOCAL);
  (Ord('W') shl U0) or (Ord('R') shl U1) or (Ord('I') shl U2) or (Ord('T') shl U3):
    if (SEnd - S = 5) and ((Ord(S[4]) and %11011111) = Ord('E')) then
      Exit(PASCAL_RESERVED_WORD_WRITE);
  (Ord('D') shl U0) or (Ord('O') shl U1) or (Ord('W') shl U2) or (Ord('N') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'TO') then
      Exit(PASCAL_RESERVED_WORD_DOWNTO);
  (Ord('I') shl U0) or (Ord('N') shl U1) or (Ord('L') shl U2) or (Ord('I') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'NE') then
      Exit(PASCAL_RESERVED_WORD_INLINE);
  (Ord('O') shl U0) or (Ord('B') shl U1) or (Ord('J') shl U2) or (Ord('E') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'CT') then
      Exit(PASCAL_RESERVED_WORD_INLINE);
  (Ord('P') shl U0) or (Ord('A') shl U1) or (Ord('C') shl U2) or (Ord('K') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ED') then
      Exit(PASCAL_RESERVED_WORD_INLINE);
  (Ord('R') shl U0) or (Ord('E') shl U1) or (Ord('C') shl U2) or (Ord('O') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'RD') then
      Exit(PASCAL_RESERVED_WORD_INLINE);
  (Ord('R') shl U0) or (Ord('E') shl U1) or (Ord('P') shl U2) or (Ord('E') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'AT') then
      Exit(PASCAL_RESERVED_WORD_REPEAT);
  (Ord('S') shl U0) or (Ord('T') shl U1) or (Ord('R') shl U2) or (Ord('I') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'NG') then begin
      Exit(PASCAL_RESERVED_WORD_STRING);
    end else if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'CT') then
      Exit(PASCAL_RESERVED_WORD_STRICT);
  (Ord('E') shl U0) or (Ord('X') shl U1) or (Ord('C') shl U2) or (Ord('E') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'PT') then
      Exit(PASCAL_RESERVED_WORD_EXCEPT);
  (Ord('E') shl U0) or (Ord('X') shl U1) or (Ord('P') shl U2) or (Ord('O') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'RT') then begin
      Exit(PASCAL_RESERVED_WORD_EXPORT);
    end else if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'RTS') then
      Exit(PASCAL_RESERVED_WORD_EXPORTS);
  (Ord('H') shl U0) or (Ord('E') shl U1) or (Ord('L') shl U2) or (Ord('P') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ER') then
      Exit(PASCAL_RESERVED_WORD_HELPER);
  (Ord('P') shl U0) or (Ord('A') shl U1) or (Ord('S') shl U2) or (Ord('C') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'AL') then
      Exit(PASCAL_RESERVED_WORD_PASCAL);
  (Ord('P') shl U0) or (Ord('U') shl U1) or (Ord('B') shl U2) or (Ord('L') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'IC') then begin
      Exit(PASCAL_RESERVED_WORD_PUBLIC);
    end else if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ISHED') then
      Exit(PASCAL_RESERVED_WORD_PUBLISHED);
  (Ord('R') shl U0) or (Ord('E') shl U1) or (Ord('S') shl U2) or (Ord('U') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'LT') then
      Exit(PASCAL_RESERVED_WORD_RESULT);
  (Ord('S') shl U0) or (Ord('T') shl U1) or (Ord('A') shl U2) or (Ord('T') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'IC') then
      Exit(PASCAL_RESERVED_WORD_STATIC);
  (Ord('S') shl U0) or (Ord('T') shl U1) or (Ord('O') shl U2) or (Ord('R') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ED') then
      Exit(PASCAL_RESERVED_WORD_STORED);
  (Ord('W') shl U0) or (Ord('I') shl U1) or (Ord('N') shl U2) or (Ord('A') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'PI') then
      Exit(PASCAL_RESERVED_WORD_WINAPI);
  (Ord('P') shl U0) or (Ord('R') shl U1) or (Ord('O') shl U2) or (Ord('G') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'RAM') then
      Exit(PASCAL_RESERVED_WORD_PROGRAM);
  (Ord('F') shl U0) or (Ord('I') shl U1) or (Ord('N') shl U2) or (Ord('A') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'LLY') then begin
      Exit(PASCAL_RESERVED_WORD_PROGRAM);
    end else if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'lization') then
      Exit(PASCAL_RESERVED_WORD_FINALIZATION);
  (Ord('L') shl U0) or (Ord('I') shl U1) or (Ord('B') shl U2) or (Ord('R') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ARY') then
      Exit(PASCAL_RESERVED_WORD_LIBRARY);
  (Ord('C') shl U0) or (Ord('P') shl U1) or (Ord('P') shl U2) or (Ord('D') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ECL') then
      Exit(PASCAL_RESERVED_WORD_CPPDECL);
  (Ord('D') shl U0) or (Ord('E') shl U1) or (Ord('F') shl U2) or (Ord('A') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ULT') then
      Exit(PASCAL_RESERVED_WORD_DEFAULT);
  (Ord('D') shl U0) or (Ord('Y') shl U1) or (Ord('N') shl U2) or (Ord('A') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'MIC') then
      Exit(PASCAL_RESERVED_WORD_DYNAMIC);
  (Ord('F') shl U0) or (Ord('O') shl U1) or (Ord('R') shl U2) or (Ord('W') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ARD') then
      Exit(PASCAL_RESERVED_WORD_FORWARD);
  (Ord('G') shl U0) or (Ord('E') shl U1) or (Ord('N') shl U2) or (Ord('E') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'RIC') then
      Exit(PASCAL_RESERVED_WORD_GENERIC);
  (Ord('I') shl U0) or (Ord('O') shl U1) or (Ord('C') shl U2) or (Ord('H') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ECH') then
      Exit(PASCAL_RESERVED_WORD_IOCHECK);
  (Ord('M') shl U0) or (Ord('E') shl U1) or (Ord('S') shl U2) or (Ord('S') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'AGE') then
      Exit(PASCAL_RESERVED_WORD_MESSAGE);
  (Ord('P') shl U0) or (Ord('R') shl U1) or (Ord('I') shl U2) or (Ord('V') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ATE') then
      Exit(PASCAL_RESERVED_WORD_PRIVATE);
  (Ord('S') shl U0) or (Ord('T') shl U1) or (Ord('D') shl U2) or (Ord('C') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ALL') then
      Exit(PASCAL_RESERVED_WORD_STDCALL);
  (Ord('V') shl U0) or (Ord('A') shl U1) or (Ord('R') shl U2) or (Ord('A') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'RGS') then
      Exit(PASCAL_RESERVED_WORD_VARARGS);
  (Ord('V') shl U0) or (Ord('I') shl U1) or (Ord('R') shl U2) or (Ord('T') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'UAL') then
      Exit(PASCAL_RESERVED_WORD_VIRTUAL);
  (Ord('A') shl U0) or (Ord('B') shl U1) or (Ord('S') shl U2) or (Ord('O') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'LUTE') then
      Exit(PASCAL_RESERVED_WORD_ABSOLUTE);
  (Ord('F') shl U0) or (Ord('U') shl U1) or (Ord('N') shl U2) or (Ord('C') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'TION') then
      Exit(PASCAL_RESERVED_WORD_FUNCTION);
  (Ord('O') shl U0) or (Ord('P') shl U1) or (Ord('E') shl U2) or (Ord('R') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ATOR') then
      Exit(PASCAL_RESERVED_WORD_OPERATOR);
  (Ord('P') shl U0) or (Ord('R') shl U1) or (Ord('O') shl U2) or (Ord('P') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ERTY') then
      Exit(PASCAL_RESERVED_WORD_PROPERTY);
  (Ord('I') shl U0) or (Ord('N') shl U1) or (Ord('H') shl U2) or (Ord('E') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'RITED') then
      Exit(PASCAL_RESERVED_WORD_INHERITED);
  (Ord('I') shl U0) or (Ord('N') shl U1) or (Ord('T') shl U2) or (Ord('E') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'RFACE') then begin
      Exit(PASCAL_RESERVED_WORD_INTERFACE);
    end else if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'RRUPT') then
      Exit(PASCAL_RESERVED_WORD_INTERRUPT);
  (Ord('P') shl U0) or (Ord('R') shl U1) or (Ord('O') shl U2) or (Ord('C') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'EDURE') then
      Exit(PASCAL_RESERVED_WORD_PROCEDURE);
  (Ord('T') shl U0) or (Ord('H') shl U1) or (Ord('R') shl U2) or (Ord('E') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ADVAR') then
      Exit(PASCAL_RESERVED_WORD_THREADVAR);
  (Ord('D') shl U0) or (Ord('E') shl U1) or (Ord('S') shl U2) or (Ord('T') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'RUCTOR') then
      Exit(PASCAL_RESERVED_WORD_DESTRUCTOR);
  (Ord('R') shl U0) or (Ord('E') shl U1) or (Ord('I') shl U2) or (Ord('N') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'TRODUCE') then
      Exit(PASCAL_RESERVED_WORD_REINTRODUCE);
  (Ord('D') shl U0) or (Ord('I') shl U1) or (Ord('S') shl U2) or (Ord('P') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'INTERFACE') then
      Exit(PASCAL_RESERVED_WORD_DISPINTERFACE);
  (Ord('I') shl U0) or (Ord('M') shl U1) or (Ord('P') shl U2) or (Ord('L') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'EMENTATION') then begin
      Exit(PASCAL_RESERVED_WORD_IMPLEMENTATION);
    end else if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'EMENTS') then
      Exit(PASCAL_RESERVED_WORD_IMPLEMENTS);
  (Ord('I') shl U0) or (Ord('N') shl U1) or (Ord('I') shl U2) or (Ord('T') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'IALIZATION') then
      Exit(PASCAL_RESERVED_WORD_INITIALIZATION);
  (Ord('R') shl U0) or (Ord('E') shl U1) or (Ord('S') shl U2) or (Ord('O') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'URCESTRING') then
      Exit(PASCAL_RESERVED_WORD_RESOURCESTRING);
  (Ord('A') shl U0) or (Ord('B') shl U1) or (Ord('S') shl U2) or (Ord('T') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'RACT') then
      Exit(PASCAL_RESERVED_WORD_ABSTRACT);
  (Ord('A') shl U0) or (Ord('S') shl U1) or (Ord('S') shl U2) or (Ord('E') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'MBLER') then
      Exit(PASCAL_RESERVED_WORD_ASSEMBLER);
  (Ord('B') shl U0) or (Ord('I') shl U1) or (Ord('T') shl U2) or (Ord('P') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ACKED') then
      Exit(PASCAL_RESERVED_WORD_BITPACKED);
  (Ord('C') shl U0) or (Ord('O') shl U1) or (Ord('N') shl U2) or (Ord('T') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'INUE') then
      Exit(PASCAL_RESERVED_WORD_CONTINUE);
  (Ord('D') shl U0) or (Ord('E') shl U1) or (Ord('P') shl U2) or (Ord('R') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'DEPRECATED') then
      Exit(PASCAL_RESERVED_WORD_DEPRECATED);
  (Ord('E') shl U0) or (Ord('N') shl U1) or (Ord('U') shl U2) or (Ord('M') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ERATOR') then
      Exit(PASCAL_RESERVED_WORD_ENUMERATOR);
  (Ord('E') shl U0) or (Ord('X') shl U1) or (Ord('P') shl U2) or (Ord('E') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'RIMENTAL') then
      Exit(PASCAL_RESERVED_WORD_EXPERIMENTAL);
  (Ord('E') shl U0) or (Ord('X') shl U1) or (Ord('T') shl U2) or (Ord('E') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'RNAL') then
      Exit(PASCAL_RESERVED_WORD_EXTERNAL);
  (Ord('F') shl U0) or (Ord('A') shl U1) or (Ord('R') shl U2) or (Ord('1') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, '6') then
      Exit(PASCAL_RESERVED_WORD_FAR16);
  (Ord('N') shl U0) or (Ord('O') shl U1) or (Ord('D') shl U2) or (Ord('E') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'FAULT') then
      Exit(PASCAL_RESERVED_WORD_NODEFAULT);
  (Ord('N') shl U0) or (Ord('O') shl U1) or (Ord('R') shl U2) or (Ord('E') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'TURN') then
      Exit(PASCAL_RESERVED_WORD_NORETURN);
  (Ord('N') shl U0) or (Ord('O') shl U1) or (Ord('S') shl U2) or (Ord('T') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ACKFRAME') then
      Exit(PASCAL_RESERVED_WORD_NOSTACKFRAME);
  (Ord('O') shl U0) or (Ord('L') shl U1) or (Ord('D') shl U2) or (Ord('F') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'PCCALL') then
      Exit(PASCAL_RESERVED_WORD_OLDFPCCALL);
  (Ord('O') shl U0) or (Ord('T') shl U1) or (Ord('H') shl U2) or (Ord('E') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'RWISE') then
      Exit(PASCAL_RESERVED_WORD_OTHERWISE);
  (Ord('O') shl U0) or (Ord('V') shl U1) or (Ord('E') shl U2) or (Ord('R') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'LOAD') then begin
      Exit(PASCAL_RESERVED_WORD_OVERLOAD);
    end else if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'RIDE') then
      Exit(PASCAL_RESERVED_WORD_OVERRIDE);
  (Ord('P') shl U0) or (Ord('L') shl U1) or (Ord('A') shl U2) or (Ord('T') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'FORM') then
      Exit(PASCAL_RESERVED_WORD_PLATFORM);
  (Ord('P') shl U0) or (Ord('R') shl U1) or (Ord('O') shl U2) or (Ord('T') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'ECTED') then
      Exit(PASCAL_RESERVED_WORD_PROTECTED);
  (Ord('R') shl U0) or (Ord('E') shl U1) or (Ord('G') shl U2) or (Ord('I') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'STER') then
      Exit(PASCAL_RESERVED_WORD_REGISTER);
  (Ord('S') shl U0) or (Ord('A') shl U1) or (Ord('F') shl U2) or (Ord('E') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'CALL') then
      Exit(PASCAL_RESERVED_WORD_SAFECALL);
  (Ord('S') shl U0) or (Ord('A') shl U1) or (Ord('V') shl U2) or (Ord('E') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'REGISTERS') then
      Exit(PASCAL_RESERVED_WORD_SAVEREGISTERS);
  (Ord('S') shl U0) or (Ord('O') shl U1) or (Ord('F') shl U2) or (Ord('T') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'FLOAT') then
      Exit(PASCAL_RESERVED_WORD_SOFTFLOAT);
  (Ord('S') shl U0) or (Ord('P') shl U1) or (Ord('E') shl U2) or (Ord('C') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'IALIZE') then
      Exit(PASCAL_RESERVED_WORD_SPECIALIZE);
  (Ord('U') shl U0) or (Ord('N') shl U1) or (Ord('A') shl U2) or (Ord('L') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'IGNED') then
      Exit(PASCAL_RESERVED_WORD_UNALIGNED);
  (Ord('U') shl U0) or (Ord('N') shl U1) or (Ord('I') shl U2) or (Ord('M') shl U3):
    if IsInsensitiveCaseStringMatch(S + 4, SEnd, 'PLEMENTED') then
      Exit(PASCAL_RESERVED_WORD_UNIMPLEMENTED);
  end;
  Exit(PASCAL_TOKEN_IDENTIFIER);
end;

function IdentifierOrReservedWord(
                FirstBytes: UInt32;
                SourceEnd: PAnsiChar;
                var Token: TPascalToken): PAnsiChar;
begin
  Token.Kind := PASCAL_TOKEN_IDENTIFIER;
  // no 1-letter reserved words
  if not (AnsiChar((FirstBytes shr U1) and $ff) in ['a'..'z', 'A'..'Z', '_', '0'..'9']) then
    Exit(Token.S + 1);
  if not (AnsiChar((FirstBytes shr U2) and $ff) in ['a'..'z', 'A'..'Z', '_', '0'..'9']) then begin
    case FirstBytes and ((%11011111 shl U0) or (%11011111 shl U1)) of
    (Ord('D') shl U0) or (Ord('O') shl U1):
      Token.Kind := PASCAL_RESERVED_WORD_DO;
    (Ord('I') shl U0) or (Ord('F') shl U1):
      Token.Kind := PASCAL_RESERVED_WORD_IF;
    (Ord('I') shl U0) or (Ord('N') shl U1):
      Token.Kind := PASCAL_RESERVED_WORD_IN;
    (Ord('O') shl U0) or (Ord('F') shl U1):
      Token.Kind := PASCAL_RESERVED_WORD_OF;
    (Ord('O') shl U0) or (Ord('R') shl U1):
      Token.Kind := PASCAL_RESERVED_WORD_OR;
    (Ord('T') shl U0) or (Ord('O') shl U1):
      Token.Kind := PASCAL_RESERVED_WORD_TO;
    (Ord('A') shl U0) or (Ord('S') shl U1):
      Token.Kind := PASCAL_RESERVED_WORD_AS;
    (Ord('I') shl U0) or (Ord('S') shl U1):
      Token.Kind := PASCAL_RESERVED_WORD_IS;
    (Ord('O') shl U0) or (Ord('N') shl U1):
      Token.Kind := PASCAL_RESERVED_WORD_ON;
    end;
    Exit(Token.S + 2);
  end;
  if not (AnsiChar((FirstBytes shr U3) and $ff) in ['a'..'z', 'A'..'Z', '_', '0'..'9']) then begin
    case FirstBytes and ((%11011111 shl U0) or (%11011111 shl U1) or (%11011111 shl U2)) of
    (Ord('A') shl U0) or (Ord('N') shl U1) or (Ord('D') shl U2):
      Token.Kind := PASCAL_RESERVED_WORD_AND;
    (Ord('A') shl U0) or (Ord('S') shl U1) or (Ord('M') shl U2):
      Token.Kind := PASCAL_RESERVED_WORD_ASM;
    (Ord('D') shl U0) or (Ord('I') shl U1) or (Ord('V') shl U2):
      Token.Kind := PASCAL_RESERVED_WORD_DIV;
    (Ord('E') shl U0) or (Ord('N') shl U1) or (Ord('D') shl U2):
      Token.Kind := PASCAL_RESERVED_WORD_END;
    (Ord('F') shl U0) or (Ord('O') shl U1) or (Ord('R') shl U2):
      Token.Kind := PASCAL_RESERVED_WORD_FOR;
    (Ord('M') shl U0) or (Ord('O') shl U1) or (Ord('D') shl U2):
      Token.Kind := PASCAL_RESERVED_WORD_MOD;
    (Ord('N') shl U0) or (Ord('I') shl U1) or (Ord('L') shl U2):
      Token.Kind := PASCAL_RESERVED_WORD_NIL;
    (Ord('N') shl U0) or (Ord('O') shl U1) or (Ord('T') shl U2):
      Token.Kind := PASCAL_RESERVED_WORD_NOT;
    (Ord('S') shl U0) or (Ord('E') shl U1) or (Ord('T') shl U2):
      Token.Kind := PASCAL_RESERVED_WORD_SET;
    (Ord('S') shl U0) or (Ord('H') shl U1) or (Ord('L') shl U2):
      Token.Kind := PASCAL_RESERVED_WORD_SHL;
    (Ord('S') shl U0) or (Ord('H') shl U1) or (Ord('R') shl U2):
      Token.Kind := PASCAL_RESERVED_WORD_SHR;
    (Ord('V') shl U0) or (Ord('A') shl U1) or (Ord('R') shl U2):
      Token.Kind := PASCAL_RESERVED_WORD_VAR;
    (Ord('X') shl U0) or (Ord('O') shl U1) or (Ord('R') shl U2):
      Token.Kind := PASCAL_RESERVED_WORD_XOR;
    (Ord('O') shl U0) or (Ord('U') shl U1) or (Ord('T') shl U2):
      Token.Kind := PASCAL_RESERVED_WORD_OUT;
    (Ord('T') shl U0) or (Ord('R') shl U1) or (Ord('Y') shl U2):
      Token.Kind := PASCAL_RESERVED_WORD_TRY;
    (Ord('F') shl U0) or (Ord('A') shl U1) or (Ord('R') shl U2):
      Token.Kind := PASCAL_RESERVED_WORD_FAR;
    end;
    Exit(Token.S + 3);
  end;
  SourceEnd := SkipIdentifier(Token.S + 4, SourceEnd, 0);
  Token.Kind := ClassifyReservedWord(FirstBytes, Token.S, SourceEnd);
  Exit(SourceEnd);
end;

function NumberLiteral(
                FirstBytes: UInt32;
                SourceEnd: PAnsiChar;
                var Token: TPascalToken): PAnsiChar; inline;
begin
  Exit(Token.S + 1);
end;

function StringInSingleQuotes(
                FirstBytes: UInt32;
                SourceEnd: PAnsiChar;
                var Token: TPascalToken): PAnsiChar; inline;
begin
  Exit(Token.S + 1);
end;

function SkipLine(S, SEnd: PAnsiChar{; out LineEnd: PAnsiChar}): PAnsiChar; inline;
begin
  while S < SEnd do begin
    case S^ of
      #10: begin
             Exit(S);
             //LineEnd := S;
             Exit(S + 1);
           end;
      #13: begin
             Exit(S);
             //LineEnd := S;
             if (S + 1 < SEnd) and (S[1] = #10) then
               Exit(S + 2);
             Exit(S + 1);
           end;
    end;
    Inc(S);
  end;
  //LineEnd := SEnd;
  Exit(SEnd);
end;

function SingleLineComment(
                FirstBytes: UInt32;
                SourceEnd: PAnsiChar;
                var Token: TPascalToken): PAnsiChar; inline;
begin
  Exit(Token.S + 2);
end;

function _NextPascalToken(
                FirstBytes: UInt32;
                SourceEnd: PAnsiChar;
                var Token: TPascalToken): PAnsiChar; inline;
begin
  case AnsiChar((FirstBytes shr U0) and $ff) of
  'a'..'z', 'A'..'Z', '_': Exit(IdentifierOrReservedWord(FirstBytes, SourceEnd, Token));
  '0'..'9': Exit(NumberLiteral(FirstBytes, SourceEnd, Token));
  '''': Exit(StringInSingleQuotes(FirstBytes, SourceEnd, Token));
  '/': if AnsiChar((FirstBytes shr U1) and $ff) <> '/' then begin
         Token.Kind := PASCAL_SPECIAL_SYMBOL_SLASH;
         Exit(Token.S + 1);
       end else begin
         Token.Kind := PASCAL_COMMENT_SINGLE_LINE;
         Exit(SkipLine(Token.S, SourceEnd));
       end;
  // '#': character code
  // '$': hex digit sequence
  // '&': octal digit sequence or identifier
  // '%': bin digit sequence
  // '{': multiline comment
  // '(': (* comment, (. or just (
  end;
  Token.Kind := PASCAL_TOKEN_UNKNOWN;
  Exit(Token.S + 1);
end;

function NextPascalToken(Cursor, SourceEnd: PAnsiChar;
                         out Token: TPascalToken;
                         out NextCursor: PAnsiChar): Boolean;
begin
  if Cursor >= SourceEnd then begin
    Token.Kind := PASCAL_TOKEN_EOS;
    Token.S := SourceEnd;
    Token.SEnd := SourceEnd;
    Exit(False);
  end;
  Token.S := Cursor;
  {Token.Kind := _NextPascalToken(
          _Read4Bytes(Cursor, SourceEnd),
          SourceEnd,
          Token);}
  if Cursor + 4 <= SourceEnd then begin
    SourceEnd := _NextPascalToken(PUInt32(Cursor)^, SourceEnd, Token);
  end else case SourceEnd - Cursor of
    0: SourceEnd := _NextPascalToken(0, SourceEnd, Token);
    1: SourceEnd := _NextPascalToken(
        PByte(Cursor)[0] shl U0,
        SourceEnd, Token);
    2: SourceEnd := _NextPascalToken(
        (PByte(Cursor)[0] shl U0) or (PByte(Cursor)[1] shl U1),
        SourceEnd, Token);
    3: SourceEnd := _NextPascalToken(
        (PByte(Cursor)[0] shl U0) or (PByte(Cursor)[1] shl U1) or
        (PByte(Cursor)[2] shl U2),
        SourceEnd, Token);
  else // ???
    Token.SEnd := SourceEnd;
    NextCursor := SourceEnd;
    Exit(False);
  end;
  Token.SEnd := SourceEnd;
  NextCursor := SourceEnd;
  Exit(True);
end;

end.
