// http://unicode.org/faq/utf_bom.html
unit dutf16;

interface

type
TFpcFile = file of byte;

TUtf16Reader = object
public
  procedure Init(const FileName: WideString; BOM: Boolean = True); overload;
  procedure Init(var F: TFpcFile; BOM: Boolean = True); overload;
  procedure Done;
  function Eof: Boolean;
  function ReadLine: WideString;
private const
  BUFFER_SIZE = 1024; // must be even for correct loading surrogate pairs
private
  // Opened file
  FFile: TFpcFile;
  // FBuffer[FPos .. FPos+FSize-1] is buffered input for proccessing
  FBuffer: array[0 .. BUFFER_SIZE - 1] of WideChar;
  FPos: LongInt;
  FSize: LongInt;
  // Is used for building line
  FLine: WideString;
  procedure Prepare;
  function ProcessBuffer: Boolean; // Result = end of line found?
  procedure ReadBuffer;
  procedure AppendFromBuffer(Finish: LongInt; Skip: LongInt);
end;

implementation

procedure TUtf16Reader.Init(const FileName: WideString; BOM: Boolean = True);
begin
  Assign(FFile, FileName);
  Reset(FFile);
  FPos := 0;
  FSize := 0;
end;

procedure TUtf16Reader.Init(var F: TFpcFile; BOM: Boolean = True); overload;
begin
  FFile := F;
  FPos := 0;
  FSize := 0;
end;

procedure TUtf16Reader.Done;
begin
  Close(FFile);
end;

function TUtf16Reader.Eof: Boolean;
begin
  Result := (FPos >= FSize) and System.Eof(FFile);
end;

function TUtf16Reader.ReadLine: WideString;
begin
  FLine := '';
  while not Eof do begin
    if ProcessBuffer then
      Exit(FLine);
    ReadBuffer;
  end;
  Exit(FLine);
end;

procedure TUtf16Reader.Prepare;
begin
end;

function TUtf16Reader.ProcessBuffer: Boolean;
var
  I: LongInt;
begin
  // 16-bits words in surrogate pairs are in ranges
  //   D800 .. DBFF and DC00 .. DFFF
  // No one new line codepage contains is in these ranges, so we may
  // not parse surrogate pairs for detecting end of line
  for I := FPos to FSize - 1 do begin
    if (FBuffer[I] = WideChar($0A)) or
       (FBuffer[I] = WideChar($85)) or
       (FBuffer[I] = WideChar($2028)) or
       (FBuffer[I] = WideChar($2029)) then begin
      AppendFromBuffer(I, 1);
      Exit(True);
    end;
    if FBuffer[I] = WideChar($0D) then begin
      if (I + 1 < FSize) and (FBuffer[I + 1] = WideChar($0A)) then begin
        AppendFromBuffer(I, 2);
      end else
        AppendFromBuffer(I, 1);
      Exit(True);
    end;
  end;
  if FSize - 1 >= FPos then
    AppendFromBuffer(FSize - 1, 0);
  Exit(False);
end;

procedure TUtf16Reader.ReadBuffer;
begin
  BlockRead(FFile, FBuffer[0], BUFFER_SIZE * SizeOf(WideChar), FSize);
  FPos := 0;
  FSize := FSize div 2; // TODO odd number of bytes => error invalid utf-16
end;

procedure TUtf16Reader.AppendFromBuffer(Finish: LongInt; Skip: LongInt);
var
  L, A: LongInt;
begin
  L := Length(FLine);
  A := Finish - FPos + 1;
  SetLength(FLine, L + A);
  Move(FBuffer[FPos], FLine[L], A);
  FPos := Finish + 1 + Skip;
end;

end.
