dart/pkg/compiler/lib/src/scanner/utf8_bytes_scanner.dart

// Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

part of scanner;

/**
 * Scanner that reads from a UTF-8 encoded list of bytes and creates tokens
 * that points to substrings.
 */
class Utf8BytesScanner extends ArrayBasedScanner {
  /**
   * The file content.
   *
   * The content is zero-terminated.
   */
  List<int> bytes;

  /**
   * Points to the offset of the last byte returned by [advance].
   *
   * After invoking [currentAsUnicode], the [byteOffset] points to the last
   * byte that is part of the (unicode or ASCII) character. That way, [advance]
   * can always increase the byte offset by 1.
   */
  int byteOffset = -1;

  /**
   * The getter [scanOffset] is expected to return the index where the current
   * character *starts*. In case of a non-ascii character, after invoking
   * [currentAsUnicode], the byte offset points to the *last* byte.
   *
   * This field keeps track of the number of bytes for the current unicode
   * character. For example, if bytes 7,8,9 encode one unicode character, the
   * [byteOffset] is 9 (after invoking [currentAsUnicode]). The [scanSlack]
   * will be 2, so that [scanOffset] returns 7.
   */
  int scanSlack = 0;

  /**
   * Holds the [byteOffset] value for which the current [scanSlack] is valid.
   */
  int scanSlackOffset = -1;

  /**
   * Returns the byte offset of the first byte that belongs to the current
   * character.
   */
  int get scanOffset {
    if (byteOffset == scanSlackOffset) {
      return byteOffset - scanSlack;
    } else {
      return byteOffset;
    }
  }

  /**
   * The difference between the number of bytes and the number of corresponding
   * string characters, up to the current [byteOffset].
   */
  int utf8Slack = 0;

  /**
   * Creates a new Utf8BytesScanner. The source file is expected to be a
   * [Utf8BytesSourceFile] that holds a list of UTF-8 bytes. Otherwise the
   * string text of the source file is decoded.
   *
   * The list of UTF-8 bytes [file.slowUtf8Bytes()] is expected to return an
   * array whose last element is '0' to signal the end of the file. If this
   * is not the case, the entire array is copied before scanning.
   */
  Utf8BytesScanner(SourceFile file, {bool includeComments: false})
      : bytes = file.slowUtf8ZeroTerminatedBytes(),
        super(file, includeComments) {
    assert(bytes.last == 0);
    // Skip a leading BOM.
    if (_containsBomAt(0)) byteOffset += 3;
  }

  /**
   * Creates a new Utf8BytesScanner from a list of UTF-8 bytes.
   *
   * The last element of the list is expected to be '0' to signal the end of
   * the file. If this is not the case, the entire array is copied before
   * scanning.
   */
  Utf8BytesScanner.fromBytes(List<int> zeroTerminatedBytes,
                             {bool includeComments: false})
      : this.bytes = zeroTerminatedBytes,
        super(null, includeComments) {
    assert(bytes.last == 0);
  }

  bool _containsBomAt(int offset) {
    const BOM_UTF8 = const [0xEF, 0xBB, 0xBF];

    return offset + 3 < bytes.length &&
        bytes[offset] == BOM_UTF8[0] &&
        bytes[offset + 1] == BOM_UTF8[1] &&
        bytes[offset + 2] == BOM_UTF8[2];
  }

  int advance() => bytes[++byteOffset];

  int peek() => bytes[byteOffset + 1];

  /**
   * Returns the unicode code point starting at the byte offset [startOffset]
   * with the byte [nextByte]. If [advance] is true the current [byteOffset]
   * is advanced to the last byte of the code point.
   */
  int nextCodePoint(int startOffset, int nextByte, bool advance) {
    // The number of 1s in the first byte indicate the number of bytes, at
    // least 2.
    int numBytes = 2;
    int bit = 0x20;
    while ((nextByte & bit) != 0) {
      numBytes++;
      bit >>= 1;
    }
    int end = startOffset + numBytes;
    if (advance) {
      byteOffset = end - 1;
    }
    // TODO(lry): measurably slow, decode creates first a Utf8Decoder and a
    // _Utf8Decoder instance. Also the sublist is eagerly allocated.
    String codePoint = UTF8.decode(bytes.sublist(startOffset, end));
    if (codePoint.length == 0) {
      // The UTF-8 decoder discards leading BOM characters.
      // TODO(floitsch): don't just assume that removed characters were the
      // BOM.
      assert(_containsBomAt(startOffset));
      codePoint = new String.fromCharCode(UNICODE_BOM_CHARACTER_RUNE);
    }
    if (codePoint.length == 1) {
      if (advance) {
        utf8Slack += (numBytes - 1);
        scanSlack = numBytes - 1;
        scanSlackOffset = byteOffset;
      }
      return codePoint.codeUnitAt(0);
    } else if (codePoint.length == 2) {
      if (advance) {
        utf8Slack += (numBytes - 2);
        scanSlack = numBytes - 1;
        scanSlackOffset = byteOffset;
        stringOffsetSlackOffset = byteOffset;
      }
      // In case of a surrogate pair, return a single code point.
      return codePoint.runes.single;
    } else {
      throw "Invalid UTF-8 byte sequence: ${bytes.sublist(startOffset, end)}";
    }
  }

  int lastUnicodeOffset = -1;
  int currentAsUnicode(int next) {
    if (next < 128) return next;
    // Check if currentAsUnicode was already invoked.
    if (byteOffset == lastUnicodeOffset) return next;
    int res = nextCodePoint(byteOffset, next, true);
    lastUnicodeOffset = byteOffset;
    return res;
  }

  void handleUnicode(int startScanOffset) {
    int end = byteOffset;
    // TODO(lry): this measurably slows down the scanner for files with unicode.
    String s = UTF8.decode(bytes.sublist(startScanOffset, end));
    utf8Slack += (end - startScanOffset) - s.length;
  }

  /**
   * This field remembers the byte offset of the last character decoded with
   * [nextCodePoint] that used two code units in UTF-16.
   *
   * [nextCodePoint] returns a single code point for each unicode character,
   * even if it needs two code units in UTF-16.
   *
   * For example, '\u{1d11e}' uses 4 bytes in UTF-8, and two code units in
   * UTF-16. The [utf8Slack] is therefore 2. After invoking [nextCodePoint], the
   * [byteOffset] points to the last (of 4) bytes. The [stringOffset] should
   * return the offset of the first one, which is one position more left than
   * the [utf8Slack].
   */
  int stringOffsetSlackOffset = -1;

  int get stringOffset {
    if (stringOffsetSlackOffset == byteOffset) {
      return byteOffset - utf8Slack - 1;
    } else {
      return byteOffset - utf8Slack;
    }
  }

  Token firstToken() => tokens.next;
  Token previousToken() => tail;

  void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly,
                            [int extraOffset = 0]) {
    tail.next = new StringToken.fromUtf8Bytes(
        info, bytes, start, byteOffset + extraOffset, asciiOnly, tokenStart);
    tail = tail.next;
  }

  bool atEndOfFile() => byteOffset >= bytes.length - 1;
}