blob: 61e6bdf465a3d0db59ed39714dcfd62f538ced1d [file] [log] [blame]
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
library utf.utf82_test;
import 'package:test/test.dart';
import 'package:utf/utf.dart';
import 'expect.dart' as expect;
const String testEnglishPhrase = 'The quick brown fox jumps over the lazy dog.';
const testEnglishUtf8 = <int>[
0x54, 0x68, 0x65, 0x20, 0x71, 0x75, 0x69, 0x63, // 8
0x6b, 0x20, 0x62, 0x72, 0x6f, 0x77, 0x6e, 0x20,
0x66, 0x6f, 0x78, 0x20, 0x6a, 0x75, 0x6d, 0x70,
0x73, 0x20, 0x6f, 0x76, 0x65, 0x72, 0x20, 0x74,
0x68, 0x65, 0x20, 0x6c, 0x61, 0x7a, 0x79, 0x20,
0x64, 0x6f, 0x67, 0x2e
];
const String testDanishPhrase = 'Quizdeltagerne spiste jordbær med '
'fløde mens cirkusklovnen Wolther spillede på xylofon.';
const testDanishUtf8 = <int>[
0x51, 0x75, 0x69, 0x7a, 0x64, 0x65, 0x6c, 0x74, // 8
0x61, 0x67, 0x65, 0x72, 0x6e, 0x65, 0x20, 0x73,
0x70, 0x69, 0x73, 0x74, 0x65, 0x20, 0x6a, 0x6f,
0x72, 0x64, 0x62, 0xc3, 0xa6, 0x72, 0x20, 0x6d,
0x65, 0x64, 0x20, 0x66, 0x6c, 0xc3, 0xb8, 0x64,
0x65, 0x20, 0x6d, 0x65, 0x6e, 0x73, 0x20, 0x63,
0x69, 0x72, 0x6b, 0x75, 0x73, 0x6b, 0x6c, 0x6f,
0x76, 0x6e, 0x65, 0x6e, 0x20, 0x57, 0x6f, 0x6c,
0x74, 0x68, 0x65, 0x72, 0x20, 0x73, 0x70, 0x69,
0x6c, 0x6c, 0x65, 0x64, 0x65, 0x20, 0x70, 0xc3,
0xa5, 0x20, 0x78, 0x79, 0x6c, 0x6f, 0x66, 0x6f,
0x6e, 0x2e
];
// unusual formatting due to strange editor interaction w/ text direction.
const String testHebrewPhrase =
'דג סקרן שט בים מאוכזב ולפתע מצא לו חברה איך הקליטה';
const testHebrewUtf8 = <int>[
0xd7, 0x93, 0xd7, 0x92, 0x20, 0xd7, 0xa1, 0xd7, // 8
0xa7, 0xd7, 0xa8, 0xd7, 0x9f, 0x20, 0xd7, 0xa9,
0xd7, 0x98, 0x20, 0xd7, 0x91, 0xd7, 0x99, 0xd7,
0x9d, 0x20, 0xd7, 0x9e, 0xd7, 0x90, 0xd7, 0x95,
0xd7, 0x9b, 0xd7, 0x96, 0xd7, 0x91, 0x20, 0xd7,
0x95, 0xd7, 0x9c, 0xd7, 0xa4, 0xd7, 0xaa, 0xd7,
0xa2, 0x20, 0xd7, 0x9e, 0xd7, 0xa6, 0xd7, 0x90,
0x20, 0xd7, 0x9c, 0xd7, 0x95, 0x20, 0xd7, 0x97,
0xd7, 0x91, 0xd7, 0xa8, 0xd7, 0x94, 0x20, 0xd7,
0x90, 0xd7, 0x99, 0xd7, 0x9a, 0x20, 0xd7, 0x94,
0xd7, 0xa7, 0xd7, 0x9c, 0xd7, 0x99, 0xd7, 0x98,
0xd7, 0x94
];
const String testRussianPhrase = 'Съешь же ещё этих мягких '
'французских булок да выпей чаю';
const testRussianUtf8 = <int>[
0xd0, 0xa1, 0xd1, 0x8a, 0xd0, 0xb5, 0xd1, 0x88, // 8
0xd1, 0x8c, 0x20, 0xd0, 0xb6, 0xd0, 0xb5, 0x20,
0xd0, 0xb5, 0xd1, 0x89, 0xd1, 0x91, 0x20, 0xd1,
0x8d, 0xd1, 0x82, 0xd0, 0xb8, 0xd1, 0x85, 0x20,
0xd0, 0xbc, 0xd1, 0x8f, 0xd0, 0xb3, 0xd0, 0xba,
0xd0, 0xb8, 0xd1, 0x85, 0x20, 0xd1, 0x84, 0xd1,
0x80, 0xd0, 0xb0, 0xd0, 0xbd, 0xd1, 0x86, 0xd1,
0x83, 0xd0, 0xb7, 0xd1, 0x81, 0xd0, 0xba, 0xd0,
0xb8, 0xd1, 0x85, 0x20, 0xd0, 0xb1, 0xd1, 0x83,
0xd0, 0xbb, 0xd0, 0xbe, 0xd0, 0xba, 0x20, 0xd0,
0xb4, 0xd0, 0xb0, 0x20, 0xd0, 0xb2, 0xd1, 0x8b,
0xd0, 0xbf, 0xd0, 0xb5, 0xd0, 0xb9, 0x20, 0xd1,
0x87, 0xd0, 0xb0, 0xd1, 0x8e
];
const String testGreekPhrase = 'Γαζέες καὶ μυρτιὲς δὲν θὰ βρῶ πιὰ '
'στὸ χρυσαφὶ ξέφωτο';
const testGreekUtf8 = <int>[
0xce, 0x93, 0xce, 0xb1, 0xce, 0xb6, 0xce, 0xad, // 8
0xce, 0xb5, 0xcf, 0x82, 0x20, 0xce, 0xba, 0xce,
0xb1, 0xe1, 0xbd, 0xb6, 0x20, 0xce, 0xbc, 0xcf,
0x85, 0xcf, 0x81, 0xcf, 0x84, 0xce, 0xb9, 0xe1,
0xbd, 0xb2, 0xcf, 0x82, 0x20, 0xce, 0xb4, 0xe1,
0xbd, 0xb2, 0xce, 0xbd, 0x20, 0xce, 0xb8, 0xe1,
0xbd, 0xb0, 0x20, 0xce, 0xb2, 0xcf, 0x81, 0xe1,
0xbf, 0xb6, 0x20, 0xcf, 0x80, 0xce, 0xb9, 0xe1,
0xbd, 0xb0, 0x20, 0xcf, 0x83, 0xcf, 0x84, 0xe1,
0xbd, 0xb8, 0x20, 0xcf, 0x87, 0xcf, 0x81, 0xcf,
0x85, 0xcf, 0x83, 0xce, 0xb1, 0xcf, 0x86, 0xe1,
0xbd, 0xb6, 0x20, 0xce, 0xbe, 0xce, 0xad, 0xcf,
0x86, 0xcf, 0x89, 0xcf, 0x84, 0xce, 0xbf
];
const String testKatakanaPhrase = '''
イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム
ウヰノオクヤマ ケフコエテ アサキユメミシ ヱヒモセスン''';
const testKatakanaUtf8 = <int>[
0xe3, 0x82, 0xa4, 0xe3, 0x83, 0xad, 0xe3, 0x83, // 8
0x8f, 0xe3, 0x83, 0x8b, 0xe3, 0x83, 0x9b, 0xe3,
0x83, 0x98, 0xe3, 0x83, 0x88, 0x20, 0xe3, 0x83,
0x81, 0xe3, 0x83, 0xaa, 0xe3, 0x83, 0x8c, 0xe3,
0x83, 0xab, 0xe3, 0x83, 0xb2, 0x20, 0xe3, 0x83,
0xaf, 0xe3, 0x82, 0xab, 0xe3, 0x83, 0xa8, 0xe3,
0x82, 0xbf, 0xe3, 0x83, 0xac, 0xe3, 0x82, 0xbd,
0x20, 0xe3, 0x83, 0x84, 0xe3, 0x83, 0x8d, 0xe3,
0x83, 0x8a, 0xe3, 0x83, 0xa9, 0xe3, 0x83, 0xa0,
0x0a, 0xe3, 0x82, 0xa6, 0xe3, 0x83, 0xb0, 0xe3,
0x83, 0x8e, 0xe3, 0x82, 0xaa, 0xe3, 0x82, 0xaf,
0xe3, 0x83, 0xa4, 0xe3, 0x83, 0x9e, 0x20, 0xe3,
0x82, 0xb1, 0xe3, 0x83, 0x95, 0xe3, 0x82, 0xb3,
0xe3, 0x82, 0xa8, 0xe3, 0x83, 0x86, 0x20, 0xe3,
0x82, 0xa2, 0xe3, 0x82, 0xb5, 0xe3, 0x82, 0xad,
0xe3, 0x83, 0xa6, 0xe3, 0x83, 0xa1, 0xe3, 0x83,
0x9f, 0xe3, 0x82, 0xb7, 0x20, 0xe3, 0x83, 0xb1,
0xe3, 0x83, 0x92, 0xe3, 0x83, 0xa2, 0xe3, 0x82,
0xbb, 0xe3, 0x82, 0xb9, 0xe3, 0x83, 0xb3
];
void main() {
test('utf8 bytes to codepoints', testUtf8bytesToCodepoints);
test('utf8 bytes to string', testUtf8BytesToString);
test('encode to utf8', testEncodeToUtf8);
test('iterable methods', testIterableMethods);
}
void testEncodeToUtf8() {
expect.listEquals(
testEnglishUtf8, encodeUtf8(testEnglishPhrase), 'english to utf8');
expect.listEquals(
testDanishUtf8, encodeUtf8(testDanishPhrase), 'encode danish to utf8');
expect.listEquals(
testHebrewUtf8, encodeUtf8(testHebrewPhrase), 'Hebrew to utf8');
expect.listEquals(
testRussianUtf8, encodeUtf8(testRussianPhrase), 'Russian to utf8');
expect.listEquals(
testGreekUtf8, encodeUtf8(testGreekPhrase), 'Greek to utf8');
expect.listEquals(
testKatakanaUtf8, encodeUtf8(testKatakanaPhrase), 'Katakana to utf8');
}
void testUtf8bytesToCodepoints() {
expect.listEquals(
[954, 972, 963, 956, 949],
utf8ToCodepoints(
[0xce, 0xba, 0xcf, 0x8c, 0xcf, 0x83, 0xce, 0xbc, 0xce, 0xb5]),
'κόσμε');
// boundary conditions: First possible sequence of a certain length
expect.listEquals([], utf8ToCodepoints([]), 'no input');
expect.listEquals([0x0], utf8ToCodepoints([0x0]), '0');
expect.listEquals([0x80], utf8ToCodepoints([0xc2, 0x80]), '80');
expect.listEquals([0x800], utf8ToCodepoints([0xe0, 0xa0, 0x80]), '800');
expect.listEquals(
[0x10000], utf8ToCodepoints([0xf0, 0x90, 0x80, 0x80]), '10000');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xf8, 0x88, 0x80, 0x80, 0x80]), '200000');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xfc, 0x84, 0x80, 0x80, 0x80, 0x80]), '4000000');
// boundary conditions: Last possible sequence of a certain length
expect.listEquals([0x7f], utf8ToCodepoints([0x7f]), '7f');
expect.listEquals([0x7ff], utf8ToCodepoints([0xdf, 0xbf]), '7ff');
expect.listEquals([0xffff], utf8ToCodepoints([0xef, 0xbf, 0xbf]), 'ffff');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xf7, 0xbf, 0xbf, 0xbf]), '1fffff');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xfb, 0xbf, 0xbf, 0xbf, 0xbf]), '3ffffff');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xfd, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf]), '4000000');
// other boundary conditions
expect.listEquals([0xd7ff], utf8ToCodepoints([0xed, 0x9f, 0xbf]), 'd7ff');
expect.listEquals([0xe000], utf8ToCodepoints([0xee, 0x80, 0x80]), 'e000');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xef, 0xbf, 0xbd]), 'fffd');
expect.listEquals(
[0x10ffff], utf8ToCodepoints([0xf4, 0x8f, 0xbf, 0xbf]), '10ffff');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xf4, 0x90, 0x80, 0x80]), '110000');
// unexpected continuation bytes
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0x80]), '80 => replacement character');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xbf]), 'bf => replacement character');
var allContinuationBytes = <int>[];
var matchingReplacementChars = <int>[];
for (var i = 0x80; i < 0xc0; i++) {
allContinuationBytes.add(i);
matchingReplacementChars.add(UNICODE_REPLACEMENT_CHARACTER_CODEPOINT);
}
expect.listEquals(
matchingReplacementChars,
utf8ToCodepoints(allContinuationBytes),
'80 - bf => replacement character x 64');
var allFirstTwoByteSeq = <int>[];
matchingReplacementChars = <int>[];
for (var i = 0xc0; i < 0xe0; i++) {
allFirstTwoByteSeq.addAll([i, 0x20]);
matchingReplacementChars.addAll([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]);
}
expect.listEquals(
matchingReplacementChars,
utf8ToCodepoints(allFirstTwoByteSeq),
'c0 - df + space => replacement character + space x 32');
var allFirstThreeByteSeq = <int>[];
matchingReplacementChars = <int>[];
for (var i = 0xe0; i < 0xf0; i++) {
allFirstThreeByteSeq.addAll([i, 0x20]);
matchingReplacementChars.addAll([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]);
}
expect.listEquals(
matchingReplacementChars,
utf8ToCodepoints(allFirstThreeByteSeq),
'e0 - ef + space => replacement character x 16');
var allFirstFourByteSeq = <int>[];
matchingReplacementChars = <int>[];
for (var i = 0xf0; i < 0xf8; i++) {
allFirstFourByteSeq.addAll([i, 0x20]);
matchingReplacementChars.addAll([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]);
}
expect.listEquals(
matchingReplacementChars,
utf8ToCodepoints(allFirstFourByteSeq),
'f0 - f7 + space => replacement character x 8');
var allFirstFiveByteSeq = <int>[];
matchingReplacementChars = <int>[];
for (var i = 0xf8; i < 0xfc; i++) {
allFirstFiveByteSeq.addAll([i, 0x20]);
matchingReplacementChars.addAll([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]);
}
expect.listEquals(
matchingReplacementChars,
utf8ToCodepoints(allFirstFiveByteSeq),
'f8 - fb + space => replacement character x 4');
var allFirstSixByteSeq = <int>[];
matchingReplacementChars = <int>[];
for (var i = 0xfc; i < 0xfe; i++) {
allFirstSixByteSeq.addAll([i, 0x20]);
matchingReplacementChars.addAll([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]);
}
expect.listEquals(
matchingReplacementChars,
utf8ToCodepoints(allFirstSixByteSeq),
'fc - fd + space => replacement character x 2');
// Sequences with last continuation byte missing
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xc2]), '2-byte sequence with last byte missing');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xe0, 0x80]), '3-byte sequence with last byte missing');
expect.listEquals(
[UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xf0, 0x80, 0x80]),
'4-byte sequence with last byte missing');
expect.listEquals(
[UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xf8, 0x88, 0x80, 0x80]),
'5-byte sequence with last byte missing');
expect.listEquals(
[UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80]),
'6-byte sequence with last byte missing');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xdf]), '2-byte sequence with last byte missing (hi)');
expect.listEquals(
[UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xef, 0xbf]),
'3-byte sequence with last byte missing (hi)');
expect.listEquals(
[UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xf7, 0xbf, 0xbf]),
'4-byte sequence with last byte missing (hi)');
expect.listEquals(
[UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xfb, 0xbf, 0xbf, 0xbf]),
'5-byte sequence with last byte missing (hi)');
expect.listEquals(
[UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xfd, 0xbf, 0xbf, 0xbf, 0xbf]),
'6-byte sequence with last byte missing (hi)');
// Concatenation of incomplete sequences
expect.listEquals(
[
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT
],
utf8ToCodepoints([
0xc2, // 1
0xe0, 0x80,
0xf0, 0x80, 0x80,
0xf8, 0x88, 0x80, 0x80,
0xfc, 0x80, 0x80, 0x80, 0x80,
0xdf,
0xef, 0xbf,
0xf7, 0xbf, 0xbf,
0xfb, 0xbf, 0xbf, 0xbf,
0xfd, 0xbf, 0xbf, 0xbf, 0xbf
]),
'Concatenation of incomplete sequences');
// Impossible bytes
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xfe]), 'fe');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xff]), 'ff');
expect.listEquals([
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT
], utf8ToCodepoints([0xfe, 0xfe, 0xff, 0xff]), 'fe fe ff ff');
// Overlong sequences
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xc0, 0xaf]), 'c0 af');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xe0, 0x80, 0xaf]), 'e0 80 af');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xf0, 0x80, 0x80, 0xaf]), 'f0 80 80 af');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xf8, 0x80, 0x80, 0x80, 0xaf]), 'f8 80 80 80 af');
expect.listEquals(
[UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80, 0xaf]),
'fc 80 80 80 80 af');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xc1, 0xbf]), 'c1 bf');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xe0, 0x9f, 0xbf]), 'e0 9f bf');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xf0, 0x8f, 0xbf, 0xbf]), 'f0 8f bf bf');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xf8, 0x87, 0xbf, 0xbf, 0xbf]), 'f8 87 bf bf bf');
expect.listEquals(
[UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xfc, 0x83, 0xbf, 0xbf, 0xbf, 0xbf]),
'fc 83 bf bf bf bf');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xc0, 0x80]), 'c0 80');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xe0, 0x80, 0x80]), 'e0 80 80');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xf0, 0x80, 0x80, 0x80]), 'f0 80 80 80');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xf8, 0x80, 0x80, 0x80, 0x80]), 'f8 80 80 80 80');
expect.listEquals(
[UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80, 0x80]),
'fc 80 80 80 80 80');
// Illegal code positions
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xed, 0xa0, 0x80]), 'U+D800');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xed, 0xad, 0xbf]), 'U+DB7F');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xed, 0xae, 0x80]), 'U+DB80');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xed, 0xaf, 0xbf]), 'U+DBFF');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xed, 0xb0, 0x80]), 'U+DC00');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xed, 0xbe, 0x80]), 'U+DF80');
expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT],
utf8ToCodepoints([0xed, 0xbf, 0xbf]), 'U+DFFF');
// Paired UTF-16 surrogates
expect.listEquals([
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT
], utf8ToCodepoints([0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80]), 'U+D800 U+DC00');
expect.listEquals([
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT
], utf8ToCodepoints([0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf]), 'U+D800 U+DFFF');
expect.listEquals([
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT
], utf8ToCodepoints([0xed, 0xad, 0xbf, 0xed, 0xb0, 0x80]), 'U+DB7F U+DC00');
expect.listEquals([
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT
], utf8ToCodepoints([0xed, 0xad, 0xbf, 0xed, 0xbf, 0xbf]), 'U+DB7F U+DFFF');
expect.listEquals([
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT
], utf8ToCodepoints([0xed, 0xae, 0x80, 0xed, 0xb0, 0x80]), 'U+DB80 U+DC00');
expect.listEquals([
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT
], utf8ToCodepoints([0xed, 0xae, 0x80, 0xed, 0xbf, 0xbf]), 'U+DB80 U+DFFF');
expect.listEquals([
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT
], utf8ToCodepoints([0xed, 0xaf, 0xbf, 0xed, 0xb0, 0x80]), 'U+DBFF U+DC00');
expect.listEquals([
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT,
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT
], utf8ToCodepoints([0xed, 0xaf, 0xbf, 0xed, 0xbf, 0xbf]), 'U+DBFF U+DFFF');
// Other illegal code positions (???)
expect.listEquals([0xfffe], utf8ToCodepoints([0xef, 0xbf, 0xbe]), 'U+FFFE');
expect.listEquals([0xffff], utf8ToCodepoints([0xef, 0xbf, 0xbf]), 'U+FFFF');
}
void testUtf8BytesToString() {
expect.stringEquals(
testEnglishPhrase, decodeUtf8(testEnglishUtf8), 'English');
expect.stringEquals(testDanishPhrase, decodeUtf8(testDanishUtf8), 'Danish');
expect.stringEquals(testHebrewPhrase, decodeUtf8(testHebrewUtf8), 'Hebrew');
expect.stringEquals(
testRussianPhrase, decodeUtf8(testRussianUtf8), 'Russian');
expect.stringEquals(testGreekPhrase, decodeUtf8(testGreekUtf8), 'Greek');
expect.stringEquals(
testKatakanaPhrase, decodeUtf8(testKatakanaUtf8), 'Katakana');
}
void testIterableMethods() {
var englishDecoder = decodeUtf8AsIterable(testEnglishUtf8);
// get the first character
expect.equals(testEnglishUtf8[0], englishDecoder.first);
// get the whole translation using the Iterable interface
expect.stringEquals(
testEnglishPhrase, String.fromCharCodes(List<int>.from(englishDecoder)));
var kataDecoder = decodeUtf8AsIterable(testKatakanaUtf8);
// get the first character
expect.equals(testKatakanaPhrase.codeUnits[0], kataDecoder.first);
// get the whole translation using the Iterable interface
expect.stringEquals(
testKatakanaPhrase, String.fromCharCodes(List<int>.from(kataDecoder)));
}