Adjust UTF-8 tests to match WHATWG standard expectations.
This adjusts all UTF-8 tests to the new semantics in the breaking change described here: https://github.com/dart-lang/sdk/issues/41100 This has three parts: - Unpaired surrogates are encoded as replacement characters, and encoded surrogates are considered malformed input when decoding. - Decoding errors are generally reported on the position of the byte that conclusively makes the input malformed. - The number of replacement characters emitted by the decoder is generally one per unfinished sequence or undecodable byte. The code changes to implement the new semantics are placed in subsequent commits. Change-Id: I4cc8ce660e39287e734070764ab8e1f0ebb8b9e0 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/143815 Reviewed-by: Lasse R.H. Nielsen <lrn@google.com>
This commit is contained in:
Aske Simon Christensen
committed by
commit-bot@chromium.org
commit-bot@chromium.org
parent
049949fdf4
commit
f5bf50e7a4
@@ -46,11 +46,14 @@ String decodeAllowMalformed(List<int> bytes, int chunkSize) {
|
||||
return buffer.toString();
|
||||
}
|
||||
|
||||
final TESTS = [
|
||||
final TESTS0 = [
|
||||
// Unfinished UTF-8 sequences.
|
||||
[0xc3],
|
||||
[0xE2, 0x82],
|
||||
[0xF0, 0xA4, 0xAD],
|
||||
[0xF0, 0xA4, 0xAD]
|
||||
];
|
||||
|
||||
final TESTS1 = [
|
||||
// Overlong encoding of euro-sign.
|
||||
[0xF0, 0x82, 0x82, 0xAC],
|
||||
// Other overlong/unfinished sequences.
|
||||
@@ -77,11 +80,11 @@ final TESTS2 = [
|
||||
// Test that 0xC0|1, 0x80 does not eat the next character.
|
||||
[
|
||||
[0xC0, 0x80, 0x61],
|
||||
"Xa"
|
||||
"XXa"
|
||||
],
|
||||
[
|
||||
[0xC1, 0x80, 0x61],
|
||||
"Xa"
|
||||
"XXa"
|
||||
],
|
||||
// 0xF5 .. 0xFF never appear in valid UTF-8 sequences.
|
||||
[
|
||||
@@ -220,44 +223,49 @@ final TESTS2 = [
|
||||
];
|
||||
|
||||
main() {
|
||||
var allTests = TESTS.expand((test) {
|
||||
var allTests = [...TESTS0, ...TESTS1].expand((test) {
|
||||
// Pairs of test and expected string output when malformed strings are
|
||||
// allowed. Replacement character: U+FFFD
|
||||
// allowed. Replacement character: U+FFFD, one per unfinished sequence or
|
||||
// undecodable byte.
|
||||
String replacement =
|
||||
TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length;
|
||||
return [
|
||||
[test, "\u{FFFD}"],
|
||||
[test, "${replacement}"],
|
||||
[
|
||||
new List<int>.from([0x61])..addAll(test),
|
||||
"a\u{FFFD}"
|
||||
[0x61, ...test],
|
||||
"a${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0x61])
|
||||
..addAll(test)
|
||||
..add(0x61),
|
||||
"a\u{FFFD}a"
|
||||
],
|
||||
[new List<int>.from(test)..add(0x61), "\u{FFFD}a"],
|
||||
[new List<int>.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"],
|
||||
[
|
||||
new List<int>.from(test)
|
||||
..add(0x61)
|
||||
..addAll(test),
|
||||
"\u{FFFD}a\u{FFFD}"
|
||||
[0x61, ...test, 0x61],
|
||||
"a${replacement}a"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0xc3, 0xa5])..addAll(test),
|
||||
"å\u{FFFD}"
|
||||
[...test, 0x61],
|
||||
"${replacement}a"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]),
|
||||
"å\u{FFFD}å"
|
||||
[...test, ...test],
|
||||
"${replacement}${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from(test)..addAll([0xc3, 0xa5]),
|
||||
"\u{FFFD}å"
|
||||
[...test, 0x61, ...test],
|
||||
"${replacement}a${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from(test)..addAll([0xc3, 0xa5])..addAll(test),
|
||||
"\u{FFFD}å\u{FFFD}"
|
||||
[0xc3, 0xa5, ...test],
|
||||
"å${replacement}"
|
||||
],
|
||||
[
|
||||
[0xc3, 0xa5, ...test, 0xc3, 0xa5],
|
||||
"å${replacement}å"
|
||||
],
|
||||
[
|
||||
[...test, 0xc3, 0xa5],
|
||||
"${replacement}å"
|
||||
],
|
||||
[
|
||||
[...test, 0xc3, 0xa5, ...test],
|
||||
"${replacement}å${replacement}"
|
||||
]
|
||||
];
|
||||
});
|
||||
|
||||
@@ -28,11 +28,14 @@ String decodeAllowMalformed(List<int> bytes) {
|
||||
return buffer.toString();
|
||||
}
|
||||
|
||||
final TESTS = [
|
||||
final TESTS0 = [
|
||||
// Unfinished UTF-8 sequences.
|
||||
[0xc3],
|
||||
[0xE2, 0x82],
|
||||
[0xF0, 0xA4, 0xAD],
|
||||
[0xF0, 0xA4, 0xAD]
|
||||
];
|
||||
|
||||
final TESTS1 = [
|
||||
// Overlong encoding of euro-sign.
|
||||
[0xF0, 0x82, 0x82, 0xAC],
|
||||
// Other overlong/unfinished sequences.
|
||||
@@ -59,11 +62,11 @@ final TESTS2 = [
|
||||
// Test that 0xC0|1, 0x80 does not eat the next character.
|
||||
[
|
||||
[0xC0, 0x80, 0x61],
|
||||
"Xa"
|
||||
"XXa"
|
||||
],
|
||||
[
|
||||
[0xC1, 0x80, 0x61],
|
||||
"Xa"
|
||||
"XXa"
|
||||
],
|
||||
// 0xF5 .. 0xFF never appear in valid UTF-8 sequences.
|
||||
[
|
||||
@@ -202,44 +205,49 @@ final TESTS2 = [
|
||||
];
|
||||
|
||||
main() {
|
||||
var allTests = TESTS.expand((test) {
|
||||
var allTests = [...TESTS0, ...TESTS1].expand((test) {
|
||||
// Pairs of test and expected string output when malformed strings are
|
||||
// allowed. Replacement character: U+FFFD
|
||||
// allowed. Replacement character: U+FFFD, one per unfinished sequence or
|
||||
// undecodable byte.
|
||||
String replacement =
|
||||
TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length;
|
||||
return [
|
||||
[test, "\u{FFFD}"],
|
||||
[test, "${replacement}"],
|
||||
[
|
||||
new List<int>.from([0x61])..addAll(test),
|
||||
"a\u{FFFD}"
|
||||
[0x61, ...test],
|
||||
"a${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0x61])
|
||||
..addAll(test)
|
||||
..add(0x61),
|
||||
"a\u{FFFD}a"
|
||||
],
|
||||
[new List<int>.from(test)..add(0x61), "\u{FFFD}a"],
|
||||
[new List<int>.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"],
|
||||
[
|
||||
new List<int>.from(test)
|
||||
..add(0x61)
|
||||
..addAll(test),
|
||||
"\u{FFFD}a\u{FFFD}"
|
||||
[0x61, ...test, 0x61],
|
||||
"a${replacement}a"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0xc3, 0xa5])..addAll(test),
|
||||
"å\u{FFFD}"
|
||||
[...test, 0x61],
|
||||
"${replacement}a"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]),
|
||||
"å\u{FFFD}å"
|
||||
[...test, ...test],
|
||||
"${replacement}${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from(test)..addAll([0xc3, 0xa5]),
|
||||
"\u{FFFD}å"
|
||||
[...test, 0x61, ...test],
|
||||
"${replacement}a${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from(test)..addAll([0xc3, 0xa5])..addAll(test),
|
||||
"\u{FFFD}å\u{FFFD}"
|
||||
[0xc3, 0xa5, ...test],
|
||||
"å${replacement}"
|
||||
],
|
||||
[
|
||||
[0xc3, 0xa5, ...test, 0xc3, 0xa5],
|
||||
"å${replacement}å"
|
||||
],
|
||||
[
|
||||
[...test, 0xc3, 0xa5],
|
||||
"${replacement}å"
|
||||
],
|
||||
[
|
||||
[...test, 0xc3, 0xa5, ...test],
|
||||
"${replacement}å${replacement}"
|
||||
]
|
||||
];
|
||||
});
|
||||
|
||||
@@ -49,11 +49,14 @@ String decodeAllowMalformed2(List<int> inputBytes) {
|
||||
return utf8.decode(bytes);
|
||||
}
|
||||
|
||||
final TESTS = [
|
||||
final TESTS0 = [
|
||||
// Unfinished UTF-8 sequences.
|
||||
[0xc3],
|
||||
[0xE2, 0x82],
|
||||
[0xF0, 0xA4, 0xAD],
|
||||
[0xF0, 0xA4, 0xAD]
|
||||
];
|
||||
|
||||
final TESTS1 = [
|
||||
// Overlong encoding of euro-sign.
|
||||
[0xF0, 0x82, 0x82, 0xAC],
|
||||
// Other overlong/unfinished sequences.
|
||||
@@ -80,11 +83,11 @@ final TESTS2 = [
|
||||
// Test that 0xC0|1, 0x80 does not eat the next character.
|
||||
[
|
||||
[0xC0, 0x80, 0x61],
|
||||
"Xa"
|
||||
"XXa"
|
||||
],
|
||||
[
|
||||
[0xC1, 0x80, 0x61],
|
||||
"Xa"
|
||||
"XXa"
|
||||
],
|
||||
// 0xF5 .. 0xFF never appear in valid UTF-8 sequences.
|
||||
[
|
||||
@@ -223,44 +226,49 @@ final TESTS2 = [
|
||||
];
|
||||
|
||||
main() {
|
||||
var allTests = TESTS.expand((test) {
|
||||
var allTests = [...TESTS0, ...TESTS1].expand((test) {
|
||||
// Pairs of test and expected string output when malformed strings are
|
||||
// allowed. Replacement character: U+FFFD
|
||||
// allowed. Replacement character: U+FFFD, one per unfinished sequence or
|
||||
// undecodable byte.
|
||||
String replacement =
|
||||
TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length;
|
||||
return [
|
||||
[test, "\u{FFFD}"],
|
||||
[test, "${replacement}"],
|
||||
[
|
||||
new List<int>.from([0x61])..addAll(test),
|
||||
"a\u{FFFD}"
|
||||
[0x61, ...test],
|
||||
"a${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0x61])
|
||||
..addAll(test)
|
||||
..add(0x61),
|
||||
"a\u{FFFD}a"
|
||||
],
|
||||
[new List<int>.from(test)..add(0x61), "\u{FFFD}a"],
|
||||
[new List<int>.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"],
|
||||
[
|
||||
new List<int>.from(test)
|
||||
..add(0x61)
|
||||
..addAll(test),
|
||||
"\u{FFFD}a\u{FFFD}"
|
||||
[0x61, ...test, 0x61],
|
||||
"a${replacement}a"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0xc3, 0xa5])..addAll(test),
|
||||
"å\u{FFFD}"
|
||||
[...test, 0x61],
|
||||
"${replacement}a"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]),
|
||||
"å\u{FFFD}å"
|
||||
[...test, ...test],
|
||||
"${replacement}${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from(test)..addAll([0xc3, 0xa5]),
|
||||
"\u{FFFD}å"
|
||||
[...test, 0x61, ...test],
|
||||
"${replacement}a${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from(test)..addAll([0xc3, 0xa5])..addAll(test),
|
||||
"\u{FFFD}å\u{FFFD}"
|
||||
[0xc3, 0xa5, ...test],
|
||||
"å${replacement}"
|
||||
],
|
||||
[
|
||||
[0xc3, 0xa5, ...test, 0xc3, 0xa5],
|
||||
"å${replacement}å"
|
||||
],
|
||||
[
|
||||
[...test, 0xc3, 0xa5],
|
||||
"${replacement}å"
|
||||
],
|
||||
[
|
||||
[...test, 0xc3, 0xa5, ...test],
|
||||
"${replacement}å${replacement}"
|
||||
]
|
||||
];
|
||||
});
|
||||
|
||||
@@ -112,8 +112,7 @@ main() {
|
||||
const LEADING_SURROGATE = 0xd801;
|
||||
const TRAILING_SURROGATE = 0xdc12;
|
||||
const UTF8_ENCODING = const [0xf0, 0x90, 0x90, 0x92];
|
||||
const UTF8_LEADING = const [0xed, 0xa0, 0x81];
|
||||
const UTF8_TRAILING = const [0xed, 0xb0, 0x92];
|
||||
const UTF8_REPLACEMENT = const [0xef, 0xbf, 0xbd];
|
||||
const CHAR_A = 0x61;
|
||||
|
||||
// Test surrogates at all kinds of locations.
|
||||
@@ -129,17 +128,17 @@ main() {
|
||||
codeUnits[i] = LEADING_SURROGATE;
|
||||
var str = new String.fromCharCodes(codeUnits);
|
||||
var bytes = new List.filled(i + 3, CHAR_A);
|
||||
bytes[i] = UTF8_LEADING[0];
|
||||
bytes[i + 1] = UTF8_LEADING[1];
|
||||
bytes[i + 2] = UTF8_LEADING[2];
|
||||
bytes[i] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
||||
runTest([bytes, str]);
|
||||
|
||||
codeUnits[i] = TRAILING_SURROGATE;
|
||||
str = new String.fromCharCodes(codeUnits);
|
||||
bytes = new List.filled(i + 3, CHAR_A);
|
||||
bytes[i] = UTF8_TRAILING[0];
|
||||
bytes[i + 1] = UTF8_TRAILING[1];
|
||||
bytes[i + 2] = UTF8_TRAILING[2];
|
||||
bytes[i] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
||||
runTest([bytes, str]);
|
||||
|
||||
codeUnits.add(CHAR_A);
|
||||
@@ -158,36 +157,36 @@ main() {
|
||||
codeUnits[i + 1] = TRAILING_SURROGATE;
|
||||
str = new String.fromCharCodes(codeUnits);
|
||||
bytes = new List.filled(i + 6, CHAR_A);
|
||||
bytes[i] = UTF8_TRAILING[0];
|
||||
bytes[i + 1] = UTF8_TRAILING[1];
|
||||
bytes[i + 2] = UTF8_TRAILING[2];
|
||||
bytes[i + 3] = UTF8_TRAILING[0];
|
||||
bytes[i + 4] = UTF8_TRAILING[1];
|
||||
bytes[i + 5] = UTF8_TRAILING[2];
|
||||
bytes[i] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
||||
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
||||
runTest([bytes, str]);
|
||||
|
||||
codeUnits[i] = LEADING_SURROGATE;
|
||||
codeUnits[i + 1] = LEADING_SURROGATE;
|
||||
str = new String.fromCharCodes(codeUnits);
|
||||
bytes = new List.filled(i + 6, CHAR_A);
|
||||
bytes[i] = UTF8_LEADING[0];
|
||||
bytes[i + 1] = UTF8_LEADING[1];
|
||||
bytes[i + 2] = UTF8_LEADING[2];
|
||||
bytes[i + 3] = UTF8_LEADING[0];
|
||||
bytes[i + 4] = UTF8_LEADING[1];
|
||||
bytes[i + 5] = UTF8_LEADING[2];
|
||||
bytes[i] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
||||
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
||||
runTest([bytes, str]);
|
||||
|
||||
codeUnits[i] = TRAILING_SURROGATE;
|
||||
codeUnits[i + 1] = LEADING_SURROGATE;
|
||||
str = new String.fromCharCodes(codeUnits);
|
||||
bytes = new List.filled(i + 6, CHAR_A);
|
||||
bytes[i] = UTF8_TRAILING[0];
|
||||
bytes[i + 1] = UTF8_TRAILING[1];
|
||||
bytes[i + 2] = UTF8_TRAILING[2];
|
||||
bytes[i + 3] = UTF8_LEADING[0];
|
||||
bytes[i + 4] = UTF8_LEADING[1];
|
||||
bytes[i + 5] = UTF8_LEADING[2];
|
||||
bytes[i] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
||||
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
||||
runTest([bytes, str]);
|
||||
|
||||
codeUnits.add(CHAR_A);
|
||||
@@ -210,12 +209,12 @@ main() {
|
||||
codeUnits[i + 2] = CHAR_A; // Add trailing 'a'.
|
||||
str = new String.fromCharCodes(codeUnits);
|
||||
bytes = new List.filled(i + 7, CHAR_A);
|
||||
bytes[i] = UTF8_TRAILING[0];
|
||||
bytes[i + 1] = UTF8_TRAILING[1];
|
||||
bytes[i + 2] = UTF8_TRAILING[2];
|
||||
bytes[i + 3] = UTF8_TRAILING[0];
|
||||
bytes[i + 4] = UTF8_TRAILING[1];
|
||||
bytes[i + 5] = UTF8_TRAILING[2];
|
||||
bytes[i] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
||||
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
||||
runTest([bytes, str]);
|
||||
|
||||
codeUnits[i] = LEADING_SURROGATE;
|
||||
@@ -223,12 +222,12 @@ main() {
|
||||
codeUnits[i + 2] = CHAR_A; // Add trailing 'a'.
|
||||
str = new String.fromCharCodes(codeUnits);
|
||||
bytes = new List.filled(i + 7, CHAR_A);
|
||||
bytes[i] = UTF8_LEADING[0];
|
||||
bytes[i + 1] = UTF8_LEADING[1];
|
||||
bytes[i + 2] = UTF8_LEADING[2];
|
||||
bytes[i + 3] = UTF8_LEADING[0];
|
||||
bytes[i + 4] = UTF8_LEADING[1];
|
||||
bytes[i + 5] = UTF8_LEADING[2];
|
||||
bytes[i] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
||||
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
||||
runTest([bytes, str]);
|
||||
|
||||
codeUnits[i] = TRAILING_SURROGATE;
|
||||
@@ -236,12 +235,12 @@ main() {
|
||||
codeUnits[i + 2] = CHAR_A; // Add trailing 'a'.
|
||||
str = new String.fromCharCodes(codeUnits);
|
||||
bytes = new List.filled(i + 7, CHAR_A);
|
||||
bytes[i] = UTF8_TRAILING[0];
|
||||
bytes[i + 1] = UTF8_TRAILING[1];
|
||||
bytes[i + 2] = UTF8_TRAILING[2];
|
||||
bytes[i + 3] = UTF8_LEADING[0];
|
||||
bytes[i + 4] = UTF8_LEADING[1];
|
||||
bytes[i + 5] = UTF8_LEADING[2];
|
||||
bytes[i] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
||||
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
||||
runTest([bytes, str]);
|
||||
|
||||
// Make sure the invariant is correct.
|
||||
|
||||
@@ -284,69 +284,69 @@ void jsonThrows(String name, String codeString) {
|
||||
void testMalformed() {
|
||||
// Overlong encodings.
|
||||
jsonMalformedTest(
|
||||
"overlong-0-2", "@\uFFFD@", [0x22, 0x40, 0xc0, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest(
|
||||
"overlong-0-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x80, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-0-4", "@\uFFFD@",
|
||||
"overlong-0-2", "@\uFFFD\uFFFD@", [0x22, 0x40, 0xc0, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-0-3", "@\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xe0, 0x80, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-0-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x80, 0x80, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest(
|
||||
"overlong-7f-2", "@\uFFFD@", [0x22, 0x40, 0xc1, 0xbf, 0x40, 0x22]);
|
||||
jsonMalformedTest(
|
||||
"overlong-7f-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x81, 0xbf, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-7f-4", "@\uFFFD@",
|
||||
"overlong-7f-2", "@\uFFFD\uFFFD@", [0x22, 0x40, 0xc1, 0xbf, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-7f-3", "@\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xe0, 0x81, 0xbf, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-7f-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x81, 0xbf, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest(
|
||||
"overlong-80-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x82, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-80-4", "@\uFFFD@",
|
||||
jsonMalformedTest("overlong-80-3", "@\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xe0, 0x82, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-80-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x82, 0x80, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest(
|
||||
"overlong-7ff-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x9f, 0xbf, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-7ff-4", "@\uFFFD@",
|
||||
jsonMalformedTest("overlong-7ff-3", "@\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xe0, 0x9f, 0xbf, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-7ff-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x9f, 0xbf, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("overlong-800-4", "@\uFFFD@",
|
||||
jsonMalformedTest("overlong-800-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0xa0, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-ffff-4", "@\uFFFD@",
|
||||
jsonMalformedTest("overlong-ffff-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xf0, 0x8f, 0xbf, 0xbf, 0x40, 0x22]);
|
||||
|
||||
// Unterminated multibyte sequences.
|
||||
jsonMalformedTest(
|
||||
"unterminated-2-normal", "@\uFFFD@", [0x22, 0x40, 0xc0, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-3-normal", "@\uFFFD@",
|
||||
jsonMalformedTest("unterminated-3-normal", "@\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xe0, 0x80, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-4-normal", "@\uFFFD@",
|
||||
jsonMalformedTest("unterminated-4-normal", "@\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x80, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-2-multi", "@\uFFFD\x80@",
|
||||
[0x22, 0x40, 0xc0, 0xc2, 0x80, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-3-multi", "@\uFFFD\x80@",
|
||||
jsonMalformedTest("unterminated-3-multi", "@\uFFFD\uFFFD\x80@",
|
||||
[0x22, 0x40, 0xe0, 0x80, 0xc2, 0x80, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-4-multi", "@\uFFFD\x80@",
|
||||
jsonMalformedTest("unterminated-4-multi", "@\uFFFD\uFFFD\uFFFD\x80@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x80, 0xc2, 0x80, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-2-escape", "@\uFFFD\n@",
|
||||
[0x22, 0x40, 0xc0, 0x5c, 0x6e, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-3-escape", "@\uFFFD\n@",
|
||||
jsonMalformedTest("unterminated-3-escape", "@\uFFFD\uFFFD\n@",
|
||||
[0x22, 0x40, 0xe0, 0x80, 0x5c, 0x6e, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-4-escape", "@\uFFFD\n@",
|
||||
jsonMalformedTest("unterminated-4-escape", "@\uFFFD\uFFFD\uFFFD\n@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x80, 0x5c, 0x6e, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-2-end", "@\uFFFD", [0x22, 0x40, 0xc0, 0x22]);
|
||||
|
||||
jsonMalformedTest(
|
||||
"unterminated-3-end", "@\uFFFD", [0x22, 0x40, 0xe0, 0x80, 0x22]);
|
||||
"unterminated-3-end", "@\uFFFD\uFFFD", [0x22, 0x40, 0xe0, 0x80, 0x22]);
|
||||
|
||||
jsonMalformedTest(
|
||||
"unterminated-4-end", "@\uFFFD", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x22]);
|
||||
jsonMalformedTest("unterminated-4-end", "@\uFFFD\uFFFD\uFFFD",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x80, 0x22]);
|
||||
|
||||
// Unexpected continuation byte
|
||||
// - after a normal character.
|
||||
@@ -372,13 +372,13 @@ void testMalformed() {
|
||||
"leading-2", "@\uFFFD\x80@", [0x22, 0x40, 0xc0, 0xc2, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("leading-3-1", "@\uFFFD\x80@",
|
||||
[0x22, 0x40, 0xe0, 0xc2, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("leading-3-2", "@\uFFFD\x80@",
|
||||
jsonMalformedTest("leading-3-2", "@\uFFFD\uFFFD\x80@",
|
||||
[0x22, 0x40, 0xe0, 0x80, 0xc2, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("leading-4-1", "@\uFFFD\x80@",
|
||||
[0x22, 0x40, 0xf0, 0xc2, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("leading-4-2", "@\uFFFD\x80@",
|
||||
jsonMalformedTest("leading-4-2", "@\uFFFD\uFFFD\x80@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0xc2, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("leading-4-3", "@\uFFFD\x80@",
|
||||
jsonMalformedTest("leading-4-3", "@\uFFFD\uFFFD\uFFFD\x80@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x80, 0xc2, 0x80, 0x40, 0x22]);
|
||||
|
||||
// Overlong encodings of ASCII outside of strings always fail.
|
||||
|
||||
@@ -28,11 +28,14 @@ String decodeAllowMalformed4(List<int> bytes) {
|
||||
return new Utf8Codec(allowMalformed: true).decoder.convert(bytes);
|
||||
}
|
||||
|
||||
final TESTS = [
|
||||
final TESTS0 = [
|
||||
// Unfinished UTF-8 sequences.
|
||||
[0xc3],
|
||||
[0xE2, 0x82],
|
||||
[0xF0, 0xA4, 0xAD],
|
||||
[0xF0, 0xA4, 0xAD]
|
||||
];
|
||||
|
||||
final TESTS1 = [
|
||||
// Overlong encoding of euro-sign.
|
||||
[0xF0, 0x82, 0x82, 0xAC],
|
||||
// Other overlong/unfinished sequences.
|
||||
@@ -64,11 +67,11 @@ final TESTS2 = [
|
||||
// Test that 0xC0|1, 0x80 does not eat the next character.
|
||||
[
|
||||
[0xC0, 0x80, 0x61],
|
||||
"Xa"
|
||||
"XXa"
|
||||
],
|
||||
[
|
||||
[0xC1, 0x80, 0x61],
|
||||
"Xa"
|
||||
"XXa"
|
||||
],
|
||||
// 0xF5 .. 0xFF never appear in valid UTF-8 sequences.
|
||||
[
|
||||
@@ -207,44 +210,49 @@ final TESTS2 = [
|
||||
];
|
||||
|
||||
main() {
|
||||
var allTests = TESTS.expand((test) {
|
||||
var allTests = [...TESTS0, ...TESTS1].expand((test) {
|
||||
// Pairs of test and expected string output when malformed strings are
|
||||
// allowed. Replacement character: U+FFFD
|
||||
// allowed. Replacement character: U+FFFD, one per unfinished sequence or
|
||||
// undecodable byte.
|
||||
String replacement =
|
||||
TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length;
|
||||
return [
|
||||
[test, "\u{FFFD}"],
|
||||
[test, "${replacement}"],
|
||||
[
|
||||
new List<int>.from([0x61])..addAll(test),
|
||||
"a\u{FFFD}"
|
||||
[0x61, ...test],
|
||||
"a${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0x61])
|
||||
..addAll(test)
|
||||
..add(0x61),
|
||||
"a\u{FFFD}a"
|
||||
],
|
||||
[new List<int>.from(test)..add(0x61), "\u{FFFD}a"],
|
||||
[new List<int>.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"],
|
||||
[
|
||||
new List<int>.from(test)
|
||||
..add(0x61)
|
||||
..addAll(test),
|
||||
"\u{FFFD}a\u{FFFD}"
|
||||
[0x61, ...test, 0x61],
|
||||
"a${replacement}a"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0xc3, 0xa5])..addAll(test),
|
||||
"å\u{FFFD}"
|
||||
[...test, 0x61],
|
||||
"${replacement}a"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]),
|
||||
"å\u{FFFD}å"
|
||||
[...test, ...test],
|
||||
"${replacement}${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from(test)..addAll([0xc3, 0xa5]),
|
||||
"\u{FFFD}å"
|
||||
[...test, 0x61, ...test],
|
||||
"${replacement}a${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from(test)..addAll([0xc3, 0xa5])..addAll(test),
|
||||
"\u{FFFD}å\u{FFFD}"
|
||||
[0xc3, 0xa5, ...test],
|
||||
"å${replacement}"
|
||||
],
|
||||
[
|
||||
[0xc3, 0xa5, ...test, 0xc3, 0xa5],
|
||||
"å${replacement}å"
|
||||
],
|
||||
[
|
||||
[...test, 0xc3, 0xa5],
|
||||
"${replacement}å"
|
||||
],
|
||||
[
|
||||
[...test, 0xc3, 0xa5, ...test],
|
||||
"${replacement}å${replacement}"
|
||||
]
|
||||
];
|
||||
});
|
||||
|
||||
@@ -727,10 +727,14 @@ void testUtf8bytesToCodepoints(List<int> utf8ToRunes(List<int> utf8)) {
|
||||
Expect.listEquals([0xe000], utf8ToRunes([0xee, 0x80, 0x80]), "e000");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xef, 0xbf, 0xbd]), "fffd");
|
||||
Expect
|
||||
.listEquals([0x10ffff], utf8ToRunes([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xf4, 0x90, 0x80, 0x80]), "110000");
|
||||
Expect.listEquals(
|
||||
[0x10ffff], utf8ToRunes([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
], utf8ToRunes([0xf4, 0x90, 0x80, 0x80]), "110000");
|
||||
|
||||
// unexpected continuation bytes
|
||||
Expect.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0x80]),
|
||||
@@ -795,12 +799,15 @@ void testUtf8bytesToCodepoints(List<int> utf8ToRunes(List<int> utf8)) {
|
||||
// Sequences with last continuation byte missing
|
||||
Expect.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xc2]),
|
||||
"2-byte sequence with last byte missing");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xe0, 0x80]), "3-byte sequence with last byte missing");
|
||||
Expect.listEquals(
|
||||
[unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xf0, 0x80, 0x80]),
|
||||
"4-byte sequence with last byte missing");
|
||||
[unicodeReplacementCharacterRune, unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xe0, 0x80]),
|
||||
"3-byte sequence with last byte missing");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
], utf8ToRunes([0xf0, 0x80, 0x80]), "4-byte sequence with last byte missing");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
@@ -871,6 +878,9 @@ void testUtf8bytesToCodepoints(List<int> utf8ToRunes(List<int> utf8)) {
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
],
|
||||
utf8ToRunes([
|
||||
@@ -890,8 +900,8 @@ void testUtf8bytesToCodepoints(List<int> utf8ToRunes(List<int> utf8)) {
|
||||
0x80,
|
||||
0x80,
|
||||
0xdf,
|
||||
0xef,
|
||||
0xbf,
|
||||
0xef, // These two bytes form one incomplete sequence.
|
||||
0xbf, // All others form one per byte.
|
||||
0xf7,
|
||||
0xbf,
|
||||
0xbf,
|
||||
@@ -908,10 +918,10 @@ void testUtf8bytesToCodepoints(List<int> utf8ToRunes(List<int> utf8)) {
|
||||
"Concatenation of incomplete sequences");
|
||||
|
||||
// Impossible bytes
|
||||
Expect
|
||||
.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xfe]), "fe");
|
||||
Expect
|
||||
.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xff]), "ff");
|
||||
Expect.listEquals(
|
||||
[unicodeReplacementCharacterRune], utf8ToRunes([0xfe]), "fe");
|
||||
Expect.listEquals(
|
||||
[unicodeReplacementCharacterRune], utf8ToRunes([0xff]), "ff");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
@@ -921,11 +931,20 @@ void testUtf8bytesToCodepoints(List<int> utf8ToRunes(List<int> utf8)) {
|
||||
|
||||
// Overlong sequences
|
||||
Expect.listEquals(
|
||||
[unicodeReplacementCharacterRune], utf8ToRunes([0xc0, 0xaf]), "c0 af");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xe0, 0x80, 0xaf]), "e0 80 af");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af");
|
||||
[unicodeReplacementCharacterRune, unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xc0, 0xaf]),
|
||||
"c0 af");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
], utf8ToRunes([0xe0, 0x80, 0xaf]), "e0 80 af");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
], utf8ToRunes([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
@@ -943,11 +962,20 @@ void testUtf8bytesToCodepoints(List<int> utf8ToRunes(List<int> utf8)) {
|
||||
], utf8ToRunes([0xfc, 0x80, 0x80, 0x80, 0x80, 0xaf]), "fc 80 80 80 80 af");
|
||||
|
||||
Expect.listEquals(
|
||||
[unicodeReplacementCharacterRune], utf8ToRunes([0xc1, 0xbf]), "c1 bf");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xe0, 0x9f, 0xbf]), "e0 9f bf");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf");
|
||||
[unicodeReplacementCharacterRune, unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xc1, 0xbf]),
|
||||
"c1 bf");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
], utf8ToRunes([0xe0, 0x9f, 0xbf]), "e0 9f bf");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
], utf8ToRunes([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
@@ -965,11 +993,20 @@ void testUtf8bytesToCodepoints(List<int> utf8ToRunes(List<int> utf8)) {
|
||||
], utf8ToRunes([0xfc, 0x83, 0xbf, 0xbf, 0xbf, 0xbf]), "fc 83 bf bf bf bf");
|
||||
|
||||
Expect.listEquals(
|
||||
[unicodeReplacementCharacterRune], utf8ToRunes([0xc0, 0x80]), "c0 80");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xe0, 0x80, 0x80]), "e0 80 80");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80");
|
||||
[unicodeReplacementCharacterRune, unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xc0, 0x80]),
|
||||
"c0 80");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
], utf8ToRunes([0xe0, 0x80, 0x80]), "e0 80 80");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
], utf8ToRunes([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
|
||||
@@ -9,7 +9,7 @@ import 'dart:convert';
|
||||
|
||||
main() {
|
||||
for (int i = 0; i <= 0x10FFFF; i++) {
|
||||
if (i == unicodeBomCharacterRune) continue;
|
||||
if (i == unicodeBomCharacterRune || (i & 0x1FF800) == 0xD800) continue;
|
||||
Expect.equals(
|
||||
i, utf8.decode(utf8.encode(new String.fromCharCode(i))).runes.first);
|
||||
}
|
||||
|
||||
@@ -25,8 +25,8 @@ void testEncodeSlice() {
|
||||
String ascii = "ABCDE";
|
||||
Expect.listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii));
|
||||
Expect.listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0));
|
||||
Expect
|
||||
.listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0, 5));
|
||||
Expect.listEquals(
|
||||
[0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0, 5));
|
||||
Expect.listEquals([0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 1));
|
||||
Expect.listEquals([0x41, 0x42, 0x43, 0x44], encoder.convert(ascii, 0, 4));
|
||||
Expect.listEquals([0x42, 0x43, 0x44], encoder.convert(ascii, 1, 4));
|
||||
@@ -50,6 +50,6 @@ void testEncodeSlice() {
|
||||
Expect.listEquals(
|
||||
[0xc2, 0x82, 0xe1, 0x81, 0x81], encoder.convert(unicode, 1, 3));
|
||||
// Split in the middle of a surrogate pair.
|
||||
Expect.listEquals([0xc2, 0x82, 0xe1, 0x81, 0x81, 0xed, 0xa0, 0x80],
|
||||
Expect.listEquals([0xc2, 0x82, 0xe1, 0x81, 0x81, 0xef, 0xbf, 0xbd],
|
||||
encoder.convert(unicode, 1, 4));
|
||||
}
|
||||
|
||||
@@ -73,28 +73,29 @@ void testErrorOffset() {
|
||||
|
||||
// Bad encoding, points to first bad byte.
|
||||
testExn([0x80, 0x00], 0);
|
||||
testExn([0xC0, 0x00], 1);
|
||||
testExn([0xE0, 0x00], 1);
|
||||
testExn([0xE0, 0x80, 0x00], 2);
|
||||
testExn([0xF0, 0x00], 1);
|
||||
testExn([0xF0, 0x80, 0x00], 2);
|
||||
testExn([0xF0, 0x80, 0x80, 0x00], 3);
|
||||
testExn([0xC2, 0x00], 1);
|
||||
testExn([0xE2, 0x00], 1);
|
||||
testExn([0xE2, 0x80, 0x00], 2);
|
||||
testExn([0xF2, 0x00], 1);
|
||||
testExn([0xF2, 0x80, 0x00], 2);
|
||||
testExn([0xF2, 0x80, 0x80, 0x00], 3);
|
||||
testExn([0xF8, 0x00], 0);
|
||||
// Short encoding, points to end.
|
||||
testExn([0xC0], 1);
|
||||
testExn([0xE0], 1);
|
||||
testExn([0xE0, 0x80], 2);
|
||||
testExn([0xF0], 1);
|
||||
testExn([0xF0, 0x80], 2);
|
||||
testExn([0xF0, 0x80, 0x80], 3);
|
||||
// Overlong encoding, points to start of encoding.
|
||||
testExn([0xC2], 1);
|
||||
testExn([0xE2], 1);
|
||||
testExn([0xE2, 0x80], 2);
|
||||
testExn([0xF2], 1);
|
||||
testExn([0xF2, 0x80], 2);
|
||||
testExn([0xF2, 0x80, 0x80], 3);
|
||||
// Overlong encoding, points to byte that gave enough information to conclude
|
||||
// that it was overlong.
|
||||
testExn([0xC0, 0x80], 0);
|
||||
testExn([0xC1, 0xBF], 0);
|
||||
testExn([0xE0, 0x80, 0x80], 0);
|
||||
testExn([0xE0, 0x9F, 0xBF], 0);
|
||||
testExn([0xF0, 0x80, 0x80, 0x80], 0);
|
||||
testExn([0xF0, 0x8F, 0xBF, 0xBF], 0);
|
||||
testExn([0xE0, 0x80, 0x80], 1);
|
||||
testExn([0xE0, 0x9F, 0xBF], 1);
|
||||
testExn([0xF0, 0x80, 0x80, 0x80], 1);
|
||||
testExn([0xF0, 0x8F, 0xBF, 0xBF], 1);
|
||||
// Invalid character (value too large, over 0x10FFFF).
|
||||
testExn([0xF4, 0x90, 0x80, 0x80], 0);
|
||||
testExn([0xF4, 0x90, 0x80, 0x80], 1);
|
||||
testExn([0xF7, 0xBF, 0xBF, 0xBF], 0);
|
||||
}
|
||||
|
||||
@@ -46,11 +46,14 @@ String decodeAllowMalformed(List<int> bytes, int chunkSize) {
|
||||
return buffer.toString();
|
||||
}
|
||||
|
||||
final TESTS = [
|
||||
final TESTS0 = [
|
||||
// Unfinished UTF-8 sequences.
|
||||
[0xc3],
|
||||
[0xE2, 0x82],
|
||||
[0xF0, 0xA4, 0xAD],
|
||||
[0xF0, 0xA4, 0xAD]
|
||||
];
|
||||
|
||||
final TESTS1 = [
|
||||
// Overlong encoding of euro-sign.
|
||||
[0xF0, 0x82, 0x82, 0xAC],
|
||||
// Other overlong/unfinished sequences.
|
||||
@@ -77,11 +80,11 @@ final TESTS2 = [
|
||||
// Test that 0xC0|1, 0x80 does not eat the next character.
|
||||
[
|
||||
[0xC0, 0x80, 0x61],
|
||||
"Xa"
|
||||
"XXa"
|
||||
],
|
||||
[
|
||||
[0xC1, 0x80, 0x61],
|
||||
"Xa"
|
||||
"XXa"
|
||||
],
|
||||
// 0xF5 .. 0xFF never appear in valid UTF-8 sequences.
|
||||
[
|
||||
@@ -220,44 +223,49 @@ final TESTS2 = [
|
||||
];
|
||||
|
||||
main() {
|
||||
var allTests = TESTS.expand((test) {
|
||||
var allTests = [...TESTS0, ...TESTS1].expand((test) {
|
||||
// Pairs of test and expected string output when malformed strings are
|
||||
// allowed. Replacement character: U+FFFD
|
||||
// allowed. Replacement character: U+FFFD, one per unfinished sequence or
|
||||
// undecodable byte.
|
||||
String replacement =
|
||||
TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length;
|
||||
return [
|
||||
[test, "\u{FFFD}"],
|
||||
[test, "${replacement}"],
|
||||
[
|
||||
new List<int>.from([0x61])..addAll(test),
|
||||
"a\u{FFFD}"
|
||||
[0x61, ...test],
|
||||
"a${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0x61])
|
||||
..addAll(test)
|
||||
..add(0x61),
|
||||
"a\u{FFFD}a"
|
||||
],
|
||||
[new List<int>.from(test)..add(0x61), "\u{FFFD}a"],
|
||||
[new List<int>.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"],
|
||||
[
|
||||
new List<int>.from(test)
|
||||
..add(0x61)
|
||||
..addAll(test),
|
||||
"\u{FFFD}a\u{FFFD}"
|
||||
[0x61, ...test, 0x61],
|
||||
"a${replacement}a"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0xc3, 0xa5])..addAll(test),
|
||||
"å\u{FFFD}"
|
||||
[...test, 0x61],
|
||||
"${replacement}a"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]),
|
||||
"å\u{FFFD}å"
|
||||
[...test, ...test],
|
||||
"${replacement}${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from(test)..addAll([0xc3, 0xa5]),
|
||||
"\u{FFFD}å"
|
||||
[...test, 0x61, ...test],
|
||||
"${replacement}a${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from(test)..addAll([0xc3, 0xa5])..addAll(test),
|
||||
"\u{FFFD}å\u{FFFD}"
|
||||
[0xc3, 0xa5, ...test],
|
||||
"å${replacement}"
|
||||
],
|
||||
[
|
||||
[0xc3, 0xa5, ...test, 0xc3, 0xa5],
|
||||
"å${replacement}å"
|
||||
],
|
||||
[
|
||||
[...test, 0xc3, 0xa5],
|
||||
"${replacement}å"
|
||||
],
|
||||
[
|
||||
[...test, 0xc3, 0xa5, ...test],
|
||||
"${replacement}å${replacement}"
|
||||
]
|
||||
];
|
||||
});
|
||||
|
||||
@@ -28,11 +28,14 @@ String decodeAllowMalformed(List<int> bytes) {
|
||||
return buffer.toString();
|
||||
}
|
||||
|
||||
final TESTS = [
|
||||
final TESTS0 = [
|
||||
// Unfinished UTF-8 sequences.
|
||||
[0xc3],
|
||||
[0xE2, 0x82],
|
||||
[0xF0, 0xA4, 0xAD],
|
||||
[0xF0, 0xA4, 0xAD]
|
||||
];
|
||||
|
||||
final TESTS1 = [
|
||||
// Overlong encoding of euro-sign.
|
||||
[0xF0, 0x82, 0x82, 0xAC],
|
||||
// Other overlong/unfinished sequences.
|
||||
@@ -59,11 +62,11 @@ final TESTS2 = [
|
||||
// Test that 0xC0|1, 0x80 does not eat the next character.
|
||||
[
|
||||
[0xC0, 0x80, 0x61],
|
||||
"Xa"
|
||||
"XXa"
|
||||
],
|
||||
[
|
||||
[0xC1, 0x80, 0x61],
|
||||
"Xa"
|
||||
"XXa"
|
||||
],
|
||||
// 0xF5 .. 0xFF never appear in valid UTF-8 sequences.
|
||||
[
|
||||
@@ -202,44 +205,49 @@ final TESTS2 = [
|
||||
];
|
||||
|
||||
main() {
|
||||
var allTests = TESTS.expand((test) {
|
||||
var allTests = [...TESTS0, ...TESTS1].expand((test) {
|
||||
// Pairs of test and expected string output when malformed strings are
|
||||
// allowed. Replacement character: U+FFFD
|
||||
// allowed. Replacement character: U+FFFD, one per unfinished sequence or
|
||||
// undecodable byte.
|
||||
String replacement =
|
||||
TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length;
|
||||
return [
|
||||
[test, "\u{FFFD}"],
|
||||
[test, "${replacement}"],
|
||||
[
|
||||
new List<int>.from([0x61])..addAll(test),
|
||||
"a\u{FFFD}"
|
||||
[0x61, ...test],
|
||||
"a${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0x61])
|
||||
..addAll(test)
|
||||
..add(0x61),
|
||||
"a\u{FFFD}a"
|
||||
],
|
||||
[new List<int>.from(test)..add(0x61), "\u{FFFD}a"],
|
||||
[new List<int>.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"],
|
||||
[
|
||||
new List<int>.from(test)
|
||||
..add(0x61)
|
||||
..addAll(test),
|
||||
"\u{FFFD}a\u{FFFD}"
|
||||
[0x61, ...test, 0x61],
|
||||
"a${replacement}a"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0xc3, 0xa5])..addAll(test),
|
||||
"å\u{FFFD}"
|
||||
[...test, 0x61],
|
||||
"${replacement}a"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]),
|
||||
"å\u{FFFD}å"
|
||||
[...test, ...test],
|
||||
"${replacement}${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from(test)..addAll([0xc3, 0xa5]),
|
||||
"\u{FFFD}å"
|
||||
[...test, 0x61, ...test],
|
||||
"${replacement}a${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from(test)..addAll([0xc3, 0xa5])..addAll(test),
|
||||
"\u{FFFD}å\u{FFFD}"
|
||||
[0xc3, 0xa5, ...test],
|
||||
"å${replacement}"
|
||||
],
|
||||
[
|
||||
[0xc3, 0xa5, ...test, 0xc3, 0xa5],
|
||||
"å${replacement}å"
|
||||
],
|
||||
[
|
||||
[...test, 0xc3, 0xa5],
|
||||
"${replacement}å"
|
||||
],
|
||||
[
|
||||
[...test, 0xc3, 0xa5, ...test],
|
||||
"${replacement}å${replacement}"
|
||||
]
|
||||
];
|
||||
});
|
||||
|
||||
@@ -49,11 +49,14 @@ String decodeAllowMalformed2(List<int> inputBytes) {
|
||||
return utf8.decode(bytes);
|
||||
}
|
||||
|
||||
final TESTS = [
|
||||
final TESTS0 = [
|
||||
// Unfinished UTF-8 sequences.
|
||||
[0xc3],
|
||||
[0xE2, 0x82],
|
||||
[0xF0, 0xA4, 0xAD],
|
||||
[0xF0, 0xA4, 0xAD]
|
||||
];
|
||||
|
||||
final TESTS1 = [
|
||||
// Overlong encoding of euro-sign.
|
||||
[0xF0, 0x82, 0x82, 0xAC],
|
||||
// Other overlong/unfinished sequences.
|
||||
@@ -80,11 +83,11 @@ final TESTS2 = [
|
||||
// Test that 0xC0|1, 0x80 does not eat the next character.
|
||||
[
|
||||
[0xC0, 0x80, 0x61],
|
||||
"Xa"
|
||||
"XXa"
|
||||
],
|
||||
[
|
||||
[0xC1, 0x80, 0x61],
|
||||
"Xa"
|
||||
"XXa"
|
||||
],
|
||||
// 0xF5 .. 0xFF never appear in valid UTF-8 sequences.
|
||||
[
|
||||
@@ -223,44 +226,49 @@ final TESTS2 = [
|
||||
];
|
||||
|
||||
main() {
|
||||
var allTests = TESTS.expand((test) {
|
||||
var allTests = [...TESTS0, ...TESTS1].expand((test) {
|
||||
// Pairs of test and expected string output when malformed strings are
|
||||
// allowed. Replacement character: U+FFFD
|
||||
// allowed. Replacement character: U+FFFD, one per unfinished sequence or
|
||||
// undecodable byte.
|
||||
String replacement =
|
||||
TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length;
|
||||
return [
|
||||
[test, "\u{FFFD}"],
|
||||
[test, "${replacement}"],
|
||||
[
|
||||
new List<int>.from([0x61])..addAll(test),
|
||||
"a\u{FFFD}"
|
||||
[0x61, ...test],
|
||||
"a${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0x61])
|
||||
..addAll(test)
|
||||
..add(0x61),
|
||||
"a\u{FFFD}a"
|
||||
],
|
||||
[new List<int>.from(test)..add(0x61), "\u{FFFD}a"],
|
||||
[new List<int>.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"],
|
||||
[
|
||||
new List<int>.from(test)
|
||||
..add(0x61)
|
||||
..addAll(test),
|
||||
"\u{FFFD}a\u{FFFD}"
|
||||
[0x61, ...test, 0x61],
|
||||
"a${replacement}a"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0xc3, 0xa5])..addAll(test),
|
||||
"å\u{FFFD}"
|
||||
[...test, 0x61],
|
||||
"${replacement}a"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]),
|
||||
"å\u{FFFD}å"
|
||||
[...test, ...test],
|
||||
"${replacement}${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from(test)..addAll([0xc3, 0xa5]),
|
||||
"\u{FFFD}å"
|
||||
[...test, 0x61, ...test],
|
||||
"${replacement}a${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from(test)..addAll([0xc3, 0xa5])..addAll(test),
|
||||
"\u{FFFD}å\u{FFFD}"
|
||||
[0xc3, 0xa5, ...test],
|
||||
"å${replacement}"
|
||||
],
|
||||
[
|
||||
[0xc3, 0xa5, ...test, 0xc3, 0xa5],
|
||||
"å${replacement}å"
|
||||
],
|
||||
[
|
||||
[...test, 0xc3, 0xa5],
|
||||
"${replacement}å"
|
||||
],
|
||||
[
|
||||
[...test, 0xc3, 0xa5, ...test],
|
||||
"${replacement}å${replacement}"
|
||||
]
|
||||
];
|
||||
});
|
||||
|
||||
@@ -112,8 +112,7 @@ main() {
|
||||
const LEADING_SURROGATE = 0xd801;
|
||||
const TRAILING_SURROGATE = 0xdc12;
|
||||
const UTF8_ENCODING = const [0xf0, 0x90, 0x90, 0x92];
|
||||
const UTF8_LEADING = const [0xed, 0xa0, 0x81];
|
||||
const UTF8_TRAILING = const [0xed, 0xb0, 0x92];
|
||||
const UTF8_REPLACEMENT = const [0xef, 0xbf, 0xbd];
|
||||
const CHAR_A = 0x61;
|
||||
|
||||
// Test surrogates at all kinds of locations.
|
||||
@@ -129,17 +128,17 @@ main() {
|
||||
codeUnits[i] = LEADING_SURROGATE;
|
||||
var str = new String.fromCharCodes(codeUnits);
|
||||
var bytes = new List.filled(i + 3, CHAR_A);
|
||||
bytes[i] = UTF8_LEADING[0];
|
||||
bytes[i + 1] = UTF8_LEADING[1];
|
||||
bytes[i + 2] = UTF8_LEADING[2];
|
||||
bytes[i] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
||||
runTest([bytes, str]);
|
||||
|
||||
codeUnits[i] = TRAILING_SURROGATE;
|
||||
str = new String.fromCharCodes(codeUnits);
|
||||
bytes = new List.filled(i + 3, CHAR_A);
|
||||
bytes[i] = UTF8_TRAILING[0];
|
||||
bytes[i + 1] = UTF8_TRAILING[1];
|
||||
bytes[i + 2] = UTF8_TRAILING[2];
|
||||
bytes[i] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
||||
runTest([bytes, str]);
|
||||
|
||||
codeUnits.length = i + 2;
|
||||
@@ -157,36 +156,36 @@ main() {
|
||||
codeUnits[i + 1] = TRAILING_SURROGATE;
|
||||
str = new String.fromCharCodes(codeUnits);
|
||||
bytes = new List.filled(i + 6, CHAR_A);
|
||||
bytes[i] = UTF8_TRAILING[0];
|
||||
bytes[i + 1] = UTF8_TRAILING[1];
|
||||
bytes[i + 2] = UTF8_TRAILING[2];
|
||||
bytes[i + 3] = UTF8_TRAILING[0];
|
||||
bytes[i + 4] = UTF8_TRAILING[1];
|
||||
bytes[i + 5] = UTF8_TRAILING[2];
|
||||
bytes[i] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
||||
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
||||
runTest([bytes, str]);
|
||||
|
||||
codeUnits[i] = LEADING_SURROGATE;
|
||||
codeUnits[i + 1] = LEADING_SURROGATE;
|
||||
str = new String.fromCharCodes(codeUnits);
|
||||
bytes = new List.filled(i + 6, CHAR_A);
|
||||
bytes[i] = UTF8_LEADING[0];
|
||||
bytes[i + 1] = UTF8_LEADING[1];
|
||||
bytes[i + 2] = UTF8_LEADING[2];
|
||||
bytes[i + 3] = UTF8_LEADING[0];
|
||||
bytes[i + 4] = UTF8_LEADING[1];
|
||||
bytes[i + 5] = UTF8_LEADING[2];
|
||||
bytes[i] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
||||
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
||||
runTest([bytes, str]);
|
||||
|
||||
codeUnits[i] = TRAILING_SURROGATE;
|
||||
codeUnits[i + 1] = LEADING_SURROGATE;
|
||||
str = new String.fromCharCodes(codeUnits);
|
||||
bytes = new List.filled(i + 6, CHAR_A);
|
||||
bytes[i] = UTF8_TRAILING[0];
|
||||
bytes[i + 1] = UTF8_TRAILING[1];
|
||||
bytes[i + 2] = UTF8_TRAILING[2];
|
||||
bytes[i + 3] = UTF8_LEADING[0];
|
||||
bytes[i + 4] = UTF8_LEADING[1];
|
||||
bytes[i + 5] = UTF8_LEADING[2];
|
||||
bytes[i] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
||||
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
||||
runTest([bytes, str]);
|
||||
|
||||
codeUnits.length = i + 3;
|
||||
@@ -208,12 +207,12 @@ main() {
|
||||
codeUnits[i + 2] = CHAR_A; // Add trailing 'a'.
|
||||
str = new String.fromCharCodes(codeUnits);
|
||||
bytes = new List.filled(i + 7, CHAR_A);
|
||||
bytes[i] = UTF8_TRAILING[0];
|
||||
bytes[i + 1] = UTF8_TRAILING[1];
|
||||
bytes[i + 2] = UTF8_TRAILING[2];
|
||||
bytes[i + 3] = UTF8_TRAILING[0];
|
||||
bytes[i + 4] = UTF8_TRAILING[1];
|
||||
bytes[i + 5] = UTF8_TRAILING[2];
|
||||
bytes[i] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
||||
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
||||
runTest([bytes, str]);
|
||||
|
||||
codeUnits[i] = LEADING_SURROGATE;
|
||||
@@ -221,12 +220,12 @@ main() {
|
||||
codeUnits[i + 2] = CHAR_A; // Add trailing 'a'.
|
||||
str = new String.fromCharCodes(codeUnits);
|
||||
bytes = new List.filled(i + 7, CHAR_A);
|
||||
bytes[i] = UTF8_LEADING[0];
|
||||
bytes[i + 1] = UTF8_LEADING[1];
|
||||
bytes[i + 2] = UTF8_LEADING[2];
|
||||
bytes[i + 3] = UTF8_LEADING[0];
|
||||
bytes[i + 4] = UTF8_LEADING[1];
|
||||
bytes[i + 5] = UTF8_LEADING[2];
|
||||
bytes[i] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
||||
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
||||
runTest([bytes, str]);
|
||||
|
||||
codeUnits[i] = TRAILING_SURROGATE;
|
||||
@@ -234,12 +233,12 @@ main() {
|
||||
codeUnits[i + 2] = CHAR_A; // Add trailing 'a'.
|
||||
str = new String.fromCharCodes(codeUnits);
|
||||
bytes = new List.filled(i + 7, CHAR_A);
|
||||
bytes[i] = UTF8_TRAILING[0];
|
||||
bytes[i + 1] = UTF8_TRAILING[1];
|
||||
bytes[i + 2] = UTF8_TRAILING[2];
|
||||
bytes[i + 3] = UTF8_LEADING[0];
|
||||
bytes[i + 4] = UTF8_LEADING[1];
|
||||
bytes[i + 5] = UTF8_LEADING[2];
|
||||
bytes[i] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
||||
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
||||
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
||||
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
||||
runTest([bytes, str]);
|
||||
|
||||
// Make sure the invariant is correct.
|
||||
|
||||
@@ -284,69 +284,69 @@ void jsonThrows(String name, String codeString) {
|
||||
void testMalformed() {
|
||||
// Overlong encodings.
|
||||
jsonMalformedTest(
|
||||
"overlong-0-2", "@\uFFFD@", [0x22, 0x40, 0xc0, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest(
|
||||
"overlong-0-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x80, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-0-4", "@\uFFFD@",
|
||||
"overlong-0-2", "@\uFFFD\uFFFD@", [0x22, 0x40, 0xc0, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-0-3", "@\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xe0, 0x80, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-0-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x80, 0x80, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest(
|
||||
"overlong-7f-2", "@\uFFFD@", [0x22, 0x40, 0xc1, 0xbf, 0x40, 0x22]);
|
||||
jsonMalformedTest(
|
||||
"overlong-7f-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x81, 0xbf, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-7f-4", "@\uFFFD@",
|
||||
"overlong-7f-2", "@\uFFFD\uFFFD@", [0x22, 0x40, 0xc1, 0xbf, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-7f-3", "@\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xe0, 0x81, 0xbf, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-7f-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x81, 0xbf, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest(
|
||||
"overlong-80-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x82, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-80-4", "@\uFFFD@",
|
||||
jsonMalformedTest("overlong-80-3", "@\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xe0, 0x82, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-80-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x82, 0x80, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest(
|
||||
"overlong-7ff-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x9f, 0xbf, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-7ff-4", "@\uFFFD@",
|
||||
jsonMalformedTest("overlong-7ff-3", "@\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xe0, 0x9f, 0xbf, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-7ff-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x9f, 0xbf, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("overlong-800-4", "@\uFFFD@",
|
||||
jsonMalformedTest("overlong-800-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0xa0, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("overlong-ffff-4", "@\uFFFD@",
|
||||
jsonMalformedTest("overlong-ffff-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xf0, 0x8f, 0xbf, 0xbf, 0x40, 0x22]);
|
||||
|
||||
// Unterminated multibyte sequences.
|
||||
jsonMalformedTest(
|
||||
"unterminated-2-normal", "@\uFFFD@", [0x22, 0x40, 0xc0, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-3-normal", "@\uFFFD@",
|
||||
jsonMalformedTest("unterminated-3-normal", "@\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xe0, 0x80, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-4-normal", "@\uFFFD@",
|
||||
jsonMalformedTest("unterminated-4-normal", "@\uFFFD\uFFFD\uFFFD@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x80, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-2-multi", "@\uFFFD\x80@",
|
||||
[0x22, 0x40, 0xc0, 0xc2, 0x80, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-3-multi", "@\uFFFD\x80@",
|
||||
jsonMalformedTest("unterminated-3-multi", "@\uFFFD\uFFFD\x80@",
|
||||
[0x22, 0x40, 0xe0, 0x80, 0xc2, 0x80, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-4-multi", "@\uFFFD\x80@",
|
||||
jsonMalformedTest("unterminated-4-multi", "@\uFFFD\uFFFD\uFFFD\x80@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x80, 0xc2, 0x80, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-2-escape", "@\uFFFD\n@",
|
||||
[0x22, 0x40, 0xc0, 0x5c, 0x6e, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-3-escape", "@\uFFFD\n@",
|
||||
jsonMalformedTest("unterminated-3-escape", "@\uFFFD\uFFFD\n@",
|
||||
[0x22, 0x40, 0xe0, 0x80, 0x5c, 0x6e, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-4-escape", "@\uFFFD\n@",
|
||||
jsonMalformedTest("unterminated-4-escape", "@\uFFFD\uFFFD\uFFFD\n@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x80, 0x5c, 0x6e, 0x40, 0x22]);
|
||||
|
||||
jsonMalformedTest("unterminated-2-end", "@\uFFFD", [0x22, 0x40, 0xc0, 0x22]);
|
||||
|
||||
jsonMalformedTest(
|
||||
"unterminated-3-end", "@\uFFFD", [0x22, 0x40, 0xe0, 0x80, 0x22]);
|
||||
"unterminated-3-end", "@\uFFFD\uFFFD", [0x22, 0x40, 0xe0, 0x80, 0x22]);
|
||||
|
||||
jsonMalformedTest(
|
||||
"unterminated-4-end", "@\uFFFD", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x22]);
|
||||
jsonMalformedTest("unterminated-4-end", "@\uFFFD\uFFFD\uFFFD",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x80, 0x22]);
|
||||
|
||||
// Unexpected continuation byte
|
||||
// - after a normal character.
|
||||
@@ -372,13 +372,13 @@ void testMalformed() {
|
||||
"leading-2", "@\uFFFD\x80@", [0x22, 0x40, 0xc0, 0xc2, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("leading-3-1", "@\uFFFD\x80@",
|
||||
[0x22, 0x40, 0xe0, 0xc2, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("leading-3-2", "@\uFFFD\x80@",
|
||||
jsonMalformedTest("leading-3-2", "@\uFFFD\uFFFD\x80@",
|
||||
[0x22, 0x40, 0xe0, 0x80, 0xc2, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("leading-4-1", "@\uFFFD\x80@",
|
||||
[0x22, 0x40, 0xf0, 0xc2, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("leading-4-2", "@\uFFFD\x80@",
|
||||
jsonMalformedTest("leading-4-2", "@\uFFFD\uFFFD\x80@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0xc2, 0x80, 0x40, 0x22]);
|
||||
jsonMalformedTest("leading-4-3", "@\uFFFD\x80@",
|
||||
jsonMalformedTest("leading-4-3", "@\uFFFD\uFFFD\uFFFD\x80@",
|
||||
[0x22, 0x40, 0xf0, 0x80, 0x80, 0xc2, 0x80, 0x40, 0x22]);
|
||||
|
||||
// Overlong encodings of ASCII outside of strings always fail.
|
||||
|
||||
@@ -28,11 +28,14 @@ String decodeAllowMalformed4(List<int> bytes) {
|
||||
return new Utf8Codec(allowMalformed: true).decoder.convert(bytes);
|
||||
}
|
||||
|
||||
final TESTS = [
|
||||
final TESTS0 = [
|
||||
// Unfinished UTF-8 sequences.
|
||||
[0xc3],
|
||||
[0xE2, 0x82],
|
||||
[0xF0, 0xA4, 0xAD],
|
||||
[0xF0, 0xA4, 0xAD]
|
||||
];
|
||||
|
||||
final TESTS1 = [
|
||||
// Overlong encoding of euro-sign.
|
||||
[0xF0, 0x82, 0x82, 0xAC],
|
||||
// Other overlong/unfinished sequences.
|
||||
@@ -64,11 +67,11 @@ final TESTS2 = [
|
||||
// Test that 0xC0|1, 0x80 does not eat the next character.
|
||||
[
|
||||
[0xC0, 0x80, 0x61],
|
||||
"Xa"
|
||||
"XXa"
|
||||
],
|
||||
[
|
||||
[0xC1, 0x80, 0x61],
|
||||
"Xa"
|
||||
"XXa"
|
||||
],
|
||||
// 0xF5 .. 0xFF never appear in valid UTF-8 sequences.
|
||||
[
|
||||
@@ -207,44 +210,49 @@ final TESTS2 = [
|
||||
];
|
||||
|
||||
main() {
|
||||
var allTests = TESTS.expand((test) {
|
||||
var allTests = [...TESTS0, ...TESTS1].expand((test) {
|
||||
// Pairs of test and expected string output when malformed strings are
|
||||
// allowed. Replacement character: U+FFFD
|
||||
// allowed. Replacement character: U+FFFD, one per unfinished sequence or
|
||||
// undecodable byte.
|
||||
String replacement =
|
||||
TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length;
|
||||
return [
|
||||
[test, "\u{FFFD}"],
|
||||
[test, "${replacement}"],
|
||||
[
|
||||
new List<int>.from([0x61])..addAll(test),
|
||||
"a\u{FFFD}"
|
||||
[0x61, ...test],
|
||||
"a${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0x61])
|
||||
..addAll(test)
|
||||
..add(0x61),
|
||||
"a\u{FFFD}a"
|
||||
],
|
||||
[new List<int>.from(test)..add(0x61), "\u{FFFD}a"],
|
||||
[new List<int>.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"],
|
||||
[
|
||||
new List<int>.from(test)
|
||||
..add(0x61)
|
||||
..addAll(test),
|
||||
"\u{FFFD}a\u{FFFD}"
|
||||
[0x61, ...test, 0x61],
|
||||
"a${replacement}a"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0xc3, 0xa5])..addAll(test),
|
||||
"å\u{FFFD}"
|
||||
[...test, 0x61],
|
||||
"${replacement}a"
|
||||
],
|
||||
[
|
||||
new List<int>.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]),
|
||||
"å\u{FFFD}å"
|
||||
[...test, ...test],
|
||||
"${replacement}${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from(test)..addAll([0xc3, 0xa5]),
|
||||
"\u{FFFD}å"
|
||||
[...test, 0x61, ...test],
|
||||
"${replacement}a${replacement}"
|
||||
],
|
||||
[
|
||||
new List<int>.from(test)..addAll([0xc3, 0xa5])..addAll(test),
|
||||
"\u{FFFD}å\u{FFFD}"
|
||||
[0xc3, 0xa5, ...test],
|
||||
"å${replacement}"
|
||||
],
|
||||
[
|
||||
[0xc3, 0xa5, ...test, 0xc3, 0xa5],
|
||||
"å${replacement}å"
|
||||
],
|
||||
[
|
||||
[...test, 0xc3, 0xa5],
|
||||
"${replacement}å"
|
||||
],
|
||||
[
|
||||
[...test, 0xc3, 0xa5, ...test],
|
||||
"${replacement}å${replacement}"
|
||||
]
|
||||
];
|
||||
});
|
||||
|
||||
@@ -727,10 +727,14 @@ void testUtf8bytesToCodepoints(List<int> utf8ToRunes(List<int> utf8)) {
|
||||
Expect.listEquals([0xe000], utf8ToRunes([0xee, 0x80, 0x80]), "e000");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xef, 0xbf, 0xbd]), "fffd");
|
||||
Expect
|
||||
.listEquals([0x10ffff], utf8ToRunes([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xf4, 0x90, 0x80, 0x80]), "110000");
|
||||
Expect.listEquals(
|
||||
[0x10ffff], utf8ToRunes([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
], utf8ToRunes([0xf4, 0x90, 0x80, 0x80]), "110000");
|
||||
|
||||
// unexpected continuation bytes
|
||||
Expect.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0x80]),
|
||||
@@ -795,12 +799,15 @@ void testUtf8bytesToCodepoints(List<int> utf8ToRunes(List<int> utf8)) {
|
||||
// Sequences with last continuation byte missing
|
||||
Expect.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xc2]),
|
||||
"2-byte sequence with last byte missing");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xe0, 0x80]), "3-byte sequence with last byte missing");
|
||||
Expect.listEquals(
|
||||
[unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xf0, 0x80, 0x80]),
|
||||
"4-byte sequence with last byte missing");
|
||||
[unicodeReplacementCharacterRune, unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xe0, 0x80]),
|
||||
"3-byte sequence with last byte missing");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
], utf8ToRunes([0xf0, 0x80, 0x80]), "4-byte sequence with last byte missing");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
@@ -871,6 +878,9 @@ void testUtf8bytesToCodepoints(List<int> utf8ToRunes(List<int> utf8)) {
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
],
|
||||
utf8ToRunes([
|
||||
@@ -890,8 +900,8 @@ void testUtf8bytesToCodepoints(List<int> utf8ToRunes(List<int> utf8)) {
|
||||
0x80,
|
||||
0x80,
|
||||
0xdf,
|
||||
0xef,
|
||||
0xbf,
|
||||
0xef, // These two bytes form one incomplete sequence.
|
||||
0xbf, // All others form one per byte.
|
||||
0xf7,
|
||||
0xbf,
|
||||
0xbf,
|
||||
@@ -908,10 +918,10 @@ void testUtf8bytesToCodepoints(List<int> utf8ToRunes(List<int> utf8)) {
|
||||
"Concatenation of incomplete sequences");
|
||||
|
||||
// Impossible bytes
|
||||
Expect
|
||||
.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xfe]), "fe");
|
||||
Expect
|
||||
.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xff]), "ff");
|
||||
Expect.listEquals(
|
||||
[unicodeReplacementCharacterRune], utf8ToRunes([0xfe]), "fe");
|
||||
Expect.listEquals(
|
||||
[unicodeReplacementCharacterRune], utf8ToRunes([0xff]), "ff");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
@@ -921,11 +931,20 @@ void testUtf8bytesToCodepoints(List<int> utf8ToRunes(List<int> utf8)) {
|
||||
|
||||
// Overlong sequences
|
||||
Expect.listEquals(
|
||||
[unicodeReplacementCharacterRune], utf8ToRunes([0xc0, 0xaf]), "c0 af");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xe0, 0x80, 0xaf]), "e0 80 af");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af");
|
||||
[unicodeReplacementCharacterRune, unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xc0, 0xaf]),
|
||||
"c0 af");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
], utf8ToRunes([0xe0, 0x80, 0xaf]), "e0 80 af");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
], utf8ToRunes([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
@@ -943,11 +962,20 @@ void testUtf8bytesToCodepoints(List<int> utf8ToRunes(List<int> utf8)) {
|
||||
], utf8ToRunes([0xfc, 0x80, 0x80, 0x80, 0x80, 0xaf]), "fc 80 80 80 80 af");
|
||||
|
||||
Expect.listEquals(
|
||||
[unicodeReplacementCharacterRune], utf8ToRunes([0xc1, 0xbf]), "c1 bf");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xe0, 0x9f, 0xbf]), "e0 9f bf");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf");
|
||||
[unicodeReplacementCharacterRune, unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xc1, 0xbf]),
|
||||
"c1 bf");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
], utf8ToRunes([0xe0, 0x9f, 0xbf]), "e0 9f bf");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
], utf8ToRunes([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
@@ -965,11 +993,20 @@ void testUtf8bytesToCodepoints(List<int> utf8ToRunes(List<int> utf8)) {
|
||||
], utf8ToRunes([0xfc, 0x83, 0xbf, 0xbf, 0xbf, 0xbf]), "fc 83 bf bf bf bf");
|
||||
|
||||
Expect.listEquals(
|
||||
[unicodeReplacementCharacterRune], utf8ToRunes([0xc0, 0x80]), "c0 80");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xe0, 0x80, 0x80]), "e0 80 80");
|
||||
Expect.listEquals([unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80");
|
||||
[unicodeReplacementCharacterRune, unicodeReplacementCharacterRune],
|
||||
utf8ToRunes([0xc0, 0x80]),
|
||||
"c0 80");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
], utf8ToRunes([0xe0, 0x80, 0x80]), "e0 80 80");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune
|
||||
], utf8ToRunes([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80");
|
||||
Expect.listEquals([
|
||||
unicodeReplacementCharacterRune,
|
||||
unicodeReplacementCharacterRune,
|
||||
|
||||
@@ -9,7 +9,7 @@ import 'dart:convert';
|
||||
|
||||
main() {
|
||||
for (int i = 0; i <= 0x10FFFF; i++) {
|
||||
if (i == unicodeBomCharacterRune) continue;
|
||||
if (i == unicodeBomCharacterRune || (i & 0x1FF800) == 0xD800) continue;
|
||||
Expect.equals(
|
||||
i, utf8.decode(utf8.encode(new String.fromCharCode(i))).runes.first);
|
||||
}
|
||||
|
||||
@@ -25,8 +25,8 @@ void testEncodeSlice() {
|
||||
String ascii = "ABCDE";
|
||||
Expect.listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii));
|
||||
Expect.listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0));
|
||||
Expect
|
||||
.listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0, 5));
|
||||
Expect.listEquals(
|
||||
[0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0, 5));
|
||||
Expect.listEquals([0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 1));
|
||||
Expect.listEquals([0x41, 0x42, 0x43, 0x44], encoder.convert(ascii, 0, 4));
|
||||
Expect.listEquals([0x42, 0x43, 0x44], encoder.convert(ascii, 1, 4));
|
||||
@@ -50,6 +50,6 @@ void testEncodeSlice() {
|
||||
Expect.listEquals(
|
||||
[0xc2, 0x82, 0xe1, 0x81, 0x81], encoder.convert(unicode, 1, 3));
|
||||
// Split in the middle of a surrogate pair.
|
||||
Expect.listEquals([0xc2, 0x82, 0xe1, 0x81, 0x81, 0xed, 0xa0, 0x80],
|
||||
Expect.listEquals([0xc2, 0x82, 0xe1, 0x81, 0x81, 0xef, 0xbf, 0xbd],
|
||||
encoder.convert(unicode, 1, 4));
|
||||
}
|
||||
|
||||
@@ -73,28 +73,29 @@ void testErrorOffset() {
|
||||
|
||||
// Bad encoding, points to first bad byte.
|
||||
testExn([0x80, 0x00], 0);
|
||||
testExn([0xC0, 0x00], 1);
|
||||
testExn([0xE0, 0x00], 1);
|
||||
testExn([0xE0, 0x80, 0x00], 2);
|
||||
testExn([0xF0, 0x00], 1);
|
||||
testExn([0xF0, 0x80, 0x00], 2);
|
||||
testExn([0xF0, 0x80, 0x80, 0x00], 3);
|
||||
testExn([0xC2, 0x00], 1);
|
||||
testExn([0xE2, 0x00], 1);
|
||||
testExn([0xE2, 0x80, 0x00], 2);
|
||||
testExn([0xF2, 0x00], 1);
|
||||
testExn([0xF2, 0x80, 0x00], 2);
|
||||
testExn([0xF2, 0x80, 0x80, 0x00], 3);
|
||||
testExn([0xF8, 0x00], 0);
|
||||
// Short encoding, points to end.
|
||||
testExn([0xC0], 1);
|
||||
testExn([0xE0], 1);
|
||||
testExn([0xE0, 0x80], 2);
|
||||
testExn([0xF0], 1);
|
||||
testExn([0xF0, 0x80], 2);
|
||||
testExn([0xF0, 0x80, 0x80], 3);
|
||||
// Overlong encoding, points to start of encoding.
|
||||
testExn([0xC2], 1);
|
||||
testExn([0xE2], 1);
|
||||
testExn([0xE2, 0x80], 2);
|
||||
testExn([0xF2], 1);
|
||||
testExn([0xF2, 0x80], 2);
|
||||
testExn([0xF2, 0x80, 0x80], 3);
|
||||
// Overlong encoding, points to byte that gave enough information to conclude
|
||||
// that it was overlong.
|
||||
testExn([0xC0, 0x80], 0);
|
||||
testExn([0xC1, 0xBF], 0);
|
||||
testExn([0xE0, 0x80, 0x80], 0);
|
||||
testExn([0xE0, 0x9F, 0xBF], 0);
|
||||
testExn([0xF0, 0x80, 0x80, 0x80], 0);
|
||||
testExn([0xF0, 0x8F, 0xBF, 0xBF], 0);
|
||||
testExn([0xE0, 0x80, 0x80], 1);
|
||||
testExn([0xE0, 0x9F, 0xBF], 1);
|
||||
testExn([0xF0, 0x80, 0x80, 0x80], 1);
|
||||
testExn([0xF0, 0x8F, 0xBF, 0xBF], 1);
|
||||
// Invalid character (value too large, over 0x10FFFF).
|
||||
testExn([0xF4, 0x90, 0x80, 0x80], 0);
|
||||
testExn([0xF4, 0x90, 0x80, 0x80], 1);
|
||||
testExn([0xF7, 0xBF, 0xBF, 0xBF], 0);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user