From f5bf50e7a4ec7d047669c708570aee071aa4cbbd Mon Sep 17 00:00:00 2001 From: Aske Simon Christensen Date: Mon, 4 May 2020 10:48:32 +0000 Subject: [PATCH] Adjust UTF-8 tests to match WHATWG standard expectations. This adjusts all UTF-8 tests to the new semantics in the breaking change described here: https://github.com/dart-lang/sdk/issues/41100 This has three parts: - Unpaired surrogates are encoded as replacement characters, and encoded surrogates are considered malformed input when decoding. - Decoding errors are generally reported on the position of the byte that conclusively makes the input malformed. - The number of replacement characters emitted by the decoder is generally one per unfinished sequence or undecodable byte. The code changes to implement the new semantics are placed in subsequent commits. Change-Id: I4cc8ce660e39287e734070764ab8e1f0ebb8b9e0 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/143815 Reviewed-by: Lasse R.H. Nielsen --- .../chunked_conversion_utf82_test.dart | 66 +++++++------ .../chunked_conversion_utf86_test.dart | 66 +++++++------ .../chunked_conversion_utf87_test.dart | 66 +++++++------ .../chunked_conversion_utf88_test.dart | 87 ++++++++--------- tests/lib/convert/json_utf8_chunk_test.dart | 56 +++++------ tests/lib/convert/utf82_test.dart | 66 +++++++------ tests/lib/convert/utf84_test.dart | 97 +++++++++++++------ tests/lib/convert/utf85_test.dart | 2 +- tests/lib/convert/utf8_encode_test.dart | 6 +- tests/lib/convert/utf8_test.dart | 37 +++---- .../chunked_conversion_utf82_test.dart | 66 +++++++------ .../chunked_conversion_utf86_test.dart | 66 +++++++------ .../chunked_conversion_utf87_test.dart | 66 +++++++------ .../chunked_conversion_utf88_test.dart | 87 ++++++++--------- tests/lib_2/convert/json_utf8_chunk_test.dart | 56 +++++------ tests/lib_2/convert/utf82_test.dart | 66 +++++++------ tests/lib_2/convert/utf84_test.dart | 97 +++++++++++++------ tests/lib_2/convert/utf85_test.dart | 2 +- tests/lib_2/convert/utf8_encode_test.dart | 6 +- tests/lib_2/convert/utf8_test.dart | 37 +++---- 20 files changed, 618 insertions(+), 480 deletions(-) diff --git a/tests/lib/convert/chunked_conversion_utf82_test.dart b/tests/lib/convert/chunked_conversion_utf82_test.dart index bc0b200b0c2..2e07f87f447 100644 --- a/tests/lib/convert/chunked_conversion_utf82_test.dart +++ b/tests/lib/convert/chunked_conversion_utf82_test.dart @@ -46,11 +46,14 @@ String decodeAllowMalformed(List bytes, int chunkSize) { return buffer.toString(); } -final TESTS = [ +final TESTS0 = [ // Unfinished UTF-8 sequences. [0xc3], [0xE2, 0x82], - [0xF0, 0xA4, 0xAD], + [0xF0, 0xA4, 0xAD] +]; + +final TESTS1 = [ // Overlong encoding of euro-sign. [0xF0, 0x82, 0x82, 0xAC], // Other overlong/unfinished sequences. @@ -77,11 +80,11 @@ final TESTS2 = [ // Test that 0xC0|1, 0x80 does not eat the next character. [ [0xC0, 0x80, 0x61], - "Xa" + "XXa" ], [ [0xC1, 0x80, 0x61], - "Xa" + "XXa" ], // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. [ @@ -220,44 +223,49 @@ final TESTS2 = [ ]; main() { - var allTests = TESTS.expand((test) { + var allTests = [...TESTS0, ...TESTS1].expand((test) { // Pairs of test and expected string output when malformed strings are - // allowed. Replacement character: U+FFFD + // allowed. Replacement character: U+FFFD, one per unfinished sequence or + // undecodable byte. + String replacement = + TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length; return [ - [test, "\u{FFFD}"], + [test, "${replacement}"], [ - new List.from([0x61])..addAll(test), - "a\u{FFFD}" + [0x61, ...test], + "a${replacement}" ], [ - new List.from([0x61]) - ..addAll(test) - ..add(0x61), - "a\u{FFFD}a" - ], - [new List.from(test)..add(0x61), "\u{FFFD}a"], - [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], - [ - new List.from(test) - ..add(0x61) - ..addAll(test), - "\u{FFFD}a\u{FFFD}" + [0x61, ...test, 0x61], + "a${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test), - "å\u{FFFD}" + [...test, 0x61], + "${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), - "å\u{FFFD}å" + [...test, ...test], + "${replacement}${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5]), - "\u{FFFD}å" + [...test, 0x61, ...test], + "${replacement}a${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), - "\u{FFFD}å\u{FFFD}" + [0xc3, 0xa5, ...test], + "å${replacement}" + ], + [ + [0xc3, 0xa5, ...test, 0xc3, 0xa5], + "å${replacement}å" + ], + [ + [...test, 0xc3, 0xa5], + "${replacement}å" + ], + [ + [...test, 0xc3, 0xa5, ...test], + "${replacement}å${replacement}" ] ]; }); diff --git a/tests/lib/convert/chunked_conversion_utf86_test.dart b/tests/lib/convert/chunked_conversion_utf86_test.dart index a42e7523334..8c9a08c39fd 100644 --- a/tests/lib/convert/chunked_conversion_utf86_test.dart +++ b/tests/lib/convert/chunked_conversion_utf86_test.dart @@ -28,11 +28,14 @@ String decodeAllowMalformed(List bytes) { return buffer.toString(); } -final TESTS = [ +final TESTS0 = [ // Unfinished UTF-8 sequences. [0xc3], [0xE2, 0x82], - [0xF0, 0xA4, 0xAD], + [0xF0, 0xA4, 0xAD] +]; + +final TESTS1 = [ // Overlong encoding of euro-sign. [0xF0, 0x82, 0x82, 0xAC], // Other overlong/unfinished sequences. @@ -59,11 +62,11 @@ final TESTS2 = [ // Test that 0xC0|1, 0x80 does not eat the next character. [ [0xC0, 0x80, 0x61], - "Xa" + "XXa" ], [ [0xC1, 0x80, 0x61], - "Xa" + "XXa" ], // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. [ @@ -202,44 +205,49 @@ final TESTS2 = [ ]; main() { - var allTests = TESTS.expand((test) { + var allTests = [...TESTS0, ...TESTS1].expand((test) { // Pairs of test and expected string output when malformed strings are - // allowed. Replacement character: U+FFFD + // allowed. Replacement character: U+FFFD, one per unfinished sequence or + // undecodable byte. + String replacement = + TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length; return [ - [test, "\u{FFFD}"], + [test, "${replacement}"], [ - new List.from([0x61])..addAll(test), - "a\u{FFFD}" + [0x61, ...test], + "a${replacement}" ], [ - new List.from([0x61]) - ..addAll(test) - ..add(0x61), - "a\u{FFFD}a" - ], - [new List.from(test)..add(0x61), "\u{FFFD}a"], - [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], - [ - new List.from(test) - ..add(0x61) - ..addAll(test), - "\u{FFFD}a\u{FFFD}" + [0x61, ...test, 0x61], + "a${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test), - "å\u{FFFD}" + [...test, 0x61], + "${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), - "å\u{FFFD}å" + [...test, ...test], + "${replacement}${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5]), - "\u{FFFD}å" + [...test, 0x61, ...test], + "${replacement}a${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), - "\u{FFFD}å\u{FFFD}" + [0xc3, 0xa5, ...test], + "å${replacement}" + ], + [ + [0xc3, 0xa5, ...test, 0xc3, 0xa5], + "å${replacement}å" + ], + [ + [...test, 0xc3, 0xa5], + "${replacement}å" + ], + [ + [...test, 0xc3, 0xa5, ...test], + "${replacement}å${replacement}" ] ]; }); diff --git a/tests/lib/convert/chunked_conversion_utf87_test.dart b/tests/lib/convert/chunked_conversion_utf87_test.dart index ab42ee23b89..5434f1f1f43 100644 --- a/tests/lib/convert/chunked_conversion_utf87_test.dart +++ b/tests/lib/convert/chunked_conversion_utf87_test.dart @@ -49,11 +49,14 @@ String decodeAllowMalformed2(List inputBytes) { return utf8.decode(bytes); } -final TESTS = [ +final TESTS0 = [ // Unfinished UTF-8 sequences. [0xc3], [0xE2, 0x82], - [0xF0, 0xA4, 0xAD], + [0xF0, 0xA4, 0xAD] +]; + +final TESTS1 = [ // Overlong encoding of euro-sign. [0xF0, 0x82, 0x82, 0xAC], // Other overlong/unfinished sequences. @@ -80,11 +83,11 @@ final TESTS2 = [ // Test that 0xC0|1, 0x80 does not eat the next character. [ [0xC0, 0x80, 0x61], - "Xa" + "XXa" ], [ [0xC1, 0x80, 0x61], - "Xa" + "XXa" ], // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. [ @@ -223,44 +226,49 @@ final TESTS2 = [ ]; main() { - var allTests = TESTS.expand((test) { + var allTests = [...TESTS0, ...TESTS1].expand((test) { // Pairs of test and expected string output when malformed strings are - // allowed. Replacement character: U+FFFD + // allowed. Replacement character: U+FFFD, one per unfinished sequence or + // undecodable byte. + String replacement = + TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length; return [ - [test, "\u{FFFD}"], + [test, "${replacement}"], [ - new List.from([0x61])..addAll(test), - "a\u{FFFD}" + [0x61, ...test], + "a${replacement}" ], [ - new List.from([0x61]) - ..addAll(test) - ..add(0x61), - "a\u{FFFD}a" - ], - [new List.from(test)..add(0x61), "\u{FFFD}a"], - [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], - [ - new List.from(test) - ..add(0x61) - ..addAll(test), - "\u{FFFD}a\u{FFFD}" + [0x61, ...test, 0x61], + "a${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test), - "å\u{FFFD}" + [...test, 0x61], + "${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), - "å\u{FFFD}å" + [...test, ...test], + "${replacement}${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5]), - "\u{FFFD}å" + [...test, 0x61, ...test], + "${replacement}a${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), - "\u{FFFD}å\u{FFFD}" + [0xc3, 0xa5, ...test], + "å${replacement}" + ], + [ + [0xc3, 0xa5, ...test, 0xc3, 0xa5], + "å${replacement}å" + ], + [ + [...test, 0xc3, 0xa5], + "${replacement}å" + ], + [ + [...test, 0xc3, 0xa5, ...test], + "${replacement}å${replacement}" ] ]; }); diff --git a/tests/lib/convert/chunked_conversion_utf88_test.dart b/tests/lib/convert/chunked_conversion_utf88_test.dart index c0a63d734e5..93cab53b646 100644 --- a/tests/lib/convert/chunked_conversion_utf88_test.dart +++ b/tests/lib/convert/chunked_conversion_utf88_test.dart @@ -112,8 +112,7 @@ main() { const LEADING_SURROGATE = 0xd801; const TRAILING_SURROGATE = 0xdc12; const UTF8_ENCODING = const [0xf0, 0x90, 0x90, 0x92]; - const UTF8_LEADING = const [0xed, 0xa0, 0x81]; - const UTF8_TRAILING = const [0xed, 0xb0, 0x92]; + const UTF8_REPLACEMENT = const [0xef, 0xbf, 0xbd]; const CHAR_A = 0x61; // Test surrogates at all kinds of locations. @@ -129,17 +128,17 @@ main() { codeUnits[i] = LEADING_SURROGATE; var str = new String.fromCharCodes(codeUnits); var bytes = new List.filled(i + 3, CHAR_A); - bytes[i] = UTF8_LEADING[0]; - bytes[i + 1] = UTF8_LEADING[1]; - bytes[i + 2] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = TRAILING_SURROGATE; str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 3, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits.add(CHAR_A); @@ -158,36 +157,36 @@ main() { codeUnits[i + 1] = TRAILING_SURROGATE; str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 6, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; - bytes[i + 3] = UTF8_TRAILING[0]; - bytes[i + 4] = UTF8_TRAILING[1]; - bytes[i + 5] = UTF8_TRAILING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = LEADING_SURROGATE; codeUnits[i + 1] = LEADING_SURROGATE; str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 6, CHAR_A); - bytes[i] = UTF8_LEADING[0]; - bytes[i + 1] = UTF8_LEADING[1]; - bytes[i + 2] = UTF8_LEADING[2]; - bytes[i + 3] = UTF8_LEADING[0]; - bytes[i + 4] = UTF8_LEADING[1]; - bytes[i + 5] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = TRAILING_SURROGATE; codeUnits[i + 1] = LEADING_SURROGATE; str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 6, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; - bytes[i + 3] = UTF8_LEADING[0]; - bytes[i + 4] = UTF8_LEADING[1]; - bytes[i + 5] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits.add(CHAR_A); @@ -210,12 +209,12 @@ main() { codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 7, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; - bytes[i + 3] = UTF8_TRAILING[0]; - bytes[i + 4] = UTF8_TRAILING[1]; - bytes[i + 5] = UTF8_TRAILING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = LEADING_SURROGATE; @@ -223,12 +222,12 @@ main() { codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 7, CHAR_A); - bytes[i] = UTF8_LEADING[0]; - bytes[i + 1] = UTF8_LEADING[1]; - bytes[i + 2] = UTF8_LEADING[2]; - bytes[i + 3] = UTF8_LEADING[0]; - bytes[i + 4] = UTF8_LEADING[1]; - bytes[i + 5] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = TRAILING_SURROGATE; @@ -236,12 +235,12 @@ main() { codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 7, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; - bytes[i + 3] = UTF8_LEADING[0]; - bytes[i + 4] = UTF8_LEADING[1]; - bytes[i + 5] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); // Make sure the invariant is correct. diff --git a/tests/lib/convert/json_utf8_chunk_test.dart b/tests/lib/convert/json_utf8_chunk_test.dart index c18f330bc93..e75019d3e99 100644 --- a/tests/lib/convert/json_utf8_chunk_test.dart +++ b/tests/lib/convert/json_utf8_chunk_test.dart @@ -284,69 +284,69 @@ void jsonThrows(String name, String codeString) { void testMalformed() { // Overlong encodings. jsonMalformedTest( - "overlong-0-2", "@\uFFFD@", [0x22, 0x40, 0xc0, 0x80, 0x40, 0x22]); - jsonMalformedTest( - "overlong-0-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x80, 0x80, 0x40, 0x22]); - jsonMalformedTest("overlong-0-4", "@\uFFFD@", + "overlong-0-2", "@\uFFFD\uFFFD@", [0x22, 0x40, 0xc0, 0x80, 0x40, 0x22]); + jsonMalformedTest("overlong-0-3", "@\uFFFD\uFFFD\uFFFD@", + [0x22, 0x40, 0xe0, 0x80, 0x80, 0x40, 0x22]); + jsonMalformedTest("overlong-0-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x80, 0x40, 0x22]); jsonMalformedTest( - "overlong-7f-2", "@\uFFFD@", [0x22, 0x40, 0xc1, 0xbf, 0x40, 0x22]); - jsonMalformedTest( - "overlong-7f-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x81, 0xbf, 0x40, 0x22]); - jsonMalformedTest("overlong-7f-4", "@\uFFFD@", + "overlong-7f-2", "@\uFFFD\uFFFD@", [0x22, 0x40, 0xc1, 0xbf, 0x40, 0x22]); + jsonMalformedTest("overlong-7f-3", "@\uFFFD\uFFFD\uFFFD@", + [0x22, 0x40, 0xe0, 0x81, 0xbf, 0x40, 0x22]); + jsonMalformedTest("overlong-7f-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x81, 0xbf, 0x40, 0x22]); - jsonMalformedTest( - "overlong-80-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x82, 0x80, 0x40, 0x22]); - jsonMalformedTest("overlong-80-4", "@\uFFFD@", + jsonMalformedTest("overlong-80-3", "@\uFFFD\uFFFD\uFFFD@", + [0x22, 0x40, 0xe0, 0x82, 0x80, 0x40, 0x22]); + jsonMalformedTest("overlong-80-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x82, 0x80, 0x40, 0x22]); - jsonMalformedTest( - "overlong-7ff-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x9f, 0xbf, 0x40, 0x22]); - jsonMalformedTest("overlong-7ff-4", "@\uFFFD@", + jsonMalformedTest("overlong-7ff-3", "@\uFFFD\uFFFD\uFFFD@", + [0x22, 0x40, 0xe0, 0x9f, 0xbf, 0x40, 0x22]); + jsonMalformedTest("overlong-7ff-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x9f, 0xbf, 0x40, 0x22]); - jsonMalformedTest("overlong-800-4", "@\uFFFD@", + jsonMalformedTest("overlong-800-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0xa0, 0x80, 0x40, 0x22]); - jsonMalformedTest("overlong-ffff-4", "@\uFFFD@", + jsonMalformedTest("overlong-ffff-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x8f, 0xbf, 0xbf, 0x40, 0x22]); // Unterminated multibyte sequences. jsonMalformedTest( "unterminated-2-normal", "@\uFFFD@", [0x22, 0x40, 0xc0, 0x40, 0x22]); - jsonMalformedTest("unterminated-3-normal", "@\uFFFD@", + jsonMalformedTest("unterminated-3-normal", "@\uFFFD\uFFFD@", [0x22, 0x40, 0xe0, 0x80, 0x40, 0x22]); - jsonMalformedTest("unterminated-4-normal", "@\uFFFD@", + jsonMalformedTest("unterminated-4-normal", "@\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x40, 0x22]); jsonMalformedTest("unterminated-2-multi", "@\uFFFD\x80@", [0x22, 0x40, 0xc0, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("unterminated-3-multi", "@\uFFFD\x80@", + jsonMalformedTest("unterminated-3-multi", "@\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xe0, 0x80, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("unterminated-4-multi", "@\uFFFD\x80@", + jsonMalformedTest("unterminated-4-multi", "@\uFFFD\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0xc2, 0x80, 0x40, 0x22]); jsonMalformedTest("unterminated-2-escape", "@\uFFFD\n@", [0x22, 0x40, 0xc0, 0x5c, 0x6e, 0x40, 0x22]); - jsonMalformedTest("unterminated-3-escape", "@\uFFFD\n@", + jsonMalformedTest("unterminated-3-escape", "@\uFFFD\uFFFD\n@", [0x22, 0x40, 0xe0, 0x80, 0x5c, 0x6e, 0x40, 0x22]); - jsonMalformedTest("unterminated-4-escape", "@\uFFFD\n@", + jsonMalformedTest("unterminated-4-escape", "@\uFFFD\uFFFD\uFFFD\n@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x5c, 0x6e, 0x40, 0x22]); jsonMalformedTest("unterminated-2-end", "@\uFFFD", [0x22, 0x40, 0xc0, 0x22]); jsonMalformedTest( - "unterminated-3-end", "@\uFFFD", [0x22, 0x40, 0xe0, 0x80, 0x22]); + "unterminated-3-end", "@\uFFFD\uFFFD", [0x22, 0x40, 0xe0, 0x80, 0x22]); - jsonMalformedTest( - "unterminated-4-end", "@\uFFFD", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x22]); + jsonMalformedTest("unterminated-4-end", "@\uFFFD\uFFFD\uFFFD", + [0x22, 0x40, 0xf0, 0x80, 0x80, 0x22]); // Unexpected continuation byte // - after a normal character. @@ -372,13 +372,13 @@ void testMalformed() { "leading-2", "@\uFFFD\x80@", [0x22, 0x40, 0xc0, 0xc2, 0x80, 0x40, 0x22]); jsonMalformedTest("leading-3-1", "@\uFFFD\x80@", [0x22, 0x40, 0xe0, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("leading-3-2", "@\uFFFD\x80@", + jsonMalformedTest("leading-3-2", "@\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xe0, 0x80, 0xc2, 0x80, 0x40, 0x22]); jsonMalformedTest("leading-4-1", "@\uFFFD\x80@", [0x22, 0x40, 0xf0, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("leading-4-2", "@\uFFFD\x80@", + jsonMalformedTest("leading-4-2", "@\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xf0, 0x80, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("leading-4-3", "@\uFFFD\x80@", + jsonMalformedTest("leading-4-3", "@\uFFFD\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0xc2, 0x80, 0x40, 0x22]); // Overlong encodings of ASCII outside of strings always fail. diff --git a/tests/lib/convert/utf82_test.dart b/tests/lib/convert/utf82_test.dart index df08103e86d..cf0c2b0585e 100644 --- a/tests/lib/convert/utf82_test.dart +++ b/tests/lib/convert/utf82_test.dart @@ -28,11 +28,14 @@ String decodeAllowMalformed4(List bytes) { return new Utf8Codec(allowMalformed: true).decoder.convert(bytes); } -final TESTS = [ +final TESTS0 = [ // Unfinished UTF-8 sequences. [0xc3], [0xE2, 0x82], - [0xF0, 0xA4, 0xAD], + [0xF0, 0xA4, 0xAD] +]; + +final TESTS1 = [ // Overlong encoding of euro-sign. [0xF0, 0x82, 0x82, 0xAC], // Other overlong/unfinished sequences. @@ -64,11 +67,11 @@ final TESTS2 = [ // Test that 0xC0|1, 0x80 does not eat the next character. [ [0xC0, 0x80, 0x61], - "Xa" + "XXa" ], [ [0xC1, 0x80, 0x61], - "Xa" + "XXa" ], // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. [ @@ -207,44 +210,49 @@ final TESTS2 = [ ]; main() { - var allTests = TESTS.expand((test) { + var allTests = [...TESTS0, ...TESTS1].expand((test) { // Pairs of test and expected string output when malformed strings are - // allowed. Replacement character: U+FFFD + // allowed. Replacement character: U+FFFD, one per unfinished sequence or + // undecodable byte. + String replacement = + TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length; return [ - [test, "\u{FFFD}"], + [test, "${replacement}"], [ - new List.from([0x61])..addAll(test), - "a\u{FFFD}" + [0x61, ...test], + "a${replacement}" ], [ - new List.from([0x61]) - ..addAll(test) - ..add(0x61), - "a\u{FFFD}a" - ], - [new List.from(test)..add(0x61), "\u{FFFD}a"], - [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], - [ - new List.from(test) - ..add(0x61) - ..addAll(test), - "\u{FFFD}a\u{FFFD}" + [0x61, ...test, 0x61], + "a${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test), - "å\u{FFFD}" + [...test, 0x61], + "${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), - "å\u{FFFD}å" + [...test, ...test], + "${replacement}${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5]), - "\u{FFFD}å" + [...test, 0x61, ...test], + "${replacement}a${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), - "\u{FFFD}å\u{FFFD}" + [0xc3, 0xa5, ...test], + "å${replacement}" + ], + [ + [0xc3, 0xa5, ...test, 0xc3, 0xa5], + "å${replacement}å" + ], + [ + [...test, 0xc3, 0xa5], + "${replacement}å" + ], + [ + [...test, 0xc3, 0xa5, ...test], + "${replacement}å${replacement}" ] ]; }); diff --git a/tests/lib/convert/utf84_test.dart b/tests/lib/convert/utf84_test.dart index bc1db59a004..566b1fa54bd 100755 --- a/tests/lib/convert/utf84_test.dart +++ b/tests/lib/convert/utf84_test.dart @@ -727,10 +727,14 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { Expect.listEquals([0xe000], utf8ToRunes([0xee, 0x80, 0x80]), "e000"); Expect.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xef, 0xbf, 0xbd]), "fffd"); - Expect - .listEquals([0x10ffff], utf8ToRunes([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xf4, 0x90, 0x80, 0x80]), "110000"); + Expect.listEquals( + [0x10ffff], utf8ToRunes([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf4, 0x90, 0x80, 0x80]), "110000"); // unexpected continuation bytes Expect.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0x80]), @@ -795,12 +799,15 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { // Sequences with last continuation byte missing Expect.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xc2]), "2-byte sequence with last byte missing"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xe0, 0x80]), "3-byte sequence with last byte missing"); Expect.listEquals( - [unicodeReplacementCharacterRune], - utf8ToRunes([0xf0, 0x80, 0x80]), - "4-byte sequence with last byte missing"); + [unicodeReplacementCharacterRune, unicodeReplacementCharacterRune], + utf8ToRunes([0xe0, 0x80]), + "3-byte sequence with last byte missing"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf0, 0x80, 0x80]), "4-byte sequence with last byte missing"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, @@ -871,6 +878,9 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, unicodeReplacementCharacterRune ], utf8ToRunes([ @@ -890,8 +900,8 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { 0x80, 0x80, 0xdf, - 0xef, - 0xbf, + 0xef, // These two bytes form one incomplete sequence. + 0xbf, // All others form one per byte. 0xf7, 0xbf, 0xbf, @@ -908,10 +918,10 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { "Concatenation of incomplete sequences"); // Impossible bytes - Expect - .listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xfe]), "fe"); - Expect - .listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xff]), "ff"); + Expect.listEquals( + [unicodeReplacementCharacterRune], utf8ToRunes([0xfe]), "fe"); + Expect.listEquals( + [unicodeReplacementCharacterRune], utf8ToRunes([0xff]), "ff"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, @@ -921,11 +931,20 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { // Overlong sequences Expect.listEquals( - [unicodeReplacementCharacterRune], utf8ToRunes([0xc0, 0xaf]), "c0 af"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xe0, 0x80, 0xaf]), "e0 80 af"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af"); + [unicodeReplacementCharacterRune, unicodeReplacementCharacterRune], + utf8ToRunes([0xc0, 0xaf]), + "c0 af"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xe0, 0x80, 0xaf]), "e0 80 af"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, @@ -943,11 +962,20 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { ], utf8ToRunes([0xfc, 0x80, 0x80, 0x80, 0x80, 0xaf]), "fc 80 80 80 80 af"); Expect.listEquals( - [unicodeReplacementCharacterRune], utf8ToRunes([0xc1, 0xbf]), "c1 bf"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xe0, 0x9f, 0xbf]), "e0 9f bf"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf"); + [unicodeReplacementCharacterRune, unicodeReplacementCharacterRune], + utf8ToRunes([0xc1, 0xbf]), + "c1 bf"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xe0, 0x9f, 0xbf]), "e0 9f bf"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, @@ -965,11 +993,20 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { ], utf8ToRunes([0xfc, 0x83, 0xbf, 0xbf, 0xbf, 0xbf]), "fc 83 bf bf bf bf"); Expect.listEquals( - [unicodeReplacementCharacterRune], utf8ToRunes([0xc0, 0x80]), "c0 80"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xe0, 0x80, 0x80]), "e0 80 80"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80"); + [unicodeReplacementCharacterRune, unicodeReplacementCharacterRune], + utf8ToRunes([0xc0, 0x80]), + "c0 80"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xe0, 0x80, 0x80]), "e0 80 80"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, diff --git a/tests/lib/convert/utf85_test.dart b/tests/lib/convert/utf85_test.dart index 3f287e94762..a3809918a8f 100644 --- a/tests/lib/convert/utf85_test.dart +++ b/tests/lib/convert/utf85_test.dart @@ -9,7 +9,7 @@ import 'dart:convert'; main() { for (int i = 0; i <= 0x10FFFF; i++) { - if (i == unicodeBomCharacterRune) continue; + if (i == unicodeBomCharacterRune || (i & 0x1FF800) == 0xD800) continue; Expect.equals( i, utf8.decode(utf8.encode(new String.fromCharCode(i))).runes.first); } diff --git a/tests/lib/convert/utf8_encode_test.dart b/tests/lib/convert/utf8_encode_test.dart index 1e611bb575b..0a43e1452f1 100644 --- a/tests/lib/convert/utf8_encode_test.dart +++ b/tests/lib/convert/utf8_encode_test.dart @@ -25,8 +25,8 @@ void testEncodeSlice() { String ascii = "ABCDE"; Expect.listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii)); Expect.listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0)); - Expect - .listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0, 5)); + Expect.listEquals( + [0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0, 5)); Expect.listEquals([0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 1)); Expect.listEquals([0x41, 0x42, 0x43, 0x44], encoder.convert(ascii, 0, 4)); Expect.listEquals([0x42, 0x43, 0x44], encoder.convert(ascii, 1, 4)); @@ -50,6 +50,6 @@ void testEncodeSlice() { Expect.listEquals( [0xc2, 0x82, 0xe1, 0x81, 0x81], encoder.convert(unicode, 1, 3)); // Split in the middle of a surrogate pair. - Expect.listEquals([0xc2, 0x82, 0xe1, 0x81, 0x81, 0xed, 0xa0, 0x80], + Expect.listEquals([0xc2, 0x82, 0xe1, 0x81, 0x81, 0xef, 0xbf, 0xbd], encoder.convert(unicode, 1, 4)); } diff --git a/tests/lib/convert/utf8_test.dart b/tests/lib/convert/utf8_test.dart index 858c982ef22..6f97fd06770 100644 --- a/tests/lib/convert/utf8_test.dart +++ b/tests/lib/convert/utf8_test.dart @@ -73,28 +73,29 @@ void testErrorOffset() { // Bad encoding, points to first bad byte. testExn([0x80, 0x00], 0); - testExn([0xC0, 0x00], 1); - testExn([0xE0, 0x00], 1); - testExn([0xE0, 0x80, 0x00], 2); - testExn([0xF0, 0x00], 1); - testExn([0xF0, 0x80, 0x00], 2); - testExn([0xF0, 0x80, 0x80, 0x00], 3); + testExn([0xC2, 0x00], 1); + testExn([0xE2, 0x00], 1); + testExn([0xE2, 0x80, 0x00], 2); + testExn([0xF2, 0x00], 1); + testExn([0xF2, 0x80, 0x00], 2); + testExn([0xF2, 0x80, 0x80, 0x00], 3); testExn([0xF8, 0x00], 0); // Short encoding, points to end. - testExn([0xC0], 1); - testExn([0xE0], 1); - testExn([0xE0, 0x80], 2); - testExn([0xF0], 1); - testExn([0xF0, 0x80], 2); - testExn([0xF0, 0x80, 0x80], 3); - // Overlong encoding, points to start of encoding. + testExn([0xC2], 1); + testExn([0xE2], 1); + testExn([0xE2, 0x80], 2); + testExn([0xF2], 1); + testExn([0xF2, 0x80], 2); + testExn([0xF2, 0x80, 0x80], 3); + // Overlong encoding, points to byte that gave enough information to conclude + // that it was overlong. testExn([0xC0, 0x80], 0); testExn([0xC1, 0xBF], 0); - testExn([0xE0, 0x80, 0x80], 0); - testExn([0xE0, 0x9F, 0xBF], 0); - testExn([0xF0, 0x80, 0x80, 0x80], 0); - testExn([0xF0, 0x8F, 0xBF, 0xBF], 0); + testExn([0xE0, 0x80, 0x80], 1); + testExn([0xE0, 0x9F, 0xBF], 1); + testExn([0xF0, 0x80, 0x80, 0x80], 1); + testExn([0xF0, 0x8F, 0xBF, 0xBF], 1); // Invalid character (value too large, over 0x10FFFF). - testExn([0xF4, 0x90, 0x80, 0x80], 0); + testExn([0xF4, 0x90, 0x80, 0x80], 1); testExn([0xF7, 0xBF, 0xBF, 0xBF], 0); } diff --git a/tests/lib_2/convert/chunked_conversion_utf82_test.dart b/tests/lib_2/convert/chunked_conversion_utf82_test.dart index bc0b200b0c2..2e07f87f447 100644 --- a/tests/lib_2/convert/chunked_conversion_utf82_test.dart +++ b/tests/lib_2/convert/chunked_conversion_utf82_test.dart @@ -46,11 +46,14 @@ String decodeAllowMalformed(List bytes, int chunkSize) { return buffer.toString(); } -final TESTS = [ +final TESTS0 = [ // Unfinished UTF-8 sequences. [0xc3], [0xE2, 0x82], - [0xF0, 0xA4, 0xAD], + [0xF0, 0xA4, 0xAD] +]; + +final TESTS1 = [ // Overlong encoding of euro-sign. [0xF0, 0x82, 0x82, 0xAC], // Other overlong/unfinished sequences. @@ -77,11 +80,11 @@ final TESTS2 = [ // Test that 0xC0|1, 0x80 does not eat the next character. [ [0xC0, 0x80, 0x61], - "Xa" + "XXa" ], [ [0xC1, 0x80, 0x61], - "Xa" + "XXa" ], // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. [ @@ -220,44 +223,49 @@ final TESTS2 = [ ]; main() { - var allTests = TESTS.expand((test) { + var allTests = [...TESTS0, ...TESTS1].expand((test) { // Pairs of test and expected string output when malformed strings are - // allowed. Replacement character: U+FFFD + // allowed. Replacement character: U+FFFD, one per unfinished sequence or + // undecodable byte. + String replacement = + TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length; return [ - [test, "\u{FFFD}"], + [test, "${replacement}"], [ - new List.from([0x61])..addAll(test), - "a\u{FFFD}" + [0x61, ...test], + "a${replacement}" ], [ - new List.from([0x61]) - ..addAll(test) - ..add(0x61), - "a\u{FFFD}a" - ], - [new List.from(test)..add(0x61), "\u{FFFD}a"], - [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], - [ - new List.from(test) - ..add(0x61) - ..addAll(test), - "\u{FFFD}a\u{FFFD}" + [0x61, ...test, 0x61], + "a${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test), - "å\u{FFFD}" + [...test, 0x61], + "${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), - "å\u{FFFD}å" + [...test, ...test], + "${replacement}${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5]), - "\u{FFFD}å" + [...test, 0x61, ...test], + "${replacement}a${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), - "\u{FFFD}å\u{FFFD}" + [0xc3, 0xa5, ...test], + "å${replacement}" + ], + [ + [0xc3, 0xa5, ...test, 0xc3, 0xa5], + "å${replacement}å" + ], + [ + [...test, 0xc3, 0xa5], + "${replacement}å" + ], + [ + [...test, 0xc3, 0xa5, ...test], + "${replacement}å${replacement}" ] ]; }); diff --git a/tests/lib_2/convert/chunked_conversion_utf86_test.dart b/tests/lib_2/convert/chunked_conversion_utf86_test.dart index a42e7523334..8c9a08c39fd 100644 --- a/tests/lib_2/convert/chunked_conversion_utf86_test.dart +++ b/tests/lib_2/convert/chunked_conversion_utf86_test.dart @@ -28,11 +28,14 @@ String decodeAllowMalformed(List bytes) { return buffer.toString(); } -final TESTS = [ +final TESTS0 = [ // Unfinished UTF-8 sequences. [0xc3], [0xE2, 0x82], - [0xF0, 0xA4, 0xAD], + [0xF0, 0xA4, 0xAD] +]; + +final TESTS1 = [ // Overlong encoding of euro-sign. [0xF0, 0x82, 0x82, 0xAC], // Other overlong/unfinished sequences. @@ -59,11 +62,11 @@ final TESTS2 = [ // Test that 0xC0|1, 0x80 does not eat the next character. [ [0xC0, 0x80, 0x61], - "Xa" + "XXa" ], [ [0xC1, 0x80, 0x61], - "Xa" + "XXa" ], // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. [ @@ -202,44 +205,49 @@ final TESTS2 = [ ]; main() { - var allTests = TESTS.expand((test) { + var allTests = [...TESTS0, ...TESTS1].expand((test) { // Pairs of test and expected string output when malformed strings are - // allowed. Replacement character: U+FFFD + // allowed. Replacement character: U+FFFD, one per unfinished sequence or + // undecodable byte. + String replacement = + TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length; return [ - [test, "\u{FFFD}"], + [test, "${replacement}"], [ - new List.from([0x61])..addAll(test), - "a\u{FFFD}" + [0x61, ...test], + "a${replacement}" ], [ - new List.from([0x61]) - ..addAll(test) - ..add(0x61), - "a\u{FFFD}a" - ], - [new List.from(test)..add(0x61), "\u{FFFD}a"], - [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], - [ - new List.from(test) - ..add(0x61) - ..addAll(test), - "\u{FFFD}a\u{FFFD}" + [0x61, ...test, 0x61], + "a${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test), - "å\u{FFFD}" + [...test, 0x61], + "${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), - "å\u{FFFD}å" + [...test, ...test], + "${replacement}${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5]), - "\u{FFFD}å" + [...test, 0x61, ...test], + "${replacement}a${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), - "\u{FFFD}å\u{FFFD}" + [0xc3, 0xa5, ...test], + "å${replacement}" + ], + [ + [0xc3, 0xa5, ...test, 0xc3, 0xa5], + "å${replacement}å" + ], + [ + [...test, 0xc3, 0xa5], + "${replacement}å" + ], + [ + [...test, 0xc3, 0xa5, ...test], + "${replacement}å${replacement}" ] ]; }); diff --git a/tests/lib_2/convert/chunked_conversion_utf87_test.dart b/tests/lib_2/convert/chunked_conversion_utf87_test.dart index 97946902dc9..67929af0b99 100644 --- a/tests/lib_2/convert/chunked_conversion_utf87_test.dart +++ b/tests/lib_2/convert/chunked_conversion_utf87_test.dart @@ -49,11 +49,14 @@ String decodeAllowMalformed2(List inputBytes) { return utf8.decode(bytes); } -final TESTS = [ +final TESTS0 = [ // Unfinished UTF-8 sequences. [0xc3], [0xE2, 0x82], - [0xF0, 0xA4, 0xAD], + [0xF0, 0xA4, 0xAD] +]; + +final TESTS1 = [ // Overlong encoding of euro-sign. [0xF0, 0x82, 0x82, 0xAC], // Other overlong/unfinished sequences. @@ -80,11 +83,11 @@ final TESTS2 = [ // Test that 0xC0|1, 0x80 does not eat the next character. [ [0xC0, 0x80, 0x61], - "Xa" + "XXa" ], [ [0xC1, 0x80, 0x61], - "Xa" + "XXa" ], // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. [ @@ -223,44 +226,49 @@ final TESTS2 = [ ]; main() { - var allTests = TESTS.expand((test) { + var allTests = [...TESTS0, ...TESTS1].expand((test) { // Pairs of test and expected string output when malformed strings are - // allowed. Replacement character: U+FFFD + // allowed. Replacement character: U+FFFD, one per unfinished sequence or + // undecodable byte. + String replacement = + TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length; return [ - [test, "\u{FFFD}"], + [test, "${replacement}"], [ - new List.from([0x61])..addAll(test), - "a\u{FFFD}" + [0x61, ...test], + "a${replacement}" ], [ - new List.from([0x61]) - ..addAll(test) - ..add(0x61), - "a\u{FFFD}a" - ], - [new List.from(test)..add(0x61), "\u{FFFD}a"], - [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], - [ - new List.from(test) - ..add(0x61) - ..addAll(test), - "\u{FFFD}a\u{FFFD}" + [0x61, ...test, 0x61], + "a${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test), - "å\u{FFFD}" + [...test, 0x61], + "${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), - "å\u{FFFD}å" + [...test, ...test], + "${replacement}${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5]), - "\u{FFFD}å" + [...test, 0x61, ...test], + "${replacement}a${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), - "\u{FFFD}å\u{FFFD}" + [0xc3, 0xa5, ...test], + "å${replacement}" + ], + [ + [0xc3, 0xa5, ...test, 0xc3, 0xa5], + "å${replacement}å" + ], + [ + [...test, 0xc3, 0xa5], + "${replacement}å" + ], + [ + [...test, 0xc3, 0xa5, ...test], + "${replacement}å${replacement}" ] ]; }); diff --git a/tests/lib_2/convert/chunked_conversion_utf88_test.dart b/tests/lib_2/convert/chunked_conversion_utf88_test.dart index 58cecaa685e..8b19b2a15d2 100644 --- a/tests/lib_2/convert/chunked_conversion_utf88_test.dart +++ b/tests/lib_2/convert/chunked_conversion_utf88_test.dart @@ -112,8 +112,7 @@ main() { const LEADING_SURROGATE = 0xd801; const TRAILING_SURROGATE = 0xdc12; const UTF8_ENCODING = const [0xf0, 0x90, 0x90, 0x92]; - const UTF8_LEADING = const [0xed, 0xa0, 0x81]; - const UTF8_TRAILING = const [0xed, 0xb0, 0x92]; + const UTF8_REPLACEMENT = const [0xef, 0xbf, 0xbd]; const CHAR_A = 0x61; // Test surrogates at all kinds of locations. @@ -129,17 +128,17 @@ main() { codeUnits[i] = LEADING_SURROGATE; var str = new String.fromCharCodes(codeUnits); var bytes = new List.filled(i + 3, CHAR_A); - bytes[i] = UTF8_LEADING[0]; - bytes[i + 1] = UTF8_LEADING[1]; - bytes[i + 2] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = TRAILING_SURROGATE; str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 3, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits.length = i + 2; @@ -157,36 +156,36 @@ main() { codeUnits[i + 1] = TRAILING_SURROGATE; str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 6, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; - bytes[i + 3] = UTF8_TRAILING[0]; - bytes[i + 4] = UTF8_TRAILING[1]; - bytes[i + 5] = UTF8_TRAILING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = LEADING_SURROGATE; codeUnits[i + 1] = LEADING_SURROGATE; str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 6, CHAR_A); - bytes[i] = UTF8_LEADING[0]; - bytes[i + 1] = UTF8_LEADING[1]; - bytes[i + 2] = UTF8_LEADING[2]; - bytes[i + 3] = UTF8_LEADING[0]; - bytes[i + 4] = UTF8_LEADING[1]; - bytes[i + 5] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = TRAILING_SURROGATE; codeUnits[i + 1] = LEADING_SURROGATE; str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 6, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; - bytes[i + 3] = UTF8_LEADING[0]; - bytes[i + 4] = UTF8_LEADING[1]; - bytes[i + 5] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits.length = i + 3; @@ -208,12 +207,12 @@ main() { codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 7, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; - bytes[i + 3] = UTF8_TRAILING[0]; - bytes[i + 4] = UTF8_TRAILING[1]; - bytes[i + 5] = UTF8_TRAILING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = LEADING_SURROGATE; @@ -221,12 +220,12 @@ main() { codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 7, CHAR_A); - bytes[i] = UTF8_LEADING[0]; - bytes[i + 1] = UTF8_LEADING[1]; - bytes[i + 2] = UTF8_LEADING[2]; - bytes[i + 3] = UTF8_LEADING[0]; - bytes[i + 4] = UTF8_LEADING[1]; - bytes[i + 5] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = TRAILING_SURROGATE; @@ -234,12 +233,12 @@ main() { codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 7, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; - bytes[i + 3] = UTF8_LEADING[0]; - bytes[i + 4] = UTF8_LEADING[1]; - bytes[i + 5] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); // Make sure the invariant is correct. diff --git a/tests/lib_2/convert/json_utf8_chunk_test.dart b/tests/lib_2/convert/json_utf8_chunk_test.dart index ec47339ba32..cf5064ed823 100644 --- a/tests/lib_2/convert/json_utf8_chunk_test.dart +++ b/tests/lib_2/convert/json_utf8_chunk_test.dart @@ -284,69 +284,69 @@ void jsonThrows(String name, String codeString) { void testMalformed() { // Overlong encodings. jsonMalformedTest( - "overlong-0-2", "@\uFFFD@", [0x22, 0x40, 0xc0, 0x80, 0x40, 0x22]); - jsonMalformedTest( - "overlong-0-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x80, 0x80, 0x40, 0x22]); - jsonMalformedTest("overlong-0-4", "@\uFFFD@", + "overlong-0-2", "@\uFFFD\uFFFD@", [0x22, 0x40, 0xc0, 0x80, 0x40, 0x22]); + jsonMalformedTest("overlong-0-3", "@\uFFFD\uFFFD\uFFFD@", + [0x22, 0x40, 0xe0, 0x80, 0x80, 0x40, 0x22]); + jsonMalformedTest("overlong-0-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x80, 0x40, 0x22]); jsonMalformedTest( - "overlong-7f-2", "@\uFFFD@", [0x22, 0x40, 0xc1, 0xbf, 0x40, 0x22]); - jsonMalformedTest( - "overlong-7f-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x81, 0xbf, 0x40, 0x22]); - jsonMalformedTest("overlong-7f-4", "@\uFFFD@", + "overlong-7f-2", "@\uFFFD\uFFFD@", [0x22, 0x40, 0xc1, 0xbf, 0x40, 0x22]); + jsonMalformedTest("overlong-7f-3", "@\uFFFD\uFFFD\uFFFD@", + [0x22, 0x40, 0xe0, 0x81, 0xbf, 0x40, 0x22]); + jsonMalformedTest("overlong-7f-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x81, 0xbf, 0x40, 0x22]); - jsonMalformedTest( - "overlong-80-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x82, 0x80, 0x40, 0x22]); - jsonMalformedTest("overlong-80-4", "@\uFFFD@", + jsonMalformedTest("overlong-80-3", "@\uFFFD\uFFFD\uFFFD@", + [0x22, 0x40, 0xe0, 0x82, 0x80, 0x40, 0x22]); + jsonMalformedTest("overlong-80-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x82, 0x80, 0x40, 0x22]); - jsonMalformedTest( - "overlong-7ff-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x9f, 0xbf, 0x40, 0x22]); - jsonMalformedTest("overlong-7ff-4", "@\uFFFD@", + jsonMalformedTest("overlong-7ff-3", "@\uFFFD\uFFFD\uFFFD@", + [0x22, 0x40, 0xe0, 0x9f, 0xbf, 0x40, 0x22]); + jsonMalformedTest("overlong-7ff-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x9f, 0xbf, 0x40, 0x22]); - jsonMalformedTest("overlong-800-4", "@\uFFFD@", + jsonMalformedTest("overlong-800-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0xa0, 0x80, 0x40, 0x22]); - jsonMalformedTest("overlong-ffff-4", "@\uFFFD@", + jsonMalformedTest("overlong-ffff-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x8f, 0xbf, 0xbf, 0x40, 0x22]); // Unterminated multibyte sequences. jsonMalformedTest( "unterminated-2-normal", "@\uFFFD@", [0x22, 0x40, 0xc0, 0x40, 0x22]); - jsonMalformedTest("unterminated-3-normal", "@\uFFFD@", + jsonMalformedTest("unterminated-3-normal", "@\uFFFD\uFFFD@", [0x22, 0x40, 0xe0, 0x80, 0x40, 0x22]); - jsonMalformedTest("unterminated-4-normal", "@\uFFFD@", + jsonMalformedTest("unterminated-4-normal", "@\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x40, 0x22]); jsonMalformedTest("unterminated-2-multi", "@\uFFFD\x80@", [0x22, 0x40, 0xc0, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("unterminated-3-multi", "@\uFFFD\x80@", + jsonMalformedTest("unterminated-3-multi", "@\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xe0, 0x80, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("unterminated-4-multi", "@\uFFFD\x80@", + jsonMalformedTest("unterminated-4-multi", "@\uFFFD\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0xc2, 0x80, 0x40, 0x22]); jsonMalformedTest("unterminated-2-escape", "@\uFFFD\n@", [0x22, 0x40, 0xc0, 0x5c, 0x6e, 0x40, 0x22]); - jsonMalformedTest("unterminated-3-escape", "@\uFFFD\n@", + jsonMalformedTest("unterminated-3-escape", "@\uFFFD\uFFFD\n@", [0x22, 0x40, 0xe0, 0x80, 0x5c, 0x6e, 0x40, 0x22]); - jsonMalformedTest("unterminated-4-escape", "@\uFFFD\n@", + jsonMalformedTest("unterminated-4-escape", "@\uFFFD\uFFFD\uFFFD\n@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x5c, 0x6e, 0x40, 0x22]); jsonMalformedTest("unterminated-2-end", "@\uFFFD", [0x22, 0x40, 0xc0, 0x22]); jsonMalformedTest( - "unterminated-3-end", "@\uFFFD", [0x22, 0x40, 0xe0, 0x80, 0x22]); + "unterminated-3-end", "@\uFFFD\uFFFD", [0x22, 0x40, 0xe0, 0x80, 0x22]); - jsonMalformedTest( - "unterminated-4-end", "@\uFFFD", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x22]); + jsonMalformedTest("unterminated-4-end", "@\uFFFD\uFFFD\uFFFD", + [0x22, 0x40, 0xf0, 0x80, 0x80, 0x22]); // Unexpected continuation byte // - after a normal character. @@ -372,13 +372,13 @@ void testMalformed() { "leading-2", "@\uFFFD\x80@", [0x22, 0x40, 0xc0, 0xc2, 0x80, 0x40, 0x22]); jsonMalformedTest("leading-3-1", "@\uFFFD\x80@", [0x22, 0x40, 0xe0, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("leading-3-2", "@\uFFFD\x80@", + jsonMalformedTest("leading-3-2", "@\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xe0, 0x80, 0xc2, 0x80, 0x40, 0x22]); jsonMalformedTest("leading-4-1", "@\uFFFD\x80@", [0x22, 0x40, 0xf0, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("leading-4-2", "@\uFFFD\x80@", + jsonMalformedTest("leading-4-2", "@\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xf0, 0x80, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("leading-4-3", "@\uFFFD\x80@", + jsonMalformedTest("leading-4-3", "@\uFFFD\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0xc2, 0x80, 0x40, 0x22]); // Overlong encodings of ASCII outside of strings always fail. diff --git a/tests/lib_2/convert/utf82_test.dart b/tests/lib_2/convert/utf82_test.dart index df08103e86d..cf0c2b0585e 100644 --- a/tests/lib_2/convert/utf82_test.dart +++ b/tests/lib_2/convert/utf82_test.dart @@ -28,11 +28,14 @@ String decodeAllowMalformed4(List bytes) { return new Utf8Codec(allowMalformed: true).decoder.convert(bytes); } -final TESTS = [ +final TESTS0 = [ // Unfinished UTF-8 sequences. [0xc3], [0xE2, 0x82], - [0xF0, 0xA4, 0xAD], + [0xF0, 0xA4, 0xAD] +]; + +final TESTS1 = [ // Overlong encoding of euro-sign. [0xF0, 0x82, 0x82, 0xAC], // Other overlong/unfinished sequences. @@ -64,11 +67,11 @@ final TESTS2 = [ // Test that 0xC0|1, 0x80 does not eat the next character. [ [0xC0, 0x80, 0x61], - "Xa" + "XXa" ], [ [0xC1, 0x80, 0x61], - "Xa" + "XXa" ], // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. [ @@ -207,44 +210,49 @@ final TESTS2 = [ ]; main() { - var allTests = TESTS.expand((test) { + var allTests = [...TESTS0, ...TESTS1].expand((test) { // Pairs of test and expected string output when malformed strings are - // allowed. Replacement character: U+FFFD + // allowed. Replacement character: U+FFFD, one per unfinished sequence or + // undecodable byte. + String replacement = + TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length; return [ - [test, "\u{FFFD}"], + [test, "${replacement}"], [ - new List.from([0x61])..addAll(test), - "a\u{FFFD}" + [0x61, ...test], + "a${replacement}" ], [ - new List.from([0x61]) - ..addAll(test) - ..add(0x61), - "a\u{FFFD}a" - ], - [new List.from(test)..add(0x61), "\u{FFFD}a"], - [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], - [ - new List.from(test) - ..add(0x61) - ..addAll(test), - "\u{FFFD}a\u{FFFD}" + [0x61, ...test, 0x61], + "a${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test), - "å\u{FFFD}" + [...test, 0x61], + "${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), - "å\u{FFFD}å" + [...test, ...test], + "${replacement}${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5]), - "\u{FFFD}å" + [...test, 0x61, ...test], + "${replacement}a${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), - "\u{FFFD}å\u{FFFD}" + [0xc3, 0xa5, ...test], + "å${replacement}" + ], + [ + [0xc3, 0xa5, ...test, 0xc3, 0xa5], + "å${replacement}å" + ], + [ + [...test, 0xc3, 0xa5], + "${replacement}å" + ], + [ + [...test, 0xc3, 0xa5, ...test], + "${replacement}å${replacement}" ] ]; }); diff --git a/tests/lib_2/convert/utf84_test.dart b/tests/lib_2/convert/utf84_test.dart index bc1db59a004..566b1fa54bd 100755 --- a/tests/lib_2/convert/utf84_test.dart +++ b/tests/lib_2/convert/utf84_test.dart @@ -727,10 +727,14 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { Expect.listEquals([0xe000], utf8ToRunes([0xee, 0x80, 0x80]), "e000"); Expect.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xef, 0xbf, 0xbd]), "fffd"); - Expect - .listEquals([0x10ffff], utf8ToRunes([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xf4, 0x90, 0x80, 0x80]), "110000"); + Expect.listEquals( + [0x10ffff], utf8ToRunes([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf4, 0x90, 0x80, 0x80]), "110000"); // unexpected continuation bytes Expect.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0x80]), @@ -795,12 +799,15 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { // Sequences with last continuation byte missing Expect.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xc2]), "2-byte sequence with last byte missing"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xe0, 0x80]), "3-byte sequence with last byte missing"); Expect.listEquals( - [unicodeReplacementCharacterRune], - utf8ToRunes([0xf0, 0x80, 0x80]), - "4-byte sequence with last byte missing"); + [unicodeReplacementCharacterRune, unicodeReplacementCharacterRune], + utf8ToRunes([0xe0, 0x80]), + "3-byte sequence with last byte missing"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf0, 0x80, 0x80]), "4-byte sequence with last byte missing"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, @@ -871,6 +878,9 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, unicodeReplacementCharacterRune ], utf8ToRunes([ @@ -890,8 +900,8 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { 0x80, 0x80, 0xdf, - 0xef, - 0xbf, + 0xef, // These two bytes form one incomplete sequence. + 0xbf, // All others form one per byte. 0xf7, 0xbf, 0xbf, @@ -908,10 +918,10 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { "Concatenation of incomplete sequences"); // Impossible bytes - Expect - .listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xfe]), "fe"); - Expect - .listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xff]), "ff"); + Expect.listEquals( + [unicodeReplacementCharacterRune], utf8ToRunes([0xfe]), "fe"); + Expect.listEquals( + [unicodeReplacementCharacterRune], utf8ToRunes([0xff]), "ff"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, @@ -921,11 +931,20 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { // Overlong sequences Expect.listEquals( - [unicodeReplacementCharacterRune], utf8ToRunes([0xc0, 0xaf]), "c0 af"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xe0, 0x80, 0xaf]), "e0 80 af"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af"); + [unicodeReplacementCharacterRune, unicodeReplacementCharacterRune], + utf8ToRunes([0xc0, 0xaf]), + "c0 af"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xe0, 0x80, 0xaf]), "e0 80 af"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, @@ -943,11 +962,20 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { ], utf8ToRunes([0xfc, 0x80, 0x80, 0x80, 0x80, 0xaf]), "fc 80 80 80 80 af"); Expect.listEquals( - [unicodeReplacementCharacterRune], utf8ToRunes([0xc1, 0xbf]), "c1 bf"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xe0, 0x9f, 0xbf]), "e0 9f bf"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf"); + [unicodeReplacementCharacterRune, unicodeReplacementCharacterRune], + utf8ToRunes([0xc1, 0xbf]), + "c1 bf"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xe0, 0x9f, 0xbf]), "e0 9f bf"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, @@ -965,11 +993,20 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { ], utf8ToRunes([0xfc, 0x83, 0xbf, 0xbf, 0xbf, 0xbf]), "fc 83 bf bf bf bf"); Expect.listEquals( - [unicodeReplacementCharacterRune], utf8ToRunes([0xc0, 0x80]), "c0 80"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xe0, 0x80, 0x80]), "e0 80 80"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80"); + [unicodeReplacementCharacterRune, unicodeReplacementCharacterRune], + utf8ToRunes([0xc0, 0x80]), + "c0 80"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xe0, 0x80, 0x80]), "e0 80 80"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, diff --git a/tests/lib_2/convert/utf85_test.dart b/tests/lib_2/convert/utf85_test.dart index 3f287e94762..a3809918a8f 100644 --- a/tests/lib_2/convert/utf85_test.dart +++ b/tests/lib_2/convert/utf85_test.dart @@ -9,7 +9,7 @@ import 'dart:convert'; main() { for (int i = 0; i <= 0x10FFFF; i++) { - if (i == unicodeBomCharacterRune) continue; + if (i == unicodeBomCharacterRune || (i & 0x1FF800) == 0xD800) continue; Expect.equals( i, utf8.decode(utf8.encode(new String.fromCharCode(i))).runes.first); } diff --git a/tests/lib_2/convert/utf8_encode_test.dart b/tests/lib_2/convert/utf8_encode_test.dart index 1e611bb575b..0a43e1452f1 100644 --- a/tests/lib_2/convert/utf8_encode_test.dart +++ b/tests/lib_2/convert/utf8_encode_test.dart @@ -25,8 +25,8 @@ void testEncodeSlice() { String ascii = "ABCDE"; Expect.listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii)); Expect.listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0)); - Expect - .listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0, 5)); + Expect.listEquals( + [0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0, 5)); Expect.listEquals([0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 1)); Expect.listEquals([0x41, 0x42, 0x43, 0x44], encoder.convert(ascii, 0, 4)); Expect.listEquals([0x42, 0x43, 0x44], encoder.convert(ascii, 1, 4)); @@ -50,6 +50,6 @@ void testEncodeSlice() { Expect.listEquals( [0xc2, 0x82, 0xe1, 0x81, 0x81], encoder.convert(unicode, 1, 3)); // Split in the middle of a surrogate pair. - Expect.listEquals([0xc2, 0x82, 0xe1, 0x81, 0x81, 0xed, 0xa0, 0x80], + Expect.listEquals([0xc2, 0x82, 0xe1, 0x81, 0x81, 0xef, 0xbf, 0xbd], encoder.convert(unicode, 1, 4)); } diff --git a/tests/lib_2/convert/utf8_test.dart b/tests/lib_2/convert/utf8_test.dart index 858c982ef22..6f97fd06770 100644 --- a/tests/lib_2/convert/utf8_test.dart +++ b/tests/lib_2/convert/utf8_test.dart @@ -73,28 +73,29 @@ void testErrorOffset() { // Bad encoding, points to first bad byte. testExn([0x80, 0x00], 0); - testExn([0xC0, 0x00], 1); - testExn([0xE0, 0x00], 1); - testExn([0xE0, 0x80, 0x00], 2); - testExn([0xF0, 0x00], 1); - testExn([0xF0, 0x80, 0x00], 2); - testExn([0xF0, 0x80, 0x80, 0x00], 3); + testExn([0xC2, 0x00], 1); + testExn([0xE2, 0x00], 1); + testExn([0xE2, 0x80, 0x00], 2); + testExn([0xF2, 0x00], 1); + testExn([0xF2, 0x80, 0x00], 2); + testExn([0xF2, 0x80, 0x80, 0x00], 3); testExn([0xF8, 0x00], 0); // Short encoding, points to end. - testExn([0xC0], 1); - testExn([0xE0], 1); - testExn([0xE0, 0x80], 2); - testExn([0xF0], 1); - testExn([0xF0, 0x80], 2); - testExn([0xF0, 0x80, 0x80], 3); - // Overlong encoding, points to start of encoding. + testExn([0xC2], 1); + testExn([0xE2], 1); + testExn([0xE2, 0x80], 2); + testExn([0xF2], 1); + testExn([0xF2, 0x80], 2); + testExn([0xF2, 0x80, 0x80], 3); + // Overlong encoding, points to byte that gave enough information to conclude + // that it was overlong. testExn([0xC0, 0x80], 0); testExn([0xC1, 0xBF], 0); - testExn([0xE0, 0x80, 0x80], 0); - testExn([0xE0, 0x9F, 0xBF], 0); - testExn([0xF0, 0x80, 0x80, 0x80], 0); - testExn([0xF0, 0x8F, 0xBF, 0xBF], 0); + testExn([0xE0, 0x80, 0x80], 1); + testExn([0xE0, 0x9F, 0xBF], 1); + testExn([0xF0, 0x80, 0x80, 0x80], 1); + testExn([0xF0, 0x8F, 0xBF, 0xBF], 1); // Invalid character (value too large, over 0x10FFFF). - testExn([0xF4, 0x90, 0x80, 0x80], 0); + testExn([0xF4, 0x90, 0x80, 0x80], 1); testExn([0xF7, 0xBF, 0xBF, 0xBF], 0); }