diff --git a/tests/lib/convert/chunked_conversion_utf82_test.dart b/tests/lib/convert/chunked_conversion_utf82_test.dart index bc0b200b0c2..2e07f87f447 100644 --- a/tests/lib/convert/chunked_conversion_utf82_test.dart +++ b/tests/lib/convert/chunked_conversion_utf82_test.dart @@ -46,11 +46,14 @@ String decodeAllowMalformed(List bytes, int chunkSize) { return buffer.toString(); } -final TESTS = [ +final TESTS0 = [ // Unfinished UTF-8 sequences. [0xc3], [0xE2, 0x82], - [0xF0, 0xA4, 0xAD], + [0xF0, 0xA4, 0xAD] +]; + +final TESTS1 = [ // Overlong encoding of euro-sign. [0xF0, 0x82, 0x82, 0xAC], // Other overlong/unfinished sequences. @@ -77,11 +80,11 @@ final TESTS2 = [ // Test that 0xC0|1, 0x80 does not eat the next character. [ [0xC0, 0x80, 0x61], - "Xa" + "XXa" ], [ [0xC1, 0x80, 0x61], - "Xa" + "XXa" ], // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. [ @@ -220,44 +223,49 @@ final TESTS2 = [ ]; main() { - var allTests = TESTS.expand((test) { + var allTests = [...TESTS0, ...TESTS1].expand((test) { // Pairs of test and expected string output when malformed strings are - // allowed. Replacement character: U+FFFD + // allowed. Replacement character: U+FFFD, one per unfinished sequence or + // undecodable byte. + String replacement = + TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length; return [ - [test, "\u{FFFD}"], + [test, "${replacement}"], [ - new List.from([0x61])..addAll(test), - "a\u{FFFD}" + [0x61, ...test], + "a${replacement}" ], [ - new List.from([0x61]) - ..addAll(test) - ..add(0x61), - "a\u{FFFD}a" - ], - [new List.from(test)..add(0x61), "\u{FFFD}a"], - [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], - [ - new List.from(test) - ..add(0x61) - ..addAll(test), - "\u{FFFD}a\u{FFFD}" + [0x61, ...test, 0x61], + "a${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test), - "å\u{FFFD}" + [...test, 0x61], + "${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), - "å\u{FFFD}å" + [...test, ...test], + "${replacement}${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5]), - "\u{FFFD}å" + [...test, 0x61, ...test], + "${replacement}a${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), - "\u{FFFD}å\u{FFFD}" + [0xc3, 0xa5, ...test], + "å${replacement}" + ], + [ + [0xc3, 0xa5, ...test, 0xc3, 0xa5], + "å${replacement}å" + ], + [ + [...test, 0xc3, 0xa5], + "${replacement}å" + ], + [ + [...test, 0xc3, 0xa5, ...test], + "${replacement}å${replacement}" ] ]; }); diff --git a/tests/lib/convert/chunked_conversion_utf86_test.dart b/tests/lib/convert/chunked_conversion_utf86_test.dart index a42e7523334..8c9a08c39fd 100644 --- a/tests/lib/convert/chunked_conversion_utf86_test.dart +++ b/tests/lib/convert/chunked_conversion_utf86_test.dart @@ -28,11 +28,14 @@ String decodeAllowMalformed(List bytes) { return buffer.toString(); } -final TESTS = [ +final TESTS0 = [ // Unfinished UTF-8 sequences. [0xc3], [0xE2, 0x82], - [0xF0, 0xA4, 0xAD], + [0xF0, 0xA4, 0xAD] +]; + +final TESTS1 = [ // Overlong encoding of euro-sign. [0xF0, 0x82, 0x82, 0xAC], // Other overlong/unfinished sequences. @@ -59,11 +62,11 @@ final TESTS2 = [ // Test that 0xC0|1, 0x80 does not eat the next character. [ [0xC0, 0x80, 0x61], - "Xa" + "XXa" ], [ [0xC1, 0x80, 0x61], - "Xa" + "XXa" ], // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. [ @@ -202,44 +205,49 @@ final TESTS2 = [ ]; main() { - var allTests = TESTS.expand((test) { + var allTests = [...TESTS0, ...TESTS1].expand((test) { // Pairs of test and expected string output when malformed strings are - // allowed. Replacement character: U+FFFD + // allowed. Replacement character: U+FFFD, one per unfinished sequence or + // undecodable byte. + String replacement = + TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length; return [ - [test, "\u{FFFD}"], + [test, "${replacement}"], [ - new List.from([0x61])..addAll(test), - "a\u{FFFD}" + [0x61, ...test], + "a${replacement}" ], [ - new List.from([0x61]) - ..addAll(test) - ..add(0x61), - "a\u{FFFD}a" - ], - [new List.from(test)..add(0x61), "\u{FFFD}a"], - [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], - [ - new List.from(test) - ..add(0x61) - ..addAll(test), - "\u{FFFD}a\u{FFFD}" + [0x61, ...test, 0x61], + "a${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test), - "å\u{FFFD}" + [...test, 0x61], + "${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), - "å\u{FFFD}å" + [...test, ...test], + "${replacement}${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5]), - "\u{FFFD}å" + [...test, 0x61, ...test], + "${replacement}a${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), - "\u{FFFD}å\u{FFFD}" + [0xc3, 0xa5, ...test], + "å${replacement}" + ], + [ + [0xc3, 0xa5, ...test, 0xc3, 0xa5], + "å${replacement}å" + ], + [ + [...test, 0xc3, 0xa5], + "${replacement}å" + ], + [ + [...test, 0xc3, 0xa5, ...test], + "${replacement}å${replacement}" ] ]; }); diff --git a/tests/lib/convert/chunked_conversion_utf87_test.dart b/tests/lib/convert/chunked_conversion_utf87_test.dart index ab42ee23b89..5434f1f1f43 100644 --- a/tests/lib/convert/chunked_conversion_utf87_test.dart +++ b/tests/lib/convert/chunked_conversion_utf87_test.dart @@ -49,11 +49,14 @@ String decodeAllowMalformed2(List inputBytes) { return utf8.decode(bytes); } -final TESTS = [ +final TESTS0 = [ // Unfinished UTF-8 sequences. [0xc3], [0xE2, 0x82], - [0xF0, 0xA4, 0xAD], + [0xF0, 0xA4, 0xAD] +]; + +final TESTS1 = [ // Overlong encoding of euro-sign. [0xF0, 0x82, 0x82, 0xAC], // Other overlong/unfinished sequences. @@ -80,11 +83,11 @@ final TESTS2 = [ // Test that 0xC0|1, 0x80 does not eat the next character. [ [0xC0, 0x80, 0x61], - "Xa" + "XXa" ], [ [0xC1, 0x80, 0x61], - "Xa" + "XXa" ], // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. [ @@ -223,44 +226,49 @@ final TESTS2 = [ ]; main() { - var allTests = TESTS.expand((test) { + var allTests = [...TESTS0, ...TESTS1].expand((test) { // Pairs of test and expected string output when malformed strings are - // allowed. Replacement character: U+FFFD + // allowed. Replacement character: U+FFFD, one per unfinished sequence or + // undecodable byte. + String replacement = + TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length; return [ - [test, "\u{FFFD}"], + [test, "${replacement}"], [ - new List.from([0x61])..addAll(test), - "a\u{FFFD}" + [0x61, ...test], + "a${replacement}" ], [ - new List.from([0x61]) - ..addAll(test) - ..add(0x61), - "a\u{FFFD}a" - ], - [new List.from(test)..add(0x61), "\u{FFFD}a"], - [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], - [ - new List.from(test) - ..add(0x61) - ..addAll(test), - "\u{FFFD}a\u{FFFD}" + [0x61, ...test, 0x61], + "a${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test), - "å\u{FFFD}" + [...test, 0x61], + "${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), - "å\u{FFFD}å" + [...test, ...test], + "${replacement}${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5]), - "\u{FFFD}å" + [...test, 0x61, ...test], + "${replacement}a${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), - "\u{FFFD}å\u{FFFD}" + [0xc3, 0xa5, ...test], + "å${replacement}" + ], + [ + [0xc3, 0xa5, ...test, 0xc3, 0xa5], + "å${replacement}å" + ], + [ + [...test, 0xc3, 0xa5], + "${replacement}å" + ], + [ + [...test, 0xc3, 0xa5, ...test], + "${replacement}å${replacement}" ] ]; }); diff --git a/tests/lib/convert/chunked_conversion_utf88_test.dart b/tests/lib/convert/chunked_conversion_utf88_test.dart index c0a63d734e5..93cab53b646 100644 --- a/tests/lib/convert/chunked_conversion_utf88_test.dart +++ b/tests/lib/convert/chunked_conversion_utf88_test.dart @@ -112,8 +112,7 @@ main() { const LEADING_SURROGATE = 0xd801; const TRAILING_SURROGATE = 0xdc12; const UTF8_ENCODING = const [0xf0, 0x90, 0x90, 0x92]; - const UTF8_LEADING = const [0xed, 0xa0, 0x81]; - const UTF8_TRAILING = const [0xed, 0xb0, 0x92]; + const UTF8_REPLACEMENT = const [0xef, 0xbf, 0xbd]; const CHAR_A = 0x61; // Test surrogates at all kinds of locations. @@ -129,17 +128,17 @@ main() { codeUnits[i] = LEADING_SURROGATE; var str = new String.fromCharCodes(codeUnits); var bytes = new List.filled(i + 3, CHAR_A); - bytes[i] = UTF8_LEADING[0]; - bytes[i + 1] = UTF8_LEADING[1]; - bytes[i + 2] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = TRAILING_SURROGATE; str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 3, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits.add(CHAR_A); @@ -158,36 +157,36 @@ main() { codeUnits[i + 1] = TRAILING_SURROGATE; str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 6, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; - bytes[i + 3] = UTF8_TRAILING[0]; - bytes[i + 4] = UTF8_TRAILING[1]; - bytes[i + 5] = UTF8_TRAILING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = LEADING_SURROGATE; codeUnits[i + 1] = LEADING_SURROGATE; str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 6, CHAR_A); - bytes[i] = UTF8_LEADING[0]; - bytes[i + 1] = UTF8_LEADING[1]; - bytes[i + 2] = UTF8_LEADING[2]; - bytes[i + 3] = UTF8_LEADING[0]; - bytes[i + 4] = UTF8_LEADING[1]; - bytes[i + 5] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = TRAILING_SURROGATE; codeUnits[i + 1] = LEADING_SURROGATE; str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 6, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; - bytes[i + 3] = UTF8_LEADING[0]; - bytes[i + 4] = UTF8_LEADING[1]; - bytes[i + 5] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits.add(CHAR_A); @@ -210,12 +209,12 @@ main() { codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 7, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; - bytes[i + 3] = UTF8_TRAILING[0]; - bytes[i + 4] = UTF8_TRAILING[1]; - bytes[i + 5] = UTF8_TRAILING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = LEADING_SURROGATE; @@ -223,12 +222,12 @@ main() { codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 7, CHAR_A); - bytes[i] = UTF8_LEADING[0]; - bytes[i + 1] = UTF8_LEADING[1]; - bytes[i + 2] = UTF8_LEADING[2]; - bytes[i + 3] = UTF8_LEADING[0]; - bytes[i + 4] = UTF8_LEADING[1]; - bytes[i + 5] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = TRAILING_SURROGATE; @@ -236,12 +235,12 @@ main() { codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 7, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; - bytes[i + 3] = UTF8_LEADING[0]; - bytes[i + 4] = UTF8_LEADING[1]; - bytes[i + 5] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); // Make sure the invariant is correct. diff --git a/tests/lib/convert/json_utf8_chunk_test.dart b/tests/lib/convert/json_utf8_chunk_test.dart index c18f330bc93..e75019d3e99 100644 --- a/tests/lib/convert/json_utf8_chunk_test.dart +++ b/tests/lib/convert/json_utf8_chunk_test.dart @@ -284,69 +284,69 @@ void jsonThrows(String name, String codeString) { void testMalformed() { // Overlong encodings. jsonMalformedTest( - "overlong-0-2", "@\uFFFD@", [0x22, 0x40, 0xc0, 0x80, 0x40, 0x22]); - jsonMalformedTest( - "overlong-0-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x80, 0x80, 0x40, 0x22]); - jsonMalformedTest("overlong-0-4", "@\uFFFD@", + "overlong-0-2", "@\uFFFD\uFFFD@", [0x22, 0x40, 0xc0, 0x80, 0x40, 0x22]); + jsonMalformedTest("overlong-0-3", "@\uFFFD\uFFFD\uFFFD@", + [0x22, 0x40, 0xe0, 0x80, 0x80, 0x40, 0x22]); + jsonMalformedTest("overlong-0-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x80, 0x40, 0x22]); jsonMalformedTest( - "overlong-7f-2", "@\uFFFD@", [0x22, 0x40, 0xc1, 0xbf, 0x40, 0x22]); - jsonMalformedTest( - "overlong-7f-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x81, 0xbf, 0x40, 0x22]); - jsonMalformedTest("overlong-7f-4", "@\uFFFD@", + "overlong-7f-2", "@\uFFFD\uFFFD@", [0x22, 0x40, 0xc1, 0xbf, 0x40, 0x22]); + jsonMalformedTest("overlong-7f-3", "@\uFFFD\uFFFD\uFFFD@", + [0x22, 0x40, 0xe0, 0x81, 0xbf, 0x40, 0x22]); + jsonMalformedTest("overlong-7f-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x81, 0xbf, 0x40, 0x22]); - jsonMalformedTest( - "overlong-80-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x82, 0x80, 0x40, 0x22]); - jsonMalformedTest("overlong-80-4", "@\uFFFD@", + jsonMalformedTest("overlong-80-3", "@\uFFFD\uFFFD\uFFFD@", + [0x22, 0x40, 0xe0, 0x82, 0x80, 0x40, 0x22]); + jsonMalformedTest("overlong-80-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x82, 0x80, 0x40, 0x22]); - jsonMalformedTest( - "overlong-7ff-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x9f, 0xbf, 0x40, 0x22]); - jsonMalformedTest("overlong-7ff-4", "@\uFFFD@", + jsonMalformedTest("overlong-7ff-3", "@\uFFFD\uFFFD\uFFFD@", + [0x22, 0x40, 0xe0, 0x9f, 0xbf, 0x40, 0x22]); + jsonMalformedTest("overlong-7ff-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x9f, 0xbf, 0x40, 0x22]); - jsonMalformedTest("overlong-800-4", "@\uFFFD@", + jsonMalformedTest("overlong-800-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0xa0, 0x80, 0x40, 0x22]); - jsonMalformedTest("overlong-ffff-4", "@\uFFFD@", + jsonMalformedTest("overlong-ffff-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x8f, 0xbf, 0xbf, 0x40, 0x22]); // Unterminated multibyte sequences. jsonMalformedTest( "unterminated-2-normal", "@\uFFFD@", [0x22, 0x40, 0xc0, 0x40, 0x22]); - jsonMalformedTest("unterminated-3-normal", "@\uFFFD@", + jsonMalformedTest("unterminated-3-normal", "@\uFFFD\uFFFD@", [0x22, 0x40, 0xe0, 0x80, 0x40, 0x22]); - jsonMalformedTest("unterminated-4-normal", "@\uFFFD@", + jsonMalformedTest("unterminated-4-normal", "@\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x40, 0x22]); jsonMalformedTest("unterminated-2-multi", "@\uFFFD\x80@", [0x22, 0x40, 0xc0, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("unterminated-3-multi", "@\uFFFD\x80@", + jsonMalformedTest("unterminated-3-multi", "@\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xe0, 0x80, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("unterminated-4-multi", "@\uFFFD\x80@", + jsonMalformedTest("unterminated-4-multi", "@\uFFFD\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0xc2, 0x80, 0x40, 0x22]); jsonMalformedTest("unterminated-2-escape", "@\uFFFD\n@", [0x22, 0x40, 0xc0, 0x5c, 0x6e, 0x40, 0x22]); - jsonMalformedTest("unterminated-3-escape", "@\uFFFD\n@", + jsonMalformedTest("unterminated-3-escape", "@\uFFFD\uFFFD\n@", [0x22, 0x40, 0xe0, 0x80, 0x5c, 0x6e, 0x40, 0x22]); - jsonMalformedTest("unterminated-4-escape", "@\uFFFD\n@", + jsonMalformedTest("unterminated-4-escape", "@\uFFFD\uFFFD\uFFFD\n@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x5c, 0x6e, 0x40, 0x22]); jsonMalformedTest("unterminated-2-end", "@\uFFFD", [0x22, 0x40, 0xc0, 0x22]); jsonMalformedTest( - "unterminated-3-end", "@\uFFFD", [0x22, 0x40, 0xe0, 0x80, 0x22]); + "unterminated-3-end", "@\uFFFD\uFFFD", [0x22, 0x40, 0xe0, 0x80, 0x22]); - jsonMalformedTest( - "unterminated-4-end", "@\uFFFD", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x22]); + jsonMalformedTest("unterminated-4-end", "@\uFFFD\uFFFD\uFFFD", + [0x22, 0x40, 0xf0, 0x80, 0x80, 0x22]); // Unexpected continuation byte // - after a normal character. @@ -372,13 +372,13 @@ void testMalformed() { "leading-2", "@\uFFFD\x80@", [0x22, 0x40, 0xc0, 0xc2, 0x80, 0x40, 0x22]); jsonMalformedTest("leading-3-1", "@\uFFFD\x80@", [0x22, 0x40, 0xe0, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("leading-3-2", "@\uFFFD\x80@", + jsonMalformedTest("leading-3-2", "@\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xe0, 0x80, 0xc2, 0x80, 0x40, 0x22]); jsonMalformedTest("leading-4-1", "@\uFFFD\x80@", [0x22, 0x40, 0xf0, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("leading-4-2", "@\uFFFD\x80@", + jsonMalformedTest("leading-4-2", "@\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xf0, 0x80, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("leading-4-3", "@\uFFFD\x80@", + jsonMalformedTest("leading-4-3", "@\uFFFD\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0xc2, 0x80, 0x40, 0x22]); // Overlong encodings of ASCII outside of strings always fail. diff --git a/tests/lib/convert/utf82_test.dart b/tests/lib/convert/utf82_test.dart index df08103e86d..cf0c2b0585e 100644 --- a/tests/lib/convert/utf82_test.dart +++ b/tests/lib/convert/utf82_test.dart @@ -28,11 +28,14 @@ String decodeAllowMalformed4(List bytes) { return new Utf8Codec(allowMalformed: true).decoder.convert(bytes); } -final TESTS = [ +final TESTS0 = [ // Unfinished UTF-8 sequences. [0xc3], [0xE2, 0x82], - [0xF0, 0xA4, 0xAD], + [0xF0, 0xA4, 0xAD] +]; + +final TESTS1 = [ // Overlong encoding of euro-sign. [0xF0, 0x82, 0x82, 0xAC], // Other overlong/unfinished sequences. @@ -64,11 +67,11 @@ final TESTS2 = [ // Test that 0xC0|1, 0x80 does not eat the next character. [ [0xC0, 0x80, 0x61], - "Xa" + "XXa" ], [ [0xC1, 0x80, 0x61], - "Xa" + "XXa" ], // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. [ @@ -207,44 +210,49 @@ final TESTS2 = [ ]; main() { - var allTests = TESTS.expand((test) { + var allTests = [...TESTS0, ...TESTS1].expand((test) { // Pairs of test and expected string output when malformed strings are - // allowed. Replacement character: U+FFFD + // allowed. Replacement character: U+FFFD, one per unfinished sequence or + // undecodable byte. + String replacement = + TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length; return [ - [test, "\u{FFFD}"], + [test, "${replacement}"], [ - new List.from([0x61])..addAll(test), - "a\u{FFFD}" + [0x61, ...test], + "a${replacement}" ], [ - new List.from([0x61]) - ..addAll(test) - ..add(0x61), - "a\u{FFFD}a" - ], - [new List.from(test)..add(0x61), "\u{FFFD}a"], - [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], - [ - new List.from(test) - ..add(0x61) - ..addAll(test), - "\u{FFFD}a\u{FFFD}" + [0x61, ...test, 0x61], + "a${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test), - "å\u{FFFD}" + [...test, 0x61], + "${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), - "å\u{FFFD}å" + [...test, ...test], + "${replacement}${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5]), - "\u{FFFD}å" + [...test, 0x61, ...test], + "${replacement}a${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), - "\u{FFFD}å\u{FFFD}" + [0xc3, 0xa5, ...test], + "å${replacement}" + ], + [ + [0xc3, 0xa5, ...test, 0xc3, 0xa5], + "å${replacement}å" + ], + [ + [...test, 0xc3, 0xa5], + "${replacement}å" + ], + [ + [...test, 0xc3, 0xa5, ...test], + "${replacement}å${replacement}" ] ]; }); diff --git a/tests/lib/convert/utf84_test.dart b/tests/lib/convert/utf84_test.dart index bc1db59a004..566b1fa54bd 100755 --- a/tests/lib/convert/utf84_test.dart +++ b/tests/lib/convert/utf84_test.dart @@ -727,10 +727,14 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { Expect.listEquals([0xe000], utf8ToRunes([0xee, 0x80, 0x80]), "e000"); Expect.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xef, 0xbf, 0xbd]), "fffd"); - Expect - .listEquals([0x10ffff], utf8ToRunes([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xf4, 0x90, 0x80, 0x80]), "110000"); + Expect.listEquals( + [0x10ffff], utf8ToRunes([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf4, 0x90, 0x80, 0x80]), "110000"); // unexpected continuation bytes Expect.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0x80]), @@ -795,12 +799,15 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { // Sequences with last continuation byte missing Expect.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xc2]), "2-byte sequence with last byte missing"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xe0, 0x80]), "3-byte sequence with last byte missing"); Expect.listEquals( - [unicodeReplacementCharacterRune], - utf8ToRunes([0xf0, 0x80, 0x80]), - "4-byte sequence with last byte missing"); + [unicodeReplacementCharacterRune, unicodeReplacementCharacterRune], + utf8ToRunes([0xe0, 0x80]), + "3-byte sequence with last byte missing"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf0, 0x80, 0x80]), "4-byte sequence with last byte missing"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, @@ -871,6 +878,9 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, unicodeReplacementCharacterRune ], utf8ToRunes([ @@ -890,8 +900,8 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { 0x80, 0x80, 0xdf, - 0xef, - 0xbf, + 0xef, // These two bytes form one incomplete sequence. + 0xbf, // All others form one per byte. 0xf7, 0xbf, 0xbf, @@ -908,10 +918,10 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { "Concatenation of incomplete sequences"); // Impossible bytes - Expect - .listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xfe]), "fe"); - Expect - .listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xff]), "ff"); + Expect.listEquals( + [unicodeReplacementCharacterRune], utf8ToRunes([0xfe]), "fe"); + Expect.listEquals( + [unicodeReplacementCharacterRune], utf8ToRunes([0xff]), "ff"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, @@ -921,11 +931,20 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { // Overlong sequences Expect.listEquals( - [unicodeReplacementCharacterRune], utf8ToRunes([0xc0, 0xaf]), "c0 af"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xe0, 0x80, 0xaf]), "e0 80 af"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af"); + [unicodeReplacementCharacterRune, unicodeReplacementCharacterRune], + utf8ToRunes([0xc0, 0xaf]), + "c0 af"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xe0, 0x80, 0xaf]), "e0 80 af"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, @@ -943,11 +962,20 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { ], utf8ToRunes([0xfc, 0x80, 0x80, 0x80, 0x80, 0xaf]), "fc 80 80 80 80 af"); Expect.listEquals( - [unicodeReplacementCharacterRune], utf8ToRunes([0xc1, 0xbf]), "c1 bf"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xe0, 0x9f, 0xbf]), "e0 9f bf"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf"); + [unicodeReplacementCharacterRune, unicodeReplacementCharacterRune], + utf8ToRunes([0xc1, 0xbf]), + "c1 bf"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xe0, 0x9f, 0xbf]), "e0 9f bf"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, @@ -965,11 +993,20 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { ], utf8ToRunes([0xfc, 0x83, 0xbf, 0xbf, 0xbf, 0xbf]), "fc 83 bf bf bf bf"); Expect.listEquals( - [unicodeReplacementCharacterRune], utf8ToRunes([0xc0, 0x80]), "c0 80"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xe0, 0x80, 0x80]), "e0 80 80"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80"); + [unicodeReplacementCharacterRune, unicodeReplacementCharacterRune], + utf8ToRunes([0xc0, 0x80]), + "c0 80"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xe0, 0x80, 0x80]), "e0 80 80"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, diff --git a/tests/lib/convert/utf85_test.dart b/tests/lib/convert/utf85_test.dart index 3f287e94762..a3809918a8f 100644 --- a/tests/lib/convert/utf85_test.dart +++ b/tests/lib/convert/utf85_test.dart @@ -9,7 +9,7 @@ import 'dart:convert'; main() { for (int i = 0; i <= 0x10FFFF; i++) { - if (i == unicodeBomCharacterRune) continue; + if (i == unicodeBomCharacterRune || (i & 0x1FF800) == 0xD800) continue; Expect.equals( i, utf8.decode(utf8.encode(new String.fromCharCode(i))).runes.first); } diff --git a/tests/lib/convert/utf8_encode_test.dart b/tests/lib/convert/utf8_encode_test.dart index 1e611bb575b..0a43e1452f1 100644 --- a/tests/lib/convert/utf8_encode_test.dart +++ b/tests/lib/convert/utf8_encode_test.dart @@ -25,8 +25,8 @@ void testEncodeSlice() { String ascii = "ABCDE"; Expect.listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii)); Expect.listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0)); - Expect - .listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0, 5)); + Expect.listEquals( + [0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0, 5)); Expect.listEquals([0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 1)); Expect.listEquals([0x41, 0x42, 0x43, 0x44], encoder.convert(ascii, 0, 4)); Expect.listEquals([0x42, 0x43, 0x44], encoder.convert(ascii, 1, 4)); @@ -50,6 +50,6 @@ void testEncodeSlice() { Expect.listEquals( [0xc2, 0x82, 0xe1, 0x81, 0x81], encoder.convert(unicode, 1, 3)); // Split in the middle of a surrogate pair. - Expect.listEquals([0xc2, 0x82, 0xe1, 0x81, 0x81, 0xed, 0xa0, 0x80], + Expect.listEquals([0xc2, 0x82, 0xe1, 0x81, 0x81, 0xef, 0xbf, 0xbd], encoder.convert(unicode, 1, 4)); } diff --git a/tests/lib/convert/utf8_test.dart b/tests/lib/convert/utf8_test.dart index 858c982ef22..6f97fd06770 100644 --- a/tests/lib/convert/utf8_test.dart +++ b/tests/lib/convert/utf8_test.dart @@ -73,28 +73,29 @@ void testErrorOffset() { // Bad encoding, points to first bad byte. testExn([0x80, 0x00], 0); - testExn([0xC0, 0x00], 1); - testExn([0xE0, 0x00], 1); - testExn([0xE0, 0x80, 0x00], 2); - testExn([0xF0, 0x00], 1); - testExn([0xF0, 0x80, 0x00], 2); - testExn([0xF0, 0x80, 0x80, 0x00], 3); + testExn([0xC2, 0x00], 1); + testExn([0xE2, 0x00], 1); + testExn([0xE2, 0x80, 0x00], 2); + testExn([0xF2, 0x00], 1); + testExn([0xF2, 0x80, 0x00], 2); + testExn([0xF2, 0x80, 0x80, 0x00], 3); testExn([0xF8, 0x00], 0); // Short encoding, points to end. - testExn([0xC0], 1); - testExn([0xE0], 1); - testExn([0xE0, 0x80], 2); - testExn([0xF0], 1); - testExn([0xF0, 0x80], 2); - testExn([0xF0, 0x80, 0x80], 3); - // Overlong encoding, points to start of encoding. + testExn([0xC2], 1); + testExn([0xE2], 1); + testExn([0xE2, 0x80], 2); + testExn([0xF2], 1); + testExn([0xF2, 0x80], 2); + testExn([0xF2, 0x80, 0x80], 3); + // Overlong encoding, points to byte that gave enough information to conclude + // that it was overlong. testExn([0xC0, 0x80], 0); testExn([0xC1, 0xBF], 0); - testExn([0xE0, 0x80, 0x80], 0); - testExn([0xE0, 0x9F, 0xBF], 0); - testExn([0xF0, 0x80, 0x80, 0x80], 0); - testExn([0xF0, 0x8F, 0xBF, 0xBF], 0); + testExn([0xE0, 0x80, 0x80], 1); + testExn([0xE0, 0x9F, 0xBF], 1); + testExn([0xF0, 0x80, 0x80, 0x80], 1); + testExn([0xF0, 0x8F, 0xBF, 0xBF], 1); // Invalid character (value too large, over 0x10FFFF). - testExn([0xF4, 0x90, 0x80, 0x80], 0); + testExn([0xF4, 0x90, 0x80, 0x80], 1); testExn([0xF7, 0xBF, 0xBF, 0xBF], 0); } diff --git a/tests/lib_2/convert/chunked_conversion_utf82_test.dart b/tests/lib_2/convert/chunked_conversion_utf82_test.dart index bc0b200b0c2..2e07f87f447 100644 --- a/tests/lib_2/convert/chunked_conversion_utf82_test.dart +++ b/tests/lib_2/convert/chunked_conversion_utf82_test.dart @@ -46,11 +46,14 @@ String decodeAllowMalformed(List bytes, int chunkSize) { return buffer.toString(); } -final TESTS = [ +final TESTS0 = [ // Unfinished UTF-8 sequences. [0xc3], [0xE2, 0x82], - [0xF0, 0xA4, 0xAD], + [0xF0, 0xA4, 0xAD] +]; + +final TESTS1 = [ // Overlong encoding of euro-sign. [0xF0, 0x82, 0x82, 0xAC], // Other overlong/unfinished sequences. @@ -77,11 +80,11 @@ final TESTS2 = [ // Test that 0xC0|1, 0x80 does not eat the next character. [ [0xC0, 0x80, 0x61], - "Xa" + "XXa" ], [ [0xC1, 0x80, 0x61], - "Xa" + "XXa" ], // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. [ @@ -220,44 +223,49 @@ final TESTS2 = [ ]; main() { - var allTests = TESTS.expand((test) { + var allTests = [...TESTS0, ...TESTS1].expand((test) { // Pairs of test and expected string output when malformed strings are - // allowed. Replacement character: U+FFFD + // allowed. Replacement character: U+FFFD, one per unfinished sequence or + // undecodable byte. + String replacement = + TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length; return [ - [test, "\u{FFFD}"], + [test, "${replacement}"], [ - new List.from([0x61])..addAll(test), - "a\u{FFFD}" + [0x61, ...test], + "a${replacement}" ], [ - new List.from([0x61]) - ..addAll(test) - ..add(0x61), - "a\u{FFFD}a" - ], - [new List.from(test)..add(0x61), "\u{FFFD}a"], - [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], - [ - new List.from(test) - ..add(0x61) - ..addAll(test), - "\u{FFFD}a\u{FFFD}" + [0x61, ...test, 0x61], + "a${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test), - "å\u{FFFD}" + [...test, 0x61], + "${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), - "å\u{FFFD}å" + [...test, ...test], + "${replacement}${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5]), - "\u{FFFD}å" + [...test, 0x61, ...test], + "${replacement}a${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), - "\u{FFFD}å\u{FFFD}" + [0xc3, 0xa5, ...test], + "å${replacement}" + ], + [ + [0xc3, 0xa5, ...test, 0xc3, 0xa5], + "å${replacement}å" + ], + [ + [...test, 0xc3, 0xa5], + "${replacement}å" + ], + [ + [...test, 0xc3, 0xa5, ...test], + "${replacement}å${replacement}" ] ]; }); diff --git a/tests/lib_2/convert/chunked_conversion_utf86_test.dart b/tests/lib_2/convert/chunked_conversion_utf86_test.dart index a42e7523334..8c9a08c39fd 100644 --- a/tests/lib_2/convert/chunked_conversion_utf86_test.dart +++ b/tests/lib_2/convert/chunked_conversion_utf86_test.dart @@ -28,11 +28,14 @@ String decodeAllowMalformed(List bytes) { return buffer.toString(); } -final TESTS = [ +final TESTS0 = [ // Unfinished UTF-8 sequences. [0xc3], [0xE2, 0x82], - [0xF0, 0xA4, 0xAD], + [0xF0, 0xA4, 0xAD] +]; + +final TESTS1 = [ // Overlong encoding of euro-sign. [0xF0, 0x82, 0x82, 0xAC], // Other overlong/unfinished sequences. @@ -59,11 +62,11 @@ final TESTS2 = [ // Test that 0xC0|1, 0x80 does not eat the next character. [ [0xC0, 0x80, 0x61], - "Xa" + "XXa" ], [ [0xC1, 0x80, 0x61], - "Xa" + "XXa" ], // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. [ @@ -202,44 +205,49 @@ final TESTS2 = [ ]; main() { - var allTests = TESTS.expand((test) { + var allTests = [...TESTS0, ...TESTS1].expand((test) { // Pairs of test and expected string output when malformed strings are - // allowed. Replacement character: U+FFFD + // allowed. Replacement character: U+FFFD, one per unfinished sequence or + // undecodable byte. + String replacement = + TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length; return [ - [test, "\u{FFFD}"], + [test, "${replacement}"], [ - new List.from([0x61])..addAll(test), - "a\u{FFFD}" + [0x61, ...test], + "a${replacement}" ], [ - new List.from([0x61]) - ..addAll(test) - ..add(0x61), - "a\u{FFFD}a" - ], - [new List.from(test)..add(0x61), "\u{FFFD}a"], - [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], - [ - new List.from(test) - ..add(0x61) - ..addAll(test), - "\u{FFFD}a\u{FFFD}" + [0x61, ...test, 0x61], + "a${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test), - "å\u{FFFD}" + [...test, 0x61], + "${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), - "å\u{FFFD}å" + [...test, ...test], + "${replacement}${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5]), - "\u{FFFD}å" + [...test, 0x61, ...test], + "${replacement}a${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), - "\u{FFFD}å\u{FFFD}" + [0xc3, 0xa5, ...test], + "å${replacement}" + ], + [ + [0xc3, 0xa5, ...test, 0xc3, 0xa5], + "å${replacement}å" + ], + [ + [...test, 0xc3, 0xa5], + "${replacement}å" + ], + [ + [...test, 0xc3, 0xa5, ...test], + "${replacement}å${replacement}" ] ]; }); diff --git a/tests/lib_2/convert/chunked_conversion_utf87_test.dart b/tests/lib_2/convert/chunked_conversion_utf87_test.dart index 97946902dc9..67929af0b99 100644 --- a/tests/lib_2/convert/chunked_conversion_utf87_test.dart +++ b/tests/lib_2/convert/chunked_conversion_utf87_test.dart @@ -49,11 +49,14 @@ String decodeAllowMalformed2(List inputBytes) { return utf8.decode(bytes); } -final TESTS = [ +final TESTS0 = [ // Unfinished UTF-8 sequences. [0xc3], [0xE2, 0x82], - [0xF0, 0xA4, 0xAD], + [0xF0, 0xA4, 0xAD] +]; + +final TESTS1 = [ // Overlong encoding of euro-sign. [0xF0, 0x82, 0x82, 0xAC], // Other overlong/unfinished sequences. @@ -80,11 +83,11 @@ final TESTS2 = [ // Test that 0xC0|1, 0x80 does not eat the next character. [ [0xC0, 0x80, 0x61], - "Xa" + "XXa" ], [ [0xC1, 0x80, 0x61], - "Xa" + "XXa" ], // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. [ @@ -223,44 +226,49 @@ final TESTS2 = [ ]; main() { - var allTests = TESTS.expand((test) { + var allTests = [...TESTS0, ...TESTS1].expand((test) { // Pairs of test and expected string output when malformed strings are - // allowed. Replacement character: U+FFFD + // allowed. Replacement character: U+FFFD, one per unfinished sequence or + // undecodable byte. + String replacement = + TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length; return [ - [test, "\u{FFFD}"], + [test, "${replacement}"], [ - new List.from([0x61])..addAll(test), - "a\u{FFFD}" + [0x61, ...test], + "a${replacement}" ], [ - new List.from([0x61]) - ..addAll(test) - ..add(0x61), - "a\u{FFFD}a" - ], - [new List.from(test)..add(0x61), "\u{FFFD}a"], - [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], - [ - new List.from(test) - ..add(0x61) - ..addAll(test), - "\u{FFFD}a\u{FFFD}" + [0x61, ...test, 0x61], + "a${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test), - "å\u{FFFD}" + [...test, 0x61], + "${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), - "å\u{FFFD}å" + [...test, ...test], + "${replacement}${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5]), - "\u{FFFD}å" + [...test, 0x61, ...test], + "${replacement}a${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), - "\u{FFFD}å\u{FFFD}" + [0xc3, 0xa5, ...test], + "å${replacement}" + ], + [ + [0xc3, 0xa5, ...test, 0xc3, 0xa5], + "å${replacement}å" + ], + [ + [...test, 0xc3, 0xa5], + "${replacement}å" + ], + [ + [...test, 0xc3, 0xa5, ...test], + "${replacement}å${replacement}" ] ]; }); diff --git a/tests/lib_2/convert/chunked_conversion_utf88_test.dart b/tests/lib_2/convert/chunked_conversion_utf88_test.dart index 58cecaa685e..8b19b2a15d2 100644 --- a/tests/lib_2/convert/chunked_conversion_utf88_test.dart +++ b/tests/lib_2/convert/chunked_conversion_utf88_test.dart @@ -112,8 +112,7 @@ main() { const LEADING_SURROGATE = 0xd801; const TRAILING_SURROGATE = 0xdc12; const UTF8_ENCODING = const [0xf0, 0x90, 0x90, 0x92]; - const UTF8_LEADING = const [0xed, 0xa0, 0x81]; - const UTF8_TRAILING = const [0xed, 0xb0, 0x92]; + const UTF8_REPLACEMENT = const [0xef, 0xbf, 0xbd]; const CHAR_A = 0x61; // Test surrogates at all kinds of locations. @@ -129,17 +128,17 @@ main() { codeUnits[i] = LEADING_SURROGATE; var str = new String.fromCharCodes(codeUnits); var bytes = new List.filled(i + 3, CHAR_A); - bytes[i] = UTF8_LEADING[0]; - bytes[i + 1] = UTF8_LEADING[1]; - bytes[i + 2] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = TRAILING_SURROGATE; str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 3, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits.length = i + 2; @@ -157,36 +156,36 @@ main() { codeUnits[i + 1] = TRAILING_SURROGATE; str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 6, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; - bytes[i + 3] = UTF8_TRAILING[0]; - bytes[i + 4] = UTF8_TRAILING[1]; - bytes[i + 5] = UTF8_TRAILING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = LEADING_SURROGATE; codeUnits[i + 1] = LEADING_SURROGATE; str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 6, CHAR_A); - bytes[i] = UTF8_LEADING[0]; - bytes[i + 1] = UTF8_LEADING[1]; - bytes[i + 2] = UTF8_LEADING[2]; - bytes[i + 3] = UTF8_LEADING[0]; - bytes[i + 4] = UTF8_LEADING[1]; - bytes[i + 5] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = TRAILING_SURROGATE; codeUnits[i + 1] = LEADING_SURROGATE; str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 6, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; - bytes[i + 3] = UTF8_LEADING[0]; - bytes[i + 4] = UTF8_LEADING[1]; - bytes[i + 5] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits.length = i + 3; @@ -208,12 +207,12 @@ main() { codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 7, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; - bytes[i + 3] = UTF8_TRAILING[0]; - bytes[i + 4] = UTF8_TRAILING[1]; - bytes[i + 5] = UTF8_TRAILING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = LEADING_SURROGATE; @@ -221,12 +220,12 @@ main() { codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 7, CHAR_A); - bytes[i] = UTF8_LEADING[0]; - bytes[i + 1] = UTF8_LEADING[1]; - bytes[i + 2] = UTF8_LEADING[2]; - bytes[i + 3] = UTF8_LEADING[0]; - bytes[i + 4] = UTF8_LEADING[1]; - bytes[i + 5] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); codeUnits[i] = TRAILING_SURROGATE; @@ -234,12 +233,12 @@ main() { codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. str = new String.fromCharCodes(codeUnits); bytes = new List.filled(i + 7, CHAR_A); - bytes[i] = UTF8_TRAILING[0]; - bytes[i + 1] = UTF8_TRAILING[1]; - bytes[i + 2] = UTF8_TRAILING[2]; - bytes[i + 3] = UTF8_LEADING[0]; - bytes[i + 4] = UTF8_LEADING[1]; - bytes[i + 5] = UTF8_LEADING[2]; + bytes[i] = UTF8_REPLACEMENT[0]; + bytes[i + 1] = UTF8_REPLACEMENT[1]; + bytes[i + 2] = UTF8_REPLACEMENT[2]; + bytes[i + 3] = UTF8_REPLACEMENT[0]; + bytes[i + 4] = UTF8_REPLACEMENT[1]; + bytes[i + 5] = UTF8_REPLACEMENT[2]; runTest([bytes, str]); // Make sure the invariant is correct. diff --git a/tests/lib_2/convert/json_utf8_chunk_test.dart b/tests/lib_2/convert/json_utf8_chunk_test.dart index ec47339ba32..cf5064ed823 100644 --- a/tests/lib_2/convert/json_utf8_chunk_test.dart +++ b/tests/lib_2/convert/json_utf8_chunk_test.dart @@ -284,69 +284,69 @@ void jsonThrows(String name, String codeString) { void testMalformed() { // Overlong encodings. jsonMalformedTest( - "overlong-0-2", "@\uFFFD@", [0x22, 0x40, 0xc0, 0x80, 0x40, 0x22]); - jsonMalformedTest( - "overlong-0-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x80, 0x80, 0x40, 0x22]); - jsonMalformedTest("overlong-0-4", "@\uFFFD@", + "overlong-0-2", "@\uFFFD\uFFFD@", [0x22, 0x40, 0xc0, 0x80, 0x40, 0x22]); + jsonMalformedTest("overlong-0-3", "@\uFFFD\uFFFD\uFFFD@", + [0x22, 0x40, 0xe0, 0x80, 0x80, 0x40, 0x22]); + jsonMalformedTest("overlong-0-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x80, 0x40, 0x22]); jsonMalformedTest( - "overlong-7f-2", "@\uFFFD@", [0x22, 0x40, 0xc1, 0xbf, 0x40, 0x22]); - jsonMalformedTest( - "overlong-7f-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x81, 0xbf, 0x40, 0x22]); - jsonMalformedTest("overlong-7f-4", "@\uFFFD@", + "overlong-7f-2", "@\uFFFD\uFFFD@", [0x22, 0x40, 0xc1, 0xbf, 0x40, 0x22]); + jsonMalformedTest("overlong-7f-3", "@\uFFFD\uFFFD\uFFFD@", + [0x22, 0x40, 0xe0, 0x81, 0xbf, 0x40, 0x22]); + jsonMalformedTest("overlong-7f-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x81, 0xbf, 0x40, 0x22]); - jsonMalformedTest( - "overlong-80-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x82, 0x80, 0x40, 0x22]); - jsonMalformedTest("overlong-80-4", "@\uFFFD@", + jsonMalformedTest("overlong-80-3", "@\uFFFD\uFFFD\uFFFD@", + [0x22, 0x40, 0xe0, 0x82, 0x80, 0x40, 0x22]); + jsonMalformedTest("overlong-80-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x82, 0x80, 0x40, 0x22]); - jsonMalformedTest( - "overlong-7ff-3", "@\uFFFD@", [0x22, 0x40, 0xe0, 0x9f, 0xbf, 0x40, 0x22]); - jsonMalformedTest("overlong-7ff-4", "@\uFFFD@", + jsonMalformedTest("overlong-7ff-3", "@\uFFFD\uFFFD\uFFFD@", + [0x22, 0x40, 0xe0, 0x9f, 0xbf, 0x40, 0x22]); + jsonMalformedTest("overlong-7ff-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x9f, 0xbf, 0x40, 0x22]); - jsonMalformedTest("overlong-800-4", "@\uFFFD@", + jsonMalformedTest("overlong-800-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0xa0, 0x80, 0x40, 0x22]); - jsonMalformedTest("overlong-ffff-4", "@\uFFFD@", + jsonMalformedTest("overlong-ffff-4", "@\uFFFD\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x8f, 0xbf, 0xbf, 0x40, 0x22]); // Unterminated multibyte sequences. jsonMalformedTest( "unterminated-2-normal", "@\uFFFD@", [0x22, 0x40, 0xc0, 0x40, 0x22]); - jsonMalformedTest("unterminated-3-normal", "@\uFFFD@", + jsonMalformedTest("unterminated-3-normal", "@\uFFFD\uFFFD@", [0x22, 0x40, 0xe0, 0x80, 0x40, 0x22]); - jsonMalformedTest("unterminated-4-normal", "@\uFFFD@", + jsonMalformedTest("unterminated-4-normal", "@\uFFFD\uFFFD\uFFFD@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x40, 0x22]); jsonMalformedTest("unterminated-2-multi", "@\uFFFD\x80@", [0x22, 0x40, 0xc0, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("unterminated-3-multi", "@\uFFFD\x80@", + jsonMalformedTest("unterminated-3-multi", "@\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xe0, 0x80, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("unterminated-4-multi", "@\uFFFD\x80@", + jsonMalformedTest("unterminated-4-multi", "@\uFFFD\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0xc2, 0x80, 0x40, 0x22]); jsonMalformedTest("unterminated-2-escape", "@\uFFFD\n@", [0x22, 0x40, 0xc0, 0x5c, 0x6e, 0x40, 0x22]); - jsonMalformedTest("unterminated-3-escape", "@\uFFFD\n@", + jsonMalformedTest("unterminated-3-escape", "@\uFFFD\uFFFD\n@", [0x22, 0x40, 0xe0, 0x80, 0x5c, 0x6e, 0x40, 0x22]); - jsonMalformedTest("unterminated-4-escape", "@\uFFFD\n@", + jsonMalformedTest("unterminated-4-escape", "@\uFFFD\uFFFD\uFFFD\n@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x5c, 0x6e, 0x40, 0x22]); jsonMalformedTest("unterminated-2-end", "@\uFFFD", [0x22, 0x40, 0xc0, 0x22]); jsonMalformedTest( - "unterminated-3-end", "@\uFFFD", [0x22, 0x40, 0xe0, 0x80, 0x22]); + "unterminated-3-end", "@\uFFFD\uFFFD", [0x22, 0x40, 0xe0, 0x80, 0x22]); - jsonMalformedTest( - "unterminated-4-end", "@\uFFFD", [0x22, 0x40, 0xf0, 0x80, 0x80, 0x22]); + jsonMalformedTest("unterminated-4-end", "@\uFFFD\uFFFD\uFFFD", + [0x22, 0x40, 0xf0, 0x80, 0x80, 0x22]); // Unexpected continuation byte // - after a normal character. @@ -372,13 +372,13 @@ void testMalformed() { "leading-2", "@\uFFFD\x80@", [0x22, 0x40, 0xc0, 0xc2, 0x80, 0x40, 0x22]); jsonMalformedTest("leading-3-1", "@\uFFFD\x80@", [0x22, 0x40, 0xe0, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("leading-3-2", "@\uFFFD\x80@", + jsonMalformedTest("leading-3-2", "@\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xe0, 0x80, 0xc2, 0x80, 0x40, 0x22]); jsonMalformedTest("leading-4-1", "@\uFFFD\x80@", [0x22, 0x40, 0xf0, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("leading-4-2", "@\uFFFD\x80@", + jsonMalformedTest("leading-4-2", "@\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xf0, 0x80, 0xc2, 0x80, 0x40, 0x22]); - jsonMalformedTest("leading-4-3", "@\uFFFD\x80@", + jsonMalformedTest("leading-4-3", "@\uFFFD\uFFFD\uFFFD\x80@", [0x22, 0x40, 0xf0, 0x80, 0x80, 0xc2, 0x80, 0x40, 0x22]); // Overlong encodings of ASCII outside of strings always fail. diff --git a/tests/lib_2/convert/utf82_test.dart b/tests/lib_2/convert/utf82_test.dart index df08103e86d..cf0c2b0585e 100644 --- a/tests/lib_2/convert/utf82_test.dart +++ b/tests/lib_2/convert/utf82_test.dart @@ -28,11 +28,14 @@ String decodeAllowMalformed4(List bytes) { return new Utf8Codec(allowMalformed: true).decoder.convert(bytes); } -final TESTS = [ +final TESTS0 = [ // Unfinished UTF-8 sequences. [0xc3], [0xE2, 0x82], - [0xF0, 0xA4, 0xAD], + [0xF0, 0xA4, 0xAD] +]; + +final TESTS1 = [ // Overlong encoding of euro-sign. [0xF0, 0x82, 0x82, 0xAC], // Other overlong/unfinished sequences. @@ -64,11 +67,11 @@ final TESTS2 = [ // Test that 0xC0|1, 0x80 does not eat the next character. [ [0xC0, 0x80, 0x61], - "Xa" + "XXa" ], [ [0xC1, 0x80, 0x61], - "Xa" + "XXa" ], // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. [ @@ -207,44 +210,49 @@ final TESTS2 = [ ]; main() { - var allTests = TESTS.expand((test) { + var allTests = [...TESTS0, ...TESTS1].expand((test) { // Pairs of test and expected string output when malformed strings are - // allowed. Replacement character: U+FFFD + // allowed. Replacement character: U+FFFD, one per unfinished sequence or + // undecodable byte. + String replacement = + TESTS0.contains(test) ? "\u{FFFD}" : "\u{FFFD}" * test.length; return [ - [test, "\u{FFFD}"], + [test, "${replacement}"], [ - new List.from([0x61])..addAll(test), - "a\u{FFFD}" + [0x61, ...test], + "a${replacement}" ], [ - new List.from([0x61]) - ..addAll(test) - ..add(0x61), - "a\u{FFFD}a" - ], - [new List.from(test)..add(0x61), "\u{FFFD}a"], - [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], - [ - new List.from(test) - ..add(0x61) - ..addAll(test), - "\u{FFFD}a\u{FFFD}" + [0x61, ...test, 0x61], + "a${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test), - "å\u{FFFD}" + [...test, 0x61], + "${replacement}a" ], [ - new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), - "å\u{FFFD}å" + [...test, ...test], + "${replacement}${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5]), - "\u{FFFD}å" + [...test, 0x61, ...test], + "${replacement}a${replacement}" ], [ - new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), - "\u{FFFD}å\u{FFFD}" + [0xc3, 0xa5, ...test], + "å${replacement}" + ], + [ + [0xc3, 0xa5, ...test, 0xc3, 0xa5], + "å${replacement}å" + ], + [ + [...test, 0xc3, 0xa5], + "${replacement}å" + ], + [ + [...test, 0xc3, 0xa5, ...test], + "${replacement}å${replacement}" ] ]; }); diff --git a/tests/lib_2/convert/utf84_test.dart b/tests/lib_2/convert/utf84_test.dart index bc1db59a004..566b1fa54bd 100755 --- a/tests/lib_2/convert/utf84_test.dart +++ b/tests/lib_2/convert/utf84_test.dart @@ -727,10 +727,14 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { Expect.listEquals([0xe000], utf8ToRunes([0xee, 0x80, 0x80]), "e000"); Expect.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xef, 0xbf, 0xbd]), "fffd"); - Expect - .listEquals([0x10ffff], utf8ToRunes([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xf4, 0x90, 0x80, 0x80]), "110000"); + Expect.listEquals( + [0x10ffff], utf8ToRunes([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf4, 0x90, 0x80, 0x80]), "110000"); // unexpected continuation bytes Expect.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0x80]), @@ -795,12 +799,15 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { // Sequences with last continuation byte missing Expect.listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xc2]), "2-byte sequence with last byte missing"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xe0, 0x80]), "3-byte sequence with last byte missing"); Expect.listEquals( - [unicodeReplacementCharacterRune], - utf8ToRunes([0xf0, 0x80, 0x80]), - "4-byte sequence with last byte missing"); + [unicodeReplacementCharacterRune, unicodeReplacementCharacterRune], + utf8ToRunes([0xe0, 0x80]), + "3-byte sequence with last byte missing"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf0, 0x80, 0x80]), "4-byte sequence with last byte missing"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, @@ -871,6 +878,9 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, unicodeReplacementCharacterRune ], utf8ToRunes([ @@ -890,8 +900,8 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { 0x80, 0x80, 0xdf, - 0xef, - 0xbf, + 0xef, // These two bytes form one incomplete sequence. + 0xbf, // All others form one per byte. 0xf7, 0xbf, 0xbf, @@ -908,10 +918,10 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { "Concatenation of incomplete sequences"); // Impossible bytes - Expect - .listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xfe]), "fe"); - Expect - .listEquals([unicodeReplacementCharacterRune], utf8ToRunes([0xff]), "ff"); + Expect.listEquals( + [unicodeReplacementCharacterRune], utf8ToRunes([0xfe]), "fe"); + Expect.listEquals( + [unicodeReplacementCharacterRune], utf8ToRunes([0xff]), "ff"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, @@ -921,11 +931,20 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { // Overlong sequences Expect.listEquals( - [unicodeReplacementCharacterRune], utf8ToRunes([0xc0, 0xaf]), "c0 af"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xe0, 0x80, 0xaf]), "e0 80 af"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af"); + [unicodeReplacementCharacterRune, unicodeReplacementCharacterRune], + utf8ToRunes([0xc0, 0xaf]), + "c0 af"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xe0, 0x80, 0xaf]), "e0 80 af"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, @@ -943,11 +962,20 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { ], utf8ToRunes([0xfc, 0x80, 0x80, 0x80, 0x80, 0xaf]), "fc 80 80 80 80 af"); Expect.listEquals( - [unicodeReplacementCharacterRune], utf8ToRunes([0xc1, 0xbf]), "c1 bf"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xe0, 0x9f, 0xbf]), "e0 9f bf"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf"); + [unicodeReplacementCharacterRune, unicodeReplacementCharacterRune], + utf8ToRunes([0xc1, 0xbf]), + "c1 bf"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xe0, 0x9f, 0xbf]), "e0 9f bf"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, @@ -965,11 +993,20 @@ void testUtf8bytesToCodepoints(List utf8ToRunes(List utf8)) { ], utf8ToRunes([0xfc, 0x83, 0xbf, 0xbf, 0xbf, 0xbf]), "fc 83 bf bf bf bf"); Expect.listEquals( - [unicodeReplacementCharacterRune], utf8ToRunes([0xc0, 0x80]), "c0 80"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xe0, 0x80, 0x80]), "e0 80 80"); - Expect.listEquals([unicodeReplacementCharacterRune], - utf8ToRunes([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80"); + [unicodeReplacementCharacterRune, unicodeReplacementCharacterRune], + utf8ToRunes([0xc0, 0x80]), + "c0 80"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xe0, 0x80, 0x80]), "e0 80 80"); + Expect.listEquals([ + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune, + unicodeReplacementCharacterRune + ], utf8ToRunes([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80"); Expect.listEquals([ unicodeReplacementCharacterRune, unicodeReplacementCharacterRune, diff --git a/tests/lib_2/convert/utf85_test.dart b/tests/lib_2/convert/utf85_test.dart index 3f287e94762..a3809918a8f 100644 --- a/tests/lib_2/convert/utf85_test.dart +++ b/tests/lib_2/convert/utf85_test.dart @@ -9,7 +9,7 @@ import 'dart:convert'; main() { for (int i = 0; i <= 0x10FFFF; i++) { - if (i == unicodeBomCharacterRune) continue; + if (i == unicodeBomCharacterRune || (i & 0x1FF800) == 0xD800) continue; Expect.equals( i, utf8.decode(utf8.encode(new String.fromCharCode(i))).runes.first); } diff --git a/tests/lib_2/convert/utf8_encode_test.dart b/tests/lib_2/convert/utf8_encode_test.dart index 1e611bb575b..0a43e1452f1 100644 --- a/tests/lib_2/convert/utf8_encode_test.dart +++ b/tests/lib_2/convert/utf8_encode_test.dart @@ -25,8 +25,8 @@ void testEncodeSlice() { String ascii = "ABCDE"; Expect.listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii)); Expect.listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0)); - Expect - .listEquals([0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0, 5)); + Expect.listEquals( + [0x41, 0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 0, 5)); Expect.listEquals([0x42, 0x43, 0x44, 0x45], encoder.convert(ascii, 1)); Expect.listEquals([0x41, 0x42, 0x43, 0x44], encoder.convert(ascii, 0, 4)); Expect.listEquals([0x42, 0x43, 0x44], encoder.convert(ascii, 1, 4)); @@ -50,6 +50,6 @@ void testEncodeSlice() { Expect.listEquals( [0xc2, 0x82, 0xe1, 0x81, 0x81], encoder.convert(unicode, 1, 3)); // Split in the middle of a surrogate pair. - Expect.listEquals([0xc2, 0x82, 0xe1, 0x81, 0x81, 0xed, 0xa0, 0x80], + Expect.listEquals([0xc2, 0x82, 0xe1, 0x81, 0x81, 0xef, 0xbf, 0xbd], encoder.convert(unicode, 1, 4)); } diff --git a/tests/lib_2/convert/utf8_test.dart b/tests/lib_2/convert/utf8_test.dart index 858c982ef22..6f97fd06770 100644 --- a/tests/lib_2/convert/utf8_test.dart +++ b/tests/lib_2/convert/utf8_test.dart @@ -73,28 +73,29 @@ void testErrorOffset() { // Bad encoding, points to first bad byte. testExn([0x80, 0x00], 0); - testExn([0xC0, 0x00], 1); - testExn([0xE0, 0x00], 1); - testExn([0xE0, 0x80, 0x00], 2); - testExn([0xF0, 0x00], 1); - testExn([0xF0, 0x80, 0x00], 2); - testExn([0xF0, 0x80, 0x80, 0x00], 3); + testExn([0xC2, 0x00], 1); + testExn([0xE2, 0x00], 1); + testExn([0xE2, 0x80, 0x00], 2); + testExn([0xF2, 0x00], 1); + testExn([0xF2, 0x80, 0x00], 2); + testExn([0xF2, 0x80, 0x80, 0x00], 3); testExn([0xF8, 0x00], 0); // Short encoding, points to end. - testExn([0xC0], 1); - testExn([0xE0], 1); - testExn([0xE0, 0x80], 2); - testExn([0xF0], 1); - testExn([0xF0, 0x80], 2); - testExn([0xF0, 0x80, 0x80], 3); - // Overlong encoding, points to start of encoding. + testExn([0xC2], 1); + testExn([0xE2], 1); + testExn([0xE2, 0x80], 2); + testExn([0xF2], 1); + testExn([0xF2, 0x80], 2); + testExn([0xF2, 0x80, 0x80], 3); + // Overlong encoding, points to byte that gave enough information to conclude + // that it was overlong. testExn([0xC0, 0x80], 0); testExn([0xC1, 0xBF], 0); - testExn([0xE0, 0x80, 0x80], 0); - testExn([0xE0, 0x9F, 0xBF], 0); - testExn([0xF0, 0x80, 0x80, 0x80], 0); - testExn([0xF0, 0x8F, 0xBF, 0xBF], 0); + testExn([0xE0, 0x80, 0x80], 1); + testExn([0xE0, 0x9F, 0xBF], 1); + testExn([0xF0, 0x80, 0x80, 0x80], 1); + testExn([0xF0, 0x8F, 0xBF, 0xBF], 1); // Invalid character (value too large, over 0x10FFFF). - testExn([0xF4, 0x90, 0x80, 0x80], 0); + testExn([0xF4, 0x90, 0x80, 0x80], 1); testExn([0xF7, 0xBF, 0xBF, 0xBF], 0); }