Aske Simon Christensen
f5bf50e7a4
This adjusts all UTF-8 tests to the new semantics in the breaking change described here: https://github.com/dart-lang/sdk/issues/41100 This has three parts: - Unpaired surrogates are encoded as replacement characters, and encoded surrogates are considered malformed input when decoding. - Decoding errors are generally reported on the position of the byte that conclusively makes the input malformed. - The number of replacement characters emitted by the decoder is generally one per unfinished sequence or undecodable byte. The code changes to implement the new semantics are placed in subsequent commits. Change-Id: I4cc8ce660e39287e734070764ab8e1f0ebb8b9e0 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/143815 Reviewed-by: Lasse R.H. Nielsen <lrn@google.com>
248 lines
8.2 KiB
Dart
248 lines
8.2 KiB
Dart
// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
|
|
library utf8_test;
|
|
|
|
import "package:expect/expect.dart";
|
|
import 'dart:convert';
|
|
|
|
List<int> encode(String str) {
|
|
List<int> bytes;
|
|
var byteSink =
|
|
new ByteConversionSink.withCallback((result) => bytes = result);
|
|
var stringConversionSink = new Utf8Encoder().startChunkedConversion(byteSink);
|
|
stringConversionSink.add(str);
|
|
stringConversionSink.close();
|
|
return bytes;
|
|
}
|
|
|
|
List<int> encode2(String str) {
|
|
List<int> bytes;
|
|
var byteSink =
|
|
new ByteConversionSink.withCallback((result) => bytes = result);
|
|
var stringConversionSink = new Utf8Encoder().startChunkedConversion(byteSink);
|
|
ClosableStringSink stringSink = stringConversionSink.asStringSink();
|
|
stringSink.write(str);
|
|
stringSink.close();
|
|
return bytes;
|
|
}
|
|
|
|
List<int> encode3(String str) {
|
|
List<int> bytes;
|
|
var byteSink =
|
|
new ByteConversionSink.withCallback((result) => bytes = result);
|
|
var stringConversionSink = new Utf8Encoder().startChunkedConversion(byteSink);
|
|
ClosableStringSink stringSink = stringConversionSink.asStringSink();
|
|
str.codeUnits.forEach(stringSink.writeCharCode);
|
|
stringSink.close();
|
|
return bytes;
|
|
}
|
|
|
|
List<int> encode4(String str) {
|
|
List<int> bytes;
|
|
var byteSink =
|
|
new ByteConversionSink.withCallback((result) => bytes = result);
|
|
var stringConversionSink = new Utf8Encoder().startChunkedConversion(byteSink);
|
|
ClosableStringSink stringSink = stringConversionSink.asStringSink();
|
|
str.runes.forEach(stringSink.writeCharCode);
|
|
stringSink.close();
|
|
return bytes;
|
|
}
|
|
|
|
List<int> encode5(String str) {
|
|
List<int> bytes;
|
|
var byteSink =
|
|
new ByteConversionSink.withCallback((result) => bytes = result);
|
|
var stringConversionSink = new Utf8Encoder().startChunkedConversion(byteSink);
|
|
ByteConversionSink inputByteSink = stringConversionSink.asUtf8Sink(false);
|
|
List<int> tmpBytes = utf8.encode(str);
|
|
inputByteSink.add(tmpBytes);
|
|
inputByteSink.close();
|
|
return bytes;
|
|
}
|
|
|
|
List<int> encode6(String str) {
|
|
List<int> bytes;
|
|
var byteSink =
|
|
new ByteConversionSink.withCallback((result) => bytes = result);
|
|
var stringConversionSink = new Utf8Encoder().startChunkedConversion(byteSink);
|
|
ByteConversionSink inputByteSink = stringConversionSink.asUtf8Sink(false);
|
|
List<int> tmpBytes = utf8.encode(str);
|
|
tmpBytes.forEach((b) => inputByteSink.addSlice([0, b, 1], 1, 2, false));
|
|
inputByteSink.close();
|
|
return bytes;
|
|
}
|
|
|
|
List<int> encode7(String str) {
|
|
List<int> bytes;
|
|
var byteSink =
|
|
new ByteConversionSink.withCallback((result) => bytes = result);
|
|
var stringConversionSink = new Utf8Encoder().startChunkedConversion(byteSink);
|
|
stringConversionSink.addSlice("1" + str + "2", 1, str.length + 1, false);
|
|
stringConversionSink.close();
|
|
return bytes;
|
|
}
|
|
|
|
int _nextPowerOf2(v) {
|
|
assert(v > 0);
|
|
v--;
|
|
v |= v >> 1;
|
|
v |= v >> 2;
|
|
v |= v >> 4;
|
|
v |= v >> 8;
|
|
v |= v >> 16;
|
|
v++;
|
|
return v;
|
|
}
|
|
|
|
runTest(test) {
|
|
List<int> bytes = test[0];
|
|
String string = test[1];
|
|
Expect.listEquals(bytes, encode(string));
|
|
Expect.listEquals(bytes, encode2(string));
|
|
Expect.listEquals(bytes, encode3(string));
|
|
Expect.listEquals(bytes, encode4(string));
|
|
Expect.listEquals(bytes, encode5(string));
|
|
Expect.listEquals(bytes, encode6(string));
|
|
Expect.listEquals(bytes, encode7(string));
|
|
}
|
|
|
|
main() {
|
|
const LEADING_SURROGATE = 0xd801;
|
|
const TRAILING_SURROGATE = 0xdc12;
|
|
const UTF8_ENCODING = const [0xf0, 0x90, 0x90, 0x92];
|
|
const UTF8_REPLACEMENT = const [0xef, 0xbf, 0xbd];
|
|
const CHAR_A = 0x61;
|
|
|
|
// Test surrogates at all kinds of locations.
|
|
var codeUnits = <int>[];
|
|
for (int i = 0; i < 2049; i++) {
|
|
// Invariant: codeUnits[0..i - 1] is filled with CHAR_A (character 'a').
|
|
codeUnits.length = i + 1;
|
|
codeUnits[i] = CHAR_A;
|
|
|
|
// Only test for problem zones, close to powers of two.
|
|
if (i > 20 && _nextPowerOf2(i - 2) - i > 10) continue;
|
|
|
|
codeUnits[i] = LEADING_SURROGATE;
|
|
var str = new String.fromCharCodes(codeUnits);
|
|
var bytes = new List.filled(i + 3, CHAR_A);
|
|
bytes[i] = UTF8_REPLACEMENT[0];
|
|
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
|
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
|
runTest([bytes, str]);
|
|
|
|
codeUnits[i] = TRAILING_SURROGATE;
|
|
str = new String.fromCharCodes(codeUnits);
|
|
bytes = new List.filled(i + 3, CHAR_A);
|
|
bytes[i] = UTF8_REPLACEMENT[0];
|
|
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
|
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
|
runTest([bytes, str]);
|
|
|
|
codeUnits.length = i + 2;
|
|
codeUnits[i] = LEADING_SURROGATE;
|
|
codeUnits[i + 1] = TRAILING_SURROGATE;
|
|
str = new String.fromCharCodes(codeUnits);
|
|
bytes = new List.filled(i + 4, CHAR_A);
|
|
bytes[i] = UTF8_ENCODING[0];
|
|
bytes[i + 1] = UTF8_ENCODING[1];
|
|
bytes[i + 2] = UTF8_ENCODING[2];
|
|
bytes[i + 3] = UTF8_ENCODING[3];
|
|
runTest([bytes, str]);
|
|
|
|
codeUnits[i] = TRAILING_SURROGATE;
|
|
codeUnits[i + 1] = TRAILING_SURROGATE;
|
|
str = new String.fromCharCodes(codeUnits);
|
|
bytes = new List.filled(i + 6, CHAR_A);
|
|
bytes[i] = UTF8_REPLACEMENT[0];
|
|
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
|
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
|
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
|
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
|
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
|
runTest([bytes, str]);
|
|
|
|
codeUnits[i] = LEADING_SURROGATE;
|
|
codeUnits[i + 1] = LEADING_SURROGATE;
|
|
str = new String.fromCharCodes(codeUnits);
|
|
bytes = new List.filled(i + 6, CHAR_A);
|
|
bytes[i] = UTF8_REPLACEMENT[0];
|
|
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
|
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
|
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
|
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
|
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
|
runTest([bytes, str]);
|
|
|
|
codeUnits[i] = TRAILING_SURROGATE;
|
|
codeUnits[i + 1] = LEADING_SURROGATE;
|
|
str = new String.fromCharCodes(codeUnits);
|
|
bytes = new List.filled(i + 6, CHAR_A);
|
|
bytes[i] = UTF8_REPLACEMENT[0];
|
|
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
|
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
|
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
|
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
|
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
|
runTest([bytes, str]);
|
|
|
|
codeUnits.length = i + 3;
|
|
codeUnits[i] = LEADING_SURROGATE;
|
|
codeUnits[i + 1] = TRAILING_SURROGATE;
|
|
codeUnits[i + 2] = CHAR_A; // Add trailing 'a'.
|
|
str = new String.fromCharCodes(codeUnits);
|
|
bytes = new List.filled(i + 5, CHAR_A);
|
|
bytes[i] = UTF8_ENCODING[0];
|
|
bytes[i + 1] = UTF8_ENCODING[1];
|
|
bytes[i + 2] = UTF8_ENCODING[2];
|
|
bytes[i + 3] = UTF8_ENCODING[3];
|
|
// No need to assign the 'a' character. The whole list is already filled
|
|
// with it.
|
|
runTest([bytes, str]);
|
|
|
|
codeUnits[i] = TRAILING_SURROGATE;
|
|
codeUnits[i + 1] = TRAILING_SURROGATE;
|
|
codeUnits[i + 2] = CHAR_A; // Add trailing 'a'.
|
|
str = new String.fromCharCodes(codeUnits);
|
|
bytes = new List.filled(i + 7, CHAR_A);
|
|
bytes[i] = UTF8_REPLACEMENT[0];
|
|
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
|
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
|
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
|
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
|
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
|
runTest([bytes, str]);
|
|
|
|
codeUnits[i] = LEADING_SURROGATE;
|
|
codeUnits[i + 1] = LEADING_SURROGATE;
|
|
codeUnits[i + 2] = CHAR_A; // Add trailing 'a'.
|
|
str = new String.fromCharCodes(codeUnits);
|
|
bytes = new List.filled(i + 7, CHAR_A);
|
|
bytes[i] = UTF8_REPLACEMENT[0];
|
|
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
|
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
|
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
|
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
|
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
|
runTest([bytes, str]);
|
|
|
|
codeUnits[i] = TRAILING_SURROGATE;
|
|
codeUnits[i + 1] = LEADING_SURROGATE;
|
|
codeUnits[i + 2] = CHAR_A; // Add trailing 'a'.
|
|
str = new String.fromCharCodes(codeUnits);
|
|
bytes = new List.filled(i + 7, CHAR_A);
|
|
bytes[i] = UTF8_REPLACEMENT[0];
|
|
bytes[i + 1] = UTF8_REPLACEMENT[1];
|
|
bytes[i + 2] = UTF8_REPLACEMENT[2];
|
|
bytes[i + 3] = UTF8_REPLACEMENT[0];
|
|
bytes[i + 4] = UTF8_REPLACEMENT[1];
|
|
bytes[i + 5] = UTF8_REPLACEMENT[2];
|
|
runTest([bytes, str]);
|
|
|
|
// Make sure the invariant is correct.
|
|
codeUnits[i] = CHAR_A;
|
|
}
|
|
}
|