[dart2wasm] Fix bug in dart2wasm UTF-8 decoder
The UTF-8 decoder processes the input bytes in chunks of 1024 bytes. It had the assumption that processing a chunk can at most yield 1024 UTF-16 code units. But the previous chunk may have had an uninished unicode point that will (when finished in the next chunk) need to be encoded as 2 UTF-16 code units. So decoding a 1024 byte chunk may yield 1025 UTF-16 code units. => Ensure the fixed buffer can hold 1025 UTF-16 code units. Closes https://github.com/flutter/flutter/issues/180942 Change-Id: I53bb5b96647d84205153c0df6d468f2e77acef41 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/475042 Reviewed-by: Ömer Ağacan <omersa@google.com> Commit-Queue: Martin Kustermann <kustermann@google.com>
This commit is contained in:
committed by
Commit Queue
parent
c99cd19bd4
commit
2dabb7225e
@@ -2277,21 +2277,22 @@ class _Utf8Decoder {
|
||||
}
|
||||
|
||||
String result = '';
|
||||
final int chunks = length ~/ _characterArraySize;
|
||||
final int remaining = length % _characterArraySize;
|
||||
const maxBytes = _maxChunkSizeInBytes;
|
||||
final int chunks = length ~/ maxBytes;
|
||||
final int remaining = length % maxBytes;
|
||||
for (int i = 0; i < chunks; ++i) {
|
||||
result += _convertChunkInternal(
|
||||
bytes,
|
||||
start,
|
||||
start + _characterArraySize,
|
||||
start + maxBytes,
|
||||
codeUnits,
|
||||
errorOffset,
|
||||
true,
|
||||
);
|
||||
start += _characterArraySize;
|
||||
start += maxBytes;
|
||||
}
|
||||
if (remaining > 0) {
|
||||
assert(remaining < _characterArraySize);
|
||||
assert(remaining < maxBytes);
|
||||
result += _convertChunkInternal(
|
||||
bytes,
|
||||
start,
|
||||
@@ -2347,21 +2348,22 @@ class _Utf8Decoder {
|
||||
}
|
||||
|
||||
String result = '';
|
||||
final int chunks = length ~/ _characterArraySize;
|
||||
final int remaining = length % _characterArraySize;
|
||||
const maxBytes = _maxChunkSizeInBytes;
|
||||
final int chunks = length ~/ maxBytes;
|
||||
final int remaining = length % maxBytes;
|
||||
for (int i = 0; i < chunks; ++i) {
|
||||
result += _convertChunkInternal(
|
||||
bytes,
|
||||
start,
|
||||
start + _characterArraySize,
|
||||
start + maxBytes,
|
||||
codeUnits,
|
||||
errorOffset,
|
||||
true,
|
||||
);
|
||||
start += _characterArraySize;
|
||||
start += maxBytes;
|
||||
}
|
||||
if (remaining > 0) {
|
||||
assert(remaining < _characterArraySize);
|
||||
assert(remaining < maxBytes);
|
||||
result += _convertChunkInternal(
|
||||
bytes,
|
||||
start,
|
||||
@@ -2378,7 +2380,28 @@ class _Utf8Decoder {
|
||||
|
||||
@pragma('wasm:initialize-at-startup')
|
||||
static final _characterArray = WasmArray<WasmI16>(_characterArraySize);
|
||||
static const _characterArraySize = 1024;
|
||||
static const _characterArraySize = 1025;
|
||||
|
||||
// We use the pre-allocated [_characterArray] to avoid temporary allocations
|
||||
// of `WasmArray<WasmI16>` objects when calling
|
||||
// `wasm:js-string.fromCharCodeArray` builtin.
|
||||
//
|
||||
// This constant defines the maximum size of a chunk of utf-8 encoded
|
||||
// bytes that will - when decoded to UTF-16 code units - fit into the
|
||||
// [_characterArray].
|
||||
//
|
||||
// The reason we subtract one from the [_characterArraySize] is the following:
|
||||
// When processing a chunk of N bytes we may have a carryover state from the
|
||||
// previous chunk that contains an uninished unicode point that requires
|
||||
// encoding it as a surrogate pair (i.e. 2 UTF-16 code units).
|
||||
//
|
||||
// So when e.g. processing 1024 bytes, the first byte may finish a unicode
|
||||
// point that requires 2 UTF-16 code units, the remaining 1023 bytes may be
|
||||
// ascii. So the 1024 byte chunk may turn into 1025 UTF-16 code units.
|
||||
//
|
||||
// => The [_characterArraySize] must be one larger than
|
||||
// [_maxChunkSizeInBytes].
|
||||
static const _maxChunkSizeInBytes = _characterArraySize - 1;
|
||||
|
||||
@pragma('wasm:prefer-inline')
|
||||
String _convertChunkInternal(
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file
|
||||
// for details. All rights reserved. Use of this source code is governed by a
|
||||
// BSD-style license that can be found in the LICENSE file.
|
||||
|
||||
import 'dart:convert';
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:expect/expect.dart';
|
||||
|
||||
/// This is a regression test for a bug in the utf8 decoder in dart2wasm.
|
||||
/// The utf8 decoder processes input bytes in chunks of max 1024. If processing
|
||||
/// one chunk ends with an unfinished unicode point it maintains carry over
|
||||
/// state to be applied when processing the next chunk.
|
||||
///
|
||||
/// The particular bug was when the carry over state contained an unfinished
|
||||
/// unicode point that will (when it's finished in the next chunk) need to be
|
||||
/// encoded as a surrogate pair of 2 UTF-16 code units.
|
||||
///
|
||||
/// The assumption in the decoder was that 1024 byte chunk can result in max
|
||||
/// 1024 UTF-16 code units. But due to this carry over of a surrogate pair it
|
||||
/// may need a buffer of 1025 UTF-16 code units.
|
||||
///
|
||||
/// See `_Utf8Decoder._characterArray` in
|
||||
/// `sdk/lib/_internal/wasm/lib/convert_patch.dart`
|
||||
|
||||
const space = 0x20;
|
||||
const quote = 0x22;
|
||||
|
||||
final bytes = Uint8List(4096);
|
||||
final jsonBytes = Uint8List(4096 + 2);
|
||||
|
||||
// Encoding of '😔'.
|
||||
const utf8EncodedSurrogatePair = [240, 159, 152, 148];
|
||||
|
||||
main() {
|
||||
for (int i = 0; i < bytes.length - utf8EncodedSurrogatePair.length; ++i) {
|
||||
// The specific bug this is a regression test for was at i == 1021
|
||||
|
||||
// Construct utf8 encoded bytes comprised of spaces.
|
||||
bytes.fillRange(0, bytes.length, space);
|
||||
// Now inject the UTF-8 encoding of a surrogate pair into the [i, i+4]
|
||||
// range.
|
||||
bytes.setRange(
|
||||
i,
|
||||
i + utf8EncodedSurrogatePair.length,
|
||||
utf8EncodedSurrogatePair,
|
||||
);
|
||||
|
||||
// Normal conversion
|
||||
Expect.equals('😔', utf8.decode(bytes).trim());
|
||||
// Chunked conversion
|
||||
Expect.equals('😔', decodeChunked(bytes).trim());
|
||||
}
|
||||
}
|
||||
|
||||
String decodeChunked(Uint8List bytes) {
|
||||
jsonBytes[0] = quote;
|
||||
jsonBytes[jsonBytes.length - 1] = quote;
|
||||
jsonBytes.setRange(1, 1 + bytes.length, bytes);
|
||||
|
||||
// This uses internally the chunked conversion path.
|
||||
final fused = utf8.decoder.fuse(json.decoder);
|
||||
return fused.convert(jsonBytes) as String;
|
||||
}
|
||||
Reference in New Issue
Block a user