[dart2wasm] Fix bug in dart2wasm UTF-8 decoder

The UTF-8 decoder processes the input bytes in chunks of 1024 bytes. It
had the assumption that processing a chunk can at most yield 1024 UTF-16
code units. But the previous chunk may have had an uninished unicode
point that will (when finished in the next chunk) need to be encoded as
2 UTF-16 code units.

So decoding a 1024 byte chunk may yield 1025 UTF-16 code units.

=> Ensure the fixed buffer can hold 1025 UTF-16 code units.

Closes https://github.com/flutter/flutter/issues/180942

Change-Id: I53bb5b96647d84205153c0df6d468f2e77acef41
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/475042
Reviewed-by: Ömer Ağacan <omersa@google.com>
Commit-Queue: Martin Kustermann <kustermann@google.com>
This commit is contained in:
Martin Kustermann
2026-01-28 02:48:25 -08:00
committed by Commit Queue
parent c99cd19bd4
commit 2dabb7225e
2 changed files with 98 additions and 11 deletions
+34 -11
View File
@@ -2277,21 +2277,22 @@ class _Utf8Decoder {
}
String result = '';
final int chunks = length ~/ _characterArraySize;
final int remaining = length % _characterArraySize;
const maxBytes = _maxChunkSizeInBytes;
final int chunks = length ~/ maxBytes;
final int remaining = length % maxBytes;
for (int i = 0; i < chunks; ++i) {
result += _convertChunkInternal(
bytes,
start,
start + _characterArraySize,
start + maxBytes,
codeUnits,
errorOffset,
true,
);
start += _characterArraySize;
start += maxBytes;
}
if (remaining > 0) {
assert(remaining < _characterArraySize);
assert(remaining < maxBytes);
result += _convertChunkInternal(
bytes,
start,
@@ -2347,21 +2348,22 @@ class _Utf8Decoder {
}
String result = '';
final int chunks = length ~/ _characterArraySize;
final int remaining = length % _characterArraySize;
const maxBytes = _maxChunkSizeInBytes;
final int chunks = length ~/ maxBytes;
final int remaining = length % maxBytes;
for (int i = 0; i < chunks; ++i) {
result += _convertChunkInternal(
bytes,
start,
start + _characterArraySize,
start + maxBytes,
codeUnits,
errorOffset,
true,
);
start += _characterArraySize;
start += maxBytes;
}
if (remaining > 0) {
assert(remaining < _characterArraySize);
assert(remaining < maxBytes);
result += _convertChunkInternal(
bytes,
start,
@@ -2378,7 +2380,28 @@ class _Utf8Decoder {
@pragma('wasm:initialize-at-startup')
static final _characterArray = WasmArray<WasmI16>(_characterArraySize);
static const _characterArraySize = 1024;
static const _characterArraySize = 1025;
// We use the pre-allocated [_characterArray] to avoid temporary allocations
// of `WasmArray<WasmI16>` objects when calling
// `wasm:js-string.fromCharCodeArray` builtin.
//
// This constant defines the maximum size of a chunk of utf-8 encoded
// bytes that will - when decoded to UTF-16 code units - fit into the
// [_characterArray].
//
// The reason we subtract one from the [_characterArraySize] is the following:
// When processing a chunk of N bytes we may have a carryover state from the
// previous chunk that contains an uninished unicode point that requires
// encoding it as a surrogate pair (i.e. 2 UTF-16 code units).
//
// So when e.g. processing 1024 bytes, the first byte may finish a unicode
// point that requires 2 UTF-16 code units, the remaining 1023 bytes may be
// ascii. So the 1024 byte chunk may turn into 1025 UTF-16 code units.
//
// => The [_characterArraySize] must be one larger than
// [_maxChunkSizeInBytes].
static const _maxChunkSizeInBytes = _characterArraySize - 1;
@pragma('wasm:prefer-inline')
String _convertChunkInternal(
@@ -0,0 +1,64 @@
// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:convert';
import 'dart:typed_data';
import 'package:expect/expect.dart';
/// This is a regression test for a bug in the utf8 decoder in dart2wasm.
/// The utf8 decoder processes input bytes in chunks of max 1024. If processing
/// one chunk ends with an unfinished unicode point it maintains carry over
/// state to be applied when processing the next chunk.
///
/// The particular bug was when the carry over state contained an unfinished
/// unicode point that will (when it's finished in the next chunk) need to be
/// encoded as a surrogate pair of 2 UTF-16 code units.
///
/// The assumption in the decoder was that 1024 byte chunk can result in max
/// 1024 UTF-16 code units. But due to this carry over of a surrogate pair it
/// may need a buffer of 1025 UTF-16 code units.
///
/// See `_Utf8Decoder._characterArray` in
/// `sdk/lib/_internal/wasm/lib/convert_patch.dart`
const space = 0x20;
const quote = 0x22;
final bytes = Uint8List(4096);
final jsonBytes = Uint8List(4096 + 2);
// Encoding of '😔'.
const utf8EncodedSurrogatePair = [240, 159, 152, 148];
main() {
for (int i = 0; i < bytes.length - utf8EncodedSurrogatePair.length; ++i) {
// The specific bug this is a regression test for was at i == 1021
// Construct utf8 encoded bytes comprised of spaces.
bytes.fillRange(0, bytes.length, space);
// Now inject the UTF-8 encoding of a surrogate pair into the [i, i+4]
// range.
bytes.setRange(
i,
i + utf8EncodedSurrogatePair.length,
utf8EncodedSurrogatePair,
);
// Normal conversion
Expect.equals('😔', utf8.decode(bytes).trim());
// Chunked conversion
Expect.equals('😔', decodeChunked(bytes).trim());
}
}
String decodeChunked(Uint8List bytes) {
jsonBytes[0] = quote;
jsonBytes[jsonBytes.length - 1] = quote;
jsonBytes.setRange(1, 1 + bytes.length, bytes);
// This uses internally the chunked conversion path.
final fused = utf8.decoder.fuse(json.decoder);
return fused.convert(jsonBytes) as String;
}