dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[scanner] Don't give the Utf8 scanner a 0-terminated byte sequence

Browse Source 
Not having to do the read-allocate-copy dance for files to add a 0-byte
at the end results in these changes when using the CFE to compile
(a fixed version of) the CFE:

```
msec task-clock:u: -1.7356% +/- 0.2164% (-73.16 +/- 9.12)
page-faults:u: -2.6957% +/- 0.0111% (-2914.83 +/- 12.00)
cycles:u: -1.7128% +/- 0.2223% (-297927979.70 +/- 38660477.01)
instructions:u: -1.6814% +/- 0.0002% (-361315766.86 +/- 36853.71)
branch-misses:u: -3.3289% +/- 0.9669% (-2153126.00 +/- 625370.97)
seconds time elapsed: -1.7372% +/- 0.2154% (-0.07 +/- 0.01)
seconds user: -1.5998% +/- 0.2740% (-0.06 +/- 0.01)
seconds sys: -4.1451% +/- 2.9801% (-0.01 +/- 0.01)
Scavenge(   new space) goes from 62 to 61
```

TEST=Existing test coverage.

Change-Id: I8e182bcee39839f6ed1e658c30c85c40ecf0b259
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/385722
Reviewed-by: Johnni Winther <johnniwinther@google.com>
Reviewed-by: Morgan :) <davidmorgan@google.com>
Reviewed-by: Daco Harkes <dacoharkes@google.com>
Reviewed-by: Srujan Gaddam <srujzs@google.com>
Commit-Queue: Jens Johansen <jensj@google.com>
Reviewed-by: Mayank Patke <fishythefish@google.com>
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
This commit is contained in:
Jens Johansen
2024-09-25 08:33:48 +00:00
committed by Commit Queue
parent 6d150d21b3
commit 3dce89fbe0
58 changed files with 218 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/_fe_analyzer_shared/lib/src/scanner/io.dart
+4 -28
View File
@@ -4,38 +4,14 @@
library _fe_analyzer_shared.scanner.io;
import 'dart:io' show File, RandomAccessFile;
import 'dart:io' show File;
import 'dart:typed_data' show Uint8List;
Uint8List readBytesFromFileSync(Uri uri) {
RandomAccessFile file = new File.fromUri(uri).openSync();
Uint8List list;
try {
int length = file.lengthSync();
// +1 to have a 0 terminated list, see [Scanner].
list = new Uint8List(length + 1);
file.readIntoSync(list, /* start = */ 0, length);
} finally {
file.closeSync();
}
return list;
return new File.fromUri(uri).readAsBytesSync();
}
Future<Uint8List> readBytesFromFile(Uri uri,
{bool ensureZeroTermination = true}) async {
RandomAccessFile file = await new File.fromUri(uri).open();
Uint8List list;
try {
int length = await file.length();
// +1 to have a 0 terminated list, see [Scanner].
list = new Uint8List(ensureZeroTermination ? length + 1 : length);
int read = await file.readInto(list);
if (read != length) {
throw "Error reading file: ${uri}";
}
} finally {
await file.close();
}
return list;
Future<Uint8List> readBytesFromFile(Uri uri) async {
return await new File.fromUri(uri).readAsBytes();
}
pkg/_fe_analyzer_shared/lib/src/scanner/scanner.dart
-3
View File
@@ -73,9 +73,6 @@ ScannerResult scan(Uint8List bytes,
bool includeComments = false,
LanguageVersionChanged? languageVersionChanged,
bool allowLazyStrings = true}) {
if (bytes.last != 0) {
throw new ArgumentError("[bytes]: the last byte must be 0.");
}
Scanner scanner = new Utf8BytesScanner(bytes,
configuration: configuration,
includeComments: includeComments,
pkg/_fe_analyzer_shared/lib/src/scanner/string_scanner.dart
+5 -3
View File
@@ -139,8 +139,10 @@ class StringScanner extends AbstractScanner {
}
@override
// To preserve old behavior we only return true once advance has been out of
// bounds. This should probably change. It's at least used in tests
// (where the eof token has its offset reduced by one to 'fix' this.)
// This class used to enforce zero-terminated input, so we only return true
// once advance has been out of bounds.
// TODO(jensj): This should probably change.
// It's at least used in tests (where the eof token has its offset reduced
// by one to 'fix' this.)
bool atEndOfFile() => scanOffset > _stringLengthMinusOne;
}
pkg/_fe_analyzer_shared/lib/src/scanner/utf8_bytes_scanner.dart
+10 -18
View File
@@ -33,11 +33,7 @@ import 'token_impl.dart'
* that points to substrings.
*/
class Utf8BytesScanner extends AbstractScanner {
/**
* The file content.
*
* The content is zero-terminated.
*/
/// The raw file content.
final Uint8List _bytes;
final int _bytesLengthMinusOne;
@@ -86,15 +82,6 @@ class Utf8BytesScanner extends AbstractScanner {
*/
int utf8Slack = 0;
/**
* Creates a new Utf8BytesScanner. The source file is expected to be a
* [Utf8BytesSourceFile] that holds a list of UTF-8 bytes. Otherwise the
* string text of the source file is decoded.
*
* The list of UTF-8 bytes [file.slowUtf8Bytes()] is expected to return an
* array whose last element is '0' to signal the end of the file. If this
* is not the case, the entire array is copied before scanning.
*/
Utf8BytesScanner(this._bytes,
{ScannerConfiguration? configuration,
bool includeComments = false,
@@ -104,7 +91,6 @@ class Utf8BytesScanner extends AbstractScanner {
super(configuration, includeComments, languageVersionChanged,
numberOfBytesHint: _bytes.length,
allowLazyStrings: allowLazyStrings) {
assert(_bytes.last == 0);
// Skip a leading BOM.
if (containsBomAt(/* offset = */ 0)) {
byteOffset += 3;
@@ -168,10 +154,11 @@ class Utf8BytesScanner extends AbstractScanner {
} else {
expectedHighBytes = 1; // Bad code unit.
}
// TODO(jensj): Don't we need a bounds check here? Can't I crash this?
int numBytes = 0;
for (int i = 0; i < expectedHighBytes; i++) {
if (_bytes[byteOffset + i] < 0x80) {
int next = byteOffset + i;
if (next > _bytesLengthMinusOne) break;
if (_bytes[next] < 0x80) {
break;
}
numBytes++;
@@ -300,5 +287,10 @@ class Utf8BytesScanner extends AbstractScanner {
}
@override
bool atEndOfFile() => byteOffset >= _bytesLengthMinusOne;
// This class used to require zero-terminated input, so we only return true
// once advance has been out of bounds.
// TODO(jensj): This should probably change.
// It's at least used in tests (where the eof token has its offset reduced
// by one to 'fix' this.)
bool atEndOfFile() => byteOffset > _bytesLengthMinusOne;
}
pkg/_fe_analyzer_shared/test/scanner_benchmark.dart
+1 -21
View File
@@ -20,8 +20,6 @@ void main(List<String> args) {
scanType = ScanType.string;
} else if (arg == "--bytes") {
scanType = ScanType.bytes;
} else if (arg == "--bytes0") {
scanType = ScanType.bytesWith0Byte;
} else if (arg == "--stringtobytes") {
scanType = ScanType.stringAsBytes;
} else if (arg == "--count") {
@@ -41,8 +39,6 @@ void main(List<String> args) {
String content = f.readAsStringSync();
String contentZeroTerminated = content + '\x00';
Uint8List contentBytes = f.readAsBytesSync();
Uint8List zeroTerminatedBytes = new Uint8List(contentBytes.length + 1);
zeroTerminatedBytes.setRange(0, contentBytes.length, contentBytes);
int numErrors = 0;
Stopwatch stopwatch = new Stopwatch()..start();
@@ -68,22 +64,7 @@ void main(List<String> args) {
lengthProcessed = contentBytes.length;
for (int i = 0; i < iterations; i++) {
hasErrors = scan(
zeroTerminatedBytes,
configuration: new ScannerConfiguration(
enableExtensionMethods: true,
enableNonNullable: true,
enableTripleShift: true,
),
includeComments: true,
).hasErrors;
}
case ScanType.bytesWith0Byte:
lengthProcessed = contentBytes.length;
for (int i = 0; i < iterations; i++) {
Uint8List tmp = new Uint8List(contentBytes.length + 1);
tmp.setRange(0, contentBytes.length, contentBytes);
hasErrors = scan(
tmp,
contentBytes,
configuration: new ScannerConfiguration(
enableExtensionMethods: true,
enableNonNullable: true,
@@ -134,7 +115,6 @@ void main(List<String> args) {
enum ScanType {
string("string characters"),
bytes("bytes"),
bytesWith0Byte("bytes"),
stringAsBytes("string characters as bytes"),
countLfs("bytes");
pkg/build_integration/lib/file_system/multi_root.dart
+3 -2
View File
@@ -3,6 +3,7 @@
// BSD-style license that can be found in the LICENSE file.
import 'dart:async';
import 'dart:typed_data';
// ignore: implementation_imports
import 'package:front_end/src/api_unstable/build_integration.dart';
@@ -81,7 +82,7 @@ class MultiRootFileSystemEntity implements FileSystemEntity {
(await delegate).existsAsyncIfPossible();
@override
Future<List<int>> readAsBytes() async => (await delegate).readAsBytes();
Future<Uint8List> readAsBytes() async => (await delegate).readAsBytes();
@override
Future<List<int>> readAsBytesAsyncIfPossible() async =>
@@ -104,7 +105,7 @@ class MissingFileSystemEntity implements FileSystemEntity {
Future<bool> existsAsyncIfPossible() => exists();
@override
Future<List<int>> readAsBytes() =>
Future<Uint8List> readAsBytes() =>
Future.error(FileSystemException(uri, 'File not found'));
@override
pkg/build_integration/lib/file_system/single_root.dart
+3 -1
View File
@@ -2,6 +2,8 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:typed_data';
// ignore: implementation_imports
import 'package:front_end/src/api_unstable/build_integration.dart';
@@ -68,7 +70,7 @@ class SingleRootFileSystemEntity implements FileSystemEntity {
delegate.existsAsyncIfPossible();
@override
Future<List<int>> readAsBytes() async => delegate.readAsBytes();
Future<Uint8List> readAsBytes() async => delegate.readAsBytes();
@override
Future<List<int>> readAsBytesAsyncIfPossible() async =>
pkg/compiler/lib/compiler_api.dart
+4 -3
View File
@@ -5,6 +5,7 @@
library compiler;
import 'dart:async';
import 'dart:typed_data';
import 'package:front_end/src/api_unstable/dart2js.dart' as fe;
@@ -107,11 +108,11 @@ abstract class CompilerInput {
/// zero-terminated list of encoded bytes. If the input kind is
/// `InputKind.binary` the resulting list is the raw bytes from the input
/// source.
Future<Input<List<int>>> readFromUri(Uri uri,
Future<Input<Uint8List>> readFromUri(Uri uri,
{InputKind inputKind = InputKind.UTF8});
/// Register that [uri] should be an `InputKind.UTF8` input with the
/// given [source] as its zero-terminated list of contents.
/// given [source] of contents.
///
/// If [uri] was read prior to this call, this registration has no effect,
/// otherwise it is expected that a future [readFromUri] will return the
@@ -122,7 +123,7 @@ abstract class CompilerInput {
/// of source files that may not be available on disk. By using these
/// registered contents, dart2js will be able to provide accurate line/column
/// information on an error.
void registerUtf8ContentsForDiagnostics(Uri uri, List<int> source);
void registerUtf8ContentsForDiagnostics(Uri uri, Uint8List source);
}
/// Output types used in `CompilerOutput.createOutputSink`.
pkg/compiler/lib/src/compiler.dart
+2 -1
View File
@@ -6,6 +6,7 @@ library dart2js.compiler_base;
import 'dart:async' show Future;
import 'dart:convert' show jsonEncode;
import 'dart:typed_data';
import 'package:compiler/src/universe/use.dart' show StaticUse;
import 'package:front_end/src/api_unstable/dart2js.dart' as fe;
@@ -836,7 +837,7 @@ class Compiler {
}
}
Future<api.Input<List<int>>> callUserProvider(
Future<api.Input<Uint8List>> callUserProvider(
Uri uri, api.InputKind inputKind) {
try {
return userProviderTask
pkg/compiler/lib/src/io/location_provider.dart
+1 -1
View File
@@ -40,7 +40,7 @@ class LocationCollector extends CodeOutputListener implements LocationProvider {
@override
Location getLocation(int offset) {
RangeError.checkValueInInterval(offset, 0, length, 'offset');
return Source(lineStarts, const <int>[], null, null)
return Source.emptySource(lineStarts, null, null)
.getLocation(_dummyFile, offset);
}
pkg/compiler/lib/src/io/mapped_file.dart
+6 -16
View File
@@ -11,34 +11,24 @@ import 'dart:typed_data';
import 'package:compiler/src/source_file_provider.dart';
import 'package:mmap/mmap.dart';
Uint8List viewOfFile(String filename, bool zeroTerminated) {
Uint8List viewOfFile(String filename) {
final mappedFile = mmapFile(filename);
if (!zeroTerminated) {
return mappedFile.fileBytes;
}
if (mappedFile.hasZeroPadding) {
return mappedFile.fileBytesZeroTerminated;
}
// In the rare case we need a zero-terminated list and the file size
// is exactly page-aligned we need to allocate a new list with extra
// room for the terminating 0.
return Uint8List(mappedFile.fileLength + 1)
..setRange(0, mappedFile.fileLength, mappedFile.fileBytes);
return mappedFile.fileBytes;
}
class MemoryMapSourceFileByteReader implements SourceFileByteReader {
const MemoryMapSourceFileByteReader();
@override
Uint8List getBytes(String filename, {bool zeroTerminated = true}) {
Uint8List getBytes(String filename) {
if (supportsMMap) {
try {
return viewOfFile(filename, zeroTerminated);
return viewOfFile(filename);
} catch (e) {
return readAll(filename, zeroTerminated: zeroTerminated);
return readAll(filename);
}
} else {
return readAll(filename, zeroTerminated: zeroTerminated);
return readAll(filename);
}
}
}
pkg/compiler/lib/src/io/source_file.dart
+17 -35
View File
@@ -15,7 +15,7 @@ import '../../compiler_api.dart' as api show Input, InputKind;
/// Represents a file of source code. The content can be either a [String] or
/// a UTF-8 encoded [List<int>] of bytes.
abstract class SourceFile implements api.Input<List<int>>, LocationProvider {
abstract class SourceFile implements api.Input<Uint8List>, LocationProvider {
/// The absolute URI of the source file.
@override
Uri get uri;
@@ -28,11 +28,8 @@ abstract class SourceFile implements api.Input<List<int>>, LocationProvider {
kernel.Source get kernelSource {
// TODO(johnniwinther): Instead of creating a new Source object,
// we should use the one provided by the front-end.
return _cachedKernelSource ??= kernel.Source(
lineStarts,
slowUtf8ZeroTerminatedBytes(),
uri /* TODO(jensj): What is the import URI? */,
uri)
return _cachedKernelSource ??= kernel.Source(lineStarts, utf8Bytes(),
uri /* TODO(jensj): What is the import URI? */, uri)
..cachedText = slowText();
}
@@ -44,9 +41,8 @@ abstract class SourceFile implements api.Input<List<int>>, LocationProvider {
/// The text content of the file represented as a String
String slowText();
/// The content of the file represented as a UTF-8 encoded [List<int>],
/// terminated with a trailing 0 byte.
List<int> slowUtf8ZeroTerminatedBytes();
/// The content of the file represented as a UTF-8 encoded [Uint8List].
Uint8List utf8Bytes();
/// The length of the string representation of this source file, i.e.,
/// equivalent to [:slowText().length:], but faster.
@@ -169,40 +165,26 @@ abstract class SourceFile implements api.Input<List<int>>, LocationProvider {
int get lines => lineStarts.length - 1;
}
List<int> _zeroTerminateIfNecessary(List<int> bytes) {
if (bytes.length > 0 && bytes.last == 0) return bytes;
List<int> result = Uint8List(bytes.length + 1);
result.setRange(0, bytes.length, bytes);
result[result.length - 1] = 0;
return result;
}
class Utf8BytesSourceFile extends SourceFile {
@override
final Uri uri;
/// The UTF-8 encoded content of the source file.
final List<int> zeroTerminatedContent;
final Uint8List content;
/// Creates a Utf8BytesSourceFile.
///
/// If possible, the given [content] should be zero-terminated. If it isn't,
/// the constructor clones the content and adds a trailing 0.
Utf8BytesSourceFile(this.uri, List<int> content)
: this.zeroTerminatedContent = _zeroTerminateIfNecessary(content);
Utf8BytesSourceFile(this.uri, this.content) : assert(content.last != 0);
@override
List<int> get data => zeroTerminatedContent;
Uint8List get data => content;
@override
String slowText() {
// Don't convert the trailing zero byte.
return utf8.decoder
.convert(zeroTerminatedContent, 0, zeroTerminatedContent.length - 1);
return utf8.decoder.convert(content);
}
@override
List<int> slowUtf8ZeroTerminatedBytes() => zeroTerminatedContent;
Uint8List utf8Bytes() => content;
@override
String slowSubstring(int start, int end) {
@@ -244,7 +226,7 @@ class StringSourceFile extends SourceFile {
: this(Uri(path: filename), filename, text);
@override
List<int> get data => utf8.encode(text);
Uint8List get data => utf8.encode(text);
@override
int get length => text.length;
@@ -255,8 +237,8 @@ class StringSourceFile extends SourceFile {
String slowText() => text;
@override
List<int> slowUtf8ZeroTerminatedBytes() {
return _zeroTerminateIfNecessary(utf8.encode(text));
Uint8List utf8Bytes() {
return utf8.encode(text);
}
@override
@@ -267,15 +249,15 @@ class StringSourceFile extends SourceFile {
}
/// Binary input data.
class Binary implements api.Input<List<int>> {
class Binary implements api.Input<Uint8List> {
@override
final Uri uri;
List<int>? _data;
Uint8List? _data;
Binary(this.uri, List<int> data) : _data = data;
Binary(this.uri, Uint8List data) : _data = data;
@override
List<int> get data {
Uint8List get data {
if (_data != null) return _data!;
throw StateError("'get data' after 'release()'");
}
pkg/compiler/lib/src/kernel/front_end_adapter.dart
+4 -3
View File
@@ -7,6 +7,7 @@
library compiler.kernel.front_end_adapter;
import 'dart:async';
import 'dart:typed_data';
import 'package:front_end/src/api_unstable/dart2js.dart' as fe;
@@ -40,7 +41,7 @@ class _CompilerFileSystemEntity implements fe.FileSystemEntity {
@override
Future<String> readAsString() async {
api.Input<List<int>> input;
api.Input<Uint8List> input;
try {
input = await fs.inputProvider
.readFromUri(uri, inputKind: api.InputKind.UTF8);
@@ -55,8 +56,8 @@ class _CompilerFileSystemEntity implements fe.FileSystemEntity {
}
@override
Future<List<int>> readAsBytes() async {
api.Input<List<int>> input;
Future<Uint8List> readAsBytes() async {
api.Input<Uint8List> input;
try {
input = await fs.inputProvider
.readFromUri(uri, inputKind: api.InputKind.binary);
pkg/compiler/lib/src/phase/load_kernel.dart
+2 -1
View File
@@ -3,6 +3,7 @@
// BSD-style license that can be found in the LICENSE file.
import 'dart:async';
import 'dart:typed_data';
import 'package:_js_interop_checks/src/transformations/static_interop_class_eraser.dart';
import 'package:collection/collection.dart';
@@ -185,7 +186,7 @@ Future<_LoadFromKernelResult> _loadFromKernel(
ir.Component component = ir.Component();
Future<void> read(Uri uri) async {
api.Input<List<int>> input =
api.Input<Uint8List> input =
await compilerInput.readFromUri(uri, inputKind: api.InputKind.binary);
BinaryBuilder(input.data).readComponent(component);
}
pkg/compiler/lib/src/serialization/task.dart
+5 -4
View File
@@ -3,6 +3,7 @@
// BSD-style license that can be found in the LICENSE file.
import 'dart:async';
import 'dart:typed_data';
import 'package:kernel/ast.dart' as ir;
import 'package:kernel/binary/ast_from_binary.dart' as ir;
@@ -130,7 +131,7 @@ class SerializationTask extends CompilerTask {
return await measureIoSubtask('deserialize closed world', () async {
final uri = _options.dataUriForStage(CompilerStage.closedWorld);
_reporter.log('Reading data from $uri');
api.Input<List<int>> dataInput =
api.Input<Uint8List> dataInput =
await _provider.readFromUri(uri, inputKind: api.InputKind.binary);
DataSourceReader source = DataSourceReader(
BinaryDataSource(dataInput.data, stringInterner: _stringInterner),
@@ -167,7 +168,7 @@ class SerializationTask extends CompilerTask {
return await measureIoSubtask('deserialize data', () async {
final uri = _options.dataUriForStage(CompilerStage.globalInference);
_reporter.log('Reading data from $uri');
api.Input<List<int>> dataInput =
api.Input<Uint8List> dataInput =
await _provider.readFromUri(uri, inputKind: api.InputKind.binary);
DataSourceReader source = DataSourceReader(
BinaryDataSource(dataInput.data, stringInterner: _stringInterner),
@@ -234,7 +235,7 @@ class SerializationTask extends CompilerTask {
'${_options.dataUriForStage(CompilerStage.codegenSharded)}$shard');
await measureIoSubtask('deserialize codegen', () async {
_reporter.log('Reading data from ${uri}');
api.Input<List<int>> dataInput =
api.Input<Uint8List> dataInput =
await _provider.readFromUri(uri, inputKind: api.InputKind.binary);
// TODO(36983): This code is extracted because there appeared to be a
// memory leak for large buffer held by `source`.
@@ -251,7 +252,7 @@ class SerializationTask extends CompilerTask {
JsBackendStrategy backendStrategy,
JClosedWorld closedWorld,
Uri uri,
api.Input<List<int>> dataInput,
api.Input<Uint8List> dataInput,
Map<MemberEntity, Deferrable<CodegenResult>> results,
bool useDeferredSourceReads,
SourceLookup sourceLookup,
pkg/compiler/lib/src/source_file_provider.dart
+24 -35
View File
@@ -17,7 +17,7 @@ import 'io/source_file.dart';
import 'util/output_util.dart';
abstract class SourceFileByteReader {
List<int> getBytes(String filename, {bool zeroTerminated = true});
Uint8List getBytes(String filename);
}
abstract class SourceFileProvider implements api.CompilerInput {
@@ -29,11 +29,11 @@ abstract class SourceFileProvider implements api.CompilerInput {
final Set<Uri> _registeredUris = {};
final Map<Uri, Uri> _mappedUris = {};
final bool disableByteCache;
final Map<Uri, List<int>> _byteCache = {};
final Map<Uri, Uint8List> _byteCache = {};
SourceFileProvider(this.byteReader, {this.disableByteCache = true});
Future<api.Input<List<int>>> readBytesFromUri(
Future<api.Input<Uint8List>> readBytesFromUri(
Uri resourceUri, api.InputKind inputKind) {
if (!resourceUri.isAbsolute) {
resourceUri = cwd.resolveUri(resourceUri);
@@ -46,8 +46,8 @@ abstract class SourceFileProvider implements api.CompilerInput {
}
/// Adds [source] to the cache under the [resourceUri] key.
api.Input<List<int>> _sourceToFile(
Uri resourceUri, List<int> source, api.InputKind inputKind) {
api.Input<Uint8List> _sourceToFile(
Uri resourceUri, Uint8List source, api.InputKind inputKind) {
switch (inputKind) {
case api.InputKind.UTF8:
return Utf8BytesSourceFile(resourceUri, source);
@@ -57,7 +57,7 @@ abstract class SourceFileProvider implements api.CompilerInput {
}
@override
void registerUtf8ContentsForDiagnostics(Uri resourceUri, List<int> source) {
void registerUtf8ContentsForDiagnostics(Uri resourceUri, Uint8List source) {
if (!resourceUri.isAbsolute) {
resourceUri = cwd.resolveUri(resourceUri);
}
@@ -77,13 +77,12 @@ abstract class SourceFileProvider implements api.CompilerInput {
return _registeredUris.add(uri);
}
api.Input<List<int>> _readFromFileSync(Uri uri, api.InputKind inputKind) {
api.Input<Uint8List> _readFromFileSync(Uri uri, api.InputKind inputKind) {
final resourceUri = _mappedUris[uri] ?? uri;
assert(resourceUri.isScheme('file'));
List<int> source;
Uint8List source;
try {
source = byteReader.getBytes(resourceUri.toFilePath(),
zeroTerminated: inputKind == api.InputKind.UTF8);
source = byteReader.getBytes(resourceUri.toFilePath());
} on FileSystemException catch (ex) {
String? message = ex.osError?.message;
String detail = message != null ? ' ($message)' : '';
@@ -98,7 +97,7 @@ abstract class SourceFileProvider implements api.CompilerInput {
return _sourceToFile(Uri.parse(relativizeUri(uri)), source, inputKind);
}
api.Input<List<int>>? _readFromFileSyncOrNull(
api.Input<Uint8List>? _readFromFileSyncOrNull(
Uri uri, api.InputKind inputKind) {
try {
return _readFromFileSync(uri, inputKind);
@@ -109,7 +108,7 @@ abstract class SourceFileProvider implements api.CompilerInput {
/// Read [resourceUri] directly as a UTF-8 file. If reading fails, `null` is
/// returned.
api.Input<List<int>>? readUtf8FromFileSyncForTesting(Uri resourceUri) {
api.Input<Uint8List>? readUtf8FromFileSyncForTesting(Uri resourceUri) {
try {
return _readFromFileSync(resourceUri, api.InputKind.UTF8);
} catch (e) {
@@ -119,9 +118,9 @@ abstract class SourceFileProvider implements api.CompilerInput {
}
}
Future<api.Input<List<int>>> _readFromFile(
Future<api.Input<Uint8List>> _readFromFile(
Uri resourceUri, api.InputKind inputKind) {
api.Input<List<int>> input;
api.Input<Uint8List> input;
try {
input = _readFromFileSync(resourceUri, inputKind);
} catch (e) {
@@ -131,7 +130,7 @@ abstract class SourceFileProvider implements api.CompilerInput {
}
/// Get the bytes for a previously accessed UTF-8 [Uri].
api.Input<List<int>>? getUtf8SourceFile(Uri resourceUri) {
api.Input<Uint8List>? getUtf8SourceFile(Uri resourceUri) {
if (!resourceUri.isAbsolute) {
resourceUri = cwd.resolveUri(resourceUri);
}
@@ -156,23 +155,13 @@ abstract class SourceFileProvider implements api.CompilerInput {
class MemoryCopySourceFileByteReader implements SourceFileByteReader {
const MemoryCopySourceFileByteReader();
@override
List<int> getBytes(String filename, {bool zeroTerminated = true}) {
return readAll(filename, zeroTerminated: zeroTerminated);
Uint8List getBytes(String filename) {
return readAll(filename);
}
}
Uint8List readAll(String filename, {bool zeroTerminated = true}) {
RandomAccessFile file = File(filename).openSync();
int length = file.lengthSync();
int bufferLength = length;
if (zeroTerminated) {
// +1 to have a 0 terminated list, see [Scanner].
bufferLength++;
}
var buffer = Uint8List(bufferLength);
file.readIntoSync(buffer, 0, length);
file.closeSync();
return buffer;
Uint8List readAll(String filename) {
return File(filename).readAsBytesSync();
}
class CompilerSourceFileProvider extends SourceFileProvider {
@@ -182,7 +171,7 @@ class CompilerSourceFileProvider extends SourceFileProvider {
: super(byteReader);
@override
Future<api.Input<List<int>>> readFromUri(Uri uri,
Future<api.Input<Uint8List>> readFromUri(Uri uri,
{api.InputKind inputKind = api.InputKind.UTF8}) =>
readBytesFromUri(uri, inputKind);
}
@@ -284,7 +273,7 @@ class FormattingDiagnosticHandler implements api.CompilerDiagnostics {
if (uri == null) {
print('${color(message)}');
} else {
api.Input<List<int>>? file = provider.getUtf8SourceFile(uri);
api.Input<Uint8List>? file = provider.getUtf8SourceFile(uri);
if (file is SourceFile && begin != null && end != null) {
print(file.getLocationMessage(color(message), begin, end,
colorize: color));
@@ -503,7 +492,7 @@ class BazelInputProvider extends SourceFileProvider {
static Uri _resolve(String path) => Uri.base.resolve(path);
@override
Future<api.Input<List<int>>> readFromUri(Uri uri,
Future<api.Input<Uint8List>> readFromUri(Uri uri,
{api.InputKind inputKind = api.InputKind.UTF8}) async {
var resolvedUri = uri;
var path = uri.path;
@@ -517,7 +506,7 @@ class BazelInputProvider extends SourceFileProvider {
}
}
}
api.Input<List<int>> result =
api.Input<Uint8List> result =
await readBytesFromUri(resolvedUri, inputKind);
if (uri != resolvedUri) {
if (!resolvedUri.isAbsolute) {
@@ -548,7 +537,7 @@ class MultiRootInputProvider extends SourceFileProvider {
{super.disableByteCache});
@override
Future<api.Input<List<int>>> readFromUri(Uri uri,
Future<api.Input<Uint8List>> readFromUri(Uri uri,
{api.InputKind inputKind = api.InputKind.UTF8}) async {
var resolvedUri = uri;
if (resolvedUri.isScheme(markerScheme)) {
@@ -562,7 +551,7 @@ class MultiRootInputProvider extends SourceFileProvider {
}
}
}
api.Input<List<int>> result =
api.Input<Uint8List> result =
await readBytesFromUri(resolvedUri, inputKind);
_mappedUris[uri] = resolvedUri;
return result;
pkg/compiler/lib/src/util/memory_source_file_helper.dart
+5 -4
View File
@@ -5,6 +5,7 @@
library dart2js.test.memory_source_file_helper;
import 'dart:async' show Future;
import 'dart:typed_data';
export 'dart:io' show Platform;
import 'package:compiler/compiler_api.dart' as api;
@@ -26,7 +27,7 @@ class MemorySourceFileProvider extends CompilerSourceFileProvider {
MemorySourceFileProvider(Map<String, dynamic> this.memorySourceFiles);
@override
Future<api.Input<List<int>>> readBytesFromUri(
Future<api.Input<Uint8List>> readBytesFromUri(
Uri resourceUri, api.InputKind inputKind) {
if (!resourceUri.isScheme('memory')) {
return super.readBytesFromUri(resourceUri, inputKind);
@@ -39,7 +40,7 @@ class MemorySourceFileProvider extends CompilerSourceFileProvider {
return Future.error(Exception(
'No such memory file $resourceUri in ${memorySourceFiles.keys}'));
}
api.Input<List<int>> input;
api.Input<Uint8List> input;
StringSourceFile? stringFile;
registerUri(resourceUri);
if (source is String) {
@@ -60,12 +61,12 @@ class MemorySourceFileProvider extends CompilerSourceFileProvider {
}
@override
Future<api.Input<List<int>>> readFromUri(Uri resourceUri,
Future<api.Input<Uint8List>> readFromUri(Uri resourceUri,
{api.InputKind inputKind = api.InputKind.UTF8}) =>
readBytesFromUri(resourceUri, inputKind);
@override
api.Input<List<int>>? getUtf8SourceFile(Uri resourceUri) {
api.Input<Uint8List>? getUtf8SourceFile(Uri resourceUri) {
var source = memorySourceFiles[resourceUri.path];
if (source == null) return null;
return source is String
pkg/compiler/test/deferred_loading/deferred_load_id_map_helper.dart
+3 -1
View File
@@ -3,6 +3,7 @@
// BSD-style license that can be found in the LICENSE file.
import 'dart:io' hide Link;
import 'dart:typed_data';
import 'package:async_helper/async_helper.dart';
import 'package:compiler/compiler_api.dart';
@@ -54,7 +55,8 @@ Future<void> runTest(String testGroup, int shard, List<String> options,
],
outputProvider: cfeCollector,
beforeRun: (c) => compiler = c);
final cfeDill = cfeCollector.binaryOutputMap.values.first.list;
final cfeDill =
Uint8List.fromList(cfeCollector.binaryOutputMap.values.first.list);
final dillInputFiles = {cfeFilename: cfeDill};
final resultCollector = OutputCollector();
final compilerResult = await runCompiler(
pkg/compiler/test/end_to_end/dill_loader_test.dart
+3 -1
View File
@@ -2,6 +2,8 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:typed_data';
import 'package:compiler/src/elements/names.dart';
import 'package:compiler/src/util/memory_compiler.dart';
@@ -34,7 +36,7 @@ main() {
..setExitCodeOnProblem = true
..verify = true;
List<int> kernelBinary =
Uint8List kernelBinary =
serializeComponent((await kernelForProgram(uri, options))!.component!);
var compiler = compilerFor(
entryPoint: uri,
pkg/compiler/test/end_to_end/modular_loader_test.dart
+3 -1
View File
@@ -2,6 +2,8 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:typed_data';
import 'package:compiler/src/elements/names.dart';
import 'package:compiler/src/util/memory_compiler.dart';
@@ -77,7 +79,7 @@ main() {
}
/// Generate a component for a modular compilation unit.
Future<List<int>> compileUnit(List<String> inputs, Map<String, dynamic> sources,
Future<Uint8List> compileUnit(List<String> inputs, Map<String, dynamic> sources,
{List<String> deps = const []}) async {
var fs = MemoryFileSystem(_defaultDir);
sources.forEach((name, data) {
pkg/compiler/test/serialization/serialization_diff_helper.dart
+2 -1
View File
@@ -3,6 +3,7 @@
// BSD-style license that can be found in the LICENSE file.
import 'dart:io';
import 'dart:typed_data';
import 'package:compiler/compiler_api.dart' as api;
import 'package:compiler/src/commandline_options.dart';
import 'package:compiler/src/util/memory_compiler.dart';
@@ -36,7 +37,7 @@ Future<CompiledOutput> compileWithSerialization(
options: options);
Expect.isTrue(result.isSuccess);
outputProvider.binaryOutputMap.forEach((fileName, binarySink) {
memorySourceFiles[fileName.path] = binarySink.list;
memorySourceFiles[fileName.path] = Uint8List.fromList(binarySink.list);
});
return outputProvider.clear();
}
pkg/dev_compiler/lib/src/kernel/asset_file_system.dart
+2 -1
View File
@@ -5,6 +5,7 @@
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';
import 'package:async/async.dart';
import 'package:front_end/src/api_prototype/file_system.dart';
@@ -84,7 +85,7 @@ class AssetFileSystemEntity implements FileSystemEntity {
Future<bool> existsAsyncIfPossible() => exists();
@override
Future<List<int>> readAsBytes() async {
Future<Uint8List> readAsBytes() async {
return _runWithClient((httpClient) async {
var response = await httpClient.getUrl(uri);
if (response.statusCode != HttpStatus.ok) {
pkg/front_end/lib/src/api_prototype/file_system.dart
+3 -1
View File
@@ -4,6 +4,8 @@
library front_end.file_system;
import 'dart:typed_data' show Uint8List;
/// Abstract interface to file system operations.
///
/// All front end interaction with the file system goes through this interface;
@@ -66,7 +68,7 @@ abstract class FileSystemEntity {
/// If an error occurs while attempting to read the file (e.g. because no such
/// file exists, or the entity is a directory), the future is completed with
/// [FileSystemException].
Future<List<int>> readAsBytes();
Future<Uint8List> readAsBytes();
/// Attempts to access this file system entity as a file and read its contents
/// as raw bytes.
pkg/front_end/lib/src/api_prototype/language_version.dart
+2 -5
View File
@@ -68,7 +68,7 @@ Future<VersionAndPackageUri> languageVersionForUri(
int? major;
int? minor;
if (fileUri != null) {
List<int>? rawBytes;
Uint8List? rawBytes;
try {
FileSystem fileSystem = context.options.fileSystem;
rawBytes = await fileSystem.entityForUri(fileUri).readAsBytes();
@@ -76,10 +76,7 @@ Future<VersionAndPackageUri> languageVersionForUri(
rawBytes = null;
}
if (rawBytes != null) {
Uint8List zeroTerminatedBytes = new Uint8List(rawBytes.length + 1);
zeroTerminatedBytes.setRange(0, rawBytes.length, rawBytes);
scan(zeroTerminatedBytes,
scan(rawBytes,
includeComments: false,
configuration: new ScannerConfiguration(), languageVersionChanged:
(Scanner scanner, LanguageVersionToken version) {
pkg/front_end/lib/src/api_prototype/memory_file_system.dart
+1 -1
View File
@@ -93,7 +93,7 @@ class MemoryFileSystemEntity implements FileSystemEntity {
@override
// Coverage-ignore(suite): Not run.
Future<List<int>> readAsBytes() {
Future<Uint8List> readAsBytes() {
Uint8List? contents = _fileSystem._files[uri];
if (contents == null) {
return new Future.error(
pkg/front_end/lib/src/api_prototype/standard_file_system.dart
+3 -2
View File
@@ -5,6 +5,7 @@
library front_end.standard_file_system;
import 'dart:io' as io;
import 'dart:typed_data';
import '../base/file_system_dependency_tracker.dart';
import 'file_system.dart';
@@ -90,7 +91,7 @@ class _IoFileSystemEntity implements FileSystemEntity {
}
@override
Future<List<int>> readAsBytes() {
Future<Uint8List> readAsBytes() {
try {
FileSystemDependencyTracker.recordDependency(tracker, uri);
return new Future.value(new io.File.fromUri(uri).readAsBytesSync());
@@ -159,7 +160,7 @@ class DataFileSystemEntity implements FileSystemEntity {
}
@override
Future<List<int>> readAsBytes() {
Future<Uint8List> readAsBytes() {
return new Future.value(uri.data!.contentAsBytes());
}
pkg/front_end/lib/src/base/hybrid_file_system.dart
+3 -1
View File
@@ -6,6 +6,8 @@
/// sdk sources from disk.
library front_end.src.hybrid_file_system;
import 'dart:typed_data';
import '../api_prototype/file_system.dart';
import '../api_prototype/memory_file_system.dart';
import '../api_prototype/standard_file_system.dart';
@@ -64,7 +66,7 @@ class HybridFileSystemEntity implements FileSystemEntity {
(await delegate).existsAsyncIfPossible();
@override
Future<List<int>> readAsBytes() async => (await delegate).readAsBytes();
Future<Uint8List> readAsBytes() async => (await delegate).readAsBytes();
@override
// Coverage-ignore(suite): Not run.
pkg/front_end/lib/src/base/incremental_compiler.dart
+2 -1
View File
@@ -6,6 +6,7 @@ library fasta.incremental_compiler;
import 'dart:async' show Completer;
import 'dart:convert' show JsonEncoder;
import 'dart:typed_data';
import 'package:_fe_analyzer_shared/src/scanner/abstract_scanner.dart'
show ScannerConfiguration;
@@ -1219,7 +1220,7 @@ class IncrementalCompiler implements IncrementalKernelGenerator {
}
for (Uri uri in builderUris) {
List<int>? previousSource = context.uriToSource[uri]?.source;
Uint8List? previousSource = context.uriToSource[uri]?.source;
if (previousSource == null || previousSource.isEmpty) {
recorderForTesting?.recordAdvancedInvalidationResult(
AdvancedInvalidationResult.noPreviousSource);
pkg/front_end/lib/src/kernel/kernel_target.dart
+7 -5
View File
@@ -4,6 +4,8 @@
library fasta.kernel_target;
import 'dart:typed_data';
import 'package:_fe_analyzer_shared/src/messages/severity.dart' show Severity;
import 'package:kernel/ast.dart';
import 'package:kernel/class_hierarchy.dart' show ClassHierarchy;
@@ -263,7 +265,7 @@ class KernelTarget {
bool _hasAddedSources = false;
void addSourceInformation(
Uri importUri, Uri fileUri, List<int> lineStarts, List<int> sourceCode) {
Uri importUri, Uri fileUri, List<int> lineStarts, Uint8List sourceCode) {
Source source = new Source(lineStarts, sourceCode, importUri, fileUri);
uriToSource[fileUri] = source;
if (_hasAddedSources) {
@@ -271,8 +273,8 @@ class KernelTarget {
// The sources have already been added to the component in [link] so we
// have to add source directly here to create a consistent component.
component?.uriToSource[fileUri] = excludeSource
? new Source(source.lineStarts, const <int>[], source.importUri,
source.fileUri)
? new Source.emptySource(
source.lineStarts, source.importUri, source.fileUri)
: source;
}
}
@@ -869,8 +871,8 @@ class KernelTarget {
uriToSource[uri] = excludeSource
?
// Coverage-ignore(suite): Not run.
new Source(source.lineStarts, const <int>[], source.importUri,
source.fileUri)
new Source.emptySource(
source.lineStarts, source.importUri, source.fileUri)
: source;
}
pkg/front_end/lib/src/kernel/macro/macro.dart
+7 -2
View File
@@ -2,6 +2,9 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:convert';
import 'dart:typed_data';
import 'package:_fe_analyzer_shared/src/macros/uri.dart';
import 'package:_fe_analyzer_shared/src/scanner/scanner.dart';
import 'package:kernel/ast.dart';
@@ -1307,14 +1310,16 @@ class MacroApplications {
component.accept(new ReOffsetVisitor(reOffsetMaps));
ScannerResult scannerResult = scanString(source,
Uint8List sourceUtf8 = utf8.encode(source);
ScannerResult scannerResult = scan(sourceUtf8,
configuration: new ScannerConfiguration(
enableExtensionMethods: true,
enableNonNullable: true,
enableTripleShift: true,
forAugmentationLibrary: true));
_sourceLoader.target.addSourceInformation(augmentationImportUri,
augmentationFileUri, scannerResult.lineStarts, source.codeUnits);
augmentationFileUri, scannerResult.lineStarts, sourceUtf8);
for (Uri intermediateAugmentationUri in intermediateAugmentationUris) {
_sourceLoader.target
.removeSourceInformation(intermediateAugmentationUri);
pkg/front_end/lib/src/source/source_library_builder.dart
+4 -2
View File
@@ -4,7 +4,8 @@
library fasta.source_library_builder;
import 'dart:convert' show jsonEncode;
import 'dart:convert' show jsonEncode, utf8;
import 'dart:typed_data';
import 'package:_fe_analyzer_shared/src/field_promotability.dart';
import 'package:_fe_analyzer_shared/src/flow_analysis/flow_analysis_operations.dart';
@@ -563,8 +564,9 @@ class SourceLibraryBuilder extends LibraryBuilderImpl {
indexedLibrary: indexedLibrary,
omittedTypes: omittedTypeDeclarationBuilders);
addAugmentationLibrary(augmentationLibrary);
Uint8List sourceUtf8 = utf8.encode(source);
loader.registerUnparsedLibrarySource(
augmentationLibrary.compilationUnit, source);
augmentationLibrary.compilationUnit, sourceUtf8);
return augmentationLibrary;
}
pkg/front_end/lib/src/source/source_loader.dart
+5 -25
View File
@@ -1002,9 +1002,6 @@ severity: $severity
bytes = synthesizeSourceForMissingFile(compilationUnit.importUri, null);
}
if (bytes != null) {
Uint8List zeroTerminatedBytes = new Uint8List(bytes.length + 1);
zeroTerminatedBytes.setRange(0, bytes.length, bytes);
bytes = zeroTerminatedBytes;
sourceBytes[fileUri] = bytes;
}
}
@@ -1012,7 +1009,7 @@ severity: $severity
if (bytes == null) {
// If it isn't found in the cache, read the file read from the file
// system.
List<int> rawBytes;
Uint8List rawBytes;
try {
rawBytes = await fileSystem.entityForUri(fileUri).readAsBytes();
} on FileSystemException catch (e) {
@@ -1022,9 +1019,7 @@ severity: $severity
rawBytes =
synthesizeSourceForMissingFile(compilationUnit.importUri, message);
}
Uint8List zeroTerminatedBytes = new Uint8List(rawBytes.length + 1);
zeroTerminatedBytes.setRange(0, rawBytes.length, rawBytes);
bytes = zeroTerminatedBytes;
bytes = rawBytes;
sourceBytes[fileUri] = bytes;
byteCount += rawBytes.length;
}
@@ -1063,8 +1058,6 @@ severity: $severity
}, allowLazyStrings: allowLazyStrings);
Token token = result.tokens;
if (!suppressLexicalErrors) {
List<int> source = getSource(bytes);
/// We use the [importUri] of the created [Library] and not the
/// [importUri] of the [LibraryBuilder] since it might be an augmentation
/// library which is not directly part of the output.
@@ -1087,7 +1080,7 @@ severity: $severity
}
}
target.addSourceInformation(
importUri, compilationUnit.fileUri, result.lineStarts, source);
importUri, compilationUnit.fileUri, result.lineStarts, bytes);
}
compilationUnit.issuePostponedProblems();
compilationUnit.markLanguageVersionFinal();
@@ -1142,11 +1135,8 @@ severity: $severity
///
/// This is used for creating synthesized augmentation libraries.
void registerUnparsedLibrarySource(
SourceCompilationUnit compilationUnit, String source) {
List<int> codeUnits = source.codeUnits;
Uint8List bytes = new Uint8List(codeUnits.length + 1);
bytes.setRange(0, codeUnits.length, codeUnits);
sourceBytes[compilationUnit.fileUri] = bytes;
SourceCompilationUnit compilationUnit, Uint8List source) {
sourceBytes[compilationUnit.fileUri] = source;
_unparsedLibraries.addLast(compilationUnit);
}
@@ -1239,16 +1229,6 @@ severity: $severity
}
}
List<int> getSource(List<int> bytes) {
// bytes is 0-terminated. We don't want that included.
if (bytes is Uint8List) {
return new Uint8List.view(
bytes.buffer, bytes.offsetInBytes, bytes.length - 1);
}
// Coverage-ignore(suite): Not run.
return bytes.sublist(0, bytes.length - 1);
}
Future<Null> buildOutline(SourceCompilationUnit compilationUnit) async {
Token tokens = await tokenize(compilationUnit);
OutlineBuilder listener = compilationUnit.createOutlineBuilder();
pkg/front_end/lib/src/util/import_export_etc_helper.dart
+3 -5
View File
@@ -8,11 +8,11 @@ import 'dart:typed_data';
import 'parser_ast.dart';
import 'parser_ast_helper.dart';
FileInfoHelper getFileInfoHelper(Uint8List zeroTerminatedBytes) {
FileInfoHelper getFileInfoHelper(Uint8List rawBytes) {
ImportExportPartLibraryHelperVisitor visitor =
new ImportExportPartLibraryHelperVisitor();
getAST(
zeroTerminatedBytes,
rawBytes,
enableExtensionMethods: true,
enableNonNullable: true,
enableTripleShift: true,
@@ -24,9 +24,7 @@ FileInfoHelper getFileInfoHelper(Uint8List zeroTerminatedBytes) {
FileInfoHelper getFileInfoHelperFromString(String source) {
Uint8List rawBytes = utf8.encode(source);
Uint8List zeroTerminatedBytes = new Uint8List(rawBytes.length + 1);
zeroTerminatedBytes.setRange(0, rawBytes.length, rawBytes);
return getFileInfoHelper(zeroTerminatedBytes);
return getFileInfoHelper(rawBytes);
}
class FileInfoHelper {
pkg/front_end/lib/src/util/outline_extractor.dart
+3 -2
View File
@@ -3,6 +3,7 @@
// BSD-style license that can be found in the LICENSE file.
import 'dart:convert';
import 'dart:typed_data';
import 'package:_fe_analyzer_shared/src/parser/identifier_context.dart';
import 'package:_fe_analyzer_shared/src/scanner/abstract_scanner.dart'
@@ -129,7 +130,7 @@ class _Processor {
fileUri = uriTranslator.translate(importUri)!;
}
if (verbosityLevel >= 20) log("$fileUri");
final List<int> bytes =
final Uint8List bytes =
await fileSystem.entityForUri(fileUri).readAsBytes();
// TODO: Support updating the configuration; also default it to match
// the package version.
@@ -142,7 +143,7 @@ class _Processor {
textualOutline(bytes, configuration, enablePatterns: true);
textualOutlineStopwatch.stop();
if (outlined == null) throw "Textual outline returned null";
final List<int> bytes2 = utf8.encode(outlined);
final Uint8List bytes2 = utf8.encode(outlined);
getAstStopwatch.start();
List<Token> languageVersionsSeen = [];
final ParserAstNode ast = getAST(bytes2,
pkg/front_end/lib/src/util/parser_ast.dart
+2 -5
View File
@@ -25,7 +25,7 @@ import 'parser_ast_helper.dart';
// "assumed version" (from package config probably) which is then updated if
// a language version is seen which will then implicitly answer these questions.
CompilationUnitEnd getAST(
List<int> rawBytes, {
Uint8List rawBytes, {
bool includeBody = true,
bool includeComments = false,
bool enableExtensionMethods = false,
@@ -35,16 +35,13 @@ CompilationUnitEnd getAST(
List<Token>? languageVersionsSeen,
List<int>? lineStarts,
}) {
Uint8List bytes = new Uint8List(rawBytes.length + 1);
bytes.setRange(0, rawBytes.length, rawBytes);
ScannerConfiguration scannerConfiguration = new ScannerConfiguration(
enableExtensionMethods: enableExtensionMethods,
enableNonNullable: enableNonNullable,
enableTripleShift: enableTripleShift);
Utf8BytesScanner scanner = new Utf8BytesScanner(
bytes,
rawBytes,
includeComments: includeComments,
configuration: scannerConfiguration,
languageVersionChanged: (scanner, languageVersion) {
pkg/front_end/lib/src/util/textual_outline.dart
+2 -5
View File
@@ -421,7 +421,7 @@ class BoxedInt {
// "show A, B, C hide A show A" would be empty.
String? textualOutline(
List<int> rawBytes,
Uint8List rawBytes,
ScannerConfiguration configuration, {
bool throwOnUnexpected = false,
bool performModelling = false,
@@ -429,14 +429,11 @@ String? textualOutline(
required bool enablePatterns,
TextualOutlineInfoForTesting? infoForTesting,
}) {
Uint8List bytes = new Uint8List(rawBytes.length + 1);
bytes.setRange(0, rawBytes.length, rawBytes);
List<_Chunk> parsedChunks = <_Chunk>[];
BoxedInt originalPosition = new BoxedInt(0);
Utf8BytesScanner scanner = new Utf8BytesScanner(bytes,
Utf8BytesScanner scanner = new Utf8BytesScanner(rawBytes,
includeComments: false,
configuration: configuration, languageVersionChanged:
(Scanner scanner, LanguageVersionToken languageVersionToken) {
pkg/front_end/test/crashing_test_case_minimizer_impl.dart
+1 -1
View File
@@ -2273,7 +2273,7 @@ class _FakeFileSystemEntity extends FileSystemEntity {
Future<bool> existsAsyncIfPossible() => exists();
@override
Future<List<int>> readAsBytes() {
Future<Uint8List> readAsBytes() {
_ensureCachedIfOk();
Uint8List? data = fs.data[uri];
if (data == null) throw new FileSystemException(uri, "File doesn't exist.");
pkg/front_end/test/fasta/messages_suite.dart
+1 -1
View File
@@ -162,7 +162,7 @@ class MessageTestSuite extends ChainContext {
List<String> formatSpellingMistakes(spell.SpellingResult spellResult,
int offset, String message, String messageForDenyListed) {
if (source == null) {
List<int> bytes = file.readAsBytesSync();
Uint8List bytes = file.readAsBytesSync();
List<int> lineStarts = <int>[];
int indexOf = 0;
while (indexOf >= 0) {
pkg/front_end/test/fasta/testing/suite.dart
+2 -2
View File
@@ -1882,7 +1882,7 @@ class _FakeFileSystemEntity extends FileSystemEntity {
fs.data[uri] = null;
return;
}
fs.data[uri] = (await f.readAsBytes()) as Uint8List;
fs.data[uri] = await f.readAsBytes();
}
@override
@@ -1897,7 +1897,7 @@ class _FakeFileSystemEntity extends FileSystemEntity {
Future<bool> existsAsyncIfPossible() => exists();
@override
Future<List<int>> readAsBytes() async {
Future<Uint8List> readAsBytes() async {
await _ensureCachedIfOk();
Uint8List? data = fs.data[uri];
if (data == null) throw new FileSystemException(uri, "File doesn't exist.");
pkg/front_end/test/fasta/textual_outline_suite.dart
+2 -1
View File
@@ -5,6 +5,7 @@
library fasta.test.textual_outline_test;
import 'dart:io';
import 'dart:typed_data';
import 'package:_fe_analyzer_shared/src/scanner/abstract_scanner.dart'
show ScannerConfiguration;
@@ -110,7 +111,7 @@ class TextualOutline extends Step<TestDescription, TestDescription, Context> {
Map<ExperimentalFlag, bool> experimentalFlagsExplicit =
folderOptions.computeExplicitExperimentalFlags(const {});
List<int> bytes = new File.fromUri(description.uri).readAsBytesSync();
Uint8List bytes = new File.fromUri(description.uri).readAsBytesSync();
for (bool modelled in [false, true]) {
TextualOutlineInfoForTesting info = new TextualOutlineInfoForTesting();
String? result = textualOutline(
pkg/front_end/test/fasta/util/parser_ast_test.dart
+5 -4
View File
@@ -4,6 +4,7 @@
import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';
import 'package:front_end/src/util/parser_ast.dart';
import 'package:front_end/src/util/parser_ast_helper.dart';
@@ -33,7 +34,7 @@ void canParseTopLevelIshOfAllFrontendFiles() {
if (!entry.path.endsWith(".dart")) continue;
try {
processed++;
List<int> data = entry.readAsBytesSync();
Uint8List data = entry.readAsBytesSync();
CompilationUnitEnd ast = getAST(
data,
includeBody: true,
@@ -79,7 +80,7 @@ void canParseTopLevelIshOfAllFrontendFiles() {
void testTopLevelStuff() {
File file = new File.fromUri(
base.resolve("parser_ast_test_data/top_level_stuff.txt"));
List<int> data = file.readAsBytesSync();
Uint8List data = file.readAsBytesSync();
CompilationUnitEnd ast = getAST(data,
includeBody: true,
includeComments: true,
@@ -161,7 +162,7 @@ void testTopLevelStuff() {
void testClassStuff() {
File file = new File.fromUri(base.resolve("parser_ast_test_data/class.txt"));
List<int> data = file.readAsBytesSync();
Uint8List data = file.readAsBytesSync();
CompilationUnitEnd ast = getAST(data,
includeBody: true,
includeComments: true,
@@ -245,7 +246,7 @@ void testClassStuff() {
void testMixinStuff() {
File file = new File.fromUri(base.resolve("parser_ast_test_data/mixin.txt"));
List<int> data = file.readAsBytesSync();
Uint8List data = file.readAsBytesSync();
CompilationUnitEnd ast = getAST(data,
includeBody: true,
includeComments: true,
pkg/front_end/test/find_all_subclasses_tool.dart
+4 -6
View File
@@ -25,13 +25,11 @@ Future<Set<Class>> getAllTokens() async {
/// Compiles either a File or a Directory and finds all subtypes of (including)
/// the specified [className] in a file containing [classFilename].
Future<Set<Class>> findIn(
FileSystemEntity where, String className, String classFilename) async {
Directory where, String className, String classFilename) async {
List<Uri> files = [];
if (where is File) {
files.add(where.uri);
} else if (where is Directory) {
for (FileSystemEntity subEntity in where.listSync(recursive: true)) {
if (subEntity is File) {
for (FileSystemEntity subEntity in where.listSync(recursive: true)) {
if (subEntity is File) {
if (subEntity.path.toLowerCase().endsWith(".dart")) {
files.add(subEntity.uri);
}
}
pkg/front_end/test/lint_suite.dart
+2 -7
View File
@@ -65,7 +65,7 @@ class LintTestDescription extends TestDescription {
}
class LintTestCache {
List<int>? rawBytes;
Uint8List? rawBytes;
late List<int> lineStarts;
Source? source;
Token? firstToken;
@@ -133,12 +133,7 @@ class LintStep extends Step<LintTestDescription, LintTestDescription, Context> {
LintTestDescription description, Context context) async {
if (description.cache.rawBytes == null) {
File f = new File.fromUri(description.uri);
description.cache.rawBytes = f.readAsBytesSync();
Uint8List bytes = new Uint8List(description.cache.rawBytes!.length + 1);
bytes.setRange(
0, description.cache.rawBytes!.length, description.cache.rawBytes!);
Uint8List bytes = description.cache.rawBytes = f.readAsBytesSync();
Utf8BytesScanner scanner = new Utf8BytesScanner(
bytes,
configuration: const ScannerConfiguration(
pkg/front_end/test/parser_suite.dart
+7 -10
View File
@@ -153,7 +153,7 @@ class ParserAstStep extends Step<TestDescription, TestDescription, Context> {
TestDescription description, Context context) {
Uri uri = description.uri;
File f = new File.fromUri(uri);
List<int> rawBytes = f.readAsBytesSync();
Uint8List rawBytes = f.readAsBytesSync();
ParserAstNode ast = getAST(rawBytes);
if (ast.what != "CompilationUnit") {
throw "Expected a single element for 'CompilationUnit' "
@@ -181,7 +181,7 @@ class ListenerStep extends Step<TestDescription, TestDescription, Context> {
Token firstToken = scanUri(uri, shortName, lineStarts: lineStarts);
File f = new File.fromUri(uri);
List<int> rawBytes = f.readAsBytesSync();
Uint8List rawBytes = f.readAsBytesSync();
Source source = new Source(lineStarts, rawBytes, uri, uri);
String shortNameId = "${suiteName}/${shortName}";
ParserTestListenerWithMessageFormatting parserTestListener =
@@ -240,7 +240,7 @@ class IntertwinedStep extends Step<TestDescription, TestDescription, Context> {
scanUri(description.uri, description.shortName, lineStarts: lineStarts);
File f = new File.fromUri(description.uri);
List<int> rawBytes = f.readAsBytesSync();
Uint8List rawBytes = f.readAsBytesSync();
Source source =
new Source(lineStarts, rawBytes, description.uri, description.uri);
@@ -416,7 +416,7 @@ Token scanUri(Uri uri, String shortName, {List<int>? lineStarts}) {
}
File f = new File.fromUri(uri);
List<int> rawBytes = f.readAsBytesSync();
Uint8List rawBytes = f.readAsBytesSync();
return scanRawBytes(rawBytes, config, lineStarts);
}
@@ -427,12 +427,9 @@ bool shouldAllowPatterns(String shortName) {
}
Token scanRawBytes(
List<int> rawBytes, ScannerConfiguration config, List<int>? lineStarts) {
Uint8List bytes = new Uint8List(rawBytes.length + 1);
bytes.setRange(0, rawBytes.length, rawBytes);
Utf8BytesScanner scanner =
new Utf8BytesScanner(bytes, includeComments: true, configuration: config);
Uint8List rawBytes, ScannerConfiguration config, List<int>? lineStarts) {
Utf8BytesScanner scanner = new Utf8BytesScanner(rawBytes,
includeComments: true, configuration: config);
Token firstToken = scanner.tokenize();
if (lineStarts != null) {
lineStarts.addAll(scanner.lineStarts);
pkg/front_end/test/parser_test_listener_creator.dart
+3 -6
View File
@@ -27,12 +27,9 @@ String generateTestListener(Uri repoDir) {
final StringBuffer out = new StringBuffer();
File f = new File.fromUri(
repoDir.resolve("pkg/_fe_analyzer_shared/lib/src/parser/listener.dart"));
List<int> rawBytes = f.readAsBytesSync();
Uint8List bytes = new Uint8List(rawBytes.length + 1);
bytes.setRange(0, rawBytes.length, rawBytes);
Utf8BytesScanner scanner = new Utf8BytesScanner(bytes, includeComments: true);
Uint8List rawBytes = f.readAsBytesSync();
Utf8BytesScanner scanner =
new Utf8BytesScanner(rawBytes, includeComments: true);
Token firstToken = scanner.tokenize();
out.write(r"""
pkg/front_end/test/parser_test_parser_creator.dart
+3 -6
View File
@@ -27,12 +27,9 @@ String generateTestParser(Uri repoDir) {
StringBuffer out = new StringBuffer();
File f = new File.fromUri(repoDir
.resolve("pkg/_fe_analyzer_shared/lib/src/parser/parser_impl.dart"));
List<int> rawBytes = f.readAsBytesSync();
Uint8List bytes = new Uint8List(rawBytes.length + 1);
bytes.setRange(0, rawBytes.length, rawBytes);
Utf8BytesScanner scanner = new Utf8BytesScanner(bytes, includeComments: true);
Uint8List rawBytes = f.readAsBytesSync();
Utf8BytesScanner scanner =
new Utf8BytesScanner(rawBytes, includeComments: true);
Token firstToken = scanner.tokenize();
out.write(r"""
pkg/front_end/test/scanner_fasta_test.dart
+7 -10
View File
@@ -29,10 +29,8 @@ void main() {
});
}
Uint8List encodeAsNullTerminatedUtf8(String source) {
final sourceBytes = utf8.encode(source);
return Uint8List(sourceBytes.length + 1)
..setRange(0, sourceBytes.length, sourceBytes);
Uint8List encodeAsUtf8(String source) {
return utf8.encode(source);
}
@reflectiveTest
@@ -46,8 +44,7 @@ class ScannerTest_Fasta_FuzzTestAPI {
expect(result.hasErrors, isFalse);
expect(result.tokens.type, same(Keyword.CLASS));
// UTF8 encode source with trailing zero
Uint8List bytes = encodeAsNullTerminatedUtf8(source);
Uint8List bytes = encodeAsUtf8(source);
result = usedForFuzzTesting.scan(bytes);
expect(result.hasErrors, isFalse);
@@ -60,7 +57,7 @@ class ScannerTest_Fasta_UTF8 extends ScannerTest_Fasta {
@override
Token scanWithListener(String source, ErrorListener listener,
{ScannerConfiguration? configuration}) {
var bytes = encodeAsNullTerminatedUtf8(source);
var bytes = encodeAsUtf8(source);
var result =
scan(bytes, configuration: configuration, includeComments: true);
var token = result.tokens;
@@ -103,10 +100,10 @@ class ScannerTest_Fasta_UTF8 extends ScannerTest_Fasta {
}
for (int byte0 = 1; byte0 <= 0xFF; ++byte0) {
Uint8List bytes = Uint8List(2)..[0] = byte0;
Uint8List bytes = Uint8List(1)..[0] = byte0;
scanBytes(bytes);
for (int byte1 = 1; byte1 <= 0xFF; ++byte1) {
Uint8List bytes = Uint8List(3)
Uint8List bytes = Uint8List(2)
..[0] = byte0
..[1] = byte1;
scanBytes(bytes);
@@ -708,7 +705,7 @@ class ScannerTest_Fasta_Direct_UTF8 extends ScannerTest_Fasta_Direct {
@override
ScannerResult scanSource(source,
{bool includeComments = true, bool? enableTripleShift}) {
Uint8List encoded = encodeAsNullTerminatedUtf8(source);
Uint8List encoded = encodeAsUtf8(source);
ScannerConfiguration? configuration;
if (enableTripleShift == true) {
pkg/front_end/test/spelling_test_base.dart
+2 -5
View File
@@ -73,13 +73,10 @@ class SpellTest extends Step<TestDescription, TestDescription, SpellContext> {
Future<Result<TestDescription>> run(
TestDescription description, SpellContext context) async {
File f = new File.fromUri(description.uri);
List<int> rawBytes = f.readAsBytesSync();
Uint8List bytes = new Uint8List(rawBytes.length + 1);
bytes.setRange(0, rawBytes.length, rawBytes);
Uint8List rawBytes = f.readAsBytesSync();
Utf8BytesScanner scanner =
new Utf8BytesScanner(bytes, includeComments: true);
new Utf8BytesScanner(rawBytes, includeComments: true);
Token firstToken = scanner.tokenize();
Token? token = firstToken;
pkg/front_end/test/utils/kernel_chain.dart
+2 -6
View File
@@ -334,12 +334,8 @@ class ErrorCommentChecker
File f = new File.fromUri(uri);
if (!f.existsSync()) return const {};
Uint8List rawBytes = f.readAsBytesSync();
Uint8List bytes = new Uint8List(rawBytes.length + 1);
bytes.setRange(0, rawBytes.length, rawBytes);
Utf8BytesScanner scanner = new Utf8BytesScanner(
bytes,
rawBytes,
configuration: const ScannerConfiguration(
enableExtensionMethods: true,
enableNonNullable: true,
@@ -354,7 +350,7 @@ class ErrorCommentChecker
Token? token = firstToken;
Token? previousToken;
Source lineStartsHelper = new Source(lineStarts, const [], null, null);
Source lineStartsHelper = new Source.emptySource(lineStarts, null, null);
Map<int, List<CommentToken>> linesToComments = {};
while (token != null && !token.isEof) {
CommentToken? precedingComments = token.precedingComments;
pkg/front_end/tool/_fasta/parser_ast_helper_creator.dart
+3 -6
View File
@@ -27,12 +27,9 @@ String generateAstHelper(Uri repoDir) {
StringBuffer out = new StringBuffer();
File f = new File.fromUri(
repoDir.resolve("pkg/_fe_analyzer_shared/lib/src/parser/listener.dart"));
List<int> rawBytes = f.readAsBytesSync();
Uint8List bytes = new Uint8List(rawBytes.length + 1);
bytes.setRange(0, rawBytes.length, rawBytes);
Utf8BytesScanner scanner = new Utf8BytesScanner(bytes, includeComments: true);
Uint8List rawBytes = f.readAsBytesSync();
Utf8BytesScanner scanner =
new Utf8BytesScanner(rawBytes, includeComments: true);
Token firstToken = scanner.tokenize();
out.write(r"""
pkg/front_end/tool/dart_doctest_impl.dart
+1 -4
View File
@@ -530,16 +530,13 @@ List<Test> extractTests(Uint8List rawBytes, Uri uriForReporting) {
}
Token scanRawBytes(Uint8List rawBytes, {List<int>? lineStarts}) {
Uint8List bytes = new Uint8List(rawBytes.length + 1);
bytes.setRange(0, rawBytes.length, rawBytes);
ScannerConfiguration scannerConfiguration = new ScannerConfiguration(
enableExtensionMethods: true,
enableNonNullable: true,
enableTripleShift: true);
Utf8BytesScanner scanner = new Utf8BytesScanner(
bytes,
rawBytes,
includeComments: true,
configuration: scannerConfiguration,
languageVersionChanged: (scanner, languageVersion) {
pkg/front_end/tool/incremental_perf.dart
+2 -1
View File
@@ -44,6 +44,7 @@ library front_end.tool.incremental_perf;
import 'dart:convert';
import 'dart:io' hide FileSystemEntity;
import 'dart:typed_data';
import 'package:args/args.dart';
import 'package:front_end/src/api_prototype/front_end.dart';
@@ -255,7 +256,7 @@ class OverlayFileSystemEntity implements FileSystemEntity {
(await delegate).existsAsyncIfPossible();
@override
Future<List<int>> readAsBytes() async => (await delegate).readAsBytes();
Future<Uint8List> readAsBytes() async => (await delegate).readAsBytes();
@override
Future<List<int>> readAsBytesAsyncIfPossible() async =>
pkg/front_end/tool/perf.dart
+1 -5
View File
@@ -212,11 +212,7 @@ Set<Source> scanReachableFiles(Uri entryUri) {
/// Loads the file contents of all [files] as bytes.
Set<Uint8List> loadFileContentsAsBytes(Set<Source> files) {
return files.map((Source source) {
final bytes = utf8.encode(source.contents.data);
// CFE needs files to e 0-terminated.
return Uint8List(bytes.length + 1)
..setRange(0, bytes.length, bytes)
..[bytes.length] = 0;
return utf8.encode(source.contents.data);
}).toSet();
}
pkg/frontend_server/lib/src/binary_protocol.dart
+1 -1
View File
@@ -189,7 +189,7 @@ class _FileSystemEntity implements fe.FileSystemEntity {
Future<bool> existsAsyncIfPossible() => exists();
@override
Future<List<int>> readAsBytes() async {
Future<Uint8List> readAsBytes() async {
final Uint8List? storedBytes = _fileSystem._dills[uri];
if (storedBytes != null) {
return storedBytes;
pkg/kernel/lib/ast.dart
+6 -1
View File
@@ -66,6 +66,7 @@ library kernel.ast;
import 'dart:collection' show ListBase;
import 'dart:convert' show utf8;
import 'dart:typed_data';
import 'package:_fe_analyzer_shared/src/type_inference/nullability_suffix.dart';
import 'package:_fe_analyzer_shared/src/type_inference/type_analyzer_operations.dart'
@@ -14826,10 +14827,11 @@ class _ChildReplacer extends Transformer {
}
class Source {
static final Uint8List _emptySource = new Uint8List(0);
final List<int>? lineStarts;
/// A UTF8 encoding of the original source file.
final List<int> source;
final Uint8List source;
final Uri? importUri;
@@ -14841,6 +14843,9 @@ class Source {
Source(this.lineStarts, this.source, this.importUri, this.fileUri);
Source.emptySource(this.lineStarts, this.importUri, this.fileUri)
: source = _emptySource;
/// Return the text corresponding to [line] which is a 1-based line
/// number. The returned line contains no line separators.
String? getTextLine(int line) {
pkg/kernel/lib/binary/ast_to_binary.dart
+2 -2
View File
@@ -879,8 +879,8 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
!(includeSources &&
_sourcesFromRealImplementation.length > i &&
_sourcesFromRealImplementation[i] == true)) {
source = new Source(
<int>[], const <int>[], source?.importUri, source?.fileUri);
source =
new Source.emptySource(<int>[], source?.importUri, source?.fileUri);
}
String uriAsString = "$uri";
pkg/kernel/test/load_concat_dill_keeps_source_test.dart
+2 -2
View File
@@ -40,7 +40,7 @@ void main() {
cPartial1.uriToSource[uri1] =
new Source([42, 2 * 42], utf8.encode("source #1"), uri1, uri1);
cPartial1.uriToSource[uri2] =
new Source([43, 3 * 43], const <int>[], uri1, uri1);
new Source.emptySource([43, 3 * 43], uri1, uri1);
List<int> partial1Serialized = serialize(cPartial1);
expectSource(partial1Serialized, true, false);
@@ -48,7 +48,7 @@ void main() {
..setMainMethodAndMode(null, false, NonNullableByDefaultCompiledMode.Strong)
..libraries.add(library2);
cPartial2.uriToSource[uri1] =
new Source([42, 2 * 42], const <int>[], uri1, uri1);
new Source.emptySource([42, 2 * 42], uri1, uri1);
cPartial2.uriToSource[uri2] =
new Source([43, 3 * 43], utf8.encode("source #2"), uri1, uri1);
List<int> partial2Serialized = serialize(cPartial2);
pkg/vm/lib/http_filesystem.dart
+3 -2
View File
@@ -4,6 +4,7 @@
import 'dart:async';
import 'dart:io' as io;
import 'dart:typed_data';
import 'package:front_end/src/api_unstable/vm.dart';
@@ -42,7 +43,7 @@ class HttpFileSystemEntity implements FileSystemEntity {
Future<bool> existsAsyncIfPossible() => exists();
@override
Future<List<int>> readAsBytes() async {
Future<Uint8List> readAsBytes() async {
return connectAndRun((io.HttpClient httpClient) async {
io.HttpClientRequest request = await httpClient.getUrl(uri);
io.HttpClientResponse response = await request.close();
@@ -51,7 +52,7 @@ class HttpFileSystemEntity implements FileSystemEntity {
throw new FileSystemException(uri, response.toString());
}
List<List<int>> list = await response.toList();
return list.expand((list) => list).toList();
return new Uint8List.fromList(list.expand((list) => list).toList());
});
}