Build macro kernels.

They are not used yet.

Change-Id: I7849bf845f161f5c48e8ec9ded5900a3486148c1
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/237483
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
Commit-Queue: Konstantin Shcheglov <scheglov@google.com>
This commit is contained in:
Konstantin Shcheglov
2022-03-16 16:55:14 +00:00
committed by Commit Bot
parent fc513d9448
commit 1a1701c2b5
13 changed files with 473 additions and 1 deletions
@@ -14,6 +14,7 @@ import 'package:analyzer/src/dart/analysis/driver_based_analysis_context.dart';
import 'package:analyzer/src/dart/analysis/file_content_cache.dart';
import 'package:analyzer/src/dart/analysis/performance_logger.dart';
import 'package:analyzer/src/generated/engine.dart' show AnalysisOptionsImpl;
import 'package:analyzer/src/summary2/macro.dart';
import 'package:analyzer/src/util/sdk.dart';
/// An implementation of [AnalysisContextCollection].
@@ -42,6 +43,7 @@ class AnalysisContextCollectionImpl implements AnalysisContextCollection {
AnalysisDriverScheduler? scheduler,
FileContentCache? fileContentCache,
void Function(AnalysisOptionsImpl)? updateAnalysisOptions,
MacroKernelBuilder? macroKernelBuilder,
}) : resourceProvider =
resourceProvider ?? PhysicalResourceProvider.INSTANCE {
sdkPath ??= getSdkPath();
@@ -74,6 +76,7 @@ class AnalysisContextCollectionImpl implements AnalysisContextCollection {
scheduler: scheduler,
updateAnalysisOptions: updateAnalysisOptions,
fileContentCache: fileContentCache,
macroKernelBuilder: macroKernelBuilder,
);
contexts.add(context);
}
@@ -25,6 +25,7 @@ import 'package:analyzer/src/generated/source.dart';
import 'package:analyzer/src/hint/sdk_constraint_extractor.dart';
import 'package:analyzer/src/summary/package_bundle_reader.dart';
import 'package:analyzer/src/summary/summary_sdk.dart';
import 'package:analyzer/src/summary2/macro.dart';
import 'package:analyzer/src/summary2/package_bundle_format.dart';
import 'package:analyzer/src/task/options.dart';
import 'package:analyzer/src/util/file_paths.dart' as file_paths;
@@ -58,6 +59,7 @@ class ContextBuilderImpl implements ContextBuilder {
String? sdkSummaryPath,
void Function(AnalysisOptionsImpl)? updateAnalysisOptions,
FileContentCache? fileContentCache,
MacroKernelBuilder? macroKernelBuilder,
}) {
// TODO(scheglov) Remove this, and make `sdkPath` required.
sdkPath ??= getSdkPath();
@@ -115,6 +117,7 @@ class ContextBuilderImpl implements ContextBuilder {
externalSummaries: summaryData,
retainDataForTesting: retainDataForTesting,
fileContentCache: fileContentCache,
macroKernelBuilder: macroKernelBuilder,
);
if (declaredVariables != null) {
@@ -42,6 +42,7 @@ import 'package:analyzer/src/summary/format.dart';
import 'package:analyzer/src/summary/idl.dart';
import 'package:analyzer/src/summary/package_bundle_reader.dart';
import 'package:analyzer/src/summary2/ast_binary_flags.dart';
import 'package:analyzer/src/summary2/macro.dart';
import 'package:analyzer/src/util/file_paths.dart' as file_paths;
import 'package:analyzer/src/util/performance/operation_performance.dart';
import 'package:meta/meta.dart';
@@ -82,7 +83,7 @@ import 'package:meta/meta.dart';
/// TODO(scheglov) Clean up the list of implicitly analyzed files.
class AnalysisDriver implements AnalysisDriverGeneric {
/// The version of data format, should be incremented on every format change.
static const int DATA_VERSION = 210;
static const int DATA_VERSION = 211;
static const bool _applyFileChangesSynchronously = true;
@@ -124,6 +125,8 @@ class AnalysisDriver implements AnalysisDriverGeneric {
/// from file paths.
SourceFactory _sourceFactory;
final MacroKernelBuilder? macroKernelBuilder;
/// The declared environment variables.
DeclaredVariables declaredVariables = DeclaredVariables();
@@ -257,6 +260,7 @@ class AnalysisDriver implements AnalysisDriverGeneric {
required SourceFactory sourceFactory,
required AnalysisOptionsImpl analysisOptions,
required Packages packages,
this.macroKernelBuilder,
FileContentCache? fileContentCache,
bool enableIndex = false,
SummaryDataStore? externalSummaries,
@@ -322,7 +326,9 @@ class AnalysisDriver implements AnalysisDriverGeneric {
analysisOptions: _analysisOptions,
declaredVariables: declaredVariables,
sourceFactory: _sourceFactory,
macroKernelBuilder: macroKernelBuilder,
externalSummaries: _externalSummaries,
fileSystemState: _fsState,
);
}
@@ -40,6 +40,7 @@ import 'package:analyzer/src/workspace/workspace.dart';
import 'package:collection/collection.dart';
import 'package:convert/convert.dart';
import 'package:meta/meta.dart';
import 'package:path/path.dart' as package_path;
import 'package:pub_semver/pub_semver.dart';
var counterFileStateRefresh = 0;
@@ -599,6 +600,7 @@ class FileState {
var exports = <UnlinkedNamespaceDirective>[];
var imports = <UnlinkedNamespaceDirective>[];
var parts = <String>[];
var macroClasses = <MacroClass>[];
var hasDartCoreImport = false;
var hasLibraryDirective = false;
var hasPartOfDirective = false;
@@ -621,6 +623,25 @@ class FileState {
hasPartOfDirective = true;
}
}
for (var declaration in unit.declarations) {
if (declaration is ClassDeclarationImpl) {
if (declaration.macroKeyword != null) {
var constructors = declaration.members
.whereType<ConstructorDeclaration>()
.map((e) => e.name?.name ?? '')
.where((e) => !e.startsWith('_'))
.toList();
if (constructors.isNotEmpty) {
macroClasses.add(
MacroClass(
name: declaration.name.name,
constructors: constructors,
),
);
}
}
}
}
if (!hasDartCoreImport) {
imports.add(
UnlinkedNamespaceDirective(
@@ -637,6 +658,7 @@ class FileState {
imports: imports,
informativeBytes: writeUnitInformative(unit),
lineStarts: Uint32List.fromList(unit.lineInfo.lineStarts),
macroClasses: macroClasses,
partOfName: null,
partOfUri: null,
parts: parts,
@@ -764,6 +786,8 @@ class FileSystemState {
_testView = FileSystemStateTestView(this);
}
package_path.Context get pathContext => _resourceProvider.pathContext;
@visibleForTesting
FileSystemStateTestView get test => _testView;
@@ -22,7 +22,9 @@ import 'package:analyzer/src/summary/package_bundle_reader.dart';
import 'package:analyzer/src/summary2/bundle_reader.dart';
import 'package:analyzer/src/summary2/link.dart' as link2;
import 'package:analyzer/src/summary2/linked_element_factory.dart';
import 'package:analyzer/src/summary2/macro.dart';
import 'package:analyzer/src/summary2/reference.dart';
import 'package:path/src/context.dart';
var counterLinkedLibraries = 0;
var counterLoadedLibraries = 0;
@@ -39,6 +41,8 @@ class LibraryContext {
final LibraryContextTestView testView;
final PerformanceLog logger;
final ByteStore byteStore;
final FileSystemState fileSystemState;
final MacroKernelBuilder? macroKernelBuilder;
final SummaryDataStore store = SummaryDataStore();
late final AnalysisContextImpl analysisContext;
@@ -49,9 +53,11 @@ class LibraryContext {
required AnalysisSessionImpl analysisSession,
required PerformanceLog logger,
required ByteStore byteStore,
required this.fileSystemState,
required AnalysisOptionsImpl analysisOptions,
required DeclaredVariables declaredVariables,
required SourceFactory sourceFactory,
this.macroKernelBuilder,
required SummaryDataStore? externalSummaries,
}) : logger = logger,
byteStore = byteStore {
@@ -120,9 +126,28 @@ class LibraryContext {
cycle.directDependencies.forEach(loadBundle);
var unitsInformativeBytes = <Uri, Uint8List>{};
var macroLibraries = <MacroLibrary>[];
for (var library in cycle.libraries) {
var macroClasses = <MacroClass>[];
for (var file in library.libraryFiles) {
unitsInformativeBytes[file.uri] = file.unlinked2.informativeBytes;
for (var macroClass in file.unlinked2.macroClasses) {
macroClasses.add(
MacroClass(
name: macroClass.name,
constructors: macroClass.constructors,
),
);
}
}
if (macroClasses.isNotEmpty) {
macroLibraries.add(
MacroLibrary(
uri: library.uri,
path: library.path,
classes: macroClasses,
),
);
}
}
@@ -206,6 +231,17 @@ class LibraryContext {
),
);
}
final macroKernelBuilder = this.macroKernelBuilder;
if (macroKernelBuilder != null && macroLibraries.isNotEmpty) {
var macroKernelKey = cycle.transitiveSignature + '.macro_kernel';
var macroKernelBytes = macroKernelBuilder.build(
fileSystem: _MacroFileSystem(fileSystemState),
libraries: macroLibraries,
);
byteStore.put(macroKernelKey, macroKernelBytes);
bytesPut += macroKernelBytes.length;
}
}
logger.run('Prepare linked bundles', () {
@@ -259,3 +295,30 @@ class LibraryContext {
class LibraryContextTestView {
final List<Set<String>> linkedCycles = [];
}
class _MacroFileEntry implements MacroFileEntry {
final FileState fileState;
_MacroFileEntry(this.fileState);
@override
String get content => fileState.content;
@override
bool get exists => fileState.exists;
}
class _MacroFileSystem implements MacroFileSystem {
final FileSystemState fileSystemState;
_MacroFileSystem(this.fileSystemState);
@override
Context get pathContext => fileSystemState.pathContext;
@override
MacroFileEntry getFile(String path) {
var fileState = fileSystemState.getFileForPath(path);
return _MacroFileEntry(fileState);
}
}
@@ -65,6 +65,31 @@ class AnalysisDriverUnlinkedUnit {
}
}
/// Unlinked information about a `macro` class.
class MacroClass {
final String name;
final List<String> constructors;
MacroClass({
required this.name,
required this.constructors,
});
factory MacroClass.read(
SummaryDataReader reader,
) {
return MacroClass(
name: reader.readStringUtf8(),
constructors: reader.readStringUtf8List(),
);
}
void write(BufferedSink sink) {
sink.writeStringUtf8(name);
sink.writeStringUtf8Iterable(constructors);
}
}
/// Unlinked information about a namespace directive.
class UnlinkedNamespaceDirective {
/// The configurations that control which library will actually be used.
@@ -159,6 +184,9 @@ class UnlinkedUnit {
/// Offsets of the first character of each line in the source code.
final Uint32List lineStarts;
/// The list of `macro` classes.
final List<MacroClass> macroClasses;
/// The library name of the `part of my.name;` directive.
final String? partOfName;
@@ -176,6 +204,7 @@ class UnlinkedUnit {
required this.imports,
required this.informativeBytes,
required this.lineStarts,
required this.macroClasses,
required this.partOfName,
required this.partOfUri,
required this.parts,
@@ -194,6 +223,9 @@ class UnlinkedUnit {
),
informativeBytes: reader.readUint8List(),
lineStarts: reader.readUInt30List(),
macroClasses: reader.readTypedList(
() => MacroClass.read(reader),
),
partOfName: reader.readOptionalStringUtf8(),
partOfUri: reader.readOptionalStringUtf8(),
parts: reader.readStringUtf8List(),
@@ -212,6 +244,9 @@ class UnlinkedUnit {
});
sink.writeUint8List(informativeBytes);
sink.writeUint30List(lineStarts);
sink.writeList<MacroClass>(macroClasses, (x) {
x.write(sink);
});
sink.writeOptionalStringUtf8(partOfName);
sink.writeOptionalStringUtf8(partOfUri);
sink.writeStringUtf8Iterable(parts);
@@ -962,6 +962,7 @@ class _FileStateUnlinked {
imports: imports,
informativeBytes: writeUnitInformative(unit),
lineStarts: Uint32List.fromList(unit.lineInfo.lineStarts),
macroClasses: [],
partOfName: partOfName,
partOfUri: partOfUriStr,
parts: parts,
+50
View File
@@ -0,0 +1,50 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:typed_data';
import 'package:path/path.dart' as package_path;
class MacroClass {
final String name;
final List<String> constructors;
MacroClass({
required this.name,
required this.constructors,
});
}
abstract class MacroFileEntry {
String get content;
/// When CFE searches for `package_config.json` we need to check this.
bool get exists;
}
abstract class MacroFileSystem {
/// Used to convert `file:` URIs into paths.
package_path.Context get pathContext;
MacroFileEntry getFile(String path);
}
abstract class MacroKernelBuilder {
Uint8List build({
required MacroFileSystem fileSystem,
required List<MacroLibrary> libraries,
});
}
class MacroLibrary {
final Uri uri;
final String path;
final List<MacroClass> classes;
MacroLibrary({
required this.uri,
required this.path,
required this.classes,
});
}
+6
View File
@@ -24,7 +24,13 @@ dev_dependencies:
path: ../analyzer_utilities
args: ^2.0.0
async: ^2.5.0
front_end:
path: ../front_end
kernel:
path: ../kernel
linter: ^1.12.0
matcher: ^0.12.10
test: ^1.16.0
test_reflective_loader: ^0.2.0
vm:
path: ../vm
@@ -12,6 +12,7 @@ import 'package:analyzer/src/dart/analysis/driver.dart';
import 'package:analyzer/src/dart/analysis/driver_based_analysis_context.dart';
import 'package:analyzer/src/dart/analysis/experiments.dart';
import 'package:analyzer/src/generated/engine.dart' show AnalysisOptionsImpl;
import 'package:analyzer/src/summary2/macro.dart';
import 'package:analyzer/src/test_utilities/mock_packages.dart';
import 'package:analyzer/src/test_utilities/mock_sdk.dart';
import 'package:analyzer/src/test_utilities/package_config_file_builder.dart';
@@ -26,6 +27,7 @@ import 'package:meta/meta.dart';
import 'package:test/test.dart';
import '../../../generated/test_support.dart';
import '../../summary/repository_macro_kernel_builder.dart';
import 'context_collection_resolution_caching.dart';
import 'resolution.dart';
@@ -139,6 +141,8 @@ abstract class ContextResolutionTest
_declaredVariables = map;
}
MacroKernelBuilder? get macroKernelBuilder => null;
bool get retainDataForTesting => false;
Folder get sdkRoot => newFolder('/sdk');
@@ -246,6 +250,7 @@ abstract class ContextResolutionTest
retainDataForTesting: retainDataForTesting,
sdkPath: sdkRoot.path,
updateAnalysisOptions: updateAnalysisOptions,
macroKernelBuilder: macroKernelBuilder,
);
verifyCreatedCollection();
@@ -321,6 +326,7 @@ class PubPackageResolutionTest extends ContextResolutionTest {
bool ffi = false,
bool js = false,
bool meta = false,
MacrosEnvironment? macrosEnvironment,
}) {
config = config.copy();
@@ -354,6 +360,15 @@ class PubPackageResolutionTest extends ContextResolutionTest {
config.add(name: 'meta', rootPath: metaPath);
}
if (macrosEnvironment != null) {
var packagesRootFolder = getFolder(packagesRootPath);
macrosEnvironment.packageSharedFolder.copyTo(packagesRootFolder);
config.add(
name: '_fe_analyzer_shared',
rootPath: getFolder('$packagesRootPath/_fe_analyzer_shared').path,
);
}
var path = '$testPackageRootPath/.dart_tool/package_config.json';
writePackageConfig(path, config);
}
@@ -0,0 +1,54 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'package:analyzer/src/summary2/macro.dart';
import 'package:test_reflective_loader/test_reflective_loader.dart';
import '../../summary/repository_macro_kernel_builder.dart';
import 'context_collection_resolution.dart';
main() {
try {
MacrosEnvironment.instance;
} catch (_) {
print('Cannot initialize environment. Skip macros tests.');
return;
}
defineReflectiveSuite(() {
defineReflectiveTests(MacroResolutionTest);
});
}
@reflectiveTest
class MacroResolutionTest extends PubPackageResolutionTest {
@override
MacroKernelBuilder? get macroKernelBuilder {
return DartRepositoryMacroKernelBuilder(
MacrosEnvironment.instance.platformDillBytes,
);
}
@override
void setUp() {
super.setUp();
writeTestPackageConfig(
PackageConfigFileBuilder(),
macrosEnvironment: MacrosEnvironment.instance,
);
}
test_0() async {
await assertNoErrorsInCode(r'''
import 'dart:async';
import 'package:_fe_analyzer_shared/src/macros/api.dart';
macro class EmptyMacro implements ClassTypesMacro {
const EmptyMacro();
FutureOr<void> buildTypesForClass(clazz, builder) {}
}
''');
}
}
@@ -44,6 +44,7 @@ import 'language_version_test.dart' as language_version;
import 'library_element_test.dart' as library_element;
import 'local_function_test.dart' as local_function;
import 'local_variable_test.dart' as local_variable;
import 'macro_test.dart' as macro;
import 'metadata_test.dart' as metadata;
import 'method_declaration_test.dart' as method_declaration;
import 'method_invocation_test.dart' as method_invocation;
@@ -108,6 +109,7 @@ main() {
library_element.main();
local_function.main();
local_variable.main();
macro.main();
metadata.main();
method_declaration.main();
method_invocation.main();
@@ -0,0 +1,210 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
// ignore: deprecated_member_use
import 'dart:cli' as cli;
import 'dart:convert';
import 'dart:io' as io;
import 'dart:typed_data';
import 'package:_fe_analyzer_shared/src/macros/bootstrap.dart';
import 'package:_fe_analyzer_shared/src/macros/executor/serialization.dart';
import 'package:analyzer/file_system/file_system.dart';
import 'package:analyzer/file_system/memory_file_system.dart';
import 'package:analyzer/file_system/physical_file_system.dart';
import 'package:analyzer/src/summary2/macro.dart';
import 'package:analyzer/src/util/uri.dart';
import 'package:analyzer_utilities/package_root.dart' as package_root;
import 'package:front_end/src/api_prototype/compiler_options.dart' as fe;
import 'package:front_end/src/api_prototype/file_system.dart' as fe;
import 'package:front_end/src/fasta/kernel/utils.dart' as fe;
import 'package:kernel/target/targets.dart' as fe;
import 'package:path/path.dart' as package_path;
import 'package:vm/kernel_front_end.dart' as fe;
import 'package:vm/target/vm.dart' as fe;
final Uri _platformDillUri = Uri.parse('org-dartlang-sdk://vm.dill');
/// Implementation of [MacroKernelBuilder] that can run in Dart SDK repository.
///
/// This is a temporary implementation, to be replaced with a more stable
/// approach, e.g. a `dart:` API for compilation, shipping `front_end`
/// with SDK, etc.
class DartRepositoryMacroKernelBuilder implements MacroKernelBuilder {
final Uint8List platformDillBytes;
DartRepositoryMacroKernelBuilder(this.platformDillBytes);
@override
Uint8List build({
required MacroFileSystem fileSystem,
required List<MacroLibrary> libraries,
}) {
var options = fe.CompilerOptions()
..sdkSummary = _platformDillUri
..target = fe.VmTarget(fe.TargetFlags(enableNullSafety: true));
var macroMainContent = bootstrapMacroIsolate(
{
for (var library in libraries)
library.uri.toString(): {
for (var c in library.classes) c.name: c.constructors
},
},
SerializationMode.byteDataClient,
);
var macroMainBytes = utf8.encode(macroMainContent) as Uint8List;
var macroMainPath = libraries.first.path + '.macro';
var macroMainUri = fileSystem.pathContext.toUri(macroMainPath);
options
..fileSystem = _FileSystem(
fileSystem,
platformDillBytes,
macroMainUri,
macroMainBytes,
);
// TODO(scheglov) For now we convert async into sync.
// ignore: deprecated_member_use
var compilationResults = cli.waitFor(
fe.compileToKernel(
macroMainUri,
options,
environmentDefines: {},
),
);
return fe.serializeComponent(
compilationResults.component!,
filter: (library) {
return !library.importUri.isScheme('dart');
},
includeSources: false,
);
}
}
/// Environment for compiling macros to kernels, expecting that we run
/// a test in the Dart SDK repository.
///
/// Just like [DartRepositoryMacroKernelBuilder], this is a temporary
/// implementation.
class MacrosEnvironment {
static late final instance = MacrosEnvironment._();
final _resourceProvider = MemoryResourceProvider(context: package_path.posix);
late final Uint8List platformDillBytes;
MacrosEnvironment._() {
var physical = PhysicalResourceProvider.INSTANCE;
var packageRoot = physical.pathContext.normalize(package_root.packageRoot);
physical
.getFolder(packageRoot)
.getChildAssumingFolder('_fe_analyzer_shared/lib/src/macros')
.copyTo(
packageSharedFolder.getChildAssumingFolder('lib/src'),
);
platformDillBytes = physical
.getFile(io.Platform.resolvedExecutable)
.parent
.parent
.getChildAssumingFolder('lib')
.getChildAssumingFolder('_internal')
.getChildAssumingFile('vm_platform_strong.dill')
.readAsBytesSync();
}
Folder get packageSharedFolder {
return _resourceProvider.getFolder('/packages/_fe_analyzer_shared');
}
}
class _BytesFileSystemEntity implements fe.FileSystemEntity {
@override
final Uri uri;
final Uint8List bytes;
_BytesFileSystemEntity(this.uri, this.bytes);
@override
Future<bool> exists() async => true;
@override
Future<bool> existsAsyncIfPossible() => exists();
@override
Future<List<int>> readAsBytes() async => bytes;
@override
Future<List<int>> readAsBytesAsyncIfPossible() => readAsBytes();
@override
Future<String> readAsString() async {
var bytes = await readAsBytes();
return utf8.decode(bytes);
}
}
class _FileSystem implements fe.FileSystem {
final MacroFileSystem fileSystem;
final Uint8List platformDillBytes;
final Uri macroMainUri;
final Uint8List macroMainBytes;
_FileSystem(
this.fileSystem,
this.platformDillBytes,
this.macroMainUri,
this.macroMainBytes,
);
@override
fe.FileSystemEntity entityForUri(Uri uri) {
if (uri == _platformDillUri) {
return _BytesFileSystemEntity(uri, platformDillBytes);
} else if (uri == macroMainUri) {
return _BytesFileSystemEntity(uri, macroMainBytes);
} else if (uri.isScheme('file')) {
var path = fileUriToNormalizedPath(fileSystem.pathContext, uri);
return _FileSystemEntity(
uri,
fileSystem.getFile(path),
);
} else {
throw fe.FileSystemException(uri, 'Only supports file: URIs');
}
}
}
class _FileSystemEntity implements fe.FileSystemEntity {
@override
final Uri uri;
final MacroFileEntry file;
_FileSystemEntity(this.uri, this.file);
@override
Future<bool> exists() async => file.exists;
@override
Future<bool> existsAsyncIfPossible() => exists();
@override
Future<List<int>> readAsBytes() async {
var string = await readAsString();
return utf8.encode(string);
}
@override
Future<List<int>> readAsBytesAsyncIfPossible() => readAsBytes();
@override
Future<String> readAsString() async => file.content;
}