diff --git a/pkg/cfg/lib/front_end/ast_to_ir.dart b/pkg/cfg/lib/front_end/ast_to_ir.dart index 66e4adacdb0..87cbb863205 100644 --- a/pkg/cfg/lib/front_end/ast_to_ir.dart +++ b/pkg/cfg/lib/front_end/ast_to_ir.dart @@ -116,7 +116,7 @@ class AstToIr extends ast.RecursiveVisitor { } if (builder.hasOpenBlock) { builder.addNullConstant(); - _buildReturn(); + builder.addReturn(); } return builder.done(); } @@ -152,13 +152,6 @@ class AstToIr extends ast.RecursiveVisitor { } } - void _buildReturn() { - if (function.isSuspendable) { - builder.addLeaveSuspendableFunction(function.returnType); - } - builder.addReturn(); - } - void _buildImplicitGetter(ast.Field node) { final field = CField(node); if (node.isStatic) { @@ -506,7 +499,7 @@ class AstToIr extends ast.RecursiveVisitor { final value = builder.pop(); _generateNonLocalControlTransfer(node, null, () { builder.push(value); - _buildReturn(); + builder.addReturn(); }); } @@ -1824,7 +1817,39 @@ class AstToIr extends ast.RecursiveVisitor { builder.pop(); return; } - builder.addSuspend(node.isYieldStar ? .yieldStar : .yield, const TopType()); + switch (function.asyncMarker) { + case .AsyncStar: + // yield/yield* statement acts as a return statement if subscription + // to the async* Stream is cancelled. + final canceledBlock = builder.newTargetBlock(); + final continueBlock = builder.newTargetBlock(); + + // Suspend will evaluate to true if subscription to async* Stream is cancelled. + builder.addSuspend( + node.isYieldStar ? .asyncYieldStar : .asyncYield, + const BoolType(), + ); + builder.addBranch(canceledBlock, continueBlock); + + builder.startBlock(canceledBlock); + _generateNonLocalControlTransfer(node, null, () { + builder.addNullConstant(); + builder.addReturn(); + }); + + builder.startBlock(continueBlock); + break; + + case .SyncStar: + builder.addSuspend( + node.isYieldStar ? .syncYieldStar : .syncYield, + const TopType(), + ); + break; + + default: + throw 'Unexpected YieldStatement in $function with ${function.asyncMarker}'; + } } } diff --git a/pkg/cfg/lib/ir/flow_graph_builder.dart b/pkg/cfg/lib/ir/flow_graph_builder.dart index 522ef386a00..9e8b1db1c86 100644 --- a/pkg/cfg/lib/ir/flow_graph_builder.dart +++ b/pkg/cfg/lib/ir/flow_graph_builder.dart @@ -622,19 +622,6 @@ class FlowGraphBuilder { appendInstruction(instr); } - /// Append [LeaveSuspendableFunction] to the graph. - void addLeaveSuspendableFunction(CType type) { - final returnValue = pop(); - final instr = LeaveSuspendableFunction( - graph, - currentSourcePosition, - type, - returnValue, - ); - push(instr); - appendInstruction(instr); - } - /// Append [Suspend] to the graph. Suspend addSuspend(SuspendOpcode op, CType type) { final typeArguments = (op == .awaitWithTypeCheck) ? pop() : null; diff --git a/pkg/cfg/lib/ir/flow_graph_checker.dart b/pkg/cfg/lib/ir/flow_graph_checker.dart index 021f9bea721..dbe76d01747 100644 --- a/pkg/cfg/lib/ir/flow_graph_checker.dart +++ b/pkg/cfg/lib/ir/flow_graph_checker.dart @@ -413,11 +413,6 @@ final class FlowGraphChecker extends Pass implements InstructionVisitor { verifyTypeArgumentsInput(instr.typeArguments, instr); } - @override - void visitLeaveSuspendableFunction(LeaveSuspendableFunction instr) { - assert(graph.function.isSuspendable); - } - @override void visitSuspend(Suspend instr) { assert(graph.function.isSuspendable); @@ -426,8 +421,11 @@ final class FlowGraphChecker extends Pass implements InstructionVisitor { case .await || .awaitWithTypeCheck: assert(asyncMarker == .Async || asyncMarker == .AsyncStar); break; - case .yield || .yieldStar: - assert(asyncMarker == .AsyncStar || asyncMarker == .SyncStar); + case .asyncYield || .asyncYieldStar: + assert(asyncMarker == .AsyncStar); + break; + case .syncYield || .syncYieldStar: + assert(asyncMarker == .SyncStar); break; } if (instr.op == .awaitWithTypeCheck) { diff --git a/pkg/cfg/lib/ir/instructions.dart b/pkg/cfg/lib/ir/instructions.dart index 7e4ba33cc94..9907d953565 100644 --- a/pkg/cfg/lib/ir/instructions.dart +++ b/pkg/cfg/lib/ir/instructions.dart @@ -605,6 +605,8 @@ final class Return extends Instruction setInputAt(0, value); } + Definition get value => inputDefAt(0); + @override R accept(InstructionVisitor v) => v.visitReturn(this); } @@ -1412,29 +1414,15 @@ final class EnterSuspendableFunction extends Instruction R accept(InstructionVisitor v) => v.visitEnterSuspendableFunction(this); } -/// Leave a suspendable function. -final class LeaveSuspendableFunction extends Definition - with CanThrow, HasSideEffects { - @override - final CType type; - - LeaveSuspendableFunction( - super.graph, - super.sourcePosition, - this.type, - Definition returnValue, - ) : super(inputCount: 1) { - setInputAt(0, returnValue); - } - - Definition get returnValue => inputDefAt(0); - - @override - R accept(InstructionVisitor v) => v.visitLeaveSuspendableFunction(this); +enum SuspendOpcode { + await, + awaitWithTypeCheck, + asyncYield, + asyncYieldStar, + syncYield, + syncYieldStar, } -enum SuspendOpcode { await, awaitWithTypeCheck, yield, yieldStar } - /// A point where execution of a suspendable function can be suspended /// and resumed (await, yield or yield*). final class Suspend extends Definition with CanThrow, HasSideEffects { diff --git a/pkg/cfg/lib/ir/types.dart b/pkg/cfg/lib/ir/types.dart index d315c404d17..ab349904aed 100644 --- a/pkg/cfg/lib/ir/types.dart +++ b/pkg/cfg/lib/ir/types.dart @@ -75,6 +75,9 @@ sealed class CType { /// Return non-nullable variant of this type (if possible). CType get toNonNullableType; + /// Returns true if value of this type can be `Future`. + bool get canBeFuture; + @override bool operator ==(Object other) => other is CType && @@ -104,6 +107,9 @@ final class IntType extends CType { @override CType get toNonNullableType => this; + @override + bool get canBeFuture => false; + @override String toString() => 'int'; } @@ -127,6 +133,9 @@ final class DoubleType extends CType { @override CType get toNonNullableType => this; + @override + bool get canBeFuture => false; + @override String toString() => 'double'; } @@ -150,6 +159,9 @@ final class BoolType extends CType { @override CType get toNonNullableType => this; + @override + bool get canBeFuture => false; + @override String toString() => 'bool'; } @@ -173,6 +185,9 @@ final class StringType extends CType { @override CType get toNonNullableType => this; + @override + bool get canBeFuture => false; + @override String toString() => 'String'; } @@ -196,6 +211,9 @@ final class ObjectType extends CType { @override CType get toNonNullableType => this; + @override + bool get canBeFuture => true; + @override String toString() => 'Object'; } @@ -216,6 +234,9 @@ final class NullType extends CType { @override CType get toNonNullableType => const NeverType(); + @override + bool get canBeFuture => false; + @override String toString() => 'Null'; } @@ -236,6 +257,9 @@ final class NeverType extends CType { @override CType get toNonNullableType => this; + @override + bool get canBeFuture => false; + @override String toString() => 'Never'; } @@ -258,6 +282,9 @@ final class TopType extends CType { @override CType get toNonNullableType => const ObjectType(); + @override + bool get canBeFuture => true; + @override String toString() => ''; } @@ -278,6 +305,9 @@ final class StaticType extends CType { @override CType get toNonNullableType => CType.fromStaticType(dartType.toNonNull()); + @override + bool get canBeFuture => true; + @override String toString() => dartType.getDisplayString(); } @@ -301,6 +331,9 @@ sealed class ExtendedType extends CType { @override CType get toNonNullableType => this; + @override + bool get canBeFuture => false; + @override bool operator ==(Object other) => other is ExtendedType && this.kind == other.kind; diff --git a/pkg/cfg/lib/ir/visitor.dart b/pkg/cfg/lib/ir/visitor.dart index cf5736a690f..dba32550f19 100644 --- a/pkg/cfg/lib/ir/visitor.dart +++ b/pkg/cfg/lib/ir/visitor.dart @@ -44,7 +44,6 @@ abstract interface class InstructionVisitor { R visitAllocateMapLiteral(AllocateMapLiteral instr); R visitStringInterpolation(StringInterpolation instr); R visitEnterSuspendableFunction(EnterSuspendableFunction instr); - R visitLeaveSuspendableFunction(LeaveSuspendableFunction instr); R visitSuspend(Suspend instr); R visitBinaryIntOp(BinaryIntOp instr); R visitUnaryIntOp(UnaryIntOp instr); @@ -120,8 +119,6 @@ abstract mixin class DefaultInstructionVisitor defaultInstruction(instr); R visitEnterSuspendableFunction(EnterSuspendableFunction instr) => defaultInstruction(instr); - R visitLeaveSuspendableFunction(LeaveSuspendableFunction instr) => - defaultInstruction(instr); R visitSuspend(Suspend instr) => defaultInstruction(instr); R visitBinaryIntOp(BinaryIntOp instr) => defaultInstruction(instr); R visitUnaryIntOp(UnaryIntOp instr) => defaultInstruction(instr); diff --git a/pkg/cfg/lib/passes/constant_propagation.dart b/pkg/cfg/lib/passes/constant_propagation.dart index f9472f1226e..244844e9380 100644 --- a/pkg/cfg/lib/passes/constant_propagation.dart +++ b/pkg/cfg/lib/passes/constant_propagation.dart @@ -450,11 +450,6 @@ final class ConstantPropagation extends Pass @override void visitEnterSuspendableFunction(EnterSuspendableFunction instr) {} - @override - void visitLeaveSuspendableFunction(LeaveSuspendableFunction instr) { - _setNonConstant(instr); - } - @override void visitSuspend(Suspend instr) { _setNonConstant(instr); diff --git a/pkg/cfg/lib/passes/simplification.dart b/pkg/cfg/lib/passes/simplification.dart index 2608a32d90d..470cb7e15ca 100644 --- a/pkg/cfg/lib/passes/simplification.dart +++ b/pkg/cfg/lib/passes/simplification.dart @@ -280,10 +280,6 @@ final class Simplification extends Pass Instruction visitEnterSuspendableFunction(EnterSuspendableFunction instr) => instr; - @override - Instruction visitLeaveSuspendableFunction(LeaveSuspendableFunction instr) => - instr; - @override Instruction visitSuspend(Suspend instr) => instr; diff --git a/pkg/cfg/testcases/suspendable.dart b/pkg/cfg/testcases/suspendable.dart index 74760887edd..65df97342db 100644 --- a/pkg/cfg/testcases/suspendable.dart +++ b/pkg/cfg/testcases/suspendable.dart @@ -23,11 +23,25 @@ Future async5(T x) async { } Stream asyncStar1(int a) async* { + print('before'); yield a; + print('after'); } Stream asyncStar2(Stream a) async* { + print('before'); yield* a; + print('after'); +} + +Stream asyncStar3(int a) async* { + try { + print('before'); + yield a; + print('after'); + } finally { + print('finally'); + } } Iterable syncStar1(int a) sync* { diff --git a/pkg/cfg/testcases/suspendable.dart.expect b/pkg/cfg/testcases/suspendable.dart.expect index 11dd3cee6c9..8ca4ad09575 100644 --- a/pkg/cfg/testcases/suspendable.dart.expect +++ b/pkg/cfg/testcases/suspendable.dart.expect @@ -3,8 +3,7 @@ B0 = EntryBlock() v1 = Constant() v3 = Constant(42) EnterSuspendableFunction(v1) - v4 = LeaveSuspendableFunction(v3) - Return(v4) + Return(v3) --- async2 B0 = EntryBlock() @@ -15,8 +14,7 @@ B0 = EntryBlock() v6 = Suspend await(v1) v8 = Suspend await(v2) v9 = BinaryIntOp +(v6, v8) - v10 = LeaveSuspendableFunction(v9) - Return(v10) + Return(v9) --- async3 B0 = EntryBlock() @@ -27,8 +25,7 @@ B0 = EntryBlock() EnterSuspendableFunction(v2) v6 = Suspend awaitWithTypeCheck(v1, v5) DirectCall print(v6) - v11 = LeaveSuspendableFunction(v10) - Return(v11) + Return(v10) --- async4 B0 = EntryBlock() @@ -40,8 +37,7 @@ B0 = EntryBlock() EnterSuspendableFunction(v6) v8 = TypeArguments(v5, v4, ) v10 = DirectCall _GrowableList._literal1(v8, v2) - v11 = LeaveSuspendableFunction(v10) - Return(v11) + Return(v10) --- async5 B0 = EntryBlock() @@ -54,28 +50,72 @@ B0 = EntryBlock() v9 = TypeArguments(v8, v4, ) v10 = Suspend awaitWithTypeCheck(v2, v9) DirectCall print(v10) - v14 = LeaveSuspendableFunction(v8) - Return(v14) + Return(v8) --- asyncStar1 -B0 = EntryBlock() +B0 = EntryBlock() dominates:(B8, B7) v2 = Constant() - v6 = Constant(null) + v4 = Constant("before") + v11 = Constant(null) + v13 = Constant("after") v1 = Parameter(a) EnterSuspendableFunction(v2) - Suspend yield(v1) - v7 = LeaveSuspendableFunction(v6) - Return(v7) + DirectCall print(v4) + v9 = Suspend asyncYield(v1) + Branch(v9, true: B7, false: B8) +B7 = TargetBlock() idom:B0 + Return(v11) +B8 = TargetBlock() idom:B0 + DirectCall print(v13) + Return(v11) --- asyncStar2 -B0 = EntryBlock() +B0 = EntryBlock() dominates:(B8, B7) v2 = Constant() - v6 = Constant(null) + v4 = Constant("before") + v11 = Constant(null) + v13 = Constant("after") v1 = Parameter(a) EnterSuspendableFunction(v2) - Suspend yieldStar(v1) - v7 = LeaveSuspendableFunction(v6) - Return(v7) + DirectCall print(v4) + v9 = Suspend asyncYieldStar(v1) + Branch(v9, true: B7, false: B8) +B7 = TargetBlock() idom:B0 + Return(v11) +B8 = TargetBlock() idom:B0 + DirectCall print(v13) + Return(v11) + +--- asyncStar3 +B0 = EntryBlock() dominates:(B5, B4) + v2 = Constant() + v7 = Constant("before") + v16 = Constant("after") + v22 = Constant("finally") + v28 = Constant(null) + v1 = Parameter(a) + EnterSuspendableFunction(v2) + TryEntry(try-body: B4, catch-block: B5) +B4 = TargetBlock() exception-handler:B5 idom:B0 dominates:(B11, B10) + DirectCall print(v7) + v12 = Suspend asyncYield(v1) + Branch(v12, true: B10, false: B11) +B10 = TargetBlock() exception-handler:B5 idom:B4 dominates:(B14) + Goto(B14) +B14 = JoinBlock(B10) idom:B10 + DirectCall print(v22) + Return(v28) +B11 = TargetBlock() exception-handler:B5 idom:B4 dominates:(B18) + DirectCall print(v16) + Goto(B18) +B18 = JoinBlock(B11) idom:B11 + DirectCall print(v22) + Return(v28) +B5 = CatchBlock() idom:B0 + v20 = Parameter(#exception) + v21 = Parameter(#stackTrace) + DirectCall print(v22) + Throw(v20, v21) --- syncStar1 B0 = EntryBlock() @@ -83,9 +123,8 @@ B0 = EntryBlock() v6 = Constant(null) v1 = Parameter(a) EnterSuspendableFunction(v2) - Suspend yield(v1) - v7 = LeaveSuspendableFunction(v6) - Return(v7) + Suspend syncYield(v1) + Return(v6) --- syncStar2 B0 = EntryBlock() @@ -93,9 +132,8 @@ B0 = EntryBlock() v6 = Constant(null) v1 = Parameter(a) EnterSuspendableFunction(v2) - Suspend yieldStar(v1) - v7 = LeaveSuspendableFunction(v6) - Return(v7) + Suspend syncYieldStar(v1) + Return(v6) --- main B0 = EntryBlock() diff --git a/pkg/native_compiler/lib/back_end/arm64/assembler.dart b/pkg/native_compiler/lib/back_end/arm64/assembler.dart index badaa2f1d3f..324055cc8fd 100644 --- a/pkg/native_compiler/lib/back_end/arm64/assembler.dart +++ b/pkg/native_compiler/lib/back_end/arm64/assembler.dart @@ -522,6 +522,17 @@ final class Arm64Assembler extends Assembler with Uint32OutputBuffer { b(label, condition); } + @override + void branchIfBoolIs(Register left, bool right, Label label) { + // Test bool value bit: 0 = true, 1 = false. + final boolValueBit = boolValueBitPosition(log2wordSize); + if (right) { + tbz(left, boolValueBit, label); + } else { + tbnz(left, boolValueBit, label); + } + } + @override void loadFromPool(Register reg, Object obj) { int poolIndex = objectPool.getObject(obj); @@ -757,6 +768,20 @@ final class Arm64Assembler extends Assembler with Uint32OutputBuffer { blr(LR); } + // TODO: remove after all stubs are implemented in the compiler + void callVmStub(StubCode vmStub) { + loadFromPool(codeReg, vmStub); + ldr(LR, fieldAddress(codeReg, vmOffsets.Code_entry_point_offset.first)); + blr(LR); + } + + // TODO: remove after all stubs are implemented in the compiler + void jumpVmStub(StubCode vmStub) { + loadFromPool(codeReg, vmStub); + ldr(LR, fieldAddress(codeReg, vmOffsets.Code_entry_point_offset.first)); + br(LR); + } + @override void unimplemented(String message) { loadConstant(R0, ConstantValue.fromString(message)); diff --git a/pkg/native_compiler/lib/back_end/arm64/code_generator.dart b/pkg/native_compiler/lib/back_end/arm64/code_generator.dart index 9a4b06f6694..32649b0f266 100644 --- a/pkg/native_compiler/lib/back_end/arm64/code_generator.dart +++ b/pkg/native_compiler/lib/back_end/arm64/code_generator.dart @@ -7,6 +7,7 @@ import 'dart:math' as math; import 'package:cfg/ir/constant_value.dart'; import 'package:cfg/ir/field.dart'; import 'package:cfg/ir/functions.dart'; +import 'package:cfg/ir/global_context.dart'; import 'package:cfg/ir/instructions.dart'; import 'package:cfg/ir/types.dart'; import 'package:cfg/utils/misc.dart'; @@ -25,6 +26,39 @@ final class Arm64CodeGenerator extends CodeGenerator { final FunctionRegistry functionRegistry; late final Arm64Assembler _asm; + late final CFunction _asyncStarStreamControllerAdd = functionRegistry + .getFunction( + GlobalContext.instance.coreTypes.index.getProcedure( + 'dart:async', + '_AsyncStarStreamController', + 'add', + ), + ); + late final CFunction _asyncStarStreamControllerAddStream = functionRegistry + .getFunction( + GlobalContext.instance.coreTypes.index.getProcedure( + 'dart:async', + '_AsyncStarStreamController', + 'addStream', + ), + ); + + late final CField _syncStarIteratorCurrent = CField( + GlobalContext.instance.coreTypes.index.getField( + 'dart:async', + '_SyncStarIterator', + '_current', + ), + ); + + late final CField _syncStarIteratorYieldStarIterable = CField( + GlobalContext.instance.coreTypes.index.getField( + 'dart:async', + '_SyncStarIterator', + '_yieldStarIterable', + ), + ); + Arm64CodeGenerator(super.backEndState, this.functionRegistry); @override @@ -297,17 +331,11 @@ final class Arm64CodeGenerator extends CodeGenerator { @override void visitBranch(Branch instr) { final cond = inputReg(instr, 0); - final boolValueBit = boolValueBitPosition(log2wordSize); _generateBranch(instr.trueSuccessor, instr.falseSuccessor, ( bool value, Label label, ) { - // Test bool value bit: 0 = true, 1 = false. - if (value) { - _asm.tbz(cond, boolValueBit, label); - } else { - _asm.tbnz(cond, boolValueBit, label); - } + _asm.branchIfBoolIs(cond, value, label); }); } @@ -425,6 +453,24 @@ final class Arm64CodeGenerator extends CodeGenerator { @override void visitReturn(Return instr) { assert(inputReg(instr, 0) == returnReg); + switch (graph.function.asyncMarker) { + case .Async: + _asm.jumpVmStub( + instr.value.type.canBeFuture + ? StubCode.ReturnAsync + : StubCode.ReturnAsyncNotFuture, + ); + return; + case .AsyncStar: + _asm.jumpVmStub(StubCode.ReturnAsyncStar); + return; + case .SyncStar: + // Overwrite the return value to indicate the end of iteration. + _asm.loadConstant(returnReg, ConstantValue.fromBool(false)); + break; + case .Sync: + break; + } _asm.leaveDartFrame(); _asm.ret(); } @@ -1076,12 +1122,7 @@ final class Arm64CodeGenerator extends CodeGenerator { }); _asm.loadFromPool(TypeTestingStub.subtypeTestCacheReg, stc); - _asm.loadFromPool(codeReg, stub); - _asm.ldr( - tempReg, - _asm.fieldAddress(codeReg, vmOffsets.Code_entry_point_offset.first), - ); - _asm.blr(tempReg); + _asm.callVmStub(stub); _asm.cmp(TypeTestingStub.subtypeTestCacheResultReg, nullReg); _asm.b(slowPath, .equal); _asm.mov(resultReg, TypeTestingStub.subtypeTestCacheResultReg); @@ -1618,21 +1659,94 @@ final class Arm64CodeGenerator extends CodeGenerator { @override void visitEnterSuspendableFunction(EnterSuspendableFunction instr) { - _asm.unimplemented( - 'Unimplemented: code generation for EnterSuspendableFunction', - ); - } - - @override - void visitLeaveSuspendableFunction(LeaveSuspendableFunction instr) { - _asm.unimplemented( - 'Unimplemented: code generation for LeaveSuspendableFunction', - ); + final asyncMarker = graph.function.asyncMarker; + final stub = switch (asyncMarker) { + .Async => StubCode.InitAsync, + .AsyncStar => StubCode.InitAsyncStar, + .SyncStar => StubCode.InitSyncStar, + .Sync => throw 'Unexpected async marker', + }; + _asm.callVmStub(stub); + // Suspend async* and sync* functions at the beginning. + if (asyncMarker == .AsyncStar) { + _asm.mov(SuspendStub.argumentReg, nullReg); + _asm.callVmStub(StubCode.YieldAsyncStar); + } else if (asyncMarker == .SyncStar) { + _asm.mov(SuspendStub.argumentReg, nullReg); + _asm.callVmStub(StubCode.SuspendSyncStarAtStart); + } } @override void visitSuspend(Suspend instr) { - _asm.unimplemented('Unimplemented: code generation for Suspend'); + void loadFunctionData(Register dst) { + _asm.ldr(tempReg, _asm.address(FP, stackFrame.suspendStateOffsetFromFP)); + _asm.ldr( + dst, + _asm.fieldAddress(tempReg, vmOffsets.SuspendState_function_data_offset), + ); + } + + switch (instr.op) { + case .await: + _asm.callVmStub(StubCode.Await); + break; + case .awaitWithTypeCheck: + _asm.callVmStub(StubCode.AwaitWithTypeCheck); + break; + case .asyncYield || .asyncYieldStar: + // Load controller from suspend state. + loadFunctionData(tempReg); + // Call controller.add or addStream. + assert(stackFrame.maxArgumentsStackSlots >= 2); + _asm.stp( + SuspendStub.argumentReg, + tempReg, + RegOffsetAddress(stackPointerReg, 0), + ); + _callFunction( + instr.op == .asyncYield + ? _asyncStarStreamControllerAdd + : _asyncStarStreamControllerAddStream, + ); + // It returns true if subscription was canceled. + final done = Label(); + _asm.branchIfBoolIs(returnReg, true, done); + // Suspend. + _asm.mov(SuspendStub.argumentReg, nullReg); + _asm.callVmStub(StubCode.YieldAsyncStar); + _asm.bind(done); + break; + case .syncYield || .syncYieldStar: + // Load iterator from suspend state. + final iteratorReg = temporaryReg(instr, 0); + final scratch1Reg = temporaryReg(instr, 1); + final scratch2Reg = temporaryReg(instr, 2); + loadFunctionData(iteratorReg); + // Set _SyncStarIterator._current or _yieldStarIterable. + _asm.str( + SuspendStub.argumentReg, + _asm.fieldAddress( + iteratorReg, + objectLayout.getFieldOffset( + instr.op == .syncYield + ? _syncStarIteratorCurrent + : _syncStarIteratorYieldStarIterable, + ), + ), + ); + _writeBarrier( + iteratorReg, + SuspendStub.argumentReg, + scratch1Reg, + scratch2Reg, + valueCanBeSmi: _canBeSmi(instr.operand), + ); + // Suspend. + _asm.mov(SuspendStub.argumentReg, nullReg); + _asm.callVmStub(StubCode.SuspendSyncStarAtYield); + break; + } } @override diff --git a/pkg/native_compiler/lib/back_end/arm64/constraints.dart b/pkg/native_compiler/lib/back_end/arm64/constraints.dart index 03289c69b63..1772c7457c3 100644 --- a/pkg/native_compiler/lib/back_end/arm64/constraints.dart +++ b/pkg/native_compiler/lib/back_end/arm64/constraints.dart @@ -16,17 +16,19 @@ final class Arm64Constraints extends Constraints { // TODO: enable returning unboxed FP values on FP register. static const bool returnFPValuesOnFPRegister = false; - late final volatileRegisters = [ + late final allRegisters = [ ...getAllocatableRegisters(), ...getAllocatableFPRegisters(), ]; + // TODO:add callee-save registers + late final volatileRegisters = allRegisters; - late final volatileRegistersExceptReturnReg = volatileRegisters - .where((r) => r != returnReg) - .toList(); - late final volatileRegistersExceptFPReturnReg = volatileRegisters - .where((r) => r != returnFPReg) - .toList(); + late final volatileRegistersExceptReturnReg = volatileRegistersExcept( + returnReg, + ); + late final volatileRegistersExceptFPReturnReg = volatileRegistersExcept( + returnFPReg, + ); List? _parameters; @@ -44,6 +46,19 @@ final class Arm64Constraints extends Constraints { @override List getAllocatableFPRegisters() => allocatableFPRegisters; + List volatileRegistersExcept(PhysicalRegister reg) => [ + for (final r in volatileRegisters) + if (r != reg) r, + ]; + + List allRegistersExcept( + Constraint? result, + List inputs, + ) => [ + for (final r in allRegisters) + if (r != result && !inputs.contains(r)) r, + ]; + // TODO: pass arguments on registers // TODO: add callee-save registers InstructionConstraints callConstraints(CallInstruction instr) { @@ -404,14 +419,25 @@ final class Arm64Constraints extends Constraints { @override InstructionConstraints? visitEnterSuspendableFunction( EnterSuspendableFunction instr, - ) => const InstructionConstraints(null, [anyCpuRegister]); + ) { + final inputs = [InitSuspendableFunctionStub.typeArgsReg]; + return InstructionConstraints( + null, + inputs, + allRegistersExcept(null, inputs), + ); + } @override - InstructionConstraints? visitLeaveSuspendableFunction( - LeaveSuspendableFunction instr, - ) => const InstructionConstraints(anyCpuRegister, [anyCpuRegister]); - - @override - InstructionConstraints? visitSuspend(Suspend instr) => - const InstructionConstraints(anyCpuRegister, [anyCpuRegister]); + InstructionConstraints? visitSuspend(Suspend instr) { + final inputs = [ + SuspendStub.argumentReg, + if (instr.op == .awaitWithTypeCheck) SuspendStub.typeArgsReg, + ]; + return InstructionConstraints( + returnReg, + inputs, + allRegistersExcept(returnReg, inputs), + ); + } } diff --git a/pkg/native_compiler/lib/back_end/arm64/stack_frame.dart b/pkg/native_compiler/lib/back_end/arm64/stack_frame.dart index 0e20ff13b07..eed3792f82c 100644 --- a/pkg/native_compiler/lib/back_end/arm64/stack_frame.dart +++ b/pkg/native_compiler/lib/back_end/arm64/stack_frame.dart @@ -19,6 +19,7 @@ import 'package:native_compiler/back_end/stack_frame.dart'; /// FP -> [saved FP] /// [Code] /// [saved tagged ObjectPool] +/// [suspend state (only for async/async*/sync* functions)] /// [shadow space for optional parameters] /// [spill slot 0] /// ... @@ -27,6 +28,9 @@ import 'package:native_compiler/back_end/stack_frame.dart'; /// ``` /// TODO: add catch block entry parameters area. final class Arm64StackFrame extends StackFrame { + /// Stack frame alignment. + static const int alignment = 2 * wordSize; + /// Number of fixed frame slots; distance between the last parameter /// slot and the first spill slot in words. static const int numberOfFixedSlots = 4; @@ -37,11 +41,15 @@ final class Arm64StackFrame extends StackFrame { /// Offset of the saved pool pointer relative to FP. static const int poolPointerOffsetFromFP = -2 * wordSize; - /// Offset of the first shadow parameter, relative to FP - static const int shadowParametersOffsetFromFP = -3 * wordSize; + /// Offset of the suspend state, relative to FP + static const int _suspendStateOffsetFromFP = -3 * wordSize; - /// Stack frame alignment. - static const int alignment = 2 * wordSize; + /// Number of stack slots used by suspend state. + late final int _suspendStateStackSlots = (function.isSuspendable ? 1 : 0); + + /// Offset of the first shadow parameter, relative to FP + late final int _shadowParametersOffsetFromFP = + _suspendStateOffsetFromFP - _suspendStateStackSlots * wordSize; /// Number of stack slots reserved for shadow parameters. late final int _shadowParametersStackSlots = @@ -52,7 +60,7 @@ final class Arm64StackFrame extends StackFrame { : 0; late final int _firstSpillSlotOffsetFromFP = - shadowParametersOffsetFromFP - _shadowParametersStackSlots * wordSize; + _shadowParametersOffsetFromFP - _shadowParametersStackSlots * wordSize; Arm64StackFrame(super.function); @@ -72,6 +80,8 @@ final class Arm64StackFrame extends StackFrame { return 4; // Result + 3 arguments for InstantiateType runtime call. case TypeTest(): return 6; // Result + 5 arguments for Instanceof runtime call. + case Suspend(:var op) when op == .asyncYield || op == .asyncYieldStar: + return 2; // 2 arguments for _AsyncStarStreamController.add/addStream call. default: return 0; } @@ -97,6 +107,12 @@ final class Arm64StackFrame extends StackFrame { } } + @override + int get suspendStateOffsetFromFP { + assert(function.isSuspendable); + return _suspendStateOffsetFromFP; + } + @override int shadowParameterOffsetFromFP(int paramIndex) { assert( @@ -104,13 +120,16 @@ final class Arm64StackFrame extends StackFrame { ); assert(paramIndex >= argumentRegisters.length); assert(paramIndex < function.numberOfParameters); - return shadowParametersOffsetFromFP - + return _shadowParametersOffsetFromFP - (paramIndex - argumentRegisters.length) * wordSize; } @override int get frameSizeToAllocate => roundUp( - (_shadowParametersStackSlots + usedSpillSlots + maxArgumentsStackSlots) * + (_suspendStateStackSlots + + _shadowParametersStackSlots + + usedSpillSlots + + maxArgumentsStackSlots) * wordSize, alignment, ); diff --git a/pkg/native_compiler/lib/back_end/arm64/stub_code_generator.dart b/pkg/native_compiler/lib/back_end/arm64/stub_code_generator.dart index 4aaab8d4e6a..e3ca82d55f6 100644 --- a/pkg/native_compiler/lib/back_end/arm64/stub_code_generator.dart +++ b/pkg/native_compiler/lib/back_end/arm64/stub_code_generator.dart @@ -118,6 +118,15 @@ final class TypeTestingStub { static const Register subtypeTestCacheResultReg = R7; } +final class InitSuspendableFunctionStub { + static const Register typeArgsReg = R0; +} + +final class SuspendStub { + static const Register argumentReg = R0; + static const Register typeArgsReg = R1; +} + final class Arm64StubFactory extends StubFactory { final VMOffsets vmOffsets; final ObjectLayout objectLayout; diff --git a/pkg/native_compiler/lib/back_end/assembler.dart b/pkg/native_compiler/lib/back_end/assembler.dart index 54bc212d280..e1c621bdaf6 100644 --- a/pkg/native_compiler/lib/back_end/assembler.dart +++ b/pkg/native_compiler/lib/back_end/assembler.dart @@ -152,6 +152,9 @@ abstract base class Assembler { void jump(Label label); void branchIf(Condition condition, Label label); + /// Jump to [label] if Dart `bool` value in [left] matches [right]. + void branchIfBoolIs(Register left, bool right, Label label); + void loadFromPool(Register reg, Object obj); void loadConstant(Register reg, ConstantValue value); diff --git a/pkg/native_compiler/lib/back_end/register_allocator.dart b/pkg/native_compiler/lib/back_end/register_allocator.dart index c3faf32e6aa..1a5dbb0d7f5 100644 --- a/pkg/native_compiler/lib/back_end/register_allocator.dart +++ b/pkg/native_compiler/lib/back_end/register_allocator.dart @@ -359,8 +359,13 @@ final class LinearScanRegisterAllocator extends RegisterAllocator { ); } } else if (constr is ParameterStackLocation) { - assert(liveRange.splitFrom == null); - liveRange.spillSlot = constr; + if (!graph.function.isSuspendable) { + // Lock spill slot to the parameter location on the stack, + // unless the current function is suspendable + // (suspend/resume does not preserve parameters on the stack). + assert(liveRange.splitFrom == null); + liveRange.spillSlot = constr; + } _operandLocations[operandId] = constr; if (instr.hasUses) { final loc = liveRange.addUse(pos + 1, constr); diff --git a/pkg/native_compiler/lib/back_end/stack_frame.dart b/pkg/native_compiler/lib/back_end/stack_frame.dart index 71aeca4b8c4..5fe19a843e3 100644 --- a/pkg/native_compiler/lib/back_end/stack_frame.dart +++ b/pkg/native_compiler/lib/back_end/stack_frame.dart @@ -42,6 +42,9 @@ abstract base class StackFrame { /// Should be used only after the frame is finalized. int offsetFromFP(StackLocation location); + /// Offset of the suspend state relative to the frame pointer, in bytes. + int get suspendStateOffsetFromFP; + /// Offset of the shadow parameter relative to the frame pointer, in bytes. /// Should be used only after the frame is finalized. int shadowParameterOffsetFromFP(int paramIndex); diff --git a/pkg/native_compiler/lib/snapshot/snapshot.dart b/pkg/native_compiler/lib/snapshot/snapshot.dart index 22c59594378..be54fc2aa80 100644 --- a/pkg/native_compiler/lib/snapshot/snapshot.dart +++ b/pkg/native_compiler/lib/snapshot/snapshot.dart @@ -162,12 +162,23 @@ class SnapshotSerializer { addBaseObject(const ast.NullType()); addBaseObject(const ast.NeverType.nonNullable()); addBaseObject(ast.ListConstant(const ast.DynamicType(), const [])); - // TODO: generate these stubs instead of referencig them from the VM. + // TODO: generate these stubs instead of referencing them from the VM. addBaseObject(StubCode.Subtype1TestCache); addBaseObject(StubCode.Subtype2TestCache); addBaseObject(StubCode.Subtype3TestCache); addBaseObject(StubCode.Subtype4TestCache); addBaseObject(StubCode.Subtype6TestCache); + addBaseObject(StubCode.InitAsync); + addBaseObject(StubCode.InitAsyncStar); + addBaseObject(StubCode.InitSyncStar); + addBaseObject(StubCode.Await); + addBaseObject(StubCode.AwaitWithTypeCheck); + addBaseObject(StubCode.YieldAsyncStar); + addBaseObject(StubCode.SuspendSyncStarAtStart); + addBaseObject(StubCode.SuspendSyncStarAtYield); + addBaseObject(StubCode.ReturnAsync); + addBaseObject(StubCode.ReturnAsyncNotFuture); + addBaseObject(StubCode.ReturnAsyncStar); numObjects = numBaseObjects; } diff --git a/runtime/vm/module_snapshot.cc b/runtime/vm/module_snapshot.cc index 06d619780dc..cf138db5527 100644 --- a/runtime/vm/module_snapshot.cc +++ b/runtime/vm/module_snapshot.cc @@ -1493,6 +1493,21 @@ void Deserializer::Deserialize() { AddBaseObject(StubCode::Subtype3TestCache()); AddBaseObject(StubCode::Subtype4TestCache()); AddBaseObject(StubCode::Subtype6TestCache()); + AddBaseObject(Code::Handle(zone(), object_store->init_async_stub())); + AddBaseObject(Code::Handle(zone(), object_store->init_async_star_stub())); + AddBaseObject(Code::Handle(zone(), object_store->init_sync_star_stub())); + AddBaseObject(Code::Handle(zone(), object_store->await_stub())); + AddBaseObject( + Code::Handle(zone(), object_store->await_with_type_check_stub())); + AddBaseObject(Code::Handle(zone(), object_store->yield_async_star_stub())); + AddBaseObject( + Code::Handle(zone(), object_store->suspend_sync_star_at_start_stub())); + AddBaseObject( + Code::Handle(zone(), object_store->suspend_sync_star_at_yield_stub())); + AddBaseObject(Code::Handle(zone(), object_store->return_async_stub())); + AddBaseObject( + Code::Handle(zone(), object_store->return_async_not_future_stub())); + AddBaseObject(Code::Handle(zone(), object_store->return_async_star_stub())); if (num_base_objects_ != (next_ref_index_ - kFirstReference)) { FATAL("Snapshot expects %" Pd