diff --git a/pkg/cfg/lib/front_end/ast_to_ir.dart b/pkg/cfg/lib/front_end/ast_to_ir.dart index 7f86c7c1630..8222f45395b 100644 --- a/pkg/cfg/lib/front_end/ast_to_ir.dart +++ b/pkg/cfg/lib/front_end/ast_to_ir.dart @@ -366,12 +366,12 @@ class AstToIr extends ast.RecursiveVisitor { } return switch (typeParametersStyle) { .separateFunctionAndClassTypeParameters => [ - visitor.containsFunctionTypeParams - ? functionTypeParameters! - : builder.graph.getConstant(ConstantValue.fromNull()), visitor.containsClassTypeParams ? classTypeParameters! : builder.graph.getConstant(ConstantValue.fromNull()), + visitor.containsFunctionTypeParams + ? functionTypeParameters! + : builder.graph.getConstant(ConstantValue.fromNull()), ], }; } diff --git a/pkg/cfg/lib/ir/instructions.dart b/pkg/cfg/lib/ir/instructions.dart index d2539593e9e..7fdcff779ab 100644 --- a/pkg/cfg/lib/ir/instructions.dart +++ b/pkg/cfg/lib/ir/instructions.dart @@ -1727,8 +1727,10 @@ enum ParallelMoveStage { output, // Spill output of the instruction. spill, - // Split live ranges. + // Split live ranges between instructions. split, + // Split live ranges at the next instruction. + splitLate, // Moves at control flow edges (including phi moves). control, // Move instruction inputs to their fixed locations. diff --git a/pkg/cfg/testcases/expressions.dart.expect b/pkg/cfg/testcases/expressions.dart.expect index b8fb2310693..9652b83a487 100644 --- a/pkg/cfg/testcases/expressions.dart.expect +++ b/pkg/cfg/testcases/expressions.dart.expect @@ -18,7 +18,7 @@ B0 = EntryBlock() v4 = TypeParameters(v1) v7 = DirectCall _GrowableList.(v5, v6) DirectCall print(v7) - v10 = TypeArguments(v4, v9, ) + v10 = TypeArguments(v9, v4, ) v11 = DirectCall _GrowableList.(v10, v6) DirectCall print(v11) v17 = DirectCall _GrowableList._literal3(v13, v14, v15, v16) @@ -46,7 +46,7 @@ B0 = EntryBlock() v7 = TypeParameters(v1) v9 = AllocateMapLiteral(v8) DirectCall print(v9) - v12 = TypeArguments(v7, v11, ) + v12 = TypeArguments(v11, v7, ) v13 = AllocateMapLiteral(v12) DirectCall print(v13) v20 = AllocateMapLiteral(v15, v16, v17, v18, v19) diff --git a/pkg/cfg/testcases/types.dart.expect b/pkg/cfg/testcases/types.dart.expect index 28e071db09f..607ee236ef0 100644 --- a/pkg/cfg/testcases/types.dart.expect +++ b/pkg/cfg/testcases/types.dart.expect @@ -15,7 +15,7 @@ B0 = EntryBlock() v1 = Parameter(this) v3 = TypeParameters(v1) InterfaceCall A.foo(v4, v1, v6) - v9 = TypeArguments(v8, v3, >) + v9 = TypeArguments(v3, v8, >) InterfaceCall A.foo(v9, v1, v11) Return(v8) @@ -27,10 +27,10 @@ B0 = EntryBlock() dominates:(B12, B14, B11) v3 = Parameter(o) TypeParameters(v1) v7 = TypeParameters(v2) - v10 = TypeTest(v3, v9, v7, List) + v10 = TypeTest(v3, v7, v9, List) Branch(v10, true: B11, false: B12) B11 = TargetBlock() idom:B0 - v17 = TypeCast(v3, v9, v7, List, unchecked) + v17 = TypeCast(v3, v7, v9, List, unchecked) TypeCast(v17, Map) Goto(B14) B12 = TargetBlock() idom:B0 @@ -47,9 +47,9 @@ B0 = EntryBlock() v4 = TypeParameters(v1) v6 = TypeParameters(v2) DirectCall print(v7) - v10 = TypeLiteral(A.T%, v9, v6) + v10 = TypeLiteral(A.T%, v6, v9) DirectCall print(v10) - v12 = TypeLiteral(Map, v4, v6) + v12 = TypeLiteral(Map, v6, v4) DirectCall print(v12) Return(v9) @@ -58,7 +58,7 @@ B0 = EntryBlock() v4 = Constant(null) v1 = Parameter(#functionTypeParameters) v3 = TypeParameters(v1) - v5 = TypeArguments(v3, v4, ) + v5 = TypeArguments(v4, v3, ) v6 = AllocateObject A(v5) DirectCall A._(v6) Return(v6) diff --git a/pkg/native_compiler/lib/back_end/arm64/assembler.dart b/pkg/native_compiler/lib/back_end/arm64/assembler.dart index b50fa713f5a..9a0920071f9 100644 --- a/pkg/native_compiler/lib/back_end/arm64/assembler.dart +++ b/pkg/native_compiler/lib/back_end/arm64/assembler.dart @@ -685,6 +685,26 @@ final class Arm64Assembler extends Assembler with Uint32OutputBuffer { } } + @override + void cmpImmediate( + Register reg, + int value, [ + OperandSize sz = OperandSize.s64, + Register scratch = temp2Reg, + ]) { + assert(sz.is32or64); + assert(_isInt(sz.bitWidth, value) || _isUint(sz.bitWidth, value)); + if (canEncodeImm12(value)) { + cmp(reg, Immediate(value), sz); + } else if (canEncodeImm12(-value)) { + cmn(reg, Immediate(-value), sz); + } else { + assert(reg != scratch); + loadImmediate(scratch, value); + cmp(reg, scratch, sz); + } + } + @override void andImmediate( Register dst, @@ -810,6 +830,16 @@ final class Arm64Assembler extends Assembler with Uint32OutputBuffer { addImmediate(resultReg, resultReg, heapObjectTag); } + void loadClassId(Register result, Register object) { + ldr(result, fieldAddress(object, vmOffsets.Object_tags_offset)); + ubfx( + result, + result, + vmOffsets.UntaggedObject_kClassIdTagPos, + vmOffsets.UntaggedObject_kClassIdTagSize, + ); + } + // [rd] and [rn] can be SP if [o] is Immediate or ExtRegOperand. // For an unmodified rm in this case, use ExtRegOperand(rm, Extend.UXTX, 0). void add( diff --git a/pkg/native_compiler/lib/back_end/arm64/code_generator.dart b/pkg/native_compiler/lib/back_end/arm64/code_generator.dart index a016287f9ec..7ad1b3cae88 100644 --- a/pkg/native_compiler/lib/back_end/arm64/code_generator.dart +++ b/pkg/native_compiler/lib/back_end/arm64/code_generator.dart @@ -814,14 +814,253 @@ final class Arm64CodeGenerator extends CodeGenerator { _asm.unimplemented('Unimplemented: code generation for NullCheck'); } + int _getNumberOfInputsForSubtypeTestCache( + ast.DartType type, { + required bool hasInstantiatorTypeArgs, + required bool hasFunctionTypeArgs, + }) { + if (type is ast.ExtensionType) { + type = type.extensionTypeErasure; + } + switch (type) { + case ast.NullType(): + case ast.NeverType(): + case ast.InterfaceType() when type.classNode.typeParameters.isEmpty: + return 1; + case ast.InterfaceType(): + case ast.FutureOrType(): + if (hasFunctionTypeArgs) { + return 4; + } + if (hasInstantiatorTypeArgs) { + return 3; + } + return 2; + case ast.FunctionType(): + case ast.RecordType(): + case ast.TypeParameterType(): + return 6; + case ast.ExtensionType(): + case ast.DynamicType(): + case ast.VoidType(): + case ast.StructuralParameterType(): + case ast.IntersectionType(): + case ast.TypedefType(): + case ast.InvalidType(): + case ast.AuxiliaryType(): + case ast.ExperimentalType(): + throw 'Unexpected type ${type.runtimeType} $type'; + } + } + @override void visitTypeCast(TypeCast instr) { - _asm.unimplemented('Unimplemented: code generation for TypeCast'); + final operandReg = inputReg(instr, 0); + final resultReg = outputReg(instr); + if (operandReg != resultReg) { + _asm.mov(resultReg, operandReg); + } + + if (!instr.isChecked) { + return; + } + + final done = Label(); + late final Label slowPath = addSlowPath(() { + _asm.unimplemented( + 'Unimplemented: code generation for TypeCast slow path', + ); + _asm.b(done); + }); + + // Handle a few built-in types, use TTS for other types. + final type = instr.testedType; + switch (type) { + case ObjectType(): + _asm.cmp(resultReg, nullReg); + _asm.b(slowPath, .equal); + case NullType(): + _asm.cmp(resultReg, nullReg); + _asm.b(slowPath, .notEqual); + case IntType(): + _asm.tbz(resultReg, smiBit, done); + _asm.loadClassId(tempReg, resultReg); + _asm.cmpImmediate(tempReg, ClassId.MintCid.index); + _asm.b(slowPath, .notEqual); + case DoubleType(): + _asm.tbz(resultReg, smiBit, slowPath); + _asm.loadClassId(tempReg, resultReg); + _asm.cmpImmediate(tempReg, ClassId.DoubleCid.index); + _asm.b(slowPath, .notEqual); + case BoolType(): + _asm.tbz(resultReg, smiBit, slowPath); + _asm.loadClassId(tempReg, resultReg); + _asm.cmpImmediate(tempReg, ClassId.BoolCid.index); + _asm.b(slowPath, .notEqual); + case StringType(): + _asm.tbz(resultReg, smiBit, slowPath); + _asm.loadClassId(tempReg, resultReg); + _asm.cmpImmediate(tempReg, ClassId.OneByteStringCid.index); + _asm.b(done, .equal); + _asm.cmpImmediate(tempReg, ClassId.TwoByteStringCid.index); + _asm.b(slowPath, .notEqual); + default: + _asm.tbz( + resultReg, + smiBit, + const IntType().isSubtypeOf(type) ? done : slowPath, + ); + if (type.isNullable) { + _asm.cmp(resultReg, nullReg); + _asm.b(done, .equal); + } + final dartType = type.dartType; + if (dartType is ast.TypeParameterType) { + final declaration = dartType.parameter.declaration; + assert(instr.inputCount == 3); + final instantiatorTypeArgsReg = inputReg(instr, 1); + final functionTypeArgsReg = inputReg(instr, 2); + final typeArgsReg = (declaration is ast.Class) + ? instantiatorTypeArgsReg + : functionTypeArgsReg; + final index = computeIndexOfTypeParameter(dartType.parameter); + _asm.cmp(typeArgsReg, nullReg); + _asm.b(done, .equal); + _asm.ldr( + TypeTestingStub.dstTypeReg, + _asm.address( + typeArgsReg, + vmOffsets.TypeArguments_types_offset + + index * objectLayout.compressedWordSize, + ), + ); + } else { + _asm.loadFromPool(TypeTestingStub.dstTypeReg, dartType); + } + _asm.ldr( + tempReg, + _asm.address( + TypeTestingStub.dstTypeReg, + vmOffsets.AbstractType_type_test_stub_entry_point_offset, + ), + ); + bool isNullConstant(Definition def) => + def is Constant && def.value.isNull; + final hasInstantiatorTypeArgs = + instr.inputCount > 1 && !isNullConstant(instr.inputDefAt(1)); + final hasFunctionTypeArgs = + instr.inputCount > 1 && !isNullConstant(instr.inputDefAt(2)); + final stc = SubtypeTestCache( + _getNumberOfInputsForSubtypeTestCache( + dartType, + hasInstantiatorTypeArgs: hasInstantiatorTypeArgs, + hasFunctionTypeArgs: hasFunctionTypeArgs, + ), + ); + _asm.loadFromPool(TypeTestingStub.subtypeTestCacheReg, stc); + _asm.blr(tempReg); + } + + _asm.bind(done); } @override void visitTypeTest(TypeTest instr) { - _asm.unimplemented('Unimplemented: code generation for TypeTest'); + final operandReg = inputReg(instr, 0); + final resultReg = outputReg(instr); + final doneFalse = Label(); + final doneTrue = Label(); + final done = Label(); + + late final Label slowPath = addSlowPath(() { + _asm.unimplemented( + 'Unimplemented: code generation for TypeTest slow path', + ); + _asm.b(done); + }); + + // Handle a few built-in types, use STC for other types. + final type = instr.testedType; + switch (type) { + case ObjectType(): + _asm.cmp(operandReg, nullReg); + _asm.b(doneTrue, .notEqual); + case NullType(): + _asm.cmp(resultReg, nullReg); + _asm.b(doneTrue, .equal); + case IntType(): + _asm.tbz(resultReg, smiBit, doneTrue); + _asm.loadClassId(tempReg, resultReg); + _asm.cmpImmediate(tempReg, ClassId.MintCid.index); + _asm.b(doneTrue, .equal); + case DoubleType(): + _asm.tbz(resultReg, smiBit, doneFalse); + _asm.loadClassId(tempReg, resultReg); + _asm.cmpImmediate(tempReg, ClassId.DoubleCid.index); + _asm.b(doneTrue, .equal); + case BoolType(): + _asm.tbz(resultReg, smiBit, doneFalse); + _asm.loadClassId(tempReg, resultReg); + _asm.cmpImmediate(tempReg, ClassId.BoolCid.index); + _asm.b(doneTrue, .equal); + case StringType(): + _asm.tbz(resultReg, smiBit, doneFalse); + _asm.loadClassId(tempReg, resultReg); + _asm.cmpImmediate(tempReg, ClassId.OneByteStringCid.index); + _asm.b(doneTrue, .equal); + _asm.cmpImmediate(tempReg, ClassId.TwoByteStringCid.index); + _asm.b(doneTrue, .equal); + default: + _asm.tbz( + resultReg, + smiBit, + const IntType().isSubtypeOf(type) ? doneTrue : doneFalse, + ); + if (type.isNullable) { + _asm.cmp(resultReg, nullReg); + _asm.b(doneTrue, .equal); + } + bool isNullConstant(Definition def) => + def is Constant && def.value.isNull; + final hasInstantiatorTypeArgs = + instr.inputCount > 1 && !isNullConstant(instr.inputDefAt(1)); + final hasFunctionTypeArgs = + instr.inputCount > 1 && !isNullConstant(instr.inputDefAt(2)); + final stc = SubtypeTestCache( + _getNumberOfInputsForSubtypeTestCache( + type.dartType, + hasInstantiatorTypeArgs: hasInstantiatorTypeArgs, + hasFunctionTypeArgs: hasFunctionTypeArgs, + ), + ); + final stub = switch (stc.numInputs) { + 1 => StubCode.Subtype1TestCache, + 2 => StubCode.Subtype2TestCache, + 3 => StubCode.Subtype3TestCache, + 4 => StubCode.Subtype4TestCache, + 6 => StubCode.Subtype6TestCache, + _ => + throw 'Unexpected number of SubtypeTestCache inputs ${stc.numInputs} (type $type)', + }; + _asm.loadFromPool(TypeTestingStub.subtypeTestCacheReg, stc); + _asm.loadFromPool(codeReg, stub); + _asm.ldr( + tempReg, + _asm.fieldAddress(codeReg, vmOffsets.Code_entry_point_offset.first), + ); + _asm.blr(tempReg); + _asm.cmp(TypeTestingStub.subtypeTestCacheResultReg, nullReg); + _asm.b(slowPath, .equal); + _asm.mov(resultReg, TypeTestingStub.subtypeTestCacheResultReg); + _asm.b(done); + } + + _asm.bind(doneFalse); + _asm.loadConstant(resultReg, ConstantValue.fromBool(false)); + _asm.b(done); + _asm.bind(doneTrue); + _asm.loadConstant(resultReg, ConstantValue.fromBool(true)); + _asm.bind(done); } @override @@ -1225,6 +1464,13 @@ final class Arm64CodeGenerator extends CodeGenerator { ); } + @override + Location getMoveTempRegister(RegisterClass registerClass) => + switch (registerClass) { + .cpu => tempReg, + .fpu => fpTempReg, + }; + @override void generateMove(Location from, Location to) { switch (from) { @@ -1265,15 +1511,6 @@ final class Arm64CodeGenerator extends CodeGenerator { 'Unimplemented: code generation for generateLoadConstant', ); } - - @override - void generatePush(Location loc) { - _asm.unimplemented('Unimplemented: code generation for generatePush'); - } - - void generatePop(Location loc) { - _asm.unimplemented('Unimplemented: code generation for generatePop'); - } } extension on ComparisonOpcode { diff --git a/pkg/native_compiler/lib/back_end/arm64/constraints.dart b/pkg/native_compiler/lib/back_end/arm64/constraints.dart index 1a513304b00..1032cfbebfc 100644 --- a/pkg/native_compiler/lib/back_end/arm64/constraints.dart +++ b/pkg/native_compiler/lib/back_end/arm64/constraints.dart @@ -204,24 +204,80 @@ final class Arm64Constraints extends Constraints { const InstructionConstraints(anyCpuRegister, [anyCpuRegister]); @override - InstructionConstraints? visitTypeCast(TypeCast instr) => - InstructionConstraints(anyCpuRegister, [ - anyCpuRegister, - if (instr.inputCount > 1) ...[ - anyRegisterOrImmediate(instr.inputDefAt(1)), - anyRegisterOrImmediate(instr.inputDefAt(2)), + InstructionConstraints? visitTypeCast(TypeCast instr) { + final callsTypeTestingStub = + instr.isChecked && + switch (instr.testedType) { + ObjectType() || + NullType() || + IntType() || + DoubleType() || + BoolType() || + StringType() => false, + _ => true, + }; + if (callsTypeTestingStub) { + return InstructionConstraints( + TypeTestingStub.instanceReg, + [ + TypeTestingStub.instanceReg, + if (instr.inputCount > 1) ...const [ + TypeTestingStub.instantiatorTypeArgumentsReg, + TypeTestingStub.functionTypeArgumentsReg, + ], ], - ]); + const [ + TypeTestingStub.dstTypeReg, + TypeTestingStub.subtypeTestCacheReg, + TypeTestingStub.scratchReg, + ], + ); + } + return InstructionConstraints(anyCpuRegister, [ + anyCpuRegister, + if (instr.inputCount > 1) ...[ + anyRegisterOrImmediate(instr.inputDefAt(1)), + anyRegisterOrImmediate(instr.inputDefAt(2)), + ], + ]); + } @override - InstructionConstraints? visitTypeTest(TypeTest instr) => - InstructionConstraints(anyCpuRegister, [ - anyCpuRegister, - if (instr.inputCount > 1) ...[ - anyRegisterOrImmediate(instr.inputDefAt(1)), - anyRegisterOrImmediate(instr.inputDefAt(2)), + InstructionConstraints? visitTypeTest(TypeTest instr) { + final callsSubtypeTestCacheStub = switch (instr.testedType) { + ObjectType() || + NullType() || + IntType() || + DoubleType() || + BoolType() || + StringType() => false, + _ => true, + }; + if (callsSubtypeTestCacheStub) { + return InstructionConstraints( + TypeTestingStub.subtypeTestCacheResultReg, + [ + TypeTestingStub.instanceReg, + if (instr.inputCount > 1) ...const [ + TypeTestingStub.instantiatorTypeArgumentsReg, + TypeTestingStub.functionTypeArgumentsReg, + ], ], - ]); + const [ + TypeTestingStub.dstTypeReg, + TypeTestingStub.subtypeTestCacheReg, + TypeTestingStub.scratchReg, + ], + ); + } + return InstructionConstraints(anyCpuRegister, [ + anyCpuRegister, + if (instr.inputCount > 1) ...[ + anyRegisterOrImmediate(instr.inputDefAt(1)), + anyRegisterOrImmediate(instr.inputDefAt(2)), + ], + ]); + } @override InstructionConstraints? visitTypeArguments(TypeArguments instr) => diff --git a/pkg/native_compiler/lib/back_end/arm64/stub_code_generator.dart b/pkg/native_compiler/lib/back_end/arm64/stub_code_generator.dart index 0262ee8bd46..4aaab8d4e6a 100644 --- a/pkg/native_compiler/lib/back_end/arm64/stub_code_generator.dart +++ b/pkg/native_compiler/lib/back_end/arm64/stub_code_generator.dart @@ -108,6 +108,16 @@ final class WriteBarrierStub extends Arm64StubCodeGenerator { } } +final class TypeTestingStub { + static const Register instanceReg = R0; + static const Register dstTypeReg = R8; + static const Register instantiatorTypeArgumentsReg = R2; + static const Register functionTypeArgumentsReg = R1; + static const Register subtypeTestCacheReg = R3; + static const Register scratchReg = R4; + static const Register subtypeTestCacheResultReg = R7; +} + final class Arm64StubFactory extends StubFactory { final VMOffsets vmOffsets; final ObjectLayout objectLayout; diff --git a/pkg/native_compiler/lib/back_end/assembler.dart b/pkg/native_compiler/lib/back_end/assembler.dart index 05f2ac78d41..54bc212d280 100644 --- a/pkg/native_compiler/lib/back_end/assembler.dart +++ b/pkg/native_compiler/lib/back_end/assembler.dart @@ -174,6 +174,13 @@ abstract base class Assembler { OperandSize sz = OperandSize.s64, ]); + /// compare [reg] with arbitrary integer [value]. + void cmpImmediate( + Register reg, + int value, [ + OperandSize sz = OperandSize.s64, + ]); + /// [dst] = bitwise and ([src], arbitrary integer [value]). void andImmediate( Register dst, diff --git a/pkg/native_compiler/lib/back_end/code_generator.dart b/pkg/native_compiler/lib/back_end/code_generator.dart index 40e9172bb0b..b176d69f4f9 100644 --- a/pkg/native_compiler/lib/back_end/code_generator.dart +++ b/pkg/native_compiler/lib/back_end/code_generator.dart @@ -220,19 +220,14 @@ abstract base class CodeGenerator extends Pass } } } - for (final move in instr.moves) { - if (move is Move) { - final from = move.from.physicalLocation; - final to = move.to.physicalLocation; - - if (map.containsKey(from)) { - if (map.containsKey(to)) { - _generateDependentMoves(from, to, map); - } else { - generateMove(from, to); - map.remove(from); - } - } + while (map.isNotEmpty) { + final from = map.keys.first; + final to = map[from]!; + if (map.containsKey(to)) { + _generateDependentMoves(from, to, map); + } else { + generateMove(from, to); + map.remove(from); } } for (final move in instr.moves) { @@ -248,23 +243,28 @@ abstract base class CodeGenerator extends Pass Map moves, ) { assert(from != to); + assert(moves[from] == to); final pendingList = [from]; final pendingSet = {from}; // Visit the chain of dependent moves until it ends or cycle is found. while (moves.containsKey(to)) { if (pendingSet.contains(to)) { - // Moves form a cycle. Save value on the stack to generate moves. + // Moves form a cycle. Save value to the temporary register to generate moves. // TODO: regalloc should provide scratch register(s) for // ParallelMove instructions if there are available registers. // TODO: we can also allocate a scratch register from ParallelMove // itself, resusing source registers which are already moved out or // destination registers which are not moved in yet. - // TODO: as a last resort, allocate a scratch space on the stack and - // avoid any push/pop. - generatePush(to); - for (final from in pendingList.reversed) { + final temp = getMoveTempRegister( + (to is FPRegister || moves[to] is FPRegister) + ? RegisterClass.fpu + : RegisterClass.cpu, + ); + generateMove(to, temp); + while (pendingList.isNotEmpty) { + from = pendingList.removeLast(); if (from == to) { - generatePop(moves.remove(from)!); + generateMove(temp, moves.remove(from)!); break; } generateMove(from, moves.remove(from)!); @@ -281,10 +281,9 @@ abstract base class CodeGenerator extends Pass } } + Location getMoveTempRegister(RegisterClass registerClass); void generateMove(Location from, Location to); void generateLoadConstant(ConstantValue value, Location to); - void generatePush(Location loc); - void generatePop(Location loc); @override void visitTypeParameters(TypeParameters instr) => diff --git a/pkg/native_compiler/lib/back_end/object_pool.dart b/pkg/native_compiler/lib/back_end/object_pool.dart index faad2bba268..9ee4f4924bf 100644 --- a/pkg/native_compiler/lib/back_end/object_pool.dart +++ b/pkg/native_compiler/lib/back_end/object_pool.dart @@ -122,3 +122,13 @@ final class StaticFieldOffset extends SpecializedEntry { bool operator ==(Object other) => other is StaticFieldOffset && this.field == other.field; } + +/// Object pool entry representing a subtype test cache. +/// This is not a specialized entry, it is encoded as a regular object reference. +final class SubtypeTestCache { + final int numInputs; + SubtypeTestCache(this.numInputs); + + // Use identity hashCode and == as separate subtype test caches are + // used for each type check. +} diff --git a/pkg/native_compiler/lib/back_end/register_allocator.dart b/pkg/native_compiler/lib/back_end/register_allocator.dart index 675fb87a0d7..2aa127dbeb6 100644 --- a/pkg/native_compiler/lib/back_end/register_allocator.dart +++ b/pkg/native_compiler/lib/back_end/register_allocator.dart @@ -161,7 +161,7 @@ final class LinearScanRegisterAllocator extends RegisterAllocator { } errorContext.annotator = (Instruction instr) => - '[${instructionPos(instr)}]'; + (instr is ParallelMove) ? null : '[${instructionPos(instr)}]'; } void buildLiveRanges(SSALivenessAnalysis liveness) { @@ -170,6 +170,7 @@ final class LinearScanRegisterAllocator extends RegisterAllocator { _fpuRegLiveRanges = List.filled(constraints.getNumberOfFPRegisters(), null); for (final block in backEndState.codeGenBlockOrder.reversed) { + currentBlock = block; final blockStart = blockStartPos(block); final blockEnd = blockEndPos(block); @@ -203,6 +204,7 @@ final class LinearScanRegisterAllocator extends RegisterAllocator { } for (final instr in block.reversed) { + currentInstruction = instr; if (instr is CallInstruction) { backEndState.stackFrame.allocateArgumentsSlots(instr); } @@ -811,15 +813,25 @@ final class LinearScanRegisterAllocator extends RegisterAllocator { if (instr is JoinBlock && instr.hasPhis) { instr = instr.phis.last; } + ParallelMoveStage stage; if (instr is! Goto && next.start.isOdd) { instr = _nextInstruction(instr); + stage = ParallelMoveStage.split; + } else { + stage = ParallelMoveStage.splitLate; } _insertMoveBefore( instr, - ParallelMoveStage.split, + stage, liveRange.allocatedLocation!, next.allocatedLocation!, ); + if (trace) { + print( + 'Insert split move at ${next.start} (before ${IrToText.instruction(instr)})' + ' ${liveRange} ${liveRange.allocatedLocation} => $next ${next.allocatedLocation}', + ); + } } liveRange = next; } diff --git a/pkg/native_compiler/lib/runtime/type_utils.dart b/pkg/native_compiler/lib/runtime/type_utils.dart index 65a975a8929..f891c91eb48 100644 --- a/pkg/native_compiler/lib/runtime/type_utils.dart +++ b/pkg/native_compiler/lib/runtime/type_utils.dart @@ -100,6 +100,23 @@ bool isAllDynamic(List typeArgs) { return true; } +/// Calculate index of [tp] in the type arguments vector. +int computeIndexOfTypeParameter(ast.TypeParameter tp) { + final decl = tp.declaration!; + int index = decl.typeParameters.indexOf(tp); + assert(index >= 0); + if (decl is ast.LocalFunction) { + ast.TreeNode node = decl.parent!; + while (node is! ast.Member) { + if (node is ast.FunctionNode) { + index += node.typeParameters.length; + } + node = node.parent!; + } + } + return index; +} + /// Returns true if [field] has a non-trivial initializer. /// /// VM does not allow field initializer functions for fields diff --git a/pkg/native_compiler/lib/snapshot/snapshot.dart b/pkg/native_compiler/lib/snapshot/snapshot.dart index f51c19962c9..5a3ad49e1b2 100644 --- a/pkg/native_compiler/lib/snapshot/snapshot.dart +++ b/pkg/native_compiler/lib/snapshot/snapshot.dart @@ -21,6 +21,7 @@ import 'package:native_compiler/configuration.dart'; import 'package:native_compiler/runtime/names.dart'; import 'package:native_compiler/runtime/object_layout.dart'; import 'package:native_compiler/runtime/type_utils.dart'; +import 'package:native_compiler/runtime/vm_defs.dart'; /// Kinds of Dart snapshots. /// Should match Snapshot::Kind enum in runtime/vm/snapshot.h. @@ -77,6 +78,7 @@ enum PredefinedClusters { typeArguments, codes, icDatas, + subtypeTestCaches, objectPools, instances, // Separate cluster for every class. } @@ -160,6 +162,12 @@ class SnapshotSerializer { addBaseObject(const ast.NullType()); addBaseObject(const ast.NeverType.nonNullable()); addBaseObject(ast.ListConstant(const ast.DynamicType(), const [])); + // TODO: generate these stubs instead of referencig them from the VM. + addBaseObject(StubCode.Subtype1TestCache); + addBaseObject(StubCode.Subtype2TestCache); + addBaseObject(StubCode.Subtype3TestCache); + addBaseObject(StubCode.Subtype4TestCache); + addBaseObject(StubCode.Subtype6TestCache); numObjects = numBaseObjects; } @@ -342,6 +350,9 @@ class SnapshotSerializer { // Generated code and object pool Code() => getPredefinedCluster(PredefinedClusters.codes), ICData() => getPredefinedCluster(PredefinedClusters.icDatas), + SubtypeTestCache() => getPredefinedCluster( + PredefinedClusters.subtypeTestCaches, + ), ObjectPool() => getPredefinedCluster(PredefinedClusters.objectPools), _ => throw 'Unxpected ${obj.runtimeType} $obj', }; @@ -381,6 +392,7 @@ class SnapshotSerializer { .typeParameterTypes => throw 'Unimplemented cluster $clusterId', .codes => CodeSerializationCluster(), .icDatas => ICDataSerializationCluster(), + .subtypeTestCaches => SubtypeTestCacheSerializationCluster(), .objectPools => ObjectPoolSerializationCluster(), .instances => throw 'Each class has a separate instance cluster', }; @@ -1270,6 +1282,36 @@ final class ICDataSerializationCluster extends SerializationCluster { } } +final class SubtypeTestCacheSerializationCluster extends SerializationCluster { + final List _objects = []; + + @override + void trace(SnapshotSerializer serializer, Object object) { + final obj = object as SubtypeTestCache; + _objects.add(obj); + } + + @override + void writePreLoad(SnapshotSerializer serializer) { + serializer.writeUint(PredefinedClusters.subtypeTestCaches.index); + } + + @override + void writeAlloc(SnapshotSerializer serializer) { + serializer.writeUint(_objects.length); + for (final obj in _objects) { + serializer.assignRef(obj); + } + } + + @override + void writeFill(SnapshotSerializer serializer) { + for (final obj in _objects) { + serializer.writeUint(obj.numInputs); + } + } +} + final class ObjectPoolSerializationCluster extends SerializationCluster { final List _objects = []; final Map icDatas = {}; diff --git a/pkg/native_compiler/lib/utils/interval_list.dart b/pkg/native_compiler/lib/utils/interval_list.dart index 68c65087e27..745dd510a1d 100644 --- a/pkg/native_compiler/lib/utils/interval_list.dart +++ b/pkg/native_compiler/lib/utils/interval_list.dart @@ -58,7 +58,7 @@ class IntervalList { assert(start < end); if (!isEmpty) { // Intervals should be added in descending order. - assert(end < endAt(0)); + assert(end <= endAt(0)); if (start == startAt(0)) { // Ignore nested interval. return; diff --git a/pkg/native_compiler/test/back_end/arm64/assembler_test.dart b/pkg/native_compiler/test/back_end/arm64/assembler_test.dart index 282f34667e8..d263735d00e 100644 --- a/pkg/native_compiler/test/back_end/arm64/assembler_test.dart +++ b/pkg/native_compiler/test/back_end/arm64/assembler_test.dart @@ -311,6 +311,34 @@ void main() { 'sub csp, fp, r17 uxtx 0\n', ); }); + test('cmpImmediate', () { + asm.cmpImmediate(R1, 0); + asm.cmpImmediate(R3, 0xabc); + asm.cmpImmediate(R4, 0xabc000); + asm.cmpImmediate(R0, 0x1001); + asm.cmpImmediate(R6, 0x11223344_55667788); + asm.cmpImmediate(FP, -1); + asm.cmpImmediate(R4, -0xabc); + asm.cmpImmediate(R5, -0xabc000); + asm.cmpImmediate(R7, -0x1001); + expectDisassembly( + 'cmp r1, #0x0\n' + 'cmp r3, #0xabc\n' + 'cmp r4, #0xabc000\n' + 'movz r17, #0x1001\n' + 'cmp r0, r17\n' + 'movz r17, #0x7788\n' + 'movk r17, #0x5566 lsl 16\n' + 'movk r17, #0x3344 lsl 32\n' + 'movk r17, #0x1122 lsl 48\n' + 'cmp r6, r17\n' + 'cmn fp, #0x1\n' + 'cmn r4, #0xabc\n' + 'cmn r5, #0xabc000\n' + 'movn r17, #0x1000\n' + 'cmp r7, r17\n', + ); + }); test('andImmediate', () { asm.andImmediate(R1, R2, 0); asm.andImmediate(R1, R2, 0, .u32); @@ -444,6 +472,21 @@ void main() { 'add r0, r0, #0x1\n', ); }); + test('loadClassId', () { + asm.loadClassId(R0, R0); + asm.loadClassId(R1, R5); + final lowBit = vmOffsets.UntaggedObject_kClassIdTagPos; + final highBit = + vmOffsets.UntaggedObject_kClassIdTagPos + + vmOffsets.UntaggedObject_kClassIdTagSize - + 1; + expectDisassembly( + 'ldr r0, [r0, #${vmOffsets.Object_tags_offset - heapObjectTag}]\n' + 'ubfm r0, r0, #$lowBit, #$highBit\n' + 'ldr r1, [r5, #${vmOffsets.Object_tags_offset - heapObjectTag}]\n' + 'ubfm r1, r1, #$lowBit, #$highBit\n', + ); + }); }); group('instruction', () { diff --git a/pkg/native_compiler/testcases/lowering_test.dart.expect b/pkg/native_compiler/testcases/lowering_test.dart.expect index 70ef400a80a..1935dcb017e 100644 --- a/pkg/native_compiler/testcases/lowering_test.dart.expect +++ b/pkg/native_compiler/testcases/lowering_test.dart.expect @@ -11,20 +11,28 @@ B0 = EntryBlock() B0 = EntryBlock() v9 = Constant(null) v1 = Parameter(#functionTypeParameters) # RA: param[0] <- () - ParallelMove output(param[0] -> vloc:R0) + ParallelMove output(param[0] -> vloc:R5) v2 = Parameter(this) # RA: param[1] <- () - ParallelMove output(param[1] -> vloc:R1) + ParallelMove output(param[1] -> vloc:R0) v3 = Parameter(x) # RA: param[2] <- () - ParallelMove output(param[2] -> vloc:R2) - v20 = LoadInstanceField(C.#typeArguments, v2) # RA: R1 <- (R1) - ParallelMove spill(R1 -> stack[0]) - v10 = TypeTest(v3, v9, v20, List) # RA: R3 <- (R2, -, R1) - DirectCall print(v10) # RA: R0 <- (R3) temps: [R1, R2, R3, R4, R5, R6, R7, R8, R9, R10, R11, R12, R13, R14, R19, R20, R23, R25, V0, V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V11, V12, V13, V14, V15, V16, V17, V18, V19, V20, V21, V22, V23, V24, V25, V26, V27, V28, V29, V30] - ParallelMove split(param[0] -> R1, param[2] -> R2) - v13 = TypeTest(v3, v1, v9, List) # RA: R0 <- (R2, R1, -) - DirectCall print(v13) # RA: R0 <- (R0) temps: [R1, R2, R3, R4, R5, R6, R7, R8, R9, R10, R11, R12, R13, R14, R19, R20, R23, R25, V0, V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V11, V12, V13, V14, V15, V16, V17, V18, V19, V20, V21, V22, V23, V24, V25, V26, V27, V28, V29, V30] - ParallelMove split(param[0] -> R2, param[2] -> R3, stack[0] -> R1) - v16 = TypeCast(v3, v1, v20, Map) # RA: R0 <- (R3, R2, R1) + ParallelMove output(param[2] -> vloc:R6) + v20 = LoadInstanceField(C.#typeArguments, v2) # RA: R2 <- (R0) + ParallelMove spill(R2 -> stack[0]) + ParallelMove splitLate(R2 -> R9) + ParallelMove input(vloc:R6 -> R0, vloc:R2 -> R2, NullConstant(null) -> R1) + v10 = TypeTest(v3, v20, v9, List) # RA: R7 <- (R0, R2, R1) temps: [R8, R3, R4] + ParallelMove output(R7 -> vloc:R7) + DirectCall print(v10) # RA: R0 <- (R7) temps: [R1, R2, R3, R4, R5, R6, R7, R8, R9, R10, R11, R12, R13, R14, R19, R20, R23, R25, V0, V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V11, V12, V13, V14, V15, V16, V17, V18, V19, V20, V21, V22, V23, V24, V25, V26, V27, V28, V29, V30] + ParallelMove split(param[0] -> R1, param[2] -> R5) + ParallelMove splitLate(R1 -> R6) + ParallelMove input(vloc:R5 -> R0, NullConstant(null) -> R2, vloc:R1 -> R1) + v13 = TypeTest(v3, v9, v1, List) # RA: R7 <- (R0, R2, R1) temps: [R8, R3, R4] + ParallelMove output(R7 -> vloc:R7) + DirectCall print(v13) # RA: R0 <- (R7) temps: [R1, R2, R3, R4, R5, R6, R7, R8, R9, R10, R11, R12, R13, R14, R19, R20, R23, R25, V0, V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V11, V12, V13, V14, V15, V16, V17, V18, V19, V20, V21, V22, V23, V24, V25, V26, V27, V28, V29, V30] + ParallelMove split(param[0] -> R3, param[2] -> R1, stack[0] -> R2) + ParallelMove input(vloc:R1 -> R0, vloc:R2 -> R2, vloc:R3 -> R1) + v16 = TypeCast(v3, v20, v1, Map) # RA: R0 <- (R0, R2, R1) temps: [R8, R3, R4] + ParallelMove output(R0 -> vloc:R0) DirectCall print(v16) # RA: R0 <- (R0) temps: [R1, R2, R3, R4, R5, R6, R7, R8, R9, R10, R11, R12, R13, R14, R19, R20, R23, R25, V0, V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V11, V12, V13, V14, V15, V16, V17, V18, V19, V20, V21, V22, V23, V24, V25, V26, V27, V28, V29, V30] ParallelMove input(NullConstant(null) -> R0) Return(v9) # RA: (R0) @@ -215,7 +223,7 @@ B0 = EntryBlock() ParallelMove output(R0 -> vloc:R0) DirectCall print(v7) # RA: R0 <- (R0) temps: [R1, R2, R3, R4, R5, R6, R7, R8, R9, R10, R11, R12, R13, R14, R19, R20, R23, R25, V0, V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V11, V12, V13, V14, V15, V16, V17, V18, V19, V20, V21, V22, V23, V24, V25, V26, V27, V28, V29, V30] ParallelMove split(param[0] -> R1) - v10 = TypeArguments(v1, v9, ) # RA: R0 <- (R1, -) + v10 = TypeArguments(v9, v1, ) # RA: R0 <- (-, R1) ParallelMove spill(R0 -> stack[0]) v11 = DirectCall _GrowableList.(v10, v6) # RA: R0 <- (R0, -) temps: [R1, R2, R3, R4, R5, R6, R7, R8, R9, R10, R11, R12, R13, R14, R19, R20, R23, R25, V0, V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V11, V12, V13, V14, V15, V16, V17, V18, V19, V20, V21, V22, V23, V24, V25, V26, V27, V28, V29, V30] ParallelMove output(R0 -> vloc:R0) @@ -292,7 +300,7 @@ B0 = EntryBlock() ParallelMove output(R0 -> vloc:R0) DirectCall print(v32) # RA: R0 <- (R0) temps: [R1, R2, R3, R4, R5, R6, R7, R8, R9, R10, R11, R12, R13, R14, R19, R20, R23, R25, V0, V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V11, V12, V13, V14, V15, V16, V17, V18, V19, V20, V21, V22, V23, V24, V25, V26, V27, V28, V29, V30] ParallelMove split(param[0] -> R1) - v12 = TypeArguments(v1, v11, ) # RA: R0 <- (R1, -) + v12 = TypeArguments(v11, v1, ) # RA: R0 <- (-, R1) ParallelMove spill(R0 -> stack[0]) v33 = DirectCall Map._fromLiteral(v12, v31) # RA: R0 <- (R0, -) temps: [R1, R2, R3, R4, R5, R6, R7, R8, R9, R10, R11, R12, R13, R14, R19, R20, R23, R25, V0, V1, V2, V3, V4, V5, V6, V7, V8, V9, V10, V11, V12, V13, V14, V15, V16, V17, V18, V19, V20, V21, V22, V23, V24, V25, V26, V27, V28, V29, V30] ParallelMove output(R0 -> vloc:R0) diff --git a/runtime/vm/module_snapshot.cc b/runtime/vm/module_snapshot.cc index a121e39fdc0..90c7d8b46a9 100644 --- a/runtime/vm/module_snapshot.cc +++ b/runtime/vm/module_snapshot.cc @@ -73,6 +73,7 @@ class ModuleSnapshot : public AllStatic { kTypeArguments, kCodes, kICDatas, + kSubtypeTestCaches, kObjectPools, kInstances, }; @@ -1127,6 +1128,30 @@ class ICDataDeserializationCluster : public DeserializationCluster { } }; +class SubtypeTestCacheDeserializationCluster : public DeserializationCluster { + public: + SubtypeTestCacheDeserializationCluster() + : DeserializationCluster("SubtypeTestCache") {} + ~SubtypeTestCacheDeserializationCluster() {} + + void ReadAlloc(Deserializer* d) override { + ReadAllocFixedSize(d, SubtypeTestCache::InstanceSize()); + } + + void ReadFill(Deserializer* d_) override { + Deserializer::Local d(d_); + + for (intptr_t id = start_index_, n = stop_index_; id < n; id++) { + SubtypeTestCachePtr stc = static_cast(d.Ref(id)); + Deserializer::InitializeHeader(stc, kSubtypeTestCacheCid, + SubtypeTestCache::InstanceSize()); + stc->untag()->cache_ = Object::empty_subtype_test_cache_array().ptr(); + stc->untag()->num_inputs_ = d.ReadUnsigned(); + stc->untag()->num_occupied_ = 0; + } + } +}; + class ObjectPoolDeserializationCluster : public DeserializationCluster { public: ObjectPoolDeserializationCluster() : DeserializationCluster("ObjectPool") {} @@ -1350,6 +1375,8 @@ DeserializationCluster* Deserializer::ReadCluster() { return new (Z) CodeDeserializationCluster(Z); case ModuleSnapshot::kICDatas: return new (Z) ICDataDeserializationCluster(); + case ModuleSnapshot::kSubtypeTestCaches: + return new (Z) SubtypeTestCacheDeserializationCluster(); case ModuleSnapshot::kObjectPools: return new (Z) ObjectPoolDeserializationCluster(); case ModuleSnapshot::kInstances: { @@ -1406,6 +1433,11 @@ void Deserializer::Deserialize() { AddBaseObject(Type::Handle(zone(), object_store->null_type())); AddBaseObject(Type::Handle(zone(), object_store->never_type())); AddBaseObject(Object::empty_array()); + AddBaseObject(StubCode::Subtype1TestCache()); + AddBaseObject(StubCode::Subtype2TestCache()); + AddBaseObject(StubCode::Subtype3TestCache()); + AddBaseObject(StubCode::Subtype4TestCache()); + AddBaseObject(StubCode::Subtype6TestCache()); if (num_base_objects_ != (next_ref_index_ - kFirstReference)) { FATAL("Snapshot expects %" Pd diff --git a/runtime/vm/raw_object.h b/runtime/vm/raw_object.h index 4274c8e3240..acc0a66d46a 100644 --- a/runtime/vm/raw_object.h +++ b/runtime/vm/raw_object.h @@ -58,6 +58,7 @@ class ListDeserializationCluster; class MapDeserializationCluster; class ObjectPoolDeserializationCluster; class SetDeserializationCluster; +class SubtypeTestCacheDeserializationCluster; class TypeArgumentsDeserializationCluster; } // namespace module_snapshot @@ -2789,6 +2790,7 @@ class UntaggedSubtypeTestCache : public UntaggedObject { uint32_t num_occupied_; friend class Interpreter; + friend class module_snapshot::SubtypeTestCacheDeserializationCluster; }; class UntaggedLoadingUnit : public UntaggedObject {