diff --git a/runtime/vm/compiler/assembler/assembler_riscv.cc b/runtime/vm/compiler/assembler/assembler_riscv.cc index 987508b4897..0fa2904f6dc 100644 --- a/runtime/vm/compiler/assembler/assembler_riscv.cc +++ b/runtime/vm/compiler/assembler/assembler_riscv.cc @@ -2014,6 +2014,79 @@ void MicroAssembler::ssamoswapd(Register rd, } #endif // XLEN >= 64 +void MicroAssembler::vsetvli(Register rd, + Register rs1, + ElementWidth sew, + LengthMultiplier lmul, + TailMode vta, + MaskMode vma) { + ASSERT(Supports(RV_V)); + intx_t vtypei = (vma << 7) | (vta << 6) | (sew << 3) | (lmul << 0); + EmitIType(vtypei, rs1, OPCFG, rd, OPV); +} + +void MicroAssembler::vle8v(VRegister vd, Address rs1, VectorMask vm) { + ASSERT(Supports(RV_V)); + ASSERT(rs1.offset() == 0); + Emit32(EncodeOpcode(LOADFP) | EncodeVd(vd) | EncodeRs1(rs1.base()) | + EncodeFunct3(E8) | vm); +} + +void MicroAssembler::vle16v(VRegister vd, Address rs1, VectorMask vm) { + ASSERT(Supports(RV_V)); + ASSERT(rs1.offset() == 0); + Emit32(EncodeOpcode(LOADFP) | EncodeVd(vd) | EncodeRs1(rs1.base()) | + EncodeFunct3(E16) | vm); +} + +void MicroAssembler::vle32v(VRegister vd, Address rs1, VectorMask vm) { + ASSERT(Supports(RV_V)); + ASSERT(rs1.offset() == 0); + Emit32(EncodeOpcode(LOADFP) | EncodeVd(vd) | EncodeRs1(rs1.base()) | + EncodeFunct3(E32) | vm); +} + +void MicroAssembler::vle64v(VRegister vd, Address rs1, VectorMask vm) { + ASSERT(Supports(RV_V)); + ASSERT(rs1.offset() == 0); + Emit32(EncodeOpcode(LOADFP) | EncodeVd(vd) | EncodeRs1(rs1.base()) | + EncodeFunct3(E64) | vm); +} + +void MicroAssembler::vse8v(VRegister vs3, Address rs1, VectorMask vm) { + ASSERT(Supports(RV_V)); + ASSERT(rs1.offset() == 0); + Emit32(EncodeOpcode(STOREFP) | EncodeVs3(vs3) | EncodeRs1(rs1.base()) | + EncodeFunct3(E8) | vm); +} + +void MicroAssembler::vse16v(VRegister vs3, Address rs1, VectorMask vm) { + ASSERT(Supports(RV_V)); + ASSERT(rs1.offset() == 0); + Emit32(EncodeOpcode(STOREFP) | EncodeVs3(vs3) | EncodeRs1(rs1.base()) | + EncodeFunct3(E16) | vm); +} + +void MicroAssembler::vse32v(VRegister vs3, Address rs1, VectorMask vm) { + ASSERT(Supports(RV_V)); + ASSERT(rs1.offset() == 0); + Emit32(EncodeOpcode(STOREFP) | EncodeVs3(vs3) | EncodeRs1(rs1.base()) | + EncodeFunct3(E32) | vm); +} + +void MicroAssembler::vse64v(VRegister vs3, Address rs1, VectorMask vm) { + ASSERT(Supports(RV_V)); + ASSERT(rs1.offset() == 0); + Emit32(EncodeOpcode(STOREFP) | EncodeVs3(vs3) | EncodeRs1(rs1.base()) | + EncodeFunct3(E64) | vm); +} + +void MicroAssembler::vmvvx(VRegister vd, Register rs1, VectorMask vm) { + ASSERT(Supports(RV_V)); + Emit32(EncodeOpcode(OPV) | EncodeVd(vd) | EncodeRs1(rs1) | + EncodeFunct3(OPIVX) | EncodeFunct6(VMV) | vm); +} + void MicroAssembler::lb(Register rd, Address addr, std::memory_order order) { ASSERT(addr.offset() == 0); ASSERT((order == std::memory_order_acquire) || diff --git a/runtime/vm/compiler/assembler/assembler_riscv.h b/runtime/vm/compiler/assembler/assembler_riscv.h index d80e33a68e3..027cb8c3b94 100644 --- a/runtime/vm/compiler/assembler/assembler_riscv.h +++ b/runtime/vm/compiler/assembler/assembler_riscv.h @@ -723,6 +723,23 @@ class MicroAssembler : public AssemblerBase { std::memory_order order = std::memory_order_relaxed); #endif // XLEN >= 64 + // ==== RV32V: Vectors ==== + void vsetvli(Register rd, + Register rs1, + ElementWidth sew, + LengthMultiplier lmul, + TailMode vta, + MaskMode vma); + void vle8v(VRegister vd, Address rs1, VectorMask vm = unmasked); + void vle16v(VRegister vd, Address rs1, VectorMask vm = unmasked); + void vle32v(VRegister vd, Address rs1, VectorMask vm = unmasked); + void vle64v(VRegister vd, Address rs1, VectorMask vm = unmasked); + void vse8v(VRegister vs3, Address rs1, VectorMask vm = unmasked); + void vse16v(VRegister vs3, Address rs1, VectorMask vm = unmasked); + void vse32v(VRegister vs3, Address rs1, VectorMask vm = unmasked); + void vse64v(VRegister vs3, Address rs1, VectorMask vm = unmasked); + void vmvvx(VRegister vd, Register rs1, VectorMask vm = unmasked); + // ==== Zalasr: Load-acquire, store-release ==== void lb(Register rd, Address addr, std::memory_order order); void lh(Register rd, Address addr, std::memory_order order); diff --git a/runtime/vm/compiler/assembler/assembler_riscv_test.cc b/runtime/vm/compiler/assembler/assembler_riscv_test.cc index 964e0b4b589..dfe40d24e5b 100644 --- a/runtime/vm/compiler/assembler/assembler_riscv_test.cc +++ b/runtime/vm/compiler/assembler/assembler_riscv_test.cc @@ -8479,6 +8479,261 @@ ASSEMBLER_TEST_RUN(DoubleLessOrEqualQuiet, test) { EXPECT_EQ(0, CallI(test->entry(), qNAN, -3.0)); } +ASSEMBLER_TEST_GENERATE(VectorMemoryCopy, assembler) { + __ SetExtensions(RV_GC | RV_V); + Label loop; + __ Bind(&loop); + __ vsetvli(A4, A2, e8, m8, ta, ma); + __ vle8v(V0, Address(A1)); + __ add(A1, A1, A4); + __ sub(A2, A2, A4); + __ vse8v(V0, Address(A0)); + __ add(A0, A0, A4); + __ bnez(A2, &loop); + __ ret(); +} +ASSEMBLER_TEST_RUN(VectorMemoryCopy, test) { + EXPECT_DISASSEMBLY( + "0c367757 vsetvli tmp2, a2, e8, m8, ta, ma\n" + "02058007 vle8.v v0, (a1)\n" + " 95ba add a1, a1, tmp2\n" + " 8e19 sub a2, a2, tmp2\n" + "02050027 vse8.v v0, (a0)\n" + " 953a add a0, a0, tmp2\n" + " f67d bnez a2, -18\n" + " 8082 ret\n"); + + intptr_t len = 1000; + uint8_t* src = reinterpret_cast(malloc(len)); + uint8_t* dst = reinterpret_cast(malloc(len)); + for (intptr_t i = 0; i < len; i++) { + src[i] = i & 0xFF; + dst[i] = 0xFF; + } + + Call(test->entry(), reinterpret_cast(dst), + reinterpret_cast(src), len); + for (intptr_t i = 0; i < len; i++) { + EXPECT_EQ(i & 0xFF, src[i]); + EXPECT_EQ(i & 0xFF, dst[i]); + } + + // AVL < VLEN + dst[0] = 0xFF; + Call(test->entry(), reinterpret_cast(dst), + reinterpret_cast(src), 1); + EXPECT_EQ(0, dst[0]); + + // AVL = 0 + dst[0] = 0xFF; + Call(test->entry(), reinterpret_cast(dst), + reinterpret_cast(src), 0); + EXPECT_EQ(0xFF, dst[0]); + + free(src); + free(dst); +} + +ASSEMBLER_TEST_GENERATE(VectorMemorySet8, assembler) { + __ SetExtensions(RV_GC | RV_V); + Label loop; + __ Bind(&loop); + __ vsetvli(A4, A2, e8, m8, ta, ma); + __ vmvvx(V0, A1); + __ sub(A2, A2, A4); + __ vse8v(V0, Address(A0)); + __ add(A0, A0, A4); + __ bnez(A2, &loop); + __ ret(); +} +ASSEMBLER_TEST_RUN(VectorMemorySet8, test) { + EXPECT_DISASSEMBLY( + "0c367757 vsetvli tmp2, a2, e8, m8, ta, ma\n" + "5e05c057 vmv.v.x v0, a1\n" + " 8e19 sub a2, a2, tmp2\n" + "02050027 vse8.v v0, (a0)\n" + " 953a add a0, a0, tmp2\n" + " fa65 bnez a2, -16\n" + " 8082 ret\n"); + + intptr_t len = 100; + uint8_t* dst = reinterpret_cast(malloc(len * 1)); + for (intptr_t i = 0; i < len; i++) { + dst[i] = 0; + } + + Call(test->entry(), reinterpret_cast(dst), 0x12, len); + for (intptr_t i = 0; i < len; i++) { + EXPECT_EQ(0x12u, dst[i]); + } + + // AVL < VLEN + dst[0] = 0; + dst[1] = 0; + Call(test->entry(), reinterpret_cast(dst), 0x23, 1); + EXPECT_EQ(0x23u, dst[0]); + EXPECT_EQ(0u, dst[1]); + + // AVL = 0 + dst[0] = 0; + Call(test->entry(), reinterpret_cast(dst), 0xFF, 0); + EXPECT_EQ(0u, dst[0]); + + free(dst); +} + +ASSEMBLER_TEST_GENERATE(VectorMemorySet16, assembler) { + __ SetExtensions(RV_GC | RV_V); + Label loop; + __ Bind(&loop); + __ vsetvli(A4, A2, e16, m8, ta, ma); + __ vmvvx(V0, A1); + __ sub(A2, A2, A4); + __ vse16v(V0, Address(A0)); + __ slli(A4, A4, 1); + __ add(A0, A0, A4); + __ bnez(A2, &loop); + __ ret(); +} +ASSEMBLER_TEST_RUN(VectorMemorySet16, test) { + EXPECT_DISASSEMBLY( + "0cb67757 vsetvli tmp2, a2, e16, m8, ta, ma\n" + "5e05c057 vmv.v.x v0, a1\n" + " 8e19 sub a2, a2, tmp2\n" + "02055027 vse16.v v0, (a0)\n" + " 0706 slli tmp2, tmp2, 0x1\n" + " 953a add a0, a0, tmp2\n" + " f67d bnez a2, -18\n" + " 8082 ret\n"); + + intptr_t len = 100; + uint16_t* dst = reinterpret_cast(malloc(len * 2)); + for (intptr_t i = 0; i < len; i++) { + dst[i] = 0; + } + + Call(test->entry(), reinterpret_cast(dst), 0x1234, len); + for (intptr_t i = 0; i < len; i++) { + EXPECT_EQ(0x1234u, dst[i]); + } + + // AVL < VLEN + dst[0] = 0; + dst[1] = 0; + Call(test->entry(), reinterpret_cast(dst), 0x2345, 1); + EXPECT_EQ(0x2345u, dst[0]); + EXPECT_EQ(0u, dst[1]); + + // AVL = 0 + dst[0] = 0; + Call(test->entry(), reinterpret_cast(dst), 0xFFFF, 0); + EXPECT_EQ(0u, dst[0]); + + free(dst); +} + +ASSEMBLER_TEST_GENERATE(VectorMemorySet32, assembler) { + __ SetExtensions(RV_GC | RV_V); + Label loop; + __ Bind(&loop); + __ vsetvli(A4, A2, e32, m8, ta, ma); + __ vmvvx(V0, A1); + __ sub(A2, A2, A4); + __ vse32v(V0, Address(A0)); + __ slli(A4, A4, 2); + __ add(A0, A0, A4); + __ bnez(A2, &loop); + __ ret(); +} +ASSEMBLER_TEST_RUN(VectorMemorySet32, test) { + EXPECT_DISASSEMBLY( + "0d367757 vsetvli tmp2, a2, e32, m8, ta, ma\n" + "5e05c057 vmv.v.x v0, a1\n" + " 8e19 sub a2, a2, tmp2\n" + "02056027 vse32.v v0, (a0)\n" + " 070a slli tmp2, tmp2, 0x2\n" + " 953a add a0, a0, tmp2\n" + " f67d bnez a2, -18\n" + " 8082 ret\n"); + + intptr_t len = 100; + uint32_t* dst = reinterpret_cast(malloc(len * 4)); + for (intptr_t i = 0; i < len; i++) { + dst[i] = 0; + } + + Call(test->entry(), reinterpret_cast(dst), 0x12345678, len); + for (intptr_t i = 0; i < len; i++) { + EXPECT_EQ(0x12345678u, dst[i]); + } + + // AVL < VLEN + dst[0] = 0; + dst[1] = 0; + Call(test->entry(), reinterpret_cast(dst), 0x23456789, 1); + EXPECT_EQ(0x23456789u, dst[0]); + EXPECT_EQ(0u, dst[1]); + + // AVL = 0 + dst[0] = 0; + Call(test->entry(), reinterpret_cast(dst), 0xFFFFFFFF, 0); + EXPECT_EQ(0u, dst[0]); + + free(dst); +} + +#if XLEN >= 64 +ASSEMBLER_TEST_GENERATE(VectorMemorySet64, assembler) { + __ SetExtensions(RV_GC | RV_V); + Label loop; + __ Bind(&loop); + __ vsetvli(A4, A2, e64, m8, ta, ma); + __ vmvvx(V0, A1); + __ sub(A2, A2, A4); + __ vse64v(V0, Address(A0)); + __ slli(A4, A4, 3); + __ add(A0, A0, A4); + __ bnez(A2, &loop); + __ ret(); +} +ASSEMBLER_TEST_RUN(VectorMemorySet64, test) { + EXPECT_DISASSEMBLY( + "0db67757 vsetvli tmp2, a2, e64, m8, ta, ma\n" + "5e05c057 vmv.v.x v0, a1\n" + " 8e19 sub a2, a2, tmp2\n" + "02057027 vse64.v v0, (a0)\n" + " 070e slli tmp2, tmp2, 0x3\n" + " 953a add a0, a0, tmp2\n" + " f67d bnez a2, -18\n" + " 8082 ret\n"); + + intptr_t len = 100; + uint64_t* dst = reinterpret_cast(malloc(len * 8)); + for (intptr_t i = 0; i < len; i++) { + dst[i] = 0; + } + + Call(test->entry(), reinterpret_cast(dst), 0x1234567812345678, len); + for (intptr_t i = 0; i < len; i++) { + EXPECT_EQ(0x1234567812345678u, dst[i]); + } + + // AVL < VLEN + dst[0] = 0; + dst[1] = 0; + Call(test->entry(), reinterpret_cast(dst), 0x2345678923456789, 1); + EXPECT_EQ(0x2345678923456789u, dst[0]); + EXPECT_EQ(0u, dst[1]); + + // AVL = 0 + dst[0] = 0; + Call(test->entry(), reinterpret_cast(dst), 0xFFFFFFFF, 0); + EXPECT_EQ(0u, dst[0]); + + free(dst); +} +#endif + ASSEMBLER_TEST_GENERATE(LoadByteAcquire, assembler) { __ SetExtensions(RV_GC | RV_Zalasr); __ lb(A0, Address(A1), std::memory_order_acquire); diff --git a/runtime/vm/compiler/assembler/disassembler_riscv.cc b/runtime/vm/compiler/assembler/disassembler_riscv.cc index c29ebcc2799..89b291832cb 100644 --- a/runtime/vm/compiler/assembler/disassembler_riscv.cc +++ b/runtime/vm/compiler/assembler/disassembler_riscv.cc @@ -91,6 +91,9 @@ class RISCVDisassembler { void DisassembleFNMADD(Instr instr); void DisassembleFNMSUB(Instr instr); void DisassembleOPFP(Instr instr); + void DisassembleOPV(Instr instr); + void DisassembleOPV_CFG(Instr instr); + void DisassembleOPV_IVX(Instr instr); void UnknownInstruction(Instr instr); void UnknownInstruction(CInstr instr); @@ -181,6 +184,9 @@ void RISCVDisassembler::DisassembleInstruction(Instr instr) { case OPFP: DisassembleOPFP(instr); break; + case OPV: + DisassembleOPV(instr); + break; default: if ((instr.encoding() == 0) || (instr.encoding() == static_cast(-1))) { @@ -563,12 +569,30 @@ void RISCVDisassembler::DisassembleLOAD(Instr instr) { void RISCVDisassembler::DisassembleLOADFP(Instr instr) { switch (instr.funct3()) { + case H: + Print("flh 'frd, 'iimm('rs1)", instr, RV_Zfhmin); + break; case S: Print("flw 'frd, 'iimm('rs1)", instr, RV_F); break; case D: Print("fld 'frd, 'iimm('rs1)", instr, RV_D); break; + case Q: + Print("flq 'frd, 'iimm('rs1)", instr, RV_Q); + break; + case E8: + Print("vle8.v 'vd, ('rs1)'vm", instr, RV_V); + break; + case E16: + Print("vle16.v 'vd, ('rs1)'vm", instr, RV_V); + break; + case E32: + Print("vle32.v 'vd, ('rs1)'vm", instr, RV_V); + break; + case E64: + Print("vle64.v 'vd, ('rs1)'vm", instr, RV_V); + break; default: UnknownInstruction(instr); } @@ -597,12 +621,30 @@ void RISCVDisassembler::DisassembleSTORE(Instr instr) { void RISCVDisassembler::DisassembleSTOREFP(Instr instr) { switch (instr.funct3()) { + case H: + Print("fsh 'frs2, 'simm('rs1)", instr, RV_Zfhmin); + break; case S: Print("fsw 'frs2, 'simm('rs1)", instr, RV_F); break; case D: Print("fsd 'frs2, 'simm('rs1)", instr, RV_D); break; + case Q: + Print("fsq 'frs2, 'simm('rs1)", instr, RV_Q); + break; + case E8: + Print("vse8.v 'vs3, ('rs1)'vm", instr, RV_V); + break; + case E16: + Print("vse16.v 'vs3, ('rs1)'vm", instr, RV_V); + break; + case E32: + Print("vse32.v 'vs3, ('rs1)'vm", instr, RV_V); + break; + case E64: + Print("vse64.v 'vs3, ('rs1)'vm", instr, RV_V); + break; default: UnknownInstruction(instr); } @@ -1800,6 +1842,40 @@ void RISCVDisassembler::DisassembleOPFP(Instr instr) { } } +void RISCVDisassembler::DisassembleOPV(Instr instr) { + switch (instr.funct3()) { + case OPCFG: + DisassembleOPV_CFG(instr); + break; + case OPIVX: + DisassembleOPV_IVX(instr); + break; + default: + UnknownInstruction(instr); + } +} + +void RISCVDisassembler::DisassembleOPV_CFG(Instr instr) { + if ((instr.encoding() & 0x80000000) == 0) { + Print("vsetvli 'rd, 'rs1, 'vtypei", instr, RV_V); + } else { + // vsetivli + // vsetvl + UnknownInstruction(instr); + } +} + +void RISCVDisassembler::DisassembleOPV_IVX(Instr instr) { + switch (instr.funct6()) { + case VMV: + Print("vmv.v.x 'vd, 'rs1'vm", instr, RV_V); + break; + default: + UnknownInstruction(instr); + break; + } +} + void RISCVDisassembler::UnknownInstruction(Instr instr) { if (instr.encoding() == 0) { Print("trap", instr, RV_I); @@ -1991,6 +2067,88 @@ const char* RISCVDisassembler::PrintOption(const char* format, Instr instr) { break; } return format + 4; + } else if (STRING_STARTS_WITH(format, "vd")) { + Printf("%s", vector_reg_names[instr.vd()]); + return format + 2; + } else if (STRING_STARTS_WITH(format, "vs1")) { + Printf("%s", vector_reg_names[instr.vs1()]); + return format + 3; + } else if (STRING_STARTS_WITH(format, "vs2")) { + Printf("%s", vector_reg_names[instr.vs2()]); + return format + 3; + } else if (STRING_STARTS_WITH(format, "vs3")) { + Printf("%s", vector_reg_names[instr.vs3()]); + return format + 3; + } else if (STRING_STARTS_WITH(format, "vtypei")) { + intx_t vtypei = instr.itype_imm(); + + // SEW + switch ((vtypei >> 3) & 0b111) { + case e8: + Printf("e8"); + break; + case e16: + Printf("e16"); + break; + case e32: + Printf("e32"); + break; + case e64: + Printf("e64"); + break; + default: + Printf("invalid sew"); + break; + } + + // LMUL + switch ((vtypei >> 0) & 0b111) { + case mf8: + Printf(", mf8"); + break; + case mf4: + Printf(", mf4"); + break; + case mf2: + Printf(", mf2"); + break; + case m1: + Printf(", m1"); + break; + case m2: + Printf(", m2"); + break; + case m4: + Printf(", m4"); + break; + case m8: + Printf(", m8"); + break; + default: + Printf(", invalid lmul"); + break; + } + + // VTA + if ((vtypei & (1 << 6)) == 0) { + Printf(", tu"); + } else { + Printf(", ta"); + } + + // VMA + if ((vtypei & (1 << 7)) == 0) { + Printf(", mu"); + } else { + Printf(", ma"); + } + + return format + 6; + } else if (STRING_STARTS_WITH(format, "vm")) { + if (instr.vm()) { + Printf(", v0.t"); + } + return format + 2; } FATAL("Bad format %s\n", format); diff --git a/runtime/vm/constants_riscv.cc b/runtime/vm/constants_riscv.cc index 49415a28889..a52f0c45bc1 100644 --- a/runtime/vm/constants_riscv.cc +++ b/runtime/vm/constants_riscv.cc @@ -29,6 +29,12 @@ const char* const fpu_reg_names[kNumberOfFpuRegisters] = { "fs8", "fs9", "fs10", "fs11", "ft8", "ft9", "ft10", "ft11", }; +const char* const vector_reg_names[kNumberOfVectorRegisters] = { + "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", + "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", + "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", +}; + const Register CallingConventions::ArgumentRegisters[] = { // A3/A4/A5 are assigned to TMP/TMP2/PP. This assignment is important for // reducing code size. To minimize distruption to the rest of the compiler, diff --git a/runtime/vm/constants_riscv.h b/runtime/vm/constants_riscv.h index a17b3702425..e96b24b1749 100644 --- a/runtime/vm/constants_riscv.h +++ b/runtime/vm/constants_riscv.h @@ -131,6 +131,43 @@ enum FRegister { kNoFpuRegister = -1, }; +enum VRegister { + V0 = 0, + V1 = 1, + V2 = 2, + V3 = 3, + V4 = 4, + V5 = 5, + V6 = 6, + V7 = 7, + V8 = 8, + V9 = 9, + V10 = 10, + V11 = 11, + V12 = 12, + V13 = 13, + V14 = 14, + V15 = 15, + V16 = 16, + V17 = 17, + V18 = 18, + V19 = 19, + V20 = 20, + V21 = 21, + V22 = 22, + V23 = 23, + V24 = 24, + V25 = 25, + V26 = 26, + V27 = 27, + V28 = 28, + V29 = 29, + V30 = 30, + V31 = 31, + kNumberOfVectorRegisters = 32, + kNoVectorRegister = -1, +}; + // Register alias for floating point scratch register. const FRegister FTMP = FT11; @@ -143,6 +180,7 @@ typedef double fpu_register_t; extern const char* const cpu_reg_names[kNumberOfCpuRegisters]; extern const char* const cpu_reg_abi_names[kNumberOfCpuRegisters]; extern const char* const fpu_reg_names[kNumberOfFpuRegisters]; +extern const char* const vector_reg_names[kNumberOfVectorRegisters]; // Register aliases. constexpr Register TMP = A3; // Used as scratch register by assembler. @@ -741,6 +779,7 @@ enum Opcode { FNMSUB = 0b1001011, FNMADD = 0b1001111, OPFP = 0b1010011, + OPV = 0b1010111, }; enum Funct12 { @@ -825,8 +864,15 @@ enum Funct3 { WIDTH32 = 0b010, WIDTH64 = 0b011, + H = 0b001, S = 0b010, D = 0b011, + Q = 0b100, + E8 = 0b000, + E16 = 0b101, + E32 = 0b110, + E64 = 0b111, + J = 0b000, JN = 0b001, JX = 0b010, @@ -867,6 +913,15 @@ enum Funct3 { CZEROEQZ = 0b101, CZERONEZ = 0b111, + + OPIVV = 0b000, + OPFVV = 0b001, + OPMVV = 0b010, + OPIVI = 0b011, + OPIVX = 0b100, + OPFVF = 0b101, + OPMVX = 0b110, + OPCFG = 0b111, }; enum Funct7 { @@ -922,6 +977,21 @@ enum Funct7 { SSPUSH = 0b1100111, }; +enum Funct6 { + VADD = 0b000000, + VSUB = 0b000010, + VRSUB = 0b000011, + VMINU = 0b000100, + VMIN = 0b000101, + VMAXU = 0b000110, + VMAX = 0b000111, + VAND = 0b001001, + VOR = 0b001010, + VXOR = 0b001011, + + VMV = 0b010111, +}; + enum Funct5 { LR = 0b00010, SC = 0b00011, @@ -986,6 +1056,45 @@ enum HartEffects { const intptr_t kReleaseShift = 25; const intptr_t kAcquireShift = 26; +enum ElementWidth { + e8 = 0b000, + e16 = 0b001, + e32 = 0b010, + e64 = 0b011, + + reservedsew1 = 0b100, + reservedsew2 = 0b101, + reservedsew3 = 0b110, + reservedsew4 = 0b111, +}; + +enum LengthMultiplier { + mf8 = 0b101, + mf4 = 0b110, + mf2 = 0b111, + m1 = 0b000, + m2 = 0b001, + m4 = 0b010, + m8 = 0b011, + + reservedlmul1 = 0b100, +}; + +enum MaskMode { + mu = 0, // Mask undisturbed + ma = 1, // Mask agnostic +}; + +enum TailMode { + tu = 0, // Tail undisturbed + ta = 1, // Mask agnosticagnostic +}; + +enum VectorMask { + v0t = 0, + unmasked = 1 << 25, +}; + constexpr uint32_t kFlisConstants[32] = { 0xbf800000, // -1.0 0x00800000, // min positive normal @@ -1075,6 +1184,10 @@ DEFINE_REG_ENCODING(FRegister, FRd, 7) DEFINE_REG_ENCODING(FRegister, FRs1, 15) DEFINE_REG_ENCODING(FRegister, FRs2, 20) DEFINE_REG_ENCODING(FRegister, FRs3, 27) +DEFINE_REG_ENCODING(VRegister, Vd, 7) +DEFINE_REG_ENCODING(VRegister, Vs1, 15) +DEFINE_REG_ENCODING(VRegister, Vs2, 20) +DEFINE_REG_ENCODING(VRegister, Vs3, 7) #undef DEFINE_REG_ENCODING #define DEFINE_FUNCT_ENCODING(type, name, shift, mask) \ @@ -1093,6 +1206,7 @@ DEFINE_FUNCT_ENCODING(Opcode, Opcode, 0, 0x7F) DEFINE_FUNCT_ENCODING(Funct2, Funct2, 25, 0x3) DEFINE_FUNCT_ENCODING(Funct3, Funct3, 12, 0x7) DEFINE_FUNCT_ENCODING(Funct5, Funct5, 27, 0x1F) +DEFINE_FUNCT_ENCODING(Funct6, Funct6, 26, 0x3F) DEFINE_FUNCT_ENCODING(Funct7, Funct7, 25, 0x7F) DEFINE_FUNCT_ENCODING(Funct12, Funct12, 20, 0xFFF) #if XLEN == 32 @@ -1217,9 +1331,16 @@ class Instr { FRegister frs2() const { return DecodeFRs2(encoding_); } FRegister frs3() const { return DecodeFRs3(encoding_); } + VRegister vd() const { return DecodeVd(encoding_); } + VRegister vs1() const { return DecodeVs1(encoding_); } + VRegister vs2() const { return DecodeVs2(encoding_); } + VRegister vs3() const { return DecodeVs3(encoding_); } + bool vm() const { return (encoding_ & 1 << 25) == 0; } + Funct2 funct2() const { return DecodeFunct2(encoding_); } Funct3 funct3() const { return DecodeFunct3(encoding_); } Funct5 funct5() const { return DecodeFunct5(encoding_); } + Funct6 funct6() const { return DecodeFunct6(encoding_); } Funct7 funct7() const { return DecodeFunct7(encoding_); } Funct12 funct12() const { return DecodeFunct12(encoding_); } @@ -1763,26 +1884,28 @@ static constexpr Extension RV_M(1); // Multiply/divide static constexpr Extension RV_A(2); // Atomic static constexpr Extension RV_F(3); // Single-precision floating point static constexpr Extension RV_D(4); // Double-precision floating point -static constexpr Extension RV_C(5); // Compressed instructions +static constexpr Extension RV_Q(5); // Quad-precision floating point +static constexpr Extension RV_C(6); // Compressed instructions static constexpr ExtensionSet RV_G = RV_I | RV_M | RV_A | RV_F | RV_D; static constexpr ExtensionSet RV_GC = RV_G | RV_C; static constexpr ExtensionSet RVA20 = RV_GC; -static constexpr Extension RV_Zba(6); // Address generation -static constexpr Extension RV_Zbb(7); // Basic bit-manipulation -static constexpr Extension RV_Zbs(8); // Single-bit instructions -static constexpr Extension RV_Zbc(9); // Carry-less multiplication +static constexpr Extension RV_Zba(7); // Address generation +static constexpr Extension RV_Zbb(8); // Basic bit-manipulation +static constexpr Extension RV_Zbs(9); // Single-bit instructions +static constexpr Extension RV_Zbc(10); // Carry-less multiplication static constexpr ExtensionSet RV_B = RV_Zba | RV_Zbb | RV_Zbs; static constexpr ExtensionSet RV_GCB = RV_GC | RV_B; static constexpr ExtensionSet RVA22 = RV_GCB; -static constexpr Extension RV_V(10); // Vector -static constexpr Extension RV_Zicond(11); // Integer conditional operations -static constexpr Extension RV_Zcb(12); // More compressed instructions -static constexpr Extension RV_Zfa(13); // Additional floating-point +static constexpr Extension RV_V(11); // Vector +static constexpr Extension RV_Zicond(12); // Integer conditional operations +static constexpr Extension RV_Zcb(13); // More compressed instructions +static constexpr Extension RV_Zfa(14); // Additional floating-point static constexpr ExtensionSet RVA23 = RV_GCB | RV_V | RV_Zicond | RV_Zcb | RV_Zfa; -static constexpr Extension RV_Zicfiss(14); // Shadow stack -static constexpr Extension RV_Zabha(15); // Byte and halfword AMOs -static constexpr Extension RV_Zalasr(16); // Load-acquire, store-release +static constexpr Extension RV_Zicfiss(15); // Shadow stack +static constexpr Extension RV_Zabha(16); // Byte and halfword AMOs +static constexpr Extension RV_Zalasr(17); // Load-acquire, store-release +static constexpr Extension RV_Zfhmin(18); // Load-acquire, store-release #if defined(DART_TARGET_OS_ANDROID) static constexpr ExtensionSet RV_baseline = RVA23; diff --git a/runtime/vm/simulator_riscv.cc b/runtime/vm/simulator_riscv.cc index 8ed5a8962eb..20ed0380322 100644 --- a/runtime/vm/simulator_riscv.cc +++ b/runtime/vm/simulator_riscv.cc @@ -212,6 +212,11 @@ Simulator::Simulator() : random_(), memory_(FLAG_sim_buffer_memory) { // fregs_[i] = bit_cast(random_.NextUInt64()); fregs_[i] = bit_cast(kNaNBox); } + for (intptr_t i = 0; i < kNumberOfVectorRegisters; i++) { + for (intptr_t j = 0; j < VLEN / 8; j++) { + vregs_[i][j] = random_.NextUInt64(); + } + } // The sp is initialized to point to the bottom (high address) of the // allocated stack area. @@ -251,6 +256,11 @@ void Simulator::PrepareCall(PreservedRegisters* preserved) { fregs_[i] = bit_cast(kNaNBox); } } + for (intptr_t i = 0; i < kNumberOfVectorRegisters; i++) { + for (intptr_t j = 0; j < VLEN / 8; j++) { + vregs_[i][j] = random_.NextUInt64(); + } + } preserved->ssp = ssp_; #endif } @@ -272,6 +282,11 @@ void Simulator::ClobberVolatileRegisters() { fregs_[i] = bit_cast(kNaNBox); } } + for (intptr_t i = 0; i < kNumberOfVectorRegisters; i++) { + for (intptr_t j = 0; j < VLEN / 8; j++) { + vregs_[i][j] = random_.NextUInt64(); + } + } #endif } @@ -489,18 +504,90 @@ void Simulator::PrintRegisters() { static_cast(kNumberOfFpuRegisters)); for (intptr_t i = 0; i < kNumberOfCpuRegisters; i++) { #if XLEN == 32 - OS::Print("%4s: %8x %11d", cpu_reg_names[i], xregs_[i], xregs_[i]); + OS::Print("%5s: %8x %11d", cpu_reg_names[i], xregs_[i], xregs_[i]); #elif XLEN == 64 - OS::Print("%4s: %16" Px64 " %20" Pd64, cpu_reg_names[i], xregs_[i], + OS::Print("%5s: %16" Px64 " %20" Pd64, cpu_reg_names[i], xregs_[i], xregs_[i]); #endif - OS::Print(" %4s: %lf\n", fpu_reg_names[i], fregs_[i]); + OS::Print(" %5s: %lf\n", fpu_reg_names[i], fregs_[i]); } #if XLEN == 32 - OS::Print(" pc: %8x\n", pc_); + OS::Print(" pc: %8x\n", pc_); #elif XLEN == 64 - OS::Print(" pc: %16" Px64 "\n", pc_); + OS::Print(" pc: %16" Px64 "\n", pc_); #endif + + for (intptr_t i = 0; i < kNumberOfVectorRegisters; i++) { + OS::Print("%5s: ", vector_reg_names[i]); + for (intptr_t j = VLEN / 8 - 1; j >= 0; j--) { + OS::Print("%02x", (unsigned)vregs_[i][j]); + } + OS::Print("\n"); + } +#if XLEN == 32 + OS::Print(" vl: %8x %11d\n", vl_, vl_); +#elif XLEN == 64 + OS::Print(" vl: %16" Px64 " %20" Pd64 "\n", vl_, vl_); +#endif + +#if XLEN == 32 + OS::Print("vtype: %8x ", vtype_); +#elif XLEN == 64 + OS::Print("vtype: %16" Px64 " ", vtype_); +#endif + switch (vsew()) { + case e8: + OS::Print("e8"); + break; + case e16: + OS::Print("e16"); + break; + case e32: + OS::Print("e32"); + break; + case e64: + OS::Print("e64"); + break; + default: + OS::Print("invalid sew"); + break; + } + switch (vlmul()) { + case mf8: + OS::Print(", mf8"); + break; + case mf4: + OS::Print(", mf4"); + break; + case mf2: + OS::Print(", mf2"); + break; + case m1: + OS::Print(", m1"); + break; + case m2: + OS::Print(", m2"); + break; + case m4: + OS::Print(", m4"); + break; + case m8: + OS::Print(", m8"); + break; + default: + OS::Print(", invalid lmul"); + break; + } + if ((vtype_ & (1 << 6)) == 0) { + OS::Print(", tu"); + } else { + OS::Print(", ta"); + } + if ((vtype_ & (1 << 7)) == 0) { + OS::Print(", mu\n"); + } else { + OS::Print(", ma\n"); + } } void Simulator::PrintStack() { @@ -581,6 +668,9 @@ void Simulator::Interpret(Instr instr) { case OPFP: InterpretOPFP(instr); break; + case OPV: + InterpretOPV(instr); + break; default: IllegalInstruction(instr); } @@ -1347,12 +1437,36 @@ void Simulator::InterpretLOADFP(Instr instr) { case D: set_fregd(instr.frd(), MemoryRead(addr, instr.rs1())); break; + case E8: + InterpretLOADV(instr); + break; + case E16: + InterpretLOADV(instr); + break; + case E32: + InterpretLOADV(instr); + break; + case E64: + InterpretLOADV(instr); + break; default: IllegalInstruction(instr); } pc_ += instr.length(); } +template +void Simulator::InterpretLOADV(Instr instr) { + if ((instr.encoding() & 0xFFF00000) != 0x02000000) { + UNIMPLEMENTED(); // Only unmasked unit-stride implemented. + } + uintx_t base = get_xreg(instr.rs1()); + T* vd = ref_vreg(instr.vd()); + for (uintx_t i = 0; i < vl_; i++) { + vd[i] = MemoryRead(base + i * sizeof(T), instr.rs1()); + } +} + DART_FORCE_INLINE void Simulator::InterpretSTORE(Instr instr) { uintx_t addr = get_xreg(instr.rs1()) + instr.stype_imm(); @@ -1387,12 +1501,36 @@ void Simulator::InterpretSTOREFP(Instr instr) { case D: MemoryWrite(addr, get_fregd(instr.frs2()), instr.rs1()); break; + case E8: + InterpretSTOREV(instr); + break; + case E16: + InterpretSTOREV(instr); + break; + case E32: + InterpretSTOREV(instr); + break; + case E64: + InterpretSTOREV(instr); + break; default: IllegalInstruction(instr); } pc_ += instr.length(); } +template +void Simulator::InterpretSTOREV(Instr instr) { + if ((instr.encoding() & 0xFFF00000) != 0x02000000) { + UNIMPLEMENTED(); // Only unmasked unit-stride implemented. + } + uintx_t base = get_xreg(instr.rs1()); + T* vs3 = ref_vreg(instr.vs3()); + for (uintx_t i = 0; i < vl_; i++) { + MemoryWrite(base + i * sizeof(T), vs3[i], instr.rs1()); + } +} + DART_FORCE_INLINE void Simulator::InterpretOPIMM(Instr instr) { switch (instr.funct3()) { @@ -3390,6 +3528,172 @@ void Simulator::InterpretOPFP(Instr instr) { pc_ += instr.length(); } +void Simulator::InterpretOPV(Instr instr) { + switch (instr.funct3()) { + case OPIVV: + InterpretOPV_IVV(instr); + break; + case OPFVV: + InterpretOPV_FVV(instr); + break; + case OPMVV: + InterpretOPV_MVV(instr); + break; + case OPIVI: + InterpretOPV_IVI(instr); + break; + case OPIVX: + InterpretOPV_IVX(instr); + break; + case OPFVF: + InterpretOPV_FVF(instr); + break; + case OPMVX: + InterpretOPV_MVX(instr); + break; + case OPCFG: + InterpretOPV_CFG(instr); + break; + default: + IllegalInstruction(instr); + } + pc_ += instr.length(); +} + +void Simulator::InterpretOPV_IVV(Instr instr) { + IllegalInstruction(instr); +} + +void Simulator::InterpretOPV_FVV(Instr instr) { + IllegalInstruction(instr); +} + +void Simulator::InterpretOPV_MVV(Instr instr) { + IllegalInstruction(instr); +} + +void Simulator::InterpretOPV_IVI(Instr instr) { + IllegalInstruction(instr); +} + +void Simulator::InterpretOPV_IVX(Instr instr) { + switch (vsew()) { + case e8: + InterpretOPV_IVX(instr); + break; + case e16: + InterpretOPV_IVX(instr); + break; + case e32: + InterpretOPV_IVX(instr); + break; + case e64: + InterpretOPV_IVX(instr); + break; + default: + FATAL("Invalid SEW"); + } +} + +template +void Simulator::InterpretOPV_IVX(Instr instr) { + if (instr.vm()) UNIMPLEMENTED(); + sew_t rs1 = get_xreg(instr.rs1()); + sew_t* vs2 = ref_vreg(instr.vs2()); + sew_t* vd = ref_vreg(instr.vd()); + switch (instr.funct6()) { + case VADD: + for (uintx_t i = 0, n = vl_; i < n; i++) { + vd[i] = rs1 + vs2[i]; + } + break; + case VMV: + for (uintx_t i = 0, n = vl_; i < n; i++) { + vd[i] = rs1; + } + break; + default: + IllegalInstruction(instr); + } +} + +void Simulator::InterpretOPV_FVF(Instr instr) { + IllegalInstruction(instr); +} + +void Simulator::InterpretOPV_MVX(Instr instr) { + IllegalInstruction(instr); +} + +void Simulator::InterpretOPV_CFG(Instr instr) { + if ((instr.encoding() & 0x80000000) == 0) { + uintx_t avl = get_xreg(instr.rs1()); // In elements. + intx_t vtype = instr.itype_imm(); + uintx_t sew; + switch ((vtype >> 3) & 0b111) { + case e8: + sew = 8; + break; + case e16: + sew = 16; + break; + case e32: + sew = 32; + break; + case e64: + sew = 64; + break; + default: + FATAL("Invalid SEW"); + } + intx_t lmul; + switch ((vtype >> 0) & 0b111) { + case mf8: + lmul = -8; + break; + case mf4: + lmul = -4; + break; + case mf2: + lmul = -2; + break; + case m1: + lmul = 1; + break; + case m2: + lmul = 2; + break; + case m4: + lmul = 4; + break; + case m8: + lmul = 8; + break; + default: + FATAL("Invalid LMUL"); + } + uintx_t vlmax; + if (lmul < 0) { + vlmax = VLEN / sew / -lmul; + } else { + vlmax = VLEN / sew * lmul; + } + if (instr.rs1() == ZR && instr.rd() != ZR) { + vl_ = vlmax; + } else if (instr.rs1() == ZR) { + // Keep existing vl. + } else if (avl < vlmax) { + vl_ = avl; + } else { + vl_ = vlmax; + } + vtype_ = vtype; + set_xreg(instr.rd(), vl_); + } else { + IllegalInstruction(instr); + } +} + void Simulator::InterpretEBREAK(Instr instr) { PrintRegisters(); PrintStack(); @@ -3457,15 +3761,22 @@ void Simulator::MemoryWrite(uintx_t addr, type value, Register base) { } enum ControlStatusRegister { + // URW fflags = 0x001, frm = 0x002, fcsr = 0x003, - + vstart = 0x008, + vxsat = 0x009, + vxrm = 0x00A, + vcsr = 0x00F, ssp = 0x011, - + // URO cycle = 0xC00, time = 0xC01, instret = 0xC02, + vl = 0xC20, + vtype = 0xC21, + vlenb = 0xC22, #if XLEN == 32 cycleh = 0xC80, timeh = 0xC81, @@ -3485,6 +3796,12 @@ intx_t Simulator::CSRRead(uint16_t csr) { return 0; case instret: return instret_; + case vl: + return vl_; + case vtype: + return vtype_; + case vlenb: + return VLEN / 8; #if XLEN == 32 case cycleh: return (instret_ / 2) >> 32; diff --git a/runtime/vm/simulator_riscv.h b/runtime/vm/simulator_riscv.h index 2f3920ea2e1..446fcc83a00 100644 --- a/runtime/vm/simulator_riscv.h +++ b/runtime/vm/simulator_riscv.h @@ -20,6 +20,15 @@ class Mutex; class SimulatorSetjmpBuffer; class Thread; +#define ELEN 64 +#define VLEN 128 + +COMPILE_ASSERT(ELEN >= 8); +COMPILE_ASSERT(Utils::IsPowerOfTwo(ELEN)); +COMPILE_ASSERT(VLEN >= ELEN); +COMPILE_ASSERT(Utils::IsPowerOfTwo(VLEN)); +COMPILE_ASSERT(VLEN <= 0x10000); + // TODO(riscv): Dynamic rounding mode and other FSCR state. class Simulator { public: @@ -284,12 +293,27 @@ class Simulator { template void InterpretSTOREORDERED(Instr instr); void InterpretLOADFP(Instr instr); + template + void InterpretLOADV(Instr instr); void InterpretSTOREFP(Instr instr); + template + void InterpretSTOREV(Instr instr); void InterpretFMADD(Instr instr); void InterpretFMSUB(Instr instr); void InterpretFNMADD(Instr instr); void InterpretFNMSUB(Instr instr); void InterpretOPFP(Instr instr); + void InterpretOPV(Instr instr); + void InterpretOPV_IVV(Instr instr); + void InterpretOPV_FVV(Instr instr); + void InterpretOPV_MVV(Instr instr); + void InterpretOPV_IVI(Instr instr); + void InterpretOPV_IVX(Instr instr); + template + void InterpretOPV_IVX(Instr instr); + void InterpretOPV_FVF(Instr instr); + void InterpretOPV_MVX(Instr instr); + void InterpretOPV_CFG(Instr instr); DART_NORETURN void IllegalInstruction(Instr instr); DART_NORETURN void IllegalInstruction(CInstr instr); @@ -337,6 +361,17 @@ class Simulator { fregs_[rd] = bit_cast(bits64); } + template + T* ref_vreg(VRegister vd) { + return reinterpret_cast(&vregs_[vd][0]); + } + LengthMultiplier vlmul() const { + return static_cast((vtype_ >> 0) & 7); + } + ElementWidth vsew() const { + return static_cast((vtype_ >> 3) & 7); + } + // Known bad pc value to ensure that the simulator does not execute // without being properly setup. static constexpr uword kBadLR = -1; @@ -363,6 +398,11 @@ class Simulator { bool ss_enabled_ = false; uintx_t ssp_ = 0; + // V state + uint8_t vregs_[kNumberOfVectorRegisters][VLEN / 8]; + uintx_t vl_; + uintx_t vtype_; + // Simulator support. char* stack_; char* shadow_stack_;