dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[vm, compiler] Add just enough of the vector extension to implement memcpy and memset.

Browse Source 
TEST=ci, local qemu
Change-Id: I9518049ca927fa42d3c04e9e045c6cec1342c789
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/462462
Reviewed-by: Alexander Aprelev <aam@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>
This commit is contained in:
Ryan Macnak
2025-11-18 14:33:11 -08:00
committed by Commit Queue
parent fd94c57fc9
commit 61fff77154
8 changed files with 1008 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/vm/compiler/assembler/assembler_riscv.cc
+73
View File
@@ -2014,6 +2014,79 @@ void MicroAssembler::ssamoswapd(Register rd,
}
#endif // XLEN >= 64
void MicroAssembler::vsetvli(Register rd,
Register rs1,
ElementWidth sew,
LengthMultiplier lmul,
TailMode vta,
MaskMode vma) {
ASSERT(Supports(RV_V));
intx_t vtypei = (vma << 7) | (vta << 6) | (sew << 3) | (lmul << 0);
EmitIType(vtypei, rs1, OPCFG, rd, OPV);
}
void MicroAssembler::vle8v(VRegister vd, Address rs1, VectorMask vm) {
ASSERT(Supports(RV_V));
ASSERT(rs1.offset() == 0);
Emit32(EncodeOpcode(LOADFP) | EncodeVd(vd) | EncodeRs1(rs1.base()) |
EncodeFunct3(E8) | vm);
}
void MicroAssembler::vle16v(VRegister vd, Address rs1, VectorMask vm) {
ASSERT(Supports(RV_V));
ASSERT(rs1.offset() == 0);
Emit32(EncodeOpcode(LOADFP) | EncodeVd(vd) | EncodeRs1(rs1.base()) |
EncodeFunct3(E16) | vm);
}
void MicroAssembler::vle32v(VRegister vd, Address rs1, VectorMask vm) {
ASSERT(Supports(RV_V));
ASSERT(rs1.offset() == 0);
Emit32(EncodeOpcode(LOADFP) | EncodeVd(vd) | EncodeRs1(rs1.base()) |
EncodeFunct3(E32) | vm);
}
void MicroAssembler::vle64v(VRegister vd, Address rs1, VectorMask vm) {
ASSERT(Supports(RV_V));
ASSERT(rs1.offset() == 0);
Emit32(EncodeOpcode(LOADFP) | EncodeVd(vd) | EncodeRs1(rs1.base()) |
EncodeFunct3(E64) | vm);
}
void MicroAssembler::vse8v(VRegister vs3, Address rs1, VectorMask vm) {
ASSERT(Supports(RV_V));
ASSERT(rs1.offset() == 0);
Emit32(EncodeOpcode(STOREFP) | EncodeVs3(vs3) | EncodeRs1(rs1.base()) |
EncodeFunct3(E8) | vm);
}
void MicroAssembler::vse16v(VRegister vs3, Address rs1, VectorMask vm) {
ASSERT(Supports(RV_V));
ASSERT(rs1.offset() == 0);
Emit32(EncodeOpcode(STOREFP) | EncodeVs3(vs3) | EncodeRs1(rs1.base()) |
EncodeFunct3(E16) | vm);
}
void MicroAssembler::vse32v(VRegister vs3, Address rs1, VectorMask vm) {
ASSERT(Supports(RV_V));
ASSERT(rs1.offset() == 0);
Emit32(EncodeOpcode(STOREFP) | EncodeVs3(vs3) | EncodeRs1(rs1.base()) |
EncodeFunct3(E32) | vm);
}
void MicroAssembler::vse64v(VRegister vs3, Address rs1, VectorMask vm) {
ASSERT(Supports(RV_V));
ASSERT(rs1.offset() == 0);
Emit32(EncodeOpcode(STOREFP) | EncodeVs3(vs3) | EncodeRs1(rs1.base()) |
EncodeFunct3(E64) | vm);
}
void MicroAssembler::vmvvx(VRegister vd, Register rs1, VectorMask vm) {
ASSERT(Supports(RV_V));
Emit32(EncodeOpcode(OPV) | EncodeVd(vd) | EncodeRs1(rs1) |
EncodeFunct3(OPIVX) | EncodeFunct6(VMV) | vm);
}
void MicroAssembler::lb(Register rd, Address addr, std::memory_order order) {
ASSERT(addr.offset() == 0);
ASSERT((order == std::memory_order_acquire) ||
runtime/vm/compiler/assembler/assembler_riscv.h
+17
View File
@@ -723,6 +723,23 @@ class MicroAssembler : public AssemblerBase {
std::memory_order order = std::memory_order_relaxed);
#endif // XLEN >= 64
// ==== RV32V: Vectors ====
void vsetvli(Register rd,
Register rs1,
ElementWidth sew,
LengthMultiplier lmul,
TailMode vta,
MaskMode vma);
void vle8v(VRegister vd, Address rs1, VectorMask vm = unmasked);
void vle16v(VRegister vd, Address rs1, VectorMask vm = unmasked);
void vle32v(VRegister vd, Address rs1, VectorMask vm = unmasked);
void vle64v(VRegister vd, Address rs1, VectorMask vm = unmasked);
void vse8v(VRegister vs3, Address rs1, VectorMask vm = unmasked);
void vse16v(VRegister vs3, Address rs1, VectorMask vm = unmasked);
void vse32v(VRegister vs3, Address rs1, VectorMask vm = unmasked);
void vse64v(VRegister vs3, Address rs1, VectorMask vm = unmasked);
void vmvvx(VRegister vd, Register rs1, VectorMask vm = unmasked);
// ==== Zalasr: Load-acquire, store-release ====
void lb(Register rd, Address addr, std::memory_order order);
void lh(Register rd, Address addr, std::memory_order order);
runtime/vm/compiler/assembler/assembler_riscv_test.cc
+255
View File
@@ -8479,6 +8479,261 @@ ASSEMBLER_TEST_RUN(DoubleLessOrEqualQuiet, test) {
EXPECT_EQ(0, CallI(test->entry(), qNAN, -3.0));
}
ASSEMBLER_TEST_GENERATE(VectorMemoryCopy, assembler) {
__ SetExtensions(RV_GC | RV_V);
Label loop;
__ Bind(&loop);
__ vsetvli(A4, A2, e8, m8, ta, ma);
__ vle8v(V0, Address(A1));
__ add(A1, A1, A4);
__ sub(A2, A2, A4);
__ vse8v(V0, Address(A0));
__ add(A0, A0, A4);
__ bnez(A2, &loop);
__ ret();
}
ASSEMBLER_TEST_RUN(VectorMemoryCopy, test) {
EXPECT_DISASSEMBLY(
"0c367757 vsetvli tmp2, a2, e8, m8, ta, ma\n"
"02058007 vle8.v v0, (a1)\n"
" 95ba add a1, a1, tmp2\n"
" 8e19 sub a2, a2, tmp2\n"
"02050027 vse8.v v0, (a0)\n"
" 953a add a0, a0, tmp2\n"
" f67d bnez a2, -18\n"
" 8082 ret\n");
intptr_t len = 1000;
uint8_t* src = reinterpret_cast<uint8_t*>(malloc(len));
uint8_t* dst = reinterpret_cast<uint8_t*>(malloc(len));
for (intptr_t i = 0; i < len; i++) {
src[i] = i & 0xFF;
dst[i] = 0xFF;
}
Call(test->entry(), reinterpret_cast<intx_t>(dst),
reinterpret_cast<intx_t>(src), len);
for (intptr_t i = 0; i < len; i++) {
EXPECT_EQ(i & 0xFF, src[i]);
EXPECT_EQ(i & 0xFF, dst[i]);
}
// AVL < VLEN
dst[0] = 0xFF;
Call(test->entry(), reinterpret_cast<intx_t>(dst),
reinterpret_cast<intx_t>(src), 1);
EXPECT_EQ(0, dst[0]);
// AVL = 0
dst[0] = 0xFF;
Call(test->entry(), reinterpret_cast<intx_t>(dst),
reinterpret_cast<intx_t>(src), 0);
EXPECT_EQ(0xFF, dst[0]);
free(src);
free(dst);
}
ASSEMBLER_TEST_GENERATE(VectorMemorySet8, assembler) {
__ SetExtensions(RV_GC | RV_V);
Label loop;
__ Bind(&loop);
__ vsetvli(A4, A2, e8, m8, ta, ma);
__ vmvvx(V0, A1);
__ sub(A2, A2, A4);
__ vse8v(V0, Address(A0));
__ add(A0, A0, A4);
__ bnez(A2, &loop);
__ ret();
}
ASSEMBLER_TEST_RUN(VectorMemorySet8, test) {
EXPECT_DISASSEMBLY(
"0c367757 vsetvli tmp2, a2, e8, m8, ta, ma\n"
"5e05c057 vmv.v.x v0, a1\n"
" 8e19 sub a2, a2, tmp2\n"
"02050027 vse8.v v0, (a0)\n"
" 953a add a0, a0, tmp2\n"
" fa65 bnez a2, -16\n"
" 8082 ret\n");
intptr_t len = 100;
uint8_t* dst = reinterpret_cast<uint8_t*>(malloc(len * 1));
for (intptr_t i = 0; i < len; i++) {
dst[i] = 0;
}
Call(test->entry(), reinterpret_cast<intx_t>(dst), 0x12, len);
for (intptr_t i = 0; i < len; i++) {
EXPECT_EQ(0x12u, dst[i]);
}
// AVL < VLEN
dst[0] = 0;
dst[1] = 0;
Call(test->entry(), reinterpret_cast<intx_t>(dst), 0x23, 1);
EXPECT_EQ(0x23u, dst[0]);
EXPECT_EQ(0u, dst[1]);
// AVL = 0
dst[0] = 0;
Call(test->entry(), reinterpret_cast<intx_t>(dst), 0xFF, 0);
EXPECT_EQ(0u, dst[0]);
free(dst);
}
ASSEMBLER_TEST_GENERATE(VectorMemorySet16, assembler) {
__ SetExtensions(RV_GC | RV_V);
Label loop;
__ Bind(&loop);
__ vsetvli(A4, A2, e16, m8, ta, ma);
__ vmvvx(V0, A1);
__ sub(A2, A2, A4);
__ vse16v(V0, Address(A0));
__ slli(A4, A4, 1);
__ add(A0, A0, A4);
__ bnez(A2, &loop);
__ ret();
}
ASSEMBLER_TEST_RUN(VectorMemorySet16, test) {
EXPECT_DISASSEMBLY(
"0cb67757 vsetvli tmp2, a2, e16, m8, ta, ma\n"
"5e05c057 vmv.v.x v0, a1\n"
" 8e19 sub a2, a2, tmp2\n"
"02055027 vse16.v v0, (a0)\n"
" 0706 slli tmp2, tmp2, 0x1\n"
" 953a add a0, a0, tmp2\n"
" f67d bnez a2, -18\n"
" 8082 ret\n");
intptr_t len = 100;
uint16_t* dst = reinterpret_cast<uint16_t*>(malloc(len * 2));
for (intptr_t i = 0; i < len; i++) {
dst[i] = 0;
}
Call(test->entry(), reinterpret_cast<intx_t>(dst), 0x1234, len);
for (intptr_t i = 0; i < len; i++) {
EXPECT_EQ(0x1234u, dst[i]);
}
// AVL < VLEN
dst[0] = 0;
dst[1] = 0;
Call(test->entry(), reinterpret_cast<intx_t>(dst), 0x2345, 1);
EXPECT_EQ(0x2345u, dst[0]);
EXPECT_EQ(0u, dst[1]);
// AVL = 0
dst[0] = 0;
Call(test->entry(), reinterpret_cast<intx_t>(dst), 0xFFFF, 0);
EXPECT_EQ(0u, dst[0]);
free(dst);
}
ASSEMBLER_TEST_GENERATE(VectorMemorySet32, assembler) {
__ SetExtensions(RV_GC | RV_V);
Label loop;
__ Bind(&loop);
__ vsetvli(A4, A2, e32, m8, ta, ma);
__ vmvvx(V0, A1);
__ sub(A2, A2, A4);
__ vse32v(V0, Address(A0));
__ slli(A4, A4, 2);
__ add(A0, A0, A4);
__ bnez(A2, &loop);
__ ret();
}
ASSEMBLER_TEST_RUN(VectorMemorySet32, test) {
EXPECT_DISASSEMBLY(
"0d367757 vsetvli tmp2, a2, e32, m8, ta, ma\n"
"5e05c057 vmv.v.x v0, a1\n"
" 8e19 sub a2, a2, tmp2\n"
"02056027 vse32.v v0, (a0)\n"
" 070a slli tmp2, tmp2, 0x2\n"
" 953a add a0, a0, tmp2\n"
" f67d bnez a2, -18\n"
" 8082 ret\n");
intptr_t len = 100;
uint32_t* dst = reinterpret_cast<uint32_t*>(malloc(len * 4));
for (intptr_t i = 0; i < len; i++) {
dst[i] = 0;
}
Call(test->entry(), reinterpret_cast<intx_t>(dst), 0x12345678, len);
for (intptr_t i = 0; i < len; i++) {
EXPECT_EQ(0x12345678u, dst[i]);
}
// AVL < VLEN
dst[0] = 0;
dst[1] = 0;
Call(test->entry(), reinterpret_cast<intx_t>(dst), 0x23456789, 1);
EXPECT_EQ(0x23456789u, dst[0]);
EXPECT_EQ(0u, dst[1]);
// AVL = 0
dst[0] = 0;
Call(test->entry(), reinterpret_cast<intx_t>(dst), 0xFFFFFFFF, 0);
EXPECT_EQ(0u, dst[0]);
free(dst);
}
#if XLEN >= 64
ASSEMBLER_TEST_GENERATE(VectorMemorySet64, assembler) {
__ SetExtensions(RV_GC | RV_V);
Label loop;
__ Bind(&loop);
__ vsetvli(A4, A2, e64, m8, ta, ma);
__ vmvvx(V0, A1);
__ sub(A2, A2, A4);
__ vse64v(V0, Address(A0));
__ slli(A4, A4, 3);
__ add(A0, A0, A4);
__ bnez(A2, &loop);
__ ret();
}
ASSEMBLER_TEST_RUN(VectorMemorySet64, test) {
EXPECT_DISASSEMBLY(
"0db67757 vsetvli tmp2, a2, e64, m8, ta, ma\n"
"5e05c057 vmv.v.x v0, a1\n"
" 8e19 sub a2, a2, tmp2\n"
"02057027 vse64.v v0, (a0)\n"
" 070e slli tmp2, tmp2, 0x3\n"
" 953a add a0, a0, tmp2\n"
" f67d bnez a2, -18\n"
" 8082 ret\n");
intptr_t len = 100;
uint64_t* dst = reinterpret_cast<uint64_t*>(malloc(len * 8));
for (intptr_t i = 0; i < len; i++) {
dst[i] = 0;
}
Call(test->entry(), reinterpret_cast<intx_t>(dst), 0x1234567812345678, len);
for (intptr_t i = 0; i < len; i++) {
EXPECT_EQ(0x1234567812345678u, dst[i]);
}
// AVL < VLEN
dst[0] = 0;
dst[1] = 0;
Call(test->entry(), reinterpret_cast<intx_t>(dst), 0x2345678923456789, 1);
EXPECT_EQ(0x2345678923456789u, dst[0]);
EXPECT_EQ(0u, dst[1]);
// AVL = 0
dst[0] = 0;
Call(test->entry(), reinterpret_cast<intx_t>(dst), 0xFFFFFFFF, 0);
EXPECT_EQ(0u, dst[0]);
free(dst);
}
#endif
ASSEMBLER_TEST_GENERATE(LoadByteAcquire, assembler) {
__ SetExtensions(RV_GC | RV_Zalasr);
__ lb(A0, Address(A1), std::memory_order_acquire);
runtime/vm/compiler/assembler/disassembler_riscv.cc
+158
View File
@@ -91,6 +91,9 @@ class RISCVDisassembler {
void DisassembleFNMADD(Instr instr);
void DisassembleFNMSUB(Instr instr);
void DisassembleOPFP(Instr instr);
void DisassembleOPV(Instr instr);
void DisassembleOPV_CFG(Instr instr);
void DisassembleOPV_IVX(Instr instr);
void UnknownInstruction(Instr instr);
void UnknownInstruction(CInstr instr);
@@ -181,6 +184,9 @@ void RISCVDisassembler::DisassembleInstruction(Instr instr) {
case OPFP:
DisassembleOPFP(instr);
break;
case OPV:
DisassembleOPV(instr);
break;
default:
if ((instr.encoding() == 0) ||
(instr.encoding() == static_cast<uint32_t>(-1))) {
@@ -563,12 +569,30 @@ void RISCVDisassembler::DisassembleLOAD(Instr instr) {
void RISCVDisassembler::DisassembleLOADFP(Instr instr) {
switch (instr.funct3()) {
case H:
Print("flh 'frd, 'iimm('rs1)", instr, RV_Zfhmin);
break;
case S:
Print("flw 'frd, 'iimm('rs1)", instr, RV_F);
break;
case D:
Print("fld 'frd, 'iimm('rs1)", instr, RV_D);
break;
case Q:
Print("flq 'frd, 'iimm('rs1)", instr, RV_Q);
break;
case E8:
Print("vle8.v 'vd, ('rs1)'vm", instr, RV_V);
break;
case E16:
Print("vle16.v 'vd, ('rs1)'vm", instr, RV_V);
break;
case E32:
Print("vle32.v 'vd, ('rs1)'vm", instr, RV_V);
break;
case E64:
Print("vle64.v 'vd, ('rs1)'vm", instr, RV_V);
break;
default:
UnknownInstruction(instr);
}
@@ -597,12 +621,30 @@ void RISCVDisassembler::DisassembleSTORE(Instr instr) {
void RISCVDisassembler::DisassembleSTOREFP(Instr instr) {
switch (instr.funct3()) {
case H:
Print("fsh 'frs2, 'simm('rs1)", instr, RV_Zfhmin);
break;
case S:
Print("fsw 'frs2, 'simm('rs1)", instr, RV_F);
break;
case D:
Print("fsd 'frs2, 'simm('rs1)", instr, RV_D);
break;
case Q:
Print("fsq 'frs2, 'simm('rs1)", instr, RV_Q);
break;
case E8:
Print("vse8.v 'vs3, ('rs1)'vm", instr, RV_V);
break;
case E16:
Print("vse16.v 'vs3, ('rs1)'vm", instr, RV_V);
break;
case E32:
Print("vse32.v 'vs3, ('rs1)'vm", instr, RV_V);
break;
case E64:
Print("vse64.v 'vs3, ('rs1)'vm", instr, RV_V);
break;
default:
UnknownInstruction(instr);
}
@@ -1800,6 +1842,40 @@ void RISCVDisassembler::DisassembleOPFP(Instr instr) {
}
}
void RISCVDisassembler::DisassembleOPV(Instr instr) {
switch (instr.funct3()) {
case OPCFG:
DisassembleOPV_CFG(instr);
break;
case OPIVX:
DisassembleOPV_IVX(instr);
break;
default:
UnknownInstruction(instr);
}
}
void RISCVDisassembler::DisassembleOPV_CFG(Instr instr) {
if ((instr.encoding() & 0x80000000) == 0) {
Print("vsetvli 'rd, 'rs1, 'vtypei", instr, RV_V);
} else {
// vsetivli
// vsetvl
UnknownInstruction(instr);
}
}
void RISCVDisassembler::DisassembleOPV_IVX(Instr instr) {
switch (instr.funct6()) {
case VMV:
Print("vmv.v.x 'vd, 'rs1'vm", instr, RV_V);
break;
default:
UnknownInstruction(instr);
break;
}
}
void RISCVDisassembler::UnknownInstruction(Instr instr) {
if (instr.encoding() == 0) {
Print("trap", instr, RV_I);
@@ -1991,6 +2067,88 @@ const char* RISCVDisassembler::PrintOption(const char* format, Instr instr) {
break;
}
return format + 4;
} else if (STRING_STARTS_WITH(format, "vd")) {
Printf("%s", vector_reg_names[instr.vd()]);
return format + 2;
} else if (STRING_STARTS_WITH(format, "vs1")) {
Printf("%s", vector_reg_names[instr.vs1()]);
return format + 3;
} else if (STRING_STARTS_WITH(format, "vs2")) {
Printf("%s", vector_reg_names[instr.vs2()]);
return format + 3;
} else if (STRING_STARTS_WITH(format, "vs3")) {
Printf("%s", vector_reg_names[instr.vs3()]);
return format + 3;
} else if (STRING_STARTS_WITH(format, "vtypei")) {
intx_t vtypei = instr.itype_imm();
// SEW
switch ((vtypei >> 3) & 0b111) {
case e8:
Printf("e8");
break;
case e16:
Printf("e16");
break;
case e32:
Printf("e32");
break;
case e64:
Printf("e64");
break;
default:
Printf("invalid sew");
break;
}
// LMUL
switch ((vtypei >> 0) & 0b111) {
case mf8:
Printf(", mf8");
break;
case mf4:
Printf(", mf4");
break;
case mf2:
Printf(", mf2");
break;
case m1:
Printf(", m1");
break;
case m2:
Printf(", m2");
break;
case m4:
Printf(", m4");
break;
case m8:
Printf(", m8");
break;
default:
Printf(", invalid lmul");
break;
}
// VTA
if ((vtypei & (1 << 6)) == 0) {
Printf(", tu");
} else {
Printf(", ta");
}
// VMA
if ((vtypei & (1 << 7)) == 0) {
Printf(", mu");
} else {
Printf(", ma");
}
return format + 6;
} else if (STRING_STARTS_WITH(format, "vm")) {
if (instr.vm()) {
Printf(", v0.t");
}
return format + 2;
}
FATAL("Bad format %s\n", format);
runtime/vm/constants_riscv.cc
+6
View File
@@ -29,6 +29,12 @@ const char* const fpu_reg_names[kNumberOfFpuRegisters] = {
"fs8", "fs9", "fs10", "fs11", "ft8", "ft9", "ft10", "ft11",
};
const char* const vector_reg_names[kNumberOfVectorRegisters] = {
"v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
"v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21",
"v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31",
};
const Register CallingConventions::ArgumentRegisters[] = {
// A3/A4/A5 are assigned to TMP/TMP2/PP. This assignment is important for
// reducing code size. To minimize distruption to the rest of the compiler,
runtime/vm/constants_riscv.h
+135 -12
View File
@@ -131,6 +131,43 @@ enum FRegister {
kNoFpuRegister = -1,
};
enum VRegister {
V0 = 0,
V1 = 1,
V2 = 2,
V3 = 3,
V4 = 4,
V5 = 5,
V6 = 6,
V7 = 7,
V8 = 8,
V9 = 9,
V10 = 10,
V11 = 11,
V12 = 12,
V13 = 13,
V14 = 14,
V15 = 15,
V16 = 16,
V17 = 17,
V18 = 18,
V19 = 19,
V20 = 20,
V21 = 21,
V22 = 22,
V23 = 23,
V24 = 24,
V25 = 25,
V26 = 26,
V27 = 27,
V28 = 28,
V29 = 29,
V30 = 30,
V31 = 31,
kNumberOfVectorRegisters = 32,
kNoVectorRegister = -1,
};
// Register alias for floating point scratch register.
const FRegister FTMP = FT11;
@@ -143,6 +180,7 @@ typedef double fpu_register_t;
extern const char* const cpu_reg_names[kNumberOfCpuRegisters];
extern const char* const cpu_reg_abi_names[kNumberOfCpuRegisters];
extern const char* const fpu_reg_names[kNumberOfFpuRegisters];
extern const char* const vector_reg_names[kNumberOfVectorRegisters];
// Register aliases.
constexpr Register TMP = A3; // Used as scratch register by assembler.
@@ -741,6 +779,7 @@ enum Opcode {
FNMSUB = 0b1001011,
FNMADD = 0b1001111,
OPFP = 0b1010011,
OPV = 0b1010111,
};
enum Funct12 {
@@ -825,8 +864,15 @@ enum Funct3 {
WIDTH32 = 0b010,
WIDTH64 = 0b011,
H = 0b001,
S = 0b010,
D = 0b011,
Q = 0b100,
E8 = 0b000,
E16 = 0b101,
E32 = 0b110,
E64 = 0b111,
J = 0b000,
JN = 0b001,
JX = 0b010,
@@ -867,6 +913,15 @@ enum Funct3 {
CZEROEQZ = 0b101,
CZERONEZ = 0b111,
OPIVV = 0b000,
OPFVV = 0b001,
OPMVV = 0b010,
OPIVI = 0b011,
OPIVX = 0b100,
OPFVF = 0b101,
OPMVX = 0b110,
OPCFG = 0b111,
};
enum Funct7 {
@@ -922,6 +977,21 @@ enum Funct7 {
SSPUSH = 0b1100111,
};
enum Funct6 {
VADD = 0b000000,
VSUB = 0b000010,
VRSUB = 0b000011,
VMINU = 0b000100,
VMIN = 0b000101,
VMAXU = 0b000110,
VMAX = 0b000111,
VAND = 0b001001,
VOR = 0b001010,
VXOR = 0b001011,
VMV = 0b010111,
};
enum Funct5 {
LR = 0b00010,
SC = 0b00011,
@@ -986,6 +1056,45 @@ enum HartEffects {
const intptr_t kReleaseShift = 25;
const intptr_t kAcquireShift = 26;
enum ElementWidth {
e8 = 0b000,
e16 = 0b001,
e32 = 0b010,
e64 = 0b011,
reservedsew1 = 0b100,
reservedsew2 = 0b101,
reservedsew3 = 0b110,
reservedsew4 = 0b111,
};
enum LengthMultiplier {
mf8 = 0b101,
mf4 = 0b110,
mf2 = 0b111,
m1 = 0b000,
m2 = 0b001,
m4 = 0b010,
m8 = 0b011,
reservedlmul1 = 0b100,
};
enum MaskMode {
mu = 0, // Mask undisturbed
ma = 1, // Mask agnostic
};
enum TailMode {
tu = 0, // Tail undisturbed
ta = 1, // Mask agnosticagnostic
};
enum VectorMask {
v0t = 0,
unmasked = 1 << 25,
};
constexpr uint32_t kFlisConstants[32] = {
0xbf800000, // -1.0
0x00800000, // min positive normal
@@ -1075,6 +1184,10 @@ DEFINE_REG_ENCODING(FRegister, FRd, 7)
DEFINE_REG_ENCODING(FRegister, FRs1, 15)
DEFINE_REG_ENCODING(FRegister, FRs2, 20)
DEFINE_REG_ENCODING(FRegister, FRs3, 27)
DEFINE_REG_ENCODING(VRegister, Vd, 7)
DEFINE_REG_ENCODING(VRegister, Vs1, 15)
DEFINE_REG_ENCODING(VRegister, Vs2, 20)
DEFINE_REG_ENCODING(VRegister, Vs3, 7)
#undef DEFINE_REG_ENCODING
#define DEFINE_FUNCT_ENCODING(type, name, shift, mask) \
@@ -1093,6 +1206,7 @@ DEFINE_FUNCT_ENCODING(Opcode, Opcode, 0, 0x7F)
DEFINE_FUNCT_ENCODING(Funct2, Funct2, 25, 0x3)
DEFINE_FUNCT_ENCODING(Funct3, Funct3, 12, 0x7)
DEFINE_FUNCT_ENCODING(Funct5, Funct5, 27, 0x1F)
DEFINE_FUNCT_ENCODING(Funct6, Funct6, 26, 0x3F)
DEFINE_FUNCT_ENCODING(Funct7, Funct7, 25, 0x7F)
DEFINE_FUNCT_ENCODING(Funct12, Funct12, 20, 0xFFF)
#if XLEN == 32
@@ -1217,9 +1331,16 @@ class Instr {
FRegister frs2() const { return DecodeFRs2(encoding_); }
FRegister frs3() const { return DecodeFRs3(encoding_); }
VRegister vd() const { return DecodeVd(encoding_); }
VRegister vs1() const { return DecodeVs1(encoding_); }
VRegister vs2() const { return DecodeVs2(encoding_); }
VRegister vs3() const { return DecodeVs3(encoding_); }
bool vm() const { return (encoding_ & 1 << 25) == 0; }
Funct2 funct2() const { return DecodeFunct2(encoding_); }
Funct3 funct3() const { return DecodeFunct3(encoding_); }
Funct5 funct5() const { return DecodeFunct5(encoding_); }
Funct6 funct6() const { return DecodeFunct6(encoding_); }
Funct7 funct7() const { return DecodeFunct7(encoding_); }
Funct12 funct12() const { return DecodeFunct12(encoding_); }
@@ -1763,26 +1884,28 @@ static constexpr Extension RV_M(1); // Multiply/divide
static constexpr Extension RV_A(2); // Atomic
static constexpr Extension RV_F(3); // Single-precision floating point
static constexpr Extension RV_D(4); // Double-precision floating point
static constexpr Extension RV_C(5); // Compressed instructions
static constexpr Extension RV_Q(5); // Quad-precision floating point
static constexpr Extension RV_C(6); // Compressed instructions
static constexpr ExtensionSet RV_G = RV_I | RV_M | RV_A | RV_F | RV_D;
static constexpr ExtensionSet RV_GC = RV_G | RV_C;
static constexpr ExtensionSet RVA20 = RV_GC;
static constexpr Extension RV_Zba(6); // Address generation
static constexpr Extension RV_Zbb(7); // Basic bit-manipulation
static constexpr Extension RV_Zbs(8); // Single-bit instructions
static constexpr Extension RV_Zbc(9); // Carry-less multiplication
static constexpr Extension RV_Zba(7); // Address generation
static constexpr Extension RV_Zbb(8); // Basic bit-manipulation
static constexpr Extension RV_Zbs(9); // Single-bit instructions
static constexpr Extension RV_Zbc(10); // Carry-less multiplication
static constexpr ExtensionSet RV_B = RV_Zba | RV_Zbb | RV_Zbs;
static constexpr ExtensionSet RV_GCB = RV_GC | RV_B;
static constexpr ExtensionSet RVA22 = RV_GCB;
static constexpr Extension RV_V(10); // Vector
static constexpr Extension RV_Zicond(11); // Integer conditional operations
static constexpr Extension RV_Zcb(12); // More compressed instructions
static constexpr Extension RV_Zfa(13); // Additional floating-point
static constexpr Extension RV_V(11); // Vector
static constexpr Extension RV_Zicond(12); // Integer conditional operations
static constexpr Extension RV_Zcb(13); // More compressed instructions
static constexpr Extension RV_Zfa(14); // Additional floating-point
static constexpr ExtensionSet RVA23 =
RV_GCB | RV_V | RV_Zicond | RV_Zcb | RV_Zfa;
static constexpr Extension RV_Zicfiss(14); // Shadow stack
static constexpr Extension RV_Zabha(15); // Byte and halfword AMOs
static constexpr Extension RV_Zalasr(16); // Load-acquire, store-release
static constexpr Extension RV_Zicfiss(15); // Shadow stack
static constexpr Extension RV_Zabha(16); // Byte and halfword AMOs
static constexpr Extension RV_Zalasr(17); // Load-acquire, store-release
static constexpr Extension RV_Zfhmin(18); // Load-acquire, store-release
#if defined(DART_TARGET_OS_ANDROID)
static constexpr ExtensionSet RV_baseline = RVA23;
runtime/vm/simulator_riscv.cc
+324 -7
View File
@@ -212,6 +212,11 @@ Simulator::Simulator() : random_(), memory_(FLAG_sim_buffer_memory) {
// fregs_[i] = bit_cast<double>(random_.NextUInt64());
fregs_[i] = bit_cast<double>(kNaNBox);
}
for (intptr_t i = 0; i < kNumberOfVectorRegisters; i++) {
for (intptr_t j = 0; j < VLEN / 8; j++) {
vregs_[i][j] = random_.NextUInt64();
}
}
// The sp is initialized to point to the bottom (high address) of the
// allocated stack area.
@@ -251,6 +256,11 @@ void Simulator::PrepareCall(PreservedRegisters* preserved) {
fregs_[i] = bit_cast<double>(kNaNBox);
}
}
for (intptr_t i = 0; i < kNumberOfVectorRegisters; i++) {
for (intptr_t j = 0; j < VLEN / 8; j++) {
vregs_[i][j] = random_.NextUInt64();
}
}
preserved->ssp = ssp_;
#endif
}
@@ -272,6 +282,11 @@ void Simulator::ClobberVolatileRegisters() {
fregs_[i] = bit_cast<double>(kNaNBox);
}
}
for (intptr_t i = 0; i < kNumberOfVectorRegisters; i++) {
for (intptr_t j = 0; j < VLEN / 8; j++) {
vregs_[i][j] = random_.NextUInt64();
}
}
#endif
}
@@ -489,18 +504,90 @@ void Simulator::PrintRegisters() {
static_cast<intptr_t>(kNumberOfFpuRegisters));
for (intptr_t i = 0; i < kNumberOfCpuRegisters; i++) {
#if XLEN == 32
OS::Print("%4s: %8x %11d", cpu_reg_names[i], xregs_[i], xregs_[i]);
OS::Print("%5s: %8x %11d", cpu_reg_names[i], xregs_[i], xregs_[i]);
#elif XLEN == 64
OS::Print("%4s: %16" Px64 " %20" Pd64, cpu_reg_names[i], xregs_[i],
OS::Print("%5s: %16" Px64 " %20" Pd64, cpu_reg_names[i], xregs_[i],
xregs_[i]);
#endif
OS::Print(" %4s: %lf\n", fpu_reg_names[i], fregs_[i]);
OS::Print(" %5s: %lf\n", fpu_reg_names[i], fregs_[i]);
}
#if XLEN == 32
OS::Print(" pc: %8x\n", pc_);
OS::Print(" pc: %8x\n", pc_);
#elif XLEN == 64
OS::Print(" pc: %16" Px64 "\n", pc_);
OS::Print(" pc: %16" Px64 "\n", pc_);
#endif
for (intptr_t i = 0; i < kNumberOfVectorRegisters; i++) {
OS::Print("%5s: ", vector_reg_names[i]);
for (intptr_t j = VLEN / 8 - 1; j >= 0; j--) {
OS::Print("%02x", (unsigned)vregs_[i][j]);
}
OS::Print("\n");
}
#if XLEN == 32
OS::Print(" vl: %8x %11d\n", vl_, vl_);
#elif XLEN == 64
OS::Print(" vl: %16" Px64 " %20" Pd64 "\n", vl_, vl_);
#endif
#if XLEN == 32
OS::Print("vtype: %8x ", vtype_);
#elif XLEN == 64
OS::Print("vtype: %16" Px64 " ", vtype_);
#endif
switch (vsew()) {
case e8:
OS::Print("e8");
break;
case e16:
OS::Print("e16");
break;
case e32:
OS::Print("e32");
break;
case e64:
OS::Print("e64");
break;
default:
OS::Print("invalid sew");
break;
}
switch (vlmul()) {
case mf8:
OS::Print(", mf8");
break;
case mf4:
OS::Print(", mf4");
break;
case mf2:
OS::Print(", mf2");
break;
case m1:
OS::Print(", m1");
break;
case m2:
OS::Print(", m2");
break;
case m4:
OS::Print(", m4");
break;
case m8:
OS::Print(", m8");
break;
default:
OS::Print(", invalid lmul");
break;
}
if ((vtype_ & (1 << 6)) == 0) {
OS::Print(", tu");
} else {
OS::Print(", ta");
}
if ((vtype_ & (1 << 7)) == 0) {
OS::Print(", mu\n");
} else {
OS::Print(", ma\n");
}
}
void Simulator::PrintStack() {
@@ -581,6 +668,9 @@ void Simulator::Interpret(Instr instr) {
case OPFP:
InterpretOPFP(instr);
break;
case OPV:
InterpretOPV(instr);
break;
default:
IllegalInstruction(instr);
}
@@ -1347,12 +1437,36 @@ void Simulator::InterpretLOADFP(Instr instr) {
case D:
set_fregd(instr.frd(), MemoryRead<double>(addr, instr.rs1()));
break;
case E8:
InterpretLOADV<uint8_t>(instr);
break;
case E16:
InterpretLOADV<uint16_t>(instr);
break;
case E32:
InterpretLOADV<uint32_t>(instr);
break;
case E64:
InterpretLOADV<uint64_t>(instr);
break;
default:
IllegalInstruction(instr);
}
pc_ += instr.length();
}
template <typename T>
void Simulator::InterpretLOADV(Instr instr) {
if ((instr.encoding() & 0xFFF00000) != 0x02000000) {
UNIMPLEMENTED(); // Only unmasked unit-stride implemented.
}
uintx_t base = get_xreg(instr.rs1());
T* vd = ref_vreg<T>(instr.vd());
for (uintx_t i = 0; i < vl_; i++) {
vd[i] = MemoryRead<T>(base + i * sizeof(T), instr.rs1());
}
}
DART_FORCE_INLINE
void Simulator::InterpretSTORE(Instr instr) {
uintx_t addr = get_xreg(instr.rs1()) + instr.stype_imm();
@@ -1387,12 +1501,36 @@ void Simulator::InterpretSTOREFP(Instr instr) {
case D:
MemoryWrite<double>(addr, get_fregd(instr.frs2()), instr.rs1());
break;
case E8:
InterpretSTOREV<uint8_t>(instr);
break;
case E16:
InterpretSTOREV<uint16_t>(instr);
break;
case E32:
InterpretSTOREV<uint32_t>(instr);
break;
case E64:
InterpretSTOREV<uint64_t>(instr);
break;
default:
IllegalInstruction(instr);
}
pc_ += instr.length();
}
template <typename T>
void Simulator::InterpretSTOREV(Instr instr) {
if ((instr.encoding() & 0xFFF00000) != 0x02000000) {
UNIMPLEMENTED(); // Only unmasked unit-stride implemented.
}
uintx_t base = get_xreg(instr.rs1());
T* vs3 = ref_vreg<T>(instr.vs3());
for (uintx_t i = 0; i < vl_; i++) {
MemoryWrite<T>(base + i * sizeof(T), vs3[i], instr.rs1());
}
}
DART_FORCE_INLINE
void Simulator::InterpretOPIMM(Instr instr) {
switch (instr.funct3()) {
@@ -3390,6 +3528,172 @@ void Simulator::InterpretOPFP(Instr instr) {
pc_ += instr.length();
}
void Simulator::InterpretOPV(Instr instr) {
switch (instr.funct3()) {
case OPIVV:
InterpretOPV_IVV(instr);
break;
case OPFVV:
InterpretOPV_FVV(instr);
break;
case OPMVV:
InterpretOPV_MVV(instr);
break;
case OPIVI:
InterpretOPV_IVI(instr);
break;
case OPIVX:
InterpretOPV_IVX(instr);
break;
case OPFVF:
InterpretOPV_FVF(instr);
break;
case OPMVX:
InterpretOPV_MVX(instr);
break;
case OPCFG:
InterpretOPV_CFG(instr);
break;
default:
IllegalInstruction(instr);
}
pc_ += instr.length();
}
void Simulator::InterpretOPV_IVV(Instr instr) {
IllegalInstruction(instr);
}
void Simulator::InterpretOPV_FVV(Instr instr) {
IllegalInstruction(instr);
}
void Simulator::InterpretOPV_MVV(Instr instr) {
IllegalInstruction(instr);
}
void Simulator::InterpretOPV_IVI(Instr instr) {
IllegalInstruction(instr);
}
void Simulator::InterpretOPV_IVX(Instr instr) {
switch (vsew()) {
case e8:
InterpretOPV_IVX<uint8_t>(instr);
break;
case e16:
InterpretOPV_IVX<uint16_t>(instr);
break;
case e32:
InterpretOPV_IVX<uint32_t>(instr);
break;
case e64:
InterpretOPV_IVX<uint64_t>(instr);
break;
default:
FATAL("Invalid SEW");
}
}
template <typename sew_t>
void Simulator::InterpretOPV_IVX(Instr instr) {
if (instr.vm()) UNIMPLEMENTED();
sew_t rs1 = get_xreg(instr.rs1());
sew_t* vs2 = ref_vreg<sew_t>(instr.vs2());
sew_t* vd = ref_vreg<sew_t>(instr.vd());
switch (instr.funct6()) {
case VADD:
for (uintx_t i = 0, n = vl_; i < n; i++) {
vd[i] = rs1 + vs2[i];
}
break;
case VMV:
for (uintx_t i = 0, n = vl_; i < n; i++) {
vd[i] = rs1;
}
break;
default:
IllegalInstruction(instr);
}
}
void Simulator::InterpretOPV_FVF(Instr instr) {
IllegalInstruction(instr);
}
void Simulator::InterpretOPV_MVX(Instr instr) {
IllegalInstruction(instr);
}
void Simulator::InterpretOPV_CFG(Instr instr) {
if ((instr.encoding() & 0x80000000) == 0) {
uintx_t avl = get_xreg(instr.rs1()); // In elements.
intx_t vtype = instr.itype_imm();
uintx_t sew;
switch ((vtype >> 3) & 0b111) {
case e8:
sew = 8;
break;
case e16:
sew = 16;
break;
case e32:
sew = 32;
break;
case e64:
sew = 64;
break;
default:
FATAL("Invalid SEW");
}
intx_t lmul;
switch ((vtype >> 0) & 0b111) {
case mf8:
lmul = -8;
break;
case mf4:
lmul = -4;
break;
case mf2:
lmul = -2;
break;
case m1:
lmul = 1;
break;
case m2:
lmul = 2;
break;
case m4:
lmul = 4;
break;
case m8:
lmul = 8;
break;
default:
FATAL("Invalid LMUL");
}
uintx_t vlmax;
if (lmul < 0) {
vlmax = VLEN / sew / -lmul;
} else {
vlmax = VLEN / sew * lmul;
}
if (instr.rs1() == ZR && instr.rd() != ZR) {
vl_ = vlmax;
} else if (instr.rs1() == ZR) {
// Keep existing vl.
} else if (avl < vlmax) {
vl_ = avl;
} else {
vl_ = vlmax;
}
vtype_ = vtype;
set_xreg(instr.rd(), vl_);
} else {
IllegalInstruction(instr);
}
}
void Simulator::InterpretEBREAK(Instr instr) {
PrintRegisters();
PrintStack();
@@ -3457,15 +3761,22 @@ void Simulator::MemoryWrite(uintx_t addr, type value, Register base) {
}
enum ControlStatusRegister {
// URW
fflags = 0x001,
frm = 0x002,
fcsr = 0x003,
vstart = 0x008,
vxsat = 0x009,
vxrm = 0x00A,
vcsr = 0x00F,
ssp = 0x011,
// URO
cycle = 0xC00,
time = 0xC01,
instret = 0xC02,
vl = 0xC20,
vtype = 0xC21,
vlenb = 0xC22,
#if XLEN == 32
cycleh = 0xC80,
timeh = 0xC81,
@@ -3485,6 +3796,12 @@ intx_t Simulator::CSRRead(uint16_t csr) {
return 0;
case instret:
return instret_;
case vl:
return vl_;
case vtype:
return vtype_;
case vlenb:
return VLEN / 8;
#if XLEN == 32
case cycleh:
return (instret_ / 2) >> 32;
runtime/vm/simulator_riscv.h
+40
View File
@@ -20,6 +20,15 @@ class Mutex;
class SimulatorSetjmpBuffer;
class Thread;
#define ELEN 64
#define VLEN 128
COMPILE_ASSERT(ELEN >= 8);
COMPILE_ASSERT(Utils::IsPowerOfTwo(ELEN));
COMPILE_ASSERT(VLEN >= ELEN);
COMPILE_ASSERT(Utils::IsPowerOfTwo(VLEN));
COMPILE_ASSERT(VLEN <= 0x10000);
// TODO(riscv): Dynamic rounding mode and other FSCR state.
class Simulator {
public:
@@ -284,12 +293,27 @@ class Simulator {
template <typename type>
void InterpretSTOREORDERED(Instr instr);
void InterpretLOADFP(Instr instr);
template <typename type>
void InterpretLOADV(Instr instr);
void InterpretSTOREFP(Instr instr);
template <typename type>
void InterpretSTOREV(Instr instr);
void InterpretFMADD(Instr instr);
void InterpretFMSUB(Instr instr);
void InterpretFNMADD(Instr instr);
void InterpretFNMSUB(Instr instr);
void InterpretOPFP(Instr instr);
void InterpretOPV(Instr instr);
void InterpretOPV_IVV(Instr instr);
void InterpretOPV_FVV(Instr instr);
void InterpretOPV_MVV(Instr instr);
void InterpretOPV_IVI(Instr instr);
void InterpretOPV_IVX(Instr instr);
template <typename sew_t>
void InterpretOPV_IVX(Instr instr);
void InterpretOPV_FVF(Instr instr);
void InterpretOPV_MVX(Instr instr);
void InterpretOPV_CFG(Instr instr);
DART_NORETURN void IllegalInstruction(Instr instr);
DART_NORETURN void IllegalInstruction(CInstr instr);
@@ -337,6 +361,17 @@ class Simulator {
fregs_[rd] = bit_cast<double>(bits64);
}
template <typename T>
T* ref_vreg(VRegister vd) {
return reinterpret_cast<T*>(&vregs_[vd][0]);
}
LengthMultiplier vlmul() const {
return static_cast<LengthMultiplier>((vtype_ >> 0) & 7);
}
ElementWidth vsew() const {
return static_cast<ElementWidth>((vtype_ >> 3) & 7);
}
// Known bad pc value to ensure that the simulator does not execute
// without being properly setup.
static constexpr uword kBadLR = -1;
@@ -363,6 +398,11 @@ class Simulator {
bool ss_enabled_ = false;
uintx_t ssp_ = 0;
// V state
uint8_t vregs_[kNumberOfVectorRegisters][VLEN / 8];
uintx_t vl_;
uintx_t vtype_;
// Simulator support.
char* stack_;
char* shadow_stack_;