[vm] Add support for the Zalasr extension.

TEST=ci
Change-Id: Ic1a98751ac6b46310e420b44f487720f0dc237c8
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/366162
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Alexander Aprelev <aam@google.com>

runtime/vm/compiler/assembler/assembler_riscv.cc

@@ -1634,6 +1634,72 @@ void MicroAssembler::bseti(Register rd, Register rs1, intx_t shamt) {
   EmitRType(BSET, shamt, rs1, F3_BSET, rd, OPIMM);
 }
 
+void MicroAssembler::lb(Register rd, Address addr, std::memory_order order) {
+  ASSERT(addr.offset() == 0);
+  ASSERT((order == std::memory_order_acquire) ||
+         (order == std::memory_order_acq_rel));
+  ASSERT(Supports(RV_Zalasr));
+  EmitRType(LOADORDERED, order, ZR, addr.base(), WIDTH8, rd, AMO);
+}
+
+void MicroAssembler::lh(Register rd, Address addr, std::memory_order order) {
+  ASSERT(addr.offset() == 0);
+  ASSERT((order == std::memory_order_acquire) ||
+         (order == std::memory_order_acq_rel));
+  ASSERT(Supports(RV_Zalasr));
+  EmitRType(LOADORDERED, order, ZR, addr.base(), WIDTH16, rd, AMO);
+}
+
+void MicroAssembler::lw(Register rd, Address addr, std::memory_order order) {
+  ASSERT(addr.offset() == 0);
+  ASSERT((order == std::memory_order_acquire) ||
+         (order == std::memory_order_acq_rel));
+  ASSERT(Supports(RV_Zalasr));
+  EmitRType(LOADORDERED, order, ZR, addr.base(), WIDTH32, rd, AMO);
+}
+
+void MicroAssembler::sb(Register rs2, Address addr, std::memory_order order) {
+  ASSERT(addr.offset() == 0);
+  ASSERT((order == std::memory_order_release) ||
+         (order == std::memory_order_acq_rel));
+  ASSERT(Supports(RV_Zalasr));
+  EmitRType(STOREORDERED, order, rs2, addr.base(), WIDTH8, ZR, AMO);
+}
+
+void MicroAssembler::sh(Register rs2, Address addr, std::memory_order order) {
+  ASSERT(addr.offset() == 0);
+  ASSERT((order == std::memory_order_release) ||
+         (order == std::memory_order_acq_rel));
+  ASSERT(Supports(RV_Zalasr));
+  EmitRType(STOREORDERED, order, rs2, addr.base(), WIDTH16, ZR, AMO);
+}
+
+void MicroAssembler::sw(Register rs2, Address addr, std::memory_order order) {
+  ASSERT(addr.offset() == 0);
+  ASSERT((order == std::memory_order_release) ||
+         (order == std::memory_order_acq_rel));
+  ASSERT(Supports(RV_Zalasr));
+  EmitRType(STOREORDERED, order, rs2, addr.base(), WIDTH32, ZR, AMO);
+}
+
+#if XLEN >= 64
+void MicroAssembler::ld(Register rd, Address addr, std::memory_order order) {
+  ASSERT(addr.offset() == 0);
+  ASSERT((order == std::memory_order_acquire) ||
+         (order == std::memory_order_acq_rel));
+  ASSERT(Supports(RV_Zalasr));
+  EmitRType(LOADORDERED, order, ZR, addr.base(), WIDTH64, rd, AMO);
+}
+
+void MicroAssembler::sd(Register rs2, Address addr, std::memory_order order) {
+  ASSERT(addr.offset() == 0);
+  ASSERT((order == std::memory_order_release) ||
+         (order == std::memory_order_acq_rel));
+  ASSERT(Supports(RV_Zalasr));
+  EmitRType(STOREORDERED, order, rs2, addr.base(), WIDTH64, ZR, AMO);
+}
+#endif
+
 void MicroAssembler::c_lwsp(Register rd, Address addr) {
   ASSERT(rd != ZR);
   ASSERT(addr.base() == SP);
@@ -2582,8 +2648,31 @@ void Assembler::LoadAcquire(Register dst,
                             const Address& address,
                             OperandSize size) {
   ASSERT(dst != address.base());
-  Load(dst, address, size);
-  fence(HartEffects::kRead, HartEffects::kMemory);
+
+  if (Supports(RV_Zalasr)) {
+    Address addr = PrepareAtomicOffset(address.base(), address.offset());
+    switch (size) {
+#if XLEN == 64
+      case kEightBytes:
+        ld(dst, addr, std::memory_order_acquire);
+        break;
+#endif
+      case kFourBytes:
+        lw(dst, addr, std::memory_order_acquire);
+        break;
+      case kTwoBytes:
+        lh(dst, addr, std::memory_order_acquire);
+        break;
+      case kByte:
+        lb(dst, addr, std::memory_order_acquire);
+        break;
+      default:
+        UNREACHABLE();
+    }
+  } else {
+    Load(dst, address, size);
+    fence(HartEffects::kRead, HartEffects::kMemory);
+  }
 
   if (FLAG_target_thread_sanitizer) {
     if (address.offset() == 0) {
@@ -2598,8 +2687,33 @@ void Assembler::LoadAcquire(Register dst,
 void Assembler::StoreRelease(Register src,
                              const Address& address,
                              OperandSize size) {
-  fence(HartEffects::kMemory, HartEffects::kWrite);
-  Store(src, address, size);
+  if (Supports(RV_Zalasr)) {
+    Address addr = PrepareAtomicOffset(address.base(), address.offset());
+    switch (size) {
+#if XLEN == 64
+      case kEightBytes:
+        sd(src, addr, std::memory_order_release);
+        break;
+#endif
+      case kUnsignedFourBytes:
+      case kFourBytes:
+        sw(src, addr, std::memory_order_release);
+        break;
+      case kUnsignedTwoBytes:
+      case kTwoBytes:
+        sh(src, addr, std::memory_order_release);
+        break;
+      case kUnsignedByte:
+      case kByte:
+        sb(src, addr, std::memory_order_release);
+        break;
+      default:
+        UNREACHABLE();
+    }
+  } else {
+    fence(HartEffects::kMemory, HartEffects::kWrite);
+    Store(src, address, size);
+  }
 }
 
 void Assembler::CompareWithMemoryValue(Register value,
@@ -3165,6 +3279,15 @@ Address Assembler::PrepareLargeOffset(Register base, int32_t offset) {
   return Address(TMP2, lo);
 }
 
+Address Assembler::PrepareAtomicOffset(Register base, int32_t offset) {
+  ASSERT(base != TMP2);
+  if (offset == 0) {
+    return Address(base, 0);
+  }
+  AddImmediate(TMP2, base, offset);
+  return Address(TMP2, 0);
+}
+
 void Assembler::Load(Register dest, const Address& address, OperandSize sz) {
   Address addr = PrepareLargeOffset(address.base(), address.offset());
   switch (sz) {

runtime/vm/compiler/assembler/assembler_riscv.h

@@ -603,6 +603,19 @@ class MicroAssembler : public AssemblerBase {
   void bset(Register rd, Register rs1, Register rs2);
   void bseti(Register rd, Register rs1, intx_t shamt);
 
+  // ==== Zalasr: Load-acquire, store-release ====
+  void lb(Register rd, Address addr, std::memory_order order);
+  void lh(Register rd, Address addr, std::memory_order order);
+  void lw(Register rd, Address addr, std::memory_order order);
+  void sb(Register rs2, Address addr, std::memory_order order);
+  void sh(Register rs2, Address addr, std::memory_order order);
+  void sw(Register rs2, Address addr, std::memory_order order);
+
+#if XLEN >= 64
+  void ld(Register rd, Address addr, std::memory_order order);
+  void sd(Register rs2, Address addr, std::memory_order order);
+#endif
+
   // ==== Dart Simulator Debugging ====
   void SimulatorPrintObject(Register rs1);
 
@@ -1092,6 +1105,7 @@ class Assembler : public MicroAssembler {
                         OperandSize sz = kWordBytes) override;
 
   Address PrepareLargeOffset(Register base, int32_t offset);
+  Address PrepareAtomicOffset(Register base, int32_t offset);
   void Load(Register dest,
             const Address& address,
             OperandSize sz = kWordBytes) override;

runtime/vm/compiler/assembler/assembler_riscv_test.cc

@@ -7112,6 +7112,124 @@ ASSEMBLER_TEST_RUN(BitSetImmediate2, test) {
   EXPECT_EQ(-1, Call(test->entry(), -1));
 }
 
+ASSEMBLER_TEST_GENERATE(LoadByteAcquire, assembler) {
+  __ SetExtensions(RV_GC | RV_Zalasr);
+  __ lb(A0, Address(A1), std::memory_order_acquire);
+  __ ret();
+}
+ASSEMBLER_TEST_RUN(LoadByteAcquire, test) {
+  EXPECT_DISASSEMBLY(
+      "3405852f lb.aq a0, (a1)\n"
+      "    8082 ret\n");
+
+  int8_t data = -42;
+  EXPECT_EQ(-42, Call(test->entry(), 0, reinterpret_cast<intx_t>(&data)));
+}
+
+ASSEMBLER_TEST_GENERATE(LoadHalfwordAcquire, assembler) {
+  __ SetExtensions(RV_GC | RV_Zalasr);
+  __ lh(A0, Address(A1), std::memory_order_acquire);
+  __ ret();
+}
+ASSEMBLER_TEST_RUN(LoadHalfwordAcquire, test) {
+  EXPECT_DISASSEMBLY(
+      "3405952f lh.aq a0, (a1)\n"
+      "    8082 ret\n");
+
+  int16_t data = -42;
+  EXPECT_EQ(-42, Call(test->entry(), 0, reinterpret_cast<intx_t>(&data)));
+}
+
+ASSEMBLER_TEST_GENERATE(LoadWordAcquire, assembler) {
+  __ SetExtensions(RV_GC | RV_Zalasr);
+  __ lw(A0, Address(A1), std::memory_order_acquire);
+  __ ret();
+}
+ASSEMBLER_TEST_RUN(LoadWordAcquire, test) {
+  EXPECT_DISASSEMBLY(
+      "3405a52f lw.aq a0, (a1)\n"
+      "    8082 ret\n");
+
+  int32_t data = -42;
+  EXPECT_EQ(-42, Call(test->entry(), 0, reinterpret_cast<intx_t>(&data)));
+}
+
+ASSEMBLER_TEST_GENERATE(StoreByteRelease, assembler) {
+  __ SetExtensions(RV_GC | RV_Zalasr);
+  __ sb(A0, Address(A1), std::memory_order_release);
+  __ ret();
+}
+ASSEMBLER_TEST_RUN(StoreByteRelease, test) {
+  EXPECT_DISASSEMBLY(
+      "3aa5802f sb.rl a0, (a1)\n"
+      "    8082 ret\n");
+
+  int8_t data = 0;
+  EXPECT_EQ(-42, Call(test->entry(), -42, reinterpret_cast<intx_t>(&data)));
+  EXPECT_EQ(-42, data);
+}
+
+ASSEMBLER_TEST_GENERATE(StoreHalfwordRelease, assembler) {
+  __ SetExtensions(RV_GC | RV_Zalasr);
+  __ sh(A0, Address(A1), std::memory_order_release);
+  __ ret();
+}
+ASSEMBLER_TEST_RUN(StoreHalfwordRelease, test) {
+  EXPECT_DISASSEMBLY(
+      "3aa5902f sh.rl a0, (a1)\n"
+      "    8082 ret\n");
+
+  int16_t data = 0;
+  EXPECT_EQ(-42, Call(test->entry(), -42, reinterpret_cast<intx_t>(&data)));
+  EXPECT_EQ(-42, data);
+}
+
+ASSEMBLER_TEST_GENERATE(StoreWordRelease, assembler) {
+  __ SetExtensions(RV_GC | RV_Zalasr);
+  __ sw(A0, Address(A1), std::memory_order_release);
+  __ ret();
+}
+ASSEMBLER_TEST_RUN(StoreWordRelease, test) {
+  EXPECT_DISASSEMBLY(
+      "3aa5a02f sw.rl a0, (a1)\n"
+      "    8082 ret\n");
+
+  int32_t data = 0;
+  EXPECT_EQ(-42, Call(test->entry(), -42, reinterpret_cast<intx_t>(&data)));
+  EXPECT_EQ(-42, data);
+}
+
+#if XLEN >= 64
+ASSEMBLER_TEST_GENERATE(LoadDoubleWordAcquire, assembler) {
+  __ SetExtensions(RV_GC | RV_Zalasr);
+  __ ld(A0, Address(A1), std::memory_order_acquire);
+  __ ret();
+}
+ASSEMBLER_TEST_RUN(LoadDoubleWordAcquire, test) {
+  EXPECT_DISASSEMBLY(
+      "3405b52f ld.aq a0, (a1)\n"
+      "    8082 ret\n");
+
+  int64_t data = -42;
+  EXPECT_EQ(-42, Call(test->entry(), 0, reinterpret_cast<intx_t>(&data)));
+}
+
+ASSEMBLER_TEST_GENERATE(StoreDoubleWordRelease, assembler) {
+  __ SetExtensions(RV_GC | RV_Zalasr);
+  __ sd(A0, Address(A1), std::memory_order_release);
+  __ ret();
+}
+ASSEMBLER_TEST_RUN(StoreDoubleWordRelease, test) {
+  EXPECT_DISASSEMBLY(
+      "3aa5b02f sd.rl a0, (a1)\n"
+      "    8082 ret\n");
+
+  int64_t data = 0;
+  EXPECT_EQ(-42, Call(test->entry(), -42, reinterpret_cast<intx_t>(&data)));
+  EXPECT_EQ(-42, data);
+}
+#endif  // XLEN >= 64
+
 ASSEMBLER_TEST_GENERATE(LoadImmediate_MaxInt32, assembler) {
   FLAG_use_compressed_instructions = true;
   __ SetExtensions(RV_GC);

runtime/vm/compiler/assembler/disassembler_riscv.cc

@@ -79,6 +79,8 @@ class RISCVDisassembler {
   void DisassembleMISCMEM(Instr instr);
   void DisassembleSYSTEM(Instr instr);
   void DisassembleAMO(Instr instr);
+  void DisassembleAMO8(Instr instr);
+  void DisassembleAMO16(Instr instr);
   void DisassembleAMO32(Instr instr);
   void DisassembleAMO64(Instr instr);
   void DisassembleLOADFP(Instr instr);
@@ -1097,6 +1099,12 @@ void RISCVDisassembler::DisassembleSYSTEM(Instr instr) {
 
 void RISCVDisassembler::DisassembleAMO(Instr instr) {
   switch (instr.funct3()) {
+    case WIDTH8:
+      DisassembleAMO8(instr);
+      break;
+    case WIDTH16:
+      DisassembleAMO16(instr);
+      break;
     case WIDTH32:
       DisassembleAMO32(instr);
       break;
@@ -1108,6 +1116,32 @@ void RISCVDisassembler::DisassembleAMO(Instr instr) {
   }
 }
 
+void RISCVDisassembler::DisassembleAMO8(Instr instr) {
+  switch (instr.funct5()) {
+    case LOADORDERED:
+      Print("lb'order 'rd, ('rs1)", instr, RV_Zalasr);
+      break;
+    case STOREORDERED:
+      Print("sb'order 'rs2, ('rs1)", instr, RV_Zalasr);
+      break;
+    default:
+      UnknownInstruction(instr);
+  }
+}
+
+void RISCVDisassembler::DisassembleAMO16(Instr instr) {
+  switch (instr.funct5()) {
+    case LOADORDERED:
+      Print("lh'order 'rd, ('rs1)", instr, RV_Zalasr);
+      break;
+    case STOREORDERED:
+      Print("sh'order 'rs2, ('rs1)", instr, RV_Zalasr);
+      break;
+    default:
+      UnknownInstruction(instr);
+  }
+}
+
 void RISCVDisassembler::DisassembleAMO32(Instr instr) {
   switch (instr.funct5()) {
     case LR:
@@ -1143,6 +1177,12 @@ void RISCVDisassembler::DisassembleAMO32(Instr instr) {
     case AMOMAXU:
       Print("amomaxu.w'order 'rd, 'rs2, ('rs1)", instr, RV_A);
       break;
+    case LOADORDERED:
+      Print("lw'order 'rd, ('rs1)", instr, RV_Zalasr);
+      break;
+    case STOREORDERED:
+      Print("sw'order 'rs2, ('rs1)", instr, RV_Zalasr);
+      break;
     default:
       UnknownInstruction(instr);
   }
@@ -1184,6 +1224,12 @@ void RISCVDisassembler::DisassembleAMO64(Instr instr) {
     case AMOMAXU:
       Print("amomaxu.d'order 'rd, 'rs2, ('rs1)", instr, RV_A);
       break;
+    case LOADORDERED:
+      Print("ld'order 'rd, ('rs1)", instr, RV_Zalasr);
+      break;
+    case STOREORDERED:
+      Print("sd'order 'rs2, ('rs1)", instr, RV_Zalasr);
+      break;
 #endif
     default:
       UnknownInstruction(instr);

runtime/vm/constants_riscv.h

@@ -799,6 +799,8 @@ enum Funct3 {
   REMW = 0b110,
   REMUW = 0b111,
 
+  WIDTH8 = 0b000,
+  WIDTH16 = 0b001,
   WIDTH32 = 0b010,
   WIDTH64 = 0b011,
 
@@ -898,6 +900,8 @@ enum Funct5 {
   AMOMAX = 0b10100,
   AMOMINU = 0b11000,
   AMOMAXU = 0b11100,
+  LOADORDERED = 0b00110,
+  STOREORDERED = 0b00111,
 };
 
 enum Funct2 {
@@ -1597,7 +1601,8 @@ static constexpr Extension RV_Zbb(7);  // Basic bit-manipulation
 static constexpr Extension RV_Zbs(8);  // Single-bit instructions
 static constexpr ExtensionSet RV_B = RV_Zba | RV_Zbb | RV_Zbs;
 static constexpr ExtensionSet RV_GCB = RV_GC | RV_B;
-static constexpr Extension RV_Zbc(9);  // Carry-less multiplication
+static constexpr Extension RV_Zbc(9);      // Carry-less multiplication
+static constexpr Extension RV_Zalasr(10);  // Load-acquire, store-release
 
 #undef R
 

runtime/vm/simulator_riscv.cc

@@ -2088,6 +2088,12 @@ void Simulator::InterpretECALL(Instr instr) {
 
 void Simulator::InterpretAMO(Instr instr) {
   switch (instr.funct3()) {
+    case WIDTH8:
+      InterpretAMO8(instr);
+      break;
+    case WIDTH16:
+      InterpretAMO16(instr);
+      break;
     case WIDTH32:
       InterpretAMO32(instr);
       break;
@@ -2226,6 +2232,56 @@ void Simulator::InterpretAMOMAX(Instr instr) {
   set_xreg(instr.rd(), sign_extend(expected));
 }
 
+template <typename type>
+void Simulator::InterpretLOADORDERED(Instr instr) {
+  uintx_t addr = get_xreg(instr.rs1());
+  if ((addr & (sizeof(type) - 1)) != 0) {
+    FATAL("Misaligned atomic memory operation");
+  }
+  std::atomic<type>* atomic = reinterpret_cast<std::atomic<type>*>(addr);
+  type value = atomic->load(instr.memory_order());
+  set_xreg(instr.rd(), sign_extend(value));
+}
+
+template <typename type>
+void Simulator::InterpretSTOREORDERED(Instr instr) {
+  uintx_t addr = get_xreg(instr.rs1());
+  if ((addr & (sizeof(type) - 1)) != 0) {
+    FATAL("Misaligned atomic memory operation");
+  }
+  type value = get_xreg(instr.rs2());
+  std::atomic<type>* atomic = reinterpret_cast<std::atomic<type>*>(addr);
+  atomic->store(value, instr.memory_order());
+}
+
+void Simulator::InterpretAMO8(Instr instr) {
+  switch (instr.funct5()) {
+    case LOADORDERED:
+      InterpretLOADORDERED<int8_t>(instr);
+      break;
+    case STOREORDERED:
+      InterpretSTOREORDERED<int8_t>(instr);
+      break;
+    default:
+      IllegalInstruction(instr);
+  }
+  pc_ += instr.length();
+}
+
+void Simulator::InterpretAMO16(Instr instr) {
+  switch (instr.funct5()) {
+    case LOADORDERED:
+      InterpretLOADORDERED<int16_t>(instr);
+      break;
+    case STOREORDERED:
+      InterpretSTOREORDERED<int16_t>(instr);
+      break;
+    default:
+      IllegalInstruction(instr);
+  }
+  pc_ += instr.length();
+}
+
 void Simulator::InterpretAMO32(Instr instr) {
   switch (instr.funct5()) {
     case LR:
@@ -2261,6 +2317,12 @@ void Simulator::InterpretAMO32(Instr instr) {
     case AMOMAXU:
       InterpretAMOMAX<uint32_t>(instr);
       break;
+    case LOADORDERED:
+      InterpretLOADORDERED<int32_t>(instr);
+      break;
+    case STOREORDERED:
+      InterpretSTOREORDERED<int32_t>(instr);
+      break;
     default:
       IllegalInstruction(instr);
   }
@@ -2303,6 +2365,12 @@ void Simulator::InterpretAMO64(Instr instr) {
     case AMOMAXU:
       InterpretAMOMAX<uint64_t>(instr);
       break;
+    case LOADORDERED:
+      InterpretLOADORDERED<int64_t>(instr);
+      break;
+    case STOREORDERED:
+      InterpretSTOREORDERED<int64_t>(instr);
+      break;
 #endif  // XLEN >= 64
     default:
       IllegalInstruction(instr);

runtime/vm/simulator_riscv.h

@@ -225,6 +225,8 @@ class Simulator {
   void InterpretEBREAK(Instr instr);
   void InterpretEBREAK(CInstr instr);
   void InterpretAMO(Instr instr);
+  void InterpretAMO8(Instr instr);
+  void InterpretAMO16(Instr instr);
   void InterpretAMO32(Instr instr);
   void InterpretAMO64(Instr instr);
   template <typename type>
@@ -249,6 +251,10 @@ class Simulator {
   void InterpretAMOMINU(Instr instr);
   template <typename type>
   void InterpretAMOMAXU(Instr instr);
+  template <typename type>
+  void InterpretLOADORDERED(Instr instr);
+  template <typename type>
+  void InterpretSTOREORDERED(Instr instr);
   void InterpretLOADFP(Instr instr);
   void InterpretSTOREFP(Instr instr);
   void InterpretFMADD(Instr instr);