[vm, gc] Make the generational barriers in the runtime multi-thread safe.

TEST=ci
Bug: https://github.com/dart-lang/sdk/issues/46909
Change-Id: I9e26b88779ff9039ab7723d07ef9f8568e6595d8
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/210283
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Martin Kustermann <kustermann@google.com>

runtime/vm/heap/scavenger.cc

@@ -291,7 +291,7 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
     if (!obj->IsNewObject() || visiting_old_object_->untag()->IsRemembered()) {
       return;
     }
-    visiting_old_object_->untag()->SetRememberedBit();
+    visiting_old_object_->untag()->SetRememberedBitUnsynchronized();
     thread_->StoreBufferAddObjectGC(visiting_old_object_);
   }
 

runtime/vm/raw_object.h

@@ -317,10 +317,14 @@ class UntaggedObject {
     ASSERT(IsOldObject());
     return !tags_.Read<OldAndNotRememberedBit>();
   }
-  void SetRememberedBit() {
+  bool TryAcquireRememberedBit() {
+    ASSERT(!IsCardRemembered());
+    return tags_.TryClear<OldAndNotRememberedBit>();
+  }
+  void SetRememberedBitUnsynchronized() {
     ASSERT(!IsRemembered());
     ASSERT(!IsCardRemembered());
-    tags_.UpdateBool<OldAndNotRememberedBit>(false);
+    tags_.UpdateUnsynchronized<OldAndNotRememberedBit>(false);
   }
   void ClearRememberedBit() {
     ASSERT(IsOldObject());
@@ -328,10 +332,10 @@ class UntaggedObject {
   }
 
   DART_FORCE_INLINE
-  void AddToRememberedSet(Thread* thread) {
-    ASSERT(!this->IsRemembered());
-    this->SetRememberedBit();
-    thread->StoreBufferAddObject(ObjectPtr(this));
+  void EnsureInRememberedSet(Thread* thread) {
+    if (TryAcquireRememberedBit()) {
+      thread->StoreBufferAddObject(ObjectPtr(this));
+    }
   }
 
   bool IsCardRemembered() const { return tags_.Read<CardRememberedBit>(); }
@@ -665,7 +669,7 @@ class UntaggedObject {
       if (value->IsNewObject()) {
         // Generational barrier: record when a store creates an
         // old-and-not-remembered -> new reference.
-        AddToRememberedSet(thread);
+        EnsureInRememberedSet(thread);
       } else {
         // Incremental barrier: record when a store creates an
         // old -> old-and-not-marked reference.
@@ -695,11 +699,9 @@ class UntaggedObject {
       if (value->IsNewObject()) {
         // Generational barrier: record when a store creates an
         // old-and-not-remembered -> new reference.
-        ASSERT(!this->IsRemembered());
         if (this->IsCardRemembered()) {
           RememberCard(addr);
-        } else {
-          this->SetRememberedBit();
+        } else if (this->TryAcquireRememberedBit()) {
           thread->StoreBufferAddObject(static_cast<ObjectPtr>(this));
         }
       } else {

runtime/vm/runtime_entry.cc

@@ -483,7 +483,7 @@ DEFINE_LEAF_RUNTIME_ENTRY(uword /*ObjectPtr*/,
   }
 
   if (add_to_remembered_set) {
-    object->untag()->AddToRememberedSet(thread);
+    object->untag()->EnsureInRememberedSet(thread);
   }
 
   // For incremental write barrier elimination, we need to ensure that the

runtime/vm/thread.cc

@@ -688,9 +688,7 @@ class RestoreWriteBarrierInvariantVisitor : public ObjectPointerVisitor {
 
       switch (op_) {
         case Thread::RestoreWriteBarrierInvariantOp::kAddToRememberedSet:
-          if (!obj->untag()->IsRemembered()) {
-            obj->untag()->AddToRememberedSet(current_);
-          }
+          obj->untag()->EnsureInRememberedSet(current_);
           if (current_->is_marking()) {
             current_->DeferredMarkingStackAddObject(obj);
           }