Support verified heap pointer writes on ia32.

With --verified_mem, use VerifiedMemory to duplicate all pointer writes in the heap, and verify that no unaccounted writes occurred.

R=iposva@google.com

Review URL: https://codereview.chromium.org//711833002

git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@41700 260f80e4-7a28-3924-810f-c04153c831b5

runtime/vm/assembler_ia32.cc

@@ -13,6 +13,7 @@
 #include "vm/runtime_entry.h"
 #include "vm/stack_frame.h"
 #include "vm/stub_code.h"
+#include "vm/verified_memory.h"
 
 namespace dart {
 
@@ -2198,13 +2199,26 @@ void Assembler::StoreIntoObjectFilter(Register object,
 }
 
 
+void Assembler::VerifiedWrite(const Address& dest, Register value) {
+  // TODO(koda): Verify previous value.
+  movl(dest, value);
+  if (VerifiedMemory::enabled()) {
+    Register temp = (value == EDX) ? ECX : EDX;
+    pushl(temp);
+    leal(temp, dest);
+    movl(Address(temp, VerifiedMemory::offset()), value);
+    popl(temp);
+  }
+}
+
+
 // Destroys the value register.
 void Assembler::StoreIntoObject(Register object,
                                 const Address& dest,
                                 Register value,
                                 bool can_value_be_smi) {
   ASSERT(object != value);
-  movl(dest, value);
+  VerifiedWrite(dest, value);
   Label done;
   if (can_value_be_smi) {
     StoreIntoObjectFilter(object, value, &done);
@@ -2230,7 +2244,7 @@ void Assembler::StoreIntoObject(Register object,
 void Assembler::StoreIntoObjectNoBarrier(Register object,
                                          const Address& dest,
                                          Register value) {
-  movl(dest, value);
+  VerifiedWrite(dest, value);
 #if defined(DEBUG)
   Label done;
   pushl(value);
@@ -2243,24 +2257,47 @@ void Assembler::StoreIntoObjectNoBarrier(Register object,
 }
 
 
+void Assembler::UnverifiedStoreOldObject(const Address& dest,
+                                         const Object& value) {
+  ASSERT(value.IsOld());
+  ASSERT(!value.InVMHeap());
+  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
+  EmitUint8(0xC7);
+  EmitOperand(0, dest);
+  buffer_.EmitObject(value);
+}
+
+
 void Assembler::StoreIntoObjectNoBarrier(Register object,
                                          const Address& dest,
                                          const Object& value) {
   if (value.IsSmi() || value.InVMHeap()) {
-    movl(dest, Immediate(reinterpret_cast<int32_t>(value.raw())));
+    // TODO(koda): Verify previous value.
+    Immediate imm_value(reinterpret_cast<int32_t>(value.raw()));
+    movl(dest, imm_value);
+    if (VerifiedMemory::enabled()) {
+      Register temp = ECX;
+      pushl(temp);
+      leal(temp, dest);
+      movl(Address(temp, VerifiedMemory::offset()), imm_value);
+      popl(temp);
+    }
   } else {
-    ASSERT(value.IsOld());
-    AssemblerBuffer::EnsureCapacity ensured(&buffer_);
-    EmitUint8(0xC7);
-    EmitOperand(0, dest);
-    buffer_.EmitObject(value);
+    UnverifiedStoreOldObject(dest, value);
+    if (VerifiedMemory::enabled()) {
+      Register temp = EDX;
+      pushl(temp);
+      leal(temp, dest);
+      UnverifiedStoreOldObject(Address(temp, VerifiedMemory::offset()), value);
+      popl(temp);
+    }
   }
   // No store buffer update.
 }
 
 
 void Assembler::StoreIntoSmiField(const Address& dest, Register value) {
-  movl(dest, value);
+  VerifiedWrite(dest, value);
 #if defined(DEBUG)
   Label done;
   testl(value, Immediate(kHeapObjectTag));
@@ -2271,9 +2308,34 @@ void Assembler::StoreIntoSmiField(const Address& dest, Register value) {
 }
 
 
+void Assembler::ZeroSmiField(const Address& dest) {
+  Immediate zero(Smi::RawValue(0));
+  // TODO(koda): Verify previous value.
+  movl(dest, zero);
+  if (VerifiedMemory::enabled()) {
+    Register temp = ECX;
+    pushl(temp);
+    leal(temp, dest);
+    movl(Address(temp, VerifiedMemory::offset()), zero);
+    popl(temp);
+  }
+}
+
+
 void Assembler::IncrementSmiField(const Address& dest, int32_t increment) {
+  // Note: FlowGraphCompiler::EdgeCounterIncrementSizeInBytes depends on
+  // the length of this instruction sequence.
+  //
   // TODO(koda): Implement testl for addresses and check that dest is a smi.
-  addl(dest, Immediate(Smi::RawValue(increment)));
+  Immediate inc_imm(Smi::RawValue(increment));
+  addl(dest, inc_imm);
+  if (VerifiedMemory::enabled()) {
+    Register temp = ECX;
+    pushl(temp);
+    leal(temp, dest);
+    addl(Address(temp, VerifiedMemory::offset()), inc_imm);
+    popl(temp);
+  }
 }
 
 

runtime/vm/assembler_ia32.h

@@ -679,6 +679,7 @@ class Assembler : public ValueObject {
 
   // Stores a Smi value into a heap object field that always contains a Smi.
   void StoreIntoSmiField(const Address& dest, Register value);
+  void ZeroSmiField(const Address& dest);
   // Increments a Smi field. Leaves flags in same state as an 'addl'.
   void IncrementSmiField(const Address& dest, int32_t increment);
 
@@ -928,6 +929,10 @@ class Assembler : public ValueObject {
                                   Register value,
                                   Label* no_update);
 
+  // Private helpers for write barrier verification.
+  void VerifiedWrite(const Address& dest, Register value);
+  void UnverifiedStoreOldObject(const Address& dest, const Object& value);
+
   int32_t jit_cookie();
 
   AssemblerBuffer buffer_;

runtime/vm/code_patcher.h

@@ -92,9 +92,6 @@ class CodePatcher : public AllStatic {
   static void InsertCallAt(uword start, uword target);
 
   static RawObject* GetEdgeCounterAt(uword pc, const Code& code);
-#if defined(TARGET_ARCH_IA32)
-  static int32_t EdgeCounterIncrementSizeInBytes();
-#endif  // TARGET_ARCH_IA32
 
   static int32_t GetPoolOffsetAt(uword return_address);
   static void SetPoolOffsetAt(uword return_address, int32_t offset);

runtime/vm/code_patcher_ia32.cc

@@ -9,6 +9,7 @@
 #include "vm/code_patcher.h"
 #include "vm/cpu.h"
 #include "vm/dart_entry.h"
+#include "vm/flow_graph_compiler.h"
 #include "vm/instructions.h"
 #include "vm/object.h"
 #include "vm/raw_object.h"
@@ -275,7 +276,7 @@ intptr_t CodePatcher::InstanceCallSizeInBytes() {
 class EdgeCounter : public ValueObject {
  public:
   EdgeCounter(uword pc, const Code& ignored)
-      : end_(pc - CodePatcher::EdgeCounterIncrementSizeInBytes()) {
+      : end_(pc - FlowGraphCompiler::EdgeCounterIncrementSizeInBytes()) {
     ASSERT(IsValid(end_));
   }
 
@@ -292,14 +293,6 @@ class EdgeCounter : public ValueObject {
 };
 
 
-int32_t CodePatcher::EdgeCounterIncrementSizeInBytes() {
-  // The edge counter load is followed by the fixed-size edge counter
-  // incrementing code:
-  //     83 40 0b 02               add [eax+0xb],0x2
-  return 4;
-}
-
-
 RawObject* CodePatcher::GetEdgeCounterAt(uword pc, const Code& code) {
   ASSERT(code.ContainsInstructionAt(pc));
   EdgeCounter counter(pc, code);

runtime/vm/flow_graph_compiler.h

@@ -384,6 +384,10 @@ class FlowGraphCompiler : public ValueObject {
 
   void EmitEdgeCounter();
 
+#if defined(TARGET_ARCH_IA32)
+  static int32_t EdgeCounterIncrementSizeInBytes();
+#endif  // TARGET_ARCH_IA32
+
   void EmitOptimizedInstanceCall(ExternalLabel* target_label,
                                  const ICData& ic_data,
                                  intptr_t argument_count,

runtime/vm/flow_graph_compiler_ia32.cc

@@ -21,6 +21,7 @@
 #include "vm/stack_frame.h"
 #include "vm/stub_code.h"
 #include "vm/symbols.h"
+#include "vm/verified_memory.h"
 
 namespace dart {
 
@@ -1234,7 +1235,12 @@ void FlowGraphCompiler::EmitEdgeCounter() {
 #endif  // DEBUG
   __ IncrementSmiField(FieldAddress(EAX, Array::element_offset(0)), 1);
   DEBUG_ASSERT((assembler_->CodeSize() - increment_start) ==
-               CodePatcher::EdgeCounterIncrementSizeInBytes());
+               EdgeCounterIncrementSizeInBytes());
+}
+
+
+int32_t FlowGraphCompiler::EdgeCounterIncrementSizeInBytes() {
+  return VerifiedMemory::enabled() ? 16 : 4;
 }
 
 

runtime/vm/intermediate_language_ia32.cc

@@ -2080,13 +2080,13 @@ static void InlineArrayAllocation(FlowGraphCompiler* compiler,
       intptr_t current_offset = 0;
       __ movl(EBX, raw_null);
       while (current_offset < array_size) {
-        __ movl(Address(EDI, current_offset), EBX);
+        __ StoreIntoObjectNoBarrier(EAX, Address(EDI, current_offset), EBX);
         current_offset += kWordSize;
       }
     } else {
       Label init_loop;
       __ Bind(&init_loop);
-      __ movl(Address(EDI, 0), raw_null);
+      __ StoreIntoObjectNoBarrier(EAX, Address(EDI, 0), Object::null_object());
       __ addl(EDI, Immediate(kWordSize));
       __ cmpl(EDI, EBX);
       __ j(BELOW, &init_loop, Assembler::kNearJump);

runtime/vm/intrinsifier_ia32.cc

@@ -114,7 +114,10 @@ void Intrinsifier::GrowableArray_Allocate(Assembler* assembler) {
   // Try allocating in new space.
   const Class& cls = Class::Handle(
       Isolate::Current()->object_store()->growable_object_array_class());
-  __ TryAllocate(cls, &fall_through, Assembler::kNearJump, EAX, EBX);
+  const bool jump_length = VerifiedMemory::enabled() ?
+      Assembler::kFarJump :
+      Assembler::kNearJump;
+  __ TryAllocate(cls, &fall_through, jump_length, EAX, EBX);
 
   // Store backing array object in growable array object.
   __ movl(EBX, Address(ESP, kArrayOffset));  // data argument.
@@ -132,9 +135,7 @@ void Intrinsifier::GrowableArray_Allocate(Assembler* assembler) {
       FieldAddress(EAX, GrowableObjectArray::type_arguments_offset()),
       EBX);
 
-  // Set the length field in the growable array object to 0.
-  __ movl(FieldAddress(EAX, GrowableObjectArray::length_offset()),
-          Immediate(0));
+  __ ZeroSmiField(FieldAddress(EAX, GrowableObjectArray::length_offset()));
   __ ret();  // returns the newly allocated object in EAX.
 
   __ Bind(&fall_through);
@@ -199,7 +200,8 @@ void Intrinsifier::GrowableArraySetLength(Assembler* assembler) {
   __ movl(EBX, Address(ESP, + 1 * kWordSize));  // Length value.
   __ testl(EBX, Immediate(kSmiTagMask));
   __ j(NOT_ZERO, &fall_through, Assembler::kNearJump);  // Non-smi length.
-  __ movl(FieldAddress(EAX, GrowableObjectArray::length_offset()), EBX);
+  FieldAddress length_field(EAX, GrowableObjectArray::length_offset());
+  __ StoreIntoSmiField(length_field, EBX);
   __ ret();
   __ Bind(&fall_through);
 }
@@ -1828,7 +1830,7 @@ void Intrinsifier::OneByteString_getHashCode(Assembler* assembler) {
   __ incl(EAX);
   __ Bind(&set_hash_code);
   __ SmiTag(EAX);
-  __ movl(FieldAddress(EBX, String::hash_offset()), EAX);
+  __ StoreIntoSmiField(FieldAddress(EBX, String::hash_offset()), EAX);
   __ ret();
 }
 
@@ -1901,7 +1903,7 @@ static void TryAllocateOnebyteString(Assembler* assembler,
                               FieldAddress(EAX, String::length_offset()),
                               EDI);
   // Clear hash.
-  __ movl(FieldAddress(EAX, String::hash_offset()), Immediate(0));
+  __ ZeroSmiField(FieldAddress(EAX, String::hash_offset()));
   __ jmp(ok, Assembler::kNearJump);
 
   __ Bind(&pop_and_fail);

runtime/vm/object.cc

@@ -44,6 +44,7 @@
 #include "vm/tags.h"
 #include "vm/timer.h"
 #include "vm/unicode.h"
+#include "vm/verified_memory.h"
 #include "vm/weak_code.h"
 
 namespace dart {
@@ -1698,6 +1699,7 @@ void Object::InitializeObject(uword address, intptr_t class_id, intptr_t size) {
   tags = RawObject::ClassIdTag::update(class_id, tags);
   tags = RawObject::SizeTag::update(size, tags);
   reinterpret_cast<RawObject*>(address)->tags_ = tags;
+  VerifiedMemory::Accept(address, size);
 }
 
 
@@ -1807,6 +1809,7 @@ RawObject* Object::Clone(const Object& orig, Heap::Space space) {
   memmove(reinterpret_cast<uint8_t*>(clone_addr + kHeaderSizeInBytes),
           reinterpret_cast<uint8_t*>(orig_addr + kHeaderSizeInBytes),
           size - kHeaderSizeInBytes);
+  VerifiedMemory::Accept(clone_addr, size);
   // Add clone to store buffer, if needed.
   if (!raw_clone->IsOldObject()) {
     // No need to remember an object in new space.
@@ -12164,6 +12167,7 @@ RawCode* Code::FinalizeCode(const char* name,
   MemoryRegion region(reinterpret_cast<void*>(instrs.EntryPoint()),
                       instrs.size());
   assembler->FinalizeInstructions(region);
+  VerifiedMemory::Accept(region.start(), region.size());
   CPU::FlushICache(instrs.EntryPoint(), instrs.size());
 
   code.set_compile_timestamp(OS::GetCurrentTimeMicros());
@@ -18804,6 +18808,8 @@ RawArray* Array::New(intptr_t class_id, intptr_t len, Heap::Space space) {
                          space));
     NoGCScope no_gc;
     raw->StoreSmi(&(raw->ptr()->length_), Smi::New(len));
+    VerifiedMemory::Accept(reinterpret_cast<uword>(raw->ptr()),
+                           Array::InstanceSize(len));
     return raw;
   }
 }

runtime/vm/object.h

@@ -21,6 +21,7 @@
 #include "vm/report.h"
 #include "vm/scanner.h"
 #include "vm/tags.h"
+#include "vm/verified_memory.h"
 
 namespace dart {
 
@@ -610,6 +611,7 @@ class Object {
     ASSERT(Contains(reinterpret_cast<uword>(to)));
     if (raw()->IsNewObject()) {
       memmove(const_cast<RawObject**>(to), from, count * kWordSize);
+      VerifiedMemory::Accept(reinterpret_cast<uword>(to), count * kWordSize);
     } else {
       for (intptr_t i = 0; i < count; ++i) {
         StorePointer(&to[i], from[i]);
@@ -1586,10 +1588,6 @@ class TypeArguments : public Object {
   static const intptr_t kBytesPerElement = kWordSize;
   static const intptr_t kMaxElements = kSmiMax / kBytesPerElement;
 
-  static intptr_t length_offset() {
-    return OFFSET_OF(RawTypeArguments, length_);
-  }
-
   static intptr_t InstanceSize() {
     ASSERT(sizeof(RawTypeArguments) ==
            OFFSET_OF_RETURNED_VALUE(RawTypeArguments, types));

runtime/vm/pages.cc

@@ -11,6 +11,7 @@
 #include "vm/lockers.h"
 #include "vm/object.h"
 #include "vm/thread.h"
+#include "vm/verified_memory.h"
 #include "vm/virtual_memory.h"
 
 namespace dart {
@@ -59,7 +60,7 @@ HeapPage* HeapPage::Initialize(VirtualMemory* memory, PageType type) {
 
 HeapPage* HeapPage::Allocate(intptr_t size_in_words, PageType type) {
   VirtualMemory* memory =
-      VirtualMemory::Reserve(size_in_words << kWordSizeLog2);
+      VerifiedMemory::Reserve(size_in_words << kWordSizeLog2);
   return Initialize(memory, type);
 }
 

runtime/vm/raw_object.h

@@ -8,8 +8,9 @@
 #include "platform/assert.h"
 #include "vm/atomic.h"
 #include "vm/globals.h"
-#include "vm/token.h"
 #include "vm/snapshot.h"
+#include "vm/token.h"
+#include "vm/verified_memory.h"
 
 namespace dart {
 
@@ -487,7 +488,7 @@ class RawObject {
   void StorePointer(type const* addr, type value) {
     // Ensure that this object contains the addr.
     ASSERT(Contains(reinterpret_cast<uword>(addr)));
-    *const_cast<type*>(addr) = value;
+    VerifiedMemory::Write(const_cast<type*>(addr), value);
     // Filter stores based on source and target.
     if (!value->IsHeapObject()) return;
     if (value->IsNewObject() && this->IsOldObject() &&
@@ -502,7 +503,7 @@ class RawObject {
   void StoreSmi(RawSmi* const* addr, RawSmi* value) {
     // Can't use Contains, as array length is initialized through this method.
     ASSERT(reinterpret_cast<uword>(addr) >= RawObject::ToAddr(this));
-    *const_cast<RawSmi**>(addr) = value;
+    VerifiedMemory::Write(const_cast<RawSmi**>(addr), value);
   }
 
   friend class Api;

runtime/vm/scavenger.cc

@@ -16,6 +16,7 @@
 #include "vm/object_id_ring.h"
 #include "vm/stack_frame.h"
 #include "vm/store_buffer.h"
+#include "vm/verified_memory.h"
 #include "vm/verifier.h"
 #include "vm/visitor.h"
 #include "vm/weak_table.h"
@@ -227,6 +228,7 @@ class ScavengerVisitor : public ObjectPointerVisitor {
       memmove(reinterpret_cast<void*>(new_addr),
               reinterpret_cast<void*>(raw_addr),
               size);
+      VerifiedMemory::Accept(new_addr, size);
       // Remember forwarding address.
       ForwardTo(raw_addr, new_addr);
     }
@@ -235,6 +237,7 @@ class ScavengerVisitor : public ObjectPointerVisitor {
     *p = new_obj;
     // Update the store buffer as needed.
     if (visiting_old_object_ != NULL) {
+      VerifiedMemory::Accept(reinterpret_cast<uword>(p), sizeof(*p));
       UpdateStoreBuffer(p, new_obj);
     }
   }
@@ -356,7 +359,7 @@ SemiSpace* SemiSpace::New(intptr_t size_in_words) {
     return new SemiSpace(NULL);
   } else {
     intptr_t size_in_bytes = size_in_words << kWordSizeLog2;
-    VirtualMemory* reserved = VirtualMemory::Reserve(size_in_bytes);
+    VirtualMemory* reserved = VerifiedMemory::Reserve(size_in_bytes);
     if ((reserved == NULL) || !reserved->Commit(false)) {  // Not executable.
       // TODO(koda): If cache_ is not empty, we could try to delete it.
       delete reserved;
@@ -486,7 +489,7 @@ void Scavenger::Epilogue(Isolate* isolate,
     // objects candidates for promotion next time.
     survivor_end_ = end_;
   }
-
+  VerifiedMemory::Accept(to_->start(), to_->end() - to_->start());
 #if defined(DEBUG)
   // We can only safely verify the store buffers from old space if there is no
   // concurrent old space task. At the same time we prevent new tasks from

runtime/vm/snapshot.cc

@@ -15,6 +15,7 @@
 #include "vm/object_store.h"
 #include "vm/snapshot_ids.h"
 #include "vm/symbols.h"
+#include "vm/verified_memory.h"
 #include "vm/version.h"
 
 namespace dart {
@@ -847,6 +848,7 @@ RawObject* SnapshotReader::AllocateUninitialized(intptr_t class_id,
   // Make sure to initialize the last word, as this can be left untouched in
   // case the object deserialized has an alignment tail.
   *reinterpret_cast<RawObject**>(address + size - kWordSize) = Object::null();
+  VerifiedMemory::Accept(address, size);
 
   RawObject* raw_obj = reinterpret_cast<RawObject*>(address + kHeapObjectTag);
   uword tags = 0;

runtime/vm/stub_code_ia32.cc

@@ -1540,7 +1540,7 @@ void StubCode::GenerateZeroArgsUnoptimizedStaticCallStub(Assembler* assembler) {
   __ addl(EAX, Immediate(Smi::RawValue(1)));
   __ movl(EDI, Immediate(Smi::RawValue(Smi::kMaxValue)));
   __ cmovno(EDI, EAX);
-  __ movl(Address(EBX, count_offset), EDI);
+  __ StoreIntoSmiField(Address(EBX, count_offset), EDI);
 
   // Load arguments descriptor into EDX.
   __ movl(EDX, FieldAddress(ECX, ICData::arguments_descriptor_offset()));

runtime/vm/verified_memory.h

@@ -81,6 +81,8 @@ class VerifiedMemory : public AllStatic {
 #endif
 
   friend class Assembler;  // To use enabled/offset when generating code.
+  friend class FlowGraphCompiler;  // To compute edge counter code size.
+  friend class Intrinsifier;  // To know whether a jump is near or far.
 };
 
 }  // namespace dart

runtime/vm/verifier.cc

@@ -46,6 +46,7 @@ void VerifyObjectVisitor::VisitObject(RawObject* raw_obj) {
 
 void VerifyPointersVisitor::VisitPointers(RawObject** first, RawObject** last) {
   for (RawObject** current = first; current <= last; current++) {
+    VerifiedMemory::Verify(reinterpret_cast<uword>(current), kWordSize);
     RawObject* raw_obj = *current;
     if (raw_obj->IsHeapObject()) {
       if (!allocated_set_->Contains(raw_obj)) {

tests/standalone/standalone.status

@@ -160,3 +160,5 @@ io/skipping_dart2js_compilations_test: Fail # Issue 19551.
 [ $system != linux ]
 io/server_socket_reference_issue21383_and_issue21384_test: Skip # Not supported on other platforms so far
 
+[ $arch != ia32 && $mode == debug ]
+verified_mem_test: Skip  # Not yet implemented.

tests/standalone/verified_mem_test.dart

@@ -0,0 +1,17 @@
+// Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+//
+// Test write barrier verification mode.
+// VMOptions=--verified_mem --verify_before_gc --verify_after_gc
+
+var a = [];
+
+void main() {
+  for (int i = 0; i < 123; ++i) {
+    a.add(new List(12345));
+  }
+  for (int i = 0; i < 12345; ++i) {
+    a[0] = new List(100000);
+  }
+}