[vm] Fail gracefully when unable to reserve address space for the compresssed heap.

TEST=ulimit -v 900000 && dart hello.dart
Bug: https://github.com/dart-lang/sdk/issues/45132
Change-Id: Ifcb0ad26f59ef4269b3e60dccc844d34406f4fcd
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/189162
Reviewed-by: Liam Appelbe <liama@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>

runtime/vm/virtual_memory_posix.cc

@@ -76,10 +76,18 @@ intptr_t VirtualMemory::CalculatePageSize() {
 void VirtualMemory::Init() {
 #if defined(DART_COMPRESSED_POINTERS)
   if (VirtualMemoryCompressedHeap::GetRegion() == nullptr) {
-    VirtualMemoryCompressedHeap::Init(GenericMapAligned(
+    void* address = GenericMapAligned(
         PROT_READ | PROT_WRITE, kCompressedHeapSize, kCompressedHeapAlignment,
         kCompressedHeapSize + kCompressedHeapAlignment,
-        MAP_PRIVATE | MAP_ANONYMOUS));
+        MAP_PRIVATE | MAP_ANONYMOUS);
+    if (address == nullptr) {
+      int error = errno;
+      const int kBufferSize = 1024;
+      char error_buf[kBufferSize];
+      FATAL2("Failed to reserve region for compressed heap: %d (%s)", error,
+             Utils::StrError(error, error_buf, kBufferSize));
+    }
+    VirtualMemoryCompressedHeap::Init(address);
   }
 #endif  // defined(DART_COMPRESSED_POINTERS)
 
@@ -328,7 +336,7 @@ VirtualMemory* VirtualMemory::AllocateAligned(intptr_t size,
 #endif  // defined(HOST_OS_MACOS)
   void* address =
       GenericMapAligned(prot, size, alignment, allocated_size, map_flags);
-  if (address == MAP_FAILED) {
+  if (address == nullptr) {
     return nullptr;
   }
 

runtime/vm/virtual_memory_win.cc

@@ -55,9 +55,20 @@ void VirtualMemory::Init() {
   page_size_ = CalculatePageSize();
 
 #if defined(DART_COMPRESSED_POINTERS)
-  VirtualMemoryCompressedHeap::Init(AllocateAlignedImpl(
-      kCompressedHeapSize, kCompressedHeapAlignment,
-      kCompressedHeapSize + kCompressedHeapAlignment, PAGE_READWRITE, nullptr));
+  if (VirtualMemoryCompressedHeap::GetRegion() == nullptr) {
+    void* address =
+        AllocateAlignedImpl(kCompressedHeapSize, kCompressedHeapAlignment,
+                            kCompressedHeapSize + kCompressedHeapAlignment,
+                            PAGE_READWRITE, nullptr);
+    if (address == nullptr) {
+      int error = GetLastError();
+      const int kBufferSize = 1024;
+      char error_buf[kBufferSize];
+      FATAL2("Failed to reserve region for compressed heap: %d (%s)", error,
+             Utils::StrError(error, error_buf, kBufferSize));
+    }
+    VirtualMemoryCompressedHeap::Init(address);
+  }
 #endif  // defined(DART_COMPRESSED_POINTERS)
 }