dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[vm, gc] Common page structure for both generations.

Browse Source 
- Prep work for non-moving promotion.
 - The generations still have different alignment offsets.
 - Old-space pages now participate in the page cache.
 - Fix boolean/null bit tricks to assert the right requirements on the alignment of a page's first object.

TEST=ci
Change-Id: I4369d8c6af73228e162c226d411914868bafed33
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/260401
Reviewed-by: Siva Annamalai <asiva@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>
This commit is contained in:
Ryan Macnak
2022-09-27 21:34:05 +00:00
committed by Commit Queue
parent 3616bd5992
commit efbf68f306
47 changed files with 1048 additions and 1058 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/vm/compiler/relocation.cc
+1 -1
View File
@@ -344,7 +344,7 @@ void CodeRelocator::ResolveCallToDestination(UnresolvedCall* unresolved_call,
auto const caller = unresolved_call->caller;
uword addr = Code::PayloadStartOf(caller) + call_offset;
if (FLAG_write_protect_code) {
addr -= OldPage::Of(Code::InstructionsOf(caller))->AliasOffset();
addr -= Page::Of(Code::InstructionsOf(caller))->AliasOffset();
}
if (unresolved_call->is_tail_call) {
PcRelativeTailCallPattern call(addr);
runtime/vm/compiler/relocation_test.cc
+3 -3
View File
@@ -136,7 +136,7 @@ struct RelocatorTestHelper {
if (FLAG_write_protect_code && FLAG_dual_map_code) {
auto& instructions = Instructions::Handle(code.instructions());
instructions ^= OldPage::ToExecutable(instructions.ptr());
instructions ^= Page::ToExecutable(instructions.ptr());
code.set_instructions(instructions);
}
if (FLAG_disassemble) {
@@ -245,7 +245,7 @@ struct RelocatorTestHelper {
const auto current_size =
ImageWriter::SizeInSnapshot(Code::InstructionsOf(entry.code));
const auto alias_offset =
OldPage::Of(Code::InstructionsOf(entry.code))->AliasOffset();
Page::Of(Code::InstructionsOf(entry.code))->AliasOffset();
memmove(
reinterpret_cast<void*>(addr),
reinterpret_cast<void*>(Instructions::PayloadStart(
@@ -262,7 +262,7 @@ struct RelocatorTestHelper {
const uword address = UntaggedObject::ToAddr(instructions.ptr());
const auto size = instructions.ptr()->untag()->HeapSize();
instructions =
Instructions::RawCast(OldPage::ToExecutable(instructions.ptr()));
Instructions::RawCast(Page::ToExecutable(instructions.ptr()));
const auto prot = FLAG_dual_map_code ? VirtualMemory::kReadOnly
: VirtualMemory::kReadExecute;
runtime/vm/compiler/runtime_api.cc
+3 -3
View File
@@ -328,9 +328,9 @@ intptr_t RuntimeEntry::argument_count() const {
namespace target {
const word kOldPageSize = dart::kOldPageSize;
const word kOldPageSizeInWords = dart::kOldPageSize / kWordSize;
const word kOldPageMask = dart::kOldPageMask;
const word kPageSize = dart::kPageSize;
const word kPageSizeInWords = dart::kPageSize / kWordSize;
const word kPageMask = dart::kPageMask;
static word TranslateOffsetInWordsToHost(word offset) {
RELEASE_ASSERT((offset % kCompressedWordSize) == 0);
runtime/vm/compiler/runtime_api.h
+7 -7
View File
@@ -77,9 +77,9 @@ extern InvalidClass kUWordMax;
extern InvalidClass kNewObjectAlignmentOffset;
extern InvalidClass kOldObjectAlignmentOffset;
extern InvalidClass kNewObjectBitPosition;
extern InvalidClass kOldPageSize;
extern InvalidClass kOldPageSizeInWords;
extern InvalidClass kOldPageMask;
extern InvalidClass kPageSize;
extern InvalidClass kPageSizeInWords;
extern InvalidClass kPageMask;
extern InvalidClass kObjectAlignment;
extern InvalidClass kObjectAlignmentLog2;
extern InvalidClass kObjectAlignmentMask;
@@ -301,9 +301,9 @@ constexpr word kSmiMax = (static_cast<uword>(1) << kSmiBits) - 1;
constexpr word kSmiMin = -(static_cast<uword>(1) << kSmiBits);
// Information about heap pages.
extern const word kOldPageSize;
extern const word kOldPageSizeInWords;
extern const word kOldPageMask;
extern const word kPageSize;
extern const word kPageSizeInWords;
extern const word kPageMask;
static constexpr intptr_t kObjectAlignment = ObjectAlignment::kObjectAlignment;
@@ -1462,7 +1462,7 @@ class ClosureData : public AllStatic {
FINAL_CLASS();
};
class OldPage : public AllStatic {
class Page : public AllStatic {
public:
static const word kBytesPerCardLog2;
runtime/vm/compiler/runtime_offsets_extracted.h
+60 -80
View File
@@ -73,7 +73,7 @@ static constexpr dart::compiler::target::word
Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
Instructions_kNonBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word String_kMaxElements = 536870911;
@@ -176,7 +176,7 @@ static constexpr dart::compiler::target::word
GrowableObjectArray_length_offset = 8;
static constexpr dart::compiler::target::word
GrowableObjectArray_type_arguments_offset = 4;
static constexpr dart::compiler::target::word OldPage_card_table_offset = 20;
static constexpr dart::compiler::target::word Page_card_table_offset = 16;
static constexpr dart::compiler::target::word
CallSiteData_arguments_descriptor_offset = 8;
static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3;
@@ -733,7 +733,7 @@ static constexpr dart::compiler::target::word
Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word String_kMaxElements =
@@ -837,7 +837,7 @@ static constexpr dart::compiler::target::word
GrowableObjectArray_length_offset = 16;
static constexpr dart::compiler::target::word
GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word OldPage_card_table_offset = 40;
static constexpr dart::compiler::target::word Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3;
@@ -1400,7 +1400,7 @@ static constexpr dart::compiler::target::word
Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
Instructions_kNonBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word String_kMaxElements = 536870911;
@@ -1503,7 +1503,7 @@ static constexpr dart::compiler::target::word
GrowableObjectArray_length_offset = 8;
static constexpr dart::compiler::target::word
GrowableObjectArray_type_arguments_offset = 4;
static constexpr dart::compiler::target::word OldPage_card_table_offset = 20;
static constexpr dart::compiler::target::word Page_card_table_offset = 16;
static constexpr dart::compiler::target::word
CallSiteData_arguments_descriptor_offset = 8;
static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3;
@@ -2057,7 +2057,7 @@ static constexpr dart::compiler::target::word
Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word String_kMaxElements =
@@ -2161,7 +2161,7 @@ static constexpr dart::compiler::target::word
GrowableObjectArray_length_offset = 16;
static constexpr dart::compiler::target::word
GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word OldPage_card_table_offset = 40;
static constexpr dart::compiler::target::word Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3;
@@ -2725,7 +2725,7 @@ static constexpr dart::compiler::target::word
Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word String_kMaxElements = 536870911;
@@ -2828,7 +2828,7 @@ static constexpr dart::compiler::target::word
GrowableObjectArray_length_offset = 12;
static constexpr dart::compiler::target::word
GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word OldPage_card_table_offset = 40;
static constexpr dart::compiler::target::word Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3;
@@ -3391,7 +3391,7 @@ static constexpr dart::compiler::target::word
Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word String_kMaxElements = 536870911;
@@ -3494,7 +3494,7 @@ static constexpr dart::compiler::target::word
GrowableObjectArray_length_offset = 12;
static constexpr dart::compiler::target::word
GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word OldPage_card_table_offset = 40;
static constexpr dart::compiler::target::word Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3;
@@ -4058,7 +4058,7 @@ static constexpr dart::compiler::target::word
Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
Instructions_kNonBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word String_kMaxElements = 536870911;
@@ -4161,7 +4161,7 @@ static constexpr dart::compiler::target::word
GrowableObjectArray_length_offset = 8;
static constexpr dart::compiler::target::word
GrowableObjectArray_type_arguments_offset = 4;
static constexpr dart::compiler::target::word OldPage_card_table_offset = 20;
static constexpr dart::compiler::target::word Page_card_table_offset = 16;
static constexpr dart::compiler::target::word
CallSiteData_arguments_descriptor_offset = 8;
static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3;
@@ -4720,7 +4720,7 @@ static constexpr dart::compiler::target::word
Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word String_kMaxElements =
@@ -4824,7 +4824,7 @@ static constexpr dart::compiler::target::word
GrowableObjectArray_length_offset = 16;
static constexpr dart::compiler::target::word
GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word OldPage_card_table_offset = 40;
static constexpr dart::compiler::target::word Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3;
@@ -5387,7 +5387,7 @@ static constexpr dart::compiler::target::word
Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
Instructions_kNonBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word String_kMaxElements = 536870911;
@@ -5488,7 +5488,7 @@ static constexpr dart::compiler::target::word
GrowableObjectArray_length_offset = 8;
static constexpr dart::compiler::target::word
GrowableObjectArray_type_arguments_offset = 4;
static constexpr dart::compiler::target::word OldPage_card_table_offset = 20;
static constexpr dart::compiler::target::word Page_card_table_offset = 16;
static constexpr dart::compiler::target::word
CallSiteData_arguments_descriptor_offset = 8;
static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3;
@@ -6039,7 +6039,7 @@ static constexpr dart::compiler::target::word
Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word String_kMaxElements =
@@ -6141,7 +6141,7 @@ static constexpr dart::compiler::target::word
GrowableObjectArray_length_offset = 16;
static constexpr dart::compiler::target::word
GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word OldPage_card_table_offset = 40;
static constexpr dart::compiler::target::word Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3;
@@ -6698,7 +6698,7 @@ static constexpr dart::compiler::target::word
Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
Instructions_kNonBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word String_kMaxElements = 536870911;
@@ -6799,7 +6799,7 @@ static constexpr dart::compiler::target::word
GrowableObjectArray_length_offset = 8;
static constexpr dart::compiler::target::word
GrowableObjectArray_type_arguments_offset = 4;
static constexpr dart::compiler::target::word OldPage_card_table_offset = 20;
static constexpr dart::compiler::target::word Page_card_table_offset = 16;
static constexpr dart::compiler::target::word
CallSiteData_arguments_descriptor_offset = 8;
static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3;
@@ -7347,7 +7347,7 @@ static constexpr dart::compiler::target::word
Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word String_kMaxElements =
@@ -7449,7 +7449,7 @@ static constexpr dart::compiler::target::word
GrowableObjectArray_length_offset = 16;
static constexpr dart::compiler::target::word
GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word OldPage_card_table_offset = 40;
static constexpr dart::compiler::target::word Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3;
@@ -8007,7 +8007,7 @@ static constexpr dart::compiler::target::word
Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word String_kMaxElements = 536870911;
@@ -8108,7 +8108,7 @@ static constexpr dart::compiler::target::word
GrowableObjectArray_length_offset = 12;
static constexpr dart::compiler::target::word
GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word OldPage_card_table_offset = 40;
static constexpr dart::compiler::target::word Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3;
@@ -8665,7 +8665,7 @@ static constexpr dart::compiler::target::word
Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word String_kMaxElements = 536870911;
@@ -8766,7 +8766,7 @@ static constexpr dart::compiler::target::word
GrowableObjectArray_length_offset = 12;
static constexpr dart::compiler::target::word
GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word OldPage_card_table_offset = 40;
static constexpr dart::compiler::target::word Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3;
@@ -9324,7 +9324,7 @@ static constexpr dart::compiler::target::word
Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
Instructions_kNonBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word String_kMaxElements = 536870911;
@@ -9425,7 +9425,7 @@ static constexpr dart::compiler::target::word
GrowableObjectArray_length_offset = 8;
static constexpr dart::compiler::target::word
GrowableObjectArray_type_arguments_offset = 4;
static constexpr dart::compiler::target::word OldPage_card_table_offset = 20;
static constexpr dart::compiler::target::word Page_card_table_offset = 16;
static constexpr dart::compiler::target::word
CallSiteData_arguments_descriptor_offset = 8;
static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3;
@@ -9978,7 +9978,7 @@ static constexpr dart::compiler::target::word
Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word OldPage_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word Page_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word String_kMaxElements =
@@ -10080,7 +10080,7 @@ static constexpr dart::compiler::target::word
GrowableObjectArray_length_offset = 16;
static constexpr dart::compiler::target::word
GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word OldPage_card_table_offset = 40;
static constexpr dart::compiler::target::word Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word ICData_NumArgsTestedMask = 3;
@@ -10648,7 +10648,7 @@ static constexpr dart::compiler::target::word
AOT_Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
AOT_Instructions_kNonBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
AOT_NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word AOT_String_kMaxElements =
@@ -10756,8 +10756,7 @@ static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_length_offset = 8;
static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_type_arguments_offset = 4;
static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset =
20;
static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 16;
static constexpr dart::compiler::target::word
AOT_CallSiteData_arguments_descriptor_offset = 8;
static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3;
@@ -11382,8 +11381,7 @@ static constexpr dart::compiler::target::word
AOT_Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
AOT_Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 =
10;
static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
AOT_NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word AOT_String_kMaxElements =
@@ -11492,8 +11490,7 @@ static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_length_offset = 16;
static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset =
40;
static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
AOT_CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3;
@@ -12125,8 +12122,7 @@ static constexpr dart::compiler::target::word
AOT_Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
AOT_Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 =
10;
static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
AOT_NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word AOT_String_kMaxElements =
@@ -12235,8 +12231,7 @@ static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_length_offset = 16;
static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset =
40;
static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
AOT_CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3;
@@ -12866,7 +12861,7 @@ static constexpr dart::compiler::target::word
AOT_Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
AOT_Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
AOT_NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word AOT_String_kMaxElements =
@@ -12975,8 +12970,7 @@ static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_length_offset = 12;
static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset =
40;
static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
AOT_CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3;
@@ -13605,7 +13599,7 @@ static constexpr dart::compiler::target::word
AOT_Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
AOT_Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
AOT_NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word AOT_String_kMaxElements =
@@ -13714,8 +13708,7 @@ static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_length_offset = 12;
static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset =
40;
static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
AOT_CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3;
@@ -14345,7 +14338,7 @@ static constexpr dart::compiler::target::word
AOT_Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
AOT_Instructions_kNonBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
AOT_NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word AOT_String_kMaxElements =
@@ -14453,8 +14446,7 @@ static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_length_offset = 8;
static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_type_arguments_offset = 4;
static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset =
20;
static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 16;
static constexpr dart::compiler::target::word
AOT_CallSiteData_arguments_descriptor_offset = 8;
static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3;
@@ -15081,8 +15073,7 @@ static constexpr dart::compiler::target::word
AOT_Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
AOT_Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 =
10;
static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
AOT_NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word AOT_String_kMaxElements =
@@ -15191,8 +15182,7 @@ static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_length_offset = 16;
static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset =
40;
static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
AOT_CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3;
@@ -15821,7 +15811,7 @@ static constexpr dart::compiler::target::word
AOT_Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
AOT_Instructions_kNonBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
AOT_NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word AOT_String_kMaxElements =
@@ -15927,8 +15917,7 @@ static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_length_offset = 8;
static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_type_arguments_offset = 4;
static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset =
20;
static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 16;
static constexpr dart::compiler::target::word
AOT_CallSiteData_arguments_descriptor_offset = 8;
static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3;
@@ -16546,8 +16535,7 @@ static constexpr dart::compiler::target::word
AOT_Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
AOT_Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 =
10;
static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
AOT_NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word AOT_String_kMaxElements =
@@ -16654,8 +16642,7 @@ static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_length_offset = 16;
static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset =
40;
static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
AOT_CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3;
@@ -17280,8 +17267,7 @@ static constexpr dart::compiler::target::word
AOT_Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
AOT_Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 =
10;
static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
AOT_NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word AOT_String_kMaxElements =
@@ -17388,8 +17374,7 @@ static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_length_offset = 16;
static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset =
40;
static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
AOT_CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3;
@@ -18012,7 +17997,7 @@ static constexpr dart::compiler::target::word
AOT_Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
AOT_Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
AOT_NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word AOT_String_kMaxElements =
@@ -18119,8 +18104,7 @@ static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_length_offset = 12;
static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset =
40;
static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
AOT_CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3;
@@ -18742,7 +18726,7 @@ static constexpr dart::compiler::target::word
AOT_Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
AOT_Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
AOT_NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word AOT_String_kMaxElements =
@@ -18849,8 +18833,7 @@ static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_length_offset = 12;
static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset =
40;
static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
AOT_CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3;
@@ -19473,7 +19456,7 @@ static constexpr dart::compiler::target::word
AOT_Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
AOT_Instructions_kNonBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
AOT_NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word AOT_String_kMaxElements =
@@ -19579,8 +19562,7 @@ static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_length_offset = 8;
static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_type_arguments_offset = 4;
static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset =
20;
static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 16;
static constexpr dart::compiler::target::word
AOT_CallSiteData_arguments_descriptor_offset = 8;
static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3;
@@ -20200,8 +20182,7 @@ static constexpr dart::compiler::target::word
AOT_Instructions_kBarePayloadAlignment = 4;
static constexpr dart::compiler::target::word
AOT_Instructions_kNonBarePayloadAlignment = 8;
static constexpr dart::compiler::target::word AOT_OldPage_kBytesPerCardLog2 =
10;
static constexpr dart::compiler::target::word AOT_Page_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
AOT_NativeEntry_kNumCallWrapperArguments = 2;
static constexpr dart::compiler::target::word AOT_String_kMaxElements =
@@ -20308,8 +20289,7 @@ static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_length_offset = 16;
static constexpr dart::compiler::target::word
AOT_GrowableObjectArray_type_arguments_offset = 8;
static constexpr dart::compiler::target::word AOT_OldPage_card_table_offset =
40;
static constexpr dart::compiler::target::word AOT_Page_card_table_offset = 32;
static constexpr dart::compiler::target::word
AOT_CallSiteData_arguments_descriptor_offset = 16;
static constexpr dart::compiler::target::word AOT_ICData_NumArgsTestedMask = 3;
runtime/vm/compiler/runtime_offsets_list.h
+2 -2
View File
@@ -75,7 +75,7 @@
CONSTANT(Instructions, kPolymorphicEntryOffsetAOT) \
CONSTANT(Instructions, kBarePayloadAlignment) \
CONSTANT(Instructions, kNonBarePayloadAlignment) \
CONSTANT(OldPage, kBytesPerCardLog2) \
CONSTANT(Page, kBytesPerCardLog2) \
CONSTANT(NativeEntry, kNumCallWrapperArguments) \
CONSTANT(String, kMaxElements) \
CONSTANT(SubtypeTestCache, kFunctionTypeArguments) \
@@ -141,7 +141,7 @@
FIELD(GrowableObjectArray, data_offset) \
FIELD(GrowableObjectArray, length_offset) \
FIELD(GrowableObjectArray, type_arguments_offset) \
FIELD(OldPage, card_table_offset) \
FIELD(Page, card_table_offset) \
FIELD(CallSiteData, arguments_descriptor_offset) \
FIELD(ICData, NumArgsTestedMask) \
FIELD(ICData, NumArgsTestedShift) \
runtime/vm/compiler/stub_code_compiler_arm.cc
+5 -5
View File
@@ -1719,20 +1719,20 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler,
// Get card table.
__ Bind(&remember_card);
__ AndImmediate(TMP, R1, target::kOldPageMask); // OldPage.
__ AndImmediate(TMP, R1, target::kPageMask); // Page.
__ ldr(TMP,
Address(TMP, target::OldPage::card_table_offset())); // Card table.
Address(TMP, target::Page::card_table_offset())); // Card table.
__ cmp(TMP, Operand(0));
__ b(&remember_card_slow, EQ);
// Dirty the card.
__ AndImmediate(TMP, R1, target::kOldPageMask); // OldPage.
__ AndImmediate(TMP, R1, target::kPageMask); // Page.
__ sub(R9, R9, Operand(TMP)); // Offset in page.
__ ldr(TMP,
Address(TMP, target::OldPage::card_table_offset())); // Card table.
Address(TMP, target::Page::card_table_offset())); // Card table.
__ add(TMP, TMP,
Operand(R9, LSR,
target::OldPage::kBytesPerCardLog2)); // Card address.
target::Page::kBytesPerCardLog2)); // Card address.
__ strb(R1,
Address(TMP, 0)); // Low byte of R0 is non-zero from object tag.
__ Ret();
runtime/vm/compiler/stub_code_compiler_arm64.cc
+5 -5
View File
@@ -2024,19 +2024,19 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler,
// Get card table.
__ Bind(&remember_card);
__ AndImmediate(TMP, R1, target::kOldPageMask); // OldPage.
__ AndImmediate(TMP, R1, target::kPageMask); // Page.
__ ldr(TMP,
Address(TMP, target::OldPage::card_table_offset())); // Card table.
Address(TMP, target::Page::card_table_offset())); // Card table.
__ cbz(&remember_card_slow, TMP);
// Dirty the card.
__ AndImmediate(TMP, R1, target::kOldPageMask); // OldPage.
__ AndImmediate(TMP, R1, target::kPageMask); // Page.
__ sub(R25, R25, Operand(TMP)); // Offset in page.
__ ldr(TMP,
Address(TMP, target::OldPage::card_table_offset())); // Card table.
Address(TMP, target::Page::card_table_offset())); // Card table.
__ add(TMP, TMP,
Operand(R25, LSR,
target::OldPage::kBytesPerCardLog2)); // Card address.
target::Page::kBytesPerCardLog2)); // Card address.
__ str(R1, Address(TMP, 0),
kUnsignedByte); // Low byte of R1 is non-zero from object tag.
__ ret();
runtime/vm/compiler/stub_code_compiler_ia32.cc
+4 -4
View File
@@ -1454,17 +1454,17 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler,
// Get card table.
__ Bind(&remember_card);
__ movl(EAX, EDX); // Object.
__ andl(EAX, Immediate(target::kOldPageMask)); // OldPage.
__ cmpl(Address(EAX, target::OldPage::card_table_offset()), Immediate(0));
__ andl(EAX, Immediate(target::kPageMask)); // Page.
__ cmpl(Address(EAX, target::Page::card_table_offset()), Immediate(0));
__ j(EQUAL, &remember_card_slow, Assembler::kNearJump);
// Dirty the card.
__ subl(EDI, EAX); // Offset in page.
__ movl(EAX,
Address(EAX, target::OldPage::card_table_offset())); // Card table.
Address(EAX, target::Page::card_table_offset())); // Card table.
__ shrl(
EDI,
Immediate(target::OldPage::kBytesPerCardLog2)); // Index in card table.
Immediate(target::Page::kBytesPerCardLog2)); // Index in card table.
__ movb(Address(EAX, EDI, TIMES_1, 0), Immediate(1));
__ popl(ECX);
__ popl(EAX);
runtime/vm/compiler/stub_code_compiler_riscv.cc
+5 -7
View File
@@ -1842,17 +1842,15 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler,
// Get card table.
__ Bind(&remember_card);
__ AndImmediate(TMP, A0, target::kOldPageMask); // OldPage.
__ lx(TMP,
Address(TMP, target::OldPage::card_table_offset())); // Card table.
__ AndImmediate(TMP, A0, target::kPageMask); // Page.
__ lx(TMP, Address(TMP, target::Page::card_table_offset())); // Card table.
__ beqz(TMP, &remember_card_slow);
// Dirty the card.
__ AndImmediate(TMP, A0, target::kOldPageMask); // OldPage.
__ AndImmediate(TMP, A0, target::kPageMask); // Page.
__ sub(A6, A6, TMP); // Offset in page.
__ lx(TMP,
Address(TMP, target::OldPage::card_table_offset())); // Card table.
__ srli(A6, A6, target::OldPage::kBytesPerCardLog2);
__ lx(TMP, Address(TMP, target::Page::card_table_offset())); // Card table.
__ srli(A6, A6, target::Page::kBytesPerCardLog2);
__ add(TMP, TMP, A6); // Card address.
__ sb(A0, Address(TMP, 0)); // Low byte of A0 is non-zero from object tag.
__ ret();
runtime/vm/compiler/stub_code_compiler_x64.cc
+4 -4
View File
@@ -1928,17 +1928,17 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler,
// Get card table.
__ Bind(&remember_card);
__ movq(TMP, RDX); // Object.
__ andq(TMP, Immediate(target::kOldPageMask)); // OldPage.
__ cmpq(Address(TMP, target::OldPage::card_table_offset()), Immediate(0));
__ andq(TMP, Immediate(target::kPageMask)); // Page.
__ cmpq(Address(TMP, target::Page::card_table_offset()), Immediate(0));
__ j(EQUAL, &remember_card_slow, Assembler::kNearJump);
// Dirty the card.
__ subq(R13, TMP); // Offset in page.
__ movq(TMP,
Address(TMP, target::OldPage::card_table_offset())); // Card table.
Address(TMP, target::Page::card_table_offset())); // Card table.
__ shrq(
R13,
Immediate(target::OldPage::kBytesPerCardLog2)); // Index in card table.
Immediate(target::Page::kBytesPerCardLog2)); // Index in card table.
__ movb(Address(TMP, R13, TIMES_1, 0), Immediate(1));
__ ret();
runtime/vm/dart.cc
+2 -2
View File
@@ -326,7 +326,7 @@ char* Dart::DartInit(const Dart_InitializeParams* params) {
Api::Init();
NativeSymbolResolver::Init();
NOT_IN_PRODUCT(Profiler::Init());
SemiSpace::Init();
Page::Init();
NOT_IN_PRODUCT(Metric::Init());
StoreBuffer::Init();
MarkingStack::Init();
@@ -773,7 +773,7 @@ char* Dart::Cleanup() {
MarkingStack::Cleanup();
StoreBuffer::Cleanup();
Object::Cleanup();
SemiSpace::Cleanup();
Page::Cleanup();
StubCode::Cleanup();
#if defined(SUPPORT_TIMELINE)
if (FLAG_trace_shutdown) {
runtime/vm/dart_api_impl.cc
+1 -1
View File
@@ -1820,7 +1820,7 @@ DART_EXPORT void Dart_NotifyIdle(int64_t deadline) {
DART_EXPORT void Dart_NotifyLowMemory() {
API_TIMELINE_BEGIN_END(Thread::Current());
SemiSpace::ClearCache();
Page::ClearCache();
Zone::ClearCache();
// For each isolate's global variables, we might also clear:
runtime/vm/globals.h
+1 -1
View File
@@ -56,7 +56,7 @@ const intptr_t kBytesPerBigIntDigit = 4;
// The default old gen heap size in MB, where 0 -- unlimited.
// 32-bit: OS limit is 2 or 3 GB
// 64-bit: Linux's limit is
// sysctl vm.max_map_count (default 2^16) * 512 KB OldPages = 32 GB
// sysctl vm.max_map_count (default 2^16) * 512 KB Pages = 32 GB
// Set the VM limit below the OS limit to increase the likelihood of failing
// gracefully with a Dart OutOfMemory exception instead of SIGABORT.
const intptr_t kDefaultMaxOldGenHeapSize = (kWordSize <= 4) ? 1536 : 30720;
runtime/vm/heap/compactor.cc
+32 -34
View File
@@ -19,11 +19,11 @@ DEFINE_FLAG(bool,
false,
"Force compaction to move every movable object");
// Each OldPage is divided into blocks of size kBlockSize. Each object belongs
// Each Page is divided into blocks of size kBlockSize. Each object belongs
// to the block containing its header word (so up to kBlockSize +
// kAllocatablePageSize - 2 * kObjectAlignment bytes belong to the same block).
// During compaction, all live objects in the same block will slide such that
// they all end up on the same OldPage, and all gaps within the block will be
// they all end up on the same Page, and all gaps within the block will be
// closed. During sliding, a bitvector is computed that indictates which
// allocation units are live, so the new address of any object in the block can
// be found by adding the number of live allocation units before the object to
@@ -95,7 +95,7 @@ class ForwardingPage {
uword Lookup(uword old_addr) { return BlockFor(old_addr)->Lookup(old_addr); }
ForwardingBlock* BlockFor(uword old_addr) {
intptr_t page_offset = old_addr & ~kOldPageMask;
intptr_t page_offset = old_addr & ~kPageMask;
intptr_t block_number = page_offset / kBlockSize;
ASSERT(block_number >= 0);
ASSERT(block_number <= kBlocksPerPage);
@@ -109,17 +109,17 @@ class ForwardingPage {
DISALLOW_IMPLICIT_CONSTRUCTORS(ForwardingPage);
};
void OldPage::AllocateForwardingPage() {
void Page::AllocateForwardingPage() {
ASSERT(forwarding_page_ == NULL);
ASSERT((object_start() + sizeof(ForwardingPage)) < object_end());
ASSERT(Utils::IsAligned(sizeof(ForwardingPage), kObjectAlignment));
object_end_ -= sizeof(ForwardingPage);
forwarding_page_ = reinterpret_cast<ForwardingPage*>(object_end_);
top_ -= sizeof(ForwardingPage);
forwarding_page_ = reinterpret_cast<ForwardingPage*>(top_);
}
struct Partition {
OldPage* head;
OldPage* tail;
Page* head;
Page* tail;
};
class CompactorTask : public ThreadPool::Task {
@@ -150,8 +150,8 @@ class CompactorTask : public ThreadPool::Task {
void RunEnteredIsolateGroup();
private:
void PlanPage(OldPage* page);
void SlidePage(OldPage* page);
void PlanPage(Page* page);
void SlidePage(Page* page);
uword PlanBlock(uword first_object, ForwardingPage* forwarding_page);
uword SlideBlock(uword first_object, ForwardingPage* forwarding_page);
void PlanMoveToContiguousSize(intptr_t size);
@@ -165,7 +165,7 @@ class CompactorTask : public ThreadPool::Task {
intptr_t num_tasks_;
Partition* partitions_;
FreeList* freelist_;
OldPage* free_page_;
Page* free_page_;
uword free_current_;
uword free_end_;
@@ -178,15 +178,13 @@ class CompactorTask : public ThreadPool::Task {
// time, keeping blocks from spanning page boundaries (see ForwardingBlock).
// Free space at the end of a page that is too small for the next block is
// added to the freelist.
void GCCompactor::Compact(OldPage* pages,
FreeList* freelist,
Mutex* pages_lock) {
void GCCompactor::Compact(Page* pages, FreeList* freelist, Mutex* pages_lock) {
SetupImagePageBoundaries();
// Divide the heap.
// TODO(30978): Try to divide based on live bytes or with work stealing.
intptr_t num_pages = 0;
for (OldPage* page = pages; page != NULL; page = page->next()) {
for (Page* page = pages; page != NULL; page = page->next()) {
num_pages++;
}
@@ -202,8 +200,8 @@ void GCCompactor::Compact(OldPage* pages,
const intptr_t pages_per_task = num_pages / num_tasks;
intptr_t task_index = 0;
intptr_t page_index = 0;
OldPage* page = pages;
OldPage* prev = NULL;
Page* page = pages;
Page* prev = NULL;
while (task_index < num_tasks) {
if (page_index % pages_per_task == 0) {
partitions[task_index].head = page;
@@ -231,8 +229,8 @@ void GCCompactor::Compact(OldPage* pages,
task_index++) {
const intptr_t pages_per_task = num_pages / num_tasks;
for (intptr_t j = 0; j < pages_per_task; j++) {
OldPage* page = heap_->old_space()->AllocatePage(OldPage::kData,
/* link */ false);
Page* page = heap_->old_space()->AllocatePage(Page::kData,
/* link */ false);
if (page == nullptr) {
oom = true;
@@ -333,12 +331,12 @@ void GCCompactor::Compact(OldPage* pages,
// Free empty pages.
for (intptr_t task_index = 0; task_index < num_tasks; task_index++) {
OldPage* page = partitions[task_index].tail->next();
Page* page = partitions[task_index].tail->next();
while (page != NULL) {
OldPage* next = page->next();
Page* next = page->next();
heap_->old_space()->IncreaseCapacityInWordsLocked(
-(page->memory_->size() >> kWordSizeLog2));
page->Deallocate();
page->Deallocate(/*can_use_cache*/ true);
page = next;
}
}
@@ -385,12 +383,12 @@ void CompactorTask::RunEnteredIsolateGroup() {
if (planning_task >= num_tasks_) break;
TIMELINE_FUNCTION_GC_DURATION(thread, "Plan");
OldPage* head = partitions_[planning_task].head;
Page* head = partitions_[planning_task].head;
free_page_ = head;
free_current_ = head->object_start();
free_end_ = head->object_end();
for (OldPage* page = head; page != NULL; page = page->next()) {
for (Page* page = head; page != NULL; page = page->next()) {
PlanPage(page);
}
}
@@ -402,12 +400,12 @@ void CompactorTask::RunEnteredIsolateGroup() {
if (sliding_task >= num_tasks_) break;
TIMELINE_FUNCTION_GC_DURATION(thread, "Slide");
OldPage* head = partitions_[sliding_task].head;
Page* head = partitions_[sliding_task].head;
free_page_ = head;
free_current_ = head->object_start();
free_end_ = head->object_end();
for (OldPage* page = head; page != NULL; page = page->next()) {
for (Page* page = head; page != NULL; page = page->next()) {
SlidePage(page);
}
@@ -431,7 +429,7 @@ void CompactorTask::RunEnteredIsolateGroup() {
switch (forwarding_task) {
case 0: {
TIMELINE_FUNCTION_GC_DURATION(thread, "ForwardLargePages");
for (OldPage* large_page =
for (Page* large_page =
isolate_group_->heap()->old_space()->large_pages_;
large_page != NULL; large_page = large_page->next()) {
large_page->VisitObjectPointers(compactor_);
@@ -479,7 +477,7 @@ void CompactorTask::RunEnteredIsolateGroup() {
}
}
void CompactorTask::PlanPage(OldPage* page) {
void CompactorTask::PlanPage(Page* page) {
uword current = page->object_start();
uword end = page->object_end();
@@ -491,7 +489,7 @@ void CompactorTask::PlanPage(OldPage* page) {
}
}
void CompactorTask::SlidePage(OldPage* page) {
void CompactorTask::SlidePage(Page* page) {
uword current = page->object_start();
uword end = page->object_end();
@@ -552,7 +550,7 @@ uword CompactorTask::SlideBlock(uword first_object,
// to a new page. But if we exactly hit the end of the previous page
// then free_current could be at the start of the next page, so we
// subtract 1.
ASSERT(OldPage::Of(free_current_ - 1) != OldPage::Of(new_addr));
ASSERT(Page::Of(free_current_ - 1) != Page::Of(new_addr));
intptr_t free_remaining = free_end_ - free_current_;
// Add any leftover at the end of a page to the free list.
if (free_remaining > 0) {
@@ -593,7 +591,7 @@ uword CompactorTask::SlideBlock(uword first_object,
void CompactorTask::PlanMoveToContiguousSize(intptr_t size) {
// Move the free cursor to ensure 'size' bytes of contiguous space.
ASSERT(size <= kOldPageSize);
ASSERT(size <= kPageSize);
// Check if the current free page has enough space.
intptr_t free_remaining = free_end_ - free_current_;
@@ -611,7 +609,7 @@ void CompactorTask::PlanMoveToContiguousSize(intptr_t size) {
void GCCompactor::SetupImagePageBoundaries() {
MallocGrowableArray<ImagePageRange> ranges(4);
OldPage* image_page =
Page* image_page =
Dart::vm_isolate_group()->heap()->old_space()->image_pages_;
while (image_page != NULL) {
ImagePageRange range = {image_page->object_start(),
@@ -656,7 +654,7 @@ void GCCompactor::ForwardPointer(ObjectPtr* ptr) {
}
}
OldPage* page = OldPage::Of(old_target);
Page* page = Page::Of(old_target);
ForwardingPage* forwarding_page = page->forwarding_page();
if (forwarding_page == NULL) {
return; // Not moved (VM isolate, large page, code page).
@@ -692,7 +690,7 @@ void GCCompactor::ForwardCompressedPointer(uword heap_base,
}
}
OldPage* page = OldPage::Of(old_target);
Page* page = Page::Of(old_target);
ForwardingPage* forwarding_page = page->forwarding_page();
if (forwarding_page == NULL) {
return; // Not moved (VM isolate, large page, code page).
runtime/vm/heap/compactor.h
+2 -2
View File
@@ -17,7 +17,7 @@ namespace dart {
// Forward declarations.
class FreeList;
class Heap;
class OldPage;
class Page;
// Implements a sliding compactor.
class GCCompactor : public ValueObject,
@@ -30,7 +30,7 @@ class GCCompactor : public ValueObject,
heap_(heap) {}
~GCCompactor() { free(image_page_ranges_); }
void Compact(OldPage* pages, FreeList* freelist, Mutex* mutex);
void Compact(Page* pages, FreeList* freelist, Mutex* mutex);
private:
friend class CompactorTask;
runtime/vm/heap/freelist.h
+1 -1
View File
@@ -133,7 +133,7 @@ class FreeList {
return result;
}
// Ensures OldPage::VisitObjects can successful walk over a partially
// Ensures Page::VisitObjects can successful walk over a partially
// allocated bump region.
void MakeIterable() {
if (top_ < end_) {
runtime/vm/heap/heap.cc
+6 -19
View File
@@ -88,10 +88,10 @@ uword Heap::AllocateNew(Thread* thread, intptr_t size) {
// It is possible a GC doesn't clear enough space.
// In that case, we must fall through and allocate into old space.
return AllocateOld(thread, size, OldPage::kData);
return AllocateOld(thread, size, Page::kData);
}
uword Heap::AllocateOld(Thread* thread, intptr_t size, OldPage::PageType type) {
uword Heap::AllocateOld(Thread* thread, intptr_t size, Page::PageType type) {
ASSERT(thread->no_safepoint_scope_depth() == 0);
if (!thread->force_growth()) {
CollectForDebugging(thread);
@@ -221,7 +221,7 @@ bool Heap::OldContains(uword addr) const {
}
bool Heap::CodeContains(uword addr) const {
return old_space_.Contains(addr, OldPage::kExecutable);
return old_space_.Contains(addr, Page::kExecutable);
}
bool Heap::DataContains(uword addr) const {
@@ -346,14 +346,14 @@ void Heap::VisitObjectPointers(ObjectPointerVisitor* visitor) {
InstructionsPtr Heap::FindObjectInCodeSpace(FindObjectVisitor* visitor) const {
// Only executable pages can have RawInstructions objects.
ObjectPtr raw_obj = old_space_.FindObject(visitor, OldPage::kExecutable);
ObjectPtr raw_obj = old_space_.FindObject(visitor, Page::kExecutable);
ASSERT((raw_obj == Object::null()) ||
(raw_obj->GetClassId() == kInstructionsCid));
return static_cast<InstructionsPtr>(raw_obj);
}
ObjectPtr Heap::FindOldObject(FindObjectVisitor* visitor) const {
return old_space_.FindObject(visitor, OldPage::kData);
return old_space_.FindObject(visitor, Page::kData);
}
ObjectPtr Heap::FindNewObject(FindObjectVisitor* visitor) {
@@ -432,7 +432,7 @@ void Heap::NotifyIdle(int64_t deadline) {
}
if (OS::GetCurrentMonotonicMicros() < deadline) {
SemiSpace::ClearCache();
Page::ClearCache();
}
}
@@ -709,19 +709,6 @@ void Heap::Init(IsolateGroup* isolate_group,
isolate_group->set_heap(std::move(heap));
}
const char* Heap::RegionName(Space space) {
switch (space) {
case kNew:
return "dart-newspace";
case kOld:
return "dart-oldspace";
case kCode:
return "dart-codespace";
default:
UNREACHABLE();
}
}
void Heap::AddRegionsToObjectSet(ObjectSet* set) const {
new_space_.AddRegionsToObjectSet(set);
old_space_.AddRegionsToObjectSet(set);
runtime/vm/heap/heap.h
+4 -7
View File
@@ -65,13 +65,13 @@ class Heap {
case kNew:
// Do not attempt to allocate very large objects in new space.
if (!IsAllocatableInNewSpace(size)) {
return AllocateOld(thread, size, OldPage::kData);
return AllocateOld(thread, size, Page::kData);
}
return AllocateNew(thread, size);
case kOld:
return AllocateOld(thread, size, OldPage::kData);
return AllocateOld(thread, size, Page::kData);
case kCode:
return AllocateOld(thread, size, OldPage::kExecutable);
return AllocateOld(thread, size, Page::kExecutable);
default:
UNREACHABLE();
}
@@ -150,9 +150,6 @@ class Heap {
intptr_t max_new_gen_words,
intptr_t max_old_gen_words);
// Returns a suitable name for a VM region in the heap.
static const char* RegionName(Space space);
// Verify that all pointers in the heap point to the heap.
bool Verify(MarkExpectation mark_expectation = kForbidMarked);
@@ -324,7 +321,7 @@ class Heap {
intptr_t max_old_gen_words);
uword AllocateNew(Thread* thread, intptr_t size);
uword AllocateOld(Thread* thread, intptr_t size, OldPage::PageType type);
uword AllocateOld(Thread* thread, intptr_t size, Page::PageType type);
// Visit all pointers. Caller must ensure concurrent sweeper is not running,
// and the visitor must not allocate.
runtime/vm/heap/heap_sources.gni
+2
View File
@@ -17,6 +17,8 @@ heap_sources = [
"heap.h",
"marker.cc",
"marker.h",
"page.cc",
"page.h",
"pages.cc",
"pages.h",
"pointer_block.cc",
runtime/vm/heap/marker.cc
+3 -3
View File
@@ -376,7 +376,7 @@ class MarkingVisitorBase : public ObjectPointerVisitor {
static bool TryAcquireMarkBit(ObjectPtr raw_obj) {
if (FLAG_write_protect_code && raw_obj->IsInstructions()) {
// A non-writable alias mapping may exist for instruction pages.
raw_obj = OldPage::ToWritable(raw_obj);
raw_obj = Page::ToWritable(raw_obj);
}
if (!sync) {
raw_obj->untag()->SetMarkBitUnsynchronized();
@@ -495,7 +495,7 @@ void GCMarker::ResetSlices() {
root_slices_finished_ = 0;
root_slices_count_ = kNumFixedRootSlices;
new_page_ = heap_->new_space()->head();
for (NewPage* p = new_page_; p != nullptr; p = p->next()) {
for (Page* p = new_page_; p != nullptr; p = p->next()) {
root_slices_count_++;
}
@@ -518,7 +518,7 @@ void GCMarker::IterateRoots(ObjectPointerVisitor* visitor) {
break;
}
default: {
NewPage* page;
Page* page;
{
MonitorLocker ml(&root_slices_monitor_);
page = new_page_;
runtime/vm/heap/marker.h
+2 -2
View File
@@ -20,7 +20,7 @@ class ObjectPointerVisitor;
class PageSpace;
template <bool sync>
class MarkingVisitorBase;
class NewPage;
class Page;
class Thread;
// The class GCMarker is used to mark reachable old generation objects as part
@@ -73,7 +73,7 @@ class GCMarker {
GCLinkedLists global_list_;
MarkingVisitorBase<true>** visitors_;
NewPage* new_page_;
Page* new_page_;
Monitor root_slices_monitor_;
RelaxedAtomic<intptr_t> root_slices_started_;
intptr_t root_slices_finished_;
runtime/vm/heap/page.cc
+286
View File
@@ -0,0 +1,286 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "vm/heap/page.h"
#include "platform/assert.h"
#include "platform/leak_sanitizer.h"
#include "vm/dart.h"
#include "vm/heap/become.h"
#include "vm/heap/compactor.h"
#include "vm/heap/marker.h"
#include "vm/heap/safepoint.h"
#include "vm/heap/sweeper.h"
#include "vm/lockers.h"
#include "vm/log.h"
#include "vm/object.h"
#include "vm/object_set.h"
#include "vm/os_thread.h"
#include "vm/virtual_memory.h"
namespace dart {
// This cache needs to be at least as big as FLAG_new_gen_semi_max_size or
// munmap will noticably impact performance.
static constexpr intptr_t kPageCacheCapacity = 8 * kWordSize;
static Mutex* page_cache_mutex = nullptr;
static VirtualMemory* page_cache[kPageCacheCapacity] = {nullptr};
static intptr_t page_cache_size = 0;
void Page::Init() {
ASSERT(page_cache_mutex == nullptr);
page_cache_mutex = new Mutex(NOT_IN_PRODUCT("page_cache_mutex"));
}
void Page::ClearCache() {
MutexLocker ml(page_cache_mutex);
ASSERT(page_cache_size >= 0);
ASSERT(page_cache_size <= kPageCacheCapacity);
while (page_cache_size > 0) {
delete page_cache[--page_cache_size];
}
}
void Page::Cleanup() {
ClearCache();
delete page_cache_mutex;
page_cache_mutex = nullptr;
}
intptr_t Page::CachedSize() {
MutexLocker ml(page_cache_mutex);
return page_cache_size * kPageSize;
}
Page* Page::Allocate(intptr_t size, PageType type, bool can_use_cache) {
const bool executable = type == kExecutable;
const bool compressed = !executable;
const char* name = executable ? "dart-code" : "dart-heap";
VirtualMemory* memory = nullptr;
if (can_use_cache) {
// We don't automatically use the cache based on size and type because a
// large page that happens to be the same size as a regular page can't
// use the cache. Large pages are expected to be zeroed on allocation but
// cached pages are dirty.
ASSERT(size == kPageSize);
ASSERT(!executable);
MutexLocker ml(page_cache_mutex);
ASSERT(page_cache_size >= 0);
ASSERT(page_cache_size <= kPageCacheCapacity);
if (page_cache_size > 0) {
memory = page_cache[--page_cache_size];
}
}
if (memory == nullptr) {
memory = VirtualMemory::AllocateAligned(size, kPageSize, executable,
compressed, name);
}
if (memory == nullptr) {
return nullptr; // Out of memory.
}
if (type == kNew) {
#if defined(DEBUG)
memset(memory->address(), Heap::kZapByte, size);
#endif
// Initialized by generated code.
MSAN_UNPOISON(memory->address(), size);
} else {
// We don't zap old-gen because we rely on implicit zero-initialization
// of large typed data arrays.
}
Page* result = reinterpret_cast<Page*>(memory->address());
ASSERT(result != NULL);
result->type_ = type;
result->memory_ = memory;
result->next_ = nullptr;
result->forwarding_page_ = nullptr;
result->card_table_ = nullptr;
result->progress_bar_ = 0;
result->owner_ = nullptr;
result->top_ = 0;
result->end_ = 0;
result->survivor_end_ = 0;
result->resolved_top_ = 0;
if (type == kNew) {
uword top = result->object_start();
uword end =
memory->end() - kNewObjectAlignmentOffset - kAllocationRedZoneSize;
result->top_ = top;
result->end_ = end;
result->survivor_end_ = top;
result->resolved_top_ = top;
}
LSAN_REGISTER_ROOT_REGION(result, sizeof(*result));
return result;
}
void Page::Deallocate(bool can_use_cache) {
if (is_image_page()) {
delete memory_;
// For a heap page from a snapshot, the Page object lives in the malloc
// heap rather than the page itself.
free(this);
return;
}
free(card_table_);
LSAN_UNREGISTER_ROOT_REGION(this, sizeof(*this));
VirtualMemory* memory = memory_;
if (can_use_cache) {
ASSERT(memory->size() == kPageSize);
ASSERT(type_ != kExecutable);
MutexLocker ml(page_cache_mutex);
ASSERT(page_cache_size >= 0);
ASSERT(page_cache_size <= kPageCacheCapacity);
if (page_cache_size < kPageCacheCapacity) {
intptr_t size = memory->size();
#if defined(DEBUG)
if (type_ == kNew) {
memset(memory->address(), Heap::kZapByte, size);
} else {
// We don't zap old-gen because we rely on implicit zero-initialization
// of large typed data arrays.
}
#endif
MSAN_POISON(memory->address(), size);
page_cache[page_cache_size++] = memory;
memory = nullptr;
}
}
delete memory;
}
void Page::VisitObjects(ObjectVisitor* visitor) const {
ASSERT(Thread::Current()->IsAtSafepoint());
NoSafepointScope no_safepoint;
uword obj_addr = object_start();
uword end_addr = object_end();
while (obj_addr < end_addr) {
ObjectPtr raw_obj = UntaggedObject::FromAddr(obj_addr);
visitor->VisitObject(raw_obj);
obj_addr += raw_obj->untag()->HeapSize();
}
ASSERT(obj_addr == end_addr);
}
void Page::VisitObjectPointers(ObjectPointerVisitor* visitor) const {
ASSERT(Thread::Current()->IsAtSafepoint() ||
(Thread::Current()->task_kind() == Thread::kCompactorTask) ||
(Thread::Current()->task_kind() == Thread::kMarkerTask));
NoSafepointScope no_safepoint;
uword obj_addr = object_start();
uword end_addr = object_end();
while (obj_addr < end_addr) {
ObjectPtr raw_obj = UntaggedObject::FromAddr(obj_addr);
obj_addr += raw_obj->untag()->VisitPointers(visitor);
}
ASSERT(obj_addr == end_addr);
}
void Page::VisitRememberedCards(ObjectPointerVisitor* visitor) {
ASSERT(Thread::Current()->IsAtSafepoint() ||
(Thread::Current()->task_kind() == Thread::kScavengerTask));
NoSafepointScope no_safepoint;
if (card_table_ == NULL) {
return;
}
ArrayPtr obj =
static_cast<ArrayPtr>(UntaggedObject::FromAddr(object_start()));
ASSERT(obj->IsArray());
ASSERT(obj->untag()->IsCardRemembered());
CompressedObjectPtr* obj_from = obj->untag()->from();
CompressedObjectPtr* obj_to =
obj->untag()->to(Smi::Value(obj->untag()->length()));
uword heap_base = obj.heap_base();
const intptr_t size = card_table_size();
for (;;) {
intptr_t i = progress_bar_.fetch_add(1);
if (i >= size) break;
if (card_table_[i] != 0) {
CompressedObjectPtr* card_from =
reinterpret_cast<CompressedObjectPtr*>(this) +
(i << kSlotsPerCardLog2);
CompressedObjectPtr* card_to =
reinterpret_cast<CompressedObjectPtr*>(card_from) +
(1 << kSlotsPerCardLog2) - 1;
// Minus 1 because to is inclusive.
if (card_from < obj_from) {
// First card overlaps with header.
card_from = obj_from;
}
if (card_to > obj_to) {
// Last card(s) may extend past the object. Array truncation can make
// this happen for more than one card.
card_to = obj_to;
}
visitor->VisitCompressedPointers(heap_base, card_from, card_to);
bool has_new_target = false;
for (CompressedObjectPtr* slot = card_from; slot <= card_to; slot++) {
if ((*slot)->IsNewObjectMayBeSmi()) {
has_new_target = true;
break;
}
}
if (!has_new_target) {
card_table_[i] = 0;
}
}
}
}
void Page::ResetProgressBar() {
progress_bar_ = 0;
}
ObjectPtr Page::FindObject(FindObjectVisitor* visitor) const {
uword obj_addr = object_start();
uword end_addr = object_end();
if (visitor->VisitRange(obj_addr, end_addr)) {
while (obj_addr < end_addr) {
ObjectPtr raw_obj = UntaggedObject::FromAddr(obj_addr);
uword next_obj_addr = obj_addr + raw_obj->untag()->HeapSize();
if (visitor->VisitRange(obj_addr, next_obj_addr) &&
raw_obj->untag()->FindObject(visitor)) {
return raw_obj; // Found object, return it.
}
obj_addr = next_obj_addr;
}
ASSERT(obj_addr == end_addr);
}
return Object::null();
}
void Page::WriteProtect(bool read_only) {
ASSERT(!is_image_page());
VirtualMemory::Protection prot;
if (read_only) {
if ((type_ == kExecutable) && (memory_->AliasOffset() == 0)) {
prot = VirtualMemory::kReadExecute;
} else {
prot = VirtualMemory::kReadOnly;
}
} else {
prot = VirtualMemory::kReadWrite;
}
memory_->Protect(prot);
}
} // namespace dart
runtime/vm/heap/page.h
+340
View File
@@ -0,0 +1,340 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#ifndef RUNTIME_VM_HEAP_PAGE_H_
#define RUNTIME_VM_HEAP_PAGE_H_
#include "platform/atomic.h"
#include "vm/globals.h"
#include "vm/heap/spaces.h"
#include "vm/pointer_tagging.h"
#include "vm/raw_object.h"
#include "vm/virtual_memory.h"
namespace dart {
class ForwardingPage;
class ObjectVisitor;
class ObjectPointerVisitor;
class FindObjectVisitor;
class Thread;
// Pages are allocated with kPageSize alignment so that the Page of any object
// can be computed by masking the object with kPageMask. This does not apply to
// image pages, whose address is choosen by the system loader rather than the
// Dart VM.
static constexpr intptr_t kPageSize = 512 * KB;
static constexpr intptr_t kPageSizeInWords = kPageSize / kWordSize;
static constexpr intptr_t kPageMask = ~(kPageSize - 1);
// See ForwardingBlock and CountingBlock.
static constexpr intptr_t kBitVectorWordsPerBlock = 1;
static constexpr intptr_t kBlockSize =
kObjectAlignment * kBitsPerWord * kBitVectorWordsPerBlock;
static constexpr intptr_t kBlockMask = ~(kBlockSize - 1);
static constexpr intptr_t kBlocksPerPage = kPageSize / kBlockSize;
// Simplify initialization in allocation stubs by ensuring it is safe
// to overshoot the object end by up to kAllocationRedZoneSize. (Just as the
// stack red zone allows one to overshoot the stack pointer.)
static constexpr intptr_t kAllocationRedZoneSize = kObjectAlignment;
// A Page is the granuitary at which the Dart heap allocates memory from the OS.
// Pages are usually of size kPageSize, except large objects are allocated on
// their own Page sized to the object.
//
// +----------------------+ <- start
// | struct Page (header) |
// +----------------------+
// | alignment gap |
// +----------------------+ <- object_start
// | objects |
// | ... |
// | ... |
// +----------------------+ <- object_end / top_
// | available |
// +----------------------+ <- end_
// | red zone or |
// | forwarding table |
// +----------------------+ <- memory_->end()
class Page {
public:
static void Init();
static void ClearCache();
static intptr_t CachedSize();
static void Cleanup();
enum PageType : uword { kExecutable = 0, kData, kNew };
Page* next() const { return next_; }
void set_next(Page* next) { next_ = next; }
uword start() const { return memory_->start(); }
uword end() const { return memory_->end(); }
bool Contains(uword addr) const { return memory_->Contains(addr); }
intptr_t AliasOffset() const { return memory_->AliasOffset(); }
uword object_start() const {
return type_ == kNew ? new_object_start() : old_object_start();
}
uword old_object_start() const {
return memory_->start() + OldObjectStartOffset();
}
uword new_object_start() const {
return memory_->start() + NewObjectStartOffset();
}
uword object_end() const {
if (owner_ != NULL) return owner_->top();
return top_;
}
intptr_t used() const { return object_end() - object_start(); }
ForwardingPage* forwarding_page() const { return forwarding_page_; }
void AllocateForwardingPage();
PageType type() const { return type_; }
bool is_image_page() const { return !memory_->vm_owns_region(); }
void VisitObjects(ObjectVisitor* visitor) const;
void VisitObjectPointers(ObjectPointerVisitor* visitor) const;
ObjectPtr FindObject(FindObjectVisitor* visitor) const;
void WriteProtect(bool read_only);
static intptr_t OldObjectStartOffset() {
return Utils::RoundUp(sizeof(Page) - kOldObjectAlignmentOffset,
kObjectStartAlignment) +
kOldObjectAlignmentOffset;
}
static intptr_t NewObjectStartOffset() {
// Note weaker alignment because the bool/null offset tricks don't apply to
// new-space.
return Utils::RoundUp(sizeof(Page) - kNewObjectAlignmentOffset,
kObjectAlignment) +
kNewObjectAlignmentOffset;
}
// Warning: This does not work for objects on image pages because image pages
// are not aligned. However, it works for objects on large pages, because
// only one object is allocated per large page.
static Page* Of(ObjectPtr obj) {
ASSERT(obj->IsHeapObject());
return reinterpret_cast<Page*>(static_cast<uword>(obj) & kPageMask);
}
// Warning: This does not work for addresses on image pages or on large pages.
static Page* Of(uword addr) {
return reinterpret_cast<Page*>(addr & kPageMask);
}
// Warning: This does not work for objects on image pages.
static ObjectPtr ToExecutable(ObjectPtr obj) {
Page* page = Of(obj);
VirtualMemory* memory = page->memory_;
const intptr_t alias_offset = memory->AliasOffset();
if (alias_offset == 0) {
return obj; // Not aliased.
}
uword addr = UntaggedObject::ToAddr(obj);
if (memory->Contains(addr)) {
return UntaggedObject::FromAddr(addr + alias_offset);
}
// obj is executable.
ASSERT(memory->ContainsAlias(addr));
return obj;
}
// Warning: This does not work for objects on image pages.
static ObjectPtr ToWritable(ObjectPtr obj) {
Page* page = Of(obj);
VirtualMemory* memory = page->memory_;
const intptr_t alias_offset = memory->AliasOffset();
if (alias_offset == 0) {
return obj; // Not aliased.
}
uword addr = UntaggedObject::ToAddr(obj);
if (memory->ContainsAlias(addr)) {
return UntaggedObject::FromAddr(addr - alias_offset);
}
// obj is writable.
ASSERT(memory->Contains(addr));
return obj;
}
// 1 card = 128 slots.
static const intptr_t kSlotsPerCardLog2 = 7;
static const intptr_t kBytesPerCardLog2 =
kCompressedWordSizeLog2 + kSlotsPerCardLog2;
intptr_t card_table_size() const {
return memory_->size() >> kBytesPerCardLog2;
}
static intptr_t card_table_offset() { return OFFSET_OF(Page, card_table_); }
void RememberCard(ObjectPtr const* slot) {
ASSERT(Contains(reinterpret_cast<uword>(slot)));
if (card_table_ == NULL) {
card_table_ = reinterpret_cast<uint8_t*>(
calloc(card_table_size(), sizeof(uint8_t)));
}
intptr_t offset =
reinterpret_cast<uword>(slot) - reinterpret_cast<uword>(this);
intptr_t index = offset >> kBytesPerCardLog2;
ASSERT((index >= 0) && (index < card_table_size()));
card_table_[index] = 1;
}
bool IsCardRemembered(ObjectPtr const* slot) {
ASSERT(Contains(reinterpret_cast<uword>(slot)));
if (card_table_ == NULL) {
return false;
}
intptr_t offset =
reinterpret_cast<uword>(slot) - reinterpret_cast<uword>(this);
intptr_t index = offset >> kBytesPerCardLog2;
ASSERT((index >= 0) && (index < card_table_size()));
return card_table_[index] != 0;
}
#if defined(DART_COMPRESSED_POINTERS)
void RememberCard(CompressedObjectPtr const* slot) {
ASSERT(Contains(reinterpret_cast<uword>(slot)));
if (card_table_ == NULL) {
card_table_ = reinterpret_cast<uint8_t*>(
calloc(card_table_size(), sizeof(uint8_t)));
}
intptr_t offset =
reinterpret_cast<uword>(slot) - reinterpret_cast<uword>(this);
intptr_t index = offset >> kBytesPerCardLog2;
ASSERT((index >= 0) && (index < card_table_size()));
card_table_[index] = 1;
}
bool IsCardRemembered(CompressedObjectPtr const* slot) {
ASSERT(Contains(reinterpret_cast<uword>(slot)));
if (card_table_ == NULL) {
return false;
}
intptr_t offset =
reinterpret_cast<uword>(slot) - reinterpret_cast<uword>(this);
intptr_t index = offset >> kBytesPerCardLog2;
ASSERT((index >= 0) && (index < card_table_size()));
return card_table_[index] != 0;
}
#endif
void VisitRememberedCards(ObjectPointerVisitor* visitor);
void ResetProgressBar();
Thread* owner() const {
return owner_;
}
// Remember the limit to which objects have been copied.
void RecordSurvivors() {
survivor_end_ = object_end();
}
// Move survivor end to the end of the to_ space, making all surviving
// objects candidates for promotion next time.
void EarlyTenure() {
survivor_end_ = end_;
}
uword promo_candidate_words() const {
return (survivor_end_ - object_start()) / kWordSize;
}
void Acquire(Thread* thread) {
ASSERT(owner_ == nullptr);
owner_ = thread;
thread->set_top(top_);
thread->set_end(end_);
}
void Release(Thread* thread) {
ASSERT(owner_ == thread);
owner_ = nullptr;
top_ = thread->top();
thread->set_top(0);
thread->set_end(0);
}
void Release() {
if (owner_ != nullptr) {
Release(owner_);
}
}
uword TryAllocateGC(intptr_t size) {
ASSERT(owner_ == nullptr);
uword result = top_;
uword new_top = result + size;
if (LIKELY(new_top <= end_)) {
top_ = new_top;
return result;
}
return 0;
}
void Unallocate(uword addr, intptr_t size) {
ASSERT((addr + size) == top_);
top_ -= size;
}
bool IsSurvivor(uword raw_addr) const {
return raw_addr < survivor_end_;
}
bool IsResolved() const {
return top_ == resolved_top_;
}
private:
void set_object_end(uword value) {
ASSERT((value & kObjectAlignmentMask) == kOldObjectAlignmentOffset);
top_ = value;
}
// Returns NULL on OOM.
static Page* Allocate(intptr_t size, PageType type, bool can_use_cache);
// Deallocate the virtual memory backing this page. The page pointer to this
// page becomes immediately inaccessible.
void Deallocate(bool can_use_cache);
PageType type_;
VirtualMemory* memory_;
Page* next_;
ForwardingPage* forwarding_page_;
uint8_t* card_table_; // Remembered set, not marking.
RelaxedAtomic<intptr_t> progress_bar_;
// The thread using this page for allocation, otherwise NULL.
Thread* owner_;
// The address of the next allocation. If owner is non-NULL, this value is
// stale and the current value is at owner->top_. Called "NEXT" in the
// original Cheney paper.
uword top_;
// The address after the last allocatable byte in this page.
uword end_;
// Objects below this address have survived a scavenge.
uword survivor_end_;
// A pointer to the first unprocessed object. Resolution completes when this
// value meets the allocation top. Called "SCAN" in the original Cheney paper.
uword resolved_top_;
template <bool>
friend class ScavengerVisitorBase;
friend class SemiSpace;
friend class PageSpace;
friend class GCCompactor;
DISALLOW_ALLOCATION();
DISALLOW_IMPLICIT_CONSTRUCTORS(Page);
};
} // namespace dart
#endif // RUNTIME_VM_HEAP_PAGE_H_
runtime/vm/heap/pages.cc
+97 -275
View File
@@ -43,180 +43,6 @@ DEFINE_FLAG(bool,
"Print free list statistics after a GC");
DEFINE_FLAG(bool, log_growth, false, "Log PageSpace growth policy decisions.");
OldPage* OldPage::Allocate(intptr_t size_in_words,
PageType type,
const char* name) {
const bool executable = type == kExecutable;
const bool compressed = !executable;
VirtualMemory* memory = VirtualMemory::AllocateAligned(
size_in_words << kWordSizeLog2, kOldPageSize, executable, compressed,
name);
if (memory == NULL) {
return NULL;
}
OldPage* result = reinterpret_cast<OldPage*>(memory->address());
ASSERT(result != NULL);
result->memory_ = memory;
result->next_ = NULL;
result->used_in_bytes_ = 0;
result->forwarding_page_ = NULL;
result->card_table_ = NULL;
result->progress_bar_ = 0;
result->type_ = type;
LSAN_REGISTER_ROOT_REGION(result, sizeof(*result));
return result;
}
void OldPage::Deallocate() {
if (card_table_ != NULL) {
free(card_table_);
card_table_ = NULL;
}
bool image_page = is_image_page();
if (!image_page) {
LSAN_UNREGISTER_ROOT_REGION(this, sizeof(*this));
}
// For a regular heap pages, the memory for this object will become
// unavailable after the delete below.
delete memory_;
// For a heap page from a snapshot, the OldPage object lives in the malloc
// heap rather than the page itself.
if (image_page) {
free(this);
}
}
void OldPage::VisitObjects(ObjectVisitor* visitor) const {
ASSERT(Thread::Current()->IsAtSafepoint());
NoSafepointScope no_safepoint;
uword obj_addr = object_start();
uword end_addr = object_end();
while (obj_addr < end_addr) {
ObjectPtr raw_obj = UntaggedObject::FromAddr(obj_addr);
visitor->VisitObject(raw_obj);
obj_addr += raw_obj->untag()->HeapSize();
}
ASSERT(obj_addr == end_addr);
}
void OldPage::VisitObjectPointers(ObjectPointerVisitor* visitor) const {
ASSERT(Thread::Current()->IsAtSafepoint() ||
(Thread::Current()->task_kind() == Thread::kCompactorTask));
NoSafepointScope no_safepoint;
uword obj_addr = object_start();
uword end_addr = object_end();
while (obj_addr < end_addr) {
ObjectPtr raw_obj = UntaggedObject::FromAddr(obj_addr);
obj_addr += raw_obj->untag()->VisitPointers(visitor);
}
ASSERT(obj_addr == end_addr);
}
void OldPage::VisitRememberedCards(ObjectPointerVisitor* visitor) {
ASSERT(Thread::Current()->IsAtSafepoint() ||
(Thread::Current()->task_kind() == Thread::kScavengerTask));
NoSafepointScope no_safepoint;
if (card_table_ == NULL) {
return;
}
ArrayPtr obj =
static_cast<ArrayPtr>(UntaggedObject::FromAddr(object_start()));
ASSERT(obj->IsArray());
ASSERT(obj->untag()->IsCardRemembered());
CompressedObjectPtr* obj_from = obj->untag()->from();
CompressedObjectPtr* obj_to =
obj->untag()->to(Smi::Value(obj->untag()->length()));
uword heap_base = obj.heap_base();
const intptr_t size = card_table_size();
for (;;) {
intptr_t i = progress_bar_.fetch_add(1);
if (i >= size) break;
if (card_table_[i] != 0) {
CompressedObjectPtr* card_from =
reinterpret_cast<CompressedObjectPtr*>(this) +
(i << kSlotsPerCardLog2);
CompressedObjectPtr* card_to =
reinterpret_cast<CompressedObjectPtr*>(card_from) +
(1 << kSlotsPerCardLog2) - 1;
// Minus 1 because to is inclusive.
if (card_from < obj_from) {
// First card overlaps with header.
card_from = obj_from;
}
if (card_to > obj_to) {
// Last card(s) may extend past the object. Array truncation can make
// this happen for more than one card.
card_to = obj_to;
}
visitor->VisitCompressedPointers(heap_base, card_from, card_to);
bool has_new_target = false;
for (CompressedObjectPtr* slot = card_from; slot <= card_to; slot++) {
if ((*slot)->IsNewObjectMayBeSmi()) {
has_new_target = true;
break;
}
}
if (!has_new_target) {
card_table_[i] = 0;
}
}
}
}
void OldPage::ResetProgressBar() {
progress_bar_ = 0;
}
ObjectPtr OldPage::FindObject(FindObjectVisitor* visitor) const {
uword obj_addr = object_start();
uword end_addr = object_end();
if (visitor->VisitRange(obj_addr, end_addr)) {
while (obj_addr < end_addr) {
ObjectPtr raw_obj = UntaggedObject::FromAddr(obj_addr);
uword next_obj_addr = obj_addr + raw_obj->untag()->HeapSize();
if (visitor->VisitRange(obj_addr, next_obj_addr) &&
raw_obj->untag()->FindObject(visitor)) {
return raw_obj; // Found object, return it.
}
obj_addr = next_obj_addr;
}
ASSERT(obj_addr == end_addr);
}
return Object::null();
}
void OldPage::WriteProtect(bool read_only) {
ASSERT(!is_image_page());
VirtualMemory::Protection prot;
if (read_only) {
if ((type_ == kExecutable) && (memory_->AliasOffset() == 0)) {
prot = VirtualMemory::kReadExecute;
} else {
prot = VirtualMemory::kReadOnly;
}
} else {
prot = VirtualMemory::kReadWrite;
}
memory_->Protect(prot);
}
// The initial estimate of how many words we can mark per microsecond (usage
// before / mark-sweep time). This is a conservative value observed running
// Flutter on a Nexus 4. After the first mark-sweep, we instead use a value
@@ -266,21 +92,21 @@ PageSpace::~PageSpace() {
ml.Wait();
}
}
FreePages(pages_);
FreePages(exec_pages_);
FreePages(large_pages_);
FreePages(image_pages_);
FreePages(pages_, /*can_use_cache*/ true);
FreePages(exec_pages_, /*can_use_cache*/ false);
FreePages(large_pages_, /*can_use_cache*/ false);
FreePages(image_pages_, /*can_use_cache*/ false);
ASSERT(marker_ == NULL);
delete[] freelists_;
}
intptr_t PageSpace::LargePageSizeInWordsFor(intptr_t size) {
intptr_t page_size = Utils::RoundUp(size + OldPage::ObjectStartOffset(),
intptr_t page_size = Utils::RoundUp(size + Page::OldObjectStartOffset(),
VirtualMemory::PageSize());
return page_size >> kWordSizeLog2;
}
void PageSpace::AddPageLocked(OldPage* page) {
void PageSpace::AddPageLocked(Page* page) {
if (pages_ == nullptr) {
pages_ = page;
} else {
@@ -289,7 +115,7 @@ void PageSpace::AddPageLocked(OldPage* page) {
pages_tail_ = page;
}
void PageSpace::AddLargePageLocked(OldPage* page) {
void PageSpace::AddLargePageLocked(Page* page) {
if (large_pages_ == nullptr) {
large_pages_ = page;
} else {
@@ -298,7 +124,7 @@ void PageSpace::AddLargePageLocked(OldPage* page) {
large_pages_tail_ = page;
}
void PageSpace::AddExecPageLocked(OldPage* page) {
void PageSpace::AddExecPageLocked(Page* page) {
if (exec_pages_ == nullptr) {
exec_pages_ = page;
} else {
@@ -313,7 +139,7 @@ void PageSpace::AddExecPageLocked(OldPage* page) {
exec_pages_tail_ = page;
}
void PageSpace::RemovePageLocked(OldPage* page, OldPage* previous_page) {
void PageSpace::RemovePageLocked(Page* page, Page* previous_page) {
if (previous_page != NULL) {
previous_page->set_next(page->next());
} else {
@@ -324,7 +150,7 @@ void PageSpace::RemovePageLocked(OldPage* page, OldPage* previous_page) {
}
}
void PageSpace::RemoveLargePageLocked(OldPage* page, OldPage* previous_page) {
void PageSpace::RemoveLargePageLocked(Page* page, Page* previous_page) {
if (previous_page != NULL) {
previous_page->set_next(page->next());
} else {
@@ -335,7 +161,7 @@ void PageSpace::RemoveLargePageLocked(OldPage* page, OldPage* previous_page) {
}
}
void PageSpace::RemoveExecPageLocked(OldPage* page, OldPage* previous_page) {
void PageSpace::RemoveExecPageLocked(Page* page, Page* previous_page) {
if (previous_page != NULL) {
previous_page->set_next(page->next());
} else {
@@ -346,20 +172,19 @@ void PageSpace::RemoveExecPageLocked(OldPage* page, OldPage* previous_page) {
}
}
OldPage* PageSpace::AllocatePage(OldPage::PageType type, bool link) {
Page* PageSpace::AllocatePage(Page::PageType type, bool link) {
{
MutexLocker ml(&pages_lock_);
if (!CanIncreaseCapacityInWordsLocked(kOldPageSizeInWords)) {
if (!CanIncreaseCapacityInWordsLocked(kPageSizeInWords)) {
return nullptr;
}
IncreaseCapacityInWordsLocked(kOldPageSizeInWords);
IncreaseCapacityInWordsLocked(kPageSizeInWords);
}
const bool is_exec = (type == OldPage::kExecutable);
const char* name = Heap::RegionName(is_exec ? Heap::kCode : Heap::kOld);
OldPage* page = OldPage::Allocate(kOldPageSizeInWords, type, name);
const bool is_exec = (type == Page::kExecutable);
Page* page = Page::Allocate(kPageSize, type, /*can_use_cache*/ !is_exec);
if (page == nullptr) {
RELEASE_ASSERT(!FLAG_abort_on_oom);
IncreaseCapacityInWords(-kOldPageSizeInWords);
IncreaseCapacityInWords(-kPageSizeInWords);
return nullptr;
}
@@ -373,14 +198,14 @@ OldPage* PageSpace::AllocatePage(OldPage::PageType type, bool link) {
}
page->set_object_end(page->memory_->end());
if ((type != OldPage::kExecutable) && (heap_ != nullptr) &&
if ((type != Page::kExecutable) && (heap_ != nullptr) &&
(!heap_->is_vm_isolate())) {
page->AllocateForwardingPage();
}
return page;
}
OldPage* PageSpace::AllocateLargePage(intptr_t size, OldPage::PageType type) {
Page* PageSpace::AllocateLargePage(intptr_t size, Page::PageType type) {
const intptr_t page_size_in_words = LargePageSizeInWordsFor(size);
{
MutexLocker ml(&pages_lock_);
@@ -389,9 +214,9 @@ OldPage* PageSpace::AllocateLargePage(intptr_t size, OldPage::PageType type) {
}
IncreaseCapacityInWordsLocked(page_size_in_words);
}
const bool is_exec = (type == OldPage::kExecutable);
const char* name = Heap::RegionName(is_exec ? Heap::kCode : Heap::kOld);
OldPage* page = OldPage::Allocate(page_size_in_words, type, name);
const bool is_exec = (type == Page::kExecutable);
Page* page = Page::Allocate(page_size_in_words << kWordSizeLog2, type,
/*can_use_cache*/ false);
MutexLocker ml(&pages_lock_);
if (page == nullptr) {
@@ -415,7 +240,7 @@ OldPage* PageSpace::AllocateLargePage(intptr_t size, OldPage::PageType type) {
return page;
}
void PageSpace::TruncateLargePage(OldPage* page,
void PageSpace::TruncateLargePage(Page* page,
intptr_t new_object_size_in_bytes) {
const intptr_t old_object_size_in_bytes =
page->object_end() - page->object_start();
@@ -431,8 +256,8 @@ void PageSpace::TruncateLargePage(OldPage* page,
}
}
void PageSpace::FreePage(OldPage* page, OldPage* previous_page) {
bool is_exec = (page->type() == OldPage::kExecutable);
void PageSpace::FreePage(Page* page, Page* previous_page) {
bool is_exec = (page->type() == Page::kExecutable);
{
MutexLocker ml(&pages_lock_);
IncreaseCapacityInWordsLocked(-(page->memory_->size() >> kWordSizeLog2));
@@ -442,30 +267,29 @@ void PageSpace::FreePage(OldPage* page, OldPage* previous_page) {
RemovePageLocked(page, previous_page);
}
}
// TODO(iposva): Consider adding to a pool of empty pages.
page->Deallocate();
page->Deallocate(/*can_use_cache*/ !is_exec);
}
void PageSpace::FreeLargePage(OldPage* page, OldPage* previous_page) {
ASSERT(page->type() != OldPage::kExecutable);
void PageSpace::FreeLargePage(Page* page, Page* previous_page) {
ASSERT(page->type() != Page::kExecutable);
MutexLocker ml(&pages_lock_);
IncreaseCapacityInWordsLocked(-(page->memory_->size() >> kWordSizeLog2));
RemoveLargePageLocked(page, previous_page);
page->Deallocate();
page->Deallocate(/*can_use_cache*/ false);
}
void PageSpace::FreePages(OldPage* pages) {
OldPage* page = pages;
void PageSpace::FreePages(Page* pages, bool can_use_cache) {
Page* page = pages;
while (page != NULL) {
OldPage* next = page->next();
page->Deallocate();
Page* next = page->next();
page->Deallocate(can_use_cache);
page = next;
}
}
uword PageSpace::TryAllocateInFreshPage(intptr_t size,
FreeList* freelist,
OldPage::PageType type,
Page::PageType type,
GrowthPolicy growth_policy,
bool is_locked) {
ASSERT(Heap::IsAllocatableViaFreeLists(size));
@@ -474,7 +298,7 @@ uword PageSpace::TryAllocateInFreshPage(intptr_t size,
ASSERT(!Thread::Current()->force_growth());
if (heap_ != nullptr) { // Some unit tests.
heap_->CheckConcurrentMarking(Thread::Current(), GCReason::kOldSpace,
kOldPageSize);
kPageSize);
}
}
@@ -482,10 +306,10 @@ uword PageSpace::TryAllocateInFreshPage(intptr_t size,
SpaceUsage after_allocation = GetCurrentUsage();
after_allocation.used_in_words += size >> kWordSizeLog2;
// Can we grow by one page?
after_allocation.capacity_in_words += kOldPageSizeInWords;
after_allocation.capacity_in_words += kPageSizeInWords;
if (growth_policy == kForceGrowth ||
!page_space_controller_.ReachedHardThreshold(after_allocation)) {
OldPage* page = AllocatePage(type);
Page* page = AllocatePage(type);
if (page == NULL) {
return 0;
}
@@ -508,7 +332,7 @@ uword PageSpace::TryAllocateInFreshPage(intptr_t size,
}
uword PageSpace::TryAllocateInFreshLargePage(intptr_t size,
OldPage::PageType type,
Page::PageType type,
GrowthPolicy growth_policy) {
ASSERT(!Heap::IsAllocatableViaFreeLists(size));
@@ -532,7 +356,7 @@ uword PageSpace::TryAllocateInFreshLargePage(intptr_t size,
after_allocation.capacity_in_words += page_size_in_words;
if (growth_policy == kForceGrowth ||
!page_space_controller_.ReachedHardThreshold(after_allocation)) {
OldPage* page = AllocateLargePage(size, type);
Page* page = AllocateLargePage(size, type);
if (page != NULL) {
result = page->object_start();
// Note: usage_.capacity_in_words is increased by AllocateLargePage.
@@ -544,7 +368,7 @@ uword PageSpace::TryAllocateInFreshLargePage(intptr_t size,
uword PageSpace::TryAllocateInternal(intptr_t size,
FreeList* freelist,
OldPage::PageType type,
Page::PageType type,
GrowthPolicy growth_policy,
bool is_protected,
bool is_locked) {
@@ -586,7 +410,7 @@ class BasePageIterator : ValueObject {
public:
explicit BasePageIterator(const PageSpace* space) : space_(space) {}
OldPage* page() const { return page_; }
Page* page() const { return page_; }
bool Done() const { return page_ == NULL; }
@@ -630,7 +454,7 @@ class BasePageIterator : ValueObject {
const PageSpace* space_ = nullptr;
List list_;
OldPage* page_ = nullptr;
Page* page_ = nullptr;
};
// Provides unsafe access to all pages. Assumes pages are walkable.
@@ -665,7 +489,7 @@ class ExclusiveCodePageIterator : ValueObject {
space_->MakeIterable();
page_ = space_->exec_pages_;
}
OldPage* page() const { return page_; }
Page* page() const { return page_; }
bool Done() const { return page_ == NULL; }
void Advance() {
ASSERT(!Done());
@@ -676,7 +500,7 @@ class ExclusiveCodePageIterator : ValueObject {
const PageSpace* space_;
MutexLocker ml_;
NoSafepointScope no_safepoint;
OldPage* page_;
Page* page_;
};
void PageSpace::MakeIterable() const {
@@ -740,8 +564,8 @@ bool PageSpace::ContainsUnsafe(uword addr) const {
return false;
}
bool PageSpace::Contains(uword addr, OldPage::PageType type) const {
if (type == OldPage::kExecutable) {
bool PageSpace::Contains(uword addr, Page::PageType type) const {
if (type == Page::kExecutable) {
// Fast path executable pages.
for (ExclusiveCodePageIterator it(this); !it.Done(); it.Advance()) {
if (it.page()->Contains(addr)) {
@@ -760,8 +584,7 @@ bool PageSpace::Contains(uword addr, OldPage::PageType type) const {
bool PageSpace::DataContains(uword addr) const {
for (ExclusivePageIterator it(this); !it.Done(); it.Advance()) {
if ((it.page()->type() != OldPage::kExecutable) &&
it.page()->Contains(addr)) {
if ((it.page()->type() != Page::kExecutable) && it.page()->Contains(addr)) {
return true;
}
}
@@ -818,9 +641,9 @@ void PageSpace::VisitRememberedCards(ObjectPointerVisitor* visitor) const {
// Large pages may be added concurrently due to promotion in another scavenge
// worker, so terminate the traversal when we hit the tail we saw while
// holding the pages lock, instead of at NULL, otherwise we are racing when we
// read OldPage::next_ and OldPage::remembered_cards_.
OldPage* page;
OldPage* tail;
// read Page::next_ and Page::remembered_cards_.
Page* page;
Page* tail;
{
MutexLocker ml(&pages_lock_);
page = large_pages_;
@@ -834,14 +657,14 @@ void PageSpace::VisitRememberedCards(ObjectPointerVisitor* visitor) const {
}
void PageSpace::ResetProgressBars() const {
for (OldPage* page = large_pages_; page != NULL; page = page->next()) {
for (Page* page = large_pages_; page != NULL; page = page->next()) {
page->ResetProgressBar();
}
}
ObjectPtr PageSpace::FindObject(FindObjectVisitor* visitor,
OldPage::PageType type) const {
if (type == OldPage::kExecutable) {
Page::PageType type) const {
if (type == Page::kExecutable) {
// Fast path executable pages.
for (ExclusiveCodePageIterator it(this); !it.Done(); it.Advance()) {
ObjectPtr obj = it.page()->FindObject(visitor);
@@ -920,7 +743,7 @@ void PageSpace::PrintHeapMapToJSONStream(IsolateGroup* isolate_group,
heap_map.AddProperty("freeClassId", static_cast<intptr_t>(kFreeListElement));
heap_map.AddProperty("unitSizeBytes",
static_cast<intptr_t>(kObjectAlignment));
heap_map.AddProperty("pageSizeBytes", kOldPageSizeInWords * kWordSize);
heap_map.AddProperty("pageSizeBytes", kPageSizeInWords * kWordSize);
{
JSONObject class_list(&heap_map, "classList");
isolate_group->class_table()->PrintToJSONObject(&class_list);
@@ -933,7 +756,7 @@ void PageSpace::PrintHeapMapToJSONStream(IsolateGroup* isolate_group,
MutexLocker ml(&pages_lock_);
MakeIterable();
JSONArray all_pages(&heap_map, "pages");
for (OldPage* page = pages_; page != NULL; page = page->next()) {
for (Page* page = pages_; page != NULL; page = page->next()) {
JSONObject page_container(&all_pages);
page_container.AddPropertyF("objectStart", "0x%" Px "",
page->object_start());
@@ -941,7 +764,7 @@ void PageSpace::PrintHeapMapToJSONStream(IsolateGroup* isolate_group,
HeapMapAsJSONVisitor printer(&page_map);
page->VisitObjects(&printer);
}
for (OldPage* page = exec_pages_; page != NULL; page = page->next()) {
for (Page* page = exec_pages_; page != NULL; page = page->next()) {
JSONObject page_container(&all_pages);
page_container.AddPropertyF("objectStart", "0x%" Px "",
page->object_start());
@@ -958,15 +781,15 @@ void PageSpace::WriteProtectCode(bool read_only) {
MutexLocker ml(&pages_lock_);
NoSafepointScope no_safepoint;
// No need to go through all of the data pages first.
OldPage* page = exec_pages_;
Page* page = exec_pages_;
while (page != NULL) {
ASSERT(page->type() == OldPage::kExecutable);
ASSERT(page->type() == Page::kExecutable);
page->WriteProtect(read_only);
page = page->next();
}
page = large_pages_;
while (page != NULL) {
if (page->type() == OldPage::kExecutable) {
if (page->type() == Page::kExecutable) {
page->WriteProtect(read_only);
}
page = page->next();
@@ -1009,7 +832,7 @@ bool PageSpace::ShouldPerformIdleMarkCompact(int64_t deadline) {
// Discount two pages to account for the newest data and code pages, whose
// partial use doesn't indicate fragmentation.
const intptr_t excess_in_words =
usage_.capacity_in_words - usage_.used_in_words - 2 * kOldPageSizeInWords;
usage_.capacity_in_words - usage_.used_in_words - 2 * kPageSizeInWords;
const double excess_ratio = static_cast<double>(excess_in_words) /
static_cast<double>(usage_.capacity_in_words);
const bool fragmented = excess_ratio > 0.05;
@@ -1073,7 +896,7 @@ void PageSpace::TryReleaseReservation() {
uword addr = reinterpret_cast<uword>(oom_reservation_);
intptr_t size = oom_reservation_->HeapSize();
oom_reservation_ = nullptr;
freelists_[OldPage::kData].Free(addr, size);
freelists_[Page::kData].Free(addr, size);
}
bool PageSpace::MarkReservation() {
@@ -1089,7 +912,7 @@ bool PageSpace::MarkReservation() {
void PageSpace::TryReserveForOOM() {
if (oom_reservation_ == nullptr) {
uword addr = TryAllocate(kOOMReservationSize, OldPage::kData,
uword addr = TryAllocate(kOOMReservationSize, Page::kData,
kForceGrowth /* Don't re-enter GC */);
if (addr != 0) {
oom_reservation_ = FreeListElement::AsElement(addr, kOOMReservationSize);
@@ -1229,12 +1052,12 @@ void PageSpace::CollectGarbageHelper(Thread* thread,
// code protection.
TIMELINE_FUNCTION_GC_DURATION(thread, "SweepExecutable");
GCSweeper sweeper;
OldPage* prev_page = NULL;
OldPage* page = exec_pages_;
FreeList* freelist = &freelists_[OldPage::kExecutable];
Page* prev_page = NULL;
Page* page = exec_pages_;
FreeList* freelist = &freelists_[Page::kExecutable];
MutexLocker ml(freelist->mutex());
while (page != NULL) {
OldPage* next_page = page->next();
Page* next_page = page->next();
bool page_in_use = sweeper.SweepPage(page, freelist, true /*is_locked*/);
if (page_in_use) {
prev_page = page;
@@ -1313,17 +1136,17 @@ void PageSpace::SweepLarge() {
GCSweeper sweeper;
MutexLocker ml(&pages_lock_);
while (sweep_large_ != nullptr) {
OldPage* page = sweep_large_;
Page* page = sweep_large_;
sweep_large_ = page->next();
page->set_next(nullptr);
ASSERT(page->type() == OldPage::kData);
ASSERT(page->type() == Page::kData);
ml.Unlock();
intptr_t words_to_end = sweeper.SweepLargePage(page);
intptr_t size;
if (words_to_end == 0) {
size = page->memory_->size();
page->Deallocate();
page->Deallocate(/*can_use_cache*/ false);
ml.Lock();
IncreaseCapacityInWordsLocked(-(size >> kWordSizeLog2));
} else {
@@ -1349,10 +1172,10 @@ void PageSpace::Sweep(bool exclusive) {
MutexLocker ml(&pages_lock_);
while (sweep_regular_ != nullptr) {
OldPage* page = sweep_regular_;
Page* page = sweep_regular_;
sweep_regular_ = page->next();
page->set_next(nullptr);
ASSERT(page->type() == OldPage::kData);
ASSERT(page->type() == Page::kData);
ml.Unlock();
// Cycle through the shards round-robin so that free space is roughly
@@ -1363,7 +1186,7 @@ void PageSpace::Sweep(bool exclusive) {
intptr_t size;
if (!page_in_use) {
size = page->memory_->size();
page->Deallocate();
page->Deallocate(/*can_use_cache*/ true);
}
ml.Lock();
@@ -1389,7 +1212,7 @@ void PageSpace::ConcurrentSweep(IsolateGroup* isolate_group) {
void PageSpace::Compact(Thread* thread) {
thread->isolate_group()->set_compaction_in_progress(true);
GCCompactor compactor(thread, heap_);
compactor.Compact(pages_, &freelists_[OldPage::kData], &pages_lock_);
compactor.Compact(pages_, &freelists_[Page::kData], &pages_lock_);
thread->isolate_group()->set_compaction_in_progress(false);
if (FLAG_verify_after_gc) {
@@ -1415,8 +1238,8 @@ uword PageSpace::TryAllocateDataBumpLocked(FreeList* freelist, intptr_t size) {
// side-effect of populating the freelist with a large block. The next
// bump allocation request will have a chance to consume that block.
// TODO(koda): Could take freelist lock just once instead of twice.
return TryAllocateInFreshPage(size, freelist, OldPage::kData,
kForceGrowth, true /* is_locked*/);
return TryAllocateInFreshPage(size, freelist, Page::kData, kForceGrowth,
true /* is_locked*/);
}
intptr_t block_size = block->HeapSize();
if (remaining > 0) {
@@ -1462,30 +1285,29 @@ ObjectPtr PageSpace::AllocateSnapshot(intptr_t size) {
}
void PageSpace::SetupImagePage(void* pointer, uword size, bool is_executable) {
// Setup a OldPage so precompiled Instructions can be traversed.
// Instructions are contiguous at [pointer, pointer + size). OldPage
// Setup a Page so precompiled Instructions can be traversed.
// Instructions are contiguous at [pointer, pointer + size). Page
// expects to find objects at [memory->start() + ObjectStartOffset,
// memory->end()).
uword offset = OldPage::ObjectStartOffset();
uword offset = Page::OldObjectStartOffset();
pointer = reinterpret_cast<void*>(reinterpret_cast<uword>(pointer) - offset);
ASSERT(Utils::IsAligned(pointer, kObjectAlignment));
size += offset;
VirtualMemory* memory = VirtualMemory::ForImagePage(pointer, size);
ASSERT(memory != NULL);
OldPage* page = reinterpret_cast<OldPage*>(malloc(sizeof(OldPage)));
Page* page = reinterpret_cast<Page*>(malloc(sizeof(Page)));
page->type_ = is_executable ? Page::kExecutable : Page::kData;
page->memory_ = memory;
page->next_ = NULL;
page->object_end_ = memory->end();
page->used_in_bytes_ = page->object_end_ - page->object_start();
page->forwarding_page_ = NULL;
page->card_table_ = NULL;
page->next_ = nullptr;
page->forwarding_page_ = nullptr;
page->card_table_ = nullptr;
page->progress_bar_ = 0;
if (is_executable) {
page->type_ = OldPage::kExecutable;
} else {
page->type_ = OldPage::kData;
}
page->owner_ = nullptr;
page->top_ = memory->end();
page->end_ = memory->end();
page->survivor_end_ = 0;
page->resolved_top_ = 0;
MutexLocker ml(&pages_lock_);
page->next_ = image_pages_;
@@ -1494,7 +1316,7 @@ void PageSpace::SetupImagePage(void* pointer, uword size, bool is_executable) {
bool PageSpace::IsObjectFromImagePages(dart::ObjectPtr object) {
uword object_addr = UntaggedObject::ToAddr(object);
OldPage* image_page = image_pages_;
Page* image_page = image_pages_;
while (image_page != nullptr) {
if (image_page->Contains(object_addr)) {
return true;
@@ -1585,7 +1407,7 @@ void PageSpaceController::EvaluateGarbageCollection(SpaceUsage before,
(static_cast<intptr_t>(after.CombinedUsedInWords() /
desired_utilization_) -
(after.CombinedUsedInWords())) /
kOldPageSizeInWords;
kPageSizeInWords;
if (garbage_ratio == 0) {
// No garbage in the previous cycle so it would be hard to compute a
// grow_heap size based on estimated garbage so we use growth ratio
@@ -1600,8 +1422,8 @@ void PageSpaceController::EvaluateGarbageCollection(SpaceUsage before,
intptr_t local_grow_heap = 0;
while (min < max) {
local_grow_heap = (max + min) / 2;
const intptr_t limit = after.CombinedUsedInWords() +
(local_grow_heap * kOldPageSizeInWords);
const intptr_t limit =
after.CombinedUsedInWords() + (local_grow_heap * kPageSizeInWords);
const intptr_t allocated_before_next_gc =
limit - (after.CombinedUsedInWords());
const double estimated_garbage = k * allocated_before_next_gc;
@@ -1630,7 +1452,7 @@ void PageSpaceController::EvaluateGarbageCollection(SpaceUsage before,
ASSERT(grow_heap >= 0);
// Fraction of asymptote used.
double f = static_cast<double>(after.CombinedUsedInWords() +
(kOldPageSizeInWords * grow_heap)) /
(kPageSizeInWords * grow_heap)) /
static_cast<double>(max_capacity_in_words);
ASSERT(f >= 0.0);
// Increase weight at the high end.
@@ -1641,7 +1463,7 @@ void PageSpaceController::EvaluateGarbageCollection(SpaceUsage before,
// Discount growth more the closer we get to the desired asymptote.
grow_heap = static_cast<intptr_t>(grow_heap * f);
// Minimum growth step after reaching the asymptote.
intptr_t min_step = (2 * MB) / kOldPageSize;
intptr_t min_step = (2 * MB) / kPageSize;
grow_heap = Utils::Maximum(min_step, grow_heap);
}
@@ -1658,7 +1480,7 @@ void PageSpaceController::EvaluateAfterLoading(SpaceUsage after) {
growth_in_pages = (static_cast<intptr_t>(after.CombinedUsedInWords() /
desired_utilization_) -
(after.CombinedUsedInWords())) /
kOldPageSizeInWords;
kPageSizeInWords;
}
// Apply growth cap.
@@ -1674,7 +1496,7 @@ void PageSpaceController::RecordUpdate(SpaceUsage before,
const char* reason) {
// Save final threshold compared before growing.
intptr_t threshold =
after.CombinedUsedInWords() + (kOldPageSizeInWords * growth_in_pages);
after.CombinedUsedInWords() + (kPageSizeInWords * growth_in_pages);
#if defined(TARGET_ARCH_IA32)
bool concurrent_mark = false;
@@ -1691,7 +1513,7 @@ void PageSpaceController::RecordUpdate(SpaceUsage before,
// Set a tight idle threshold.
idle_gc_threshold_in_words_ =
after.CombinedUsedInWords() + (2 * kOldPageSizeInWords);
after.CombinedUsedInWords() + (2 * kPageSizeInWords);
#if defined(SUPPORT_TIMELINE)
Thread* thread = Thread::Current();
runtime/vm/heap/pages.h
+35 -226
View File
@@ -8,6 +8,7 @@
#include "platform/atomic.h"
#include "vm/globals.h"
#include "vm/heap/freelist.h"
#include "vm/heap/page.h"
#include "vm/heap/spaces.h"
#include "vm/lockers.h"
#include "vm/ring_buffer.h"
@@ -26,196 +27,6 @@ class ObjectSet;
class ForwardingPage;
class GCMarker;
static constexpr intptr_t kOldPageSize = 512 * KB;
static constexpr intptr_t kOldPageSizeInWords = kOldPageSize / kWordSize;
static constexpr intptr_t kOldPageMask = ~(kOldPageSize - 1);
static constexpr intptr_t kBitVectorWordsPerBlock = 1;
static constexpr intptr_t kBlockSize =
kObjectAlignment * kBitsPerWord * kBitVectorWordsPerBlock;
static constexpr intptr_t kBlockMask = ~(kBlockSize - 1);
static constexpr intptr_t kBlocksPerPage = kOldPageSize / kBlockSize;
// A page containing old generation objects.
class OldPage {
public:
enum PageType { kExecutable = 0, kData };
OldPage* next() const { return next_; }
void set_next(OldPage* next) { next_ = next; }
bool Contains(uword addr) const { return memory_->Contains(addr); }
intptr_t AliasOffset() const { return memory_->AliasOffset(); }
uword object_start() const { return memory_->start() + ObjectStartOffset(); }
uword object_end() const { return object_end_; }
uword used_in_bytes() const { return used_in_bytes_; }
void set_used_in_bytes(uword value) {
ASSERT(Utils::IsAligned(value, kObjectAlignment));
used_in_bytes_ = value;
}
ForwardingPage* forwarding_page() const { return forwarding_page_; }
void AllocateForwardingPage();
PageType type() const { return type_; }
bool is_image_page() const { return !memory_->vm_owns_region(); }
void VisitObjects(ObjectVisitor* visitor) const;
void VisitObjectPointers(ObjectPointerVisitor* visitor) const;
ObjectPtr FindObject(FindObjectVisitor* visitor) const;
void WriteProtect(bool read_only);
static intptr_t ObjectStartOffset() {
return Utils::RoundUp(sizeof(OldPage), kMaxObjectAlignment);
}
// Warning: This does not work for objects on image pages because image pages
// are not aligned. However, it works for objects on large pages, because
// only one object is allocated per large page.
static OldPage* Of(ObjectPtr obj) {
ASSERT(obj->IsHeapObject());
ASSERT(obj->IsOldObject());
return reinterpret_cast<OldPage*>(static_cast<uword>(obj) & kOldPageMask);
}
// Warning: This does not work for addresses on image pages or on large pages.
static OldPage* Of(uword addr) {
return reinterpret_cast<OldPage*>(addr & kOldPageMask);
}
// Warning: This does not work for objects on image pages.
static ObjectPtr ToExecutable(ObjectPtr obj) {
OldPage* page = Of(obj);
VirtualMemory* memory = page->memory_;
const intptr_t alias_offset = memory->AliasOffset();
if (alias_offset == 0) {
return obj; // Not aliased.
}
uword addr = UntaggedObject::ToAddr(obj);
if (memory->Contains(addr)) {
return UntaggedObject::FromAddr(addr + alias_offset);
}
// obj is executable.
ASSERT(memory->ContainsAlias(addr));
return obj;
}
// Warning: This does not work for objects on image pages.
static ObjectPtr ToWritable(ObjectPtr obj) {
OldPage* page = Of(obj);
VirtualMemory* memory = page->memory_;
const intptr_t alias_offset = memory->AliasOffset();
if (alias_offset == 0) {
return obj; // Not aliased.
}
uword addr = UntaggedObject::ToAddr(obj);
if (memory->ContainsAlias(addr)) {
return UntaggedObject::FromAddr(addr - alias_offset);
}
// obj is writable.
ASSERT(memory->Contains(addr));
return obj;
}
// 1 card = 128 slots.
static const intptr_t kSlotsPerCardLog2 = 7;
static const intptr_t kBytesPerCardLog2 =
kCompressedWordSizeLog2 + kSlotsPerCardLog2;
intptr_t card_table_size() const {
return memory_->size() >> kBytesPerCardLog2;
}
static intptr_t card_table_offset() {
return OFFSET_OF(OldPage, card_table_);
}
void RememberCard(ObjectPtr const* slot) {
ASSERT(Contains(reinterpret_cast<uword>(slot)));
if (card_table_ == NULL) {
card_table_ = reinterpret_cast<uint8_t*>(
calloc(card_table_size(), sizeof(uint8_t)));
}
intptr_t offset =
reinterpret_cast<uword>(slot) - reinterpret_cast<uword>(this);
intptr_t index = offset >> kBytesPerCardLog2;
ASSERT((index >= 0) && (index < card_table_size()));
card_table_[index] = 1;
}
bool IsCardRemembered(ObjectPtr const* slot) {
ASSERT(Contains(reinterpret_cast<uword>(slot)));
if (card_table_ == NULL) {
return false;
}
intptr_t offset =
reinterpret_cast<uword>(slot) - reinterpret_cast<uword>(this);
intptr_t index = offset >> kBytesPerCardLog2;
ASSERT((index >= 0) && (index < card_table_size()));
return card_table_[index] != 0;
}
#if defined(DART_COMPRESSED_POINTERS)
void RememberCard(CompressedObjectPtr const* slot) {
ASSERT(Contains(reinterpret_cast<uword>(slot)));
if (card_table_ == NULL) {
card_table_ = reinterpret_cast<uint8_t*>(
calloc(card_table_size(), sizeof(uint8_t)));
}
intptr_t offset =
reinterpret_cast<uword>(slot) - reinterpret_cast<uword>(this);
intptr_t index = offset >> kBytesPerCardLog2;
ASSERT((index >= 0) && (index < card_table_size()));
card_table_[index] = 1;
}
bool IsCardRemembered(CompressedObjectPtr const* slot) {
ASSERT(Contains(reinterpret_cast<uword>(slot)));
if (card_table_ == NULL) {
return false;
}
intptr_t offset =
reinterpret_cast<uword>(slot) - reinterpret_cast<uword>(this);
intptr_t index = offset >> kBytesPerCardLog2;
ASSERT((index >= 0) && (index < card_table_size()));
return card_table_[index] != 0;
}
#endif
void VisitRememberedCards(ObjectPointerVisitor* visitor);
void ResetProgressBar();
private:
void set_object_end(uword value) {
ASSERT((value & kObjectAlignmentMask) == kOldObjectAlignmentOffset);
object_end_ = value;
}
// Returns NULL on OOM.
static OldPage* Allocate(intptr_t size_in_words,
PageType type,
const char* name);
// Deallocate the virtual memory backing this page. The page pointer to this
// page becomes immediately inaccessible.
void Deallocate();
VirtualMemory* memory_;
OldPage* next_;
uword object_end_;
uword used_in_bytes_;
ForwardingPage* forwarding_page_;
uint8_t* card_table_; // Remembered set, not marking.
RelaxedAtomic<intptr_t> progress_bar_;
PageType type_;
friend class PageSpace;
friend class GCCompactor;
DISALLOW_ALLOCATION();
DISALLOW_IMPLICIT_CONSTRUCTORS(OldPage);
};
// The history holds the timing information of the last garbage collection
// runs.
class PageSpaceGarbageCollectionHistory {
@@ -328,10 +139,9 @@ class PageSpace {
~PageSpace();
uword TryAllocate(intptr_t size,
OldPage::PageType type = OldPage::kData,
Page::PageType type = Page::kData,
GrowthPolicy growth_policy = kControlGrowth) {
bool is_protected =
(type == OldPage::kExecutable) && FLAG_write_protect_code;
bool is_protected = (type == Page::kExecutable) && FLAG_write_protect_code;
bool is_locked = false;
return TryAllocateInternal(size, &freelists_[type], type, growth_policy,
is_protected, is_locked);
@@ -381,7 +191,7 @@ class PageSpace {
int64_t ImageInWords() const {
int64_t size = 0;
MutexLocker ml(&pages_lock_);
for (OldPage* page = image_pages_; page != nullptr; page = page->next()) {
for (Page* page = image_pages_; page != nullptr; page = page->next()) {
size += page->memory_->size();
}
return size >> kWordSizeLog2;
@@ -389,7 +199,7 @@ class PageSpace {
bool Contains(uword addr) const;
bool ContainsUnsafe(uword addr) const;
bool Contains(uword addr, OldPage::PageType type) const;
bool Contains(uword addr, Page::PageType type) const;
bool DataContains(uword addr) const;
bool IsValidAddress(uword addr) const { return Contains(addr); }
@@ -401,8 +211,7 @@ class PageSpace {
void VisitRememberedCards(ObjectPointerVisitor* visitor) const;
void ResetProgressBars() const;
ObjectPtr FindObject(FindObjectVisitor* visitor,
OldPage::PageType type) const;
ObjectPtr FindObject(FindObjectVisitor* visitor, Page::PageType type) const;
// Collect the garbage in the page space using mark-sweep or mark-compact.
void CollectGarbage(Thread* thread, bool compact, bool finalize);
@@ -460,7 +269,7 @@ class PageSpace {
// Bulk data allocation.
FreeList* DataFreeList(intptr_t i = 0) {
return &freelists_[OldPage::kData + i];
return &freelists_[Page::kData + i];
}
void AcquireLock(FreeList* freelist);
void ReleaseLock(FreeList* freelist);
@@ -470,7 +279,7 @@ class PageSpace {
GrowthPolicy growth_policy) {
bool is_protected = false;
bool is_locked = true;
return TryAllocateInternal(size, freelist, OldPage::kData, growth_policy,
return TryAllocateInternal(size, freelist, Page::kData, growth_policy,
is_protected, is_locked);
}
@@ -490,7 +299,7 @@ class PageSpace {
// Attempt to allocate from bump block rather than normal freelist.
uword TryAllocateDataBumpLocked(intptr_t size) {
return TryAllocateDataBumpLocked(&freelists_[OldPage::kData], size);
return TryAllocateDataBumpLocked(&freelists_[Page::kData], size);
}
uword TryAllocateDataBumpLocked(FreeList* freelist, intptr_t size);
DART_FORCE_INLINE
@@ -532,36 +341,36 @@ class PageSpace {
uword TryAllocateInternal(intptr_t size,
FreeList* freelist,
OldPage::PageType type,
Page::PageType type,
GrowthPolicy growth_policy,
bool is_protected,
bool is_locked);
uword TryAllocateInFreshPage(intptr_t size,
FreeList* freelist,
OldPage::PageType type,
Page::PageType type,
GrowthPolicy growth_policy,
bool is_locked);
uword TryAllocateInFreshLargePage(intptr_t size,
OldPage::PageType type,
Page::PageType type,
GrowthPolicy growth_policy);
// Makes bump block walkable; do not call concurrently with mutator.
void MakeIterable() const;
void AddPageLocked(OldPage* page);
void AddLargePageLocked(OldPage* page);
void AddExecPageLocked(OldPage* page);
void RemovePageLocked(OldPage* page, OldPage* previous_page);
void RemoveLargePageLocked(OldPage* page, OldPage* previous_page);
void RemoveExecPageLocked(OldPage* page, OldPage* previous_page);
void AddPageLocked(Page* page);
void AddLargePageLocked(Page* page);
void AddExecPageLocked(Page* page);
void RemovePageLocked(Page* page, Page* previous_page);
void RemoveLargePageLocked(Page* page, Page* previous_page);
void RemoveExecPageLocked(Page* page, Page* previous_page);
OldPage* AllocatePage(OldPage::PageType type, bool link = true);
OldPage* AllocateLargePage(intptr_t size, OldPage::PageType type);
Page* AllocatePage(Page::PageType type, bool link = true);
Page* AllocateLargePage(intptr_t size, Page::PageType type);
void TruncateLargePage(OldPage* page, intptr_t new_object_size_in_bytes);
void FreePage(OldPage* page, OldPage* previous_page);
void FreeLargePage(OldPage* page, OldPage* previous_page);
void FreePages(OldPage* pages);
void TruncateLargePage(Page* page, intptr_t new_object_size_in_bytes);
void FreePage(Page* page, Page* previous_page);
void FreeLargePage(Page* page, Page* previous_page);
void FreePages(Page* pages, bool can_use_cache);
void CollectGarbageHelper(Thread* thread, bool compact, bool finalize);
void SweepLarge();
@@ -584,9 +393,9 @@ class PageSpace {
Heap* const heap_;
// One list for executable pages at freelists_[OldPage::kExecutable].
// One list for executable pages at freelists_[Page::kExecutable].
// FLAG_scavenger_tasks count of lists for data pages starting at
// freelists_[OldPage::kData]. The sweeper inserts into the data page
// freelists_[Page::kData]. The sweeper inserts into the data page
// freelists round-robin. The scavenger workers each use one of the data
// page freelists without locking.
const intptr_t num_freelists_;
@@ -596,15 +405,15 @@ class PageSpace {
// Use ExclusivePageIterator for safe access to these.
mutable Mutex pages_lock_;
OldPage* pages_ = nullptr;
OldPage* pages_tail_ = nullptr;
OldPage* exec_pages_ = nullptr;
OldPage* exec_pages_tail_ = nullptr;
OldPage* large_pages_ = nullptr;
OldPage* large_pages_tail_ = nullptr;
OldPage* image_pages_ = nullptr;
OldPage* sweep_regular_ = nullptr;
OldPage* sweep_large_ = nullptr;
Page* pages_ = nullptr;
Page* pages_tail_ = nullptr;
Page* exec_pages_ = nullptr;
Page* exec_pages_tail_ = nullptr;
Page* large_pages_ = nullptr;
Page* large_pages_tail_ = nullptr;
Page* image_pages_ = nullptr;
Page* sweep_regular_ = nullptr;
Page* sweep_large_ = nullptr;
// Various sizes being tracked for this generation.
intptr_t max_capacity_in_words_;
runtime/vm/heap/scavenger.cc
+38 -131
View File
@@ -237,7 +237,7 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
ASSERT((obj == nullptr) || obj->IsOldObject());
visiting_old_object_ = obj;
if (obj != nullptr) {
// Card update happens in OldPage::VisitRememberedCards.
// Card update happens in Page::VisitRememberedCards.
ASSERT(!obj->untag()->IsCardRemembered());
}
}
@@ -300,7 +300,7 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
if (!scavenger_->abort_) {
ASSERT(!HasWork());
for (NewPage* page = head_; page != nullptr; page = page->next()) {
for (Page* page = head_; page != nullptr; page = page->next()) {
ASSERT(page->IsResolved());
page->RecordSurvivors();
}
@@ -320,8 +320,12 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
delayed_.Release();
}
NewPage* head() const { return head_; }
NewPage* tail() const { return tail_; }
Page* head() const {
return head_;
}
Page* tail() const {
return tail_;
}
static bool ForwardOrSetNullIfCollected(uword heap_base,
CompressedObjectPtr* ptr_address);
@@ -403,7 +407,7 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
intptr_t size = raw_obj->untag()->HeapSize(header);
uword new_addr = 0;
// Check whether object should be promoted.
if (!NewPage::Of(raw_obj)->IsSurvivor(raw_addr)) {
if (!Page::Of(raw_obj)->IsSurvivor(raw_addr)) {
// Not a survivor of a previous scavenge. Just copy the object into the
// to space.
new_addr = TryAllocateCopy(size);
@@ -542,9 +546,9 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
PromotionWorkList promoted_list_;
GCLinkedLists delayed_;
NewPage* head_ = nullptr;
NewPage* tail_ = nullptr; // Allocating from here.
NewPage* scan_ = nullptr; // Resolving from here.
Page* head_ = nullptr;
Page* tail_ = nullptr; // Allocating from here.
Page* scan_ = nullptr; // Resolving from here.
template <typename GCVisitorType>
friend void MournFinalized(GCVisitorType* visitor);
@@ -674,121 +678,23 @@ SemiSpace::SemiSpace(intptr_t max_capacity_in_words)
: max_capacity_in_words_(max_capacity_in_words), head_(nullptr) {}
SemiSpace::~SemiSpace() {
NewPage* page = head_;
Page* page = head_;
while (page != nullptr) {
NewPage* next = page->next();
page->Deallocate();
Page* next = page->next();
page->Deallocate(/*can_use_cache*/ true);
page = next;
}
}
// TODO(rmacnak): Unify this with old-space pages, and possibly zone segments.
// This cache needs to be at least as big as FLAG_new_gen_semi_max_size or
// munmap will noticably impact performance.
static constexpr intptr_t kPageCacheCapacity = 8 * kWordSize;
static Mutex* page_cache_mutex = nullptr;
static VirtualMemory* page_cache[kPageCacheCapacity] = {nullptr};
static intptr_t page_cache_size = 0;
void SemiSpace::Init() {
ASSERT(page_cache_mutex == nullptr);
page_cache_mutex = new Mutex(NOT_IN_PRODUCT("page_cache_mutex"));
}
void SemiSpace::ClearCache() {
MutexLocker ml(page_cache_mutex);
ASSERT(page_cache_size >= 0);
ASSERT(page_cache_size <= kPageCacheCapacity);
while (page_cache_size > 0) {
delete page_cache[--page_cache_size];
}
}
void SemiSpace::Cleanup() {
ClearCache();
delete page_cache_mutex;
page_cache_mutex = nullptr;
}
intptr_t SemiSpace::CachedSize() {
MutexLocker ml(page_cache_mutex);
return page_cache_size * kNewPageSize;
}
NewPage* NewPage::Allocate() {
const intptr_t size = kNewPageSize;
VirtualMemory* memory = nullptr;
{
MutexLocker ml(page_cache_mutex);
ASSERT(page_cache_size >= 0);
ASSERT(page_cache_size <= kPageCacheCapacity);
if (page_cache_size > 0) {
memory = page_cache[--page_cache_size];
}
}
if (memory == nullptr) {
const intptr_t alignment = kNewPageSize;
const bool is_executable = false;
const bool compressed = true;
const char* const name = Heap::RegionName(Heap::kNew);
memory = VirtualMemory::AllocateAligned(size, alignment, is_executable,
compressed, name);
}
if (memory == nullptr) {
return nullptr; // Out of memory.
}
#if defined(DEBUG)
memset(memory->address(), Heap::kZapByte, size);
#endif
// Initialized by generated code.
MSAN_UNPOISON(memory->address(), size);
NewPage* result = reinterpret_cast<NewPage*>(memory->address());
result->memory_ = memory;
result->next_ = nullptr;
result->owner_ = nullptr;
uword top = result->object_start();
result->top_ = top;
result->end_ = memory->end() - kNewObjectAlignmentOffset;
result->survivor_end_ = top;
result->resolved_top_ = top;
LSAN_REGISTER_ROOT_REGION(result, sizeof(*result));
return result;
}
void NewPage::Deallocate() {
LSAN_UNREGISTER_ROOT_REGION(this, sizeof(*this));
VirtualMemory* memory = memory_;
{
MutexLocker ml(page_cache_mutex);
ASSERT(page_cache_size >= 0);
ASSERT(page_cache_size <= kPageCacheCapacity);
if (page_cache_size < kPageCacheCapacity) {
intptr_t size = memory->size();
#if defined(DEBUG)
memset(memory->address(), Heap::kZapByte, size);
#endif
MSAN_POISON(memory->address(), size);
page_cache[page_cache_size++] = memory;
memory = nullptr;
}
}
delete memory;
}
NewPage* SemiSpace::TryAllocatePageLocked(bool link) {
Page* SemiSpace::TryAllocatePageLocked(bool link) {
if (capacity_in_words_ >= max_capacity_in_words_) {
return nullptr; // Full.
}
NewPage* page = NewPage::Allocate();
Page* page = Page::Allocate(kPageSize, Page::kNew, /*can_use_cache*/ true);
if (page == nullptr) {
return nullptr; // Out of memory;
}
capacity_in_words_ += kNewPageSizeInWords;
capacity_in_words_ += kPageSizeInWords;
if (link) {
if (head_ == nullptr) {
head_ = tail_ = page;
@@ -801,19 +707,19 @@ NewPage* SemiSpace::TryAllocatePageLocked(bool link) {
}
bool SemiSpace::Contains(uword addr) const {
for (NewPage* page = head_; page != nullptr; page = page->next()) {
for (Page* page = head_; page != nullptr; page = page->next()) {
if (page->Contains(addr)) return true;
}
return false;
}
void SemiSpace::WriteProtect(bool read_only) {
for (NewPage* page = head_; page != nullptr; page = page->next()) {
for (Page* page = head_; page != nullptr; page = page->next()) {
page->WriteProtect(read_only);
}
}
void SemiSpace::AddList(NewPage* head, NewPage* tail) {
void SemiSpace::AddList(Page* head, Page* tail) {
if (head == nullptr) {
return;
}
@@ -868,7 +774,7 @@ intptr_t Scavenger::NewSizeInWords(intptr_t old_size_in_words,
GCReason reason) const {
bool grow = false;
if (2 * heap_->isolate_group()->MutatorCount() >
(old_size_in_words / kNewPageSizeInWords)) {
(old_size_in_words / kPageSizeInWords)) {
// Not enough TLABs to give two to each mutator.
grow = true;
}
@@ -957,7 +863,7 @@ class CheckStoreBufferVisitor : public ObjectVisitor,
ObjectPtr raw_obj = *ptr;
if (raw_obj->IsHeapObject() && raw_obj->IsNewObject()) {
if (is_card_remembered_) {
if (!OldPage::Of(visiting_)->IsCardRemembered(ptr)) {
if (!Page::Of(visiting_)->IsCardRemembered(ptr)) {
FATAL3(
"Old object %#" Px " references new object %#" Px
", but the "
@@ -988,7 +894,7 @@ class CheckStoreBufferVisitor : public ObjectVisitor,
ObjectPtr raw_obj = ptr->Decompress(heap_base);
if (raw_obj->IsHeapObject() && raw_obj->IsNewObject()) {
if (is_card_remembered_) {
if (!OldPage::Of(visiting_)->IsCardRemembered(ptr)) {
if (!Page::Of(visiting_)->IsCardRemembered(ptr)) {
FATAL3(
"Old object %#" Px " references new object %#" Px
", but the "
@@ -1350,7 +1256,7 @@ void ScavengerVisitorBase<parallel>::ProcessToSpace() {
}
scan_->resolved_top_ = resolved_top;
NewPage* next = scan_->next();
Page* next = scan_->next();
if (next == nullptr) {
// Don't update scan_. More objects may yet be copied to this TLAB.
return;
@@ -1633,7 +1539,7 @@ void Scavenger::VisitObjectPointers(ObjectPointerVisitor* visitor) const {
ASSERT(Thread::Current()->IsAtSafepoint() ||
(Thread::Current()->task_kind() == Thread::kMarkerTask) ||
(Thread::Current()->task_kind() == Thread::kCompactorTask));
for (NewPage* page = to_->head(); page != nullptr; page = page->next()) {
for (Page* page = to_->head(); page != nullptr; page = page->next()) {
page->VisitObjectPointers(visitor);
}
}
@@ -1641,20 +1547,20 @@ void Scavenger::VisitObjectPointers(ObjectPointerVisitor* visitor) const {
void Scavenger::VisitObjects(ObjectVisitor* visitor) const {
ASSERT(Thread::Current()->IsAtSafepoint() ||
(Thread::Current()->task_kind() == Thread::kMarkerTask));
for (NewPage* page = to_->head(); page != nullptr; page = page->next()) {
for (Page* page = to_->head(); page != nullptr; page = page->next()) {
page->VisitObjects(visitor);
}
}
void Scavenger::AddRegionsToObjectSet(ObjectSet* set) const {
for (NewPage* page = to_->head(); page != nullptr; page = page->next()) {
for (Page* page = to_->head(); page != nullptr; page = page->next()) {
set->AddRegion(page->start(), page->end());
}
}
ObjectPtr Scavenger::FindObject(FindObjectVisitor* visitor) {
ASSERT(!scavenging_);
for (NewPage* page = to_->head(); page != nullptr; page = page->next()) {
for (Page* page = to_->head(); page != nullptr; page = page->next()) {
uword cur = page->object_start();
if (!visitor->VisitRange(cur, page->object_end())) continue;
while (cur < page->object_end()) {
@@ -1681,20 +1587,21 @@ void Scavenger::TryAllocateNewTLAB(Thread* thread,
if (can_safepoint && !thread->force_growth()) {
ASSERT(thread->no_safepoint_scope_depth() == 0);
heap_->CheckConcurrentMarking(thread, GCReason::kNewSpace, kNewPageSize);
heap_->CheckConcurrentMarking(thread, GCReason::kNewSpace, kPageSize);
}
MutexLocker ml(&space_lock_);
for (NewPage* page = to_->head(); page != nullptr; page = page->next()) {
for (Page* page = to_->head(); page != nullptr; page = page->next()) {
if (page->owner() != nullptr) continue;
intptr_t available = page->end() - page->object_end();
intptr_t available =
(page->end() - kAllocationRedZoneSize) - page->object_end();
if (available >= min_size) {
page->Acquire(thread);
return;
}
}
NewPage* page = to_->TryAllocatePageLocked(true);
Page* page = to_->TryAllocatePageLocked(true);
if (page == nullptr) {
return;
}
@@ -1716,7 +1623,7 @@ void Scavenger::AbandonRemainingTLABForDebugging(Thread* thread) {
void Scavenger::AbandonRemainingTLAB(Thread* thread) {
if (thread->top() == 0) return;
NewPage* page = NewPage::Of(thread->top() - 1);
Page* page = Page::Of(thread->top() - 1);
{
MutexLocker ml(&space_lock_);
page->Release(thread);
@@ -1726,7 +1633,7 @@ void Scavenger::AbandonRemainingTLAB(Thread* thread) {
template <bool parallel>
uword ScavengerVisitorBase<parallel>::TryAllocateCopySlow(intptr_t size) {
NewPage* page;
Page* page;
{
MutexLocker ml(&scavenger_->space_lock_);
page = scavenger_->to_->TryAllocatePageLocked(false);
@@ -1774,7 +1681,7 @@ void Scavenger::Scavenge(Thread* thread, GCType type, GCReason reason) {
intptr_t abandoned_bytes = 0; // TODO(rmacnak): Count fragmentation?
SpaceUsage usage_before = GetCurrentUsage();
intptr_t promo_candidate_words = 0;
for (NewPage* page = to_->head(); page != nullptr; page = page->next()) {
for (Page* page = to_->head(); page != nullptr; page = page->next()) {
page->Release();
if (early_tenure_) {
page->EarlyTenure();
@@ -1920,7 +1827,7 @@ void Scavenger::ReverseScavenge(SemiSpace** from) {
};
ReverseFromForwardingVisitor visitor;
for (NewPage* page = (*from)->head(); page != nullptr; page = page->next()) {
for (Page* page = (*from)->head(); page != nullptr; page = page->next()) {
page->VisitObjects(&visitor);
}
runtime/vm/heap/scavenger.h
+10 -154
View File
@@ -11,6 +11,7 @@
#include "vm/dart.h"
#include "vm/flags.h"
#include "vm/globals.h"
#include "vm/heap/page.h"
#include "vm/heap/spaces.h"
#include "vm/isolate.h"
#include "vm/lockers.h"
@@ -29,166 +30,19 @@ class ObjectSet;
template <bool parallel>
class ScavengerVisitorBase;
static constexpr intptr_t kNewPageSize = 512 * KB;
static constexpr intptr_t kNewPageSizeInWords = kNewPageSize / kWordSize;
static constexpr intptr_t kNewPageMask = ~(kNewPageSize - 1);
// Simplify initialization in allocation stubs by ensuring it is safe
// to overshoot the object end by up to kAllocationRedZoneSize. (Just as the
// stack red zone allows one to overshoot the stack pointer.)
static constexpr intptr_t kAllocationRedZoneSize = kObjectAlignment;
// A page containing new generation objects.
class NewPage {
public:
static NewPage* Allocate();
void Deallocate();
uword start() const { return memory_->start(); }
uword end() const { return memory_->end() - kAllocationRedZoneSize; }
bool Contains(uword addr) const { return memory_->Contains(addr); }
void WriteProtect(bool read_only) {
memory_->Protect(read_only ? VirtualMemory::kReadOnly
: VirtualMemory::kReadWrite);
}
NewPage* next() const { return next_; }
void set_next(NewPage* next) { next_ = next; }
Thread* owner() const { return owner_; }
uword object_start() const { return start() + ObjectStartOffset(); }
uword object_end() const { return owner_ != nullptr ? owner_->top() : top_; }
intptr_t used() const { return object_end() - object_start(); }
void VisitObjects(ObjectVisitor* visitor) const {
uword addr = object_start();
uword end = object_end();
while (addr < end) {
ObjectPtr obj = UntaggedObject::FromAddr(addr);
visitor->VisitObject(obj);
addr += obj->untag()->HeapSize();
}
}
void VisitObjectPointers(ObjectPointerVisitor* visitor) const {
uword addr = object_start();
uword end = object_end();
while (addr < end) {
ObjectPtr obj = UntaggedObject::FromAddr(addr);
intptr_t size = obj->untag()->VisitPointers(visitor);
addr += size;
}
}
static intptr_t ObjectStartOffset() {
return Utils::RoundUp(sizeof(NewPage), kObjectAlignment) +
kNewObjectAlignmentOffset;
}
static NewPage* Of(ObjectPtr obj) {
ASSERT(obj->IsHeapObject());
ASSERT(obj->IsNewObject());
return Of(static_cast<uword>(obj));
}
static NewPage* Of(uword addr) {
return reinterpret_cast<NewPage*>(addr & kNewPageMask);
}
// Remember the limit to which objects have been copied.
void RecordSurvivors() { survivor_end_ = object_end(); }
// Move survivor end to the end of the to_ space, making all surviving
// objects candidates for promotion next time.
void EarlyTenure() { survivor_end_ = end_; }
uword promo_candidate_words() const {
return (survivor_end_ - object_start()) / kWordSize;
}
void Acquire(Thread* thread) {
ASSERT(owner_ == nullptr);
owner_ = thread;
thread->set_top(top_);
thread->set_end(end_);
}
void Release(Thread* thread) {
ASSERT(owner_ == thread);
owner_ = nullptr;
top_ = thread->top();
thread->set_top(0);
thread->set_end(0);
}
void Release() {
if (owner_ != nullptr) {
Release(owner_);
}
}
uword TryAllocateGC(intptr_t size) {
ASSERT(owner_ == nullptr);
uword result = top_;
uword new_top = result + size;
if (LIKELY(new_top <= end_)) {
top_ = new_top;
return result;
}
return 0;
}
void Unallocate(uword addr, intptr_t size) {
ASSERT((addr + size) == top_);
top_ -= size;
}
bool IsSurvivor(uword raw_addr) const { return raw_addr < survivor_end_; }
bool IsResolved() const { return top_ == resolved_top_; }
private:
VirtualMemory* memory_;
NewPage* next_;
// The thread using this page for allocation, otherwise NULL.
Thread* owner_;
// The address of the next allocation. If owner is non-NULL, this value is
// stale and the current value is at owner->top_. Called "NEXT" in the
// original Cheney paper.
uword top_;
// The address after the last allocatable byte in this page.
uword end_;
// Objects below this address have survived a scavenge.
uword survivor_end_;
// A pointer to the first unprocessed object. Resolution completes when this
// value meets the allocation top. Called "SCAN" in the original Cheney paper.
uword resolved_top_;
template <bool>
friend class ScavengerVisitorBase;
DISALLOW_ALLOCATION();
DISALLOW_IMPLICIT_CONSTRUCTORS(NewPage);
};
class SemiSpace {
public:
static void Init();
static void ClearCache();
static void Cleanup();
static intptr_t CachedSize();
explicit SemiSpace(intptr_t max_capacity_in_words);
~SemiSpace();
NewPage* TryAllocatePageLocked(bool link);
Page* TryAllocatePageLocked(bool link);
bool Contains(uword addr) const;
void WriteProtect(bool read_only);
intptr_t used_in_words() const {
intptr_t size = 0;
for (const NewPage* p = head_; p != nullptr; p = p->next()) {
for (const Page* p = head_; p != nullptr; p = p->next()) {
size += p->used();
}
return size >> kWordSizeLog2;
@@ -196,9 +50,9 @@ class SemiSpace {
intptr_t capacity_in_words() const { return capacity_in_words_; }
intptr_t max_capacity_in_words() const { return max_capacity_in_words_; }
NewPage* head() const { return head_; }
Page* head() const { return head_; }
void AddList(NewPage* head, NewPage* tail);
void AddList(Page* head, Page* tail);
private:
// Size of NewPages in this semi-space.
@@ -207,8 +61,8 @@ class SemiSpace {
// Size of NewPages before we trigger a scavenge.
intptr_t max_capacity_in_words_;
NewPage* head_ = nullptr;
NewPage* tail_ = nullptr;
Page* head_ = nullptr;
Page* tail_ = nullptr;
};
// Statistics for a particular scavenge.
@@ -372,7 +226,9 @@ class Scavenger {
return max_pool_size > 0 ? max_pool_size : 1;
}
NewPage* head() const { return to_->head(); }
Page* head() const {
return to_->head();
}
private:
// Ids for time and data records in Heap::GCStats.
runtime/vm/heap/sweeper.cc
+5 -6
View File
@@ -15,20 +15,20 @@
namespace dart {
bool GCSweeper::SweepPage(OldPage* page, FreeList* freelist, bool locked) {
bool GCSweeper::SweepPage(Page* page, FreeList* freelist, bool locked) {
ASSERT(!page->is_image_page());
// Keep track whether this page is still in use.
intptr_t used_in_bytes = 0;
bool is_executable = (page->type() == OldPage::kExecutable);
bool is_executable = (page->type() == Page::kExecutable);
uword start = page->object_start();
uword end = page->object_end();
uword current = start;
while (current < end) {
ObjectPtr raw_obj = UntaggedObject::FromAddr(current);
ASSERT(OldPage::Of(raw_obj) == page);
ASSERT(Page::Of(raw_obj) == page);
// These acquire operations balance release operations in array
// truncaton, ensuring the writes creating the filler object are ordered
// before the writes inserting the filler object into the freelist.
@@ -76,16 +76,15 @@ bool GCSweeper::SweepPage(OldPage* page, FreeList* freelist, bool locked) {
}
ASSERT(current == end);
page->set_used_in_bytes(used_in_bytes);
return used_in_bytes != 0; // In use.
}
intptr_t GCSweeper::SweepLargePage(OldPage* page) {
intptr_t GCSweeper::SweepLargePage(Page* page) {
ASSERT(!page->is_image_page());
intptr_t words_to_end = 0;
ObjectPtr raw_obj = UntaggedObject::FromAddr(page->object_start());
ASSERT(OldPage::Of(raw_obj) == page);
ASSERT(Page::Of(raw_obj) == page);
if (raw_obj->untag()->IsMarked()) {
raw_obj->untag()->ClearMarkBit();
words_to_end = (raw_obj->untag()->HeapSize() >> kWordSizeLog2);
runtime/vm/heap/sweeper.h
+3 -3
View File
@@ -12,7 +12,7 @@ namespace dart {
// Forward declarations.
class FreeList;
class Heap;
class OldPage;
class Page;
class IsolateGroup;
class PageSpace;
@@ -28,11 +28,11 @@ class GCSweeper {
// pre-locked is indicated by the locked parameter.
// Returns true if the page is in use. Freelist is untouched if page is not
// in use.
bool SweepPage(OldPage* page, FreeList* freelist, bool locked);
bool SweepPage(Page* page, FreeList* freelist, bool locked);
// Returns the number of words from page->object_start() to the end of the
// last marked object.
intptr_t SweepLargePage(OldPage* page);
intptr_t SweepLargePage(Page* page);
// Sweep the large and regular sized data pages.
static void SweepConcurrent(IsolateGroup* isolate_group);
runtime/vm/heap/verifier.cc
+2 -2
View File
@@ -50,7 +50,7 @@ void VerifyPointersVisitor::VisitPointers(ObjectPtr* first, ObjectPtr* last) {
if (raw_obj->IsHeapObject()) {
if (!allocated_set_->Contains(raw_obj)) {
if (raw_obj->IsInstructions() &&
allocated_set_->Contains(OldPage::ToWritable(raw_obj))) {
allocated_set_->Contains(Page::ToWritable(raw_obj))) {
continue;
}
uword raw_addr = UntaggedObject::ToAddr(raw_obj);
@@ -68,7 +68,7 @@ void VerifyPointersVisitor::VisitCompressedPointers(uword heap_base,
if (raw_obj->IsHeapObject()) {
if (!allocated_set_->Contains(raw_obj)) {
if (raw_obj->IsInstructions() &&
allocated_set_->Contains(OldPage::ToWritable(raw_obj))) {
allocated_set_->Contains(Page::ToWritable(raw_obj))) {
continue;
}
uword raw_addr = UntaggedObject::ToAddr(raw_obj);
runtime/vm/image_snapshot.cc
+20 -9
View File
@@ -685,7 +685,9 @@ void ImageWriter::WriteText(bool vm) {
const bool bare_instruction_payloads = FLAG_precompiled_mode;
// Start snapshot at page boundary.
if (!EnterSection(ProgramSection::Text, vm, ImageWriter::kTextAlignment)) {
intptr_t alignment_padding = 0;
if (!EnterSection(ProgramSection::Text, vm, ImageWriter::kTextAlignment,
&alignment_padding)) {
return;
}
@@ -698,7 +700,7 @@ void ImageWriter::WriteText(bool vm) {
#endif
// This head also provides the gap to make the instructions snapshot
// look like a OldPage.
// look like a Page.
const intptr_t image_size = Utils::RoundUp(
next_text_offset_, compiler::target::ObjectAlignment::kObjectAlignment);
text_offset += WriteTargetWord(image_size);
@@ -720,7 +722,8 @@ void ImageWriter::WriteText(bool vm) {
if (profile_writer_ != nullptr) {
profile_writer_->SetObjectTypeAndName(parent_id, image_type_,
instructions_symbol);
profile_writer_->AttributeBytesTo(parent_id, Image::kHeaderSize);
profile_writer_->AttributeBytesTo(
parent_id, ImageWriter::kTextAlignment + alignment_padding);
profile_writer_->AddRoot(parent_id);
}
@@ -1319,7 +1322,8 @@ void AssemblyImageWriter::WriteROData(NonStreamingWriteStream* clustered_stream,
bool AssemblyImageWriter::EnterSection(ProgramSection section,
bool vm,
intptr_t alignment) {
intptr_t alignment,
intptr_t* alignment_padding) {
ASSERT(FLAG_precompiled_mode);
ASSERT(current_section_symbol_ == nullptr);
ASSERT(current_symbols_ == nullptr);
@@ -1363,7 +1367,10 @@ bool AssemblyImageWriter::EnterSection(ProgramSection section,
if (global_symbol) {
assembly_stream_->Printf(".globl %s\n", current_section_symbol_);
}
Align(alignment);
intptr_t padding = Align(alignment);
if (alignment_padding != nullptr) {
*alignment_padding = padding;
}
assembly_stream_->Printf("%s:\n", current_section_symbol_);
return true;
}
@@ -1652,7 +1659,8 @@ void BlobImageWriter::WriteROData(NonStreamingWriteStream* clustered_stream,
bool BlobImageWriter::EnterSection(ProgramSection section,
bool vm,
intptr_t alignment) {
intptr_t alignment,
intptr_t* alignment_padding) {
#if defined(DART_PRECOMPILER)
ASSERT_EQUAL(elf_ != nullptr, FLAG_precompiled_mode);
ASSERT(current_relocations_ == nullptr);
@@ -1691,7 +1699,10 @@ bool BlobImageWriter::EnterSection(ProgramSection section,
return false;
}
current_section_symbol_ = SectionSymbol(section, vm);
current_section_stream_->Align(alignment);
intptr_t padding = current_section_stream_->Align(alignment);
if (alignment_padding != nullptr) {
*alignment_padding = padding;
}
return true;
}
@@ -1769,8 +1780,8 @@ ImageReader::ImageReader(const uint8_t* data_image,
instructions_image_(ASSERT_NOTNULL(instructions_image)) {}
ApiErrorPtr ImageReader::VerifyAlignment() const {
if (!Utils::IsAligned(data_image_, kObjectAlignment) ||
!Utils::IsAligned(instructions_image_, kMaxObjectAlignment)) {
if (!Utils::IsAligned(data_image_, kObjectStartAlignment) ||
!Utils::IsAligned(instructions_image_, kObjectStartAlignment)) {
return ApiError::New(
String::Handle(String::New("Snapshot is misaligned", Heap::kOld)),
Heap::kOld);
runtime/vm/image_snapshot.h
+13 -10
View File
@@ -44,7 +44,7 @@ class Image : ValueObject {
: raw_memory_(raw_memory),
snapshot_size_(FieldValue(raw_memory, HeaderField::ImageSize)),
extra_info_(ExtraInfo(raw_memory_, snapshot_size_)) {
ASSERT(Utils::IsAligned(raw_memory, kMaxObjectAlignment));
ASSERT(Utils::IsAligned(raw_memory, kObjectStartAlignment));
}
// Even though an Image is read-only memory, we must return a void* here.
@@ -117,12 +117,12 @@ class Image : ValueObject {
//
// Note: Image::kHeaderSize is _not_ an architecture-dependent constant,
// and so there is no compiler::target::Image::kHeaderSize.
static constexpr intptr_t kHeaderSize = kMaxObjectAlignment;
static constexpr intptr_t kHeaderSize = kObjectStartAlignment;
// Explicitly double-checking kHeaderSize is never changed. Increasing the
// Image header size would mean objects would not start at a place expected
// by parts of the VM (like the GC) that use Image pages as HeapPages.
static_assert(kHeaderSize == kMaxObjectAlignment,
"Image page cannot be used as HeapPage");
// by parts of the VM (like the GC) that use Image pages as Pages.
static_assert(kHeaderSize == kObjectStartAlignment,
"Image page cannot be used as Page");
// Make sure that the number of fields in the Image header fit both on the
// host and target architectures.
static_assert(kHeaderFields * kWordSize <= kHeaderSize,
@@ -243,10 +243,10 @@ class ImageWriter : public ValueObject {
// BSS sections contain word-sized data.
static constexpr intptr_t kBssAlignment = compiler::target::kWordSize;
// ROData sections contain objects wrapped in an Image object.
static constexpr intptr_t kRODataAlignment = kMaxObjectAlignment;
static constexpr intptr_t kRODataAlignment = kObjectStartAlignment;
// Text sections contain objects (even in bare instructions mode) wrapped
// in an Image object.
static constexpr intptr_t kTextAlignment = kMaxObjectAlignment;
static constexpr intptr_t kTextAlignment = kObjectStartAlignment;
void ResetOffsets() {
next_data_offset_ = Image::kHeaderSize;
@@ -425,7 +425,8 @@ class ImageWriter : public ValueObject {
// this section should not be written.
virtual bool EnterSection(ProgramSection name,
bool vm,
intptr_t alignment) = 0;
intptr_t alignment,
intptr_t* alignment_padding = nullptr) = 0;
// Marks the exit from a particular ProgramSection, allowing subclasses to
// do any post-writing work.
virtual void ExitSection(ProgramSection name, bool vm, intptr_t size) = 0;
@@ -604,7 +605,8 @@ class AssemblyImageWriter : public ImageWriter {
virtual bool EnterSection(ProgramSection section,
bool vm,
intptr_t alignment);
intptr_t alignment,
intptr_t* alignment_padding = nullptr);
virtual void ExitSection(ProgramSection name, bool vm, intptr_t size);
virtual intptr_t WriteTargetWord(word value);
virtual intptr_t WriteBytes(const void* bytes, intptr_t size);
@@ -655,7 +657,8 @@ class BlobImageWriter : public ImageWriter {
virtual bool EnterSection(ProgramSection section,
bool vm,
intptr_t alignment);
intptr_t alignment,
intptr_t* alignment_padding = nullptr);
virtual void ExitSection(ProgramSection name, bool vm, intptr_t size);
virtual intptr_t WriteTargetWord(word value);
virtual intptr_t WriteBytes(const void* bytes, intptr_t size);
runtime/vm/message.h
-3
View File
@@ -21,9 +21,6 @@ namespace dart {
class JSONStream;
class PersistentHandle;
class OldPage;
class WeakTable;
class FreeList;
class Message {
public:
runtime/vm/object.cc
+1 -1
View File
@@ -17483,7 +17483,7 @@ CodePtr Code::FinalizeCode(FlowGraphCompiler* compiler,
if (FLAG_write_protect_code) {
uword address = UntaggedObject::ToAddr(instrs.ptr());
// Check if a dual mapping exists.
instrs = Instructions::RawCast(OldPage::ToExecutable(instrs.ptr()));
instrs = Instructions::RawCast(Page::ToExecutable(instrs.ptr()));
uword exec_address = UntaggedObject::ToAddr(instrs.ptr());
const bool use_dual_mapping = exec_address != address;
ASSERT(use_dual_mapping == FLAG_dual_map_code);
runtime/vm/object.h
+1 -1
View File
@@ -5450,7 +5450,7 @@ class Instructions : public Object {
static const intptr_t kMaxElements =
(kMaxInt32 - (sizeof(UntaggedInstructions) + sizeof(UntaggedObject) +
(2 * kMaxObjectAlignment)));
(2 * kObjectStartAlignment)));
// Currently, we align bare instruction payloads on 4 byte boundaries.
//
runtime/vm/object_graph.cc
+6 -6
View File
@@ -706,7 +706,7 @@ intptr_t ObjectGraph::InboundReferences(Object* obj, const Array& references) {
return visitor.length();
}
// Each OldPage is divided into blocks of size kBlockSize. Each object belongs
// Each Page is divided into blocks of size kBlockSize. Each object belongs
// to the block containing its header word.
// When generating a heap snapshot, we assign objects sequential ids in heap
// iteration order. A bitvector is computed that indicates the number of objects
@@ -763,7 +763,7 @@ class CountingPage {
}
CountingBlock* BlockFor(uword addr) {
intptr_t page_offset = addr & ~kOldPageMask;
intptr_t page_offset = addr & ~kPageMask;
intptr_t block_number = page_offset / kBlockSize;
ASSERT(block_number >= 0);
ASSERT(block_number <= kBlocksPerPage);
@@ -816,7 +816,7 @@ void HeapSnapshotWriter::SetupCountingPages() {
image_page_ranges_[i].size = 0;
}
intptr_t next_offset = 0;
OldPage* image_page =
Page* image_page =
Dart::vm_isolate_group()->heap()->old_space()->image_pages_;
while (image_page != NULL) {
RELEASE_ASSERT(next_offset <= kMaxImagePages);
@@ -836,7 +836,7 @@ void HeapSnapshotWriter::SetupCountingPages() {
next_offset++;
}
OldPage* page = isolate_group()->heap()->old_space()->pages_;
Page* page = isolate_group()->heap()->old_space()->pages_;
while (page != NULL) {
page->forwarding_page();
CountingPage* counting_page =
@@ -860,7 +860,7 @@ bool HeapSnapshotWriter::OnImagePage(ObjectPtr obj) const {
CountingPage* HeapSnapshotWriter::FindCountingPage(ObjectPtr obj) const {
if (obj->IsOldObject() && !OnImagePage(obj)) {
// On a regular or large page.
OldPage* page = OldPage::Of(obj);
Page* page = Page::Of(obj);
return reinterpret_cast<CountingPage*>(page->forwarding_page());
}
@@ -888,7 +888,7 @@ intptr_t HeapSnapshotWriter::GetObjectId(ObjectPtr obj) const {
if (FLAG_write_protect_code && obj->IsInstructions() && !OnImagePage(obj)) {
// A non-writable alias mapping may exist for instruction pages.
obj = OldPage::ToWritable(obj);
obj = Page::ToWritable(obj);
}
CountingPage* counting_page = FindCountingPage(obj);
runtime/vm/object_test.cc
+1 -1
View File
@@ -2759,7 +2759,7 @@ ISOLATE_UNIT_TEST_CASE_WITH_EXPECTATION(CodeExecutability, "Crash") {
Object::Handle(DartEntry::InvokeFunction(function, Array::empty_array()));
EXPECT_EQ(1, Smi::Cast(result).Value());
// Switch to the writeable but non-executable view of the instructions.
instructions ^= OldPage::ToWritable(instructions.ptr());
instructions ^= Page::ToWritable(instructions.ptr());
payload_start = instructions.PayloadStart();
EXPECT_EQ(instructions.ptr(), Instructions::FromPayloadStart(payload_start));
// Hook up Code and Instructions objects.
runtime/vm/pointer_tagging.h
+4 -3
View File
@@ -71,9 +71,10 @@ static constexpr intptr_t kTrueOffsetFromNull =
static constexpr intptr_t kFalseOffsetFromNull =
HostObjectAlignment::kFalseOffsetFromNull;
// The largest value of kObjectAlignment across all configurations.
static constexpr intptr_t kMaxObjectAlignment = 16;
COMPILE_ASSERT(kMaxObjectAlignment >= kObjectAlignment);
static constexpr intptr_t kObjectStartAlignment = 64;
COMPILE_ASSERT(kObjectStartAlignment >= kObjectAlignment);
COMPILE_ASSERT(kObjectStartAlignment >= 2 * kBoolValueMask);
COMPILE_ASSERT(kObjectStartAlignment >= 2 * kBoolVsNullMask);
// On all targets heap pointers are tagged by set least significant bit.
//
runtime/vm/raw_object.cc
+3 -3
View File
@@ -760,12 +760,12 @@ intptr_t UntaggedImmutableLinkedHashSet::VisitImmutableLinkedHashSetPointers(
}
void UntaggedObject::RememberCard(ObjectPtr const* slot) {
OldPage::Of(static_cast<ObjectPtr>(this))->RememberCard(slot);
Page::Of(static_cast<ObjectPtr>(this))->RememberCard(slot);
}
#if defined(DART_COMPRESSED_POINTERS)
void UntaggedObject::RememberCard(CompressedObjectPtr const* slot) {
OldPage::Of(static_cast<ObjectPtr>(this))->RememberCard(slot);
Page::Of(static_cast<ObjectPtr>(this))->RememberCard(slot);
}
#endif
@@ -777,7 +777,7 @@ DEFINE_LEAF_RUNTIME_ENTRY(void,
ObjectPtr object = static_cast<ObjectPtr>(object_in);
ASSERT(object->IsOldObject());
ASSERT(object->untag()->IsCardRemembered());
OldPage::Of(object)->RememberCard(slot);
Page::Of(object)->RememberCard(slot);
}
END_LEAF_RUNTIME_ENTRY
runtime/vm/raw_object.h
+1 -1
View File
@@ -3108,7 +3108,7 @@ class UntaggedArray : public UntaggedInstance {
friend class ReversePc;
template <typename Table, bool kAllCanonicalObjectsAreIncludedIntoSet>
friend class CanonicalSetDeserializationCluster;
friend class OldPage;
friend class Page;
friend class FastObjectCopy; // For initializing fields.
friend void UpdateLengthField(intptr_t, ObjectPtr, ObjectPtr); // length_
};
runtime/vm/service.cc
+4 -6
View File
@@ -4592,10 +4592,8 @@ static void AddVMMappings(JSONArray* rss_children) {
// [anon:dart-*] - as labelled (Android)
if ((strcmp(property, "Rss:") == 0) && (size != 0) &&
(strcmp(path, "(deleted)") != 0) && (strcmp(path, "[heap]") != 0) &&
(strcmp(path, "") != 0) &&
(strcmp(path, "[anon:dart-newspace]") != 0) &&
(strcmp(path, "[anon:dart-oldspace]") != 0) &&
(strcmp(path, "[anon:dart-codespace]") != 0) &&
(strcmp(path, "") != 0) && (strcmp(path, "[anon:dart-heap]") != 0) &&
(strcmp(path, "[anon:dart-code]") != 0) &&
(strcmp(path, "[anon:dart-profiler]") != 0) &&
(strcmp(path, "[anon:dart-timeline]") != 0) &&
(strcmp(path, "[anon:dart-zone]") != 0)) {
@@ -4680,9 +4678,9 @@ static intptr_t GetProcessMemoryUsageHelper(JSONStream* js) {
{
JSONObject semi(&vm_children);
semi.AddProperty("name", "SemiSpace Cache");
semi.AddProperty("name", "Page Cache");
semi.AddProperty("description", "Cached heap regions");
intptr_t size = SemiSpace::CachedSize();
intptr_t size = Page::CachedSize();
vm_size += size;
semi.AddProperty64("size", size);
JSONArray(&semi, "children");
runtime/vm/snapshot.h
+1 -1
View File
@@ -83,7 +83,7 @@ class Snapshot {
if (!IncludesCode(kind())) {
return NULL;
}
uword offset = Utils::RoundUp(length(), kMaxObjectAlignment);
uword offset = Utils::RoundUp(length(), kObjectStartAlignment);
return Addr() + offset;
}
runtime/vm/virtual_memory_compressed.cc
+14 -15
View File
@@ -41,7 +41,7 @@ void VirtualMemoryCompressedHeap::Init(void* compressed_heap_region,
memset(pages_, 0, kCompressedHeapBitmapSize);
ASSERT(size > 0);
ASSERT(size <= kCompressedHeapSize);
for (intptr_t page_id = size / kCompressedHeapPageSize;
for (intptr_t page_id = size / kCompressedPageSize;
page_id < kCompressedHeapNumPages; page_id++) {
SetPageUsed(page_id);
}
@@ -50,8 +50,8 @@ void VirtualMemoryCompressedHeap::Init(void* compressed_heap_region,
ASSERT(base_ != 0);
ASSERT(size_ != 0);
ASSERT(size_ <= kCompressedHeapSize);
ASSERT(Utils::IsAligned(base_, kCompressedHeapPageSize));
ASSERT(Utils::IsAligned(size_, kCompressedHeapPageSize));
ASSERT(Utils::IsAligned(base_, kCompressedPageSize));
ASSERT(Utils::IsAligned(size_, kCompressedPageSize));
// base_ is not necessarily 4GB-aligned, because on some systems we can't make
// a large enough reservation to guarantee it. Instead, we have only the
// weaker property that all addresses in [base_, base_ + size_) have the same
@@ -78,11 +78,10 @@ void* VirtualMemoryCompressedHeap::GetRegion() {
MemoryRegion VirtualMemoryCompressedHeap::Allocate(intptr_t size,
intptr_t alignment) {
ASSERT(alignment <= kCompressedHeapAlignment);
const intptr_t allocated_size = Utils::RoundUp(size, kCompressedHeapPageSize);
uword pages = allocated_size / kCompressedHeapPageSize;
uword page_alignment = alignment > kCompressedHeapPageSize
? alignment / kCompressedHeapPageSize
: 1;
const intptr_t allocated_size = Utils::RoundUp(size, kCompressedPageSize);
uword pages = allocated_size / kCompressedPageSize;
uword page_alignment =
alignment > kCompressedPageSize ? alignment / kCompressedPageSize : 1;
MutexLocker ml(mutex_);
// Find a gap with enough empty pages, using the bitmap. Note that reading
@@ -105,7 +104,7 @@ MemoryRegion VirtualMemoryCompressedHeap::Allocate(intptr_t size,
// Make sure we're not trying to allocate past the end of the heap.
uword end = page_id + pages;
if (end > kCompressedHeapSize / kCompressedHeapPageSize) {
if (end > kCompressedHeapSize / kCompressedPageSize) {
return MemoryRegion();
}
@@ -120,19 +119,19 @@ MemoryRegion VirtualMemoryCompressedHeap::Allocate(intptr_t size,
++minimum_free_page_id_;
}
uword address = base_ + page_id * kCompressedHeapPageSize;
ASSERT(Utils::IsAligned(address, kCompressedHeapPageSize));
uword address = base_ + page_id * kCompressedPageSize;
ASSERT(Utils::IsAligned(address, kCompressedPageSize));
return MemoryRegion(reinterpret_cast<void*>(address), allocated_size);
}
void VirtualMemoryCompressedHeap::Free(void* address, intptr_t size) {
uword start = reinterpret_cast<uword>(address);
ASSERT(Utils::IsAligned(start, kCompressedHeapPageSize));
ASSERT(Utils::IsAligned(size, kCompressedHeapPageSize));
ASSERT(Utils::IsAligned(start, kCompressedPageSize));
ASSERT(Utils::IsAligned(size, kCompressedPageSize));
MutexLocker ml(mutex_);
ASSERT(start >= base_);
uword page_id = (start - base_) / kCompressedHeapPageSize;
uword end = page_id + size / kCompressedHeapPageSize;
uword page_id = (start - base_) / kCompressedPageSize;
uword end = page_id + size / kCompressedPageSize;
for (uword i = page_id; i < end; ++i) {
ClearPageUsed(i);
}
runtime/vm/virtual_memory_compressed.h
+2 -2
View File
@@ -14,9 +14,9 @@ namespace dart {
#if defined(DART_COMPRESSED_POINTERS)
static constexpr intptr_t kCompressedHeapSize = 4 * GB;
static constexpr intptr_t kCompressedHeapAlignment = 4 * GB;
static constexpr intptr_t kCompressedHeapPageSize = kOldPageSize;
static constexpr intptr_t kCompressedPageSize = kPageSize;
static constexpr intptr_t kCompressedHeapNumPages =
kCompressedHeapSize / kOldPageSize;
kCompressedHeapSize / kPageSize;
static constexpr intptr_t kCompressedHeapBitmapSize =
kCompressedHeapNumPages / 8;
runtime/vm/virtual_memory_posix.cc
+5 -5
View File
@@ -133,10 +133,10 @@ void VirtualMemory::Init() {
// not cross a 4GB boundary. The subregion itself is not necessarily
// 4GB-aligned.
for (size_t allocated_size = kCompressedHeapSize + kCompressedHeapAlignment;
allocated_size >= kCompressedHeapPageSize; allocated_size >>= 1) {
allocated_size >= kCompressedPageSize; allocated_size >>= 1) {
void* address = GenericMapAligned(
nullptr, PROT_NONE, allocated_size, kCompressedHeapPageSize,
allocated_size + kCompressedHeapPageSize,
nullptr, PROT_NONE, allocated_size, kCompressedPageSize,
allocated_size + kCompressedPageSize,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE);
if (address == nullptr) continue;
@@ -177,7 +177,7 @@ void VirtualMemory::Init() {
// Also detect for missing support of memfd_create syscall.
if (FLAG_dual_map_code) {
intptr_t size = PageSize();
intptr_t alignment = kOldPageSize;
intptr_t alignment = kPageSize;
bool executable = true;
bool compressed = false;
VirtualMemory* vm =
@@ -206,7 +206,7 @@ void VirtualMemory::Init() {
int count = fscanf(fp, "%zu", &max_map_count);
fclose(fp);
if (count == 1) {
size_t max_heap_pages = FLAG_old_gen_heap_size * MB / kOldPageSize;
size_t max_heap_pages = FLAG_old_gen_heap_size * MB / kPageSize;
if (max_map_count < max_heap_pages) {
OS::PrintErr(
"warning: vm.max_map_count (%zu) is not large enough to support "
runtime/vm/virtual_memory_test.cc
+1 -1
View File
@@ -52,7 +52,7 @@ VM_UNIT_TEST_CASE(AllocateVirtualMemory) {
}
VM_UNIT_TEST_CASE(AllocateAlignedVirtualMemory) {
intptr_t kHeapPageSize = kOldPageSize;
intptr_t kHeapPageSize = kPageSize;
intptr_t kVirtualPageSize = 4096;
intptr_t kIterations = kHeapPageSize / kVirtualPageSize;