diff --git a/runtime/platform/utils.h b/runtime/platform/utils.h index 57bb31ae060..baa4ff4888a 100644 --- a/runtime/platform/utils.h +++ b/runtime/platform/utils.h @@ -387,29 +387,6 @@ class Utils { return ((mask >> position) & 1) != 0; } - // Decode integer in SLEB128 format from |data| and update |byte_index|. - template - static ValueType DecodeSLEB128(const uint8_t* data, - const intptr_t data_length, - intptr_t* byte_index) { - using Unsigned = typename std::make_unsigned::type; - ASSERT(*byte_index < data_length); - uword shift = 0; - Unsigned value = 0; - uint8_t part = 0; - do { - part = data[(*byte_index)++]; - value |= static_cast(part & 0x7f) << shift; - shift += 7; - } while ((part & 0x80) != 0); - - if ((shift < (sizeof(ValueType) * CHAR_BIT)) && ((part & 0x40) != 0)) { - const Unsigned kMax = std::numeric_limits::max(); - value |= static_cast(kMax << shift); - } - return static_cast(value); - } - static char* StrError(int err, char* buffer, size_t bufsize); // Not all platforms support strndup. diff --git a/runtime/vm/bitmap.cc b/runtime/vm/bitmap.cc index d5e25dd2be4..334c46b969c 100644 --- a/runtime/vm/bitmap.cc +++ b/runtime/vm/bitmap.cc @@ -77,7 +77,7 @@ void BitmapBuilder::Print() const { } } -void BitmapBuilder::AppendAsBytesTo(GrowableArray* bytes) const { +void BitmapBuilder::AppendAsBytesTo(BaseWriteStream* stream) const { // Early return if there are no bits in the payload to copy. if (Length() == 0) return; @@ -94,19 +94,20 @@ void BitmapBuilder::AppendAsBytesTo(GrowableArray* bytes) const { payload_size = total_size; extra_size = 0; } +#if defined(DEBUG) + // Make sure any bits in the payload beyond the bit length if we're not + // appending trailing zeroes are cleared to ensure deterministic snapshots. + if (extra_size == 0 && Length() % kBitsPerByte != 0) { + const int8_t mask = (1 << (Length() % kBitsPerByte)) - 1; + ASSERT_EQUAL(data_[payload_size - 1], (data_[payload_size - 1] & mask)); + } +#endif for (intptr_t i = 0; i < payload_size; i++) { - bytes->Add(data_[i]); + stream->WriteByte(data_[i]); } for (intptr_t i = 0; i < extra_size; i++) { - bytes->Add(0U); + stream->WriteByte(0U); } - // Make sure any bits in the payload beyond the bit length are cleared to - // ensure deterministic snapshots. -#if defined(DEBUG) - if (Length() % kBitsPerByte == 0) return; - const int8_t mask = (1 << (Length() % kBitsPerByte)) - 1; - ASSERT(bytes->Last() == (bytes->Last() & mask)); -#endif } bool BitmapBuilder::GetBit(intptr_t bit_offset) const { diff --git a/runtime/vm/bitmap.h b/runtime/vm/bitmap.h index 878a834b608..ed0800943d6 100644 --- a/runtime/vm/bitmap.h +++ b/runtime/vm/bitmap.h @@ -6,7 +6,7 @@ #define RUNTIME_VM_BITMAP_H_ #include "vm/allocation.h" -#include "vm/growable_array.h" +#include "vm/datastream.h" #include "vm/thread_state.h" #include "vm/zone.h" @@ -44,7 +44,7 @@ class BitmapBuilder : public ZoneAllocated { void SetRange(intptr_t min, intptr_t max, bool value); void Print() const; - void AppendAsBytesTo(GrowableArray* bytes) const; + void AppendAsBytesTo(BaseWriteStream* stream) const; private: static const intptr_t kInitialSizeInBytes = 16; diff --git a/runtime/vm/bitmap_test.cc b/runtime/vm/bitmap_test.cc index 2c7f3d9e064..c3cd20380f3 100644 --- a/runtime/vm/bitmap_test.cc +++ b/runtime/vm/bitmap_test.cc @@ -16,8 +16,8 @@ namespace dart { static const uint32_t kTestPcOffset = 0x4; static const intptr_t kTestSpillSlotBitCount = 0; -static CompressedStackMapsPtr MapsFromBuilder(BitmapBuilder* bmap) { - CompressedStackMapsBuilder builder; +static CompressedStackMapsPtr MapsFromBuilder(Zone* zone, BitmapBuilder* bmap) { + CompressedStackMapsBuilder builder(zone); builder.AddEntry(kTestPcOffset, bmap, kTestSpillSlotBitCount); return builder.Finalize(); } @@ -51,8 +51,9 @@ ISOLATE_UNIT_TEST_CASE(BitmapBuilder) { } // Create a CompressedStackMaps object and verify its contents. - const auto& maps1 = CompressedStackMaps::Handle(MapsFromBuilder(builder1)); - CompressedStackMapsIterator it1(maps1); + const auto& maps1 = CompressedStackMaps::Handle( + thread->zone(), MapsFromBuilder(thread->zone(), builder1)); + CompressedStackMaps::Iterator it1(thread, maps1); EXPECT(it1.MoveNext()); EXPECT_EQ(kTestPcOffset, it1.pc_offset()); @@ -83,8 +84,9 @@ ISOLATE_UNIT_TEST_CASE(BitmapBuilder) { EXPECT(!builder1->Get(i)); } - const auto& maps2 = CompressedStackMaps::Handle(MapsFromBuilder(builder1)); - CompressedStackMapsIterator it2(maps2); + const auto& maps2 = CompressedStackMaps::Handle( + thread->zone(), MapsFromBuilder(thread->zone(), builder1)); + CompressedStackMaps::Iterator it2(thread, maps2); EXPECT(it2.MoveNext()); EXPECT_EQ(kTestPcOffset, it2.pc_offset()); diff --git a/runtime/vm/code_descriptors.cc b/runtime/vm/code_descriptors.cc index 2dc55844ec5..edce815b09c 100644 --- a/runtime/vm/code_descriptors.cc +++ b/runtime/vm/code_descriptors.cc @@ -35,14 +35,13 @@ void DescriptorList::AddDescriptor(PcDescriptorsLayout::Kind kind, PcDescriptorsLayout::KindAndMetadata::Encode(kind, try_index, yield_index); - PcDescriptors::EncodeInteger(&encoded_data_, kind_and_metadata); - PcDescriptors::EncodeInteger(&encoded_data_, pc_offset - prev_pc_offset); + encoded_data_.WriteSLEB128(kind_and_metadata); + encoded_data_.WriteSLEB128(pc_offset - prev_pc_offset); prev_pc_offset = pc_offset; if (!FLAG_precompiled_mode) { - PcDescriptors::EncodeInteger(&encoded_data_, deopt_id - prev_deopt_id); - PcDescriptors::EncodeInteger(&encoded_data_, - token_pos.value() - prev_token_pos); + encoded_data_.WriteSLEB128(deopt_id - prev_deopt_id); + encoded_data_.WriteSLEB128(token_pos.value() - prev_token_pos); prev_deopt_id = deopt_id; prev_token_pos = token_pos.value(); } @@ -50,22 +49,11 @@ void DescriptorList::AddDescriptor(PcDescriptorsLayout::Kind kind, } PcDescriptorsPtr DescriptorList::FinalizePcDescriptors(uword entry_point) { - if (encoded_data_.length() == 0) { + if (encoded_data_.bytes_written() == 0) { return Object::empty_descriptors().raw(); } - return PcDescriptors::New(&encoded_data_); -} - -// Encode unsigned integer |value| in LEB128 format and store into |data|. -void CompressedStackMapsBuilder::EncodeLEB128(GrowableArray* data, - uintptr_t value) { - while (true) { - uint8_t part = value & 0x7f; - value >>= 7; - if (value != 0) part |= 0x80; - data->Add(part); - if (value == 0) break; - } + return PcDescriptors::New(encoded_data_.buffer(), + encoded_data_.bytes_written()); } void CompressedStackMapsBuilder::AddEntry(intptr_t pc_offset, @@ -74,179 +62,22 @@ void CompressedStackMapsBuilder::AddEntry(intptr_t pc_offset, ASSERT(bitmap != nullptr); ASSERT(pc_offset > last_pc_offset_); ASSERT(spill_slot_bit_count >= 0 && spill_slot_bit_count <= bitmap->Length()); - auto const pc_delta = pc_offset - last_pc_offset_; - auto const non_spill_slot_bit_count = bitmap->Length() - spill_slot_bit_count; - EncodeLEB128(&encoded_bytes_, pc_delta); - EncodeLEB128(&encoded_bytes_, spill_slot_bit_count); - EncodeLEB128(&encoded_bytes_, non_spill_slot_bit_count); + const uword pc_delta = pc_offset - last_pc_offset_; + const uword non_spill_slot_bit_count = + bitmap->Length() - spill_slot_bit_count; + encoded_bytes_.WriteLEB128(pc_delta); + encoded_bytes_.WriteLEB128(spill_slot_bit_count); + encoded_bytes_.WriteLEB128(non_spill_slot_bit_count); bitmap->AppendAsBytesTo(&encoded_bytes_); last_pc_offset_ = pc_offset; } CompressedStackMapsPtr CompressedStackMapsBuilder::Finalize() const { - if (encoded_bytes_.length() == 0) return CompressedStackMaps::null(); - return CompressedStackMaps::NewInlined(encoded_bytes_); -} - -CompressedStackMapsIterator::CompressedStackMapsIterator( - const CompressedStackMaps& maps, - const CompressedStackMaps& global_table) - : maps_(maps), - bits_container_(maps_.UsesGlobalTable() ? global_table : maps_) { - ASSERT(!maps_.IsGlobalTable()); - ASSERT(!maps_.UsesGlobalTable() || bits_container_.IsGlobalTable()); -} - -CompressedStackMapsIterator::CompressedStackMapsIterator( - const CompressedStackMaps& maps) - : CompressedStackMapsIterator( - maps, - // Only look up the global table if the map will end up using it. - maps.UsesGlobalTable() ? CompressedStackMaps::Handle( - Thread::Current() - ->isolate() - ->object_store() - ->canonicalized_stack_map_entries()) - : Object::null_compressed_stack_maps()) {} - -CompressedStackMapsIterator::CompressedStackMapsIterator( - const CompressedStackMapsIterator& it) - : maps_(it.maps_), - bits_container_(it.bits_container_), - next_offset_(it.next_offset_), - current_pc_offset_(it.current_pc_offset_), - current_global_table_offset_(it.current_global_table_offset_), - current_spill_slot_bit_count_(it.current_spill_slot_bit_count_), - current_non_spill_slot_bit_count_(it.current_spill_slot_bit_count_), - current_bits_offset_(it.current_bits_offset_) {} - -// Decode unsigned integer in LEB128 format from the payload of |maps| and -// update |byte_index|. -uintptr_t CompressedStackMapsIterator::DecodeLEB128( - const CompressedStackMaps& maps, - uintptr_t* byte_index) { - uword shift = 0; - uintptr_t value = 0; - uint8_t part = 0; - do { - ASSERT(*byte_index < maps.payload_size()); - part = maps.PayloadByte((*byte_index)++); - value |= static_cast(part & 0x7f) << shift; - shift += 7; - } while ((part & 0x80) != 0); - - return value; -} - -bool CompressedStackMapsIterator::MoveNext() { - // Empty CompressedStackMaps are represented as null values. - if (maps_.IsNull() || next_offset_ >= maps_.payload_size()) return false; - uintptr_t offset = next_offset_; - - auto const pc_delta = DecodeLEB128(maps_, &offset); - ASSERT(pc_delta <= (kMaxUint32 - current_pc_offset_)); - current_pc_offset_ += pc_delta; - - // Table-using CSMs have a table offset after the PC offset delta, whereas - // the post-delta part of inlined entries has the same information as - // global table entries. - if (maps_.UsesGlobalTable()) { - current_global_table_offset_ = DecodeLEB128(maps_, &offset); - ASSERT(current_global_table_offset_ < bits_container_.payload_size()); - - // Since generally we only use entries in the GC and the GC only needs - // the rest of the entry information if the PC offset matches, we lazily - // load and cache the information stored in the global object when it is - // actually requested. - current_spill_slot_bit_count_ = -1; - current_non_spill_slot_bit_count_ = -1; - current_bits_offset_ = -1; - } else { - current_spill_slot_bit_count_ = DecodeLEB128(maps_, &offset); - ASSERT(current_spill_slot_bit_count_ >= 0); - - current_non_spill_slot_bit_count_ = DecodeLEB128(maps_, &offset); - ASSERT(current_non_spill_slot_bit_count_ >= 0); - - const auto stackmap_bits = - current_spill_slot_bit_count_ + current_non_spill_slot_bit_count_; - const uintptr_t stackmap_size = - Utils::RoundUp(stackmap_bits, kBitsPerByte) >> kBitsPerByteLog2; - ASSERT(stackmap_size <= (maps_.payload_size() - offset)); - - current_bits_offset_ = offset; - offset += stackmap_size; + if (encoded_bytes_.bytes_written() == 0) { + return Object::empty_compressed_stackmaps().raw(); } - - next_offset_ = offset; - return true; -} - -intptr_t CompressedStackMapsIterator::Length() { - EnsureFullyLoadedEntry(); - return current_spill_slot_bit_count_ + current_non_spill_slot_bit_count_; -} -intptr_t CompressedStackMapsIterator::SpillSlotBitCount() { - EnsureFullyLoadedEntry(); - return current_spill_slot_bit_count_; -} - -bool CompressedStackMapsIterator::IsObject(intptr_t bit_index) { - EnsureFullyLoadedEntry(); - ASSERT(!bits_container_.IsNull()); - ASSERT(bit_index >= 0 && bit_index < Length()); - const intptr_t byte_index = bit_index >> kBitsPerByteLog2; - const intptr_t bit_remainder = bit_index & (kBitsPerByte - 1); - uint8_t byte_mask = 1U << bit_remainder; - const intptr_t byte_offset = current_bits_offset_ + byte_index; - return (bits_container_.PayloadByte(byte_offset) & byte_mask) != 0; -} - -void CompressedStackMapsIterator::LazyLoadGlobalTableEntry() { - ASSERT(maps_.UsesGlobalTable() && bits_container_.IsGlobalTable()); - ASSERT(HasLoadedEntry()); - ASSERT(current_global_table_offset_ < bits_container_.payload_size()); - - uintptr_t offset = current_global_table_offset_; - current_spill_slot_bit_count_ = DecodeLEB128(bits_container_, &offset); - ASSERT(current_spill_slot_bit_count_ >= 0); - - current_non_spill_slot_bit_count_ = DecodeLEB128(bits_container_, &offset); - ASSERT(current_non_spill_slot_bit_count_ >= 0); - - const auto stackmap_bits = Length(); - const uintptr_t stackmap_size = - Utils::RoundUp(stackmap_bits, kBitsPerByte) >> kBitsPerByteLog2; - ASSERT(stackmap_size <= (bits_container_.payload_size() - offset)); - - current_bits_offset_ = offset; -} - -const char* CompressedStackMapsIterator::ToCString(Zone* zone) const { - ZoneTextBuffer b(zone, 100); - CompressedStackMapsIterator it(*this); - // If we haven't loaded an entry yet, do so (but don't skip the current - // one if we have!) - if (!it.HasLoadedEntry()) { - if (!it.MoveNext()) return b.buffer(); - } - bool first_entry = true; - do { - if (first_entry) { - first_entry = false; - } else { - b.AddString("\n"); - } - b.Printf("0x%08x: ", it.pc_offset()); - for (intptr_t i = 0, n = it.Length(); i < n; i++) { - b.AddString(it.IsObject(i) ? "1" : "0"); - } - } while (it.MoveNext()); - return b.buffer(); -} - -const char* CompressedStackMapsIterator::ToCString() const { - return ToCString(Thread::Current()->zone()); + return CompressedStackMaps::NewInlined(encoded_bytes_.buffer(), + encoded_bytes_.bytes_written()); } ExceptionHandlersPtr ExceptionHandlerList::FinalizeExceptionHandlers( diff --git a/runtime/vm/code_descriptors.h b/runtime/vm/code_descriptors.h index bef3ef84cbb..f42190bb4f7 100644 --- a/runtime/vm/code_descriptors.h +++ b/runtime/vm/code_descriptors.h @@ -18,8 +18,8 @@ static const intptr_t kInvalidTryIndex = -1; class DescriptorList : public ZoneAllocated { public: - explicit DescriptorList(intptr_t initial_capacity) - : encoded_data_(initial_capacity), + explicit DescriptorList(Zone* zone) + : encoded_data_(zone, kInitialStreamSize), prev_pc_offset(0), prev_deopt_id(0), prev_token_pos(0) {} @@ -36,7 +36,9 @@ class DescriptorList : public ZoneAllocated { PcDescriptorsPtr FinalizePcDescriptors(uword entry_point); private: - GrowableArray encoded_data_; + static constexpr intptr_t kInitialStreamSize = 64; + + ZoneWriteStream encoded_data_; intptr_t prev_pc_offset; intptr_t prev_deopt_id; @@ -47,9 +49,8 @@ class DescriptorList : public ZoneAllocated { class CompressedStackMapsBuilder : public ZoneAllocated { public: - CompressedStackMapsBuilder() : encoded_bytes_() {} - - static void EncodeLEB128(GrowableArray* data, uintptr_t value); + explicit CompressedStackMapsBuilder(Zone* zone) + : encoded_bytes_(zone, kInitialStreamSize) {} void AddEntry(intptr_t pc_offset, BitmapBuilder* bitmap, @@ -58,82 +59,13 @@ class CompressedStackMapsBuilder : public ZoneAllocated { CompressedStackMapsPtr Finalize() const; private: + static constexpr intptr_t kInitialStreamSize = 16; + + ZoneWriteStream encoded_bytes_; intptr_t last_pc_offset_ = 0; - GrowableArray encoded_bytes_; DISALLOW_COPY_AND_ASSIGN(CompressedStackMapsBuilder); }; -class CompressedStackMapsIterator : public ValueObject { - public: - // We use the null value to represent CompressedStackMaps with no - // entries, so any CompressedStackMaps arguments to constructors can be null. - CompressedStackMapsIterator(const CompressedStackMaps& maps, - const CompressedStackMaps& global_table); - explicit CompressedStackMapsIterator(const CompressedStackMaps& maps); - - explicit CompressedStackMapsIterator(const CompressedStackMapsIterator& it); - - // Loads the next entry from [maps_], if any. If [maps_] is the null - // value, this always returns false. - bool MoveNext(); - - // Finds the entry with the given PC offset starting at the current - // position of the iterator. If [maps_] is the null value, this always - // returns false. - bool Find(uint32_t pc_offset) { - // We should never have an entry with a PC offset of 0 inside an - // non-empty CSM, so fail. - if (pc_offset == 0) return false; - do { - if (current_pc_offset_ >= pc_offset) break; - } while (MoveNext()); - return current_pc_offset_ == pc_offset; - } - - // Methods for accessing parts of an entry should not be called until - // a successful MoveNext() or Find() call has been made. - - uint32_t pc_offset() const { - ASSERT(HasLoadedEntry()); - return current_pc_offset_; - } - // We lazily load and cache information from the global table if the - // CSM uses it, so these methods cannot be const. - intptr_t Length(); - intptr_t SpillSlotBitCount(); - bool IsObject(intptr_t bit_offset); - - void EnsureFullyLoadedEntry() { - ASSERT(HasLoadedEntry()); - if (current_spill_slot_bit_count_ < 0) { - LazyLoadGlobalTableEntry(); - } - ASSERT(current_spill_slot_bit_count_ >= 0); - } - - const char* ToCString(Zone* zone) const; - const char* ToCString() const; - - private: - static uintptr_t DecodeLEB128(const CompressedStackMaps& data, - uintptr_t* byte_index); - bool HasLoadedEntry() const { return next_offset_ > 0; } - void LazyLoadGlobalTableEntry(); - - const CompressedStackMaps& maps_; - const CompressedStackMaps& bits_container_; - - uintptr_t next_offset_ = 0; - uint32_t current_pc_offset_ = 0; - // Only used when looking up non-PC information in the global table. - uintptr_t current_global_table_offset_ = 0; - intptr_t current_spill_slot_bit_count_ = -1; - intptr_t current_non_spill_slot_bit_count_ = -1; - intptr_t current_bits_offset_ = -1; - - friend class StackMapEntry; -}; - class ExceptionHandlerList : public ZoneAllocated { public: struct HandlerDesc { diff --git a/runtime/vm/code_descriptors_test.cc b/runtime/vm/code_descriptors_test.cc index 8c192b5705c..d8e2c410f82 100644 --- a/runtime/vm/code_descriptors_test.cc +++ b/runtime/vm/code_descriptors_test.cc @@ -100,7 +100,7 @@ TEST_CASE(StackMapGC) { int call_count = 0; PcDescriptors::Iterator iter(descriptors, PcDescriptorsLayout::kUnoptStaticCall); - CompressedStackMapsBuilder compressed_maps_builder; + CompressedStackMapsBuilder compressed_maps_builder(thread->zone()); while (iter.MoveNext()) { compressed_maps_builder.AddEntry(iter.PcOffset(), stack_bitmap, 0); ++call_count; @@ -121,7 +121,7 @@ TEST_CASE(StackMapGC) { } ISOLATE_UNIT_TEST_CASE(DescriptorList_TokenPositions) { - DescriptorList* descriptors = new DescriptorList(64); + DescriptorList* descriptors = new DescriptorList(thread->zone()); ASSERT(descriptors != NULL); const intptr_t token_positions[] = { kMinInt32, diff --git a/runtime/vm/compiler/assembler/disassembler.cc b/runtime/vm/compiler/assembler/disassembler.cc index 0edfb702602..e7e73a5ae97 100644 --- a/runtime/vm/compiler/assembler/disassembler.cc +++ b/runtime/vm/compiler/assembler/disassembler.cc @@ -220,7 +220,8 @@ void Disassembler::DisassembleCodeHelper(const char* function_fullname, const char* function_info, const Code& code, bool optimized) { - Zone* zone = Thread::Current()->zone(); + Thread* thread = Thread::Current(); + Zone* zone = thread->zone(); LocalVarDescriptors& var_descriptors = LocalVarDescriptors::Handle(zone); if (FLAG_print_variable_descriptors) { var_descriptors = code.GetLocalVarDescriptors(); @@ -290,13 +291,16 @@ void Disassembler::DisassembleCodeHelper(const char* function_fullname, } #endif // !defined(DART_PRECOMPILED_RUNTIME) - THR_Print("StackMaps for function '%s' {\n", function_fullname); - if (code.compressed_stackmaps() != CompressedStackMaps::null()) { + { const auto& stackmaps = CompressedStackMaps::Handle(zone, code.compressed_stackmaps()); - THR_Print("%s\n", stackmaps.ToCString()); + CompressedStackMaps::Iterator it(thread, stackmaps); + TextBuffer buffer(100); + buffer.Printf("StackMaps for function '%s' {\n", function_fullname); + it.WriteToBuffer(&buffer, "\n"); + buffer.AddString("}\n"); + THR_Print("%s", buffer.buffer()); } - THR_Print("}\n"); if (FLAG_print_variable_descriptors) { THR_Print("Variable Descriptors for function '%s' {\n", function_fullname); diff --git a/runtime/vm/compiler/backend/flow_graph_compiler.cc b/runtime/vm/compiler/backend/flow_graph_compiler.cc index 361ccbe9bd3..6a01f8a9dad 100644 --- a/runtime/vm/compiler/backend/flow_graph_compiler.cc +++ b/runtime/vm/compiler/backend/flow_graph_compiler.cc @@ -195,7 +195,9 @@ bool FlowGraphCompiler::IsPotentialUnboxedField(const Field& field) { } void FlowGraphCompiler::InitCompiler() { - pc_descriptors_list_ = new (zone()) DescriptorList(64); + compressed_stackmaps_builder_ = + new (zone()) CompressedStackMapsBuilder(zone()); + pc_descriptors_list_ = new (zone()) DescriptorList(zone()); exception_handlers_list_ = new (zone()) ExceptionHandlerList(); #if defined(DART_PRECOMPILER) catch_entry_moves_maps_builder_ = new (zone()) CatchEntryMovesMapBuilder(); @@ -1001,8 +1003,8 @@ void FlowGraphCompiler::RecordSafepoint(LocationSummary* locs, bitmap->Set(bitmap->Length(), true); } - compressed_stackmaps_builder()->AddEntry(assembler()->CodeSize(), bitmap, - spill_area_size); + compressed_stackmaps_builder_->AddEntry(assembler()->CodeSize(), bitmap, + spill_area_size); } } @@ -1157,15 +1159,11 @@ ArrayPtr FlowGraphCompiler::CreateDeoptInfo(compiler::Assembler* assembler) { } void FlowGraphCompiler::FinalizeStackMaps(const Code& code) { - if (compressed_stackmaps_builder_ == NULL) { - code.set_compressed_stackmaps( - CompressedStackMaps::Handle(CompressedStackMaps::null())); - } else { - // Finalize the compressed stack maps and add it to the code object. - const auto& maps = - CompressedStackMaps::Handle(compressed_stackmaps_builder_->Finalize()); - code.set_compressed_stackmaps(maps); - } + ASSERT(compressed_stackmaps_builder_ != NULL); + // Finalize the compressed stack maps and add it to the code object. + const auto& maps = + CompressedStackMaps::Handle(compressed_stackmaps_builder_->Finalize()); + code.set_compressed_stackmaps(maps); } void FlowGraphCompiler::FinalizeVarDescriptors(const Code& code) { diff --git a/runtime/vm/compiler/backend/flow_graph_compiler.h b/runtime/vm/compiler/backend/flow_graph_compiler.h index 9b1764e6235..cff19a26819 100644 --- a/runtime/vm/compiler/backend/flow_graph_compiler.h +++ b/runtime/vm/compiler/backend/flow_graph_compiler.h @@ -1108,13 +1108,6 @@ class FlowGraphCompiler : public ValueObject { intptr_t GetOptimizationThreshold() const; - CompressedStackMapsBuilder* compressed_stackmaps_builder() { - if (compressed_stackmaps_builder_ == NULL) { - compressed_stackmaps_builder_ = new CompressedStackMapsBuilder(); - } - return compressed_stackmaps_builder_; - } - #if defined(DEBUG) void FrameStateUpdateWith(Instruction* instr); void FrameStatePush(Definition* defn); diff --git a/runtime/vm/compiler/frontend/bytecode_reader.cc b/runtime/vm/compiler/frontend/bytecode_reader.cc index e0d3ea991f2..f130a7ada70 100644 --- a/runtime/vm/compiler/frontend/bytecode_reader.cc +++ b/runtime/vm/compiler/frontend/bytecode_reader.cc @@ -973,7 +973,7 @@ void BytecodeReaderHelper::ReadExceptionsTable(const Bytecode& bytecode, const ObjectPool& pool = ObjectPool::Handle(Z, bytecode.object_pool()); AbstractType& handler_type = AbstractType::Handle(Z); Array& handler_types = Array::Handle(Z); - DescriptorList* pc_descriptors_list = new (Z) DescriptorList(64); + DescriptorList* pc_descriptors_list = new (Z) DescriptorList(Z); ExceptionHandlerList* exception_handlers_list = new (Z) ExceptionHandlerList(); diff --git a/runtime/vm/datastream.h b/runtime/vm/datastream.h index 81f2c8251da..7da4532621b 100644 --- a/runtime/vm/datastream.h +++ b/runtime/vm/datastream.h @@ -23,16 +23,38 @@ static const int8_t kMaxDataPerByte = (~kMinDataPerByte & kByteMask); // NOLINT static const uint8_t kEndByteMarker = (255 - kMaxDataPerByte); static const uint8_t kEndUnsignedByteMarker = (255 - kMaxUnsignedDataPerByte); +struct LEB128Constants : AllStatic { + // Convenience template for ensuring non-signed types trigger SFINAE. + template + using only_if_signed = + typename std::enable_if::value, S>::type; + + // Convenience template for ensuring signed types trigger SFINAE. + template + using only_if_unsigned = + typename std::enable_if::value, S>::type; + + // (S)LEB128 encodes 7 bits of data per byte (hence 128). + static constexpr uint8_t kDataBitsPerByte = 7; + static constexpr uint8_t kDataByteMask = (1 << kDataBitsPerByte) - 1; + // If more data follows a given data byte, the high bit is set. + static constexpr uint8_t kMoreDataMask = (1 << kDataBitsPerByte); + // For SLEB128, the high bit in the data of the last byte is the sign bit. + static constexpr uint8_t kSignMask = (1 << (kDataBitsPerByte - 1)); +}; + +class NonStreamingWriteStream; + // Stream for reading various types from a buffer. class ReadStream : public ValueObject { public: ReadStream(const uint8_t* buffer, intptr_t size) : buffer_(buffer), current_(buffer), end_(buffer + size) {} - void SetStream(const uint8_t* buffer, intptr_t size) { - buffer_ = buffer; - current_ = buffer; - end_ = buffer + size; + // Creates a ReadStream that starts at a given position in the buffer. + ReadStream(const uint8_t* buffer, intptr_t size, intptr_t pos) + : ReadStream(buffer, size) { + SetPosition(pos); } template @@ -78,7 +100,7 @@ class ReadStream : public ValueObject { intptr_t Position() const { return current_ - buffer_; } void SetPosition(intptr_t value) { - ASSERT((end_ - buffer_) > value); + ASSERT((end_ - buffer_) >= value); current_ = buffer_ + value; } @@ -106,18 +128,71 @@ class ReadStream : public ValueObject { } uword ReadWordWith32BitReads() { - constexpr intptr_t kNumBytesPerRead32 = sizeof(uint32_t); - constexpr intptr_t kNumRead32PerWord = sizeof(uword) / kNumBytesPerRead32; - constexpr intptr_t kNumBitsPerRead32 = kNumBytesPerRead32 * kBitsPerByte; + constexpr intptr_t kNumRead32PerWord = kBitsPerWord / kBitsPerInt32; uword value = 0; for (intptr_t j = 0; j < kNumRead32PerWord; j++) { - const auto partial_value = Raw::Read(this); - value |= (static_cast(partial_value) << (j * kNumBitsPerRead32)); + const auto partial_value = Raw::Read(this); + value |= (static_cast(partial_value) << (j * kBitsPerInt32)); } return value; } + private: + using C = LEB128Constants; + + public: + template + C::only_if_unsigned ReadLEB128() { + constexpr intptr_t kBitsPerT = kBitsPerByte * sizeof(T); + T r = 0; + uint8_t s = 0; + uint8_t b; + do { + ASSERT(s < kBitsPerT); + b = ReadByte(); + r |= static_cast(b & C::kDataByteMask) << s; + s += C::kDataBitsPerByte; + } while ((b & C::kMoreDataMask) != 0); + ASSERT(s < C::kDataBitsPerByte + kBitsPerT); + return r; + } + + template + C::only_if_signed ReadLEB128() { + return bit_cast(ReadLEB128::type>()); + } + + template + C::only_if_unsigned ReadSLEB128() { + constexpr intptr_t kBitsPerT = kBitsPerByte * sizeof(T); + T r = 0; + uint8_t s = 0; + uint8_t b; + do { + ASSERT(s < kBitsPerT); + b = ReadByte(); + r |= static_cast(b & C::kDataByteMask) << s; + s += C::kDataBitsPerByte; + } while ((b & C::kMoreDataMask) != 0); + ASSERT(s < C::kDataBitsPerByte + kBitsPerT); + // At this point, [s] contains how many data bits have made it into the + // value. If the value is negative and the count of data bits is less than + // the size of the value, then we need to extend the sign by setting the + // remaining (unset) most significant bits (MSBs). + T sign_bits = 0; + if ((b & C::kSignMask) != 0 && s < kBitsPerT) { + // Create a bitmask for the current data bits and invert it. + sign_bits = ~((static_cast(1) << s) - 1); + } + return r | sign_bits; + } + + template + C::only_if_signed ReadSLEB128() { + return bit_cast(ReadSLEB128::type>()); + } + private: uint16_t Read16() { return Read16(kEndByteMarker); } @@ -128,11 +203,8 @@ class ReadStream : public ValueObject { template T Read(uint8_t end_byte_marker) { using Unsigned = typename std::make_unsigned::type; - const uint8_t* c = current_; - ASSERT(c < end_); - Unsigned b = *c++; + Unsigned b = ReadByte(); if (b > kMaxUnsignedDataPerByte) { - current_ = c; return b - end_byte_marker; } T r = 0; @@ -140,159 +212,78 @@ class ReadStream : public ValueObject { do { r |= static_cast(b) << s; s += kDataBitsPerByte; - ASSERT(c < end_); - b = *c++; + b = ReadByte(); } while (b <= kMaxUnsignedDataPerByte); - current_ = c; return r | (static_cast(b - end_byte_marker) << s); } - uint16_t Read16(uint8_t end_byte_marker) { - const uint8_t* c = current_; - ASSERT(c < end_); - uint16_t b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return b - end_byte_marker; - } - uint16_t r = b; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 7); - } +// Setting up needed variables for the unrolled loop sections below. +#define UNROLLED_INIT() \ + using Unsigned = typename std::make_unsigned::type; \ + Unsigned b = ReadByte(); \ + if (b > kMaxUnsignedDataPerByte) { \ + return b - end_byte_marker; \ + } \ + T r = b; - r |= b << 7; - ASSERT(c < end_); - b = *c++; - ASSERT(b > kMaxUnsignedDataPerByte); - current_ = c; - return r | (static_cast(b - end_byte_marker) << 14); +// Part of the unrolled loop where the loop may stop, having read the last part, +// or continue reading. +#define UNROLLED_BODY(bit_start) \ + static_assert(bit_start % kDataBitsPerByte == 0, \ + "Bit start must be a multiple of the data bits per byte"); \ + static_assert(bit_start >= 0 && bit_start < kBitsPerByte * sizeof(T), \ + "Starting unrolled body at invalid bit position"); \ + static_assert(bit_start + kDataBitsPerByte < kBitsPerByte * sizeof(T), \ + "Unrolled body should not contain final bits in value"); \ + b = ReadByte(); \ + if (b > kMaxUnsignedDataPerByte) { \ + return r | (static_cast(b - end_byte_marker) << bit_start); \ + } \ + r |= b << bit_start; + +// The end of the unrolled loop. +#define UNROLLED_END(bit_start) \ + static_assert(bit_start % kDataBitsPerByte == 0, \ + "Bit start must be a multiple of the data bits per byte"); \ + static_assert(bit_start >= 0 && bit_start < kBitsPerByte * sizeof(T), \ + "Starting unrolled end at invalid bit position"); \ + static_assert(bit_start + kDataBitsPerByte >= kBitsPerByte * sizeof(T), \ + "Unrolled end does not contain final bits in value"); \ + b = ReadByte(); \ + ASSERT(b > kMaxUnsignedDataPerByte); \ + return r | (static_cast(b - end_byte_marker) << bit_start); + + uint16_t Read16(uint8_t end_byte_marker) { + using T = uint16_t; + UNROLLED_INIT(); + UNROLLED_BODY(7); + UNROLLED_END(14); } uint32_t Read32(uint8_t end_byte_marker) { - const uint8_t* c = current_; - ASSERT(c < end_); - uint32_t b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return b - end_byte_marker; - } - - uint32_t r = b; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 7); - } - - r |= b << 7; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 14); - } - - r |= b << 14; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 21); - } - - r |= b << 21; - ASSERT(c < end_); - b = *c++; - ASSERT(b > kMaxUnsignedDataPerByte); - current_ = c; - return r | (static_cast(b - end_byte_marker) << 28); + using T = uint32_t; + UNROLLED_INIT(); + UNROLLED_BODY(7); + UNROLLED_BODY(14); + UNROLLED_BODY(21); + UNROLLED_END(28); } uint64_t Read64(uint8_t end_byte_marker) { - const uint8_t* c = current_; - ASSERT(c < end_); - uint64_t b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return b - end_byte_marker; - } - uint64_t r = b; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 7); - } - - r |= b << 7; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 14); - } - - r |= b << 14; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 21); - } - - r |= b << 21; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 28); - } - - r |= b << 28; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 35); - } - - r |= b << 35; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 42); - } - - r |= b << 42; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 49); - } - - r |= b << 49; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 56); - } - - r |= b << 56; - ASSERT(c < end_); - b = *c++; - ASSERT(b > kMaxUnsignedDataPerByte); - current_ = c; - return r | (static_cast(b - end_byte_marker) << 63); + using T = uint64_t; + UNROLLED_INIT(); + UNROLLED_BODY(7); + UNROLLED_BODY(14); + UNROLLED_BODY(21); + UNROLLED_BODY(28); + UNROLLED_BODY(35); + UNROLLED_BODY(42); + UNROLLED_BODY(49); + UNROLLED_BODY(56); + UNROLLED_END(63); } - uint8_t ReadByte() { + DART_FORCE_INLINE uint8_t ReadByte() { ASSERT(current_ < end_); return *current_++; } @@ -320,9 +311,11 @@ class BaseWriteStream : public ValueObject { const intptr_t position_before = Position(); const intptr_t position_after = Utils::RoundUp(position_before, alignment); const intptr_t length = position_after - position_before; - EnsureSpace(length); - memset(current_, 0, length); - SetPosition(position_after); + if (length != 0) { + EnsureSpace(length); + memset(current_, 0, length); + SetPosition(position_after); + } return length; } @@ -362,14 +355,12 @@ class BaseWriteStream : public ValueObject { }; void WriteWordWith32BitWrites(uword value) { - constexpr intptr_t kNumBytesPerWrite32 = sizeof(uint32_t); - constexpr intptr_t kNumWrite32PerWord = sizeof(uword) / kNumBytesPerWrite32; - constexpr intptr_t kNumBitsPerWrite32 = kNumBytesPerWrite32 * kBitsPerByte; + constexpr intptr_t kNumWrite32PerWord = kBitsPerWord / kBitsPerInt32; - const uint32_t mask = Utils::NBitMask(kNumBitsPerWrite32); + const uint32_t mask = Utils::NBitMask(kBitsPerInt32); for (intptr_t j = 0; j < kNumWrite32PerWord; j++) { - const uint32_t shifted_value = (value >> (j * kNumBitsPerWrite32)); - Raw::Write(this, shifted_value & mask); + const uint32_t shifted_value = (value >> (j * kBitsPerInt32)); + Raw::Write(this, shifted_value & mask); } } @@ -384,11 +375,11 @@ class BaseWriteStream : public ValueObject { } void WriteBytes(const void* addr, intptr_t len) { - EnsureSpace(len); if (len != 0) { + EnsureSpace(len); memmove(current_, addr, len); + current_ += len; } - current_ += len; } void WriteWord(uword value) { WriteFixed(value); } @@ -406,7 +397,7 @@ class BaseWriteStream : public ValueObject { // Measure. va_list measure_args; va_copy(measure_args, args); - intptr_t len = Utils::VSNPrint(NULL, 0, format, measure_args); + intptr_t len = Utils::VSNPrint(nullptr, 0, format, measure_args); va_end(measure_args); // Alloc. @@ -443,6 +434,85 @@ class BaseWriteStream : public ValueObject { void WriteString(const char* cstr) { WriteBytes(cstr, strlen(cstr)); } + private: + using C = LEB128Constants; + + public: + template + C::only_if_unsigned WriteLEB128(T value) { + T remainder = value; + bool is_last_part; + do { + uint8_t part = static_cast(remainder & C::kDataByteMask); + remainder >>= C::kDataBitsPerByte; + // For unsigned types, we're done when the remainder has no bits set. + is_last_part = remainder == static_cast(0); + if (!is_last_part) { + // Mark this part as a non-final part for this value. + part |= C::kMoreDataMask; + } + WriteByte(part); + } while (!is_last_part); + } + + template + C::only_if_signed WriteLEB128(T value) { + // If we're trying to LEB128 encode a negative value, chances are we should + // be using SLEB128 instead. + ASSERT(value >= 0); + return WriteLEB128(bit_cast::type>(value)); + } + + template + C::only_if_signed WriteSLEB128(T value) { + constexpr intptr_t kBitsPerT = kBitsPerByte * sizeof(T); + using Unsigned = typename std::make_unsigned::type; + // Record whether the original value was negative. + const bool is_negative = value < 0; + T remainder = value; + bool is_last_part; + do { + uint8_t part = static_cast(remainder & C::kDataByteMask); + remainder >>= C::kDataBitsPerByte; + // For signed types, we're done when either: + // - the remainder has all bits set and the part's sign bit is set + // for negative values, or + // - the remainder has no bits set and the part's sign bit is unset for + // non-negative values. + // If the remainder matches but the sign bit does not, we need one more + // part to set the sign bit correctly when decoding. + if (is_negative) { + // Right shifts of negative values in C are not guaranteed to be + // arithmetic. For negative values, set the [kDataBitsPerByte] most + // significant bits after shifting to ensure the value stays negative. + constexpr intptr_t preserved_bits = kBitsPerT - C::kDataBitsPerByte; + // The sign extension mask is the inverse of the preserved bits mask. + constexpr T sign_extend = + ~static_cast((static_cast(1) << preserved_bits) - 1); + // Sign extend for negative values just in case a non-arithmetic right + // shift is used by the compiler. + remainder |= sign_extend; + ASSERT(remainder < 0); // Remainder should still be negative. + is_last_part = + remainder == ~static_cast(0) && (part & C::kSignMask) != 0; + } else { + ASSERT(remainder >= 0); // Remainder should still be non-negative. + is_last_part = + (remainder == static_cast(0) && (part & C::kSignMask) == 0); + } + if (!is_last_part) { + // Mark this part as a non-final part for this value. + part |= C::kMoreDataMask; + } + WriteByte(part); + } while (!is_last_part); + } + + template + C::only_if_unsigned WriteSLEB128(T value) { + return WriteSLEB128(bit_cast::type>(value)); + } + protected: void EnsureSpace(intptr_t size_needed) { if (Remaining() >= size_needed) return; @@ -508,10 +578,7 @@ class NonStreamingWriteStream : public BaseWriteStream { class MallocWriteStream : public NonStreamingWriteStream { public: explicit MallocWriteStream(intptr_t initial_size) - : NonStreamingWriteStream(initial_size) { - // Go ahead and allocate initial space at construction. - EnsureSpace(initial_size_); - } + : NonStreamingWriteStream(initial_size) {} ~MallocWriteStream(); // Resets the stream and returns the original buffer, which is now considered @@ -537,10 +604,7 @@ class MallocWriteStream : public NonStreamingWriteStream { class ZoneWriteStream : public NonStreamingWriteStream { public: ZoneWriteStream(Zone* zone, intptr_t initial_size) - : NonStreamingWriteStream(initial_size), zone_(zone) { - // Go ahead and allocate initial space at construction. - EnsureSpace(initial_size_); - } + : NonStreamingWriteStream(initial_size), zone_(zone) {} private: virtual void Realloc(intptr_t new_size); @@ -562,10 +626,7 @@ class StreamingWriteStream : public BaseWriteStream { void* callback_data) : BaseWriteStream(initial_capacity), callback_(callback), - callback_data_(callback_data) { - // Go ahead and allocate initial space at construction. - EnsureSpace(initial_capacity); - } + callback_data_(callback_data) {} ~StreamingWriteStream(); private: diff --git a/runtime/vm/elf.cc b/runtime/vm/elf.cc index 50982857824..138d02e08dc 100644 --- a/runtime/vm/elf.cc +++ b/runtime/vm/elf.cc @@ -1106,35 +1106,8 @@ class DwarfElfStream : public DwarfWriteStream { stream_(ASSERT_NOTNULL(stream)), table_(table) {} - void sleb128(intptr_t value) { - bool is_last_part = false; - while (!is_last_part) { - uint8_t part = value & 0x7F; - value >>= 7; - if ((value == 0 && (part & 0x40) == 0) || - (value == static_cast(-1) && (part & 0x40) != 0)) { - is_last_part = true; - } else { - part |= 0x80; - } - stream_->WriteByte(part); - } - } - - void uleb128(uintptr_t value) { - bool is_last_part = false; - while (!is_last_part) { - uint8_t part = value & 0x7F; - value >>= 7; - if (value == 0) { - is_last_part = true; - } else { - part |= 0x80; - } - stream_->WriteByte(part); - } - } - + void sleb128(intptr_t value) { stream_->WriteSLEB128(value); } + void uleb128(uintptr_t value) { stream_->WriteLEB128(value); } void u1(uint8_t value) { stream_->WriteByte(value); } void u2(uint16_t value) { stream_->WriteFixed(value); } void u4(uint32_t value) { stream_->WriteFixed(value); } diff --git a/runtime/vm/kernel_binary.h b/runtime/vm/kernel_binary.h index e794427357f..1b53aaa029b 100644 --- a/runtime/vm/kernel_binary.h +++ b/runtime/vm/kernel_binary.h @@ -290,13 +290,17 @@ class Reader : public ValueObject { } intptr_t ReadSLEB128() { - const uint8_t* buffer = this->buffer(); - return Utils::DecodeSLEB128(buffer, size_, &offset_); + ReadStream stream(this->buffer(), size_, offset_); + const intptr_t result = stream.ReadSLEB128(); + offset_ = stream.Position(); + return result; } int64_t ReadSLEB128AsInt64() { - const uint8_t* buffer = this->buffer(); - return Utils::DecodeSLEB128(buffer, size_, &offset_); + ReadStream stream(this->buffer(), size_, offset_); + const int64_t result = stream.ReadSLEB128(); + offset_ = stream.Position(); + return result; } /** diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index 25776cd2370..bbbca69e5da 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -707,7 +707,7 @@ void Object::Init(Isolate* isolate) { *null_type_arguments_ = TypeArguments::null(); *empty_type_arguments_ = TypeArguments::null(); *null_abstract_type_ = AbstractType::null(); - *null_compressed_stack_maps_ = CompressedStackMaps::null(); + *null_compressed_stackmaps_ = CompressedStackMaps::null(); *bool_true_ = true_; *bool_false_ = false_; @@ -999,6 +999,19 @@ void Object::Init(Isolate* isolate) { empty_object_pool_->SetCanonical(); } + // Allocate and initialize the empty_compressed_stackmaps instance. + { + const intptr_t instance_size = CompressedStackMaps::InstanceSize(0); + uword address = heap->Allocate(instance_size, Heap::kOld); + InitializeObject(address, kCompressedStackMapsCid, instance_size); + CompressedStackMaps::initializeHandle( + empty_compressed_stackmaps_, + static_cast(address + kHeapObjectTag)); + empty_compressed_stackmaps_->StoreNonPointer( + &empty_compressed_stackmaps_->raw_ptr()->flags_and_size_, 0); + empty_compressed_stackmaps_->SetCanonical(); + } + // Allocate and initialize the empty_descriptors instance. { uword address = heap->Allocate(PcDescriptors::InstanceSize(0), Heap::kOld); @@ -1184,14 +1197,16 @@ void Object::Init(Isolate* isolate) { ASSERT(null_function_->IsFunction()); ASSERT(!null_type_arguments_->IsSmi()); ASSERT(null_type_arguments_->IsTypeArguments()); - ASSERT(!null_compressed_stack_maps_->IsSmi()); - ASSERT(null_compressed_stack_maps_->IsCompressedStackMaps()); + ASSERT(!null_compressed_stackmaps_->IsSmi()); + ASSERT(null_compressed_stackmaps_->IsCompressedStackMaps()); ASSERT(!empty_array_->IsSmi()); ASSERT(empty_array_->IsArray()); ASSERT(!zero_array_->IsSmi()); ASSERT(zero_array_->IsArray()); ASSERT(!empty_context_scope_->IsSmi()); ASSERT(empty_context_scope_->IsContextScope()); + ASSERT(!empty_compressed_stackmaps_->IsSmi()); + ASSERT(empty_compressed_stackmaps_->IsCompressedStackMaps()); ASSERT(!empty_descriptors_->IsSmi()); ASSERT(empty_descriptors_->IsPcDescriptors()); ASSERT(!empty_var_descriptors_->IsSmi()); @@ -14159,35 +14174,6 @@ const char* InstructionsSection::ToCString() const { return "InstructionsSection"; } -// Encode integer |value| in SLEB128 format and store into |data|. -static void EncodeSLEB128(GrowableArray* data, intptr_t value) { - bool is_last_part = false; - while (!is_last_part) { - uint8_t part = value & 0x7f; - value >>= 7; - if ((value == 0 && (part & 0x40) == 0) || - (value == static_cast(-1) && (part & 0x40) != 0)) { - is_last_part = true; - } else { - part |= 0x80; - } - data->Add(part); - } -} - -// Encode integer in SLEB128 format. -void PcDescriptors::EncodeInteger(GrowableArray* data, - intptr_t value) { - return EncodeSLEB128(data, value); -} - -// Decode SLEB128 encoded integer. Update byte_index to the next integer. -intptr_t PcDescriptors::DecodeInteger(intptr_t* byte_index) const { - NoSafepointScope no_safepoint; - const uint8_t* data = raw_ptr()->data(); - return Utils::DecodeSLEB128(data, Length(), byte_index); -} - ObjectPoolPtr ObjectPool::New(intptr_t len) { ASSERT(Object::object_pool_class() != Class::null()); if (len < 0 || len > kMaxElements) { @@ -14300,25 +14286,25 @@ void PcDescriptors::SetLength(intptr_t value) const { StoreNonPointer(&raw_ptr()->length_, value); } -void PcDescriptors::CopyData(GrowableArray* delta_encoded_data) { +void PcDescriptors::CopyData(const void* bytes, intptr_t size) { NoSafepointScope no_safepoint; uint8_t* data = UnsafeMutableNonPointer(&raw_ptr()->data()[0]); - for (intptr_t i = 0; i < delta_encoded_data->length(); ++i) { - data[i] = (*delta_encoded_data)[i]; - } + // We're guaranted these memory spaces do not overlap. + memcpy(data, bytes, size); // NOLINT } -PcDescriptorsPtr PcDescriptors::New(GrowableArray* data) { +PcDescriptorsPtr PcDescriptors::New(const void* delta_encoded_data, + intptr_t size) { ASSERT(Object::pc_descriptors_class() != Class::null()); Thread* thread = Thread::Current(); PcDescriptors& result = PcDescriptors::Handle(thread->zone()); { - uword size = PcDescriptors::InstanceSize(data->length()); - ObjectPtr raw = Object::Allocate(PcDescriptors::kClassId, size, Heap::kOld); + ObjectPtr raw = Object::Allocate( + PcDescriptors::kClassId, PcDescriptors::InstanceSize(size), Heap::kOld); NoSafepointScope no_safepoint; result ^= raw; - result.SetLength(data->length()); - result.CopyData(data); + result.SetLength(size); + result.CopyData(delta_encoded_data, size); } return result.raw(); } @@ -14474,45 +14460,195 @@ const char* CodeSourceMap::ToCString() const { } intptr_t CompressedStackMaps::Hashcode() const { + NoSafepointScope scope; + uint8_t* data = UnsafeMutableNonPointer(&raw_ptr()->data()[0]); + uint8_t* end = data + payload_size(); uint32_t hash = payload_size(); - for (uintptr_t i = 0; i < payload_size(); i++) { - uint8_t byte = PayloadByte(i); - hash = CombineHashes(hash, byte); + for (uint8_t* cursor = data; cursor < end; cursor++) { + hash = CombineHashes(hash, *cursor); } return FinalizeHash(hash, kHashBits); } -CompressedStackMapsPtr CompressedStackMaps::New( - const GrowableArray& payload, - bool is_global_table, - bool uses_global_table) { +CompressedStackMaps::Iterator::Iterator(const CompressedStackMaps& maps, + const CompressedStackMaps& global_table) + : maps_(maps), + bits_container_(maps_.UsesGlobalTable() ? global_table : maps_) { + ASSERT(!maps_.IsNull()); + ASSERT(!bits_container_.IsNull()); + ASSERT(!maps_.IsGlobalTable()); + ASSERT(!maps_.UsesGlobalTable() || bits_container_.IsGlobalTable()); +} + +CompressedStackMaps::Iterator::Iterator(Thread* thread, + const CompressedStackMaps& maps) + : CompressedStackMaps::Iterator( + maps, + // Only look up the global table if the map will end up using it. + maps.UsesGlobalTable() ? CompressedStackMaps::Handle( + thread->zone(), + thread->isolate() + ->object_store() + ->canonicalized_stack_map_entries()) + : Object::null_compressed_stackmaps()) {} + +CompressedStackMaps::Iterator::Iterator(const CompressedStackMaps::Iterator& it) + : maps_(it.maps_), + bits_container_(it.bits_container_), + next_offset_(it.next_offset_), + current_pc_offset_(it.current_pc_offset_), + current_global_table_offset_(it.current_global_table_offset_), + current_spill_slot_bit_count_(it.current_spill_slot_bit_count_), + current_non_spill_slot_bit_count_(it.current_spill_slot_bit_count_), + current_bits_offset_(it.current_bits_offset_) {} + +bool CompressedStackMaps::Iterator::MoveNext() { + if (next_offset_ >= maps_.payload_size()) { + return false; + } + + NoSafepointScope scope; + ReadStream stream(maps_.raw_ptr()->data(), maps_.payload_size(), + next_offset_); + + auto const pc_delta = stream.ReadLEB128(); + ASSERT(pc_delta <= (kMaxUint32 - current_pc_offset_)); + current_pc_offset_ += pc_delta; + + // Table-using CSMs have a table offset after the PC offset delta, whereas + // the post-delta part of inlined entries has the same information as + // global table entries. + if (maps_.UsesGlobalTable()) { + current_global_table_offset_ = stream.ReadLEB128(); + ASSERT(current_global_table_offset_ < bits_container_.payload_size()); + + // Since generally we only use entries in the GC and the GC only needs + // the rest of the entry information if the PC offset matches, we lazily + // load and cache the information stored in the global object when it is + // actually requested. + current_spill_slot_bit_count_ = -1; + current_non_spill_slot_bit_count_ = -1; + current_bits_offset_ = -1; + + next_offset_ = stream.Position(); + } else { + current_spill_slot_bit_count_ = stream.ReadLEB128(); + ASSERT(current_spill_slot_bit_count_ >= 0); + + current_non_spill_slot_bit_count_ = stream.ReadLEB128(); + ASSERT(current_non_spill_slot_bit_count_ >= 0); + + const auto stackmap_bits = + current_spill_slot_bit_count_ + current_non_spill_slot_bit_count_; + const uintptr_t stackmap_size = + Utils::RoundUp(stackmap_bits, kBitsPerByte) >> kBitsPerByteLog2; + ASSERT(stackmap_size <= (maps_.payload_size() - stream.Position())); + + current_bits_offset_ = stream.Position(); + next_offset_ = current_bits_offset_ + stackmap_size; + } + + return true; +} + +intptr_t CompressedStackMaps::Iterator::Length() const { + EnsureFullyLoadedEntry(); + return current_spill_slot_bit_count_ + current_non_spill_slot_bit_count_; +} +intptr_t CompressedStackMaps::Iterator::SpillSlotBitCount() const { + EnsureFullyLoadedEntry(); + return current_spill_slot_bit_count_; +} + +bool CompressedStackMaps::Iterator::IsObject(intptr_t bit_index) const { + EnsureFullyLoadedEntry(); + ASSERT(bit_index >= 0 && bit_index < Length()); + const intptr_t byte_index = bit_index >> kBitsPerByteLog2; + const intptr_t bit_remainder = bit_index & (kBitsPerByte - 1); + uint8_t byte_mask = 1U << bit_remainder; + const intptr_t byte_offset = current_bits_offset_ + byte_index; + NoSafepointScope scope; + return (bits_container_.raw_ptr()->data()[byte_offset] & byte_mask) != 0; +} + +void CompressedStackMaps::Iterator::LazyLoadGlobalTableEntry() const { + ASSERT(maps_.UsesGlobalTable()); + ASSERT(HasLoadedEntry()); + ASSERT(current_global_table_offset_ < bits_container_.payload_size()); + + NoSafepointScope scope; + ReadStream stream(bits_container_.raw_ptr()->data(), + bits_container_.payload_size(), + current_global_table_offset_); + + current_spill_slot_bit_count_ = stream.ReadLEB128(); + ASSERT(current_spill_slot_bit_count_ >= 0); + + current_non_spill_slot_bit_count_ = stream.ReadLEB128(); + ASSERT(current_non_spill_slot_bit_count_ >= 0); + + const auto stackmap_bits = Length(); + const uintptr_t stackmap_size = + Utils::RoundUp(stackmap_bits, kBitsPerByte) >> kBitsPerByteLog2; + ASSERT(stackmap_size <= (bits_container_.payload_size() - stream.Position())); + + current_bits_offset_ = stream.Position(); +} + +void CompressedStackMaps::Iterator::WriteToBuffer(BaseTextBuffer* buffer, + const char* separator) const { + CompressedStackMaps::Iterator it(*this); + // If we haven't loaded an entry yet, do so (but don't skip the current + // one if we have!) + if (!it.HasLoadedEntry()) { + if (!it.MoveNext()) return; + } + bool first_entry = true; + do { + if (!first_entry) { + buffer->AddString(separator); + } + buffer->Printf("0x%0.8" Px32 ": ", it.pc_offset()); + for (intptr_t i = 0, n = it.Length(); i < n; i++) { + buffer->AddString(it.IsObject(i) ? "1" : "0"); + } + first_entry = false; + } while (it.MoveNext()); +} + +CompressedStackMapsPtr CompressedStackMaps::New(const void* payload, + intptr_t size, + bool is_global_table, + bool uses_global_table) { ASSERT(Object::compressed_stackmaps_class() != Class::null()); // We don't currently allow both flags to be true. ASSERT(!is_global_table || !uses_global_table); - auto& result = CompressedStackMaps::Handle(); + // The canonical empty instance should be used instead. + ASSERT(size != 0); - const uintptr_t payload_size = payload.length(); - if (!CompressedStackMapsLayout::SizeField::is_valid(payload_size)) { + if (!CompressedStackMapsLayout::SizeField::is_valid(size)) { FATAL1( "Fatal error in CompressedStackMaps::New: " "invalid payload size %" Pu "\n", - payload_size); + size); } + + auto& result = CompressedStackMaps::Handle(); { // CompressedStackMaps data objects are associated with a code object, // allocate them in old generation. - ObjectPtr raw = Object::Allocate( - CompressedStackMaps::kClassId, - CompressedStackMaps::InstanceSize(payload_size), Heap::kOld); + ObjectPtr raw = + Object::Allocate(CompressedStackMaps::kClassId, + CompressedStackMaps::InstanceSize(size), Heap::kOld); NoSafepointScope no_safepoint; result ^= raw; result.StoreNonPointer( &result.raw_ptr()->flags_and_size_, CompressedStackMapsLayout::GlobalTableBit::encode(is_global_table) | CompressedStackMapsLayout::UsesTableBit::encode(uses_global_table) | - CompressedStackMapsLayout::SizeField::encode(payload_size)); + CompressedStackMapsLayout::SizeField::encode(size)); auto cursor = result.UnsafeMutableNonPointer(result.raw_ptr()->data()); - memcpy(cursor, payload.data(), payload.length()); // NOLINT + memcpy(cursor, payload, size); // NOLINT } ASSERT(!result.IsGlobalTable() || !result.UsesGlobalTable()); @@ -14522,12 +14658,16 @@ CompressedStackMapsPtr CompressedStackMaps::New( const char* CompressedStackMaps::ToCString() const { ASSERT(!IsGlobalTable()); + if (payload_size() == 0) { + return "CompressedStackMaps()"; + } auto const t = Thread::Current(); - auto zone = t->zone(); - const auto& global_table = CompressedStackMaps::Handle( - zone, t->isolate()->object_store()->canonicalized_stack_map_entries()); - CompressedStackMapsIterator it(*this, global_table); - return it.ToCString(zone); + CompressedStackMaps::Iterator it(t, *this); + ZoneTextBuffer buffer(t->zone(), 100); + buffer.AddString("CompressedStackMaps("); + it.WriteToBuffer(&buffer, ", "); + buffer.AddString(")"); + return buffer.buffer(); } StringPtr LocalVarDescriptors::GetName(intptr_t var_index) const { @@ -16479,6 +16619,7 @@ CodePtr Code::New(intptr_t pointer_offsets_length) { NOT_IN_PRODUCT(result.set_comments(Comments::New(0))); NOT_IN_PRODUCT(result.set_compile_timestamp(0)); result.set_pc_descriptors(Object::empty_descriptors()); + result.set_compressed_stackmaps(Object::empty_compressed_stackmaps()); } return result.raw(); } diff --git a/runtime/vm/object.h b/runtime/vm/object.h index e9a388b2377..230312a3a69 100644 --- a/runtime/vm/object.h +++ b/runtime/vm/object.h @@ -417,12 +417,13 @@ class Object { V(Instance, null_instance) \ V(Function, null_function) \ V(TypeArguments, null_type_arguments) \ - V(CompressedStackMaps, null_compressed_stack_maps) \ + V(CompressedStackMaps, null_compressed_stackmaps) \ V(TypeArguments, empty_type_arguments) \ V(Array, empty_array) \ V(Array, zero_array) \ V(ContextScope, empty_context_scope) \ V(ObjectPool, empty_object_pool) \ + V(CompressedStackMaps, empty_compressed_stackmaps) \ V(PcDescriptors, empty_descriptors) \ V(LocalVarDescriptors, empty_var_descriptors) \ V(ExceptionHandlers, empty_exception_handlers) \ @@ -5705,7 +5706,7 @@ class PcDescriptors : public Object { return RoundedAllocationSize(UnroundedSize(len)); } - static PcDescriptorsPtr New(GrowableArray* delta_encoded_data); + static PcDescriptorsPtr New(const void* delta_encoded_data, intptr_t size); // Verify (assert) assumptions about pc descriptors in debug mode. void Verify(const Function& function) const; @@ -5714,12 +5715,6 @@ class PcDescriptors : public Object { void PrintToJSONObject(JSONObject* jsobj, bool ref) const; - // Encode integer in SLEB128 format. - static void EncodeInteger(GrowableArray* data, intptr_t value); - - // Decode SLEB128 encoded integer. Update byte_index to the next integer. - intptr_t DecodeInteger(intptr_t* byte_index) const; - // We would have a VisitPointers function here to traverse the // pc descriptors table to visit objects if any in the table. // Note: never return a reference to a PcDescriptorsLayout::PcDescriptorRec @@ -5738,10 +5733,12 @@ class PcDescriptors : public Object { cur_yield_index_(PcDescriptorsLayout::kInvalidYieldIndex) {} bool MoveNext() { + NoSafepointScope scope; + ReadStream stream(descriptors_.raw_ptr()->data(), descriptors_.Length(), + byte_index_); // Moves to record that matches kind_mask_. while (byte_index_ < descriptors_.Length()) { - const int32_t kind_and_metadata = - descriptors_.DecodeInteger(&byte_index_); + const int32_t kind_and_metadata = stream.ReadSLEB128(); cur_kind_ = PcDescriptorsLayout::KindAndMetadata::DecodeKind(kind_and_metadata); cur_try_index_ = PcDescriptorsLayout::KindAndMetadata::DecodeTryIndex( @@ -5750,12 +5747,13 @@ class PcDescriptors : public Object { PcDescriptorsLayout::KindAndMetadata::DecodeYieldIndex( kind_and_metadata); - cur_pc_offset_ += descriptors_.DecodeInteger(&byte_index_); + cur_pc_offset_ += stream.ReadSLEB128(); if (!FLAG_precompiled_mode) { - cur_deopt_id_ += descriptors_.DecodeInteger(&byte_index_); - cur_token_pos_ += descriptors_.DecodeInteger(&byte_index_); + cur_deopt_id_ += stream.ReadSLEB128(); + cur_token_pos_ += stream.ReadSLEB128(); } + byte_index_ = stream.Position(); if ((cur_kind_ & kind_mask_) != 0) { return true; // Current is valid. @@ -5816,7 +5814,7 @@ class PcDescriptors : public Object { static PcDescriptorsPtr New(intptr_t length); void SetLength(intptr_t value) const; - void CopyData(GrowableArray* data); + void CopyData(const void* bytes, intptr_t size); FINAL_HEAP_OBJECT_IMPLEMENTATION(PcDescriptors, Object); friend class Class; @@ -5908,46 +5906,117 @@ class CompressedStackMaps : public Object { return RoundedAllocationSize(UnroundedSize(length)); } - bool UsesGlobalTable() const { return !IsNull() && UsesGlobalTable(raw()); } + bool UsesGlobalTable() const { return UsesGlobalTable(raw()); } static bool UsesGlobalTable(const CompressedStackMapsPtr raw) { return CompressedStackMapsLayout::UsesTableBit::decode( raw->ptr()->flags_and_size_); } - bool IsGlobalTable() const { return !IsNull() && IsGlobalTable(raw()); } + bool IsGlobalTable() const { return IsGlobalTable(raw()); } static bool IsGlobalTable(const CompressedStackMapsPtr raw) { return CompressedStackMapsLayout::GlobalTableBit::decode( raw->ptr()->flags_and_size_); } - static CompressedStackMapsPtr NewInlined( - const GrowableArray& bytes) { - return New(bytes, /*is_global_table=*/false, /*uses_global_table=*/false); + static CompressedStackMapsPtr NewInlined(const void* payload, intptr_t size) { + return New(payload, size, /*is_global_table=*/false, + /*uses_global_table=*/false); } - static CompressedStackMapsPtr NewUsingTable( - const GrowableArray& bytes) { - return New(bytes, /*is_global_table=*/false, /*uses_global_table=*/true); + static CompressedStackMapsPtr NewUsingTable(const void* payload, + intptr_t size) { + return New(payload, size, /*is_global_table=*/false, + /*uses_global_table=*/true); } - static CompressedStackMapsPtr NewGlobalTable( - const GrowableArray& bytes) { - return New(bytes, /*is_global_table=*/true, /*uses_global_table=*/false); + static CompressedStackMapsPtr NewGlobalTable(const void* payload, + intptr_t size) { + return New(payload, size, /*is_global_table=*/true, + /*uses_global_table=*/false); } + class Iterator : public ValueObject { + public: + Iterator(const CompressedStackMaps& maps, + const CompressedStackMaps& global_table); + Iterator(Thread* thread, const CompressedStackMaps& maps); + + explicit Iterator(const CompressedStackMaps::Iterator& it); + + // Loads the next entry from [maps_], if any. If [maps_] is the null value, + // this always returns false. + bool MoveNext(); + + // Finds the entry with the given PC offset starting at the current position + // of the iterator. If [maps_] is the null value, this always returns false. + bool Find(uint32_t pc_offset) { + // We should never have an entry with a PC offset of 0 inside an + // non-empty CSM, so fail. + if (pc_offset == 0) return false; + do { + if (current_pc_offset_ >= pc_offset) break; + } while (MoveNext()); + return current_pc_offset_ == pc_offset; + } + + // Methods for accessing parts of an entry should not be called until + // a successful MoveNext() or Find() call has been made. + + // Returns the PC offset of the loaded entry. + uint32_t pc_offset() const { + ASSERT(HasLoadedEntry()); + return current_pc_offset_; + } + + // Returns the bit length of the loaded entry. + intptr_t Length() const; + // Returns the number of spill slot bits of the loaded entry. + intptr_t SpillSlotBitCount() const; + // Returns whether the stack entry represented by the offset contains + // a tagged objecet. + bool IsObject(intptr_t bit_offset) const; + + void WriteToBuffer(BaseTextBuffer* buffer, const char* separator) const; + + private: + bool HasLoadedEntry() const { return next_offset_ > 0; } + + // Caches the corresponding values from the global table in the mutable + // fields. We lazily load these as some clients only need the PC offset. + void LazyLoadGlobalTableEntry() const; + + void EnsureFullyLoadedEntry() const { + ASSERT(HasLoadedEntry()); + if (current_spill_slot_bit_count_ < 0) { + LazyLoadGlobalTableEntry(); + ASSERT(current_spill_slot_bit_count_ >= 0); + } + } + + const CompressedStackMaps& maps_; + const CompressedStackMaps& bits_container_; + + uintptr_t next_offset_ = 0; + uint32_t current_pc_offset_ = 0; + // Only used when looking up non-PC information in the global table. + uintptr_t current_global_table_offset_ = 0; + // Marked as mutable as these fields may be updated with lazily loaded + // values from the global table when their associated accessor is called, + // but those values will never change for a given entry once loaded.. + mutable intptr_t current_spill_slot_bit_count_ = -1; + mutable intptr_t current_non_spill_slot_bit_count_ = -1; + mutable intptr_t current_bits_offset_ = -1; + + friend class StackMapEntry; + }; + private: - static CompressedStackMapsPtr New(const GrowableArray& bytes, + static CompressedStackMapsPtr New(const void* payload, + intptr_t size, bool is_global_table, bool uses_global_table); - uint8_t PayloadByte(uintptr_t offset) const { - ASSERT(offset < payload_size()); - return raw_ptr()->data()[offset]; - } - FINAL_HEAP_OBJECT_IMPLEMENTATION(CompressedStackMaps, Object); friend class Class; - friend class CompressedStackMapsIterator; // For PayloadByte - friend class StackMapEntry; // For PayloadByte }; class ExceptionHandlers : public Object { @@ -10277,7 +10346,6 @@ class TypedData : public TypedDataBase { FINAL_HEAP_OBJECT_IMPLEMENTATION(TypedData, TypedDataBase); friend class Class; - friend class CompressedStackMapsIterator; friend class ExternalTypedData; friend class TypedDataView; }; diff --git a/runtime/vm/object_test.cc b/runtime/vm/object_test.cc index 060f484345a..b0eff0e021d 100644 --- a/runtime/vm/object_test.cc +++ b/runtime/vm/object_test.cc @@ -2788,7 +2788,7 @@ ISOLATE_UNIT_TEST_CASE(ExceptionHandlers) { } ISOLATE_UNIT_TEST_CASE(PcDescriptors) { - DescriptorList* builder = new DescriptorList(0); + DescriptorList* builder = new DescriptorList(thread->zone()); // kind, pc_offset, deopt_id, token_pos, try_index, yield_index builder->AddDescriptor(PcDescriptorsLayout::kOther, 10, 1, TokenPosition(20), @@ -2858,7 +2858,7 @@ ISOLATE_UNIT_TEST_CASE(PcDescriptors) { } ISOLATE_UNIT_TEST_CASE(PcDescriptorsLargeDeltas) { - DescriptorList* builder = new DescriptorList(0); + DescriptorList* builder = new DescriptorList(thread->zone()); // kind, pc_offset, deopt_id, token_pos, try_index builder->AddDescriptor(PcDescriptorsLayout::kOther, 100, 1, diff --git a/runtime/vm/program_visitor.cc b/runtime/vm/program_visitor.cc index d2e13d22cea..483ce9b4c9b 100644 --- a/runtime/vm/program_visitor.cc +++ b/runtime/vm/program_visitor.cc @@ -451,20 +451,18 @@ void ProgramVisitor::ShareMegamorphicBuckets(Zone* zone, Isolate* isolate) { class StackMapEntry : public ZoneAllocated { public: - StackMapEntry(Zone* zone, const CompressedStackMapsIterator& it) + StackMapEntry(Zone* zone, const CompressedStackMaps::Iterator& it) : maps_(CompressedStackMaps::Handle(zone, it.maps_.raw())), bits_container_( CompressedStackMaps::Handle(zone, it.bits_container_.raw())), - spill_slot_bit_count_(it.current_spill_slot_bit_count_), - non_spill_slot_bit_count_(it.current_non_spill_slot_bit_count_), + // If the map uses the global table, this accessor call ensures the + // entry is fully loaded before we retrieve [it.current_bits_offset_]. + spill_slot_bit_count_(it.SpillSlotBitCount()), + non_spill_slot_bit_count_(it.Length() - it.SpillSlotBitCount()), bits_offset_(it.current_bits_offset_) { ASSERT(!maps_.IsNull() && !maps_.IsGlobalTable()); ASSERT(!bits_container_.IsNull()); ASSERT(!maps_.UsesGlobalTable() || bits_container_.IsGlobalTable()); - // Check that the iterator was fully loaded when we ran the initializing - // expressions above. By this point we enter the body of the constructor, - // it's too late to run EnsureFullyLoadedEntry(). - ASSERT(it.HasLoadedEntry()); ASSERT(it.current_spill_slot_bit_count_ >= 0); } @@ -475,8 +473,13 @@ class StackMapEntry : public ZoneAllocated { uint32_t hash = 0; hash = CombineHashes(hash, spill_slot_bit_count_); hash = CombineHashes(hash, non_spill_slot_bit_count_); - for (intptr_t i = 0; i < PayloadLength(); i++) { - hash = CombineHashes(hash, PayloadByte(i)); + { + NoSafepointScope scope; + auto const start = PayloadData(); + auto const end = start + PayloadLength(); + for (auto cursor = start; cursor < end; cursor++) { + hash = CombineHashes(hash, *cursor); + } } hash_ = FinalizeHash(hash, kHashBits); return hash_; @@ -490,20 +493,19 @@ class StackMapEntry : public ZoneAllocated { // Since we ensure that bits in the payload that are not part of the // actual stackmap data are cleared, we can just compare payloads by byte // instead of calling IsObject for each bit. - for (intptr_t i = 0; i < PayloadLength(); i++) { - if (PayloadByte(i) != other->PayloadByte(i)) return false; - } - return true; + NoSafepointScope scope; + return memcmp(PayloadData(), other->PayloadData(), PayloadLength()) == 0; } // Encodes this StackMapEntry to the given array of bytes and returns the // initial offset of the entry in the array. - intptr_t EncodeTo(GrowableArray* array) { - auto const current_offset = array->length(); - CompressedStackMapsBuilder::EncodeLEB128(array, spill_slot_bit_count_); - CompressedStackMapsBuilder::EncodeLEB128(array, non_spill_slot_bit_count_); - for (intptr_t i = 0; i < PayloadLength(); i++) { - array->Add(PayloadByte(i)); + intptr_t EncodeTo(NonStreamingWriteStream* stream) { + auto const current_offset = stream->Position(); + stream->WriteLEB128(spill_slot_bit_count_); + stream->WriteLEB128(non_spill_slot_bit_count_); + { + NoSafepointScope scope; + stream->WriteBytes(PayloadData(), PayloadLength()); } return current_offset; } @@ -518,8 +520,9 @@ class StackMapEntry : public ZoneAllocated { intptr_t PayloadLength() const { return Utils::RoundUp(Length(), kBitsPerByte) >> kBitsPerByteLog2; } - intptr_t PayloadByte(intptr_t offset) const { - return bits_container_.PayloadByte(bits_offset_ + offset); + const uint8_t* PayloadData() const { + ASSERT(!Thread::Current()->IsAtSafepoint()); + return bits_container_.raw()->ptr()->data() + bits_offset_; } const CompressedStackMaps& maps_; @@ -577,9 +580,9 @@ void ProgramVisitor::NormalizeAndDedupCompressedStackMaps(Zone* zone, void VisitCode(const Code& code) { compressed_stackmaps_ = code.compressed_stackmaps(); - CompressedStackMapsIterator it(compressed_stackmaps_, old_global_table_); + CompressedStackMaps::Iterator it(compressed_stackmaps_, + old_global_table_); while (it.MoveNext()) { - it.EnsureFullyLoadedEntry(); auto const entry = new (zone_) StackMapEntry(zone_, it); auto const index = entry_indices_.LookupValue(entry); if (index < 0) { @@ -598,7 +601,9 @@ void ProgramVisitor::NormalizeAndDedupCompressedStackMaps(Zone* zone, CompressedStackMapsPtr CreateGlobalTable( StackMapEntryIntMap* entry_offsets) { ASSERT(entry_offsets->IsEmpty()); - if (collected_entries_.length() == 0) return CompressedStackMaps::null(); + if (collected_entries_.length() == 0) { + return CompressedStackMaps::null(); + } // First, sort the entries from most used to least used. This way, // the most often used CSMs will have the lowest offsets, which means // they will be smaller when LEB128 encoded. @@ -606,16 +611,17 @@ void ProgramVisitor::NormalizeAndDedupCompressedStackMaps(Zone* zone, [](StackMapEntry* const* e1, StackMapEntry* const* e2) { return static_cast((*e2)->UsageCount() - (*e1)->UsageCount()); }); - GrowableArray bytes; + MallocWriteStream stream(128); // Encode the entries and record their offset in the payload. Sorting the // entries may have changed their indices, so update those as well. for (intptr_t i = 0, n = collected_entries_.length(); i < n; i++) { auto const entry = collected_entries_.At(i); entry_indices_.Update({entry, i}); - entry_offsets->Insert({entry, entry->EncodeTo(&bytes)}); + entry_offsets->Insert({entry, entry->EncodeTo(&stream)}); } const auto& data = CompressedStackMaps::Handle( - zone_, CompressedStackMaps::NewGlobalTable(bytes)); + zone_, CompressedStackMaps::NewGlobalTable(stream.buffer(), + stream.bytes_written())); return data.raw(); } @@ -690,19 +696,23 @@ void ProgramVisitor::NormalizeAndDedupCompressedStackMaps(Zone* zone, private: // Creates a normalized CSM from the given non-normalized CSM. CompressedStackMapsPtr NormalizeEntries(const CompressedStackMaps& maps) { - GrowableArray new_payload; - CompressedStackMapsIterator it(maps, old_global_table_); + if (maps.payload_size() == 0) { + // No entries, so use the canonical empty map. + return Object::empty_compressed_stackmaps().raw(); + } + MallocWriteStream new_payload(maps.payload_size()); + CompressedStackMaps::Iterator it(maps, old_global_table_); intptr_t last_offset = 0; while (it.MoveNext()) { - it.EnsureFullyLoadedEntry(); StackMapEntry entry(zone_, it); - auto const entry_offset = entry_offsets_.LookupValue(&entry); - auto const pc_delta = it.pc_offset() - last_offset; - CompressedStackMapsBuilder::EncodeLEB128(&new_payload, pc_delta); - CompressedStackMapsBuilder::EncodeLEB128(&new_payload, entry_offset); + const intptr_t entry_offset = entry_offsets_.LookupValue(&entry); + const intptr_t pc_delta = it.pc_offset() - last_offset; + new_payload.WriteLEB128(pc_delta); + new_payload.WriteLEB128(entry_offset); last_offset = it.pc_offset(); } - return CompressedStackMaps::NewUsingTable(new_payload); + return CompressedStackMaps::NewUsingTable(new_payload.buffer(), + new_payload.bytes_written()); } const CompressedStackMaps& old_global_table_; diff --git a/runtime/vm/raw_object.h b/runtime/vm/raw_object.h index 5f03f711464..498fd5dbb51 100644 --- a/runtime/vm/raw_object.h +++ b/runtime/vm/raw_object.h @@ -1847,7 +1847,9 @@ class CompressedStackMapsLayout : public ObjectLayout { sizeof(flags_and_size_) * kBitsPerByte - UsesTableBit::kNextBit> {}; + friend class Object; friend class ImageWriter; + friend class StackMapEntry; }; class LocalVarDescriptorsLayout : public ObjectLayout { diff --git a/runtime/vm/stack_frame.cc b/runtime/vm/stack_frame.cc index 9c14f6e53de..9df6e3c52c4 100644 --- a/runtime/vm/stack_frame.cc +++ b/runtime/vm/stack_frame.cc @@ -284,7 +284,7 @@ void StackFrame::VisitObjectPointers(ObjectPointerVisitor* visitor) { auto isolate = isolate_group()->isolates_.First(); global_table = isolate->object_store()->canonicalized_stack_map_entries(); - CompressedStackMapsIterator it(maps, global_table); + CompressedStackMaps::Iterator it(maps, global_table); const uword start = code.PayloadStart(); const uint32_t pc_offset = pc() - start; if (it.Find(pc_offset)) {