diff --git a/runtime/vm/datastream.h b/runtime/vm/datastream.h index a844493a6d6..6e0547bf812 100644 --- a/runtime/vm/datastream.h +++ b/runtime/vm/datastream.h @@ -15,13 +15,13 @@ namespace dart { -static const int8_t kDataBitsPerByte = 7; -static const int8_t kByteMask = (1 << kDataBitsPerByte) - 1; -static const int8_t kMaxUnsignedDataPerByte = kByteMask; -static const int8_t kMinDataPerByte = -(1 << (kDataBitsPerByte - 1)); -static const int8_t kMaxDataPerByte = (~kMinDataPerByte & kByteMask); // NOLINT -static const uint8_t kEndByteMarker = (255 - kMaxDataPerByte); -static const uint8_t kEndUnsignedByteMarker = (255 - kMaxUnsignedDataPerByte); +// (S)LEB128 encodes 7 bits of data per byte (hence 128). +static constexpr uint8_t kDataBitsPerByte = 7; +static constexpr uint8_t kDataByteMask = (1 << kDataBitsPerByte) - 1; +// If more data follows a given data byte, the high bit is set. +static constexpr uint8_t kMoreDataMask = (1 << kDataBitsPerByte); +// For SLEB128, the high bit in the data of the last byte is the sign bit. +static constexpr uint8_t kSignMask = (1 << (kDataBitsPerByte - 1)); typedef uint8_t* (*ReAlloc)(uint8_t* ptr, intptr_t old_size, intptr_t new_size); typedef void (*DeAlloc)(uint8_t* ptr); @@ -50,19 +50,19 @@ class ReadStream : public ValueObject { template class Raw<2, T> { public: - static T Read(ReadStream* st) { return bit_cast(st->Read16()); } + static T Read(ReadStream* st) { return bit_cast(st->Read16()); } }; template class Raw<4, T> { public: - static T Read(ReadStream* st) { return bit_cast(st->Read32()); } + static T Read(ReadStream* st) { return bit_cast(st->Read32()); } }; template class Raw<8, T> { public: - static T Read(ReadStream* st) { return bit_cast(st->Read64()); } + static T Read(ReadStream* st) { return bit_cast(st->Read64()); } }; // Reads 'len' bytes from the stream. @@ -74,9 +74,15 @@ class ReadStream : public ValueObject { current_ += len; } + // Reads a value of type [T] assuming an encoding of LEB128 (whether or not + // the type itself is unsigned). template T ReadUnsigned() { - return Read(kEndUnsignedByteMarker); + if (std::is_unsigned::value) { + return ReadInternal(); + } else { + return bit_cast(ReadUnsigned::type>()); + } } intptr_t Position() const { return current_ - buffer_; } @@ -103,9 +109,15 @@ class ReadStream : public ValueObject { return (end_ - current_); } + // Reads a value of type [T] assuming an encoding of SLEB128 (whether or not + // the type itself is signed). template T Read() { - return Read(kEndByteMarker); + if (std::is_signed::value) { + return ReadInternal(); + } else { + return bit_cast(Read::type>()); + } } uword ReadWordWith32BitReads() { @@ -122,179 +134,112 @@ class ReadStream : public ValueObject { } private: - uint16_t Read16() { return Read16(kEndByteMarker); } - - uint32_t Read32() { return Read32(kEndByteMarker); } - - uint64_t Read64() { return Read64(kEndByteMarker); } - template - T Read(uint8_t end_byte_marker) { + T ReadInternal() { using Unsigned = typename std::make_unsigned::type; const uint8_t* c = current_; - ASSERT(c < end_); - Unsigned b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return b - end_byte_marker; - } - T r = 0; + Unsigned r = 0; uint8_t s = 0; + uint8_t b; do { - r |= static_cast(b) << s; - s += kDataBitsPerByte; ASSERT(c < end_); b = *c++; - } while (b <= kMaxUnsignedDataPerByte); + r |= static_cast(b & kDataByteMask) << s; + s += kDataBitsPerByte; + } while ((b & kMoreDataMask) != 0); current_ = c; - return r | (static_cast(b - end_byte_marker) << s); + // At this point, [s] contains how many data bits have made it into the + // value. If the type is signed, the value negative, and the count of data + // bits is less than the size of the value, then we need to extend the sign + // by setting the remaining (unset) most significant bits (MSBs). + Unsigned sign_bits = 0; + const bool is_signed = std::is_signed::value; + if (is_signed && (b & kSignMask) != 0 && s < (kBitsPerByte * sizeof(T))) { + // Create a bitmask for the current data bits and invert it. + sign_bits = ~((static_cast(1) << s) - 1); + } + return static_cast(r | sign_bits); } - uint16_t Read16(uint8_t end_byte_marker) { - const uint8_t* c = current_; - ASSERT(c < end_); - uint16_t b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return b - end_byte_marker; - } - uint16_t r = b; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 7); - } +// Setting up needed variables for the unrolled loop sections below. +#define UNROLLED_INIT() \ + using Unsigned = typename std::make_unsigned::type; \ + const uint8_t* c = current_; \ + uint8_t b; \ + Unsigned r = 0; - r |= b << 7; - ASSERT(c < end_); - b = *c++; - ASSERT(b > kMaxUnsignedDataPerByte); - current_ = c; - return r | (static_cast(b - end_byte_marker) << 14); +// Part of the unrolled loop where the loop may stop, having read the last part, +// or continue reading. If stopping, extend the sign for signed values. +#define UNROLLED_BODY(bit_start) \ + static_assert(bit_start % kDataBitsPerByte == 0, \ + "Bit start must be a multiple of the data bits per byte"); \ + static_assert(bit_start >= 0 && bit_start < kBitsPerByte * sizeof(T), \ + "Starting unrolled body at invalid bit position"); \ + ASSERT(c < end_); \ + b = *c++; \ + r |= static_cast(b & kDataByteMask) << bit_start; \ + if ((b & kMoreDataMask) == 0) { \ + current_ = c; \ + Unsigned sign_bits = 0; \ + if (std::is_signed::value && (b & kSignMask) != 0) { \ + sign_bits = ~((static_cast(1) << (bit_start + 7)) - 1); \ + } \ + return static_cast(r | sign_bits); \ } - uint32_t Read32(uint8_t end_byte_marker) { - const uint8_t* c = current_; - ASSERT(c < end_); - uint32_t b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return b - end_byte_marker; - } +// If the unrolled end is reached, the last part always includes the original +// sign bit, so no need to sign extend. +#define UNROLLED_END(bit_start) \ + static_assert(bit_start % kDataBitsPerByte == 0, \ + "Bit start must be a multiple of the data bits per byte"); \ + static_assert(bit_start >= 0 && bit_start < kBitsPerByte * sizeof(T), \ + "Starting unrolled end at invalid bit position"); \ + static_assert(bit_start + kDataBitsPerByte >= kBitsPerByte * sizeof(T), \ + "Unrolled end does not contain final bits in value"); \ + ASSERT(c < end_); \ + b = *c++; \ + r |= static_cast(b & kDataByteMask) << bit_start; \ + ASSERT_EQUAL((b & kMoreDataMask), 0); \ + current_ = c; \ + return static_cast(r); - uint32_t r = b; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 7); - } - - r |= b << 7; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 14); - } - - r |= b << 14; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 21); - } - - r |= b << 21; - ASSERT(c < end_); - b = *c++; - ASSERT(b > kMaxUnsignedDataPerByte); - current_ = c; - return r | (static_cast(b - end_byte_marker) << 28); + template + T Read16() { + UNROLLED_INIT(); + UNROLLED_BODY(0); + UNROLLED_BODY(7); + UNROLLED_END(14); } - uint64_t Read64(uint8_t end_byte_marker) { - const uint8_t* c = current_; - ASSERT(c < end_); - uint64_t b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return b - end_byte_marker; - } - uint64_t r = b; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 7); - } - - r |= b << 7; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 14); - } - - r |= b << 14; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 21); - } - - r |= b << 21; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 28); - } - - r |= b << 28; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 35); - } - - r |= b << 35; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 42); - } - - r |= b << 42; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 49); - } - - r |= b << 49; - ASSERT(c < end_); - b = *c++; - if (b > kMaxUnsignedDataPerByte) { - current_ = c; - return r | (static_cast(b - end_byte_marker) << 56); - } - - r |= b << 56; - ASSERT(c < end_); - b = *c++; - ASSERT(b > kMaxUnsignedDataPerByte); - current_ = c; - return r | (static_cast(b - end_byte_marker) << 63); + template + T Read32() { + UNROLLED_INIT(); + UNROLLED_BODY(0); + UNROLLED_BODY(7); + UNROLLED_BODY(14); + UNROLLED_BODY(21); + UNROLLED_END(28); } + template + T Read64() { + UNROLLED_INIT(); + UNROLLED_BODY(0); + UNROLLED_BODY(7); + UNROLLED_BODY(14); + UNROLLED_BODY(21); + UNROLLED_BODY(28); + UNROLLED_BODY(35); + UNROLLED_BODY(42); + UNROLLED_BODY(49); + UNROLLED_BODY(56); + UNROLLED_END(63); + } + +#undef UNROLLED_END +#undef UNROLLED_BODY +#undef UNROLLED_INIT + uint8_t ReadByte() { ASSERT(current_ < end_); return *current_++; @@ -350,7 +295,7 @@ class WriteStream : public ValueObject { class Raw<1, T> { public: static void Write(WriteStream* st, T value) { - st->WriteByte(bit_cast(value)); + st->WriteByte(bit_cast(value)); } }; @@ -390,14 +335,17 @@ class WriteStream : public ValueObject { } } + // Writes the LEB128 encoding of [value] to the stream (whether or not the + // type [T] is unsigned). template void WriteUnsigned(T value) { ASSERT(value >= 0); - while (value > kMaxUnsignedDataPerByte) { - WriteByte(static_cast(value & kByteMask)); - value = value >> kDataBitsPerByte; + if (std::is_unsigned::value) { + WriteInternal(value); + } else { + using Unsigned = typename std::make_unsigned::type; + WriteUnsigned(bit_cast(value)); } - WriteByte(static_cast(value + kEndUnsignedByteMarker)); } void WriteBytes(const void* addr, intptr_t len) { @@ -465,14 +413,16 @@ class WriteStream : public ValueObject { current_ += len; // Not len + 1 to swallow the terminating NUL. } + // Writes the SLEB128 encoding of [value] to the stream (whether or not the + // type [T] is signed). template void Write(T value) { - T v = value; - while (v < kMinDataPerByte || v > kMaxDataPerByte) { - WriteByte(static_cast(v & kByteMask)); - v = v >> kDataBitsPerByte; + if (std::is_signed::value) { + WriteInternal(value); + } else { + using Signed = typename std::make_signed::type; + Write(bit_cast(value)); } - WriteByte(static_cast(v + kEndByteMarker)); } template @@ -487,6 +437,33 @@ class WriteStream : public ValueObject { } private: + template + void WriteInternal(T value) { + T remainder = value; + bool is_last_part; + do { + uint8_t part = static_cast(remainder & kDataByteMask); + remainder >>= kDataBitsPerByte; + // For unsigned types, we're done when the remainder has no bits set. + // For signed types, we're done when either: + // - the remainder has no bits set and the part's sign bit is unset, or + // - the remainder has all bits set and the part's sign bit is set. + // If the remainder matches but the sign bit does not, we need one more + // part to set the sign bit correctly. + is_last_part = + std::is_unsigned::value + ? remainder == static_cast(0) + : (remainder == static_cast(0) && (part & kSignMask) == 0) || + (remainder == ~static_cast(0) && + (part & kSignMask) != 0); + if (!is_last_part) { + // Mark this part as having more parts following it. + part |= kMoreDataMask; + } + WriteByte(part); + } while (!is_last_part); + } + DART_FORCE_INLINE void WriteByte(uint8_t value) { if (current_ >= end_) { Resize(1); @@ -514,7 +491,6 @@ class WriteStream : public ValueObject { ASSERT(end_ > *buffer_); } - private: uint8_t** const buffer_; uint8_t* end_; uint8_t* current_; diff --git a/runtime/vm/elf.cc b/runtime/vm/elf.cc index 606ef610c55..cce2eaccb16 100644 --- a/runtime/vm/elf.cc +++ b/runtime/vm/elf.cc @@ -962,35 +962,8 @@ class DwarfElfStream : public DwarfWriteStream { stream_(ASSERT_NOTNULL(stream)), address_map_(address_map) {} - void sleb128(intptr_t value) { - bool is_last_part = false; - while (!is_last_part) { - uint8_t part = value & 0x7F; - value >>= 7; - if ((value == 0 && (part & 0x40) == 0) || - (value == static_cast(-1) && (part & 0x40) != 0)) { - is_last_part = true; - } else { - part |= 0x80; - } - stream_->WriteFixed(part); - } - } - - void uleb128(uintptr_t value) { - bool is_last_part = false; - while (!is_last_part) { - uint8_t part = value & 0x7F; - value >>= 7; - if (value == 0) { - is_last_part = true; - } else { - part |= 0x80; - } - stream_->WriteFixed(part); - } - } - + void sleb128(intptr_t value) { stream_->Write(value); } + void uleb128(uintptr_t value) { stream_->WriteUnsigned(value); } void u1(uint8_t value) { stream_->WriteFixed(value); } // Can't use WriteFixed for these, as we may not be at aligned positions. void u2(uint16_t value) { stream_->WriteBytes(&value, sizeof(value)); }