dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity
Files
cabaa78cc57d08bcfcd75bfe99a42c19ed497d26
sdk/runtime/vm/unicode.cc
T
Aart Bik 0a42e8e062 [dart/vm] fix potential buffer overrun in unicode utils 
Rationale:
Found by libFuzzer, calling into Utf16::Encode should
ensure sufficient output buffer is available since
it accesses two uint16_t words.

https://github.com/dart-lang/sdk/issues/36235

Change-Id: I14733b0b059f9d710e022b02d143e42c8b5f91e3
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/97042
Reviewed-by: Alexander Markov <alexmarkov@google.com>
Commit-Queue: Aart Bik <ajcbik@google.com>
2019-03-15 18:18:15 +00:00

396 lines
12 KiB
C++
Raw Blame History

// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "vm/unicode.h"
#include "vm/allocation.h"
#include "vm/globals.h"
#include "vm/object.h"
namespace dart {
// clang-format off
const int8_t Utf8::kTrailBytes[256] = {
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0
};
// clang-format on
const uint32_t Utf8::kMagicBits[7] = {0, // Padding.
0x00000000, 0x00003080, 0x000E2080,
0x03C82080, 0xFA082080, 0x82082080};
// Minimum values of code points used to check shortest form.
const uint32_t Utf8::kOverlongMinimum[7] = {0, // Padding.
0x0, 0x80, 0x800,
0x10000, 0xFFFFFFFF, 0xFFFFFFFF};
// Returns the most restricted coding form in which the sequence of utf8
// characters in 'utf8_array' can be represented in, and the number of
// code units needed in that form.
intptr_t Utf8::CodeUnitCount(const uint8_t* utf8_array,
intptr_t array_len,
Type* type) {
intptr_t len = 0;
Type char_type = kLatin1;
for (intptr_t i = 0; i < array_len; i++) {
uint8_t code_unit = utf8_array[i];
if (!IsTrailByte(code_unit)) {
++len;
if (!IsLatin1SequenceStart(code_unit)) { // > U+00FF
if (IsSupplementarySequenceStart(code_unit)) { // >= U+10000
char_type = kSupplementary;
++len;
} else if (char_type == kLatin1) {
char_type = kBMP;
}
}
}
}
*type = char_type;
return len;
}
// Returns true if str is a valid NUL-terminated UTF-8 string.
bool Utf8::IsValid(const uint8_t* utf8_array, intptr_t array_len) {
intptr_t i = 0;
while (i < array_len) {
uint32_t ch = utf8_array[i] & 0xFF;
intptr_t j = 1;
if (ch >= 0x80) {
int8_t num_trail_bytes = kTrailBytes[ch];
bool is_malformed = false;
for (; j < num_trail_bytes; ++j) {
if ((i + j) < array_len) {
uint8_t code_unit = utf8_array[i + j];
is_malformed |= !IsTrailByte(code_unit);
ch = (ch << 6) + code_unit;
} else {
return false;
}
}
ch -= kMagicBits[num_trail_bytes];
if (!((is_malformed == false) && (j == num_trail_bytes) &&
!Utf::IsOutOfRange(ch) && !IsNonShortestForm(ch, j))) {
return false;
}
}
i += j;
}
return true;
}
intptr_t Utf8::Length(int32_t ch) {
if (ch <= kMaxOneByteChar) {
return 1;
} else if (ch <= kMaxTwoByteChar) {
return 2;
} else if (ch <= kMaxThreeByteChar) {
return 3;
}
ASSERT(ch <= kMaxFourByteChar);
return 4;
}
// A constant mask that can be 'and'ed with a word of data to determine if it
// is all ASCII (with no Latin1 characters).
#if defined(ARCH_IS_64_BIT)
static const uintptr_t kAsciiWordMask = DART_UINT64_C(0x8080808080808080);
#else
static const uintptr_t kAsciiWordMask = 0x80808080u;
#endif
intptr_t Utf8::Length(const String& str) {
if (str.IsOneByteString() || str.IsExternalOneByteString()) {
// For 1-byte strings, all code points < 0x80 have single-byte UTF-8
// encodings and all >= 0x80 have two-byte encodings. To get the length,
// start with the number of code points and add the number of high bits in
// the bytes.
uintptr_t char_length = str.Length();
uintptr_t length = char_length;
const uintptr_t* data;
NoSafepointScope no_safepoint;
if (str.IsOneByteString()) {
data = reinterpret_cast<const uintptr_t*>(OneByteString::DataStart(str));
} else {
data = reinterpret_cast<const uintptr_t*>(
ExternalOneByteString::DataStart(str));
}
uintptr_t i;
for (i = sizeof(uintptr_t); i <= char_length; i += sizeof(uintptr_t)) {
uintptr_t chunk = *data++;
chunk &= kAsciiWordMask;
if (chunk != 0) {
// Shuffle the bits until we have a count of bits in the low nibble.
#if defined(ARCH_IS_64_BIT)
chunk += chunk >> 32;
#endif
chunk += chunk >> 16;
chunk += chunk >> 8;
length += (chunk >> 7) & 0xf;
}
}
// Take care of the tail of the string, the last length % wordsize chars.
i -= sizeof(uintptr_t);
for (; i < char_length; i++) {
if (str.CharAt(i) > kMaxOneByteChar) length++;
}
return length;
}
// Slow case for 2-byte strings that handles surrogate pairs and longer UTF-8
// encodings.
intptr_t length = 0;
String::CodePointIterator it(str);
while (it.Next()) {
int32_t ch = it.Current();
length += Utf8::Length(ch);
}
return length;
}
intptr_t Utf8::Encode(int32_t ch, char* dst) {
static const int kMask = ~(1 << 6);
if (ch <= kMaxOneByteChar) {
dst[0] = ch;
return 1;
}
if (ch <= kMaxTwoByteChar) {
dst[0] = 0xC0 | (ch >> 6);
dst[1] = 0x80 | (ch & kMask);
return 2;
}
if (ch <= kMaxThreeByteChar) {
dst[0] = 0xE0 | (ch >> 12);
dst[1] = 0x80 | ((ch >> 6) & kMask);
dst[2] = 0x80 | (ch & kMask);
return 3;
}
ASSERT(ch <= kMaxFourByteChar);
dst[0] = 0xF0 | (ch >> 18);
dst[1] = 0x80 | ((ch >> 12) & kMask);
dst[2] = 0x80 | ((ch >> 6) & kMask);
dst[3] = 0x80 | (ch & kMask);
return 4;
}
intptr_t Utf8::Encode(const String& src, char* dst, intptr_t len) {
uintptr_t array_len = len;
intptr_t pos = 0;
ASSERT(static_cast<intptr_t>(array_len) >= Length(src));
if (src.IsOneByteString() || src.IsExternalOneByteString()) {
// For 1-byte strings, all code points < 0x80 have single-byte UTF-8
// encodings and all >= 0x80 have two-byte encodings.
const uintptr_t* data;
NoSafepointScope scope;
if (src.IsOneByteString()) {
data = reinterpret_cast<const uintptr_t*>(OneByteString::DataStart(src));
} else {
data = reinterpret_cast<const uintptr_t*>(
ExternalOneByteString::DataStart(src));
}
uintptr_t char_length = src.Length();
uintptr_t pos = 0;
ASSERT(kMaxOneByteChar + 1 == 0x80);
for (uintptr_t i = 0; i < char_length; i += sizeof(uintptr_t)) {
// Read the input one word at a time and just write it verbatim if it is
// plain ASCII, as determined by the mask.
if (i + sizeof(uintptr_t) <= char_length &&
(*data & kAsciiWordMask) == 0 &&
pos + sizeof(uintptr_t) <= array_len) {
StoreUnaligned(reinterpret_cast<uintptr_t*>(dst + pos), *data);
pos += sizeof(uintptr_t);
} else {
// Process up to one word of input that contains non-ASCII Latin1
// characters.
const uint8_t* p = reinterpret_cast<const uint8_t*>(data);
const uint8_t* limit =
Utils::Minimum(p + sizeof(uintptr_t), p + (char_length - i));
for (; p < limit; p++) {
uint8_t c = *p;
// These calls to Length and Encode get inlined and the cases for 3
// and 4 byte sequences are removed.
intptr_t bytes = Length(c);
if (pos + bytes > array_len) {
return pos;
}
Encode(c, reinterpret_cast<char*>(dst) + pos);
pos += bytes;
}
}
data++;
}
} else {
// For two-byte strings, which can contain 3 and 4-byte UTF-8 encodings,
// which can result in surrogate pairs, use the more general code.
String::CodePointIterator it(src);
while (it.Next()) {
int32_t ch = it.Current();
intptr_t num_bytes = Utf8::Length(ch);
if (pos + num_bytes > len) {
break;
}
Utf8::Encode(ch, &dst[pos]);
pos += num_bytes;
}
}
return pos;
}
intptr_t Utf8::Decode(const uint8_t* utf8_array,
intptr_t array_len,
int32_t* dst) {
uint32_t ch = utf8_array[0] & 0xFF;
intptr_t i = 1;
if (ch >= 0x80) {
intptr_t num_trail_bytes = kTrailBytes[ch];
bool is_malformed = false;
for (; i < num_trail_bytes; ++i) {
if (i < array_len) {
uint8_t code_unit = utf8_array[i];
is_malformed |= !IsTrailByte(code_unit);
ch = (ch << 6) + code_unit;
} else {
*dst = -1;
return 0;
}
}
ch -= kMagicBits[num_trail_bytes];
if (!((is_malformed == false) && (i == num_trail_bytes) &&
!Utf::IsOutOfRange(ch) && !IsNonShortestForm(ch, i))) {
*dst = -1;
return 0;
}
}
*dst = ch;
return i;
}
intptr_t Utf8::ReportInvalidByte(const uint8_t* utf8_array,
intptr_t array_len,
intptr_t len) {
intptr_t i = 0;
intptr_t j = 0;
intptr_t num_bytes;
for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
int32_t ch;
bool is_supplementary = IsSupplementarySequenceStart(utf8_array[i]);
num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
if (ch == -1) {
break; // Invalid input.
}
if (is_supplementary) {
j = j + 1;
}
}
OS::PrintErr("Invalid UTF8 sequence encountered, ");
for (intptr_t idx = 0; idx < 10 && (i + idx) < array_len; idx++) {
OS::PrintErr("(Error Code: %X + idx: %" Pd " )", utf8_array[idx + i],
(idx + i));
}
OS::PrintErr("\n");
return i;
}
bool Utf8::DecodeToLatin1(const uint8_t* utf8_array,
intptr_t array_len,
uint8_t* dst,
intptr_t len) {
intptr_t i = 0;
intptr_t j = 0;
intptr_t num_bytes;
for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
int32_t ch;
ASSERT(IsLatin1SequenceStart(utf8_array[i]));
num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
if (ch == -1) {
return false; // Invalid input.
}
ASSERT(Utf::IsLatin1(ch));
dst[j] = ch;
}
if ((i < array_len) && (j == len)) {
return false; // Output overflow.
}
return true; // Success.
}
bool Utf8::DecodeToUTF16(const uint8_t* utf8_array,
intptr_t array_len,
uint16_t* dst,
intptr_t len) {
intptr_t i = 0;
intptr_t j = 0;
intptr_t num_bytes;
for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
int32_t ch;
bool is_supplementary = IsSupplementarySequenceStart(utf8_array[i]);
num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
if (ch == -1) {
return false; // Invalid input.
}
if (is_supplementary) {
if (j == (len - 1)) return false; // Output overflow.
Utf16::Encode(ch, &dst[j]);
j = j + 1;
} else {
dst[j] = ch;
}
}
if ((i < array_len) && (j == len)) {
return false; // Output overflow.
}
return true; // Success.
}
bool Utf8::DecodeToUTF32(const uint8_t* utf8_array,
intptr_t array_len,
int32_t* dst,
intptr_t len) {
intptr_t i = 0;
intptr_t j = 0;
intptr_t num_bytes;
for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
int32_t ch;
num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
if (ch == -1) {
return false; // Invalid input.
}
dst[j] = ch;
}
if ((i < array_len) && (j == len)) {
return false; // Output overflow.
}
return true; // Success.
}
bool Utf8::DecodeCStringToUTF32(const char* str, int32_t* dst, intptr_t len) {
ASSERT(str != NULL);
intptr_t array_len = strlen(str);
const uint8_t* utf8_array = reinterpret_cast<const uint8_t*>(str);
return Utf8::DecodeToUTF32(utf8_array, array_len, dst, len);
}
void Utf16::Encode(int32_t codepoint, uint16_t* dst) {
ASSERT(codepoint > Utf16::kMaxCodeUnit);
ASSERT(dst != NULL);
dst[0] = (Utf16::kLeadSurrogateOffset + (codepoint >> 10));
dst[1] = (0xDC00 + (codepoint & 0x3FF));
}
} // namespace dart
Reference in New Issue View Git Blame Copy Permalink