From db17ef4ea30e263571b2d2e03487eb3a1967e3f2 Mon Sep 17 00:00:00 2001 From: "dcarlson@google.com" Date: Wed, 1 Feb 2012 22:25:46 +0000 Subject: [PATCH] String encoding utility methods and tests for Unicode, UTF-8, -16 and -32. BUG= TEST= Review URL: https://chromiumcodereview.appspot.com//9233041 git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@3814 260f80e4-7a28-3924-810f-c04153c831b5 --- utils/string_encoding/Unicode.dart | 37 ++ utils/string_encoding/Utf16.dart | 273 +++++++++++ utils/string_encoding/Utf32.dart | 273 +++++++++++ utils/string_encoding/Utf8.dart | 8 + utils/string_encoding/Utf8_impl.dart | 221 +++++++++ utils/string_encoding/unicode_core.dart | 181 +++++++ utils/tests/string_encoding/DUnit.dart | 117 +++++ utils/tests/string_encoding/run_tests.dart | 26 + .../string_encoding/unicode_core_tests.dart | 103 ++++ .../tests/string_encoding/unicode_tests.dart | 37 ++ utils/tests/string_encoding/utf16_tests.dart | 106 ++++ utils/tests/string_encoding/utf32_tests.dart | 158 ++++++ utils/tests/string_encoding/utf8_tests.dart | 457 ++++++++++++++++++ 13 files changed, 1997 insertions(+) create mode 100644 utils/string_encoding/Unicode.dart create mode 100644 utils/string_encoding/Utf16.dart create mode 100644 utils/string_encoding/Utf32.dart create mode 100644 utils/string_encoding/Utf8.dart create mode 100644 utils/string_encoding/Utf8_impl.dart create mode 100644 utils/string_encoding/unicode_core.dart create mode 100644 utils/tests/string_encoding/DUnit.dart create mode 100755 utils/tests/string_encoding/run_tests.dart create mode 100755 utils/tests/string_encoding/unicode_core_tests.dart create mode 100755 utils/tests/string_encoding/unicode_tests.dart create mode 100755 utils/tests/string_encoding/utf16_tests.dart create mode 100755 utils/tests/string_encoding/utf32_tests.dart create mode 100755 utils/tests/string_encoding/utf8_tests.dart diff --git a/utils/string_encoding/Unicode.dart b/utils/string_encoding/Unicode.dart new file mode 100644 index 00000000000..c0c9655a46c --- /dev/null +++ b/utils/string_encoding/Unicode.dart @@ -0,0 +1,37 @@ +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#library("unicode"); +#import("unicode_core.dart"); + +/** + * Provide Unicode codepoints for a given string. + */ +List stringToCodepoints(String str) { + List codepoints; + // TODO is16BitCodeUnit() is used to work around a bug with frog/dartc + // (http://code.google.com/p/dart/issues/detail?id=1357). Consider + // removing after this issue is resolved. + if (is16BitCodeUnit()) { + codepoints = utf16CodeUnitsToCodepoints(str.charCodes()); + } else { + codepoints = str.charCodes(); + } + return codepoints; +} + +/** + * Generate a string for the provided Unicode codepoints. + */ +String codepointsToString(List codepoints) { + // TODO is16BitCodeUnit() is used to work around a bug with frog/dartc + // (http://code.google.com/p/dart/issues/detail?id=1357). Consider + // removing after this issue is resolved. + if (is16BitCodeUnit()) { + return new String.fromCharCodes( + codepointsToUtf16CodeUnits(codepoints)); + } else { + return new String.fromCharCodes(codepoints); + } +} diff --git a/utils/string_encoding/Utf16.dart b/utils/string_encoding/Utf16.dart new file mode 100644 index 00000000000..12fde6dec97 --- /dev/null +++ b/utils/string_encoding/Utf16.dart @@ -0,0 +1,273 @@ +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#library("utf16"); +#import("unicode_core.dart"); +#import("unicode.dart"); + +/** + * Produce a String from a sequence of UTF-16 encoded bytes. The parameters + * allow an offset into a list of bytes (as int), limiting the length of the + * values be decoded and the ability of override the default Unicode + * replacement character. Set the replacementCharacter to null to throw an + * IllegalArgumentException rather than replace the bad value. + */ +String decodeFromUtf16(List bytes, [int offset = 0, int length, + int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { + List codeUnits = + _utf16ToUtf16CodeUnits(bytes, offset, length); + // TODO is16BitCodeUnit() is used to work around a bug with frog/dartc + // (http://code.google.com/p/dart/issues/detail?id=1357). Consider + // removing after this issue is resolved. + if (is16BitCodeUnit()) { + return new String.fromCharCodes(codeUnits); + } else { + return new String.fromCharCodes( + utf16CodeUnitsToCodepoints(codeUnits, 0, null, replacementCodepoint)); + } +} + +/** + * Produce a String from a sequence of UTF-16BE encoded bytes. The parameters + * allow an offset into a list of bytes (as int), limiting the length of the + * values be decoded and the ability of override the default Unicode + * replacement character. Set the replacementCharacter to null to throw an + * IllegalArgumentException rather than replace the bad value. + */ +String decodeFromUtf16be(List bytes, [int offset = 0, int length, + bool stripBom = true, + int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { + List codeUnits = + _utf16beToUtf16CodeUnits(bytes, offset, length, stripBom); + // TODO is16BitCodeUnit() is used to work around a bug with frog/dartc + // (http://code.google.com/p/dart/issues/detail?id=1357). Consider + // removing after this issue is resolved. + if (is16BitCodeUnit()) { + return new String.fromCharCodes(codeUnits); + } else { + return new String.fromCharCodes( + utf16CodeUnitsToCodepoints(codeUnits, 0, null, replacementCodepoint)); + } +} + +/** + * Produce a String from a sequence of UTF-16LE encoded bytes. The parameters + * allow an offset into a list of bytes (as int), limiting the length of the + * values be decoded and the ability of override the default Unicode + * replacement character. Set the replacementCharacter to null to throw an + * IllegalArgumentException rather than replace the bad value. + */ +String decodeFromUtf16le(List bytes, [int offset = 0, int length, + bool stripBom = true, + int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { + List codeUnits = + _utf16leToUtf16CodeUnits(bytes, offset, length, stripBom); + // TODO is16BitCodeUnit() is used to work around a bug with frog/dartc + // (http://code.google.com/p/dart/issues/detail?id=1357). Consider + // removing after this issue is resolved. + if (is16BitCodeUnit()) { + return new String.fromCharCodes(codeUnits); + } else { + return new String.fromCharCodes( + utf16CodeUnitsToCodepoints(codeUnits, 0, null, replacementCodepoint)); + } +} + +/** + * Produce a sequence of UTF-16 encoded bytes. + */ +List encodeAsUtf16(String str) => + encodeAsUtf16be(str, true); + +/** + * Produce a sequence of UTF-16BE encoded bytes. + */ +List encodeAsUtf16be(String str, [bool writeBOM = false]) { + List utf16CodeUnits = _stringToUtf16CodeUnits(str); + List encoding = + new List(2 * utf16CodeUnits.length + (writeBOM ? 2 : 0)); + int i = 0; + if (writeBOM) { + encoding[i++] = UNICODE_UTF_BOM_HI; + encoding[i++] = UNICODE_UTF_BOM_LO; + } + for (int unit in utf16CodeUnits) { + encoding[i++] = (unit & UNICODE_BYTE_ONE_MASK) >> 8; + encoding[i++] = unit & UNICODE_BYTE_ZERO_MASK; + } + return encoding; +} + +/** + * Produce a sequence of UTF-16LE encoded bytes. + */ +List encodeAsUtf16le(String str, [bool writeBOM = false]) { + List utf16CodeUnits = _stringToUtf16CodeUnits(str); + List encoding = + new List(2 * utf16CodeUnits.length + (writeBOM ? 2 : 0)); + int i = 0; + if (writeBOM) { + encoding[i++] = UNICODE_UTF_BOM_LO; + encoding[i++] = UNICODE_UTF_BOM_HI; + } + for (int unit in utf16CodeUnits) { + encoding[i++] = unit & UNICODE_BYTE_ZERO_MASK; + encoding[i++] = (unit & UNICODE_BYTE_ONE_MASK) >> 8; + } + return encoding; +} + +bool hasUtf16Bom(List utf32EncodedBytes, [int offset = 0, int length]) { + return hasUtf16beBom(utf32EncodedBytes, offset, length) || + hasUtf16leBom(utf32EncodedBytes, offset, length); +} + +bool hasUtf16beBom(List utf16EncodedBytes, [int offset = 0, int length]) { + if (!(offset >= 0)) { + throw new IllegalArgumentException("offset"); + } + + if (!(length == null || length >= 0)) { + throw new IllegalArgumentException("length"); + } + + int end = length != null ? + Math.min(utf16EncodedBytes.length, offset + length) : + utf16EncodedBytes.length; + + return (offset + 2) <= end && + utf16EncodedBytes[offset] == UNICODE_UTF_BOM_HI && + utf16EncodedBytes[offset + 1] == UNICODE_UTF_BOM_LO; +} + +bool hasUtf16leBom(List utf16EncodedBytes, [int offset = 0, int length]) { + if (!(offset >= 0)) { + throw new IllegalArgumentException("offset"); + } + + if (!(length == null || length >= 0)) { + throw new IllegalArgumentException("length"); + } + + int end = length != null ? + Math.min(utf16EncodedBytes.length, offset + length) : + utf16EncodedBytes.length; + + return (offset + 2) <= end && + utf16EncodedBytes[offset] == UNICODE_UTF_BOM_LO && + utf16EncodedBytes[offset + 1] == UNICODE_UTF_BOM_HI; +} + +int _sizeCodeUnits(int utf16CodeUnitsLength) { + int v = ((utf16CodeUnitsLength)/2).floor().toInt(); + return v; +} + +List _stringToUtf16CodeUnits(String str) { + // TODO is16BitCodeUnit() is used to work around a bug with frog/dartc + // (http://code.google.com/p/dart/issues/detail?id=1357). Consider + // removing after this issue is resolved. + if (is16BitCodeUnit()) { + return str.charCodes(); + } else { + return codepointsToUtf16CodeUnits(str.charCodes()); + } +} + +/** + * Convert UTF-16BE encoded bytes to utf16 code units by grouping 1-2 bytes + * to produce the code unit (0-(2^16)-1). + */ +List _utf16beToUtf16CodeUnits( + List utf16beEncodedBytes, [int offset = 0, int length, + bool stripBom = true]) { + if (!(offset >= 0)) { + throw new IllegalArgumentException("offset"); + } + + if (!(length == null || length >= 0)) { + throw new IllegalArgumentException("length"); + } + + int end = length != null ? + Math.min(utf16beEncodedBytes.length, offset + length) : + utf16beEncodedBytes.length; + + int i = (stripBom && hasUtf16beBom(utf16beEncodedBytes, offset, length)) ? + offset + 2 : offset; + List codeUnits = + new List(_sizeCodeUnits(end - i)); + int lastIndex = end - 1; + int j = 0; + while (i < lastIndex) { + int hi = utf16beEncodedBytes[i++]; + int lo = utf16beEncodedBytes[i++]; + codeUnits[j++] = (hi << 8) | lo; + } + return codeUnits; +} + +/** + * Convert UTF-16LE encoded bytes to utf16 code units by grouping 1-2 bytes + * to produce the code unit (0-(2^16)-1). + */ +List _utf16leToUtf16CodeUnits( + List utf16leEncodedBytes, [int offset = 0, int length, + bool stripBom = true]) { + if (!(offset >= 0)) { + throw new IllegalArgumentException("offset"); + } + + if (!(length == null || length >= 0)) { + throw new IllegalArgumentException("length"); + } + + int end = length != null ? + Math.min(utf16leEncodedBytes.length, offset + length) : + utf16leEncodedBytes.length; + + int i = (stripBom && hasUtf16leBom(utf16leEncodedBytes, offset, length)) ? + offset + 2 : offset; + List codeUnits = + new List(_sizeCodeUnits(end - i)); + int lastIndex = end - 1; + int j = 0; + while (i < lastIndex) { + int lo = utf16leEncodedBytes[i++]; + int hi = utf16leEncodedBytes[i++]; + codeUnits[j] = (hi << 8) | lo; + } + return codeUnits; +} + +/** + * Convert UTF-16 encoded bytes to utf16 code units by grouping 1-2 bytes + * to produce the code unit (0-(2^16)-1). Relies on BOM to determine + * endian-ness, and defaults to BE. + */ +List _utf16ToUtf16CodeUnits( + List utf16EncodedBytes, [int offset = 0, int length]) { + if (!(offset >= 0)) { + throw new IllegalArgumentException("offset"); + } + + if (!(length == null || length >= 0)) { + throw new IllegalArgumentException("length"); + } + + int end = length != null ? + Math.min(utf16EncodedBytes.length, offset + length) : + utf16EncodedBytes.length; + + if (hasUtf16beBom(utf16EncodedBytes, offset, length)) { + return _utf16beToUtf16CodeUnits(utf16EncodedBytes, offset + 2, + end - (offset + 2), false); + } else if (hasUtf16leBom(utf16EncodedBytes, offset, length)) { + return _utf16leToUtf16CodeUnits(utf16EncodedBytes, offset + 2, + end - (offset + 2), false); + } else { + return _utf16beToUtf16CodeUnits( + utf16EncodedBytes, offset, end - offset, false); + } +} diff --git a/utils/string_encoding/Utf32.dart b/utils/string_encoding/Utf32.dart new file mode 100644 index 00000000000..099f3eebda6 --- /dev/null +++ b/utils/string_encoding/Utf32.dart @@ -0,0 +1,273 @@ +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#library("utf32"); +#import("unicode_core.dart"); +#import("unicode.dart"); + +/** + * Produce a String from a sequence of UTF-32 encoded bytes. The parameters + * allow an offset into a list of bytes (as int), limiting the length of the + * values be decoded and the ability of override the default Unicode + * replacement character. Set the replacementCharacter to null to throw an + * IllegalArgumentException rather than replace the bad value. + */ +String decodeFromUtf32(List bytes, [int offset = 0, int length, + int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) => + codepointsToString(_utf32ToCodePoints(bytes, offset, length, + replacementCodepoint)); + +/** + * Produce a String from a sequence of UTF-32BE encoded bytes. The parameters + * allow an offset into a list of bytes (as int), limiting the length of the + * values be decoded and the ability of override the default Unicode + * replacement character. Set the replacementCharacter to null to throw an + * IllegalArgumentException rather than replace the bad value. + */ +String decodeFromUtf32be( + List bytes, [int offset = 0, int length, bool stripBom = true, + int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) => + codepointsToString(_utf32beToCodePoints(bytes, offset, length, stripBom, + replacementCodepoint)); + +/** + * Produce a String from a sequence of UTF-32LE encoded bytes. The parameters + * allow an offset into a list of bytes (as int), limiting the length of the + * values be decoded and the ability of override the default Unicode + * replacement character. Set the replacementCharacter to null to throw an + * IllegalArgumentException rather than replace the bad value. + */ +String decodeFromUtf32le( + List bytes, [int offset = 0, int length, bool stripBom = true, + int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) => + codepointsToString(_utf32leToCodePoints(bytes, offset, length, stripBom, + replacementCodepoint)); + +/** + * Produce a sequence of UTF-32 encoded bytes. + */ +List encodeAsUtf32(String str) => + encodeAsUtf32be(str, true); + +/** + * Produce a sequence of UTF-32BE encoded bytes. + */ +List encodeAsUtf32be(String str, [bool writeBOM = false]) { + List utf32CodeUnits = stringToCodepoints(str); + List encoding = new List(4 * utf32CodeUnits.length + + (writeBOM ? 4 : 0)); + int i = 0; + if (writeBOM) { + encoding[i++] = 0; + encoding[i++] = 0; + encoding[i++] = UNICODE_UTF_BOM_HI; + encoding[i++] = UNICODE_UTF_BOM_LO; + } + for (int unit in utf32CodeUnits) { + encoding[i++] = (unit >> 24) & UNICODE_BYTE_ZERO_MASK; + encoding[i++] = (unit >> 16) & UNICODE_BYTE_ZERO_MASK; + encoding[i++] = (unit >> 8) & UNICODE_BYTE_ZERO_MASK; + encoding[i++] = unit & UNICODE_BYTE_ZERO_MASK; + } + return encoding; +} + +/** + * Produce a sequence of UTF-32LE encoded bytes. + */ +List encodeAsUtf32le(String str, [bool writeBOM = false]) { + List utf32CodeUnits = stringToCodepoints(str); + List encoding = new List(4 * utf32CodeUnits.length + + (writeBOM ? 4 : 0)); + int i = 0; + if (writeBOM) { + encoding[i++] = UNICODE_UTF_BOM_LO; + encoding[i++] = UNICODE_UTF_BOM_HI; + encoding[i++] = 0; + encoding[i++] = 0; + } + for (int unit in utf32CodeUnits) { + encoding[i++] = unit & UNICODE_BYTE_ZERO_MASK; + encoding[i++] = (unit >> 8) & UNICODE_BYTE_ZERO_MASK; + encoding[i++] = (unit >> 16) & UNICODE_BYTE_ZERO_MASK; + encoding[i++] = (unit >> 24) & UNICODE_BYTE_ZERO_MASK; + } + return encoding; +} + +bool hasUtf32Bom( + List utf32EncodedBytes, [int offset = 0, int length]) { + return hasUtf32beBom(utf32EncodedBytes, offset, length) || + hasUtf32leBom(utf32EncodedBytes, offset, length); +} + +bool hasUtf32beBom(List utf32EncodedBytes, [int offset = 0, int length]) { + if (!(offset >= 0)) { + throw new IllegalArgumentException("offset"); + } + + if (!(length == null || length >= 0)) { + throw new IllegalArgumentException("length"); + } + + int end = length != null ? + Math.min(utf32EncodedBytes.length, offset + length) : + utf32EncodedBytes.length; + + return (offset + 4) <= end && + utf32EncodedBytes[offset] == 0 && + utf32EncodedBytes[offset + 1] == 0 && + utf32EncodedBytes[offset + 2] == UNICODE_UTF_BOM_HI && + utf32EncodedBytes[offset + 3] == UNICODE_UTF_BOM_LO; +} + +bool hasUtf32leBom(List utf32EncodedBytes, [int offset = 0, int length]) { + if (!(offset >= 0)) { + throw new IllegalArgumentException("offset"); + } + + if (!(length == null || length >= 0)) { + throw new IllegalArgumentException("length"); + } + + int end = length != null ? + Math.min(utf32EncodedBytes.length, offset + length) : + utf32EncodedBytes.length; + + return (offset + 4) <= end && + utf32EncodedBytes[offset] == UNICODE_UTF_BOM_LO && + utf32EncodedBytes[offset + 1] == UNICODE_UTF_BOM_HI && + utf32EncodedBytes[offset + 2] == 0 && + utf32EncodedBytes[offset + 3] == 0; +} + +void _addReplacementCodepoint(List codepointBuffer, int offset, + int replacementCodepoint) { + if(replacementCodepoint != null) { + codepointBuffer[offset] = replacementCodepoint; + } else { + throw new IllegalArgumentException("Invalid encoding"); + } +} + +int _sizeCodepoints(int utf32BytesLength) => + ((utf32BytesLength)/4).ceil().toInt(); + +/** + * Joins groups of 4 bytes (0-255) UTF-32BE to produce single code points. + */ +List _utf32beToCodePoints( + List utf32beEncodedBytes, [int offset = 0, int length, + bool stripBom = true, + int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { + if (!(offset >= 0)) { + throw new IllegalArgumentException("offset"); + } + + if (!(length == null || length >= 0)) { + throw new IllegalArgumentException("length"); + } + + int end = length != null ? + Math.min(utf32beEncodedBytes.length, offset + length) : + utf32beEncodedBytes.length; + + int i = (stripBom && hasUtf32beBom(utf32beEncodedBytes, offset, length)) ? + offset + 4 : offset; + int lastIndex = end - 3; + List codepoints = new List(_sizeCodepoints(end - i)); + int j = 0; + while (i < lastIndex) { + int value = utf32beEncodedBytes[i++]; + value = (value << 8) + utf32beEncodedBytes[i++]; + value = (value << 8) + utf32beEncodedBytes[i++]; + value = (value << 8) + utf32beEncodedBytes[i++]; + if (_validCodepoint(value)) { + codepoints[j++] = value; + } else { + _addReplacementCodepoint(codepoints, j++, replacementCodepoint); + } + } + while (j < codepoints.length) { + _addReplacementCodepoint(codepoints, j++, replacementCodepoint); + } + return codepoints; +} + +/** + * Joins groups of 4 bytes (0-255) UTF-32LE to produce single code points. + */ +List _utf32leToCodePoints( + List utf32leEncodedBytes, [int offset = 0, int length, + bool stripBom = true, + int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { + if (!(offset >= 0)) { + throw new IllegalArgumentException("offset"); + } + + if (!(length == null || length >= 0)) { + throw new IllegalArgumentException("length"); + } + + int end = length != null ? + Math.min(utf32leEncodedBytes.length, offset + length) : + utf32leEncodedBytes.length; + + int i = (stripBom && hasUtf32leBom(utf32leEncodedBytes, offset, length)) ? + offset + 4 : offset; + int lastIndex = end - 3; + List codepoints = new List(_sizeCodepoints(end - i)); + int j = 0; + while (i < lastIndex) { + int value = utf32leEncodedBytes[i+3]; + value = (value << 8) + utf32leEncodedBytes[i+2]; + value = (value << 8) + utf32leEncodedBytes[i+1]; + value = (value << 8) + utf32leEncodedBytes[i]; + i += 4; + if (_validCodepoint(value)) { + codepoints[j++] = value; + } else { + _addReplacementCodepoint(codepoints, j++, replacementCodepoint); + } + } + while (j < codepoints.length) { + _addReplacementCodepoint(codepoints, j++, replacementCodepoint); + } + return codepoints; +} + +/** + * Joins groups of 4 bytes (0-255) UTF-32 to produce single code points. + */ +List _utf32ToCodePoints(List utf32EncodedBytes, [int offset = 0, + int length, + int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { + if (!(offset >= 0)) { + throw new IllegalArgumentException("offset"); + } + + if (!(length == null || length >= 0)) { + throw new IllegalArgumentException("length"); + } + + int end = length != null ? + Math.min(utf32EncodedBytes.length, offset + length) : + utf32EncodedBytes.length; + + if (hasUtf32beBom(utf32EncodedBytes, offset, length)) { + return _utf32beToCodePoints(utf32EncodedBytes, offset + 4, + end - (offset + 4), false); + } else if (hasUtf32leBom(utf32EncodedBytes, offset, length)) { + return _utf32leToCodePoints(utf32EncodedBytes, offset + 4, + end - (offset + 4), false); + } else { + return _utf32beToCodePoints(utf32EncodedBytes, offset, end - offset); + } +} + +bool _validCodepoint(int codepoint) { + return (codepoint >= 0 && codepoint < UNICODE_UTF16_RESERVED_LO) || + (codepoint > UNICODE_UTF16_RESERVED_HI && + codepoint < UNICODE_VALID_RANGE_MAX); +} diff --git a/utils/string_encoding/Utf8.dart b/utils/string_encoding/Utf8.dart new file mode 100644 index 00000000000..21c5d8e4527 --- /dev/null +++ b/utils/string_encoding/Utf8.dart @@ -0,0 +1,8 @@ +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#library("utf8"); +#import("unicode_core.dart"); +#import("unicode.dart"); +#source("utf8_impl.dart"); diff --git a/utils/string_encoding/Utf8_impl.dart b/utils/string_encoding/Utf8_impl.dart new file mode 100644 index 00000000000..213bb72ea4a --- /dev/null +++ b/utils/string_encoding/Utf8_impl.dart @@ -0,0 +1,221 @@ +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +final int _UTF8_ONE_BYTE_MAX = 0x7f; +final int _UTF8_TWO_BYTE_MAX = 0x7ff; +final int _UTF8_THREE_BYTE_MAX = 0xffff; + +final int _UTF8_LO_SIX_BIT_MASK = 0x3f; + +final int _UTF8_FIRST_BYTE_OF_TWO_BASE = 0xc0; +final int _UTF8_FIRST_BYTE_OF_THREE_BASE = 0xe0; +final int _UTF8_FIRST_BYTE_OF_FOUR_BASE = 0xf0; +final int _UTF8_FIRST_BYTE_OF_FIVE_BASE = 0xf8; +final int _UTF8_FIRST_BYTE_OF_SIX_BASE = 0xfc; + +final int _UTF8_FIRST_BYTE_OF_TWO_MASK = 0x1f; +final int _UTF8_FIRST_BYTE_OF_THREE_MASK = 0xf; +final int _UTF8_FIRST_BYTE_OF_FOUR_MASK = 0x7; + +final int _UTF8_FIRST_BYTE_BOUND_EXCL = 0xfe; +final int _UTF8_SUBSEQUENT_BYTE_BASE = 0x80; + +/** + * Produce a String from a sequence of UTF-8 encoded bytes. The parameters + * allow an offset into a list of bytes (as int), limiting the length of the + * values be decoded and the ability of override the default Unicode + * replacement character. Set the replacementCharacter to null to throw an + * IllegalArgumentException rather than replace the bad value. + */ +String decodeFromUtf8(List bytes, [int offset = 0, int length, + int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) => + codepointsToString(_utf8ToCodepoints( + bytes, offset, length, replacementCodepoint)); + +/** + * Produce a sequence of UTF-8 encoded bytes from the provided string. + */ +List encodeAsUtf8(String str) => + _codepointsToUtf8(stringToCodepoints(str)); + +int _addToEncoding(int offset, int bytes, int value, List buffer) { + while(bytes > 0) { + buffer[offset + bytes] = _UTF8_SUBSEQUENT_BYTE_BASE | + (value & _UTF8_LO_SIX_BIT_MASK); + value = value >> 6; + bytes--; + } + return value; +} + +/** + * Encode code points as UTF-8 code units. + */ +List _codepointsToUtf8( + List codepoints, [int offset = 0, int length]) { + if (!(offset >= 0)) { + throw new IllegalArgumentException("offset"); + } + + if (!(length == null || length >= 0)) { + throw new IllegalArgumentException("length"); + } + + int end = length != null ? + Math.min(codepoints.length, offset + length) : + codepoints.length; + + int encodedLength = 0; + for (int i = offset; i < end; i++) { + int value = codepoints[i]; + if (value < 0 || value > UNICODE_VALID_RANGE_MAX) { + encodedLength += 3; + } else if (value <= _UTF8_ONE_BYTE_MAX) { + encodedLength++; + } else if (value <= _UTF8_TWO_BYTE_MAX) { + encodedLength += 2; + } else if (value <= _UTF8_THREE_BYTE_MAX) { + encodedLength += 3; + } else if (value <= UNICODE_VALID_RANGE_MAX) { + encodedLength += 4; + } + } + + List encoded = new List(encodedLength); + int insertAt = 0; + for (int i = offset; i < end; i++) { + int value = codepoints[i]; + if (value < 0 || value > UNICODE_VALID_RANGE_MAX) { + encoded.setRange(insertAt, 3, [0xef, 0xbf, 0xbd]); + insertAt += 3; + } else if (value <= _UTF8_ONE_BYTE_MAX) { + encoded[insertAt] = value; + insertAt++; + } else if (value <= _UTF8_TWO_BYTE_MAX) { + encoded[insertAt] = _UTF8_FIRST_BYTE_OF_TWO_BASE | ( + _UTF8_FIRST_BYTE_OF_TWO_MASK & + _addToEncoding(insertAt, 1, value, encoded)); + insertAt += 2; + } else if (value <= _UTF8_THREE_BYTE_MAX) { + encoded[insertAt] = _UTF8_FIRST_BYTE_OF_THREE_BASE | ( + _UTF8_FIRST_BYTE_OF_THREE_MASK & + _addToEncoding(insertAt, 2, value, encoded)); + insertAt += 3; + } else if (value <= UNICODE_VALID_RANGE_MAX) { + encoded[insertAt] = _UTF8_FIRST_BYTE_OF_FOUR_BASE | ( + _UTF8_FIRST_BYTE_OF_FOUR_MASK & + _addToEncoding(insertAt, 3, value, encoded)); + insertAt += 4; + } + } + return encoded; +} + + +// Because UTF-8 specifies byte order, we do not have to follow the pattern +// used by UTF-16 & UTF-32 regarding byte order. +List _utf8ToCodepoints( + List utf8EncodedBytes, [int offset = 0, int length, + int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { + if (!(offset >= 0)) { + throw new IllegalArgumentException("offset"); + } + + if (!(length == null || length >= 0)) { + throw new IllegalArgumentException("length"); + } + + int end = length != null ? + Math.min(utf8EncodedBytes.length, offset + length) : + utf8EncodedBytes.length; + + void addReplacementCodepoint(void f(int v), int replacementCodepoint) { + if(replacementCodepoint != null) { + f(replacementCodepoint); + } else { + throw new IllegalArgumentException("Invalid encoding"); + } + } + + void apply(void f(int v)) { + int i = offset; + while (i < end) { + int value = utf8EncodedBytes[i++]; + if (value >= 0) { + if (value <= _UTF8_ONE_BYTE_MAX) { + f(value); + } else if (value < _UTF8_FIRST_BYTE_OF_TWO_BASE) { + addReplacementCodepoint(f, replacementCodepoint); + continue; + } else { + int additionalBytes = 0; + if (value < _UTF8_FIRST_BYTE_OF_THREE_BASE) { + value -= _UTF8_FIRST_BYTE_OF_TWO_BASE; + additionalBytes = 1; + } else if (value < _UTF8_FIRST_BYTE_OF_FOUR_BASE) { + value -= _UTF8_FIRST_BYTE_OF_THREE_BASE; + additionalBytes = 2; + } else if (value < _UTF8_FIRST_BYTE_OF_FIVE_BASE) { + value -= _UTF8_FIRST_BYTE_OF_FOUR_BASE; + additionalBytes = 3; + } else if (value < _UTF8_FIRST_BYTE_OF_SIX_BASE) { + value -= _UTF8_FIRST_BYTE_OF_FIVE_BASE; + additionalBytes = 4; + } else if (value < _UTF8_FIRST_BYTE_BOUND_EXCL) { + value -= _UTF8_FIRST_BYTE_OF_SIX_BASE; + additionalBytes = 5; + } else { + addReplacementCodepoint(f, replacementCodepoint); + continue; + } + int j = 0; + while (j < additionalBytes && i < end) { + int nextValue = utf8EncodedBytes[i++]; + if (nextValue > _UTF8_ONE_BYTE_MAX && + nextValue < _UTF8_FIRST_BYTE_OF_TWO_BASE) { + value = (value << 6) | (nextValue & _UTF8_LO_SIX_BIT_MASK); + } else { + // if sequence-starting code unit, reposition cursor to start here + if (nextValue >= _UTF8_FIRST_BYTE_OF_TWO_BASE) { + i--; + } + break; + } + j++; + } + if (j == additionalBytes && ( + value < UNICODE_UTF16_RESERVED_LO || + value > UNICODE_UTF16_RESERVED_HI)) { + if ((additionalBytes == 1 && value > _UTF8_ONE_BYTE_MAX) || + (additionalBytes == 2 && value > _UTF8_TWO_BYTE_MAX) || + (additionalBytes == 3 && value > _UTF8_THREE_BYTE_MAX && + value <= UNICODE_VALID_RANGE_MAX)) { + f(value); + } else { + addReplacementCodepoint(f, replacementCodepoint); + } + } else { + addReplacementCodepoint(f, replacementCodepoint); + continue; + } + } + } else { + addReplacementCodepoint(f, replacementCodepoint); + continue; + } + } + } + + int codepointBufferLength = 0; + apply(void _(int value) { + codepointBufferLength++; + }); + + List codepointBuffer = new List(codepointBufferLength); + int i = 0; + apply(void _(int value) { + codepointBuffer[i++] = value; + }); + return codepointBuffer; +} diff --git a/utils/string_encoding/unicode_core.dart b/utils/string_encoding/unicode_core.dart new file mode 100644 index 00000000000..a39201853ea --- /dev/null +++ b/utils/string_encoding/unicode_core.dart @@ -0,0 +1,181 @@ +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#library("unicode_core"); + +/* + * Test for presence of bug related to the use of UTF-16 code units for + * Dart compiled to JS. + */ +bool _test16BitCodeUnit = null; +// TODO is16BitCodeUnit() is used to work around a bug with frog/dartc +// (http://code.google.com/p/dart/issues/detail?id=1357). Consider +// removing after this issue is resolved. +bool is16BitCodeUnit() { + if (_test16BitCodeUnit == null) { + _test16BitCodeUnit = (new String.fromCharCodes([0x1D11E])) == + (new String.fromCharCodes([0xD11E])); + } + return _test16BitCodeUnit; +} + +/** + * Invalid codepoints or encodings may be substituted with the value U+fffd. + */ +final int UNICODE_REPLACEMENT_CHARACTER_CODEPOINT = 0xfffd; +final int UNICODE_BOM = 0xfeff; +final int UNICODE_UTF_BOM_LO = 0xff; +final int UNICODE_UTF_BOM_HI = 0xfe; + +final int UNICODE_BYTE_ZERO_MASK = 0xff; +final int UNICODE_BYTE_ONE_MASK = 0xff00; +final int UNICODE_VALID_RANGE_MAX = 0x10ffff; +final int UNICODE_PLANE_ONE_MAX = 0xffff; +final int UNICODE_UTF16_RESERVED_LO = 0xd800; +final int UNICODE_UTF16_RESERVED_HI = 0xdfff; +final int UNICODE_UTF16_OFFSET = 0x10000; +final int UNICODE_UTF16_SURROGATE_UNIT_0_BASE = 0xd800; +final int UNICODE_UTF16_SURROGATE_UNIT_1_BASE = 0xdc00; +final int UNICODE_UTF16_HI_MASK = 0xffc00; +final int UNICODE_UTF16_LO_MASK = 0x3ff; + +/** + * Encode code points as UTF16 code units. + */ +List codepointsToUtf16CodeUnits( + List codepoints, [int offset = 0, int length, + int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { + if (!(offset >= 0)) { + throw new IllegalArgumentException("offset"); + } + + if (!(length == null || length >= 0)) { + throw new IllegalArgumentException("length"); + } + + int end = length != null ? + Math.min(codepoints.length, offset + length) : + codepoints.length; + + int encodedLength = 0; + for (int i = offset; i < end; i++) { + int value = codepoints[i]; + if ((value >= 0 && value < UNICODE_UTF16_RESERVED_LO) || + (value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) { + encodedLength++; + } else if (value > UNICODE_PLANE_ONE_MAX && + value <= UNICODE_VALID_RANGE_MAX) { + encodedLength += 2; + } else { + encodedLength++; + } + } + + void addReplacementCodepoint(List codepointBuffer, int offset, + int replacementCodepoint) { + if(replacementCodepoint != null) { + codepointBuffer[offset] = replacementCodepoint; + } else { + throw new IllegalArgumentException("Invalid encoding"); + } + } + List codeUnitsBuffer = new List(encodedLength); + int j = 0; + for (int i = offset; i < end; i++) { + int value = codepoints[i]; + if ((value >= 0 && value < UNICODE_UTF16_RESERVED_LO) || + (value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) { + codeUnitsBuffer[j++] = value; + } else if (value > UNICODE_PLANE_ONE_MAX && + value <= UNICODE_VALID_RANGE_MAX) { + int base = value - UNICODE_UTF16_OFFSET; + codeUnitsBuffer[j++] = UNICODE_UTF16_SURROGATE_UNIT_0_BASE + + ((base & UNICODE_UTF16_HI_MASK) >> 10); + codeUnitsBuffer[j++] = UNICODE_UTF16_SURROGATE_UNIT_1_BASE + + (base & UNICODE_UTF16_LO_MASK); + } else { + addReplacementCodepoint(codeUnitsBuffer, j++, replacementCodepoint); + } + } + return codeUnitsBuffer; +} + +/** + * Decodes the utf16 codeunits to codepoints. + */ +List utf16CodeUnitsToCodepoints( + List utf16CodeUnits, [int offset = 0, int length, + int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { + if (!(offset >= 0)) { + throw new IllegalArgumentException("offset"); + } + + if (!(length == null || length >= 0)) { + throw new IllegalArgumentException("length"); + } + + int end = length != null ? + Math.min(utf16CodeUnits.length, offset + length) : + utf16CodeUnits.length; + + void addReplacementCodepoint(void f(int v), int replacementCodepoint) { + if(replacementCodepoint != null) { + f(replacementCodepoint); + } else { + throw new IllegalArgumentException("Invalid encoding"); + } + } + + void apply(void f(int v)) { + int i = offset; + // skip the first entry if it is a BOM. + if (end > 0 && utf16CodeUnits[0] == UNICODE_BOM) { + i++; + } + while (i < end) { + int value = utf16CodeUnits[i++]; + if (value < 0) { + addReplacementCodepoint(f, replacementCodepoint); + continue; + } + if (value < UNICODE_UTF16_RESERVED_LO || + (value > UNICODE_UTF16_RESERVED_HI && + value <= UNICODE_PLANE_ONE_MAX)) { + // transfer directly + f(value); + } else if (value < UNICODE_UTF16_SURROGATE_UNIT_1_BASE && i < end) { + // merge surrogate pair + int nextValue = utf16CodeUnits[i++]; + if (nextValue >= UNICODE_UTF16_SURROGATE_UNIT_1_BASE && + nextValue <= UNICODE_UTF16_RESERVED_HI) { + value = (value - UNICODE_UTF16_SURROGATE_UNIT_0_BASE) << 10; + value += UNICODE_UTF16_OFFSET + + (nextValue - UNICODE_UTF16_SURROGATE_UNIT_1_BASE); + f(value); + } else { + if (nextValue >= UNICODE_UTF16_SURROGATE_UNIT_0_BASE && + nextValue < UNICODE_UTF16_SURROGATE_UNIT_1_BASE) { + i--; + } + addReplacementCodepoint(f, replacementCodepoint); + continue; + } + } else { + addReplacementCodepoint(f, replacementCodepoint); + continue; + } + } + } + int codepointBufferLength = 0; + apply(void _(int value) { + codepointBufferLength++; + }); + + List codepointBuffer = new List(codepointBufferLength); + int i = 0; + apply(void _(int value) { + codepointBuffer[i++] = value; + }); + return codepointBuffer; +} diff --git a/utils/tests/string_encoding/DUnit.dart b/utils/tests/string_encoding/DUnit.dart new file mode 100644 index 00000000000..fa4d39ed989 --- /dev/null +++ b/utils/tests/string_encoding/DUnit.dart @@ -0,0 +1,117 @@ +#!/usr/bin/env dart +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#library("dunit"); + +typedef void Test(); +typedef TestResult SynchTest(); +typedef Future AsynchTest(); + +class TestSuite { + TestSuite() : _tests = []; + + void registerTestClass(TestClass tests) { + tests.registerTests(this); + } + + void _registerTest(SynchTest test) { + _tests.add(test); + } + + void run() { + reportResults(runTests()); + } + + List runTests() { + List results = []; + for(Function test in _tests) { + results.add(test()); + } + return results; + } + + void reportResults(List results) { + if(results.every(bool _(TestResult r) => r is PassedTest)) { + print("OK -- ALL TESTS PASS (${results.length} run)"); + } else { + for(TestResult r in + results.filter(bool _(TestResult r) => !(r is PassedTest))) { + print(r); + } + int passedTests = + results.filter(bool _(TestResult r) => r is PassedTest).length; + int failures = + results.filter(bool _(TestResult r) => r is FailedTest).length; + int errors = + results.filter(bool _(TestResult r) => r is TestError).length; + print("FAIL -- TESTS RUN: ${results.length}"); + print(" PASSED: ${passedTests}"); + print(" FAILED: ${failures}"); + print(" ERRORS: ${errors}"); + } + } + + List _tests; +} + +interface TestResult { + String get testDescription(); +} + +class PassedTest implements TestResult { + const PassedTest(String this._testDescription); + String get testDescription() => _testDescription; + final String _testDescription; + String toString() => _testDescription; +} + +class _ExceptionResult { + const _ExceptionResult(String this._testDescription, var this._exception); + + String get testDescription() => _testDescription; + final String _testDescription; + + Object get exception() => _exception; + final _exception; +} + +class FailedTest extends _ExceptionResult implements TestResult { + FailedTest(String testDescription, var exception) : + super(testDescription, exception); + + String toString() => ">>> Test failure in ${_testDescription} " + + "with:\n${exception}\n"; +} + +class TestError extends _ExceptionResult implements TestResult { + TestError(String testDescription, var exception) : + super(testDescription, exception); + + String toString() => ">>> Test error caught in " + + "${_testDescription} with:\n${exception}\n"; +} + +class TestClass { + void register(String description, Function test, TestSuite suite) { + suite._registerTest(TestResult _() { + setUp(); + try { + test(); + tearDown(); + return new PassedTest(description); + } catch (ExpectException x) { + tearDown(); + return new FailedTest(description, x); + } catch (var x) { + tearDown(); + return new TestError(description, x); + } + }); + } + + abstract void registerTests(TestSuite suite); + void setUp() {} + void tearDown() {} +} diff --git a/utils/tests/string_encoding/run_tests.dart b/utils/tests/string_encoding/run_tests.dart new file mode 100755 index 00000000000..e42a056c599 --- /dev/null +++ b/utils/tests/string_encoding/run_tests.dart @@ -0,0 +1,26 @@ +#!/usr/bin/env dart +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#library("string_encoding_run_tests"); +#import("dunit.dart"); +#import("unicode_tests.dart", prefix: "u"); +#import("unicode_core_tests.dart", prefix: "uc"); +#import("utf16_tests.dart", prefix: "utf16"); +#import("utf32_tests.dart", prefix: "utf32"); +#import("utf8_tests.dart", prefix: "utf8"); + +void main() { + TestSuite suite = new TestSuite(); + registerTests(suite); + suite.run(); +} + +void registerTests(TestSuite suite) { + suite.registerTestClass(new u.UnicodeTests()); + suite.registerTestClass(new uc.UnicodeCoreTests()); + suite.registerTestClass(new utf16.Utf16Tests()); + suite.registerTestClass(new utf32.Utf32Tests()); + suite.registerTestClass(new utf8.Utf8Tests()); +} diff --git a/utils/tests/string_encoding/unicode_core_tests.dart b/utils/tests/string_encoding/unicode_core_tests.dart new file mode 100755 index 00000000000..1a02eb8d245 --- /dev/null +++ b/utils/tests/string_encoding/unicode_core_tests.dart @@ -0,0 +1,103 @@ +#!/usr/bin/env dart +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#library("unicode_core_tests"); +#import("dunit.dart"); +#import("../../string_encoding/unicode_core.dart"); + +void main() { + TestSuite suite = new TestSuite(); + suite.registerTestClass(new UnicodeCoreTests()); + suite.run(); +} + +class UnicodeCoreTests extends TestClass { + void registerTests(TestSuite suite) { + register("testPlatform", testPlatform, suite); + register("testCodepointsToUtf16CodeUnits", + testCodepointsToUtf16CodeUnits, suite); + register("testUtf16bytesToCodepoints", testUtf16bytesToCodepoints, suite); + } + + void testPlatform() { + Expect.isFalse(is16BitCodeUnit()); + } + + void testCodepointsToUtf16CodeUnits() { + // boundary conditions + Expect.listEquals([0x0], codepointsToUtf16CodeUnits([0x0]), "0"); + Expect.listEquals([0xd800, 0xdc00], + codepointsToUtf16CodeUnits([0x10000]), "10000"); + + Expect.listEquals([0xffff], + codepointsToUtf16CodeUnits([0xffff]), "ffff"); + Expect.listEquals([0xdbff, 0xdfff], + codepointsToUtf16CodeUnits([0x10ffff]), "10ffff"); + + Expect.listEquals([0xd7ff], + codepointsToUtf16CodeUnits([0xd7ff]), "d7ff"); + Expect.listEquals([0xe000], + codepointsToUtf16CodeUnits([0xe000]), "e000"); + + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + codepointsToUtf16CodeUnits([0xd800]), "d800"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + codepointsToUtf16CodeUnits([0xdfff]), "dfff"); + } + + void testUtf16bytesToCodepoints() { + // boundary conditions: First possible values + Expect.listEquals([0x0], utf16CodeUnitsToCodepoints([0x0]), "0"); + Expect.listEquals([0x10000], + utf16CodeUnitsToCodepoints([0xd800, 0xdc00]), "10000"); + + // boundary conditions: Last possible sequence of a certain length + Expect.listEquals([0xffff], + utf16CodeUnitsToCodepoints([0xffff]), "ffff"); + Expect.listEquals([0x10ffff], + utf16CodeUnitsToCodepoints([0xdbff, 0xdfff]), "10ffff"); + + // other boundary conditions + Expect.listEquals([0xd7ff], + utf16CodeUnitsToCodepoints([0xd7ff]), "d7ff"); + Expect.listEquals([0xe000], + utf16CodeUnitsToCodepoints([0xe000]), "e000"); + + // unexpected continuation bytes + Expect.listEquals([0xfffd], + utf16CodeUnitsToCodepoints([0xdc00]), + "dc00 first unexpected continuation byte"); + Expect.listEquals([0xfffd], + utf16CodeUnitsToCodepoints([0xdfff]), + "dfff last unexpected continuation byte"); + Expect.listEquals([0xfffd], + utf16CodeUnitsToCodepoints([0xdc00]), + "1 unexpected continuation bytes"); + Expect.listEquals([0xfffd, 0xfffd], + utf16CodeUnitsToCodepoints([0xdc00, 0xdc00]), + "2 unexpected continuation bytes"); + Expect.listEquals([0xfffd, 0xfffd ,0xfffd], + utf16CodeUnitsToCodepoints([0xdc00, 0xdc00, 0xdc00]), + "3 unexpected continuation bytes"); + + // incomplete sequences + Expect.listEquals([0xfffd], utf16CodeUnitsToCodepoints([0xd800]), + "d800 last byte missing"); + Expect.listEquals([0xfffd], utf16CodeUnitsToCodepoints([0xdbff]), + "dbff last byte missing"); + + // concatenation of incomplete sequences + Expect.listEquals([0xfffd, 0xfffd], + utf16CodeUnitsToCodepoints([0xd800, 0xdbff]), + "d800 dbff last byte missing"); + + // impossible bytes + Expect.listEquals([0xfffd], utf16CodeUnitsToCodepoints([0x110000]), + "110000 out of bounds"); + + // overlong sequences not possible in utf16 (nothing < x10000) + // illegal code positions d800-dfff not encodable (< x10000) + } +} diff --git a/utils/tests/string_encoding/unicode_tests.dart b/utils/tests/string_encoding/unicode_tests.dart new file mode 100755 index 00000000000..32c0f9c8992 --- /dev/null +++ b/utils/tests/string_encoding/unicode_tests.dart @@ -0,0 +1,37 @@ +#!/usr/bin/env dart +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#library("unicode_tests"); +#import("dunit.dart"); +#import("../../string_encoding/unicode.dart"); + +void main() { + TestSuite suite = new TestSuite(); + suite.registerTestClass(new UnicodeTests()); + suite.run(); +} + +class UnicodeTests extends TestClass { + static final String testPhrase = + "The quick brown fox jumps over the lazy dog."; + + static final List testCodepoints = const [ + 84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, + 32, 102, 111, 120, 32, 106, 117, 109, 112, 115, 32, 111, 118, 101, 114, + 32, 116, 104, 101, 32, 108, 97, 122, 121, 32, 100, 111, 103, 46]; + + void registerTests(TestSuite suite) { + register("testCodepointsToString", testCodepointsToString, suite); + register("testStringToCodepoints", testStringToCodepoints, suite); + } + + void testStringToCodepoints() { + Expect.listEquals(testCodepoints, stringToCodepoints(testPhrase)); + } + + void testCodepointsToString() { + Expect.listEquals(testPhrase, codepointsToString(testCodepoints)); + } +} diff --git a/utils/tests/string_encoding/utf16_tests.dart b/utils/tests/string_encoding/utf16_tests.dart new file mode 100755 index 00000000000..436ef21fa24 --- /dev/null +++ b/utils/tests/string_encoding/utf16_tests.dart @@ -0,0 +1,106 @@ +#!/usr/bin/env dart +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#library("utf16_tests"); +#import("dunit.dart"); +#import("../../string_encoding/utf16.dart"); + +void main() { + TestSuite suite = new TestSuite(); + suite.registerTestClass(new Utf16Tests()); + suite.run(); +} + +class Utf16Tests extends TestClass { + static final String testKoreanCharSubset = """ +가각갂갃간갅갆갇갈갉갊갋갌갍갎갏감갑값갓갔강갖갗갘같갚갛 +개객갞갟갠갡갢갣갤갥갦갧갨갩갪갫갬갭갮갯갰갱갲갳갴갵갶갷 +갸갹갺갻갼갽갾갿걀걁걂걃걄걅걆걇걈걉걊걋걌걍걎걏걐걑걒걓"""; + static final String testHanWater = "水"; + + static final List testKoreanCharSubsetUtf16be = const[ + 0xfe, 0xff, 0xac, 0x00, 0xac, 0x01, 0xac, 0x02, + 0xac, 0x03, 0xac, 0x04, 0xac, 0x05, 0xac, 0x06, + 0xac, 0x07, 0xac, 0x08, 0xac, 0x09, 0xac, 0x0a, + 0xac, 0x0b, 0xac, 0x0c, 0xac, 0x0d, 0xac, 0x0e, + 0xac, 0x0f, 0xac, 0x10, 0xac, 0x11, 0xac, 0x12, + 0xac, 0x13, 0xac, 0x14, 0xac, 0x15, 0xac, 0x16, + 0xac, 0x17, 0xac, 0x18, 0xac, 0x19, 0xac, 0x1a, + 0xac, 0x1b, 0x00, 0x0a, 0xac, 0x1c, 0xac, 0x1d, + 0xac, 0x1e, 0xac, 0x1f, 0xac, 0x20, 0xac, 0x21, + 0xac, 0x22, 0xac, 0x23, 0xac, 0x24, 0xac, 0x25, + 0xac, 0x26, 0xac, 0x27, 0xac, 0x28, 0xac, 0x29, + 0xac, 0x2a, 0xac, 0x2b, 0xac, 0x2c, 0xac, 0x2d, + 0xac, 0x2e, 0xac, 0x2f, 0xac, 0x30, 0xac, 0x31, + 0xac, 0x32, 0xac, 0x33, 0xac, 0x34, 0xac, 0x35, + 0xac, 0x36, 0xac, 0x37, 0x00, 0x0a, 0xac, 0x38, + 0xac, 0x39, 0xac, 0x3a, 0xac, 0x3b, 0xac, 0x3c, + 0xac, 0x3d, 0xac, 0x3e, 0xac, 0x3f, 0xac, 0x40, + 0xac, 0x41, 0xac, 0x42, 0xac, 0x43, 0xac, 0x44, + 0xac, 0x45, 0xac, 0x46, 0xac, 0x47, 0xac, 0x48, + 0xac, 0x49, 0xac, 0x4a, 0xac, 0x4b, 0xac, 0x4c, + 0xac, 0x4d, 0xac, 0x4e, 0xac, 0x4f, 0xac, 0x50, + 0xac, 0x51, 0xac, 0x52, 0xac, 0x53]; + + void registerTests(TestSuite suite) { + register("testEncodeToUtf16", testEncodeToUtf16, suite); + register("testUtf16BytesToString", testUtf16BytesToString, suite); + } + + void testEncodeToUtf16() { + Expect.listEquals(testKoreanCharSubsetUtf16be, + encodeAsUtf16(testKoreanCharSubset), + "encode UTF-16(BE by default) Korean"); + + Expect.listEquals([ + 0x00, 0xac, 0x01, 0xac, 0x02, 0xac, 0x03, 0xac, + 0x04, 0xac, 0x05, 0xac, 0x06, 0xac, 0x07, 0xac, + 0x08, 0xac, 0x09, 0xac, 0x0a, 0xac, 0x0b, 0xac, + 0x0c, 0xac, 0x0d, 0xac, 0x0e, 0xac, 0x0f, 0xac, + 0x10, 0xac, 0x11, 0xac, 0x12, 0xac, 0x13, 0xac, + 0x14, 0xac, 0x15, 0xac, 0x16, 0xac, 0x17, 0xac, + 0x18, 0xac, 0x19, 0xac, 0x1a, 0xac, 0x1b, 0xac, + 0x0a, 0x00, 0x1c, 0xac, 0x1d, 0xac, 0x1e, 0xac, + 0x1f, 0xac, 0x20, 0xac, 0x21, 0xac, 0x22, 0xac, + 0x23, 0xac, 0x24, 0xac, 0x25, 0xac, 0x26, 0xac, + 0x27, 0xac, 0x28, 0xac, 0x29, 0xac, 0x2a, 0xac, + 0x2b, 0xac, 0x2c, 0xac, 0x2d, 0xac, 0x2e, 0xac, + 0x2f, 0xac, 0x30, 0xac, 0x31, 0xac, 0x32, 0xac, + 0x33, 0xac, 0x34, 0xac, 0x35, 0xac, 0x36, 0xac, + 0x37, 0xac, 0x0a, 0x00, 0x38, 0xac, 0x39, 0xac, + 0x3a, 0xac, 0x3b, 0xac, 0x3c, 0xac, 0x3d, 0xac, + 0x3e, 0xac, 0x3f, 0xac, 0x40, 0xac, 0x41, 0xac, + 0x42, 0xac, 0x43, 0xac, 0x44, 0xac, 0x45, 0xac, + 0x46, 0xac, 0x47, 0xac, 0x48, 0xac, 0x49, 0xac, + 0x4a, 0xac, 0x4b, 0xac, 0x4c, 0xac, 0x4d, 0xac, + 0x4e, 0xac, 0x4f, 0xac, 0x50, 0xac, 0x51, 0xac, + 0x52, 0xac, 0x53, 0xac], encodeAsUtf16le(testKoreanCharSubset), + "encode UTF-16LE Korean"); + } + + void testUtf16BytesToString() { + Expect.stringEquals(testHanWater, decodeFromUtf16([0x6C, 0x34]), + "Water variation 1"); + Expect.stringEquals(testHanWater, decodeFromUtf16([0xFE, 0xFF, 0x6C, 0x34]), + "Water variation 2"); + Expect.stringEquals(testHanWater, decodeFromUtf16([0xFF, 0xFE, 0x34, 0x6C]), + "Water variation 3"); + + Expect.stringEquals(testHanWater, decodeFromUtf16be([0x6C, 0x34]), + "Water variation 4"); + Expect.stringEquals(testHanWater, + decodeFromUtf16be([0xFE, 0xFF, 0x6C, 0x34]), + "Water variation 5"); + + Expect.stringEquals(testHanWater, decodeFromUtf16le([0x34, 0x6C]), + "Water variation 6"); + Expect.stringEquals(testHanWater, + decodeFromUtf16le([0xFF, 0xFE, 0x34, 0x6C]), + "Water variation 7"); + + Expect.stringEquals(testKoreanCharSubset, + decodeFromUtf16(testKoreanCharSubsetUtf16be), "UTF-16BE Korean"); + } +} diff --git a/utils/tests/string_encoding/utf32_tests.dart b/utils/tests/string_encoding/utf32_tests.dart new file mode 100755 index 00000000000..e1c4e4abce4 --- /dev/null +++ b/utils/tests/string_encoding/utf32_tests.dart @@ -0,0 +1,158 @@ +#!/usr/bin/env dart +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#library("utf32_tests"); +#import("dunit.dart"); +#import("../../string_encoding/utf32.dart"); + +void main() { + TestSuite suite = new TestSuite(); + suite.registerTestClass(new Utf32Tests()); + suite.run(); +} + +class Utf32Tests extends TestClass { + static final String testKoreanCharSubset = """ +가각갂갃간갅갆갇갈갉갊갋갌갍갎갏감갑값갓갔강갖갗갘같갚갛 +개객갞갟갠갡갢갣갤갥갦갧갨갩갪갫갬갭갮갯갰갱갲갳갴갵갶갷 +갸갹갺갻갼갽갾갿걀걁걂걃걄걅걆걇걈걉걊걋걌걍걎걏걐걑걒걓"""; + static final String testHanTwice = "二"; + + static final List testKoreanCharSubsetUtf32be = const[ + 0x00, 0x00, 0xfe, 0xff, 0x00, 0x00, 0xac, 0x00, + 0x00, 0x00, 0xac, 0x01, 0x00, 0x00, 0xac, 0x02, + 0x00, 0x00, 0xac, 0x03, 0x00, 0x00, 0xac, 0x04, + 0x00, 0x00, 0xac, 0x05, 0x00, 0x00, 0xac, 0x06, + 0x00, 0x00, 0xac, 0x07, 0x00, 0x00, 0xac, 0x08, + 0x00, 0x00, 0xac, 0x09, 0x00, 0x00, 0xac, 0x0a, + 0x00, 0x00, 0xac, 0x0b, 0x00, 0x00, 0xac, 0x0c, + 0x00, 0x00, 0xac, 0x0d, 0x00, 0x00, 0xac, 0x0e, + 0x00, 0x00, 0xac, 0x0f, 0x00, 0x00, 0xac, 0x10, + 0x00, 0x00, 0xac, 0x11, 0x00, 0x00, 0xac, 0x12, + 0x00, 0x00, 0xac, 0x13, 0x00, 0x00, 0xac, 0x14, + 0x00, 0x00, 0xac, 0x15, 0x00, 0x00, 0xac, 0x16, + 0x00, 0x00, 0xac, 0x17, 0x00, 0x00, 0xac, 0x18, + 0x00, 0x00, 0xac, 0x19, 0x00, 0x00, 0xac, 0x1a, + 0x00, 0x00, 0xac, 0x1b, 0x00, 0x00, 0x00, 0x0a, + 0x00, 0x00, 0xac, 0x1c, 0x00, 0x00, 0xac, 0x1d, + 0x00, 0x00, 0xac, 0x1e, 0x00, 0x00, 0xac, 0x1f, + 0x00, 0x00, 0xac, 0x20, 0x00, 0x00, 0xac, 0x21, + 0x00, 0x00, 0xac, 0x22, 0x00, 0x00, 0xac, 0x23, + 0x00, 0x00, 0xac, 0x24, 0x00, 0x00, 0xac, 0x25, + 0x00, 0x00, 0xac, 0x26, 0x00, 0x00, 0xac, 0x27, + 0x00, 0x00, 0xac, 0x28, 0x00, 0x00, 0xac, 0x29, + 0x00, 0x00, 0xac, 0x2a, 0x00, 0x00, 0xac, 0x2b, + 0x00, 0x00, 0xac, 0x2c, 0x00, 0x00, 0xac, 0x2d, + 0x00, 0x00, 0xac, 0x2e, 0x00, 0x00, 0xac, 0x2f, + 0x00, 0x00, 0xac, 0x30, 0x00, 0x00, 0xac, 0x31, + 0x00, 0x00, 0xac, 0x32, 0x00, 0x00, 0xac, 0x33, + 0x00, 0x00, 0xac, 0x34, 0x00, 0x00, 0xac, 0x35, + 0x00, 0x00, 0xac, 0x36, 0x00, 0x00, 0xac, 0x37, + 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0xac, 0x38, + 0x00, 0x00, 0xac, 0x39, 0x00, 0x00, 0xac, 0x3a, + 0x00, 0x00, 0xac, 0x3b, 0x00, 0x00, 0xac, 0x3c, + 0x00, 0x00, 0xac, 0x3d, 0x00, 0x00, 0xac, 0x3e, + 0x00, 0x00, 0xac, 0x3f, 0x00, 0x00, 0xac, 0x40, + 0x00, 0x00, 0xac, 0x41, 0x00, 0x00, 0xac, 0x42, + 0x00, 0x00, 0xac, 0x43, 0x00, 0x00, 0xac, 0x44, + 0x00, 0x00, 0xac, 0x45, 0x00, 0x00, 0xac, 0x46, + 0x00, 0x00, 0xac, 0x47, 0x00, 0x00, 0xac, 0x48, + 0x00, 0x00, 0xac, 0x49, 0x00, 0x00, 0xac, 0x4a, + 0x00, 0x00, 0xac, 0x4b, 0x00, 0x00, 0xac, 0x4c, + 0x00, 0x00, 0xac, 0x4d, 0x00, 0x00, 0xac, 0x4e, + 0x00, 0x00, 0xac, 0x4f, 0x00, 0x00, 0xac, 0x50, + 0x00, 0x00, 0xac, 0x51, 0x00, 0x00, 0xac, 0x52, + 0x00, 0x00, 0xac, 0x53]; + + void registerTests(TestSuite suite) { + register("testUtf32BytesToString", testUtf32BytesToString, suite); + register("testEncodeToUtf32", testEncodeToUtf32, suite); + } + + void testEncodeToUtf32() { + Expect.listEquals(testKoreanCharSubsetUtf32be, + encodeAsUtf32(testKoreanCharSubset), + "encode UTF-32(BE by default) Korean"); + Expect.listEquals([ + 0x00, 0xac, 0x00, 0x00, 0x01, 0xac, 0x00, 0x00, + 0x02, 0xac, 0x00, 0x00, 0x03, 0xac, 0x00, 0x00, + 0x04, 0xac, 0x00, 0x00, 0x05, 0xac, 0x00, 0x00, + 0x06, 0xac, 0x00, 0x00, 0x07, 0xac, 0x00, 0x00, + 0x08, 0xac, 0x00, 0x00, 0x09, 0xac, 0x00, 0x00, + 0x0a, 0xac, 0x00, 0x00, 0x0b, 0xac, 0x00, 0x00, + 0x0c, 0xac, 0x00, 0x00, 0x0d, 0xac, 0x00, 0x00, + 0x0e, 0xac, 0x00, 0x00, 0x0f, 0xac, 0x00, 0x00, + 0x10, 0xac, 0x00, 0x00, 0x11, 0xac, 0x00, 0x00, + 0x12, 0xac, 0x00, 0x00, 0x13, 0xac, 0x00, 0x00, + 0x14, 0xac, 0x00, 0x00, 0x15, 0xac, 0x00, 0x00, + 0x16, 0xac, 0x00, 0x00, 0x17, 0xac, 0x00, 0x00, + 0x18, 0xac, 0x00, 0x00, 0x19, 0xac, 0x00, 0x00, + 0x1a, 0xac, 0x00, 0x00, 0x1b, 0xac, 0x00, 0x00, + 0x0a, 0x00, 0x00, 0x00, 0x1c, 0xac, 0x00, 0x00, + 0x1d, 0xac, 0x00, 0x00, 0x1e, 0xac, 0x00, 0x00, + 0x1f, 0xac, 0x00, 0x00, 0x20, 0xac, 0x00, 0x00, + 0x21, 0xac, 0x00, 0x00, 0x22, 0xac, 0x00, 0x00, + 0x23, 0xac, 0x00, 0x00, 0x24, 0xac, 0x00, 0x00, + 0x25, 0xac, 0x00, 0x00, 0x26, 0xac, 0x00, 0x00, + 0x27, 0xac, 0x00, 0x00, 0x28, 0xac, 0x00, 0x00, + 0x29, 0xac, 0x00, 0x00, 0x2a, 0xac, 0x00, 0x00, + 0x2b, 0xac, 0x00, 0x00, 0x2c, 0xac, 0x00, 0x00, + 0x2d, 0xac, 0x00, 0x00, 0x2e, 0xac, 0x00, 0x00, + 0x2f, 0xac, 0x00, 0x00, 0x30, 0xac, 0x00, 0x00, + 0x31, 0xac, 0x00, 0x00, 0x32, 0xac, 0x00, 0x00, + 0x33, 0xac, 0x00, 0x00, 0x34, 0xac, 0x00, 0x00, + 0x35, 0xac, 0x00, 0x00, 0x36, 0xac, 0x00, 0x00, + 0x37, 0xac, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, + 0x38, 0xac, 0x00, 0x00, 0x39, 0xac, 0x00, 0x00, + 0x3a, 0xac, 0x00, 0x00, 0x3b, 0xac, 0x00, 0x00, + 0x3c, 0xac, 0x00, 0x00, 0x3d, 0xac, 0x00, 0x00, + 0x3e, 0xac, 0x00, 0x00, 0x3f, 0xac, 0x00, 0x00, + 0x40, 0xac, 0x00, 0x00, 0x41, 0xac, 0x00, 0x00, + 0x42, 0xac, 0x00, 0x00, 0x43, 0xac, 0x00, 0x00, + 0x44, 0xac, 0x00, 0x00, 0x45, 0xac, 0x00, 0x00, + 0x46, 0xac, 0x00, 0x00, 0x47, 0xac, 0x00, 0x00, + 0x48, 0xac, 0x00, 0x00, 0x49, 0xac, 0x00, 0x00, + 0x4a, 0xac, 0x00, 0x00, 0x4b, 0xac, 0x00, 0x00, + 0x4c, 0xac, 0x00, 0x00, 0x4d, 0xac, 0x00, 0x00, + 0x4e, 0xac, 0x00, 0x00, 0x4f, 0xac, 0x00, 0x00, + 0x50, 0xac, 0x00, 0x00, 0x51, 0xac, 0x00, 0x00, + 0x52, 0xac, 0x00, 0x00, 0x53, 0xac, 0x00, 0x00], + encodeAsUtf32le(testKoreanCharSubset), + "encode UTF-32(LE by default) Korean"); + } + + void testUtf32BytesToString() { + Expect.stringEquals("\ufffd", decodeFromUtf32([0]), "single byte"); + Expect.stringEquals("\ufffd", decodeFromUtf32([0, 0, 0x4e]), + "short a byte"); + Expect.stringEquals("\u4e8c\ufffd", decodeFromUtf32([0, 0, 0x4e, 0x8c, 0]), + "extra byte"); + + Expect.stringEquals(testHanTwice, decodeFromUtf32([0, 0, 0x4e, 0x8c]), + "twice variation 1"); + Expect.stringEquals(testHanTwice, + decodeFromUtf32([0, 0, 0xfe, 0xff, 0, 0, 0x4e, 0x8c]), + "twice variation 2"); + Expect.stringEquals(testHanTwice, + decodeFromUtf32([0xff, 0xfe, 0, 0, 0x8c, 0x4e, 0, 0]), + "twice variation 3"); + + Expect.stringEquals(testHanTwice, decodeFromUtf32be([0, 0, 0x4e, 0x8c]), + "twice variation 4"); + Expect.stringEquals(testHanTwice, + decodeFromUtf32be([0, 0, 0xfe, 0xff, 0, 0, 0x4e, 0x8c]), + "twice variation 5"); + + Expect.stringEquals(testHanTwice, decodeFromUtf32le([0x8c, 0x4e, 0, 0]), + "twice variation 6"); + Expect.stringEquals(testHanTwice, + decodeFromUtf32le([0xff, 0xfe, 0, 0, 0x8c, 0x4e, 0, 0]), + "twice variation 7"); + + Expect.stringEquals(testKoreanCharSubset, + decodeFromUtf32(testKoreanCharSubsetUtf32be), + "UTF-32BE Korean"); + } +} diff --git a/utils/tests/string_encoding/utf8_tests.dart b/utils/tests/string_encoding/utf8_tests.dart new file mode 100755 index 00000000000..59cd27b6384 --- /dev/null +++ b/utils/tests/string_encoding/utf8_tests.dart @@ -0,0 +1,457 @@ +#!/usr/bin/env dart +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#library("utf8_tests"); +#import("dunit.dart"); +#import("../../string_encoding/unicode.dart"); +#import("../../string_encoding/unicode_core.dart"); +#source("../../string_encoding/utf8_impl.dart"); + +void main() { + TestSuite suite = new TestSuite(); + suite.registerTestClass(new Utf8Tests()); + suite.run(); +} + +class Utf8Tests extends TestClass { + static final String testEnglishPhrase = + "The quick brown fox jumps over the lazy dog."; + + static final List testEnglishUtf8 = const [ + 0x54, 0x68, 0x65, 0x20, 0x71, 0x75, 0x69, 0x63, + 0x6b, 0x20, 0x62, 0x72, 0x6f, 0x77, 0x6e, 0x20, + 0x66, 0x6f, 0x78, 0x20, 0x6a, 0x75, 0x6d, 0x70, + 0x73, 0x20, 0x6f, 0x76, 0x65, 0x72, 0x20, 0x74, + 0x68, 0x65, 0x20, 0x6c, 0x61, 0x7a, 0x79, 0x20, + 0x64, 0x6f, 0x67, 0x2e]; + + static final String testDanishPhrase = "Quizdeltagerne spiste jordbær med " + + "fløde mens cirkusklovnen Wolther spillede på xylofon."; + + static final List testDanishUtf8 = const[ + 0x51, 0x75, 0x69, 0x7a, 0x64, 0x65, 0x6c, 0x74, + 0x61, 0x67, 0x65, 0x72, 0x6e, 0x65, 0x20, 0x73, + 0x70, 0x69, 0x73, 0x74, 0x65, 0x20, 0x6a, 0x6f, + 0x72, 0x64, 0x62, 0xc3, 0xa6, 0x72, 0x20, 0x6d, + 0x65, 0x64, 0x20, 0x66, 0x6c, 0xc3, 0xb8, 0x64, + 0x65, 0x20, 0x6d, 0x65, 0x6e, 0x73, 0x20, 0x63, + 0x69, 0x72, 0x6b, 0x75, 0x73, 0x6b, 0x6c, 0x6f, + 0x76, 0x6e, 0x65, 0x6e, 0x20, 0x57, 0x6f, 0x6c, + 0x74, 0x68, 0x65, 0x72, 0x20, 0x73, 0x70, 0x69, + 0x6c, 0x6c, 0x65, 0x64, 0x65, 0x20, 0x70, 0xc3, + 0xa5, 0x20, 0x78, 0x79, 0x6c, 0x6f, 0x66, 0x6f, + 0x6e, 0x2e]; + + // unusual formatting due to strange editor interaction w/ text direction. + static final String + testHebrewPhrase = "דג סקרן שט בים מאוכזב ולפתע מצא לו חברה איך הקליטה"; + + static final List testHebrewUtf8 = const[ + 0xd7, 0x93, 0xd7, 0x92, 0x20, 0xd7, 0xa1, 0xd7, + 0xa7, 0xd7, 0xa8, 0xd7, 0x9f, 0x20, 0xd7, 0xa9, + 0xd7, 0x98, 0x20, 0xd7, 0x91, 0xd7, 0x99, 0xd7, + 0x9d, 0x20, 0xd7, 0x9e, 0xd7, 0x90, 0xd7, 0x95, + 0xd7, 0x9b, 0xd7, 0x96, 0xd7, 0x91, 0x20, 0xd7, + 0x95, 0xd7, 0x9c, 0xd7, 0xa4, 0xd7, 0xaa, 0xd7, + 0xa2, 0x20, 0xd7, 0x9e, 0xd7, 0xa6, 0xd7, 0x90, + 0x20, 0xd7, 0x9c, 0xd7, 0x95, 0x20, 0xd7, 0x97, + 0xd7, 0x91, 0xd7, 0xa8, 0xd7, 0x94, 0x20, 0xd7, + 0x90, 0xd7, 0x99, 0xd7, 0x9a, 0x20, 0xd7, 0x94, + 0xd7, 0xa7, 0xd7, 0x9c, 0xd7, 0x99, 0xd7, 0x98, + 0xd7, 0x94]; + + static final String testRussianPhrase = "Съешь же ещё этих мягких " + + "французских булок да выпей чаю"; + + static final List testRussianUtf8 = const[ + 0xd0, 0xa1, 0xd1, 0x8a, 0xd0, 0xb5, 0xd1, 0x88, + 0xd1, 0x8c, 0x20, 0xd0, 0xb6, 0xd0, 0xb5, 0x20, + 0xd0, 0xb5, 0xd1, 0x89, 0xd1, 0x91, 0x20, 0xd1, + 0x8d, 0xd1, 0x82, 0xd0, 0xb8, 0xd1, 0x85, 0x20, + 0xd0, 0xbc, 0xd1, 0x8f, 0xd0, 0xb3, 0xd0, 0xba, + 0xd0, 0xb8, 0xd1, 0x85, 0x20, 0xd1, 0x84, 0xd1, + 0x80, 0xd0, 0xb0, 0xd0, 0xbd, 0xd1, 0x86, 0xd1, + 0x83, 0xd0, 0xb7, 0xd1, 0x81, 0xd0, 0xba, 0xd0, + 0xb8, 0xd1, 0x85, 0x20, 0xd0, 0xb1, 0xd1, 0x83, + 0xd0, 0xbb, 0xd0, 0xbe, 0xd0, 0xba, 0x20, 0xd0, + 0xb4, 0xd0, 0xb0, 0x20, 0xd0, 0xb2, 0xd1, 0x8b, + 0xd0, 0xbf, 0xd0, 0xb5, 0xd0, 0xb9, 0x20, 0xd1, + 0x87, 0xd0, 0xb0, 0xd1, 0x8e]; + + static final String testGreekPhrase = "Γαζέες καὶ μυρτιὲς δὲν θὰ βρῶ πιὰ " + + "στὸ χρυσαφὶ ξέφωτο"; + + static final List testGreekUtf8 = const[ + 0xce, 0x93, 0xce, 0xb1, 0xce, 0xb6, 0xce, 0xad, + 0xce, 0xb5, 0xcf, 0x82, 0x20, 0xce, 0xba, 0xce, + 0xb1, 0xe1, 0xbd, 0xb6, 0x20, 0xce, 0xbc, 0xcf, + 0x85, 0xcf, 0x81, 0xcf, 0x84, 0xce, 0xb9, 0xe1, + 0xbd, 0xb2, 0xcf, 0x82, 0x20, 0xce, 0xb4, 0xe1, + 0xbd, 0xb2, 0xce, 0xbd, 0x20, 0xce, 0xb8, 0xe1, + 0xbd, 0xb0, 0x20, 0xce, 0xb2, 0xcf, 0x81, 0xe1, + 0xbf, 0xb6, 0x20, 0xcf, 0x80, 0xce, 0xb9, 0xe1, + 0xbd, 0xb0, 0x20, 0xcf, 0x83, 0xcf, 0x84, 0xe1, + 0xbd, 0xb8, 0x20, 0xcf, 0x87, 0xcf, 0x81, 0xcf, + 0x85, 0xcf, 0x83, 0xce, 0xb1, 0xcf, 0x86, 0xe1, + 0xbd, 0xb6, 0x20, 0xce, 0xbe, 0xce, 0xad, 0xcf, + 0x86, 0xcf, 0x89, 0xcf, 0x84, 0xce, 0xbf]; + + static final String testKatakanaPhrase = """ +イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム +ウヰノオクヤマ ケフコエテ アサキユメミシ ヱヒモセスン"""; + + static final List testKatakanaUtf8 = const[ + 0xe3, 0x82, 0xa4, 0xe3, 0x83, 0xad, 0xe3, 0x83, + 0x8f, 0xe3, 0x83, 0x8b, 0xe3, 0x83, 0x9b, 0xe3, + 0x83, 0x98, 0xe3, 0x83, 0x88, 0x20, 0xe3, 0x83, + 0x81, 0xe3, 0x83, 0xaa, 0xe3, 0x83, 0x8c, 0xe3, + 0x83, 0xab, 0xe3, 0x83, 0xb2, 0x20, 0xe3, 0x83, + 0xaf, 0xe3, 0x82, 0xab, 0xe3, 0x83, 0xa8, 0xe3, + 0x82, 0xbf, 0xe3, 0x83, 0xac, 0xe3, 0x82, 0xbd, + 0x20, 0xe3, 0x83, 0x84, 0xe3, 0x83, 0x8d, 0xe3, + 0x83, 0x8a, 0xe3, 0x83, 0xa9, 0xe3, 0x83, 0xa0, + 0x0a, 0xe3, 0x82, 0xa6, 0xe3, 0x83, 0xb0, 0xe3, + 0x83, 0x8e, 0xe3, 0x82, 0xaa, 0xe3, 0x82, 0xaf, + 0xe3, 0x83, 0xa4, 0xe3, 0x83, 0x9e, 0x20, 0xe3, + 0x82, 0xb1, 0xe3, 0x83, 0x95, 0xe3, 0x82, 0xb3, + 0xe3, 0x82, 0xa8, 0xe3, 0x83, 0x86, 0x20, 0xe3, + 0x82, 0xa2, 0xe3, 0x82, 0xb5, 0xe3, 0x82, 0xad, + 0xe3, 0x83, 0xa6, 0xe3, 0x83, 0xa1, 0xe3, 0x83, + 0x9f, 0xe3, 0x82, 0xb7, 0x20, 0xe3, 0x83, 0xb1, + 0xe3, 0x83, 0x92, 0xe3, 0x83, 0xa2, 0xe3, 0x82, + 0xbb, 0xe3, 0x82, 0xb9, 0xe3, 0x83, 0xb3]; + + void registerTests(TestSuite suite) { + register("testUtf8bytesToCodepoints", testUtf8bytesToCodepoints, suite); + register("testUtf8BytesToString", testUtf8BytesToString, suite); + register("testEncodeToUtf8", testEncodeToUtf8, suite); + } + + void testEncodeToUtf8() { + Expect.listEquals(testEnglishUtf8, encodeAsUtf8(testEnglishPhrase), + "english to utf8"); + + Expect.listEquals(testDanishUtf8, encodeAsUtf8(testDanishPhrase), + "encode danish to utf8"); + + Expect.listEquals(testHebrewUtf8, encodeAsUtf8(testHebrewPhrase), + "Hebrew to utf8"); + + Expect.listEquals(testRussianUtf8, encodeAsUtf8(testRussianPhrase), + "Russian to utf8"); + + Expect.listEquals(testGreekUtf8, encodeAsUtf8(testGreekPhrase), + "Greek to utf8"); + + Expect.listEquals(testKatakanaUtf8, encodeAsUtf8(testKatakanaPhrase), + "Katakana to utf8"); + } + + void testUtf8bytesToCodepoints() { + Expect.listEquals([954, 972, 963, 956, 949], + _utf8ToCodepoints([0xce, 0xba, 0xcf, 0x8c, 0xcf, + 0x83, 0xce, 0xbc, 0xce, 0xb5]), "κόσμε"); + + // boundary conditions: First possible sequence of a certain length + Expect.listEquals([0x0], _utf8ToCodepoints([0x0]), "0"); + Expect.listEquals([0x80], _utf8ToCodepoints([0xc2, 0x80]), "80"); + Expect.listEquals([0x800], + _utf8ToCodepoints([0xe0, 0xa0, 0x80]), "800"); + Expect.listEquals([0x10000], + _utf8ToCodepoints([0xf0, 0x90, 0x80, 0x80]), "10000"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xf8, 0x88, 0x80, 0x80, 0x80]), "200000"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xfc, 0x84, 0x80, 0x80, 0x80, 0x80]), + "4000000"); + + // boundary conditions: Last possible sequence of a certain length + Expect.listEquals([0x7f], _utf8ToCodepoints([0x7f]), "7f"); + Expect.listEquals([0x7ff], _utf8ToCodepoints([0xdf, 0xbf]), "7ff"); + Expect.listEquals([0xffff], + _utf8ToCodepoints([0xef, 0xbf, 0xbf]), "ffff"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xf7, 0xbf, 0xbf, 0xbf]), "1fffff"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xfb, 0xbf, 0xbf, 0xbf, 0xbf]), "3ffffff"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xfd, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf]), + "4000000"); + + // other boundary conditions + Expect.listEquals([0xd7ff], + _utf8ToCodepoints([0xed, 0x9f, 0xbf]), "d7ff"); + Expect.listEquals([0xe000], + _utf8ToCodepoints([0xee, 0x80, 0x80]), "e000"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xef, 0xbf, 0xbd]), "fffd"); + Expect.listEquals([0x10ffff], + _utf8ToCodepoints([0xf4, 0x8f, 0xbf, 0xbf]), "10ffff"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xf4, 0x90, 0x80, 0x80]), "110000"); + + // unexpected continuation bytes + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0x80]), "80 => replacement character"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xbf]), "bf => replacement character"); + + List allContinuationBytes = []; + List matchingReplacementChars = []; + for (int i = 0x80; i < 0xc0; i++) { + allContinuationBytes.add(i); + matchingReplacementChars.add(UNICODE_REPLACEMENT_CHARACTER_CODEPOINT); + } + Expect.listEquals(matchingReplacementChars, + _utf8ToCodepoints(allContinuationBytes), + "80 - bf => replacement character x 64"); + + List allFirstTwoByteSeq = []; + matchingReplacementChars = []; + for (int i = 0xc0; i < 0xe0; i++) { + allFirstTwoByteSeq.addAll([i, 0x20]); + matchingReplacementChars.addAll( + [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); + } + Expect.listEquals(matchingReplacementChars, + _utf8ToCodepoints(allFirstTwoByteSeq), + "c0 - df + space => replacement character + space x 32"); + + List allFirstThreeByteSeq = []; + matchingReplacementChars = []; + for (int i = 0xe0; i < 0xf0; i++) { + allFirstThreeByteSeq.addAll([i, 0x20]); + matchingReplacementChars.addAll( + [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); + } + Expect.listEquals(matchingReplacementChars, + _utf8ToCodepoints(allFirstThreeByteSeq), + "e0 - ef + space => replacement character x 16"); + + List allFirstFourByteSeq = []; + matchingReplacementChars = []; + for (int i = 0xf0; i < 0xf8; i++) { + allFirstFourByteSeq.addAll([i, 0x20]); + matchingReplacementChars.addAll( + [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); + } + Expect.listEquals(matchingReplacementChars, + _utf8ToCodepoints(allFirstFourByteSeq), + "f0 - f7 + space => replacement character x 8"); + + List allFirstFiveByteSeq = []; + matchingReplacementChars = []; + for (int i = 0xf8; i < 0xfc; i++) { + allFirstFiveByteSeq.addAll([i, 0x20]); + matchingReplacementChars.addAll( + [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); + } + Expect.listEquals(matchingReplacementChars, + _utf8ToCodepoints(allFirstFiveByteSeq), + "f8 - fb + space => replacement character x 4"); + + List allFirstSixByteSeq = []; + matchingReplacementChars = []; + for (int i = 0xfc; i < 0xfe; i++) { + allFirstSixByteSeq.addAll([i, 0x20]); + matchingReplacementChars.addAll( + [UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]); + } + Expect.listEquals(matchingReplacementChars, + _utf8ToCodepoints(allFirstSixByteSeq), + "fc - fd + space => replacement character x 2"); + + // Sequences with last continuation byte missing + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xc2]), + "2-byte sequence with last byte missing"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xe0, 0x80]), + "3-byte sequence with last byte missing"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xf0, 0x80, 0x80]), + "4-byte sequence with last byte missing"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xf8, 0x88, 0x80, 0x80]), + "5-byte sequence with last byte missing"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80]), + "6-byte sequence with last byte missing"); + + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xdf]), + "2-byte sequence with last byte missing (hi)"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xef, 0xbf]), + "3-byte sequence with last byte missing (hi)"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xf7, 0xbf, 0xbf]), + "4-byte sequence with last byte missing (hi)"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xfb, 0xbf, 0xbf, 0xbf]), + "5-byte sequence with last byte missing (hi)"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xfd, 0xbf, 0xbf, 0xbf, 0xbf]), + "6-byte sequence with last byte missing (hi)"); + + // Concatenation of incomplete sequences + Expect.listEquals( + [ UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT ], + _utf8ToCodepoints( + [ 0xc2, + 0xe0, 0x80, + 0xf0, 0x80, 0x80, + 0xf8, 0x88, 0x80, 0x80, + 0xfc, 0x80, 0x80, 0x80, 0x80, + 0xdf, + 0xef, 0xbf, + 0xf7, 0xbf, 0xbf, + 0xfb, 0xbf, 0xbf, 0xbf, + 0xfd, 0xbf, 0xbf, 0xbf, 0xbf ]), + "Concatenation of incomplete sequences"); + + // Impossible bytes + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xfe]), "fe"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xff]), "ff"); + Expect.listEquals([ + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xfe, 0xfe, 0xff, 0xff]), "fe fe ff ff"); + + // Overlong sequences + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xc0, 0xaf]), "c0 af"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xe0, 0x80, 0xaf]), "e0 80 af"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xf0, 0x80, 0x80, 0xaf]), "f0 80 80 af"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xf8, 0x80, 0x80, 0x80, 0xaf]), "f8 80 80 80 af"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80, 0xaf]), + "fc 80 80 80 80 af"); + + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xc1, 0xbf]), "c1 bf"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xe0, 0x9f, 0xbf]), "e0 9f bf"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xf0, 0x8f, 0xbf, 0xbf]), "f0 8f bf bf"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xf8, 0x87, 0xbf, 0xbf, 0xbf]), "f8 87 bf bf bf"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xfc, 0x83, 0xbf, 0xbf, 0xbf, 0xbf]), + "fc 83 bf bf bf bf"); + + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xc0, 0x80]), "c0 80"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xe0, 0x80, 0x80]), "e0 80 80"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xf0, 0x80, 0x80, 0x80]), "f0 80 80 80"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xf8, 0x80, 0x80, 0x80, 0x80]), "f8 80 80 80 80"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xfc, 0x80, 0x80, 0x80, 0x80, 0x80]), + "fc 80 80 80 80 80"); + + // Illegal code positions + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xed, 0xa0, 0x80]), "U+D800"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xed, 0xad, 0xbf]), "U+DB7F"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xed, 0xae, 0x80]), "U+DB80"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xed, 0xaf, 0xbf]), "U+DBFF"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xed, 0xb0, 0x80]), "U+DC00"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xed, 0xbe, 0x80]), "U+DF80"); + Expect.listEquals([UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xed, 0xbf, 0xbf]), "U+DFFF"); + + // Paired UTF-16 surrogates + Expect.listEquals([ + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80]), + "U+D800 U+DC00"); + Expect.listEquals([ + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf]), + "U+D800 U+DFFF"); + Expect.listEquals([ + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xed, 0xad, 0xbf, 0xed, 0xb0, 0x80]), + "U+DB7F U+DC00"); + Expect.listEquals([ + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xed, 0xad, 0xbf, 0xed, 0xbf, 0xbf]), + "U+DB7F U+DFFF"); + Expect.listEquals([ + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xed, 0xae, 0x80, 0xed, 0xb0, 0x80]), + "U+DB80 U+DC00"); + Expect.listEquals([ + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xed, 0xae, 0x80, 0xed, 0xbf, 0xbf]), + "U+DB80 U+DFFF"); + Expect.listEquals([ + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xed, 0xaf, 0xbf, 0xed, 0xb0, 0x80]), + "U+DBFF U+DC00"); + Expect.listEquals([ + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT, + UNICODE_REPLACEMENT_CHARACTER_CODEPOINT], + _utf8ToCodepoints([0xed, 0xaf, 0xbf, 0xed, 0xbf, 0xbf]), + "U+DBFF U+DFFF"); + + // Other illegal code positions (???) + Expect.listEquals([0xfffe], _utf8ToCodepoints([0xef, 0xbf, 0xbe]), + "U+FFFE"); + Expect.listEquals([0xffff], _utf8ToCodepoints([0xef, 0xbf, 0xbf]), + "U+FFFF"); + } + + void testUtf8BytesToString() { + Expect.stringEquals(testEnglishPhrase, + decodeFromUtf8(testEnglishUtf8), "English"); + + Expect.stringEquals(testDanishPhrase, + decodeFromUtf8(testDanishUtf8), "Danish"); + + Expect.stringEquals(testHebrewPhrase, + decodeFromUtf8(testHebrewUtf8), "Hebrew"); + + Expect.stringEquals(testRussianPhrase, + decodeFromUtf8(testRussianUtf8), "Russian"); + + Expect.stringEquals(testGreekPhrase, + decodeFromUtf8(testGreekUtf8), "Greek"); + + Expect.stringEquals(testKatakanaPhrase, + decodeFromUtf8(testKatakanaUtf8), "Katakana"); + } +}