String encoding utility methods and tests for Unicode, UTF-8, -16 and -32.

BUG=
TEST=

Review URL: https://chromiumcodereview.appspot.com//9233041

git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@3814 260f80e4-7a28-3924-810f-c04153c831b5

utils/string_encoding/Unicode.dart

@@ -0,0 +1,37 @@
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#library("unicode");
+#import("unicode_core.dart");
+
+/**
+ * Provide Unicode codepoints for a given string.
+ */
+List<int> stringToCodepoints(String str) {
+  List<int> codepoints;
+  // TODO is16BitCodeUnit() is used to work around a bug with frog/dartc
+  // (http://code.google.com/p/dart/issues/detail?id=1357). Consider
+  // removing after this issue is resolved.
+  if (is16BitCodeUnit()) {
+    codepoints = utf16CodeUnitsToCodepoints(str.charCodes());
+  } else {
+    codepoints = str.charCodes();
+  }
+  return codepoints;
+}
+
+/**
+ * Generate a string for the provided Unicode codepoints.
+ */
+String codepointsToString(List<int> codepoints) {
+  // TODO is16BitCodeUnit() is used to work around a bug with frog/dartc
+  // (http://code.google.com/p/dart/issues/detail?id=1357). Consider
+  // removing after this issue is resolved.
+  if (is16BitCodeUnit()) {
+    return new String.fromCharCodes(
+        codepointsToUtf16CodeUnits(codepoints));
+  } else {
+    return new String.fromCharCodes(codepoints);
+  }
+}

utils/string_encoding/Utf16.dart

@@ -0,0 +1,273 @@
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#library("utf16");
+#import("unicode_core.dart");
+#import("unicode.dart");
+
+/**
+ * Produce a String from a sequence of UTF-16 encoded bytes. The parameters
+ * allow an offset into a list of bytes (as int), limiting the length of the
+ * values be decoded and the ability of override the default Unicode
+ * replacement character. Set the replacementCharacter to null to throw an
+ * IllegalArgumentException rather than replace the bad value.
+ */
+String decodeFromUtf16(List<int> bytes, [int offset = 0, int length,
+    int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
+  List<int> codeUnits =
+      _utf16ToUtf16CodeUnits(bytes, offset, length);
+  // TODO is16BitCodeUnit() is used to work around a bug with frog/dartc
+  // (http://code.google.com/p/dart/issues/detail?id=1357). Consider
+  // removing after this issue is resolved.
+  if (is16BitCodeUnit()) {
+    return new String.fromCharCodes(codeUnits);
+  } else {
+    return new String.fromCharCodes(
+        utf16CodeUnitsToCodepoints(codeUnits, 0, null, replacementCodepoint));
+  }
+}
+
+/**
+ * Produce a String from a sequence of UTF-16BE encoded bytes. The parameters
+ * allow an offset into a list of bytes (as int), limiting the length of the
+ * values be decoded and the ability of override the default Unicode
+ * replacement character. Set the replacementCharacter to null to throw an
+ * IllegalArgumentException rather than replace the bad value.
+ */
+String decodeFromUtf16be(List<int> bytes, [int offset = 0, int length,
+    bool stripBom = true,
+    int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
+  List<int> codeUnits =
+      _utf16beToUtf16CodeUnits(bytes, offset, length, stripBom);
+  // TODO is16BitCodeUnit() is used to work around a bug with frog/dartc
+  // (http://code.google.com/p/dart/issues/detail?id=1357). Consider
+  // removing after this issue is resolved.
+  if (is16BitCodeUnit()) {
+    return new String.fromCharCodes(codeUnits);
+  } else {
+    return new String.fromCharCodes(
+        utf16CodeUnitsToCodepoints(codeUnits, 0, null, replacementCodepoint));
+  }
+}
+
+/**
+ * Produce a String from a sequence of UTF-16LE encoded bytes. The parameters
+ * allow an offset into a list of bytes (as int), limiting the length of the
+ * values be decoded and the ability of override the default Unicode
+ * replacement character. Set the replacementCharacter to null to throw an
+ * IllegalArgumentException rather than replace the bad value.
+ */
+String decodeFromUtf16le(List<int> bytes, [int offset = 0, int length,
+    bool stripBom = true,
+    int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
+  List<int> codeUnits =
+      _utf16leToUtf16CodeUnits(bytes, offset, length, stripBom);
+  // TODO is16BitCodeUnit() is used to work around a bug with frog/dartc
+  // (http://code.google.com/p/dart/issues/detail?id=1357). Consider
+  // removing after this issue is resolved.
+  if (is16BitCodeUnit()) {
+    return new String.fromCharCodes(codeUnits);
+  } else {
+    return new String.fromCharCodes(
+        utf16CodeUnitsToCodepoints(codeUnits, 0, null, replacementCodepoint));
+  }
+}
+
+/**
+ * Produce a sequence of UTF-16 encoded bytes.
+ */
+List<int> encodeAsUtf16(String str) =>
+    encodeAsUtf16be(str, true);
+
+/**
+ * Produce a sequence of UTF-16BE encoded bytes.
+ */
+List<int> encodeAsUtf16be(String str, [bool writeBOM = false]) {
+  List<int> utf16CodeUnits = _stringToUtf16CodeUnits(str);
+  List<int> encoding =
+      new List<int>(2 * utf16CodeUnits.length + (writeBOM ? 2 : 0));
+  int i = 0;
+  if (writeBOM) {
+    encoding[i++] = UNICODE_UTF_BOM_HI;
+    encoding[i++] = UNICODE_UTF_BOM_LO;
+  }
+  for (int unit in utf16CodeUnits) {
+    encoding[i++] = (unit & UNICODE_BYTE_ONE_MASK) >> 8;
+    encoding[i++] = unit & UNICODE_BYTE_ZERO_MASK;
+  }
+  return encoding;
+}
+
+/**
+ * Produce a sequence of UTF-16LE encoded bytes.
+ */
+List<int> encodeAsUtf16le(String str, [bool writeBOM = false]) {
+  List<int> utf16CodeUnits = _stringToUtf16CodeUnits(str);
+  List<int> encoding =
+      new List<int>(2 * utf16CodeUnits.length + (writeBOM ? 2 : 0));
+  int i = 0;
+  if (writeBOM) {
+    encoding[i++] = UNICODE_UTF_BOM_LO;
+    encoding[i++] = UNICODE_UTF_BOM_HI;
+  }
+  for (int unit in utf16CodeUnits) {
+    encoding[i++] = unit & UNICODE_BYTE_ZERO_MASK;
+    encoding[i++] = (unit & UNICODE_BYTE_ONE_MASK) >> 8;
+  }
+  return encoding;
+}
+
+bool hasUtf16Bom(List<int> utf32EncodedBytes, [int offset = 0, int length]) {
+  return hasUtf16beBom(utf32EncodedBytes, offset, length) ||
+      hasUtf16leBom(utf32EncodedBytes, offset, length);
+}
+
+bool hasUtf16beBom(List<int> utf16EncodedBytes, [int offset = 0, int length]) {
+  if (!(offset >= 0)) {
+    throw new IllegalArgumentException("offset");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(utf16EncodedBytes.length, offset + length) :
+      utf16EncodedBytes.length;
+
+  return (offset + 2) <= end &&
+      utf16EncodedBytes[offset] == UNICODE_UTF_BOM_HI &&
+      utf16EncodedBytes[offset + 1] == UNICODE_UTF_BOM_LO;
+}
+
+bool hasUtf16leBom(List<int> utf16EncodedBytes, [int offset = 0, int length]) {
+  if (!(offset >= 0)) {
+    throw new IllegalArgumentException("offset");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(utf16EncodedBytes.length, offset + length) :
+      utf16EncodedBytes.length;
+
+  return (offset + 2) <= end &&
+      utf16EncodedBytes[offset] == UNICODE_UTF_BOM_LO &&
+      utf16EncodedBytes[offset + 1] == UNICODE_UTF_BOM_HI;
+}
+
+int _sizeCodeUnits(int utf16CodeUnitsLength) {
+  int v = ((utf16CodeUnitsLength)/2).floor().toInt();
+  return v;
+}
+
+List<int> _stringToUtf16CodeUnits(String str) {
+  // TODO is16BitCodeUnit() is used to work around a bug with frog/dartc
+  // (http://code.google.com/p/dart/issues/detail?id=1357). Consider
+  // removing after this issue is resolved.
+  if (is16BitCodeUnit()) {
+    return str.charCodes();
+  } else {
+    return codepointsToUtf16CodeUnits(str.charCodes());
+  }
+}
+
+/**
+ * Convert UTF-16BE encoded bytes to utf16 code units by grouping 1-2 bytes
+ * to produce the code unit (0-(2^16)-1).
+ */
+List<int> _utf16beToUtf16CodeUnits(
+    List<int> utf16beEncodedBytes, [int offset = 0, int length,
+    bool stripBom = true]) {
+  if (!(offset >= 0)) {
+    throw new IllegalArgumentException("offset");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(utf16beEncodedBytes.length, offset + length) :
+      utf16beEncodedBytes.length;
+
+  int i = (stripBom && hasUtf16beBom(utf16beEncodedBytes, offset, length)) ?
+      offset + 2 : offset;
+  List<int> codeUnits =
+      new List<int>(_sizeCodeUnits(end - i));
+  int lastIndex = end - 1;
+  int j = 0;
+  while (i < lastIndex) {
+    int hi = utf16beEncodedBytes[i++];
+    int lo = utf16beEncodedBytes[i++];
+    codeUnits[j++] = (hi << 8) | lo;
+  }
+  return codeUnits;
+}
+
+/**
+ * Convert UTF-16LE encoded bytes to utf16 code units by grouping 1-2 bytes
+ * to produce the code unit (0-(2^16)-1).
+ */
+List<int> _utf16leToUtf16CodeUnits(
+    List<int> utf16leEncodedBytes, [int offset = 0, int length,
+    bool stripBom = true]) {
+  if (!(offset >= 0)) {
+    throw new IllegalArgumentException("offset");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(utf16leEncodedBytes.length, offset + length) :
+      utf16leEncodedBytes.length;
+
+  int i = (stripBom && hasUtf16leBom(utf16leEncodedBytes, offset, length)) ?
+      offset + 2 : offset;
+  List<int> codeUnits =
+      new List<int>(_sizeCodeUnits(end - i));
+  int lastIndex = end - 1;
+  int j = 0;
+  while (i < lastIndex) {
+    int lo = utf16leEncodedBytes[i++];
+    int hi = utf16leEncodedBytes[i++];
+    codeUnits[j] = (hi << 8) | lo;
+  }
+  return codeUnits;
+}
+
+/**
+ * Convert UTF-16 encoded bytes to utf16 code units by grouping 1-2 bytes
+ * to produce the code unit (0-(2^16)-1). Relies on BOM to determine
+ * endian-ness, and defaults to BE.
+ */
+List<int> _utf16ToUtf16CodeUnits(
+    List<int> utf16EncodedBytes, [int offset = 0, int length]) {
+  if (!(offset >= 0)) {
+    throw new IllegalArgumentException("offset");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(utf16EncodedBytes.length, offset + length) :
+      utf16EncodedBytes.length;
+
+  if (hasUtf16beBom(utf16EncodedBytes, offset, length)) {
+    return _utf16beToUtf16CodeUnits(utf16EncodedBytes, offset + 2,
+        end - (offset + 2), false);
+  } else if (hasUtf16leBom(utf16EncodedBytes, offset, length)) {
+    return _utf16leToUtf16CodeUnits(utf16EncodedBytes, offset + 2,
+        end - (offset + 2), false);
+  } else {
+    return _utf16beToUtf16CodeUnits(
+        utf16EncodedBytes, offset, end - offset, false);
+  }
+}

utils/string_encoding/Utf32.dart

@@ -0,0 +1,273 @@
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#library("utf32");
+#import("unicode_core.dart");
+#import("unicode.dart");
+
+/**
+ * Produce a String from a sequence of UTF-32 encoded bytes. The parameters
+ * allow an offset into a list of bytes (as int), limiting the length of the
+ * values be decoded and the ability of override the default Unicode
+ * replacement character. Set the replacementCharacter to null to throw an
+ * IllegalArgumentException rather than replace the bad value.
+ */
+String decodeFromUtf32(List<int> bytes, [int offset = 0, int length,
+    int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) =>
+    codepointsToString(_utf32ToCodePoints(bytes, offset, length,
+        replacementCodepoint));
+
+/**
+ * Produce a String from a sequence of UTF-32BE encoded bytes. The parameters
+ * allow an offset into a list of bytes (as int), limiting the length of the
+ * values be decoded and the ability of override the default Unicode
+ * replacement character. Set the replacementCharacter to null to throw an
+ * IllegalArgumentException rather than replace the bad value.
+ */
+String decodeFromUtf32be(
+    List<int> bytes, [int offset = 0, int length, bool stripBom = true,
+    int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) =>
+    codepointsToString(_utf32beToCodePoints(bytes, offset, length, stripBom,
+        replacementCodepoint));
+
+/**
+ * Produce a String from a sequence of UTF-32LE encoded bytes. The parameters
+ * allow an offset into a list of bytes (as int), limiting the length of the
+ * values be decoded and the ability of override the default Unicode
+ * replacement character. Set the replacementCharacter to null to throw an
+ * IllegalArgumentException rather than replace the bad value.
+ */
+String decodeFromUtf32le(
+    List<int> bytes, [int offset = 0, int length, bool stripBom = true,
+    int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) =>
+    codepointsToString(_utf32leToCodePoints(bytes, offset, length, stripBom,
+        replacementCodepoint));
+
+/**
+ * Produce a sequence of UTF-32 encoded bytes.
+ */
+List<int> encodeAsUtf32(String str) =>
+    encodeAsUtf32be(str, true);
+
+/**
+ * Produce a sequence of UTF-32BE encoded bytes.
+ */
+List<int> encodeAsUtf32be(String str, [bool writeBOM = false]) {
+  List<int> utf32CodeUnits = stringToCodepoints(str);
+  List<int> encoding = new List<int>(4 * utf32CodeUnits.length +
+      (writeBOM ? 4 : 0));
+  int i = 0;
+  if (writeBOM) {
+    encoding[i++] = 0;
+    encoding[i++] = 0;
+    encoding[i++] = UNICODE_UTF_BOM_HI;
+    encoding[i++] = UNICODE_UTF_BOM_LO;
+  }
+  for (int unit in utf32CodeUnits) {
+    encoding[i++] = (unit >> 24) & UNICODE_BYTE_ZERO_MASK;
+    encoding[i++] = (unit >> 16) & UNICODE_BYTE_ZERO_MASK;
+    encoding[i++] = (unit >> 8) & UNICODE_BYTE_ZERO_MASK;
+    encoding[i++] = unit & UNICODE_BYTE_ZERO_MASK;
+  }
+  return encoding;
+}
+
+/**
+ * Produce a sequence of UTF-32LE encoded bytes.
+ */
+List<int> encodeAsUtf32le(String str, [bool writeBOM = false]) {
+  List<int> utf32CodeUnits = stringToCodepoints(str);
+  List<int> encoding = new List<int>(4 * utf32CodeUnits.length +
+      (writeBOM ? 4 : 0));
+  int i = 0;
+  if (writeBOM) {
+    encoding[i++] = UNICODE_UTF_BOM_LO;
+    encoding[i++] = UNICODE_UTF_BOM_HI;
+    encoding[i++] = 0;
+    encoding[i++] = 0;
+  }
+  for (int unit in utf32CodeUnits) {
+    encoding[i++] = unit & UNICODE_BYTE_ZERO_MASK;
+    encoding[i++] = (unit >> 8) & UNICODE_BYTE_ZERO_MASK;
+    encoding[i++] = (unit >> 16) & UNICODE_BYTE_ZERO_MASK;
+    encoding[i++] = (unit >> 24) & UNICODE_BYTE_ZERO_MASK;
+  }
+  return encoding;
+}
+
+bool hasUtf32Bom(
+    List<int> utf32EncodedBytes, [int offset = 0, int length]) {
+  return hasUtf32beBom(utf32EncodedBytes, offset, length) ||
+      hasUtf32leBom(utf32EncodedBytes, offset, length);
+}
+
+bool hasUtf32beBom(List<int> utf32EncodedBytes, [int offset = 0, int length]) {
+  if (!(offset >= 0)) {
+    throw new IllegalArgumentException("offset");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(utf32EncodedBytes.length, offset + length) :
+      utf32EncodedBytes.length;
+
+  return (offset + 4) <= end &&
+      utf32EncodedBytes[offset] == 0 &&
+      utf32EncodedBytes[offset + 1] == 0 &&
+      utf32EncodedBytes[offset + 2] == UNICODE_UTF_BOM_HI &&
+      utf32EncodedBytes[offset + 3] == UNICODE_UTF_BOM_LO;
+}
+
+bool hasUtf32leBom(List<int> utf32EncodedBytes, [int offset = 0, int length]) {
+  if (!(offset >= 0)) {
+    throw new IllegalArgumentException("offset");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(utf32EncodedBytes.length, offset + length) :
+      utf32EncodedBytes.length;
+
+  return (offset + 4) <= end &&
+      utf32EncodedBytes[offset] == UNICODE_UTF_BOM_LO &&
+      utf32EncodedBytes[offset + 1] == UNICODE_UTF_BOM_HI &&
+      utf32EncodedBytes[offset + 2] == 0 &&
+      utf32EncodedBytes[offset + 3] == 0;
+}
+
+void _addReplacementCodepoint(List<int> codepointBuffer, int offset,
+    int replacementCodepoint) {
+  if(replacementCodepoint != null) {
+    codepointBuffer[offset] = replacementCodepoint;
+  } else {
+    throw new IllegalArgumentException("Invalid encoding");
+  }
+}
+
+int _sizeCodepoints(int utf32BytesLength) =>
+    ((utf32BytesLength)/4).ceil().toInt();
+
+/**
+ * Joins groups of 4 bytes (0-255) UTF-32BE to produce single code points.
+ */
+List<int> _utf32beToCodePoints(
+    List<int> utf32beEncodedBytes, [int offset = 0, int length,
+    bool stripBom = true,
+    int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
+  if (!(offset >= 0)) {
+    throw new IllegalArgumentException("offset");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(utf32beEncodedBytes.length, offset + length) :
+      utf32beEncodedBytes.length;
+
+  int i = (stripBom && hasUtf32beBom(utf32beEncodedBytes, offset, length)) ?
+      offset + 4 : offset;
+  int lastIndex = end - 3;
+  List<int> codepoints = new List<int>(_sizeCodepoints(end - i));
+  int j = 0;
+  while (i < lastIndex) {
+    int value = utf32beEncodedBytes[i++];
+    value = (value << 8) + utf32beEncodedBytes[i++];
+    value = (value << 8) + utf32beEncodedBytes[i++];
+    value = (value << 8) + utf32beEncodedBytes[i++];
+    if (_validCodepoint(value)) {
+      codepoints[j++] = value;
+    } else {
+      _addReplacementCodepoint(codepoints, j++, replacementCodepoint);
+    }
+  }
+  while (j < codepoints.length) {
+    _addReplacementCodepoint(codepoints, j++, replacementCodepoint);
+  }
+  return codepoints;
+}
+
+/**
+ * Joins groups of 4 bytes (0-255) UTF-32LE to produce single code points.
+ */
+List<int> _utf32leToCodePoints(
+    List<int> utf32leEncodedBytes, [int offset = 0, int length,
+    bool stripBom = true,
+    int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
+  if (!(offset >= 0)) {
+    throw new IllegalArgumentException("offset");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(utf32leEncodedBytes.length, offset + length) :
+      utf32leEncodedBytes.length;
+
+  int i = (stripBom && hasUtf32leBom(utf32leEncodedBytes, offset, length)) ?
+      offset + 4 : offset;
+  int lastIndex = end - 3;
+  List<int> codepoints = new List<int>(_sizeCodepoints(end - i));
+  int j = 0;
+  while (i < lastIndex) {
+    int value = utf32leEncodedBytes[i+3];
+    value = (value << 8) + utf32leEncodedBytes[i+2];
+    value = (value << 8) + utf32leEncodedBytes[i+1];
+    value = (value << 8) + utf32leEncodedBytes[i];
+    i += 4;
+    if (_validCodepoint(value)) {
+      codepoints[j++] = value;
+    } else {
+      _addReplacementCodepoint(codepoints, j++, replacementCodepoint);
+    }
+  }
+  while (j < codepoints.length) {
+    _addReplacementCodepoint(codepoints, j++, replacementCodepoint);
+  }
+  return codepoints;
+}
+
+/**
+ * Joins groups of 4 bytes (0-255) UTF-32 to produce single code points.
+ */
+List<int> _utf32ToCodePoints(List<int> utf32EncodedBytes, [int offset = 0,
+    int length,
+    int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
+  if (!(offset >= 0)) {
+    throw new IllegalArgumentException("offset");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(utf32EncodedBytes.length, offset + length) :
+      utf32EncodedBytes.length;
+
+  if (hasUtf32beBom(utf32EncodedBytes, offset, length)) {
+    return _utf32beToCodePoints(utf32EncodedBytes, offset + 4,
+        end - (offset + 4), false);
+  } else if (hasUtf32leBom(utf32EncodedBytes, offset, length)) {
+    return _utf32leToCodePoints(utf32EncodedBytes, offset + 4,
+        end - (offset + 4), false);
+  } else {
+    return _utf32beToCodePoints(utf32EncodedBytes, offset, end - offset);
+  }
+}
+
+bool _validCodepoint(int codepoint) {
+  return (codepoint >= 0 && codepoint < UNICODE_UTF16_RESERVED_LO) ||
+      (codepoint > UNICODE_UTF16_RESERVED_HI &&
+      codepoint < UNICODE_VALID_RANGE_MAX);
+}

utils/string_encoding/Utf8.dart

@@ -0,0 +1,8 @@
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#library("utf8");
+#import("unicode_core.dart");
+#import("unicode.dart");
+#source("utf8_impl.dart");

utils/string_encoding/Utf8_impl.dart

@@ -0,0 +1,221 @@
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+final int _UTF8_ONE_BYTE_MAX = 0x7f;
+final int _UTF8_TWO_BYTE_MAX = 0x7ff;
+final int _UTF8_THREE_BYTE_MAX = 0xffff;
+
+final int _UTF8_LO_SIX_BIT_MASK = 0x3f;
+
+final int _UTF8_FIRST_BYTE_OF_TWO_BASE = 0xc0;
+final int _UTF8_FIRST_BYTE_OF_THREE_BASE = 0xe0;
+final int _UTF8_FIRST_BYTE_OF_FOUR_BASE = 0xf0;
+final int _UTF8_FIRST_BYTE_OF_FIVE_BASE = 0xf8;
+final int _UTF8_FIRST_BYTE_OF_SIX_BASE = 0xfc;
+
+final int _UTF8_FIRST_BYTE_OF_TWO_MASK = 0x1f;
+final int _UTF8_FIRST_BYTE_OF_THREE_MASK = 0xf;
+final int _UTF8_FIRST_BYTE_OF_FOUR_MASK = 0x7;
+
+final int _UTF8_FIRST_BYTE_BOUND_EXCL = 0xfe;
+final int _UTF8_SUBSEQUENT_BYTE_BASE = 0x80;
+
+/**
+ * Produce a String from a sequence of UTF-8 encoded bytes. The parameters
+ * allow an offset into a list of bytes (as int), limiting the length of the
+ * values be decoded and the ability of override the default Unicode
+ * replacement character. Set the replacementCharacter to null to throw an
+ * IllegalArgumentException rather than replace the bad value.
+ */
+String decodeFromUtf8(List<int> bytes, [int offset = 0, int length,
+    int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) =>
+    codepointsToString(_utf8ToCodepoints(
+        bytes, offset, length, replacementCodepoint));
+
+/**
+ * Produce a sequence of UTF-8 encoded bytes from the provided string.
+ */
+List<int> encodeAsUtf8(String str) =>
+  _codepointsToUtf8(stringToCodepoints(str));
+
+int _addToEncoding(int offset, int bytes, int value, List<int> buffer) {
+  while(bytes > 0) {
+    buffer[offset + bytes] = _UTF8_SUBSEQUENT_BYTE_BASE |
+        (value & _UTF8_LO_SIX_BIT_MASK);
+    value = value >> 6;
+    bytes--;
+  }
+  return value;
+}
+
+/**
+ * Encode code points as UTF-8 code units.
+ */
+List<int> _codepointsToUtf8(
+    List<int> codepoints, [int offset = 0, int length]) {
+  if (!(offset >= 0)) {
+    throw new IllegalArgumentException("offset");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(codepoints.length, offset + length) :
+      codepoints.length;
+
+  int encodedLength = 0;
+  for (int i = offset; i < end; i++) {
+    int value = codepoints[i];
+    if (value < 0 || value > UNICODE_VALID_RANGE_MAX) {
+      encodedLength += 3;
+    } else if (value <= _UTF8_ONE_BYTE_MAX) {
+      encodedLength++;
+    } else if (value <= _UTF8_TWO_BYTE_MAX) {
+      encodedLength += 2;
+    } else if (value <= _UTF8_THREE_BYTE_MAX) {
+      encodedLength += 3;
+    } else if (value <= UNICODE_VALID_RANGE_MAX) {
+      encodedLength += 4;
+    }
+  }
+
+  List<int> encoded = new List<int>(encodedLength);
+  int insertAt = 0;
+  for (int i = offset; i < end; i++) {
+    int value = codepoints[i];
+    if (value < 0 || value > UNICODE_VALID_RANGE_MAX) {
+      encoded.setRange(insertAt, 3, [0xef, 0xbf, 0xbd]);
+      insertAt += 3;
+    } else if (value <= _UTF8_ONE_BYTE_MAX) {
+      encoded[insertAt] = value;
+      insertAt++;
+    } else if (value <= _UTF8_TWO_BYTE_MAX) {
+      encoded[insertAt] = _UTF8_FIRST_BYTE_OF_TWO_BASE | (
+          _UTF8_FIRST_BYTE_OF_TWO_MASK &
+          _addToEncoding(insertAt, 1, value, encoded));
+      insertAt += 2;
+    } else if (value <= _UTF8_THREE_BYTE_MAX) {
+      encoded[insertAt] = _UTF8_FIRST_BYTE_OF_THREE_BASE | (
+          _UTF8_FIRST_BYTE_OF_THREE_MASK &
+          _addToEncoding(insertAt, 2, value, encoded));
+      insertAt += 3;
+    } else if (value <= UNICODE_VALID_RANGE_MAX) {
+      encoded[insertAt] = _UTF8_FIRST_BYTE_OF_FOUR_BASE | (
+          _UTF8_FIRST_BYTE_OF_FOUR_MASK &
+          _addToEncoding(insertAt, 3, value, encoded));
+      insertAt += 4;
+    }
+  }
+  return encoded;
+}
+
+
+// Because UTF-8 specifies byte order, we do not have to follow the pattern
+// used by UTF-16 & UTF-32 regarding byte order.
+List<int> _utf8ToCodepoints(
+    List<int> utf8EncodedBytes, [int offset = 0, int length,
+    int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
+  if (!(offset >= 0)) {
+    throw new IllegalArgumentException("offset");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(utf8EncodedBytes.length, offset + length) :
+      utf8EncodedBytes.length;
+
+  void addReplacementCodepoint(void f(int v), int replacementCodepoint) {
+    if(replacementCodepoint != null) {
+      f(replacementCodepoint);
+    } else {
+      throw new IllegalArgumentException("Invalid encoding");
+    }
+  }
+
+  void apply(void f(int v)) {
+    int i = offset;
+    while (i < end) {
+      int value = utf8EncodedBytes[i++];
+      if (value >= 0) {
+        if (value <= _UTF8_ONE_BYTE_MAX) {
+          f(value);
+        } else if (value < _UTF8_FIRST_BYTE_OF_TWO_BASE) {
+          addReplacementCodepoint(f, replacementCodepoint);
+          continue;
+        } else {
+          int additionalBytes = 0;
+          if (value < _UTF8_FIRST_BYTE_OF_THREE_BASE) {
+            value -= _UTF8_FIRST_BYTE_OF_TWO_BASE;
+            additionalBytes = 1;
+          } else if (value < _UTF8_FIRST_BYTE_OF_FOUR_BASE) {
+            value -= _UTF8_FIRST_BYTE_OF_THREE_BASE;
+            additionalBytes = 2;
+          } else if (value < _UTF8_FIRST_BYTE_OF_FIVE_BASE) {
+            value -= _UTF8_FIRST_BYTE_OF_FOUR_BASE;
+            additionalBytes = 3;
+          } else if (value < _UTF8_FIRST_BYTE_OF_SIX_BASE) {
+            value -= _UTF8_FIRST_BYTE_OF_FIVE_BASE;
+            additionalBytes = 4;
+          } else if (value < _UTF8_FIRST_BYTE_BOUND_EXCL) {
+            value -= _UTF8_FIRST_BYTE_OF_SIX_BASE;
+            additionalBytes = 5;
+          } else {
+            addReplacementCodepoint(f, replacementCodepoint);
+            continue;
+          }
+          int j = 0;
+          while (j < additionalBytes && i < end) {
+            int nextValue = utf8EncodedBytes[i++];
+            if (nextValue > _UTF8_ONE_BYTE_MAX &&
+                nextValue < _UTF8_FIRST_BYTE_OF_TWO_BASE) {
+              value = (value << 6) | (nextValue & _UTF8_LO_SIX_BIT_MASK);
+            } else {
+              // if sequence-starting code unit, reposition cursor to start here
+              if (nextValue >= _UTF8_FIRST_BYTE_OF_TWO_BASE) {
+                i--;
+              }
+              break;
+            }
+            j++;
+          }
+          if (j == additionalBytes && (
+              value < UNICODE_UTF16_RESERVED_LO ||
+              value > UNICODE_UTF16_RESERVED_HI)) {
+            if ((additionalBytes == 1 && value > _UTF8_ONE_BYTE_MAX) ||
+                (additionalBytes == 2 && value > _UTF8_TWO_BYTE_MAX) ||
+                (additionalBytes == 3 && value > _UTF8_THREE_BYTE_MAX &&
+                  value <= UNICODE_VALID_RANGE_MAX)) {
+              f(value);
+            } else {
+              addReplacementCodepoint(f, replacementCodepoint);
+            }
+          } else {
+            addReplacementCodepoint(f, replacementCodepoint);
+            continue;
+          }
+        }
+      } else {
+        addReplacementCodepoint(f, replacementCodepoint);
+        continue;
+      }
+    }
+  }
+
+  int codepointBufferLength = 0;
+  apply(void _(int value) {
+      codepointBufferLength++;
+  });
+
+  List<int> codepointBuffer = new List<int>(codepointBufferLength);
+  int i = 0;
+  apply(void _(int value) {
+      codepointBuffer[i++] = value;
+  });
+  return codepointBuffer;
+}

utils/string_encoding/unicode_core.dart

@@ -0,0 +1,181 @@
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#library("unicode_core");
+
+/*
+ * Test for presence of bug related to the use of UTF-16 code units for
+ * Dart compiled to JS.
+ */
+bool _test16BitCodeUnit = null;
+// TODO is16BitCodeUnit() is used to work around a bug with frog/dartc
+// (http://code.google.com/p/dart/issues/detail?id=1357). Consider
+// removing after this issue is resolved.
+bool is16BitCodeUnit() {
+  if (_test16BitCodeUnit == null) {
+    _test16BitCodeUnit = (new String.fromCharCodes([0x1D11E])) ==
+        (new String.fromCharCodes([0xD11E]));
+  }
+  return _test16BitCodeUnit;
+}
+
+/**
+ * Invalid codepoints or encodings may be substituted with the value U+fffd.
+ */
+final int UNICODE_REPLACEMENT_CHARACTER_CODEPOINT = 0xfffd;
+final int UNICODE_BOM = 0xfeff;
+final int UNICODE_UTF_BOM_LO = 0xff;
+final int UNICODE_UTF_BOM_HI = 0xfe;
+
+final int UNICODE_BYTE_ZERO_MASK = 0xff;
+final int UNICODE_BYTE_ONE_MASK = 0xff00;
+final int UNICODE_VALID_RANGE_MAX = 0x10ffff;
+final int UNICODE_PLANE_ONE_MAX = 0xffff;
+final int UNICODE_UTF16_RESERVED_LO = 0xd800;
+final int UNICODE_UTF16_RESERVED_HI = 0xdfff;
+final int UNICODE_UTF16_OFFSET = 0x10000;
+final int UNICODE_UTF16_SURROGATE_UNIT_0_BASE = 0xd800;
+final int UNICODE_UTF16_SURROGATE_UNIT_1_BASE = 0xdc00;
+final int UNICODE_UTF16_HI_MASK = 0xffc00;
+final int UNICODE_UTF16_LO_MASK = 0x3ff;
+
+/**
+ * Encode code points as UTF16 code units.
+ */
+List<int> codepointsToUtf16CodeUnits(
+    List<int> codepoints, [int offset = 0, int length,
+    int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
+  if (!(offset >= 0)) {
+    throw new IllegalArgumentException("offset");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(codepoints.length, offset + length) :
+      codepoints.length;
+
+  int encodedLength = 0;
+  for (int i = offset; i < end; i++) {
+    int value = codepoints[i];
+    if ((value >= 0 && value < UNICODE_UTF16_RESERVED_LO) || 
+        (value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) {
+      encodedLength++;
+    } else if (value > UNICODE_PLANE_ONE_MAX && 
+        value <= UNICODE_VALID_RANGE_MAX) {
+      encodedLength += 2;
+    } else {
+      encodedLength++;
+    }
+  }
+  
+  void addReplacementCodepoint(List<int> codepointBuffer, int offset,
+      int replacementCodepoint) {
+    if(replacementCodepoint != null) {
+      codepointBuffer[offset] = replacementCodepoint;
+    } else {
+      throw new IllegalArgumentException("Invalid encoding");
+    }
+  }
+  List<int> codeUnitsBuffer = new List<int>(encodedLength);
+  int j = 0;
+  for (int i = offset; i < end; i++) {
+    int value = codepoints[i];
+    if ((value >= 0 && value < UNICODE_UTF16_RESERVED_LO) || 
+        (value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) {
+      codeUnitsBuffer[j++] = value;
+    } else if (value > UNICODE_PLANE_ONE_MAX && 
+        value <= UNICODE_VALID_RANGE_MAX) {
+      int base = value - UNICODE_UTF16_OFFSET;
+      codeUnitsBuffer[j++] = UNICODE_UTF16_SURROGATE_UNIT_0_BASE +
+          ((base & UNICODE_UTF16_HI_MASK) >> 10);
+      codeUnitsBuffer[j++] = UNICODE_UTF16_SURROGATE_UNIT_1_BASE +
+          (base & UNICODE_UTF16_LO_MASK);
+    } else {
+      addReplacementCodepoint(codeUnitsBuffer, j++, replacementCodepoint);
+    }
+  }
+  return codeUnitsBuffer;
+}
+
+/**
+ * Decodes the utf16 codeunits to codepoints.
+ */
+List<int> utf16CodeUnitsToCodepoints(
+    List<int> utf16CodeUnits, [int offset = 0, int length,
+    int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
+  if (!(offset >= 0)) {
+    throw new IllegalArgumentException("offset");
+  }
+
+  if (!(length == null || length >= 0)) {
+    throw new IllegalArgumentException("length");
+  }
+
+  int end = length != null ?
+      Math.min(utf16CodeUnits.length, offset + length) :
+      utf16CodeUnits.length;
+
+  void addReplacementCodepoint(void f(int v), int replacementCodepoint) {
+    if(replacementCodepoint != null) {
+      f(replacementCodepoint);
+    } else {
+      throw new IllegalArgumentException("Invalid encoding");
+    }
+  }
+
+  void apply(void f(int v)) {
+    int i = offset;
+    // skip the first entry if it is a BOM.
+    if (end > 0 && utf16CodeUnits[0] == UNICODE_BOM) {
+      i++;
+    }
+    while (i < end) {
+      int value = utf16CodeUnits[i++];
+      if (value < 0) {
+        addReplacementCodepoint(f, replacementCodepoint);
+        continue;
+      }
+      if (value < UNICODE_UTF16_RESERVED_LO || 
+          (value > UNICODE_UTF16_RESERVED_HI &&
+          value <= UNICODE_PLANE_ONE_MAX)) {
+        // transfer directly
+        f(value);
+      } else if (value < UNICODE_UTF16_SURROGATE_UNIT_1_BASE && i < end) {
+        // merge surrogate pair
+        int nextValue = utf16CodeUnits[i++];
+        if (nextValue >= UNICODE_UTF16_SURROGATE_UNIT_1_BASE && 
+            nextValue <= UNICODE_UTF16_RESERVED_HI) {
+          value = (value - UNICODE_UTF16_SURROGATE_UNIT_0_BASE) << 10;
+          value += UNICODE_UTF16_OFFSET +
+              (nextValue - UNICODE_UTF16_SURROGATE_UNIT_1_BASE);
+          f(value);
+        } else {
+          if (nextValue >= UNICODE_UTF16_SURROGATE_UNIT_0_BASE &&
+             nextValue < UNICODE_UTF16_SURROGATE_UNIT_1_BASE) {
+            i--;
+          }
+          addReplacementCodepoint(f, replacementCodepoint);
+          continue;
+        }
+      } else {
+        addReplacementCodepoint(f, replacementCodepoint);
+        continue;
+      }
+    }
+  }
+  int codepointBufferLength = 0;
+  apply(void _(int value) {
+      codepointBufferLength++;
+  });
+
+  List<int> codepointBuffer = new List<int>(codepointBufferLength);
+  int i = 0;
+  apply(void _(int value) {
+      codepointBuffer[i++] = value;
+  });
+  return codepointBuffer;
+}