// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. #library("unicode_core"); /* * Test for presence of bug related to the use of UTF-16 code units for * Dart compiled to JS. */ bool _test16BitCodeUnit = null; // TODO is16BitCodeUnit() is used to work around a bug with frog/dartc // (http://code.google.com/p/dart/issues/detail?id=1357). Consider // removing after this issue is resolved. bool is16BitCodeUnit() { if (_test16BitCodeUnit == null) { _test16BitCodeUnit = (new String.fromCharCodes([0x1D11E])) == (new String.fromCharCodes([0xD11E])); } return _test16BitCodeUnit; } /** * Invalid codepoints or encodings may be substituted with the value U+fffd. */ final int UNICODE_REPLACEMENT_CHARACTER_CODEPOINT = 0xfffd; final int UNICODE_BOM = 0xfeff; final int UNICODE_UTF_BOM_LO = 0xff; final int UNICODE_UTF_BOM_HI = 0xfe; final int UNICODE_BYTE_ZERO_MASK = 0xff; final int UNICODE_BYTE_ONE_MASK = 0xff00; final int UNICODE_VALID_RANGE_MAX = 0x10ffff; final int UNICODE_PLANE_ONE_MAX = 0xffff; final int UNICODE_UTF16_RESERVED_LO = 0xd800; final int UNICODE_UTF16_RESERVED_HI = 0xdfff; final int UNICODE_UTF16_OFFSET = 0x10000; final int UNICODE_UTF16_SURROGATE_UNIT_0_BASE = 0xd800; final int UNICODE_UTF16_SURROGATE_UNIT_1_BASE = 0xdc00; final int UNICODE_UTF16_HI_MASK = 0xffc00; final int UNICODE_UTF16_LO_MASK = 0x3ff; /** * Encode code points as UTF16 code units. */ List codepointsToUtf16CodeUnits( List codepoints, [int offset = 0, int length, int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { if (!(offset >= 0)) { throw new IllegalArgumentException("offset"); } if (!(length == null || length >= 0)) { throw new IllegalArgumentException("length"); } int end = length != null ? Math.min(codepoints.length, offset + length) : codepoints.length; int encodedLength = 0; for (int i = offset; i < end; i++) { int value = codepoints[i]; if ((value >= 0 && value < UNICODE_UTF16_RESERVED_LO) || (value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) { encodedLength++; } else if (value > UNICODE_PLANE_ONE_MAX && value <= UNICODE_VALID_RANGE_MAX) { encodedLength += 2; } else { encodedLength++; } } void addReplacementCodepoint(List codepointBuffer, int offset, int replacementCodepoint) { if (replacementCodepoint != null) { codepointBuffer[offset] = replacementCodepoint; } else { throw new IllegalArgumentException("Invalid encoding"); } } List codeUnitsBuffer = new List(encodedLength); int j = 0; for (int i = offset; i < end; i++) { int value = codepoints[i]; if ((value >= 0 && value < UNICODE_UTF16_RESERVED_LO) || (value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) { codeUnitsBuffer[j++] = value; } else if (value > UNICODE_PLANE_ONE_MAX && value <= UNICODE_VALID_RANGE_MAX) { int base = value - UNICODE_UTF16_OFFSET; codeUnitsBuffer[j++] = UNICODE_UTF16_SURROGATE_UNIT_0_BASE + ((base & UNICODE_UTF16_HI_MASK) >> 10); codeUnitsBuffer[j++] = UNICODE_UTF16_SURROGATE_UNIT_1_BASE + (base & UNICODE_UTF16_LO_MASK); } else { addReplacementCodepoint(codeUnitsBuffer, j++, replacementCodepoint); } } return codeUnitsBuffer; } /** * Decodes the utf16 codeunits to codepoints. */ List utf16CodeUnitsToCodepoints( List utf16CodeUnits, [int offset = 0, int length, int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { if (!(offset >= 0)) { throw new IllegalArgumentException("offset"); } if (!(length == null || length >= 0)) { throw new IllegalArgumentException("length"); } int end = length != null ? Math.min(utf16CodeUnits.length, offset + length) : utf16CodeUnits.length; void decode(void f(int v)) { int i = offset; // skip the first entry if it is a BOM. if (end > 0 && utf16CodeUnits[0] == UNICODE_BOM) { i++; } while (i < end) { int value = utf16CodeUnits[i++]; if (value < 0) { f(null); continue; } if (value < UNICODE_UTF16_RESERVED_LO || (value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) { // transfer directly f(value); } else if (value < UNICODE_UTF16_SURROGATE_UNIT_1_BASE && i < end) { // merge surrogate pair int nextValue = utf16CodeUnits[i++]; if (nextValue >= UNICODE_UTF16_SURROGATE_UNIT_1_BASE && nextValue <= UNICODE_UTF16_RESERVED_HI) { value = (value - UNICODE_UTF16_SURROGATE_UNIT_0_BASE) << 10; value += UNICODE_UTF16_OFFSET + (nextValue - UNICODE_UTF16_SURROGATE_UNIT_1_BASE); f(value); } else { if (nextValue >= UNICODE_UTF16_SURROGATE_UNIT_0_BASE && nextValue < UNICODE_UTF16_SURROGATE_UNIT_1_BASE) { i--; } f(null); continue; } } else { f(null); continue; } } } // First pass through data to 1) size the output buffer and 2) check for // special case optimization where A) the length stays the same and B) // no special replacement characters are used. If these criteria are met // we can just copy input to the output. int codepointBufferLength = 0; bool hasReplacements = false; decode(void _(int value) { codepointBufferLength++; if (value == null) { hasReplacements = true; } }); // If the string calls for replacements, but when the method is called // with replacementCodepoint explicitly set to null, then throw an exception. if (hasReplacements && replacementCodepoint == null) { throw new IllegalArgumentException("Invalid encoding"); } int _length = end - offset; List codepointBuffer = new List(codepointBufferLength); if (_length == codepointBufferLength && !hasReplacements) { codepointBuffer.setRange(0, _length, utf16CodeUnits, offset); } else { int i = 0; decode( void _(int value) { if (value != null) { codepointBuffer[i++] = value; } else { codepointBuffer[i++] = replacementCodepoint; } } ); } return codepointBuffer; }