// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. #library("unicode_core"); /* * Test for presence of bug related to the use of UTF-16 code units for * Dart compiled to JS. */ bool _test16BitCodeUnit = null; // TODO is16BitCodeUnit() is used to work around a bug with frog/dartc // (http://code.google.com/p/dart/issues/detail?id=1357). Consider // removing after this issue is resolved. bool is16BitCodeUnit() { if (_test16BitCodeUnit == null) { _test16BitCodeUnit = (new String.fromCharCodes([0x1D11E])) == (new String.fromCharCodes([0xD11E])); } return _test16BitCodeUnit; } /** * Invalid codepoints or encodings may be substituted with the value U+fffd. */ final int UNICODE_REPLACEMENT_CHARACTER_CODEPOINT = 0xfffd; final int UNICODE_BOM = 0xfeff; final int UNICODE_UTF_BOM_LO = 0xff; final int UNICODE_UTF_BOM_HI = 0xfe; final int UNICODE_BYTE_ZERO_MASK = 0xff; final int UNICODE_BYTE_ONE_MASK = 0xff00; final int UNICODE_VALID_RANGE_MAX = 0x10ffff; final int UNICODE_PLANE_ONE_MAX = 0xffff; final int UNICODE_UTF16_RESERVED_LO = 0xd800; final int UNICODE_UTF16_RESERVED_HI = 0xdfff; final int UNICODE_UTF16_OFFSET = 0x10000; final int UNICODE_UTF16_SURROGATE_UNIT_0_BASE = 0xd800; final int UNICODE_UTF16_SURROGATE_UNIT_1_BASE = 0xdc00; final int UNICODE_UTF16_HI_MASK = 0xffc00; final int UNICODE_UTF16_LO_MASK = 0x3ff; /** * Encode code points as UTF16 code units. */ List codepointsToUtf16CodeUnits( List codepoints, [int offset = 0, int length, int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { if (!(offset >= 0)) { throw new IllegalArgumentException("offset"); } if (!(length == null || length >= 0)) { throw new IllegalArgumentException("length"); } int end = length != null ? Math.min(codepoints.length, offset + length) : codepoints.length; int encodedLength = 0; for (int i = offset; i < end; i++) { int value = codepoints[i]; if ((value >= 0 && value < UNICODE_UTF16_RESERVED_LO) || (value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) { encodedLength++; } else if (value > UNICODE_PLANE_ONE_MAX && value <= UNICODE_VALID_RANGE_MAX) { encodedLength += 2; } else { encodedLength++; } } void addReplacementCodepoint(List codepointBuffer, int offset, int replacementCodepoint) { if(replacementCodepoint != null) { codepointBuffer[offset] = replacementCodepoint; } else { throw new IllegalArgumentException("Invalid encoding"); } } List codeUnitsBuffer = new List(encodedLength); int j = 0; for (int i = offset; i < end; i++) { int value = codepoints[i]; if ((value >= 0 && value < UNICODE_UTF16_RESERVED_LO) || (value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) { codeUnitsBuffer[j++] = value; } else if (value > UNICODE_PLANE_ONE_MAX && value <= UNICODE_VALID_RANGE_MAX) { int base = value - UNICODE_UTF16_OFFSET; codeUnitsBuffer[j++] = UNICODE_UTF16_SURROGATE_UNIT_0_BASE + ((base & UNICODE_UTF16_HI_MASK) >> 10); codeUnitsBuffer[j++] = UNICODE_UTF16_SURROGATE_UNIT_1_BASE + (base & UNICODE_UTF16_LO_MASK); } else { addReplacementCodepoint(codeUnitsBuffer, j++, replacementCodepoint); } } return codeUnitsBuffer; } /** * Decodes the utf16 codeunits to codepoints. */ List utf16CodeUnitsToCodepoints( List utf16CodeUnits, [int offset = 0, int length, int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) { if (!(offset >= 0)) { throw new IllegalArgumentException("offset"); } if (!(length == null || length >= 0)) { throw new IllegalArgumentException("length"); } int end = length != null ? Math.min(utf16CodeUnits.length, offset + length) : utf16CodeUnits.length; void addReplacementCodepoint(void f(int v), int replacementCodepoint) { if(replacementCodepoint != null) { f(replacementCodepoint); } else { throw new IllegalArgumentException("Invalid encoding"); } } void apply(void f(int v)) { int i = offset; // skip the first entry if it is a BOM. if (end > 0 && utf16CodeUnits[0] == UNICODE_BOM) { i++; } while (i < end) { int value = utf16CodeUnits[i++]; if (value < 0) { addReplacementCodepoint(f, replacementCodepoint); continue; } if (value < UNICODE_UTF16_RESERVED_LO || (value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) { // transfer directly f(value); } else if (value < UNICODE_UTF16_SURROGATE_UNIT_1_BASE && i < end) { // merge surrogate pair int nextValue = utf16CodeUnits[i++]; if (nextValue >= UNICODE_UTF16_SURROGATE_UNIT_1_BASE && nextValue <= UNICODE_UTF16_RESERVED_HI) { value = (value - UNICODE_UTF16_SURROGATE_UNIT_0_BASE) << 10; value += UNICODE_UTF16_OFFSET + (nextValue - UNICODE_UTF16_SURROGATE_UNIT_1_BASE); f(value); } else { if (nextValue >= UNICODE_UTF16_SURROGATE_UNIT_0_BASE && nextValue < UNICODE_UTF16_SURROGATE_UNIT_1_BASE) { i--; } addReplacementCodepoint(f, replacementCodepoint); continue; } } else { addReplacementCodepoint(f, replacementCodepoint); continue; } } } int codepointBufferLength = 0; apply(void _(int value) { codepointBufferLength++; }); List codepointBuffer = new List(codepointBufferLength); int i = 0; apply(void _(int value) { codepointBuffer[i++] = value; }); return codepointBuffer; }