Fix fused UTF-8/JSON decoding.

The VM version's parser did not allow a leading BOM, and it failed to parse a top-level integer.

Added test to check this.

Fixes #33251

Bug: http://dartbug.com/33251
Change-Id: I51e429082f0e9baac81e20f73b0885922b40b0b8
Reviewed-on: https://dart-review.googlesource.com/56860
Commit-Queue: Lasse R.H. Nielsen <lrn@google.com>
Reviewed-by: Florian Loitsch <floitsch@google.com>

pkg/expect/lib/expect.dart

@@ -515,33 +515,35 @@ class Expect {
   }
 
   /**
-   * Calls the function [f] and verifies that it throws an exception.
+   * Calls the function [f] and verifies that it throws a `T`.
    * The optional [check] function can provide additional validation
-   * that the correct exception is being thrown.  For example, to check
-   * the type of the exception you could write this:
+   * that the correct object is being thrown.  For example, to check
+   * the content of the thrown boject you could write this:
    *
-   *     Expect.throws(myThrowingFunction, (e) => e is MyException);
+   *     Expect.throws<MyException>(myThrowingFunction,
+   *          (e) => e.myMessage.contains("WARNING"));
+   *
+   * The type variable can be omitted and the type checked in [check]
+   * instead. This was traditionally done before Dart had generic methods.
    *
    * If `f` fails an expectation (i.e., throws an [ExpectException]), that
    * exception is not caught by [Expect.throws]. The test is still considered
    * failing.
    */
-  static void throws(void f(), [bool check(Object error), String reason]) {
+  static void throws<T>(void f(), [bool check(T error), String reason]) {
     String msg = reason == null ? "" : "($reason)";
-    if (f is! _Nullary) {
+    if (f is! Function()) {
       // Only throws from executing the function body should count as throwing.
       // The failure to even call `f` should throw outside the try/catch.
       _fail("Expect.throws$msg: Function f not callable with zero arguments");
     }
     try {
       f();
-    } catch (e, s) {
+    } on Object catch (e, s) {
       // A test failure doesn't count as throwing.
       if (e is ExpectException) rethrow;
-      if (check != null && !check(e)) {
-        _fail("Expect.throws$msg: Unexpected '$e'\n$s");
-      }
-      return;
+      if (e is T && (check == null || check(e))) return;
+      _fail("Expect.throws$msg: Unexpected '$e'\n$s");
     }
     _fail('Expect.throws$msg fails: Did not throw');
   }
@@ -607,8 +609,9 @@ class Expect {
 /// Used in [Expect] because [Expect.identical] shadows the real [identical].
 bool _identical(a, b) => identical(a, b);
 
-typedef _Nullary(); // Expect.throws argument must be this type.
-
+/// Exception thrown on a failed expectation check.
+///
+/// Always recognized by [Expect.throws] as an unexpected error.
 class ExpectException implements Exception {
   final String message;
   ExpectException(this.message);

runtime/lib/convert_patch.dart

@@ -63,6 +63,7 @@ class _JsonUtf8Decoder extends Converter<List<int>, Object> {
     parser.chunk = input;
     parser.chunkEnd = input.length;
     parser.parse(0);
+    parser.close();
     return parser.result;
   }
 
@@ -398,6 +399,7 @@ abstract class _ChunkedJsonParser<T> {
   static const int KWD_NULL = 0; // Prefix of "null" seen.
   static const int KWD_TRUE = 4; // Prefix of "true" seen.
   static const int KWD_FALSE = 8; // Prefix of "false" seen.
+  static const int KWD_BOM = 12; // Prefix of BOM seen.
   static const int KWD_COUNT_SHIFT = 4; // Prefix length in bits 4+.
 
   // Mask used to mask off two lower bits.
@@ -439,8 +441,11 @@ abstract class _ChunkedJsonParser<T> {
    *      ..0ddd0011 : Partial 'null' keyword.
    *      ..0ddd0111 : Partial 'true' keyword.
    *      ..0ddd1011 : Partial 'false' keyword.
-   *                   For all three keywords, the `ddd` bits encode the number
+   *      ..0ddd1111 : Partial UTF-8 BOM byte seqeuence ("\xEF\xBB\xBF").
+   *                   For all keywords, the `ddd` bits encode the number
    *                   of letters seen.
+   *                   The BOM byte sequence is only used by [_JsonUtf8Parser],
+   *                   and only at the very beginning of input.
    */
   int partialState = NO_PARTIAL;
 
@@ -789,7 +794,8 @@ abstract class _ChunkedJsonParser<T> {
     int keywordType = partialState & KWD_TYPE_MASK;
     int count = partialState >> KWD_COUNT_SHIFT;
     int keywordTypeIndex = keywordType >> KWD_TYPE_SHIFT;
-    String keyword = const ["null", "true", "false"][keywordTypeIndex];
+    String keyword =
+        const ["null", "true", "false", "\xEF\xBB\xBF"][keywordTypeIndex];
     assert(count < keyword.length);
     do {
       if (position == chunkEnd) {
@@ -798,13 +804,19 @@ abstract class _ChunkedJsonParser<T> {
         return chunkEnd;
       }
       int expectedChar = keyword.codeUnitAt(count);
-      if (getChar(position) != expectedChar) return fail(position);
+      if (getChar(position) != expectedChar) {
+        if (count == 0) {
+          assert(keywordType == KWD_BOM);
+          return position;
+        }
+        return fail(position);
+      }
       position++;
       count++;
     } while (count < keyword.length);
     if (keywordType == KWD_NULL) {
       listener.handleNull();
-    } else {
+    } else if (keywordType != KWD_BOM) {
       listener.handleBool(keywordType == KWD_TRUE);
     }
     return position;
@@ -1255,7 +1267,7 @@ abstract class _ChunkedJsonParser<T> {
         fail(position, "Missing expected digit");
       } else {
         // If it doesn't even start out as a numeral.
-        fail(position, "Unexpected character");
+        fail(position);
       }
     }
     if (digit == 0) {
@@ -1660,7 +1672,7 @@ class _Utf8StringBuffer {
           position++;
         } else {
           throw new FormatException(
-              "Unexepected UTF-8 continuation byte", utf8, position);
+              "Unexpected UTF-8 continuation byte", utf8, position);
         }
       } else if (char < 0xE0) {
         // C0-DF
@@ -1726,7 +1738,11 @@ class _JsonUtf8Parser extends _ChunkedJsonParser<List<int>> {
   int chunkEnd;
 
   _JsonUtf8Parser(_JsonListener listener, this.allowMalformed)
-      : super(listener);
+      : super(listener) {
+    // Starts out checking for an optional BOM (KWD_BOM, count = 0).
+    partialState =
+        _ChunkedJsonParser.PARTIAL_KEYWORD | _ChunkedJsonParser.KWD_BOM;
+  }
 
   int getChar(int position) => chunk[position];
 

tests/lib_2/convert/json_test.dart

@@ -104,10 +104,16 @@ String escape(String s) {
 }
 
 void testThrows(jsonText) {
+  var message = "json = '${escape(jsonText)}'";
   Expect.throwsFormatException(() => json.decode(jsonText),
-      "json = '${escape(jsonText)}'");
+      "json.decode, $message");
   Expect.throwsFormatException(() => jsonDecode(jsonText),
-      "json = '${escape(jsonText)}'");
+      "jsonDecode, $message");
+  Expect.throwsFormatException(() => json.decoder.convert(jsonText),
+      "json.decoder.convert, $message");
+  Expect.throwsFormatException(() =>
+      utf8.decoder.fuse(json.decoder).convert(utf8.encode(jsonText)),
+      "utf8.decoder.fuse(json.decoder) o utf.encode, $message");
 }
 
 testNumbers() {
@@ -187,6 +193,10 @@ testNumbers() {
   testThrows("-2.2 e+2");
   testThrows("-2.2e +2");
   testThrows("-2.2e+ 2");
+  testThrows("01");
+  testThrows("0.");
+  testThrows(".0");
+  testThrows("0.e1");
 
   testThrows("[2.,2]");
   testThrows("{2.:2}");

tests/lib_2/convert/json_utf8_test.dart

@@ -0,0 +1,77 @@
+// Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// Test that the fused UTF-8/JSON decoder accepts a leading BOM.
+library test;
+
+import "package:expect/expect.dart";
+import "dart:convert";
+
+void main() {
+  for (var parse in [parseFuse, parseSequence, parseChunked]) {
+    // Sanity checks.
+    Expect.isTrue(parse('true'.codeUnits.toList()));
+    Expect.isFalse(parse('false'.codeUnits.toList()));
+    Expect.equals(123, parse('123'.codeUnits.toList()));
+    Expect.listEquals([42], parse('[42]'.codeUnits.toList()));
+    Expect.mapEquals({"x": 42}, parse('{"x":42}'.codeUnits.toList()));
+
+    // String (0x22 = ") with UTF-8 encoded Unicode characters.
+    Expect.equals(
+        "A\xff\u1234\u{65555}",
+        parse([
+          0x22,
+          0x41,
+          0xc3,
+          0xbf,
+          0xe1,
+          0x88,
+          0xb4,
+          0xf1,
+          0xa5,
+          0x95,
+          0x95,
+          0x22
+        ]));
+
+    // BOM followed by true.
+    Expect.isTrue(parse([0xEF, 0xBB, 0xBF, 0x74, 0x72, 0x75, 0x65]));
+  }
+
+  // Do not accept BOM in non-UTF-8 decoder.
+  Expect.throws<FormatException>(
+      () => new JsonDecoder().convert("\xEF\xBB\xBFtrue"));
+  Expect.throws<FormatException>(() => new JsonDecoder().convert("\uFEFFtrue"));
+
+  // Only accept BOM first.
+  Expect.throws<FormatException>(
+      () => parseFuse(" \xEF\xBB\xBFtrue".codeUnits.toList()));
+  // Only accept BOM first.
+  Expect.throws<FormatException>(
+      () => parseFuse(" true\xEF\xBB\xBF".codeUnits.toList()));
+
+  Expect.throws<FormatException>(
+      () => parseFuse(" [\xEF\xBB\xBF]".codeUnits.toList()));
+}
+
+Object parseFuse(List<int> text) {
+  return new Utf8Decoder().fuse(new JsonDecoder()).convert(text);
+}
+
+Object parseSequence(List<int> text) {
+  return new JsonDecoder().convert(new Utf8Decoder().convert(text));
+}
+
+Object parseChunked(List<int> text) {
+  var result;
+  var sink = new Utf8Decoder().fuse(new JsonDecoder()).startChunkedConversion(
+      new ChunkedConversionSink.withCallback((List<Object> values) {
+    result = values[0];
+  }));
+  for (var i = 0; i < text.length; i++) {
+    sink.add([text[i]]);
+  }
+  sink.close();
+  return result;
+}