[dart:developer][VM/Service] Add APIs to dart:developer for recording HTTP profiling information, and for later retrieving that information

The APIs mentioned above are `addHttpClientProfilingData` and
`getHttpClientProfilingData`.

This CL also makes it so that profiling information recorded using
`addHttpClientProfilingData` is included in dart:io service extension
responses.

TEST= pkg/vm_service/test/get_http_profile_test.dart

Change-Id: I892a7a8485369bb92cbb0c086b93498bcec25d5f
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/341440
Reviewed-by: Sigmund Cherem <sigmund@google.com>
Commit-Queue: Derek Xu <derekx@google.com>
Reviewed-by: Ömer Ağacan <omersa@google.com>
Reviewed-by: Siva Annamalai <asiva@google.com>
Reviewed-by: Leaf Petersen <leafp@google.com>
Reviewed-by: Ben Konyi <bkonyi@google.com>

pkg/vm_service/test/get_http_profile_test.dart

@@ -242,8 +242,8 @@ Future<void> hasValidHttpRequests(HttpProfile profile, String method) async {
           if (method == 'POST') {
             // add() was used
             expect(
-              <int>[0, 1, 2],
               fullRequest.requestBody!,
+              <int>[0, 1, 2],
             );
           } else {
             // write() was used.

sdk/lib/_http/http_impl.dart

@@ -24,19 +24,16 @@ abstract final class HttpProfiler {
 
   static void clear() => _profile.clear();
 
-  static String toJson(int? updatedSince) {
-    return json.encode({
-      'type': _kType,
-      'timestamp': DateTime.now().microsecondsSinceEpoch,
-      'requests': [
-        for (final request in _profile.values.where(
-          (e) {
-            return (updatedSince == null) || e.lastUpdateTime >= updatedSince;
-          },
-        ))
-          request.toJson(),
-      ],
-    });
+  /// Returns a list of Maps, where each map conforms to the @HttpProfileRequest
+  /// type defined in the dart:io service extension spec.
+  static List<Map<String, dynamic>> serializeHttpProfileRequests(
+      int? updatedSince) {
+    return _profile.values
+        .where(
+          (e) => (updatedSince == null) || e.lastUpdateTime >= updatedSince,
+        )
+        .map((e) => e.toJson(ref: true))
+        .toList();
   }
 }
 
@@ -223,7 +220,7 @@ class _HttpProfileData {
     _updated();
   }
 
-  Map<String, dynamic> toJson({bool ref = true}) {
+  Map<String, dynamic> toJson({required bool ref}) {
     return <String, dynamic>{
       'type': '${ref ? '@' : ''}HttpProfileRequest',
       'id': id,

sdk/lib/developer/developer.dart

@@ -80,6 +80,7 @@ import 'dart:convert';
 import 'dart:isolate' show Isolate, RawReceivePort, SendPort;
 
 part 'extension.dart';
+part 'http_profiling.dart';
 part 'profiler.dart';
 part 'service.dart';
 part 'timeline.dart';

sdk/lib/developer/http_profiling.dart

@@ -0,0 +1,28 @@
+// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+part of dart.developer;
+
+// package:http_profile adds HTTP profiling information to this list by using
+// the [addHttpClientProfilingData] API below.
+List<Map<String, dynamic>> _developerProfilingData = <Map<String, dynamic>>[];
+
+/// Records the data associated with an HTTP request for profiling purposes.
+///
+/// This function should never be called directly. Instead, use
+/// [package:http_profile](https://pub.dev/packages/http_profile).
+@Since('3.4')
+void addHttpClientProfilingData(Map<String, dynamic> requestProfile) {
+  _developerProfilingData.add(requestProfile);
+  requestProfile['id'] = 'from_package/${_developerProfilingData.length}';
+}
+
+/// Returns the data added through [addHttpClientProfilingData].
+///
+/// This function is only meant for use by networking profilers and the format
+/// of the returned data may change over time.
+@Since('3.4')
+List<Map<String, dynamic>> getHttpClientProfilingData() {
+  return UnmodifiableListView(_developerProfilingData);
+}

sdk/lib/io/network_profiling.dart

@@ -11,6 +11,39 @@ const int _versionMinor = 0;
 const String _tcpSocket = 'tcp';
 const String _udpSocket = 'udp';
 
+// Creates a Map conforming to the HttpProfileRequest type defined in the
+// dart:io service extension spec from an element of dart:developer's
+// [_developerProfilingData].
+Future<Map<String, dynamic>> _createHttpProfileRequestFromProfileMap(
+    Map<String, dynamic> requestProfile,
+    {required bool ref}) async {
+  final responseData = requestProfile['responseData'] as Map<String, dynamic>;
+
+  return <String, dynamic>{
+    'type': '${ref ? '@' : ''}HttpProfileRequest',
+    'id': requestProfile['id']!,
+    'isolateId': requestProfile['isolateId']!,
+    'method': requestProfile['requestMethod']!,
+    'uri': requestProfile['requestUri']!,
+    'events': requestProfile['events']!,
+    'startTime': requestProfile['requestStartTimestamp']!,
+    if (requestProfile['requestEndTimestamp'] != null)
+      'endTime': requestProfile['requestEndTimestamp'],
+    'request': requestProfile['requestData']!,
+    'response': responseData,
+    if (!ref && requestProfile['requestEndTimestamp'] != null)
+      'requestBody':
+          await (requestProfile['_requestBodyStream'] as Stream<List<int>>)
+              .expand((List<int> i) => i)
+              .toList(),
+    if (!ref && responseData['endTime'] != null)
+      'responseBody':
+          await (requestProfile['_responseBodyStream'] as Stream<List<int>>)
+              .expand((List<int> i) => i)
+              .toList(),
+  };
+}
+
 @pragma('vm:entry-point', !const bool.fromEnvironment("dart.vm.product"))
 abstract class _NetworkProfiling {
   // Http relative RPCs
@@ -43,7 +76,7 @@ abstract class _NetworkProfiling {
   }
 
   static Future<ServiceExtensionResponse> _serviceExtensionHandler(
-      String method, Map<String, String> parameters) {
+      String method, Map<String, String> parameters) async {
     try {
       String responseJson;
       switch (method) {
@@ -54,14 +87,25 @@ abstract class _NetworkProfiling {
           responseJson = _getHttpEnableTimelineLogging();
           break;
         case _kGetHttpProfileRPC:
-          responseJson = HttpProfiler.toJson(
-            parameters.containsKey('updatedSince')
-                ? int.tryParse(parameters['updatedSince']!)
-                : null,
-          );
+          final updatedSince = parameters.containsKey('updatedSince')
+              ? int.tryParse(parameters['updatedSince']!)
+              : null;
+          responseJson = json.encode({
+            'type': 'HttpProfile',
+            'timestamp': DateTime.now().microsecondsSinceEpoch,
+            'requests': [
+              ...HttpProfiler.serializeHttpProfileRequests(updatedSince),
+              ...await Future.wait(getHttpClientProfilingData()
+                  .where((final Map<String, dynamic> p) =>
+                      updatedSince == null ||
+                      (p['_lastUpdateTime'] as int) >= updatedSince)
+                  .map((p) =>
+                      _createHttpProfileRequestFromProfileMap(p, ref: true)))
+            ],
+          });
           break;
         case _kGetHttpProfileRequestRPC:
-          responseJson = _getHttpProfileRequest(parameters);
+          responseJson = await _getHttpProfileRequest(parameters);
           break;
         case _kClearHttpProfileRPC:
           HttpProfiler.clear();
@@ -80,22 +124,16 @@ abstract class _NetworkProfiling {
           responseJson = getVersion();
           break;
         default:
-          return Future.value(
-            ServiceExtensionResponse.error(
-              ServiceExtensionResponse.extensionError,
-              'Method $method does not exist',
-            ),
+          return ServiceExtensionResponse.error(
+            ServiceExtensionResponse.extensionError,
+            'Method $method does not exist',
           );
       }
-      return Future.value(
-        ServiceExtensionResponse.result(responseJson),
-      );
+      return ServiceExtensionResponse.result(responseJson);
     } catch (errorMessage) {
-      return Future.value(
-        ServiceExtensionResponse.error(
-          ServiceExtensionResponse.invalidParams,
-          errorMessage.toString(),
-        ),
+      return ServiceExtensionResponse.error(
+        ServiceExtensionResponse.invalidParams,
+        errorMessage.toString(),
       );
     }
   }
@@ -134,19 +172,26 @@ String _setHttpEnableTimelineLogging(Map<String, String> parameters) {
   return _success();
 }
 
-String _getHttpProfileRequest(Map<String, String> parameters) {
+Future<String> _getHttpProfileRequest(Map<String, String> parameters) async {
   if (!parameters.containsKey('id')) {
     throw _missingArgument('id');
   }
-  String id = parameters['id']!;
+  final id = parameters['id']!;
+  final request;
+  if (id.startsWith('from_package/')) {
+    final profileMap = getHttpClientProfilingData()
+        .elementAtOrNull(int.parse(id.substring('from_package/'.length)) - 1);
+    request = profileMap == null
+        ? null
+        : await _createHttpProfileRequestFromProfileMap(profileMap, ref: false);
+  } else {
+    request = HttpProfiler.getHttpProfileRequest(id)?.toJson(ref: false);
+  }
 
-  final request = HttpProfiler.getHttpProfileRequest(id);
   if (request == null) {
     throw "Unable to find request with id: '$id'";
   }
-  return json.encode(
-    request.toJson(ref: false),
-  );
+  return json.encode(request);
 }
 
 String _socketProfilingEnabled(Map<String, String> parameters) {