[vm,dyn_modules] Fix dynamic calls from host app to a dynamic module

Fix the following cases of incorrect optimizations:

* In TFA, when anlyzing a dynamic call with unknown receiver type,
  do not assume that all possible targets can be computed at compile
  time (if there can be dynamically loaded classes).

* In the AOT, disable optimizations for dynamic calls with unique
  selectors (if there can be dynamically loaded classes).

TEST=pkg/dynamic_modules/test/data/dyn_call_from_host

Change-Id: I39d620aae3c116de03d4a2a3fd61864d88c48c8e
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/493960
Commit-Queue: Alexander Markov <alexmarkov@google.com>
Reviewed-by: Sigmund Cherem <sigmund@google.com>
Reviewed-by: Slava Egorov <vegorov@google.com>

pkg/dynamic_modules/test/data/dyn_call_from_host/dynamic_interface.yaml

@@ -0,0 +1,6 @@
+# Copyright (c) 2026, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+callable:
+  - library: 'dart:core'

pkg/dynamic_modules/test/data/dyn_call_from_host/main.dart

@@ -0,0 +1,34 @@
+// Copyright (c) 2026, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import '../../common/testing.dart' as helper;
+import 'package:expect/expect.dart';
+
+class C1 {
+  int method1() => 1;
+  int method2() => 2;
+}
+
+class C2 {
+  int method3() => 3;
+  int method4() => 4;
+}
+
+final List escape = [C1(), C2()];
+
+// Dynamic calls from the host app.
+void main() async {
+  print(escape);
+
+  final list = (await helper.load('entry1.dart')) as List;
+  dynamic o1 = list[0];
+  Expect.equals('10', o1.method1());
+  Expect.equals('20', o1.method2());
+
+  dynamic o2 = list[1];
+  Expect.equals('30', o2.method3());
+  Expect.equals('40', o2.method4());
+  Expect.equals('50', o2.method5());
+  helper.done();
+}

pkg/dynamic_modules/test/data/dyn_call_from_host/modules/entry1.dart

@@ -0,0 +1,19 @@
+// Copyright (c) 2026, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+class D1 {
+  String method1() => '10';
+  String method2() => '20';
+}
+
+class D2 {
+  String method3() => '30';
+  String method4() => '40';
+  String method5() => '50';
+}
+
+@pragma('dyn-module:entry-point')
+Object? dynamicModuleEntrypoint() {
+  return [D1(), D2()];
+}

pkg/vm/lib/transformations/type_flow/analysis.dart

@@ -701,6 +701,7 @@ final class _DispatchableInvocation extends _Invocation {
 
     assert(targets.isEmpty);
 
+    bool unknownTargets = false;
     if (receiver is ConcreteType) {
       _collectTargetsForConcreteType(receiver, targets, typeFlowAnalysis);
     } else if (receiver is SetType) {
@@ -709,6 +710,12 @@ final class _DispatchableInvocation extends _Invocation {
       }
     } else if (receiver is AnyInstanceType) {
       _collectTargetsForSelector(targets, typeFlowAnalysis);
+      // Any class from a dynamic module may have unknown target for the
+      // dynamic call with AnyInstanceType receiver.
+      unknownTargets = typeFlowAnalysis
+          .hierarchyCache
+          ._objectTFClass
+          .hasDynamicallyExtendableSubtypes;
     } else {
       assert(receiver is EmptyType);
     }
@@ -725,7 +732,7 @@ final class _DispatchableInvocation extends _Invocation {
       );
     }
 
-    return true;
+    return !unknownTargets;
   }
 
   void _collectTargetsForNull(

runtime/vm/compiler/aot/aot_call_specializer.cc

@@ -54,6 +54,10 @@ DEFINE_FLAG(int,
 static void GetUniqueDynamicTarget(IsolateGroup* isolate_group,
                                    const String& fname,
                                    Object* function) {
+  if (isolate_group->has_dynamically_extendable_classes()) {
+    *function = Object::null();
+    return;
+  }
   UniqueFunctionsMap functions_map(
       isolate_group->object_store()->unique_dynamic_targets());
   ASSERT(fname.IsSymbol());