[vm] Fix data race during TypeRef canonicalization

TEST=vm/dart/regress_50065_test
TEST=tools/test.py --repeat 100 -n dartk-tsan-linux-release-x64 vm/dart_2/regress_50065_test

Fixes https://github.com/dart-lang/sdk/issues/50065

Change-Id: I974d8c9c0a291d64318f3a96ba73148707fd4b68
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/261740
Reviewed-by: Ryan Macnak <rmacnak@google.com>
Commit-Queue: Alexander Markov <alexmarkov@google.com>

runtime/tests/vm/dart/regress_50065_test.dart

@@ -0,0 +1,31 @@
+// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// Regression test for https://github.com/dart-lang/sdk/issues/50065.
+// Runs tests/language/generic/function_bounds_test.dart in multiple isolates
+// in order to stress-test concurrent canonicalization of types.
+
+import 'dart:isolate';
+
+import 'package:expect/expect.dart';
+import '../../../../tests/language/generic/function_bounds_test.dart'
+    as function_bounds_test;
+
+const int N = 100;
+
+void run(dynamic message) {
+  function_bounds_test.main();
+}
+
+void main() async {
+  final List<ReceivePort> ports = [];
+  final List<Future> isolates = [];
+  for (int i = 0; i < N; ++i) {
+    final rp = ReceivePort();
+    isolates.add(Isolate.spawn(run, null, onExit: rp.sendPort));
+    ports.add(rp);
+  }
+  await Future.wait(isolates);
+  await Future.wait(ports.map((p) => p.first));
+}

runtime/tests/vm/dart_2/regress_50065_test.dart

@@ -0,0 +1,33 @@
+// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// Regression test for https://github.com/dart-lang/sdk/issues/50065.
+// Runs tests/language_2/generic/function_bounds_test.dart in multiple isolates
+// in order to stress-test concurrent canonicalization of types.
+
+// @dart = 2.9
+
+import 'dart:isolate';
+
+import 'package:expect/expect.dart';
+import '../../../../tests/language_2/generic/function_bounds_test.dart'
+    as function_bounds_test;
+
+const int N = 100;
+
+void run(dynamic message) {
+  function_bounds_test.main();
+}
+
+void main() async {
+  final List<ReceivePort> ports = [];
+  final List<Future> isolates = [];
+  for (int i = 0; i < N; ++i) {
+    final rp = ReceivePort();
+    isolates.add(Isolate.spawn(run, null, onExit: rp.sendPort));
+    ports.add(rp);
+  }
+  await Future.wait(isolates);
+  await Future.wait(ports.map((p) => p.first));
+}

runtime/vm/object.cc

@@ -22047,7 +22047,11 @@ AbstractTypePtr TypeRef::Canonicalize(Thread* thread, TrailPtr trail) const {
   AbstractType& ref_type = AbstractType::Handle(type());
   ASSERT(!ref_type.IsNull());
   ref_type = ref_type.Canonicalize(thread, trail);
-  set_type(ref_type);
+  {
+    SafepointMutexLocker ml(
+        thread->isolate_group()->type_canonicalization_mutex());
+    set_type(ref_type);
+  }
   return ptr();
 }