dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity
Files
d77e4eb4d67ca1ff4e40a8da512eb6d0ba86b6fc
sdk/runtime/vm/runtime_entry.cc
T
Tess Strickland 787446213c [vm/compiler] Move argument shape (not type) checks out of closures. 
This CL performs the following checks in the invoke field dispatcher for
dynamic closure calls when lazy dispatchers are enabled:

* The provided function type arguments vector (if any) has the correct
  length.

* No function type arguments should be provided if the closure has
  delayed type arguments.

* All required arguments (positional in all modes, named in appropriate
  null safety modes) have been provided by the caller.

* If there are optional positional arguments, an appropriate number
  has been provided.

* If there are optional named arguments, their names are valid.

Since the runtime already handles checking the argument shapes when lazy
dispatchers are disabled, these checks are now completely removed from
closure bodies in all cases. Thus, the only remaining checks in closure
bodies are the type checks performed by AssertSubtype and
AssertAssignable when lazy dispatchers are enabled.

Changes in the Flutter Gallery:

* ARM7, release: -3.61% instructions, -2.19% total
* ARM7, sizeopt: -3.62% instructions, -2.55% total
* ARM8, release: -3.66% instructions, -1.98% total
* ARM8, sizeopt: -3.65% instructions, -2.37% total

Most of these changes are already exercised by existing tests such as
(but not limited to):

* corelib{,_2}/dynamic_nosuchmethod_test
* language{,_2}/call/call_test
* language{,_2}/closure/tearoff_dynamic_test
* language{,_2}/generic/function_bounds_test
* language{,_2}/parameter/named_with_conversions_test
* language{,_2}/vm/no_such_args_error_message_vm_test

I've added one test to specifically check the interaction between
dynamic calls and required named parameters. There is some coverage in
other NNBD tests, but those are not directly focused on testing this
specifically.

Other changes:

* Adds initial cached ranges for certain BinarySmiOp and ShiftIntegerOp
  instructions when the RHS is a constant, to avoid false negatives for
  deoptimization and throw checks prior to range analysis.

* Adds new slots for various Function fields.

* Adds the ability to define unboxed native slots, which are always
  unboxed after retrieval even in unoptimized code. In the first
  iteration, the backend only handles loads from Uint32 unboxed native
  slots. Part of https://github.com/dart-lang/sdk/issues/42793.

* Removed the special handling for loading from non-nullable int fields
  in AOT compilation. Instead, their treatment is unified with the
  treatment of the new unboxed native fields, since the source field is
  always unboxed and the result of the load is also always unboxed, as
  code involving them is always optimized.

Bug: https://github.com/dart-lang/sdk/issues/40813
Change-Id: Ia02aa3e872c1fefd906fd67b55021ea1797556e4
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/155604
Commit-Queue: Tess Strickland <sstrickl@google.com>
Reviewed-by: Alexander Markov <alexmarkov@google.com>
2020-08-17 08:58:08 +00:00

3599 lines
141 KiB
C++
Raw Blame History

// Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "vm/runtime_entry.h"
#include "vm/code_descriptors.h"
#include "vm/code_patcher.h"
#include "vm/compiler/api/deopt_id.h"
#include "vm/compiler/api/type_check_mode.h"
#include "vm/compiler/jit/compiler.h"
#include "vm/dart_api_impl.h"
#include "vm/dart_api_state.h"
#include "vm/dart_entry.h"
#include "vm/debugger.h"
#include "vm/exceptions.h"
#include "vm/flags.h"
#include "vm/heap/verifier.h"
#include "vm/instructions.h"
#include "vm/interpreter.h"
#include "vm/kernel_isolate.h"
#include "vm/message.h"
#include "vm/message_handler.h"
#include "vm/object_store.h"
#include "vm/parser.h"
#include "vm/resolver.h"
#include "vm/service_isolate.h"
#include "vm/stack_frame.h"
#include "vm/symbols.h"
#include "vm/thread.h"
#include "vm/thread_registry.h"
#include "vm/type_testing_stubs.h"
#if !defined(DART_PRECOMPILED_RUNTIME)
#include "vm/deopt_instructions.h"
#endif // !defined(DART_PRECOMPILED_RUNTIME)
namespace dart {
DEFINE_FLAG(
int,
max_subtype_cache_entries,
100,
"Maximum number of subtype cache entries (number of checks cached).");
DEFINE_FLAG(
int,
regexp_optimization_counter_threshold,
1000,
"RegExp's usage-counter value before it is optimized, -1 means never");
DEFINE_FLAG(int,
reoptimization_counter_threshold,
4000,
"Counter threshold before a function gets reoptimized.");
DEFINE_FLAG(bool,
stress_write_barrier_elimination,
false,
"Stress test write barrier elimination.");
DEFINE_FLAG(bool, trace_deoptimization, false, "Trace deoptimization");
DEFINE_FLAG(bool,
trace_deoptimization_verbose,
false,
"Trace deoptimization verbose");
DECLARE_FLAG(bool, enable_interpreter);
DECLARE_FLAG(int, max_deoptimization_counter_threshold);
DECLARE_FLAG(bool, trace_compiler);
DECLARE_FLAG(bool, trace_optimizing_compiler);
DECLARE_FLAG(int, max_polymorphic_checks);
DEFINE_FLAG(bool, trace_osr, false, "Trace attempts at on-stack replacement.");
DEFINE_FLAG(int, gc_every, 0, "Run major GC on every N stack overflow checks");
DEFINE_FLAG(int,
stacktrace_every,
0,
"Compute debugger stacktrace on every N stack overflow checks");
DEFINE_FLAG(charp,
stacktrace_filter,
NULL,
"Compute stacktrace in named function on stack overflow checks");
DEFINE_FLAG(charp,
deoptimize_filter,
NULL,
"Deoptimize in named function on stack overflow checks");
DEFINE_FLAG(bool,
unopt_monomorphic_calls,
true,
"Enable specializing monomorphic calls from unoptimized code.");
DEFINE_FLAG(bool,
unopt_megamorphic_calls,
true,
"Enable specializing megamorphic calls from unoptimized code.");
DEFINE_FLAG(bool,
verbose_stack_overflow,
false,
"Print additional details about stack overflow.");
DECLARE_FLAG(int, reload_every);
DECLARE_FLAG(bool, reload_every_optimized);
DECLARE_FLAG(bool, reload_every_back_off);
#if defined(TESTING) || defined(DEBUG)
void VerifyOnTransition() {
Thread* thread = Thread::Current();
TransitionGeneratedToVM transition(thread);
VerifyPointersVisitor::VerifyPointers();
thread->isolate()->heap()->Verify();
}
#endif
// Add function to a class and that class to the class dictionary so that
// frame walking can be used.
const Function& RegisterFakeFunction(const char* name, const Code& code) {
Thread* thread = Thread::Current();
const String& class_name = String::Handle(Symbols::New(thread, "ownerClass"));
const Script& script = Script::Handle();
const Library& lib = Library::Handle(Library::CoreLibrary());
const Class& owner_class = Class::Handle(
Class::New(lib, class_name, script, TokenPosition::kNoSource));
const String& function_name = String::ZoneHandle(Symbols::New(thread, name));
const Function& function = Function::ZoneHandle(Function::New(
function_name, FunctionLayout::kRegularFunction, true, false, false,
false, false, owner_class, TokenPosition::kMinSource));
const Array& functions = Array::Handle(Array::New(1));
functions.SetAt(0, function);
owner_class.SetFunctions(functions);
lib.AddClass(owner_class);
function.AttachCode(code);
return function;
}
DEFINE_RUNTIME_ENTRY(RangeError, 2) {
const Instance& length = Instance::CheckedHandle(zone, arguments.ArgAt(0));
const Instance& index = Instance::CheckedHandle(zone, arguments.ArgAt(1));
if (!length.IsInteger()) {
// Throw: new ArgumentError.value(length, "length", "is not an integer");
const Array& args = Array::Handle(zone, Array::New(3));
args.SetAt(0, length);
args.SetAt(1, Symbols::Length());
args.SetAt(2, String::Handle(zone, String::New("is not an integer")));
Exceptions::ThrowByType(Exceptions::kArgumentValue, args);
}
if (!index.IsInteger()) {
// Throw: new ArgumentError.value(index, "index", "is not an integer");
const Array& args = Array::Handle(zone, Array::New(3));
args.SetAt(0, index);
args.SetAt(1, Symbols::Index());
args.SetAt(2, String::Handle(zone, String::New("is not an integer")));
Exceptions::ThrowByType(Exceptions::kArgumentValue, args);
}
// Throw: new RangeError.range(index, 0, length - 1, "length");
const Array& args = Array::Handle(zone, Array::New(4));
args.SetAt(0, index);
args.SetAt(1, Integer::Handle(zone, Integer::New(0)));
args.SetAt(
2, Integer::Handle(
zone, Integer::Cast(length).ArithmeticOp(
Token::kSUB, Integer::Handle(zone, Integer::New(1)))));
args.SetAt(3, Symbols::Length());
Exceptions::ThrowByType(Exceptions::kRange, args);
}
static void NullErrorHelper(Zone* zone, const String& selector) {
// If the selector is null, this must be a null check that wasn't due to a
// method invocation, so was due to the null check operator.
if (selector.IsNull()) {
const Array& args = Array::Handle(zone, Array::New(4));
args.SetAt(
3, String::Handle(
zone, String::New("Null check operator used on a null value")));
Exceptions::ThrowByType(Exceptions::kCast, args);
return;
}
InvocationMirror::Kind kind = InvocationMirror::kMethod;
if (Field::IsGetterName(selector)) {
kind = InvocationMirror::kGetter;
} else if (Field::IsSetterName(selector)) {
kind = InvocationMirror::kSetter;
}
const Smi& invocation_type = Smi::Handle(
zone,
Smi::New(InvocationMirror::EncodeType(InvocationMirror::kDynamic, kind)));
const Array& args = Array::Handle(zone, Array::New(7));
args.SetAt(0, /* instance */ Object::null_object());
args.SetAt(1, selector);
args.SetAt(2, invocation_type);
args.SetAt(3, /* func_type_args_length */ Object::smi_zero());
args.SetAt(4, /* func_type_args */ Object::null_object());
args.SetAt(5, /* func_args */ Object::null_object());
args.SetAt(6, /* func_arg_names */ Object::null_object());
Exceptions::ThrowByType(Exceptions::kNoSuchMethod, args);
}
DEFINE_RUNTIME_ENTRY(NullError, 0) {
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
const StackFrame* caller_frame = iterator.NextFrame();
ASSERT(caller_frame->IsDartFrame());
ASSERT(!caller_frame->is_interpreted());
const Code& code = Code::Handle(zone, caller_frame->LookupDartCode());
const uword pc_offset = caller_frame->pc() - code.PayloadStart();
if (FLAG_shared_slow_path_triggers_gc) {
isolate->heap()->CollectAllGarbage();
}
const CodeSourceMap& map =
CodeSourceMap::Handle(zone, code.code_source_map());
String& member_name = String::Handle(zone);
if (!map.IsNull()) {
CodeSourceMapReader reader(map, Array::null_array(),
Function::null_function());
const intptr_t name_index = reader.GetNullCheckNameIndexAt(pc_offset);
RELEASE_ASSERT(name_index >= 0);
const ObjectPool& pool = ObjectPool::Handle(zone, code.GetObjectPool());
member_name ^= pool.ObjectAt(name_index);
} else {
member_name = Symbols::OptimizedOut().raw();
}
NullErrorHelper(zone, member_name);
}
DEFINE_RUNTIME_ENTRY(NullErrorWithSelector, 1) {
const String& selector = String::CheckedHandle(zone, arguments.ArgAt(0));
NullErrorHelper(zone, selector);
}
DEFINE_RUNTIME_ENTRY(NullCastError, 0) {
NullErrorHelper(zone, String::null_string());
}
DEFINE_RUNTIME_ENTRY(ArgumentNullError, 0) {
const String& error = String::Handle(String::New("argument value is null"));
Exceptions::ThrowArgumentError(error);
}
DEFINE_RUNTIME_ENTRY(ArgumentError, 1) {
const Instance& value = Instance::CheckedHandle(zone, arguments.ArgAt(0));
Exceptions::ThrowArgumentError(value);
}
DEFINE_RUNTIME_ENTRY(ArgumentErrorUnboxedInt64, 0) {
// Unboxed value is passed through a dedicated slot in Thread.
int64_t unboxed_value = arguments.thread()->unboxed_int64_runtime_arg();
const Integer& value = Integer::Handle(zone, Integer::New(unboxed_value));
Exceptions::ThrowArgumentError(value);
}
DEFINE_RUNTIME_ENTRY(IntegerDivisionByZeroException, 0) {
const Array& args = Array::Handle(zone, Array::New(0));
Exceptions::ThrowByType(Exceptions::kIntegerDivisionByZeroException, args);
}
static Heap::Space SpaceForRuntimeAllocation() {
return FLAG_stress_write_barrier_elimination ? Heap::kOld : Heap::kNew;
}
// Allocation of a fixed length array of given element type.
// This runtime entry is never called for allocating a List of a generic type,
// because a prior run time call instantiates the element type if necessary.
// Arg0: array length.
// Arg1: array type arguments, i.e. vector of 1 type, the element type.
// Return value: newly allocated array of length arg0.
DEFINE_RUNTIME_ENTRY(AllocateArray, 2) {
const Instance& length = Instance::CheckedHandle(zone, arguments.ArgAt(0));
if (!length.IsInteger()) {
// Throw: new ArgumentError.value(length, "length", "is not an integer");
const Array& args = Array::Handle(zone, Array::New(3));
args.SetAt(0, length);
args.SetAt(1, Symbols::Length());
args.SetAt(2, String::Handle(zone, String::New("is not an integer")));
Exceptions::ThrowByType(Exceptions::kArgumentValue, args);
}
const int64_t len = Integer::Cast(length).AsInt64Value();
if (len < 0) {
// Throw: new RangeError.range(length, 0, Array::kMaxElements, "length");
Exceptions::ThrowRangeError("length", Integer::Cast(length), 0,
Array::kMaxElements);
}
if (len > Array::kMaxElements) {
const Instance& exception = Instance::Handle(
zone, thread->isolate()->object_store()->out_of_memory());
Exceptions::Throw(thread, exception);
}
const Array& array = Array::Handle(
zone,
Array::New(static_cast<intptr_t>(len), SpaceForRuntimeAllocation()));
arguments.SetReturn(array);
TypeArguments& element_type =
TypeArguments::CheckedHandle(zone, arguments.ArgAt(1));
// An Array is raw or takes one type argument. However, its type argument
// vector may be longer than 1 due to a type optimization reusing the type
// argument vector of the instantiator.
ASSERT(element_type.IsNull() ||
(element_type.Length() >= 1 && element_type.IsInstantiated()));
array.SetTypeArguments(element_type); // May be null.
}
// Helper returning the token position of the Dart caller.
static TokenPosition GetCallerLocation() {
DartFrameIterator iterator(Thread::Current(),
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* caller_frame = iterator.NextFrame();
ASSERT(caller_frame != NULL);
return caller_frame->GetTokenPos();
}
// Result of an invoke may be an unhandled exception, in which case we
// rethrow it.
static void ThrowIfError(const Object& result) {
if (!result.IsNull() && result.IsError()) {
Exceptions::PropagateError(Error::Cast(result));
}
}
// Allocate a new object.
// Arg0: class of the object that needs to be allocated.
// Arg1: type arguments of the object that needs to be allocated.
// Return value: newly allocated object.
DEFINE_RUNTIME_ENTRY(AllocateObject, 2) {
const Class& cls = Class::CheckedHandle(zone, arguments.ArgAt(0));
const Error& error =
Error::Handle(zone, cls.EnsureIsAllocateFinalized(thread));
ThrowIfError(error);
const Instance& instance =
Instance::Handle(zone, Instance::New(cls, SpaceForRuntimeAllocation()));
arguments.SetReturn(instance);
if (cls.NumTypeArguments() == 0) {
// No type arguments required for a non-parameterized type.
ASSERT(Instance::CheckedHandle(zone, arguments.ArgAt(1)).IsNull());
} else {
const auto& type_arguments =
TypeArguments::CheckedHandle(zone, arguments.ArgAt(1));
// Unless null (for a raw type), the type argument vector may be longer than
// necessary due to a type optimization reusing the type argument vector of
// the instantiator.
ASSERT(type_arguments.IsNull() ||
(type_arguments.IsInstantiated() &&
(type_arguments.Length() >= cls.NumTypeArguments())));
instance.SetTypeArguments(type_arguments);
}
}
DEFINE_LEAF_RUNTIME_ENTRY(uword /*ObjectPtr*/,
EnsureRememberedAndMarkingDeferred,
2,
uword /*ObjectPtr*/ object_in,
Thread* thread) {
ObjectPtr object = static_cast<ObjectPtr>(object_in);
// The allocation stubs will call this leaf method for newly allocated
// old space objects.
RELEASE_ASSERT(object->IsOldObject());
// If we eliminate a generational write barriers on allocations of an object
// we need to ensure it's either a new-space object or it has been added to
// the remebered set.
//
// NOTE: We use reinterpret_cast<>() instead of ::RawCast() to avoid handle
// allocations in debug mode. Handle allocations in leaf runtimes can cause
// memory leaks because they will allocate into a handle scope from the next
// outermost runtime code (to which the genenerated Dart code might not return
// in a long time).
bool add_to_remembered_set = true;
if (object->ptr()->IsRemembered()) {
// Objects must not be added to the remembered set twice because the
// scavenger's visitor is not idempotent.
// Might already be remembered because of type argument store in
// AllocateArray or any field in CloneContext.
add_to_remembered_set = false;
} else if (object->IsArray()) {
const intptr_t length = Array::LengthOf(static_cast<ArrayPtr>(object));
add_to_remembered_set =
compiler::target::WillAllocateNewOrRememberedArray(length);
} else if (object->IsContext()) {
const intptr_t num_context_variables =
Context::NumVariables(static_cast<ContextPtr>(object));
add_to_remembered_set =
compiler::target::WillAllocateNewOrRememberedContext(
num_context_variables);
}
if (add_to_remembered_set) {
object->ptr()->AddToRememberedSet(thread);
}
// For incremental write barrier elimination, we need to ensure that the
// allocation ends up in the new space or else the object needs to added
// to deferred marking stack so it will be [re]scanned.
if (thread->is_marking()) {
thread->DeferredMarkingStackAddObject(object);
}
return static_cast<uword>(object);
}
END_LEAF_RUNTIME_ENTRY
// Instantiate type.
// Arg0: uninstantiated type.
// Arg1: instantiator type arguments.
// Arg2: function type arguments.
// Return value: instantiated type.
DEFINE_RUNTIME_ENTRY(InstantiateType, 3) {
AbstractType& type = AbstractType::CheckedHandle(zone, arguments.ArgAt(0));
const TypeArguments& instantiator_type_arguments =
TypeArguments::CheckedHandle(zone, arguments.ArgAt(1));
const TypeArguments& function_type_arguments =
TypeArguments::CheckedHandle(zone, arguments.ArgAt(2));
ASSERT(!type.IsNull());
ASSERT(instantiator_type_arguments.IsNull() ||
instantiator_type_arguments.IsInstantiated());
ASSERT(function_type_arguments.IsNull() ||
function_type_arguments.IsInstantiated());
type = type.InstantiateFrom(instantiator_type_arguments,
function_type_arguments, kAllFree, Heap::kOld);
if (type.IsTypeRef()) {
type = TypeRef::Cast(type).type();
ASSERT(!type.IsTypeRef());
ASSERT(type.IsCanonical());
}
ASSERT(!type.IsNull() && type.IsInstantiated());
arguments.SetReturn(type);
}
// Instantiate type arguments.
// Arg0: uninstantiated type arguments.
// Arg1: instantiator type arguments.
// Arg2: function type arguments.
// Return value: instantiated type arguments.
DEFINE_RUNTIME_ENTRY(InstantiateTypeArguments, 3) {
TypeArguments& type_arguments =
TypeArguments::CheckedHandle(zone, arguments.ArgAt(0));
const TypeArguments& instantiator_type_arguments =
TypeArguments::CheckedHandle(zone, arguments.ArgAt(1));
const TypeArguments& function_type_arguments =
TypeArguments::CheckedHandle(zone, arguments.ArgAt(2));
ASSERT(!type_arguments.IsNull() && !type_arguments.IsInstantiated());
ASSERT(instantiator_type_arguments.IsNull() ||
instantiator_type_arguments.IsInstantiated());
ASSERT(function_type_arguments.IsNull() ||
function_type_arguments.IsInstantiated());
// Code inlined in the caller should have optimized the case where the
// instantiator can be reused as type argument vector.
ASSERT(!type_arguments.IsUninstantiatedIdentity());
type_arguments = type_arguments.InstantiateAndCanonicalizeFrom(
instantiator_type_arguments, function_type_arguments);
ASSERT(type_arguments.IsNull() || type_arguments.IsInstantiated());
arguments.SetReturn(type_arguments);
}
// Instantiate type.
// Arg0: instantiator type arguments
// Arg1: function type arguments
// Arg2: type to be a subtype of the other
// Arg3: type to be a supertype of the other
// Arg4: variable name of the subtype parameter
// No return value.
DEFINE_RUNTIME_ENTRY(SubtypeCheck, 5) {
const TypeArguments& instantiator_type_args =
TypeArguments::CheckedHandle(zone, arguments.ArgAt(0));
const TypeArguments& function_type_args =
TypeArguments::CheckedHandle(zone, arguments.ArgAt(1));
AbstractType& subtype = AbstractType::CheckedHandle(zone, arguments.ArgAt(2));
AbstractType& supertype =
AbstractType::CheckedHandle(zone, arguments.ArgAt(3));
const String& dst_name = String::CheckedHandle(zone, arguments.ArgAt(4));
ASSERT(!subtype.IsNull() && !subtype.IsTypeRef());
ASSERT(!supertype.IsNull() && !supertype.IsTypeRef());
// The supertype or subtype may not be instantiated.
if (AbstractType::InstantiateAndTestSubtype(
&subtype, &supertype, instantiator_type_args, function_type_args)) {
return;
}
// Throw a dynamic type error.
const TokenPosition location = GetCallerLocation();
Exceptions::CreateAndThrowTypeError(location, subtype, supertype, dst_name);
UNREACHABLE();
}
// Allocate a new SubtypeTestCache for use in interpreted implicit setters.
// Return value: newly allocated SubtypeTestCache.
DEFINE_RUNTIME_ENTRY(AllocateSubtypeTestCache, 0) {
ASSERT(FLAG_enable_interpreter);
arguments.SetReturn(SubtypeTestCache::Handle(zone, SubtypeTestCache::New()));
}
// Allocate a new context large enough to hold the given number of variables.
// Arg0: number of variables.
// Return value: newly allocated context.
DEFINE_RUNTIME_ENTRY(AllocateContext, 1) {
const Smi& num_variables = Smi::CheckedHandle(zone, arguments.ArgAt(0));
const Context& context = Context::Handle(
zone, Context::New(num_variables.Value(), SpaceForRuntimeAllocation()));
arguments.SetReturn(context);
}
// Make a copy of the given context, including the values of the captured
// variables.
// Arg0: the context to be cloned.
// Return value: newly allocated context.
DEFINE_RUNTIME_ENTRY(CloneContext, 1) {
const Context& ctx = Context::CheckedHandle(zone, arguments.ArgAt(0));
Context& cloned_ctx = Context::Handle(
zone, Context::New(ctx.num_variables(), SpaceForRuntimeAllocation()));
cloned_ctx.set_parent(Context::Handle(zone, ctx.parent()));
Object& inst = Object::Handle(zone);
for (int i = 0; i < ctx.num_variables(); i++) {
inst = ctx.At(i);
cloned_ctx.SetAt(i, inst);
}
arguments.SetReturn(cloned_ctx);
}
// Invoke field getter before dispatch.
// Arg0: instance.
// Arg1: field name (may be demangled during call).
// Return value: field value.
DEFINE_RUNTIME_ENTRY(GetFieldForDispatch, 2) {
ASSERT(FLAG_enable_interpreter);
const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
String& name = String::CheckedHandle(zone, arguments.ArgAt(1));
const Class& receiver_class = Class::Handle(zone, receiver.clazz());
if (Function::IsDynamicInvocationForwarderName(name)) {
name = Function::DemangleDynamicInvocationForwarderName(name);
arguments.SetArgAt(1, name); // Reflect change in arguments.
}
const String& getter_name = String::Handle(zone, Field::GetterName(name));
const int kTypeArgsLen = 0;
const int kNumArguments = 1;
ArgumentsDescriptor args_desc(Array::Handle(
zone, ArgumentsDescriptor::NewBoxed(kTypeArgsLen, kNumArguments)));
const Function& getter =
Function::Handle(zone, Resolver::ResolveDynamicForReceiverClass(
receiver_class, getter_name, args_desc));
ASSERT(!getter.IsNull()); // An InvokeFieldDispatcher function was created.
const Array& args = Array::Handle(zone, Array::New(kNumArguments));
args.SetAt(0, receiver);
const Object& result =
Object::Handle(zone, DartEntry::InvokeFunction(getter, args));
ThrowIfError(result);
arguments.SetReturn(result);
}
// Check that arguments are valid for the given closure.
// Arg0: function
// Arg1: arguments descriptor
// Return value: whether the arguments are valid
DEFINE_RUNTIME_ENTRY(ClosureArgumentsValid, 2) {
ASSERT(FLAG_enable_interpreter);
const auto& closure = Closure::CheckedHandle(zone, arguments.ArgAt(0));
const auto& descriptor = Array::CheckedHandle(zone, arguments.ArgAt(1));
const auto& function = Function::Handle(zone, closure.function());
const ArgumentsDescriptor args_desc(descriptor);
if (!function.AreValidArguments(args_desc, nullptr)) {
arguments.SetReturn(Bool::False());
} else if (!closure.IsGeneric(thread) && args_desc.TypeArgsLen() > 0) {
// The arguments may be valid for the closure function itself, but if the
// closure has delayed type arguments, no type arguments should be provided.
arguments.SetReturn(Bool::False());
} else {
arguments.SetReturn(Bool::True());
}
}
// Resolve 'call' function of receiver.
// Arg0: receiver (not a closure).
// Arg1: arguments descriptor
// Return value: 'call' function'.
DEFINE_RUNTIME_ENTRY(ResolveCallFunction, 2) {
ASSERT(FLAG_enable_interpreter);
const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
const Array& descriptor = Array::CheckedHandle(zone, arguments.ArgAt(1));
ArgumentsDescriptor args_desc(descriptor);
ASSERT(!receiver.IsClosure()); // Interpreter tests for closure.
Class& cls = Class::Handle(zone, receiver.clazz());
Function& call_function = Function::Handle(
zone,
Resolver::ResolveDynamicForReceiverClass(cls, Symbols::Call(), args_desc,
/*allow_add=*/false));
arguments.SetReturn(call_function);
}
// Helper routine for tracing a type check.
static void PrintTypeCheck(const char* message,
const Instance& instance,
const AbstractType& type,
const TypeArguments& instantiator_type_arguments,
const TypeArguments& function_type_arguments,
const Bool& result) {
DartFrameIterator iterator(Thread::Current(),
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* caller_frame = iterator.NextFrame();
ASSERT(caller_frame != NULL);
const AbstractType& instance_type =
AbstractType::Handle(instance.GetType(Heap::kNew));
ASSERT(instance_type.IsInstantiated() ||
(instance.IsClosure() && instance_type.IsInstantiated(kCurrentClass)));
if (type.IsInstantiated()) {
OS::PrintErr("%s: '%s' %" Pd " %s '%s' %" Pd " (pc: %#" Px ").\n", message,
String::Handle(instance_type.Name()).ToCString(),
Class::Handle(instance_type.type_class()).id(),
(result.raw() == Bool::True().raw()) ? "is" : "is !",
String::Handle(type.Name()).ToCString(),
Class::Handle(type.type_class()).id(), caller_frame->pc());
} else {
// Instantiate type before printing.
const AbstractType& instantiated_type = AbstractType::Handle(
type.InstantiateFrom(instantiator_type_arguments,
function_type_arguments, kAllFree, Heap::kOld));
OS::PrintErr("%s: '%s' %s '%s' instantiated from '%s' (pc: %#" Px ").\n",
message, String::Handle(instance_type.Name()).ToCString(),
(result.raw() == Bool::True().raw()) ? "is" : "is !",
String::Handle(instantiated_type.Name()).ToCString(),
String::Handle(type.Name()).ToCString(), caller_frame->pc());
}
const Function& function =
Function::Handle(caller_frame->LookupDartFunction());
OS::PrintErr(" -> Function %s\n", function.ToFullyQualifiedCString());
}
// This updates the type test cache, an array containing 5-value elements
// (instance class (or function if the instance is a closure), instance type
// arguments, instantiator type arguments, function type arguments,
// and test_result). It can be applied to classes with type arguments in which
// case it contains just the result of the class subtype test, not including the
// evaluation of type arguments.
// This operation is currently very slow (lookup of code is not efficient yet).
static void UpdateTypeTestCache(
Zone* zone,
Thread* thread,
const Instance& instance,
const AbstractType& type,
const TypeArguments& instantiator_type_arguments,
const TypeArguments& function_type_arguments,
const Bool& result,
const SubtypeTestCache& new_cache) {
ASSERT(!new_cache.IsNull());
Class& instance_class = Class::Handle(zone);
if (instance.IsSmi()) {
instance_class = Smi::Class();
} else {
instance_class = instance.clazz();
}
// If the type is uninstantiated and refers to parent function type
// parameters, the function_type_arguments have been canonicalized
// when concatenated.
ASSERT(function_type_arguments.IsNull() ||
function_type_arguments.IsCanonical());
auto& instance_class_id_or_function = Object::Handle(zone);
auto& instance_type_arguments = TypeArguments::Handle(zone);
auto& instance_parent_function_type_arguments = TypeArguments::Handle(zone);
auto& instance_delayed_type_arguments = TypeArguments::Handle(zone);
if (instance_class.IsClosureClass()) {
const auto& closure = Closure::Cast(instance);
const auto& closure_function = Function::Handle(zone, closure.function());
instance_class_id_or_function = closure_function.raw();
instance_type_arguments = closure.instantiator_type_arguments();
instance_parent_function_type_arguments = closure.function_type_arguments();
instance_delayed_type_arguments = closure.delayed_type_arguments();
} else {
instance_class_id_or_function = Smi::New(instance_class.id());
if (instance_class.NumTypeArguments() > 0) {
instance_type_arguments = instance.GetTypeArguments();
}
}
{
SafepointMutexLocker ml(
thread->isolate_group()->subtype_test_cache_mutex());
const intptr_t len = new_cache.NumberOfChecks();
if (len >= FLAG_max_subtype_cache_entries) {
if (FLAG_trace_type_checks) {
OS::PrintErr(
"Not updating subtype test cache as its length reached %d\n",
FLAG_max_subtype_cache_entries);
}
return;
}
ASSERT(instance_type_arguments.IsNull() ||
instance_type_arguments.IsCanonical());
ASSERT(instantiator_type_arguments.IsNull() ||
instantiator_type_arguments.IsCanonical());
ASSERT(function_type_arguments.IsNull() ||
function_type_arguments.IsCanonical());
ASSERT(instance_parent_function_type_arguments.IsNull() ||
instance_parent_function_type_arguments.IsCanonical());
ASSERT(instance_delayed_type_arguments.IsNull() ||
instance_delayed_type_arguments.IsCanonical());
auto& last_instance_class_id_or_function = Object::Handle(zone);
auto& last_instance_type_arguments = TypeArguments::Handle(zone);
auto& last_instantiator_type_arguments = TypeArguments::Handle(zone);
auto& last_function_type_arguments = TypeArguments::Handle(zone);
auto& last_instance_parent_function_type_arguments =
TypeArguments::Handle(zone);
auto& last_instance_delayed_type_arguments = TypeArguments::Handle(zone);
Bool& last_result = Bool::Handle(zone);
for (intptr_t i = 0; i < len; ++i) {
new_cache.GetCheck(
i, &last_instance_class_id_or_function, &last_instance_type_arguments,
&last_instantiator_type_arguments, &last_function_type_arguments,
&last_instance_parent_function_type_arguments,
&last_instance_delayed_type_arguments, &last_result);
if ((last_instance_class_id_or_function.raw() ==
instance_class_id_or_function.raw()) &&
(last_instance_type_arguments.raw() ==
instance_type_arguments.raw()) &&
(last_instantiator_type_arguments.raw() ==
instantiator_type_arguments.raw()) &&
(last_function_type_arguments.raw() ==
function_type_arguments.raw()) &&
(last_instance_parent_function_type_arguments.raw() ==
instance_parent_function_type_arguments.raw()) &&
(last_instance_delayed_type_arguments.raw() ==
instance_delayed_type_arguments.raw())) {
// Some other isolate might have updated the cache between entry was
// found missing and now.
return;
}
}
new_cache.AddCheck(instance_class_id_or_function, instance_type_arguments,
instantiator_type_arguments, function_type_arguments,
instance_parent_function_type_arguments,
instance_delayed_type_arguments, result);
if (FLAG_trace_type_checks) {
AbstractType& test_type = AbstractType::Handle(zone, type.raw());
if (!test_type.IsInstantiated()) {
test_type =
type.InstantiateFrom(instantiator_type_arguments,
function_type_arguments, kAllFree, Heap::kNew);
}
const auto& type_class = Class::Handle(zone, test_type.type_class());
const auto& instance_class_name =
String::Handle(zone, instance_class.Name());
OS::PrintErr(
" Updated test cache %#" Px " ix: %" Pd
" with (cid-or-fun:"
" %#" Px ", type-args: %#" Px ", i-type-args: %#" Px
", "
"f-type-args: %#" Px ", p-type-args: %#" Px
", "
"d-type-args: %#" Px
", result: %s)\n"
" instance [class: (%#" Px " '%s' cid: %" Pd
"), type-args: %#" Px
" %s]\n"
" test-type [class: (%#" Px " '%s' cid: %" Pd
"), i-type-args: %#" Px " %s, f-type-args: %#" Px " %s]\n",
static_cast<uword>(new_cache.raw()), len,
static_cast<uword>(instance_class_id_or_function.raw()),
static_cast<uword>(instance_type_arguments.raw()),
static_cast<uword>(instantiator_type_arguments.raw()),
static_cast<uword>(function_type_arguments.raw()),
static_cast<uword>(instance_parent_function_type_arguments.raw()),
static_cast<uword>(instance_delayed_type_arguments.raw()),
result.ToCString(), static_cast<uword>(instance_class.raw()),
instance_class_name.ToCString(), instance_class.id(),
static_cast<uword>(instance_type_arguments.raw()),
instance_type_arguments.ToCString(),
static_cast<uword>(type_class.raw()),
String::Handle(zone, type_class.Name()).ToCString(), type_class.id(),
static_cast<uword>(instantiator_type_arguments.raw()),
instantiator_type_arguments.ToCString(),
static_cast<uword>(function_type_arguments.raw()),
function_type_arguments.ToCString());
}
}
}
// Check that the given instance is an instance of the given type.
// Tested instance may be null, because a null test cannot always be inlined,
// e.g 'null is T' yields true if T = Null, but false if T = bool.
// Arg0: instance being checked.
// Arg1: type.
// Arg2: type arguments of the instantiator of the type.
// Arg3: type arguments of the function of the type.
// Arg4: SubtypeTestCache.
// Return value: true or false.
DEFINE_RUNTIME_ENTRY(Instanceof, 5) {
const Instance& instance = Instance::CheckedHandle(zone, arguments.ArgAt(0));
const AbstractType& type =
AbstractType::CheckedHandle(zone, arguments.ArgAt(1));
const TypeArguments& instantiator_type_arguments =
TypeArguments::CheckedHandle(zone, arguments.ArgAt(2));
const TypeArguments& function_type_arguments =
TypeArguments::CheckedHandle(zone, arguments.ArgAt(3));
const SubtypeTestCache& cache =
SubtypeTestCache::CheckedHandle(zone, arguments.ArgAt(4));
ASSERT(type.IsFinalized());
ASSERT(!type.IsDynamicType()); // No need to check assignment.
ASSERT(!cache.IsNull());
const Bool& result = Bool::Get(instance.IsInstanceOf(
type, instantiator_type_arguments, function_type_arguments));
if (FLAG_trace_type_checks) {
PrintTypeCheck("InstanceOf", instance, type, instantiator_type_arguments,
function_type_arguments, result);
}
UpdateTypeTestCache(zone, thread, instance, type, instantiator_type_arguments,
function_type_arguments, result, cache);
arguments.SetReturn(result);
}
// Check that the type of the given instance is a subtype of the given type and
// can therefore be assigned.
// Tested instance may not be null, because a null test is always inlined.
// Arg0: instance being assigned.
// Arg1: type being assigned to.
// Arg2: type arguments of the instantiator of the type being assigned to.
// Arg3: type arguments of the function of the type being assigned to.
// Arg4: name of variable being assigned to.
// Arg5: SubtypeTestCache.
// Arg6: invocation mode (see TypeCheckMode)
// Return value: instance if a subtype, otherwise throw a TypeError.
DEFINE_RUNTIME_ENTRY(TypeCheck, 7) {
const Instance& src_instance =
Instance::CheckedHandle(zone, arguments.ArgAt(0));
AbstractType& dst_type =
AbstractType::CheckedHandle(zone, arguments.ArgAt(1));
const TypeArguments& instantiator_type_arguments =
TypeArguments::CheckedHandle(zone, arguments.ArgAt(2));
const TypeArguments& function_type_arguments =
TypeArguments::CheckedHandle(zone, arguments.ArgAt(3));
String& dst_name = String::Handle(zone);
dst_name ^= arguments.ArgAt(4);
ASSERT(dst_name.IsNull() || dst_name.IsString());
SubtypeTestCache& cache = SubtypeTestCache::Handle(zone);
cache ^= arguments.ArgAt(5);
ASSERT(cache.IsNull() || cache.IsSubtypeTestCache());
const TypeCheckMode mode = static_cast<TypeCheckMode>(
Smi::CheckedHandle(zone, arguments.ArgAt(6)).Value());
#if defined(TARGET_ARCH_IA32)
ASSERT(mode == kTypeCheckFromInline);
#endif
ASSERT(!dst_type.IsDynamicType()); // No need to check assignment.
// A null instance is already detected and allowed in inlined code, unless
// strong checking is enabled.
ASSERT(!src_instance.IsNull() || isolate->null_safety());
const bool is_instance_of = src_instance.IsAssignableTo(
dst_type, instantiator_type_arguments, function_type_arguments);
if (FLAG_trace_type_checks) {
PrintTypeCheck("TypeCheck", src_instance, dst_type,
instantiator_type_arguments, function_type_arguments,
Bool::Get(is_instance_of));
}
if (!is_instance_of) {
// Throw a dynamic type error.
const TokenPosition location = GetCallerLocation();
const AbstractType& src_type =
AbstractType::Handle(zone, src_instance.GetType(Heap::kNew));
if (!dst_type.IsInstantiated()) {
// Instantiate dst_type before reporting the error.
dst_type = dst_type.InstantiateFrom(instantiator_type_arguments,
function_type_arguments, kAllFree,
Heap::kNew);
}
if (dst_name.IsNull()) {
#if !defined(TARGET_ARCH_IA32)
// Can only come here from type testing stub.
ASSERT(mode != kTypeCheckFromInline);
// Grab the [dst_name] from the pool. It's stored at one pool slot after
// the subtype-test-cache.
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* caller_frame = iterator.NextFrame();
ASSERT(!caller_frame->is_interpreted());
const Code& caller_code =
Code::Handle(zone, caller_frame->LookupDartCode());
const ObjectPool& pool =
ObjectPool::Handle(zone, caller_code.GetObjectPool());
TypeTestingStubCallPattern tts_pattern(caller_frame->pc());
const intptr_t stc_pool_idx = tts_pattern.GetSubtypeTestCachePoolIndex();
const intptr_t dst_name_idx = stc_pool_idx + 1;
dst_name ^= pool.ObjectAt(dst_name_idx);
#else
UNREACHABLE();
#endif
}
Exceptions::CreateAndThrowTypeError(location, src_type, dst_type, dst_name);
UNREACHABLE();
}
bool should_update_cache = true;
#if !defined(TARGET_ARCH_IA32)
bool would_update_cache_if_not_lazy = false;
#if !defined(DART_PRECOMPILED_RUNTIME)
if (mode == kTypeCheckFromLazySpecializeStub) {
// Checks against type parameters are done by loading the value of the type
// parameter and calling its type testing stub.
// So we have to install a specialized TTS on the value of the type
// parameter, not the parameter itself.
if (dst_type.IsTypeParameter()) {
dst_type = TypeParameter::Cast(dst_type).GetFromTypeArguments(
instantiator_type_arguments, function_type_arguments);
}
if (FLAG_trace_type_checks) {
OS::PrintErr(" Specializing type testing stub for %s\n",
dst_type.ToCString());
}
TypeTestingStubGenerator::SpecializeStubFor(thread, dst_type);
// Only create the cache if we failed to create a specialized TTS and doing
// the same check would cause an update to the cache.
would_update_cache_if_not_lazy =
(!src_instance.IsNull() &&
dst_type.type_test_stub() ==
StubCode::DefaultNullableTypeTest().raw()) ||
dst_type.type_test_stub() == StubCode::DefaultTypeTest().raw();
should_update_cache = would_update_cache_if_not_lazy && cache.IsNull();
}
// Fast path of type testing stub wasn't able to handle given type, yet it
// passed the type check. It means that fast-path was using outdated cid
// ranges and new classes appeared since the stub was generated.
// Re-generate the stub.
if ((mode == kTypeCheckFromSlowStub) && dst_type.IsType() &&
(TypeTestingStubGenerator::DefaultCodeForType(dst_type, /*lazy=*/false) !=
dst_type.type_test_stub()) &&
dst_type.IsInstantiated()) {
if (FLAG_trace_type_checks) {
OS::PrintErr(" Rebuilding type testing stub for %s\n",
dst_type.ToCString());
}
#if defined(DEBUG)
const auto& old_code = Code::Handle(dst_type.type_test_stub());
#endif
TypeTestingStubGenerator::SpecializeStubFor(thread, dst_type);
#if defined(DEBUG)
ASSERT(old_code.raw() != dst_type.type_test_stub());
#endif
// Only create the cache when we come from a normal stub.
should_update_cache = false;
}
#endif // !defined(DART_PRECOMPILED_RUNTIME)
#endif // !defined(TARGET_ARCH_IA32)
if (should_update_cache) {
if (cache.IsNull()) {
#if !defined(TARGET_ARCH_IA32)
ASSERT(mode == kTypeCheckFromSlowStub ||
(mode == kTypeCheckFromLazySpecializeStub &&
would_update_cache_if_not_lazy));
// We lazily create [SubtypeTestCache] for those call sites which actually
// need one and will patch the pool entry.
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* caller_frame = iterator.NextFrame();
ASSERT(!caller_frame->is_interpreted());
const Code& caller_code =
Code::Handle(zone, caller_frame->LookupDartCode());
const ObjectPool& pool =
ObjectPool::Handle(zone, caller_code.GetObjectPool());
TypeTestingStubCallPattern tts_pattern(caller_frame->pc());
const intptr_t stc_pool_idx = tts_pattern.GetSubtypeTestCachePoolIndex();
// Ensure we do have a STC (lazily create it if not) and all threads use
// the same STC.
{
SafepointMutexLocker ml(isolate->group()->subtype_test_cache_mutex());
cache ^= pool.ObjectAt<std::memory_order_acquire>(stc_pool_idx);
if (cache.IsNull()) {
cache = SubtypeTestCache::New();
pool.SetObjectAt<std::memory_order_release>(stc_pool_idx, cache);
}
}
#else
UNREACHABLE();
#endif
}
UpdateTypeTestCache(zone, thread, src_instance, dst_type,
instantiator_type_arguments, function_type_arguments,
Bool::True(), cache);
}
arguments.SetReturn(src_instance);
}
// Report that the type of the given object is not bool in conditional context.
// Throw assertion error if the object is null. (cf. Boolean Conversion
// in language Spec.)
// Arg0: bad object.
// Return value: none, throws TypeError or AssertionError.
DEFINE_RUNTIME_ENTRY(NonBoolTypeError, 1) {
const TokenPosition location = GetCallerLocation();
const Instance& src_instance =
Instance::CheckedHandle(zone, arguments.ArgAt(0));
if (src_instance.IsNull()) {
const Array& args = Array::Handle(zone, Array::New(5));
args.SetAt(
0, String::Handle(
zone,
String::New(
"Failed assertion: boolean expression must not be null")));
// No source code for this assertion, set url to null.
args.SetAt(1, String::Handle(zone, String::null()));
args.SetAt(2, Object::smi_zero());
args.SetAt(3, Object::smi_zero());
args.SetAt(4, String::Handle(zone, String::null()));
Exceptions::ThrowByType(Exceptions::kAssertion, args);
UNREACHABLE();
}
ASSERT(!src_instance.IsBool());
const Type& bool_interface = Type::Handle(Type::BoolType());
const AbstractType& src_type =
AbstractType::Handle(zone, src_instance.GetType(Heap::kNew));
Exceptions::CreateAndThrowTypeError(location, src_type, bool_interface,
Symbols::BooleanExpression());
UNREACHABLE();
}
DEFINE_RUNTIME_ENTRY(Throw, 1) {
const Instance& exception = Instance::CheckedHandle(zone, arguments.ArgAt(0));
Exceptions::Throw(thread, exception);
}
DEFINE_RUNTIME_ENTRY(ReThrow, 2) {
const Instance& exception = Instance::CheckedHandle(zone, arguments.ArgAt(0));
const Instance& stacktrace =
Instance::CheckedHandle(zone, arguments.ArgAt(1));
Exceptions::ReThrow(thread, exception, stacktrace);
}
// Patches static call in optimized code with the target's entry point.
// Compiles target if necessary.
DEFINE_RUNTIME_ENTRY(PatchStaticCall, 0) {
#if !defined(DART_PRECOMPILED_RUNTIME)
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* caller_frame = iterator.NextFrame();
ASSERT(caller_frame != NULL);
ASSERT(!caller_frame->is_interpreted());
const Code& caller_code = Code::Handle(zone, caller_frame->LookupDartCode());
ASSERT(!caller_code.IsNull());
ASSERT(caller_code.is_optimized());
const Function& target_function = Function::Handle(
zone, caller_code.GetStaticCallTargetFunctionAt(caller_frame->pc()));
const Code& target_code = Code::Handle(zone, target_function.EnsureHasCode());
// Before patching verify that we are not repeatedly patching to the same
// target.
ASSERT(target_code.raw() !=
CodePatcher::GetStaticCallTargetAt(caller_frame->pc(), caller_code));
CodePatcher::PatchStaticCallAt(caller_frame->pc(), caller_code, target_code);
caller_code.SetStaticCallTargetCodeAt(caller_frame->pc(), target_code);
if (FLAG_trace_patching) {
THR_Print("PatchStaticCall: patching caller pc %#" Px
""
" to '%s' new entry point %#" Px " (%s)\n",
caller_frame->pc(), target_function.ToFullyQualifiedCString(),
target_code.EntryPoint(),
target_code.is_optimized() ? "optimized" : "unoptimized");
}
arguments.SetReturn(target_code);
#else
UNREACHABLE();
#endif
}
#if defined(PRODUCT) || defined(DART_PRECOMPILED_RUNTIME)
DEFINE_RUNTIME_ENTRY(BreakpointRuntimeHandler, 0) {
UNREACHABLE();
return;
}
#else
// Gets called from debug stub when code reaches a breakpoint
// set on a runtime stub call.
DEFINE_RUNTIME_ENTRY(BreakpointRuntimeHandler, 0) {
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* caller_frame = iterator.NextFrame();
ASSERT(caller_frame != NULL);
Code& orig_stub = Code::Handle(zone);
if (!caller_frame->is_interpreted()) {
orig_stub = isolate->debugger()->GetPatchedStubAddress(caller_frame->pc());
}
const Error& error =
Error::Handle(zone, isolate->debugger()->PauseBreakpoint());
ThrowIfError(error);
arguments.SetReturn(orig_stub);
}
#endif
DEFINE_RUNTIME_ENTRY(SingleStepHandler, 0) {
#if defined(PRODUCT) || defined(DART_PRECOMPILED_RUNTIME)
UNREACHABLE();
#else
const Error& error =
Error::Handle(zone, isolate->debugger()->PauseStepping());
ThrowIfError(error);
#endif
}
// An instance call of the form o.f(...) could not be resolved. Check if
// there is a getter with the same name. If so, invoke it. If the value is
// a closure, invoke it with the given arguments. If the value is a
// non-closure, attempt to invoke "call" on it.
static bool ResolveCallThroughGetter(const Class& receiver_class,
const String& target_name,
const String& demangled,
const Array& arguments_descriptor,
Function* result) {
const String& getter_name = String::Handle(Field::GetterName(demangled));
const int kTypeArgsLen = 0;
const int kNumArguments = 1;
ArgumentsDescriptor args_desc(Array::Handle(
ArgumentsDescriptor::NewBoxed(kTypeArgsLen, kNumArguments)));
const Function& getter =
Function::Handle(Resolver::ResolveDynamicForReceiverClass(
receiver_class, getter_name, args_desc));
if (getter.IsNull() || getter.IsMethodExtractor()) {
return false;
}
// We do this on the target_name, _not_ on the demangled name, so that
// FlowGraphBuilder::BuildGraphOfInvokeFieldDispatcher can detect dynamic
// calls from the dyn: tag on the name of the dispatcher.
const Function& target_function =
Function::Handle(receiver_class.GetInvocationDispatcher(
target_name, arguments_descriptor,
FunctionLayout::kInvokeFieldDispatcher, FLAG_lazy_dispatchers));
ASSERT(!target_function.IsNull() || !FLAG_lazy_dispatchers);
if (FLAG_trace_ic) {
OS::PrintErr(
"InvokeField IC miss: adding <%s> id:%" Pd " -> <%s>\n",
receiver_class.ToCString(), receiver_class.id(),
target_function.IsNull() ? "null" : target_function.ToCString());
}
*result = target_function.raw();
return true;
}
// Handle other invocations (implicit closures, noSuchMethod).
FunctionPtr InlineCacheMissHelper(const Class& receiver_class,
const Array& args_descriptor,
const String& target_name) {
// Create a demangled version of the target_name, if necessary, This is used
// for the field getter in ResolveCallThroughGetter and as the target name
// for the NoSuchMethod dispatcher (if needed).
const String* demangled = &target_name;
if (Function::IsDynamicInvocationForwarderName(target_name)) {
demangled = &String::Handle(
Function::DemangleDynamicInvocationForwarderName(target_name));
}
Function& result = Function::Handle();
if (!ResolveCallThroughGetter(receiver_class, target_name, *demangled,
args_descriptor, &result)) {
ArgumentsDescriptor desc(args_descriptor);
const Function& target_function =
Function::Handle(receiver_class.GetInvocationDispatcher(
*demangled, args_descriptor,
FunctionLayout::kNoSuchMethodDispatcher, FLAG_lazy_dispatchers));
if (FLAG_trace_ic) {
OS::PrintErr(
"NoSuchMethod IC miss: adding <%s> id:%" Pd " -> <%s>\n",
receiver_class.ToCString(), receiver_class.id(),
target_function.IsNull() ? "null" : target_function.ToCString());
}
result = target_function.raw();
}
// May be null if --no-lazy-dispatchers, in which case dispatch will be
// handled by NoSuchMethodFromCallStub.
ASSERT(!result.IsNull() || !FLAG_lazy_dispatchers);
return result.raw();
}
static void TrySwitchInstanceCall(const ICData& ic_data,
const Function& target_function) {
#if !defined(DART_PRECOMPILED_RUNTIME)
// Monomorphic/megamorphic calls only check the receiver CID.
if (ic_data.NumArgsTested() != 1) return;
ASSERT(ic_data.rebind_rule() == ICData::kInstance);
// Monomorphic/megamorphic calls don't record exactness.
if (ic_data.is_tracking_exactness()) return;
#if !defined(PRODUCT)
// Monomorphic/megamorphic do not check the isolate's stepping flag.
if (Isolate::Current()->has_attempted_stepping()) return;
#endif
Thread* thread = Thread::Current();
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* caller_frame = iterator.NextFrame();
ASSERT(caller_frame->IsDartFrame());
// Monomorphic/megamorphic calls are only for unoptimized code.
if (caller_frame->is_interpreted()) return;
Zone* zone = thread->zone();
const Code& caller_code = Code::Handle(zone, caller_frame->LookupDartCode());
if (caller_code.is_optimized()) return;
// Code is detached from its function. This will prevent us from resetting
// the switchable call later because resets are function based and because
// the ic_data_array belongs to the function instead of the code. This should
// only happen because of reload, but it sometimes happens with KBC mixed mode
// probably through a race between foreground and background compilation.
const Function& caller_function =
Function::Handle(zone, caller_code.function());
if (caller_function.unoptimized_code() != caller_code.raw()) {
return;
}
#if !defined(PRODUCT)
// Skip functions that contain breakpoints or when debugger is in single
// stepping mode.
if (Debugger::IsDebugging(thread, caller_function)) return;
#endif
const intptr_t num_checks = ic_data.NumberOfChecks();
// Monomorphic call.
if (FLAG_unopt_monomorphic_calls && (num_checks == 1)) {
// A call site in the monomorphic state does not load the arguments
// descriptor, so do not allow transition to this state if the callee
// needs it.
if (target_function.PrologueNeedsArgumentsDescriptor()) {
return;
}
// Avoid forcing foreground compilation if target function is still
// interpreted.
if (FLAG_enable_interpreter && !target_function.HasCode()) {
return;
}
const Array& data = Array::Handle(zone, ic_data.entries());
const Code& target = Code::Handle(zone, target_function.EnsureHasCode());
CodePatcher::PatchInstanceCallAt(caller_frame->pc(), caller_code, data,
target);
if (FLAG_trace_ic) {
OS::PrintErr("Instance call at %" Px
" switching to monomorphic dispatch, %s\n",
caller_frame->pc(), ic_data.ToCString());
}
return; // Success.
}
// Megamorphic call.
if (FLAG_unopt_megamorphic_calls &&
(num_checks > FLAG_max_polymorphic_checks)) {
const String& name = String::Handle(zone, ic_data.target_name());
const Array& descriptor =
Array::Handle(zone, ic_data.arguments_descriptor());
const MegamorphicCache& cache = MegamorphicCache::Handle(
zone, MegamorphicCacheTable::Lookup(thread, name, descriptor));
ic_data.set_is_megamorphic(true);
CodePatcher::PatchInstanceCallAt(caller_frame->pc(), caller_code, cache,
StubCode::MegamorphicCall());
if (FLAG_trace_ic) {
OS::PrintErr("Instance call at %" Px
" switching to megamorphic dispatch, %s\n",
caller_frame->pc(), ic_data.ToCString());
}
return; // Success.
}
#endif // !defined(DART_PRECOMPILED_RUNTIME)
}
// Perform the subtype and return constant function based on the result.
static FunctionPtr ComputeTypeCheckTarget(const Instance& receiver,
const AbstractType& type,
const ArgumentsDescriptor& desc) {
const bool result = receiver.IsInstanceOf(type, Object::null_type_arguments(),
Object::null_type_arguments());
const ObjectStore* store = Isolate::Current()->object_store();
const Function& target =
Function::Handle(result ? store->simple_instance_of_true_function()
: store->simple_instance_of_false_function());
ASSERT(!target.IsNull());
return target.raw();
}
static FunctionPtr InlineCacheMissHandlerGivenTargetFunction(
const GrowableArray<const Instance*>& args, // Checked arguments only.
const ICData& ic_data,
intptr_t count,
const Function& target_function) {
if (target_function.IsNull()) {
return target_function.raw();
}
const Instance& receiver = *args[0];
if (args.length() == 1) {
if (ic_data.is_tracking_exactness()) {
#if !defined(DART_PRECOMPILED_RUNTIME)
const auto state = receiver.IsNull()
? StaticTypeExactnessState::NotExact()
: StaticTypeExactnessState::Compute(
Type::Cast(AbstractType::Handle(
ic_data.receivers_static_type())),
receiver);
ic_data.AddReceiverCheck(
receiver.GetClassId(), target_function, count,
/*exactness=*/state.CollapseSuperTypeExactness());
#else
UNREACHABLE();
#endif
} else {
ic_data.AddReceiverCheck(args[0]->GetClassId(), target_function, count);
}
} else {
GrowableArray<intptr_t> class_ids(args.length());
ASSERT(ic_data.NumArgsTested() == args.length());
for (intptr_t i = 0; i < args.length(); i++) {
class_ids.Add(args[i]->GetClassId());
}
ic_data.AddCheck(class_ids, target_function, count);
}
if (FLAG_trace_ic_miss_in_optimized || FLAG_trace_ic) {
DartFrameIterator iterator(Thread::Current(),
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* caller_frame = iterator.NextFrame();
ASSERT(caller_frame != NULL);
if (FLAG_trace_ic_miss_in_optimized) {
const Code& caller = Code::Handle(Code::LookupCode(caller_frame->pc()));
if (caller.is_optimized()) {
OS::PrintErr("IC miss in optimized code; call %s -> %s\n",
Function::Handle(caller.function()).ToCString(),
target_function.ToCString());
}
}
if (FLAG_trace_ic) {
OS::PrintErr("InlineCacheMissHandler %" Pd " call at %#" Px
"' "
"adding <%s> id:%" Pd " -> <%s>\n",
args.length(), caller_frame->pc(),
Class::Handle(receiver.clazz()).ToCString(),
receiver.GetClassId(), target_function.ToCString());
}
}
TrySwitchInstanceCall(ic_data, target_function);
return target_function.raw();
}
static FunctionPtr InlineCacheMissHandler(
const GrowableArray<const Instance*>& args, // Checked arguments only.
const ICData& ic_data,
intptr_t count = 1) {
Thread* thread = Thread::Current();
Zone* zone = thread->zone();
const Instance& receiver = *args[0];
ArgumentsDescriptor arguments_descriptor(
Array::Handle(zone, ic_data.arguments_descriptor()));
String& function_name = String::Handle(zone, ic_data.target_name());
ASSERT(function_name.IsSymbol());
const Class& receiver_class = Class::Handle(zone, receiver.clazz());
Function& target_function = Function::Handle(
zone, Resolver::ResolveDynamicForReceiverClass(
receiver_class, function_name, arguments_descriptor));
ObjectStore* store = thread->isolate()->object_store();
if (target_function.raw() == store->simple_instance_of_function()) {
// Replace the target function with constant function.
ASSERT(args.length() == 2);
const AbstractType& type = AbstractType::Cast(*args[1]);
target_function =
ComputeTypeCheckTarget(receiver, type, arguments_descriptor);
}
if (target_function.IsNull()) {
if (FLAG_trace_ic) {
OS::PrintErr("InlineCacheMissHandler NULL function for %s receiver: %s\n",
String::Handle(zone, ic_data.target_name()).ToCString(),
receiver.ToCString());
}
const Array& args_descriptor =
Array::Handle(zone, ic_data.arguments_descriptor());
const String& target_name = String::Handle(zone, ic_data.target_name());
target_function =
InlineCacheMissHelper(receiver_class, args_descriptor, target_name);
}
if (target_function.IsNull()) {
ASSERT(!FLAG_lazy_dispatchers);
return target_function.raw();
}
return InlineCacheMissHandlerGivenTargetFunction(args, ic_data, count,
target_function);
}
// Handles inline cache misses by updating the IC data array of the call site.
// Arg0: Receiver object.
// Arg1: IC data object.
// Returns: target function with compiled code or null.
// Modifies the instance call to hold the updated IC data array.
DEFINE_RUNTIME_ENTRY(InlineCacheMissHandlerOneArg, 2) {
const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
const ICData& ic_data = ICData::CheckedHandle(zone, arguments.ArgAt(1));
RELEASE_ASSERT(!FLAG_precompiled_mode);
GrowableArray<const Instance*> args(1);
args.Add(&receiver);
const Function& result =
Function::Handle(zone, InlineCacheMissHandler(args, ic_data));
arguments.SetReturn(result);
}
// Handles inline cache misses by updating the IC data array of the call site.
// Arg0: Receiver object.
// Arg1: Argument after receiver.
// Arg2: IC data object.
// Returns: target function with compiled code or null.
// Modifies the instance call to hold the updated IC data array.
DEFINE_RUNTIME_ENTRY(InlineCacheMissHandlerTwoArgs, 3) {
const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
const Instance& other = Instance::CheckedHandle(zone, arguments.ArgAt(1));
const ICData& ic_data = ICData::CheckedHandle(zone, arguments.ArgAt(2));
RELEASE_ASSERT(!FLAG_precompiled_mode);
GrowableArray<const Instance*> args(2);
args.Add(&receiver);
args.Add(&other);
const Function& result =
Function::Handle(zone, InlineCacheMissHandler(args, ic_data));
arguments.SetReturn(result);
}
// Handles a static call in unoptimized code that has one argument type not
// seen before. Compile the target if necessary and update the ICData.
// Arg0: argument.
// Arg1: IC data object.
DEFINE_RUNTIME_ENTRY(StaticCallMissHandlerOneArg, 2) {
const Instance& arg = Instance::CheckedHandle(zone, arguments.ArgAt(0));
const ICData& ic_data = ICData::CheckedHandle(zone, arguments.ArgAt(1));
// IC data for static call is prepopulated with the statically known target.
ASSERT(ic_data.NumberOfChecksIs(1));
const Function& target = Function::Handle(zone, ic_data.GetTargetAt(0));
target.EnsureHasCode();
ASSERT(!target.IsNull() && target.HasCode());
ic_data.AddReceiverCheck(arg.GetClassId(), target, 1);
if (FLAG_trace_ic) {
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* caller_frame = iterator.NextFrame();
ASSERT(caller_frame != NULL);
OS::PrintErr("StaticCallMissHandler at %#" Px " target %s (%" Pd ")\n",
caller_frame->pc(), target.ToCString(), arg.GetClassId());
}
arguments.SetReturn(target);
}
// Handles a static call in unoptimized code that has two argument types not
// seen before. Compile the target if necessary and update the ICData.
// Arg0: argument 0.
// Arg1: argument 1.
// Arg2: IC data object.
DEFINE_RUNTIME_ENTRY(StaticCallMissHandlerTwoArgs, 3) {
const Instance& arg0 = Instance::CheckedHandle(zone, arguments.ArgAt(0));
const Instance& arg1 = Instance::CheckedHandle(zone, arguments.ArgAt(1));
const ICData& ic_data = ICData::CheckedHandle(zone, arguments.ArgAt(2));
// IC data for static call is prepopulated with the statically known target.
ASSERT(!ic_data.NumberOfChecksIs(0));
const Function& target = Function::Handle(zone, ic_data.GetTargetAt(0));
target.EnsureHasCode();
GrowableArray<intptr_t> cids(2);
cids.Add(arg0.GetClassId());
cids.Add(arg1.GetClassId());
ic_data.AddCheck(cids, target);
if (FLAG_trace_ic) {
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* caller_frame = iterator.NextFrame();
ASSERT(caller_frame != NULL);
OS::PrintErr("StaticCallMissHandler at %#" Px " target %s (%" Pd ", %" Pd
")\n",
caller_frame->pc(), target.ToCString(), cids[0], cids[1]);
}
arguments.SetReturn(target);
}
static bool IsSingleTarget(Isolate* isolate,
Zone* zone,
intptr_t lower_cid,
intptr_t upper_cid,
const Function& target,
const String& name) {
Class& cls = Class::Handle(zone);
ClassTable* table = isolate->class_table();
Function& other_target = Function::Handle(zone);
for (intptr_t cid = lower_cid; cid <= upper_cid; cid++) {
if (!table->HasValidClassAt(cid)) continue;
cls = table->At(cid);
if (cls.is_abstract()) continue;
if (!cls.is_allocated()) continue;
other_target = Resolver::ResolveDynamicAnyArgs(zone, cls, name,
/*allow_add=*/false);
if (other_target.raw() != target.raw()) {
return false;
}
}
return true;
}
#if defined(DART_PRECOMPILED_RUNTIME)
class SavedUnlinkedCallMapKeyEqualsTraits : public AllStatic {
public:
static const char* Name() { return "SavedUnlinkedCallMapKeyEqualsTraits "; }
static bool ReportStats() { return false; }
static bool IsMatch(const Object& key1, const Object& key2) {
if (!key1.IsInteger() || !key2.IsInteger()) return false;
return Integer::Cast(key1).Equals(Integer::Cast(key2));
}
static uword Hash(const Object& key) {
return Integer::Cast(key).CanonicalizeHash();
}
};
using UnlinkedCallMap = UnorderedHashMap<SavedUnlinkedCallMapKeyEqualsTraits>;
static void SaveUnlinkedCall(Zone* zone,
Isolate* isolate,
uword frame_pc,
const UnlinkedCall& unlinked_call) {
IsolateGroup* isolate_group = isolate->group();
SafepointMutexLocker ml(isolate_group->unlinked_call_map_mutex());
if (isolate_group->saved_unlinked_calls() == Array::null()) {
const auto& initial_map =
Array::Handle(zone, HashTables::New<UnlinkedCallMap>(16, Heap::kOld));
isolate_group->set_saved_unlinked_calls(initial_map);
}
UnlinkedCallMap unlinked_call_map(zone,
isolate_group->saved_unlinked_calls());
const auto& pc = Integer::Handle(zone, Integer::NewFromUint64(frame_pc));
// Some other isolate might have updated unlinked_call_map[pc] too, but
// their update should be identical to ours.
const auto& new_or_old_value = UnlinkedCall::Handle(
zone, UnlinkedCall::RawCast(
unlinked_call_map.InsertOrGetValue(pc, unlinked_call)));
RELEASE_ASSERT(new_or_old_value.raw() == unlinked_call.raw());
isolate_group->set_saved_unlinked_calls(unlinked_call_map.Release());
}
static UnlinkedCallPtr LoadUnlinkedCall(Zone* zone,
Isolate* isolate,
uword pc) {
IsolateGroup* isolate_group = isolate->group();
SafepointMutexLocker ml(isolate_group->unlinked_call_map_mutex());
ASSERT(isolate_group->saved_unlinked_calls() != Array::null());
UnlinkedCallMap unlinked_call_map(zone,
isolate_group->saved_unlinked_calls());
const auto& pc_integer = Integer::Handle(zone, Integer::NewFromUint64(pc));
const auto& unlinked_call = UnlinkedCall::Cast(
Object::Handle(zone, unlinked_call_map.GetOrDie(pc_integer)));
isolate_group->set_saved_unlinked_calls(unlinked_call_map.Release());
return unlinked_call.raw();
}
// NOTE: Right now we never delete [UnlinkedCall] objects. They are needed while
// a call site is in Unlinked/Monomorphic/MonomorphicSmiable/SingleTarget
// states.
//
// Theoretically we could free the [UnlinkedCall] object once we transition the
// call site to use ICData/MegamorphicCache, but that would require careful
// coordination between the deleter and a possible concurrent reader.
//
// To simplify the code we decided not to do that atm (only a very small
// fraction of callsites in AOT use switchable calls, the name/args-descriptor
// objects are kept alive anyways -> there is little memory savings from
// freeing the [UnlinkedCall] objects).
#endif // defined(DART_PRECOMPILED_RUNTIME)
class SwitchableCallHandler {
public:
SwitchableCallHandler(Thread* thread,
const Instance& receiver,
NativeArguments arguments,
StackFrame* caller_frame,
const Code& caller_code,
const Function& caller_function)
: isolate_(thread->isolate()),
thread_(thread),
zone_(thread->zone()),
receiver_(receiver),
arguments_(arguments),
caller_frame_(caller_frame),
caller_code_(caller_code),
caller_function_(caller_function),
name_(String::Handle()),
args_descriptor_(Array::Handle()) {}
FunctionPtr ResolveTargetFunction(const Object& data);
void HandleMiss(const Object& old_data,
const Code& old_target,
const Function& target_function);
private:
void DoUnlinkedCall(const UnlinkedCall& unlinked,
const Function& target_function);
bool CanExtendSingleTargetRange(const String& name,
const Function& old_target,
const Function& target_function,
intptr_t* lower,
intptr_t* upper);
void DoMonomorphicMiss(const Object& data, const Function& target_function);
#if defined(DART_PRECOMPILED_RUNTIME)
void DoSingleTargetMiss(const SingleTargetCache& data,
const Function& target_function);
#endif // !defined(DART_PRECOMPILED_RUNTIME)
void DoICDataMiss(const ICData& data, const Function& target_function);
void DoMegamorphicMiss(const MegamorphicCache& data,
const Function& target_function);
Isolate* isolate_;
Thread* thread_;
Zone* zone_;
const Instance& receiver_;
NativeArguments arguments_;
StackFrame* caller_frame_;
const Code& caller_code_;
const Function& caller_function_;
// Call-site information populated during resolution.
String& name_;
Array& args_descriptor_;
bool is_monomorphic_hit_ = false;
};
void SwitchableCallHandler::DoUnlinkedCall(const UnlinkedCall& unlinked,
const Function& target_function) {
const String& name = String::Handle(zone_, unlinked.target_name());
const Array& descriptor =
Array::Handle(zone_, unlinked.arguments_descriptor());
const ICData& ic_data =
ICData::Handle(zone_, ICData::New(caller_function_, name, descriptor,
DeoptId::kNone, 1, /* args_tested */
ICData::kInstance));
if (!target_function.IsNull()) {
ic_data.AddReceiverCheck(receiver_.GetClassId(), target_function);
}
Object& object = Object::Handle(zone_, ic_data.raw());
Code& code = Code::Handle(zone_, StubCode::ICCallThroughCode().raw());
// If the target function has optional parameters or is generic, it's
// prologue requires ARGS_DESC_REG to be populated. Yet the switchable calls
// do not populate that on the call site, which is why we don't transition
// those call sites to monomorphic, but rather directly to call via stub
// (which will populate the ARGS_DESC_REG from the ICData).
//
// Because of this we also don't generate monomorphic checks for those
// functions.
if (!target_function.IsNull() &&
!target_function.PrologueNeedsArgumentsDescriptor()) {
// Patch to monomorphic call.
ASSERT(target_function.HasCode());
const Code& target_code =
Code::Handle(zone_, target_function.CurrentCode());
const Smi& expected_cid =
Smi::Handle(zone_, Smi::New(receiver_.GetClassId()));
if (unlinked.can_patch_to_monomorphic()) {
object = expected_cid.raw();
code = target_code.raw();
ASSERT(code.HasMonomorphicEntry());
} else {
object = MonomorphicSmiableCall::New(expected_cid.Value(), target_code);
code = StubCode::MonomorphicSmiableCheck().raw();
}
}
CodePatcher::PatchSwitchableCallAtWithMutatorsStopped(
thread_, caller_frame_->pc(), caller_code_, object, code);
// Return the ICData. The miss stub will jump to continue in the IC lookup
// stub.
arguments_.SetArgAt(0, StubCode::ICCallThroughCode());
arguments_.SetReturn(ic_data);
}
bool SwitchableCallHandler::CanExtendSingleTargetRange(
const String& name,
const Function& old_target,
const Function& target_function,
intptr_t* lower,
intptr_t* upper) {
if (old_target.raw() != target_function.raw()) {
return false;
}
intptr_t unchecked_lower, unchecked_upper;
if (receiver_.GetClassId() < *lower) {
unchecked_lower = receiver_.GetClassId();
unchecked_upper = *lower - 1;
*lower = receiver_.GetClassId();
} else {
unchecked_upper = receiver_.GetClassId();
unchecked_lower = *upper + 1;
*upper = receiver_.GetClassId();
}
return IsSingleTarget(isolate_, zone_, unchecked_lower, unchecked_upper,
target_function, name);
}
#if !defined(DART_PRECOMPILED_RUNTIME)
static ICDataPtr FindICDataForInstanceCall(Zone* zone,
const Code& code,
uword pc) {
uword pc_offset = pc - code.PayloadStart();
const PcDescriptors& descriptors =
PcDescriptors::Handle(zone, code.pc_descriptors());
PcDescriptors::Iterator iter(descriptors, PcDescriptorsLayout::kIcCall);
intptr_t deopt_id = -1;
while (iter.MoveNext()) {
if (iter.PcOffset() == pc_offset) {
deopt_id = iter.DeoptId();
break;
}
}
ASSERT(deopt_id != -1);
return Function::Handle(zone, code.function()).FindICData(deopt_id);
}
#endif // !defined(DART_PRECOMPILED_RUNTIME)
static FunctionPtr Resolve(Zone* zone,
const Class& receiver_class,
const String& name,
const Array& descriptor) {
ASSERT(name.IsSymbol());
ArgumentsDescriptor args_desc(descriptor);
Function& target_function =
Function::Handle(zone, Resolver::ResolveDynamicForReceiverClass(
receiver_class, name, args_desc));
if (target_function.IsNull()) {
target_function = InlineCacheMissHelper(receiver_class, descriptor, name);
if (target_function.IsNull()) {
ASSERT(!FLAG_lazy_dispatchers);
}
}
return target_function.raw();
}
void SwitchableCallHandler::DoMonomorphicMiss(const Object& data,
const Function& target_function) {
#if defined(DART_PRECOMPILED_RUNTIME)
classid_t old_expected_cid;
if (data.IsSmi()) {
old_expected_cid = Smi::Cast(data).Value();
} else {
RELEASE_ASSERT(data.IsMonomorphicSmiableCall());
old_expected_cid = MonomorphicSmiableCall::Cast(data).expected_cid();
}
const bool is_monomorphic_hit = old_expected_cid == receiver_.GetClassId();
const auto& old_receiver_class =
Class::Handle(zone_, isolate_->class_table()->At(old_expected_cid));
const auto& old_target = Function::Handle(
zone_, Resolve(zone_, old_receiver_class, name_, args_descriptor_));
const ICData& ic_data = ICData::Handle(
zone_, ICData::New(caller_function_, name_, args_descriptor_,
DeoptId::kNone, 1, /* args_tested */
ICData::kInstance));
// Add the first target.
if (!old_target.IsNull()) {
ic_data.AddReceiverCheck(old_expected_cid, old_target);
}
if (is_monomorphic_hit) {
// The site just have been updated to monomorphic state with same
// exact class id - do nothing in that case: stub will call through ic data.
arguments_.SetArgAt(0, StubCode::ICCallThroughCode());
arguments_.SetReturn(ic_data);
return;
}
intptr_t lower = old_expected_cid;
intptr_t upper = old_expected_cid;
if (CanExtendSingleTargetRange(name_, old_target, target_function, &lower,
&upper)) {
const SingleTargetCache& cache =
SingleTargetCache::Handle(zone_, SingleTargetCache::New());
const Code& code = Code::Handle(zone_, target_function.CurrentCode());
cache.set_target(code);
cache.set_entry_point(code.EntryPoint());
cache.set_lower_limit(lower);
cache.set_upper_limit(upper);
const Code& stub = StubCode::SingleTargetCall();
CodePatcher::PatchSwitchableCallAtWithMutatorsStopped(
thread_, caller_frame_->pc(), caller_code_, cache, stub);
// Return the ICData. The miss stub will jump to continue in the IC call
// stub.
arguments_.SetArgAt(0, StubCode::ICCallThroughCode());
arguments_.SetReturn(ic_data);
return;
}
// Patch to call through stub.
const Code& stub = StubCode::ICCallThroughCode();
CodePatcher::PatchSwitchableCallAtWithMutatorsStopped(
thread_, caller_frame_->pc(), caller_code_, ic_data, stub);
// Return the ICData. The miss stub will jump to continue in the IC lookup
// stub.
arguments_.SetArgAt(0, stub);
arguments_.SetReturn(ic_data);
#else // JIT
const ICData& ic_data = ICData::Handle(
zone_,
FindICDataForInstanceCall(zone_, caller_code_, caller_frame_->pc()));
RELEASE_ASSERT(!ic_data.IsNull());
ASSERT(ic_data.NumArgsTested() == 1);
const Code& stub = ic_data.is_tracking_exactness()
? StubCode::OneArgCheckInlineCacheWithExactnessCheck()
: StubCode::OneArgCheckInlineCache();
CodePatcher::PatchInstanceCallAtWithMutatorsStopped(
thread_, caller_frame_->pc(), caller_code_, ic_data, stub);
if (FLAG_trace_ic) {
OS::PrintErr("Instance call at %" Px
" switching to polymorphic dispatch, %s\n",
caller_frame_->pc(), ic_data.ToCString());
}
// ICData can be shared between unoptimized and optimized code, so beware that
// the new receiver class may have already been added through the optimized
// code.
if (!ic_data.HasReceiverClassId(receiver_.GetClassId())) {
GrowableArray<const Instance*> args(1);
args.Add(&receiver_);
// Don't count during insertion because the IC stub we continue through will
// do an increment.
InlineCacheMissHandlerGivenTargetFunction(args, ic_data, /*count=*/0,
target_function);
}
arguments_.SetArgAt(0, stub);
arguments_.SetReturn(ic_data);
#endif // defined(DART_PRECOMPILED_RUNTIME)
}
#if defined(DART_PRECOMPILED_RUNTIME)
void SwitchableCallHandler::DoSingleTargetMiss(
const SingleTargetCache& data,
const Function& target_function) {
const Code& old_target_code = Code::Handle(zone_, data.target());
const Function& old_target =
Function::Handle(zone_, Function::RawCast(old_target_code.owner()));
// We lost the original ICData when we patched to the monomorphic case.
const ICData& ic_data = ICData::Handle(
zone_, ICData::New(caller_function_, name_, args_descriptor_,
DeoptId::kNone, 1, /* args_tested */
ICData::kInstance));
if (!target_function.IsNull()) {
ic_data.AddReceiverCheck(receiver_.GetClassId(), target_function);
}
intptr_t lower = data.lower_limit();
intptr_t upper = data.upper_limit();
if (CanExtendSingleTargetRange(name_, old_target, target_function, &lower,
&upper)) {
data.set_lower_limit(lower);
data.set_upper_limit(upper);
// Return the ICData. The single target stub will jump to continue in the
// IC call stub.
arguments_.SetArgAt(0, StubCode::ICCallThroughCode());
arguments_.SetReturn(ic_data);
return;
}
// Call site is not single target, switch to call using ICData.
const Code& stub = StubCode::ICCallThroughCode();
CodePatcher::PatchSwitchableCallAtWithMutatorsStopped(
thread_, caller_frame_->pc(), caller_code_, ic_data, stub);
// Return the ICData. The single target stub will jump to continue in the
// IC call stub.
arguments_.SetArgAt(0, stub);
arguments_.SetReturn(ic_data);
}
#endif // !defined(DART_PRECOMPILED_RUNTIME)
void SwitchableCallHandler::DoICDataMiss(const ICData& ic_data,
const Function& target_function) {
const String& name = String::Handle(zone_, ic_data.target_name());
const Class& cls = Class::Handle(zone_, receiver_.clazz());
ASSERT(!cls.IsNull());
const Array& descriptor =
Array::CheckedHandle(zone_, ic_data.arguments_descriptor());
ArgumentsDescriptor args_desc(descriptor);
if (FLAG_trace_ic || FLAG_trace_ic_miss_in_optimized) {
OS::PrintErr("ICData miss, class=%s, function<%" Pd ">=%s\n",
cls.ToCString(), args_desc.TypeArgsLen(), name.ToCString());
}
if (target_function.IsNull()) {
arguments_.SetArgAt(0, StubCode::NoSuchMethodDispatcher());
arguments_.SetReturn(ic_data);
return;
}
const intptr_t number_of_checks = ic_data.NumberOfChecks();
if ((number_of_checks == 0) &&
(!FLAG_precompiled_mode || ic_data.receiver_cannot_be_smi()) &&
!target_function.PrologueNeedsArgumentsDescriptor()) {
// This call site is unlinked: transition to a monomorphic direct call.
// Note we cannot do this if the target has optional parameters because
// the monomorphic direct call does not load the arguments descriptor.
// We cannot do this if we are still in the middle of precompiling because
// the monomorphic case hides a live instance selector from the
// treeshaker.
const Code& target_code =
Code::Handle(zone_, target_function.EnsureHasCode());
const Smi& expected_cid =
Smi::Handle(zone_, Smi::New(receiver_.GetClassId()));
ASSERT(target_code.HasMonomorphicEntry());
CodePatcher::PatchSwitchableCallAtWithMutatorsStopped(
thread_, caller_frame_->pc(), caller_code_, expected_cid, target_code);
arguments_.SetArgAt(0, target_code);
arguments_.SetReturn(expected_cid);
} else {
// IC entry might have been added while we waited to get into runtime.
GrowableArray<intptr_t> class_ids(1);
class_ids.Add(receiver_.GetClassId());
if (ic_data.FindCheck(class_ids) == -1) {
ic_data.AddReceiverCheck(receiver_.GetClassId(), target_function);
}
if (number_of_checks > FLAG_max_polymorphic_checks) {
// Switch to megamorphic call.
const MegamorphicCache& cache = MegamorphicCache::Handle(
zone_, MegamorphicCacheTable::Lookup(thread_, name, descriptor));
const Code& stub = StubCode::MegamorphicCall();
CodePatcher::PatchSwitchableCallAtWithMutatorsStopped(
thread_, caller_frame_->pc(), caller_code_, cache, stub);
arguments_.SetArgAt(0, stub);
arguments_.SetReturn(cache);
} else {
arguments_.SetArgAt(0, StubCode::ICCallThroughCode());
arguments_.SetReturn(ic_data);
}
}
}
void SwitchableCallHandler::DoMegamorphicMiss(const MegamorphicCache& data,
const Function& target_function) {
const String& name = String::Handle(zone_, data.target_name());
const Class& cls = Class::Handle(zone_, receiver_.clazz());
ASSERT(!cls.IsNull());
const Array& descriptor =
Array::CheckedHandle(zone_, data.arguments_descriptor());
ArgumentsDescriptor args_desc(descriptor);
if (FLAG_trace_ic || FLAG_trace_ic_miss_in_optimized) {
OS::PrintErr("Megamorphic miss, class=%s, function<%" Pd ">=%s\n",
cls.ToCString(), args_desc.TypeArgsLen(), name.ToCString());
}
if (target_function.IsNull()) {
arguments_.SetArgAt(0, StubCode::NoSuchMethodDispatcher());
arguments_.SetReturn(data);
return;
}
// Insert function found into cache.
const Smi& class_id = Smi::Handle(zone_, Smi::New(cls.id()));
data.Insert(class_id, target_function);
arguments_.SetArgAt(0, StubCode::MegamorphicCall());
arguments_.SetReturn(data);
}
FunctionPtr SwitchableCallHandler::ResolveTargetFunction(const Object& data) {
switch (data.GetClassId()) {
case kUnlinkedCallCid: {
const auto& unlinked_call = UnlinkedCall::Cast(data);
#if defined(DART_PRECOMPILED_RUNTIME)
// When transitioning out of UnlinkedCall to other states (e.g.
// Monomorphic, MonomorphicSmiable, SingleTarget) we lose
// name/arg-descriptor in AOT mode and cannot recover it.
//
// Even if we could recover an old target function (which was missed) -
// which we cannot in AOT bare mode - we can still lose the name due to a
// dyn:* call site potentially targeting non-dyn:* targets.
//
// => We will therefore retain the unlinked call here.
//
// In JIT mode we always use ICData from the call site, which has the
// correct name/args-descriptor.
SaveUnlinkedCall(zone_, isolate_, caller_frame_->pc(), unlinked_call);
#endif // defined(DART_PRECOMPILED_RUNTIME)
name_ = unlinked_call.target_name();
args_descriptor_ = unlinked_call.arguments_descriptor();
break;
}
case kMonomorphicSmiableCallCid:
FALL_THROUGH;
#if defined(DART_PRECOMPILED_RUNTIME)
case kSmiCid:
FALL_THROUGH;
case kSingleTargetCacheCid: {
const auto& unlinked_call = UnlinkedCall::Handle(
zone_, LoadUnlinkedCall(zone_, isolate_, caller_frame_->pc()));
name_ = unlinked_call.target_name();
args_descriptor_ = unlinked_call.arguments_descriptor();
break;
}
#else
case kArrayCid: {
// ICData three-element array: Smi(receiver CID), Smi(count),
// Function(target). It is the Array from ICData::entries_.
const auto& ic_data = ICData::Handle(
zone_,
FindICDataForInstanceCall(zone_, caller_code_, caller_frame_->pc()));
RELEASE_ASSERT(!ic_data.IsNull());
name_ = ic_data.target_name();
args_descriptor_ = ic_data.arguments_descriptor();
break;
}
#endif // defined(DART_PRECOMPILED_RUNTIME)
case kICDataCid:
FALL_THROUGH;
case kMegamorphicCacheCid: {
const CallSiteData& call_site_data = CallSiteData::Cast(data);
name_ = call_site_data.target_name();
args_descriptor_ = call_site_data.arguments_descriptor();
break;
}
default:
UNREACHABLE();
}
const Class& cls = Class::Handle(zone_, receiver_.clazz());
return Resolve(zone_, cls, name_, args_descriptor_);
}
void SwitchableCallHandler::HandleMiss(const Object& old_data,
const Code& old_code,
const Function& target_function) {
switch (old_data.GetClassId()) {
case kUnlinkedCallCid:
ASSERT(old_code.raw() == StubCode::SwitchableCallMiss().raw());
DoUnlinkedCall(UnlinkedCall::Cast(old_data), target_function);
break;
case kMonomorphicSmiableCallCid:
ASSERT(old_code.raw() == StubCode::MonomorphicSmiableCheck().raw());
FALL_THROUGH;
#if defined(DART_PRECOMPILED_RUNTIME)
case kSmiCid:
DoMonomorphicMiss(old_data, target_function);
break;
case kSingleTargetCacheCid:
ASSERT(old_code.raw() == StubCode::SingleTargetCall().raw());
DoSingleTargetMiss(SingleTargetCache::Cast(old_data), target_function);
break;
#else
case kArrayCid:
// ICData three-element array: Smi(receiver CID), Smi(count),
// Function(target). It is the Array from ICData::entries_.
DoMonomorphicMiss(old_data, target_function);
break;
#endif // !defined(DART_PRECOMPILED_RUNTIME)
case kICDataCid:
ASSERT(old_code.raw() == StubCode::ICCallThroughCode().raw());
DoICDataMiss(ICData::Cast(old_data), target_function);
break;
case kMegamorphicCacheCid:
ASSERT(old_code.raw() == StubCode::MegamorphicCall().raw());
DoMegamorphicMiss(MegamorphicCache::Cast(old_data), target_function);
break;
default:
UNREACHABLE();
}
}
// Handle the first use of an instance call
// Arg1: Receiver.
// Arg0: Stub out.
// Returns: the ICData used to continue with the call.
DEFINE_RUNTIME_ENTRY(SwitchableCallMiss, 2) {
const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(1));
StackFrameIterator iterator(ValidationPolicy::kDontValidateFrames, thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* exit_frame = iterator.NextFrame();
ASSERT(exit_frame->IsExitFrame());
StackFrame* miss_handler_frame = iterator.NextFrame();
// This runtime entry can be called either from miss stub or from
// switchable_call_miss "dart" stub/function set up in
// [MegamorphicCacheTable::InitMissHandler].
ASSERT(miss_handler_frame->IsStubFrame() ||
miss_handler_frame->IsDartFrame());
StackFrame* caller_frame = iterator.NextFrame();
ASSERT(caller_frame->IsDartFrame());
const Code& caller_code = Code::Handle(zone, caller_frame->LookupDartCode());
const Function& caller_function =
Function::Handle(zone, caller_frame->LookupDartFunction());
Object& old_data = Object::Handle(zone);
Code& old_code = Code::Handle(zone);
#if defined(DART_PRECOMPILED_RUNTIME)
// Grab old_data and do potentially long-running step of resolving the
// target function before we stop mutators.
// This will reduce amount of time spent with all mutators are stopped
// hopefully leaving only code patching to be done then.
old_data =
CodePatcher::GetSwitchableCallDataAt(caller_frame->pc(), caller_code);
#else
old_code ^= CodePatcher::GetInstanceCallAt(caller_frame->pc(), caller_code,
&old_data);
#endif
SwitchableCallHandler handler(thread, receiver, arguments, caller_frame,
caller_code, caller_function);
const Function& target_function =
Function::Handle(zone, handler.ResolveTargetFunction(old_data));
thread->isolate_group()->RunWithStoppedMutators(
[&]() {
#if defined(DART_PRECOMPILED_RUNTIME)
old_data = CodePatcher::GetSwitchableCallDataAt(caller_frame->pc(),
caller_code);
#if defined(DEBUG)
old_code ^= CodePatcher::GetSwitchableCallTargetAt(caller_frame->pc(),
caller_code);
#endif
#else
old_code ^= CodePatcher::GetInstanceCallAt(caller_frame->pc(),
caller_code, &old_data);
#endif
handler.HandleMiss(old_data, old_code, target_function);
},
/*use_force_growth=*/true);
}
// Handles interpreted interface call cache miss.
// Arg0: receiver
// Arg1: target name
// Arg2: arguments descriptor
// Returns: target function (can only be null if !FLAG_lazy_dispatchers)
// Modifies the instance call table in current interpreter.
DEFINE_RUNTIME_ENTRY(InterpretedInstanceCallMissHandler, 3) {
#if defined(DART_PRECOMPILED_RUNTIME)
UNREACHABLE();
#else
ASSERT(FLAG_enable_interpreter);
const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
const String& target_name = String::CheckedHandle(zone, arguments.ArgAt(1));
const Array& arg_desc = Array::CheckedHandle(zone, arguments.ArgAt(2));
ArgumentsDescriptor arguments_descriptor(arg_desc);
Function& target_function = Function::Handle(
zone,
Resolver::ResolveDynamic(receiver, target_name, arguments_descriptor));
// TODO(regis): In order to substitute 'simple_instance_of_function', the 2nd
// arg to the call, the type, is needed.
if (target_function.IsNull()) {
const Class& receiver_class = Class::Handle(zone, receiver.clazz());
target_function =
InlineCacheMissHelper(receiver_class, arg_desc, target_name);
}
ASSERT(!target_function.IsNull() || !FLAG_lazy_dispatchers);
arguments.SetReturn(target_function);
#endif
}
// Used to find the correct receiver and function to invoke or to fall back to
// invoking noSuchMethod when lazy dispatchers are disabled. Returns the
// result of the invocation or an Error.
static ObjectPtr InvokeCallThroughGetterOrNoSuchMethod(
Zone* zone,
const Instance& receiver,
const String& target_name,
const Array& orig_arguments,
const Array& orig_arguments_desc) {
ASSERT(!FLAG_lazy_dispatchers);
const bool is_dynamic_call =
Function::IsDynamicInvocationForwarderName(target_name);
String& demangled_target_name = String::Handle(zone, target_name.raw());
if (is_dynamic_call) {
demangled_target_name =
Function::DemangleDynamicInvocationForwarderName(target_name);
}
Class& cls = Class::Handle(zone, receiver.clazz());
Function& function = Function::Handle(zone);
// Dart distinguishes getters and regular methods and allows their calls
// to mix with conversions, and its selectors are independent of arity. So do
// a zigzagged lookup to see if this call failed because of an arity mismatch,
// need for conversion, or there really is no such method.
const bool is_getter = Field::IsGetterName(demangled_target_name);
if (is_getter) {
// Tear-off of a method
// o.foo (o.get:foo) failed, closurize o.foo() if it exists.
const auto& function_name =
String::Handle(zone, Field::NameFromGetter(demangled_target_name));
while (!cls.IsNull()) {
// We don't generate dyn:* forwarders for method extractors so there is no
// need to try to find a dyn:get:foo first (see assertion below)
if (function.IsNull()) {
function = cls.LookupDynamicFunction(function_name);
}
if (!function.IsNull()) {
#if !defined(DART_PRECOMPILED_RUNTIME)
ASSERT(!kernel::NeedsDynamicInvocationForwarder(Function::Handle(
function.GetMethodExtractor(demangled_target_name))));
#endif
const Function& closure_function =
Function::Handle(zone, function.ImplicitClosureFunction());
const Object& result = Object::Handle(
zone, closure_function.ImplicitInstanceClosure(receiver));
return result.raw();
}
cls = cls.SuperClass();
}
// Fall through for noSuchMethod
} else {
// Call through field.
// o.foo(...) failed, invoke noSuchMethod is foo exists but has the wrong
// number of arguments, or try (o.foo).call(...)
if ((target_name.raw() == Symbols::Call().raw()) && receiver.IsClosure()) {
// Special case: closures are implemented with a call getter instead of a
// call method and with lazy dispatchers the field-invocation-dispatcher
// would perform the closure call.
auto& result = Object::Handle(
zone,
DartEntry::ResolveCallable(orig_arguments, orig_arguments_desc));
if (result.IsError()) {
return result.raw();
}
function ^= result.raw();
if (is_dynamic_call && !function.IsNull() &&
!function.CanReceiveDynamicInvocation()) {
ArgumentsDescriptor args_desc(orig_arguments_desc);
result = function.DoArgumentTypesMatch(orig_arguments, args_desc);
if (result.IsError()) {
return result.raw();
}
}
result = DartEntry::InvokeCallable(function, orig_arguments,
orig_arguments_desc);
return result.raw();
}
// Dynamic call sites have to use the dynamic getter as well (if it was
// created).
const auto& getter_name =
String::Handle(zone, Field::GetterName(demangled_target_name));
const auto& dyn_getter_name = String::Handle(
zone, is_dynamic_call
? Function::CreateDynamicInvocationForwarderName(getter_name)
: getter_name.raw());
ArgumentsDescriptor args_desc(orig_arguments_desc);
while (!cls.IsNull()) {
// If there is a function with the target name but mismatched arguments
// we need to call `receiver.noSuchMethod()`.
function = cls.LookupDynamicFunction(target_name);
if (!function.IsNull()) {
ASSERT(!function.AreValidArguments(args_desc, NULL));
break; // mismatch, invoke noSuchMethod
}
if (is_dynamic_call) {
function = cls.LookupDynamicFunction(demangled_target_name);
if (!function.IsNull()) {
ASSERT(!function.AreValidArguments(args_desc, NULL));
break; // mismatch, invoke noSuchMethod
}
}
// If there is a getter we need to call-through-getter.
if (is_dynamic_call) {
function = cls.LookupDynamicFunction(dyn_getter_name);
}
if (function.IsNull()) {
function = cls.LookupDynamicFunction(getter_name);
}
if (!function.IsNull()) {
const Array& getter_arguments = Array::Handle(Array::New(1));
getter_arguments.SetAt(0, receiver);
const Object& getter_result = Object::Handle(
zone, DartEntry::InvokeFunction(function, getter_arguments));
if (getter_result.IsError()) {
return getter_result.raw();
}
ASSERT(getter_result.IsNull() || getter_result.IsInstance());
orig_arguments.SetAt(args_desc.FirstArgIndex(), getter_result);
auto& result = Object::Handle(
zone,
DartEntry::ResolveCallable(orig_arguments, orig_arguments_desc));
if (result.IsError()) {
return result.raw();
}
function ^= result.raw();
if (is_dynamic_call && !function.IsNull() &&
!function.CanReceiveDynamicInvocation()) {
result = function.DoArgumentTypesMatch(orig_arguments, args_desc);
if (result.IsError()) {
return result.raw();
}
}
result = DartEntry::InvokeCallable(function, orig_arguments,
orig_arguments_desc);
return result.raw();
}
cls = cls.SuperClass();
}
}
const Object& result = Object::Handle(
zone, DartEntry::InvokeNoSuchMethod(receiver, demangled_target_name,
orig_arguments, orig_arguments_desc));
return result.raw();
}
// Invoke appropriate noSuchMethod or closure from getter.
// Arg0: receiver
// Arg1: ICData or MegamorphicCache
// Arg2: arguments descriptor array
// Arg3: arguments array
DEFINE_RUNTIME_ENTRY(NoSuchMethodFromCallStub, 4) {
ASSERT(!FLAG_lazy_dispatchers);
const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
const Object& ic_data_or_cache = Object::Handle(zone, arguments.ArgAt(1));
const Array& orig_arguments_desc =
Array::CheckedHandle(zone, arguments.ArgAt(2));
const Array& orig_arguments = Array::CheckedHandle(zone, arguments.ArgAt(3));
String& target_name = String::Handle(zone);
if (ic_data_or_cache.IsICData()) {
target_name = ICData::Cast(ic_data_or_cache).target_name();
} else {
ASSERT(ic_data_or_cache.IsMegamorphicCache());
target_name = MegamorphicCache::Cast(ic_data_or_cache).target_name();
}
const auto& result = Object::Handle(
zone,
InvokeCallThroughGetterOrNoSuchMethod(
zone, receiver, target_name, orig_arguments, orig_arguments_desc));
ThrowIfError(result);
arguments.SetReturn(result);
}
// Invoke appropriate noSuchMethod function.
// Arg0: receiver
// Arg1: function
// Arg1: arguments descriptor array.
// Arg3: arguments array.
DEFINE_RUNTIME_ENTRY(NoSuchMethodFromPrologue, 4) {
const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
const Function& function = Function::CheckedHandle(zone, arguments.ArgAt(1));
const Array& orig_arguments_desc =
Array::CheckedHandle(zone, arguments.ArgAt(2));
const Array& orig_arguments = Array::CheckedHandle(zone, arguments.ArgAt(3));
String& orig_function_name = String::Handle(zone);
if ((function.kind() == FunctionLayout::kClosureFunction) ||
(function.kind() == FunctionLayout::kImplicitClosureFunction)) {
// For closure the function name is always 'call'. Replace it with the
// name of the closurized function so that exception contains more
// relevant information.
orig_function_name = function.QualifiedUserVisibleName();
} else {
orig_function_name = function.name();
}
const Object& result = Object::Handle(
zone, DartEntry::InvokeNoSuchMethod(receiver, orig_function_name,
orig_arguments, orig_arguments_desc));
ThrowIfError(result);
arguments.SetReturn(result);
}
// Invoke appropriate noSuchMethod function (or in the case of no lazy
// dispatchers, walk the receiver to find the correct method to call).
// Arg0: receiver
// Arg1: function name.
// Arg2: arguments descriptor array.
// Arg3: arguments array.
DEFINE_RUNTIME_ENTRY(InvokeNoSuchMethod, 4) {
ASSERT(FLAG_enable_interpreter);
const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
const String& original_function_name =
String::CheckedHandle(zone, arguments.ArgAt(1));
const Array& orig_arguments_desc =
Array::CheckedHandle(zone, arguments.ArgAt(2));
const Array& orig_arguments = Array::CheckedHandle(zone, arguments.ArgAt(3));
auto& result = Object::Handle(zone);
if (!FLAG_lazy_dispatchers) {
// Failing to find the method could be due to the lack of lazy invoke field
// dispatchers, so attempt a deeper search before calling noSuchMethod.
result = InvokeCallThroughGetterOrNoSuchMethod(
zone, receiver, original_function_name, orig_arguments,
orig_arguments_desc);
} else {
result = DartEntry::InvokeNoSuchMethod(receiver, original_function_name,
orig_arguments, orig_arguments_desc);
}
ThrowIfError(result);
arguments.SetReturn(result);
}
#if !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME)
// The following code is used to stress test
// - deoptimization
// - debugger stack tracing
// - garbage collection
// - hot reload
static void HandleStackOverflowTestCases(Thread* thread) {
Isolate* isolate = thread->isolate();
if (FLAG_shared_slow_path_triggers_gc) {
isolate->heap()->CollectAllGarbage();
}
bool do_deopt = false;
bool do_stacktrace = false;
bool do_reload = false;
bool do_gc = false;
const intptr_t isolate_reload_every =
isolate->reload_every_n_stack_overflow_checks();
if ((FLAG_deoptimize_every > 0) || (FLAG_stacktrace_every > 0) ||
(FLAG_gc_every > 0) || (isolate_reload_every > 0)) {
if (!Isolate::IsVMInternalIsolate(isolate)) {
// TODO(turnidge): To make --deoptimize_every and
// --stacktrace-every faster we could move this increment/test to
// the generated code.
int32_t count = thread->IncrementAndGetStackOverflowCount();
if (FLAG_deoptimize_every > 0 && (count % FLAG_deoptimize_every) == 0) {
do_deopt = true;
}
if (FLAG_stacktrace_every > 0 && (count % FLAG_stacktrace_every) == 0) {
do_stacktrace = true;
}
if (FLAG_gc_every > 0 && (count % FLAG_gc_every) == 0) {
do_gc = true;
}
if ((isolate_reload_every > 0) && (count % isolate_reload_every) == 0) {
do_reload = isolate->CanReload();
}
}
}
if ((FLAG_deoptimize_filter != nullptr) ||
(FLAG_stacktrace_filter != nullptr) || (FLAG_reload_every != 0)) {
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* frame = iterator.NextFrame();
ASSERT(frame != nullptr);
Code& code = Code::Handle();
Function& function = Function::Handle();
if (frame->is_interpreted()) {
function = frame->LookupDartFunction();
} else {
code = frame->LookupDartCode();
ASSERT(!code.IsNull());
function = code.function();
}
ASSERT(!function.IsNull());
const char* function_name = nullptr;
if ((FLAG_deoptimize_filter != nullptr) ||
(FLAG_stacktrace_filter != nullptr)) {
function_name = function.ToFullyQualifiedCString();
ASSERT(function_name != nullptr);
}
if (!code.IsNull()) {
if (!code.is_optimized() && FLAG_reload_every_optimized) {
// Don't do the reload if we aren't inside optimized code.
do_reload = false;
}
if (code.is_optimized() && FLAG_deoptimize_filter != nullptr &&
strstr(function_name, FLAG_deoptimize_filter) != nullptr &&
!function.ForceOptimize()) {
OS::PrintErr("*** Forcing deoptimization (%s)\n",
function.ToFullyQualifiedCString());
do_deopt = true;
}
}
if (FLAG_stacktrace_filter != nullptr &&
strstr(function_name, FLAG_stacktrace_filter) != nullptr) {
OS::PrintErr("*** Computing stacktrace (%s)\n",
function.ToFullyQualifiedCString());
do_stacktrace = true;
}
}
if (do_deopt) {
// TODO(turnidge): Consider using DeoptimizeAt instead.
DeoptimizeFunctionsOnStack();
}
if (do_reload) {
JSONStream js;
// Maybe adjust the rate of future reloads.
isolate->MaybeIncreaseReloadEveryNStackOverflowChecks();
const char* script_uri;
{
NoReloadScope no_reload(isolate, thread);
const Library& lib =
Library::Handle(isolate->object_store()->_internal_library());
const Class& cls = Class::Handle(
lib.LookupClass(String::Handle(String::New("VMLibraryHooks"))));
const Function& func = Function::Handle(cls.LookupFunction(
String::Handle(String::New("get:platformScript"))));
Object& result = Object::Handle(
DartEntry::InvokeFunction(func, Object::empty_array()));
if (result.IsUnwindError()) {
Exceptions::PropagateError(Error::Cast(result));
}
if (!result.IsInstance()) {
FATAL1("Bad script uri hook: %s", result.ToCString());
}
result = DartLibraryCalls::ToString(Instance::Cast(result));
if (result.IsUnwindError()) {
Exceptions::PropagateError(Error::Cast(result));
}
if (!result.IsString()) {
FATAL1("Bad script uri hook: %s", result.ToCString());
}
script_uri = result.ToCString(); // Zone allocated.
}
// Issue a reload.
bool success = isolate->group()->ReloadSources(&js, true /* force_reload */,
script_uri);
if (!success) {
FATAL1("*** Isolate reload failed:\n%s\n", js.ToCString());
}
}
if (do_stacktrace) {
String& var_name = String::Handle();
Instance& var_value = Instance::Handle();
DebuggerStackTrace* stack = isolate->debugger()->StackTrace();
intptr_t num_frames = stack->Length();
for (intptr_t i = 0; i < num_frames; i++) {
ActivationFrame* frame = stack->FrameAt(i);
int num_vars = 0;
// Variable locations and number are unknown when precompiling.
#if !defined(DART_PRECOMPILED_RUNTIME)
// NumLocalVariables() can call EnsureHasUnoptimizedCode() for
// non-interpreted functions.
if (!frame->function().ForceOptimize()) {
if (!frame->IsInterpreted()) {
// Ensure that we have unoptimized code.
frame->function().EnsureHasCompiledUnoptimizedCode();
}
num_vars = frame->NumLocalVariables();
}
#endif
TokenPosition unused = TokenPosition::kNoSource;
for (intptr_t v = 0; v < num_vars; v++) {
frame->VariableAt(v, &var_name, &unused, &unused, &unused, &var_value);
}
}
if (FLAG_stress_async_stacks) {
isolate->debugger()->CollectAwaiterReturnStackTrace();
}
}
if (do_gc) {
isolate->heap()->CollectAllGarbage(Heap::kDebugging);
}
}
#endif // !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME)
#if !defined(DART_PRECOMPILED_RUNTIME)
static void HandleOSRRequest(Thread* thread) {
Isolate* isolate = thread->isolate();
ASSERT(isolate->use_osr());
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* frame = iterator.NextFrame();
ASSERT(frame != NULL);
const Code& code = Code::ZoneHandle(frame->LookupDartCode());
ASSERT(!code.IsNull());
ASSERT(!code.is_optimized());
const Function& function = Function::Handle(code.function());
ASSERT(!function.IsNull());
// If the code of the frame does not match the function's unoptimized code,
// we bail out since the code was reset by an isolate reload.
if (code.raw() != function.unoptimized_code()) {
return;
}
// Since the code is referenced from the frame and the ZoneHandle,
// it cannot have been removed from the function.
ASSERT(function.HasCode());
// Don't do OSR on intrinsified functions: The intrinsic code expects to be
// called like a regular function and can't be entered via OSR.
if (!Compiler::CanOptimizeFunction(thread, function) ||
function.is_intrinsic()) {
return;
}
// The unoptimized code is on the stack and should never be detached from
// the function at this point.
ASSERT(function.unoptimized_code() != Object::null());
intptr_t osr_id =
Code::Handle(function.unoptimized_code()).GetDeoptIdForOsr(frame->pc());
ASSERT(osr_id != Compiler::kNoOSRDeoptId);
if (FLAG_trace_osr) {
OS::PrintErr("Attempting OSR for %s at id=%" Pd ", count=%" Pd "\n",
function.ToFullyQualifiedCString(), osr_id,
function.usage_counter());
}
// Since the code is referenced from the frame and the ZoneHandle,
// it cannot have been removed from the function.
const Object& result = Object::Handle(
Compiler::CompileOptimizedFunction(thread, function, osr_id));
ThrowIfError(result);
if (!result.IsNull()) {
const Code& code = Code::Cast(result);
uword optimized_entry = code.EntryPoint();
frame->set_pc(optimized_entry);
frame->set_pc_marker(code.raw());
}
}
#endif // !defined(DART_PRECOMPILED_RUNTIME)
DEFINE_RUNTIME_ENTRY(AllocateMint, 0) {
if (FLAG_shared_slow_path_triggers_gc) {
isolate->heap()->CollectAllGarbage();
}
constexpr uint64_t val = 0x7fffffff7fffffff;
ASSERT(!Smi::IsValid(static_cast<int64_t>(val)));
const auto& integer_box = Integer::Handle(zone, Integer::NewFromUint64(val));
arguments.SetReturn(integer_box);
};
DEFINE_RUNTIME_ENTRY(StackOverflow, 0) {
#if defined(USING_SIMULATOR)
uword stack_pos = Simulator::Current()->get_sp();
// If simulator was never called (for example, in pure
// interpreted mode) it may return 0 as a value of SPREG.
if (stack_pos == 0) {
// Use any reasonable value which would not be treated
// as stack overflow.
stack_pos = thread->saved_stack_limit();
}
#else
uword stack_pos = OSThread::GetCurrentStackPointer();
#endif
// Always clear the stack overflow flags. They are meant for this
// particular stack overflow runtime call and are not meant to
// persist.
uword stack_overflow_flags = thread->GetAndClearStackOverflowFlags();
bool interpreter_stack_overflow = false;
#if !defined(DART_PRECOMPILED_RUNTIME)
if (FLAG_enable_interpreter) {
// Do not allocate an interpreter, if none is allocated yet.
Interpreter* interpreter = thread->interpreter();
if (interpreter != NULL) {
interpreter_stack_overflow =
interpreter->get_sp() >= interpreter->overflow_stack_limit();
}
}
#endif // !defined(DART_PRECOMPILED_RUNTIME)
// If an interrupt happens at the same time as a stack overflow, we
// process the stack overflow now and leave the interrupt for next
// time.
if (interpreter_stack_overflow || !thread->os_thread()->HasStackHeadroom() ||
IsCalleeFrameOf(thread->saved_stack_limit(), stack_pos)) {
if (FLAG_verbose_stack_overflow) {
OS::PrintErr("Stack overflow in %s\n",
interpreter_stack_overflow ? "interpreter" : "native code");
OS::PrintErr(" Native SP = %" Px ", stack limit = %" Px "\n", stack_pos,
thread->saved_stack_limit());
#if !defined(DART_PRECOMPILED_RUNTIME)
if (thread->interpreter() != nullptr) {
OS::PrintErr(" Interpreter SP = %" Px ", stack limit = %" Px "\n",
thread->interpreter()->get_sp(),
thread->interpreter()->overflow_stack_limit());
}
#endif // !defined(DART_PRECOMPILED_RUNTIME)
OS::PrintErr("Call stack:\n");
OS::PrintErr("size | frame\n");
StackFrameIterator frames(ValidationPolicy::kDontValidateFrames, thread,
StackFrameIterator::kNoCrossThreadIteration);
uword fp = stack_pos;
StackFrame* frame = frames.NextFrame();
while (frame != NULL) {
if (frame->is_interpreted() == interpreter_stack_overflow) {
uword delta = interpreter_stack_overflow ? (fp - frame->fp())
: (frame->fp() - fp);
fp = frame->fp();
OS::PrintErr("%4" Pd " %s\n", delta, frame->ToCString());
} else {
OS::PrintErr(" %s\n", frame->ToCString());
}
frame = frames.NextFrame();
}
}
// Use the preallocated stack overflow exception to avoid calling
// into dart code.
const Instance& exception =
Instance::Handle(isolate->object_store()->stack_overflow());
Exceptions::Throw(thread, exception);
UNREACHABLE();
}
#if !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME)
HandleStackOverflowTestCases(thread);
#endif // !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME)
// Handle interrupts:
// - store buffer overflow
// - OOB message (vm-service or dart:isolate)
const Error& error = Error::Handle(thread->HandleInterrupts());
ThrowIfError(error);
#if !defined(DART_PRECOMPILED_RUNTIME)
if ((stack_overflow_flags & Thread::kOsrRequest) != 0) {
HandleOSRRequest(thread);
}
#else
ASSERT((stack_overflow_flags & Thread::kOsrRequest) == 0);
#endif // !defined(DART_PRECOMPILED_RUNTIME)
}
DEFINE_RUNTIME_ENTRY(TraceICCall, 2) {
const ICData& ic_data = ICData::CheckedHandle(zone, arguments.ArgAt(0));
const Function& function = Function::CheckedHandle(zone, arguments.ArgAt(1));
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* frame = iterator.NextFrame();
ASSERT(frame != NULL);
OS::PrintErr(
"IC call @%#" Px ": ICData: %#" Px " cnt:%" Pd " nchecks: %" Pd " %s\n",
frame->pc(), static_cast<uword>(ic_data.raw()), function.usage_counter(),
ic_data.NumberOfChecks(), function.ToFullyQualifiedCString());
}
// This is called from interpreter when function usage counter reached
// compilation threshold and function needs to be compiled.
DEFINE_RUNTIME_ENTRY(CompileInterpretedFunction, 1) {
#if !defined(DART_PRECOMPILED_RUNTIME)
const Function& function = Function::CheckedHandle(zone, arguments.ArgAt(0));
ASSERT(!function.IsNull());
ASSERT(FLAG_enable_interpreter);
#if !defined(PRODUCT)
if (Debugger::IsDebugging(thread, function)) {
return;
}
#endif // !defined(PRODUCT)
if (FLAG_background_compilation) {
if (!BackgroundCompiler::IsDisabled(isolate,
/* optimizing_compilation = */ false) &&
function.is_background_optimizable()) {
// Ensure background compiler is running, if not start it.
BackgroundCompiler::Start(isolate);
// Reduce the chance of triggering a compilation while the function is
// being compiled in the background. INT32_MIN should ensure that it
// takes long time to trigger a compilation.
// Note that the background compilation queue rejects duplicate entries.
function.SetUsageCounter(INT32_MIN);
isolate->background_compiler()->Compile(function);
return;
}
}
// Reset usage counter for future optimization.
function.SetUsageCounter(0);
Object& result =
Object::Handle(zone, Compiler::CompileFunction(thread, function));
ThrowIfError(result);
#else
UNREACHABLE();
#endif // !DART_PRECOMPILED_RUNTIME
}
// This is called from function that needs to be optimized.
// The requesting function can be already optimized (reoptimization).
// Returns the Code object where to continue execution.
DEFINE_RUNTIME_ENTRY(OptimizeInvokedFunction, 1) {
#if !defined(DART_PRECOMPILED_RUNTIME)
const Function& function = Function::CheckedHandle(zone, arguments.ArgAt(0));
ASSERT(!function.IsNull());
ASSERT(function.HasCode());
if (Compiler::CanOptimizeFunction(thread, function)) {
if (FLAG_background_compilation) {
Field& field = Field::Handle(zone, isolate->GetDeoptimizingBoxedField());
while (!field.IsNull()) {
if (FLAG_trace_optimization || FLAG_trace_field_guards) {
THR_Print("Lazy disabling unboxing of %s\n", field.ToCString());
}
field.set_is_unboxing_candidate(false);
field.DeoptimizeDependentCode();
// Get next field.
field = isolate->GetDeoptimizingBoxedField();
}
if (!BackgroundCompiler::IsDisabled(isolate,
/* optimizing_compiler = */ true) &&
function.is_background_optimizable()) {
// Ensure background compiler is running, if not start it.
BackgroundCompiler::Start(isolate);
// Reduce the chance of triggering a compilation while the function is
// being compiled in the background. INT32_MIN should ensure that it
// takes long time to trigger a compilation.
// Note that the background compilation queue rejects duplicate entries.
function.SetUsageCounter(INT32_MIN);
isolate->optimizing_background_compiler()->Compile(function);
// Continue in the same code.
arguments.SetReturn(function);
return;
}
}
// Reset usage counter for reoptimization before calling optimizer to
// prevent recursive triggering of function optimization.
function.SetUsageCounter(0);
if (FLAG_trace_compiler || FLAG_trace_optimizing_compiler) {
if (function.HasOptimizedCode()) {
THR_Print("ReCompiling function: '%s' \n",
function.ToFullyQualifiedCString());
}
}
Object& result = Object::Handle(
zone, Compiler::CompileOptimizedFunction(thread, function));
ThrowIfError(result);
}
arguments.SetReturn(function);
#else
UNREACHABLE();
#endif // !DART_PRECOMPILED_RUNTIME
}
// The caller must be a static call in a Dart frame, or an entry frame.
// Patch static call to point to valid code's entry point.
DEFINE_RUNTIME_ENTRY(FixCallersTarget, 0) {
#if !defined(DART_PRECOMPILED_RUNTIME)
StackFrameIterator iterator(ValidationPolicy::kDontValidateFrames, thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* frame = iterator.NextFrame();
ASSERT(frame != NULL);
while (frame->IsStubFrame() || frame->IsExitFrame()) {
frame = iterator.NextFrame();
ASSERT(frame != NULL);
}
if (frame->IsEntryFrame()) {
// Since function's current code is always unpatched, the entry frame always
// calls to unpatched code.
UNREACHABLE();
}
ASSERT(frame->IsDartFrame());
const Code& caller_code = Code::Handle(zone, frame->LookupDartCode());
RELEASE_ASSERT(caller_code.is_optimized());
const Function& target_function = Function::Handle(
zone, caller_code.GetStaticCallTargetFunctionAt(frame->pc()));
const Code& current_target_code =
Code::Handle(zone, target_function.EnsureHasCode());
CodePatcher::PatchStaticCallAt(frame->pc(), caller_code, current_target_code);
caller_code.SetStaticCallTargetCodeAt(frame->pc(), current_target_code);
if (FLAG_trace_patching) {
OS::PrintErr(
"FixCallersTarget: caller %#" Px
" "
"target '%s' -> %#" Px " (%s)\n",
frame->pc(), target_function.ToFullyQualifiedCString(),
current_target_code.EntryPoint(),
current_target_code.is_optimized() ? "optimized" : "unoptimized");
}
ASSERT(!current_target_code.IsDisabled());
arguments.SetReturn(current_target_code);
#else
UNREACHABLE();
#endif
}
// The caller must be a monomorphic call from unoptimized code.
// Patch call to point to new target.
DEFINE_RUNTIME_ENTRY(FixCallersTargetMonomorphic, 0) {
#if !defined(DART_PRECOMPILED_RUNTIME)
StackFrameIterator iterator(ValidationPolicy::kDontValidateFrames, thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* frame = iterator.NextFrame();
ASSERT(frame != NULL);
while (frame->IsStubFrame() || frame->IsExitFrame()) {
frame = iterator.NextFrame();
ASSERT(frame != NULL);
}
if (frame->IsEntryFrame()) {
// Since function's current code is always unpatched, the entry frame always
// calls to unpatched code.
UNREACHABLE();
}
ASSERT(frame->IsDartFrame());
const Code& caller_code = Code::Handle(zone, frame->LookupDartCode());
RELEASE_ASSERT(!caller_code.is_optimized());
Object& cache = Object::Handle(zone);
const Code& old_target_code = Code::Handle(
zone, CodePatcher::GetInstanceCallAt(frame->pc(), caller_code, &cache));
const Function& target_function =
Function::Handle(zone, old_target_code.function());
const Code& current_target_code =
Code::Handle(zone, target_function.EnsureHasCode());
CodePatcher::PatchInstanceCallAt(frame->pc(), caller_code, cache,
current_target_code);
if (FLAG_trace_patching) {
OS::PrintErr(
"FixCallersTargetMonomorphic: caller %#" Px
" "
"target '%s' -> %#" Px " (%s)\n",
frame->pc(), target_function.ToFullyQualifiedCString(),
current_target_code.EntryPoint(),
current_target_code.is_optimized() ? "optimized" : "unoptimized");
}
ASSERT(!current_target_code.IsDisabled());
arguments.SetReturn(current_target_code);
#else
UNREACHABLE();
#endif
}
// The caller tried to allocate an instance via an invalidated allocation
// stub.
DEFINE_RUNTIME_ENTRY(FixAllocationStubTarget, 0) {
#if !defined(DART_PRECOMPILED_RUNTIME)
StackFrameIterator iterator(ValidationPolicy::kDontValidateFrames, thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* frame = iterator.NextFrame();
ASSERT(frame != NULL);
while (frame->IsStubFrame() || frame->IsExitFrame()) {
frame = iterator.NextFrame();
ASSERT(frame != NULL);
}
if (frame->IsEntryFrame()) {
// There must be a valid Dart frame.
UNREACHABLE();
}
ASSERT(frame->IsDartFrame());
const Code& caller_code = Code::Handle(zone, frame->LookupDartCode());
ASSERT(!caller_code.IsNull());
const Code& stub = Code::Handle(
CodePatcher::GetStaticCallTargetAt(frame->pc(), caller_code));
Class& alloc_class = Class::ZoneHandle(zone);
alloc_class ^= stub.owner();
Code& alloc_stub = Code::Handle(zone, alloc_class.allocation_stub());
if (alloc_stub.IsNull()) {
alloc_stub = StubCode::GetAllocationStubForClass(alloc_class);
ASSERT(!alloc_stub.IsDisabled());
}
CodePatcher::PatchStaticCallAt(frame->pc(), caller_code, alloc_stub);
caller_code.SetStubCallTargetCodeAt(frame->pc(), alloc_stub);
if (FLAG_trace_patching) {
OS::PrintErr("FixAllocationStubTarget: caller %#" Px
" alloc-class %s "
" -> %#" Px "\n",
frame->pc(), alloc_class.ToCString(), alloc_stub.EntryPoint());
}
arguments.SetReturn(alloc_stub);
#else
UNREACHABLE();
#endif
}
const char* DeoptReasonToCString(ICData::DeoptReasonId deopt_reason) {
switch (deopt_reason) {
#define DEOPT_REASON_TO_TEXT(name) \
case ICData::kDeopt##name: \
return #name;
DEOPT_REASONS(DEOPT_REASON_TO_TEXT)
#undef DEOPT_REASON_TO_TEXT
default:
UNREACHABLE();
return "";
}
}
void DeoptimizeAt(const Code& optimized_code, StackFrame* frame) {
ASSERT(optimized_code.is_optimized());
// Force-optimized code is optimized code which cannot deoptimize and doesn't
// have unoptimized code to fall back to.
ASSERT(!optimized_code.is_force_optimized());
Thread* thread = Thread::Current();
Zone* zone = thread->zone();
const Function& function = Function::Handle(zone, optimized_code.function());
const Error& error =
Error::Handle(zone, Compiler::EnsureUnoptimizedCode(thread, function));
if (!error.IsNull()) {
Exceptions::PropagateError(error);
}
const Code& unoptimized_code =
Code::Handle(zone, function.unoptimized_code());
ASSERT(!unoptimized_code.IsNull());
// The switch to unoptimized code may have already occurred.
if (function.HasOptimizedCode()) {
function.SwitchToUnoptimizedCode();
}
if (frame->IsMarkedForLazyDeopt()) {
// Deopt already scheduled.
if (FLAG_trace_deoptimization) {
THR_Print("Lazy deopt already scheduled for fp=%" Pp "\n", frame->fp());
}
} else {
uword deopt_pc = frame->pc();
ASSERT(optimized_code.ContainsInstructionAt(deopt_pc));
#if defined(DEBUG)
ValidateFrames();
#endif
// N.B.: Update the pending deopt table before updating the frame. The
// profiler may attempt a stack walk in between.
ASSERT(!frame->is_interpreted());
thread->isolate()->AddPendingDeopt(frame->fp(), deopt_pc);
frame->MarkForLazyDeopt();
if (FLAG_trace_deoptimization) {
THR_Print("Lazy deopt scheduled for fp=%" Pp ", pc=%" Pp "\n",
frame->fp(), deopt_pc);
}
}
// Mark code as dead (do not GC its embedded objects).
optimized_code.set_is_alive(false);
}
// Currently checks only that all optimized frames have kDeoptIndex
// and unoptimized code has the kDeoptAfter.
void DeoptimizeFunctionsOnStack() {
DartFrameIterator iterator(Thread::Current(),
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* frame = iterator.NextFrame();
Code& optimized_code = Code::Handle();
while (frame != NULL) {
if (!frame->is_interpreted()) {
optimized_code = frame->LookupDartCode();
if (optimized_code.is_optimized() &&
!optimized_code.is_force_optimized()) {
DeoptimizeAt(optimized_code, frame);
}
}
frame = iterator.NextFrame();
}
}
#if !defined(DART_PRECOMPILED_RUNTIME)
static const intptr_t kNumberOfSavedCpuRegisters = kNumberOfCpuRegisters;
static const intptr_t kNumberOfSavedFpuRegisters = kNumberOfFpuRegisters;
static void CopySavedRegisters(uword saved_registers_address,
fpu_register_t** fpu_registers,
intptr_t** cpu_registers) {
// Tell MemorySanitizer this region is initialized by generated code. This
// region isn't already (fully) unpoisoned by FrameSetIterator::Unpoison
// because it is in an exit frame and stack frame iteration doesn't have
// access to true SP for exit frames.
MSAN_UNPOISON(reinterpret_cast<void*>(saved_registers_address),
kNumberOfSavedFpuRegisters * kFpuRegisterSize +
kNumberOfSavedCpuRegisters * kWordSize);
ASSERT(sizeof(fpu_register_t) == kFpuRegisterSize);
fpu_register_t* fpu_registers_copy =
new fpu_register_t[kNumberOfSavedFpuRegisters];
ASSERT(fpu_registers_copy != NULL);
for (intptr_t i = 0; i < kNumberOfSavedFpuRegisters; i++) {
fpu_registers_copy[i] =
*reinterpret_cast<fpu_register_t*>(saved_registers_address);
saved_registers_address += kFpuRegisterSize;
}
*fpu_registers = fpu_registers_copy;
ASSERT(sizeof(intptr_t) == kWordSize);
intptr_t* cpu_registers_copy = new intptr_t[kNumberOfSavedCpuRegisters];
ASSERT(cpu_registers_copy != NULL);
for (intptr_t i = 0; i < kNumberOfSavedCpuRegisters; i++) {
cpu_registers_copy[i] =
*reinterpret_cast<intptr_t*>(saved_registers_address);
saved_registers_address += kWordSize;
}
*cpu_registers = cpu_registers_copy;
}
#endif
// Copies saved registers and caller's frame into temporary buffers.
// Returns the stack size of unoptimized frame.
// The calling code must be optimized, but its function may not have
// have optimized code if the code is OSR code, or if the code was invalidated
// through class loading/finalization or field guard.
DEFINE_LEAF_RUNTIME_ENTRY(intptr_t,
DeoptimizeCopyFrame,
2,
uword saved_registers_address,
uword is_lazy_deopt) {
#if !defined(DART_PRECOMPILED_RUNTIME)
Thread* thread = Thread::Current();
Isolate* isolate = thread->isolate();
StackZone zone(thread);
HANDLESCOPE(thread);
// All registers have been saved below last-fp as if they were locals.
const uword last_fp =
saved_registers_address + (kNumberOfSavedCpuRegisters * kWordSize) +
(kNumberOfSavedFpuRegisters * kFpuRegisterSize) -
((runtime_frame_layout.first_local_from_fp + 1) * kWordSize);
// Get optimized code and frame that need to be deoptimized.
DartFrameIterator iterator(last_fp, thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* caller_frame = iterator.NextFrame();
ASSERT(caller_frame != NULL);
const Code& optimized_code = Code::Handle(caller_frame->LookupDartCode());
ASSERT(optimized_code.is_optimized());
const Function& top_function =
Function::Handle(thread->zone(), optimized_code.function());
const bool deoptimizing_code = top_function.HasOptimizedCode();
if (FLAG_trace_deoptimization) {
const Function& function = Function::Handle(optimized_code.function());
THR_Print("== Deoptimizing code for '%s', %s, %s\n",
function.ToFullyQualifiedCString(),
deoptimizing_code ? "code & frame" : "frame",
(is_lazy_deopt != 0u) ? "lazy-deopt" : "");
}
if (is_lazy_deopt != 0u) {
uword deopt_pc = isolate->FindPendingDeopt(caller_frame->fp());
if (FLAG_trace_deoptimization) {
THR_Print("Lazy deopt fp=%" Pp " pc=%" Pp "\n", caller_frame->fp(),
deopt_pc);
}
// N.B.: Update frame before updating pending deopt table. The profiler
// may attempt a stack walk in between.
caller_frame->set_pc(deopt_pc);
ASSERT(caller_frame->pc() == deopt_pc);
ASSERT(optimized_code.ContainsInstructionAt(caller_frame->pc()));
isolate->ClearPendingDeoptsAtOrBelow(caller_frame->fp());
} else {
if (FLAG_trace_deoptimization) {
THR_Print("Eager deopt fp=%" Pp " pc=%" Pp "\n", caller_frame->fp(),
caller_frame->pc());
}
}
// Copy the saved registers from the stack.
fpu_register_t* fpu_registers;
intptr_t* cpu_registers;
CopySavedRegisters(saved_registers_address, &fpu_registers, &cpu_registers);
// Create the DeoptContext.
DeoptContext* deopt_context = new DeoptContext(
caller_frame, optimized_code, DeoptContext::kDestIsOriginalFrame,
fpu_registers, cpu_registers, is_lazy_deopt != 0, deoptimizing_code);
isolate->set_deopt_context(deopt_context);
// Stack size (FP - SP) in bytes.
return deopt_context->DestStackAdjustment() * kWordSize;
#else
UNREACHABLE();
return 0;
#endif // !DART_PRECOMPILED_RUNTIME
}
END_LEAF_RUNTIME_ENTRY
// The stack has been adjusted to fit all values for unoptimized frame.
// Fill the unoptimized frame.
DEFINE_LEAF_RUNTIME_ENTRY(void, DeoptimizeFillFrame, 1, uword last_fp) {
#if !defined(DART_PRECOMPILED_RUNTIME)
Thread* thread = Thread::Current();
Isolate* isolate = thread->isolate();
StackZone zone(thread);
HANDLESCOPE(thread);
DeoptContext* deopt_context = isolate->deopt_context();
DartFrameIterator iterator(last_fp, thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* caller_frame = iterator.NextFrame();
ASSERT(caller_frame != NULL);
#if defined(DEBUG)
{
// The code from the deopt_context.
const Code& code = Code::Handle(deopt_context->code());
// The code from our frame.
const Code& optimized_code = Code::Handle(caller_frame->LookupDartCode());
const Function& function = Function::Handle(optimized_code.function());
ASSERT(!function.IsNull());
// The code will be the same as before.
ASSERT(code.raw() == optimized_code.raw());
// Some sanity checking of the optimized code.
ASSERT(!optimized_code.IsNull() && optimized_code.is_optimized());
}
#endif
deopt_context->set_dest_frame(caller_frame);
deopt_context->FillDestFrame();
#else
UNREACHABLE();
#endif // !DART_PRECOMPILED_RUNTIME
}
END_LEAF_RUNTIME_ENTRY
// This is the last step in the deoptimization, GC can occur.
// Returns number of bytes to remove from the expression stack of the
// bottom-most deoptimized frame. Those arguments were artificially injected
// under return address to keep them discoverable by GC that can occur during
// materialization phase.
DEFINE_RUNTIME_ENTRY(DeoptimizeMaterialize, 0) {
#if !defined(DART_PRECOMPILED_RUNTIME)
#if defined(DEBUG)
{
// We may rendezvous for a safepoint at entry or GC from the allocations
// below. Check the stack is walkable.
ValidateFrames();
}
#endif
DeoptContext* deopt_context = isolate->deopt_context();
intptr_t deopt_arg_count = deopt_context->MaterializeDeferredObjects();
isolate->set_deopt_context(NULL);
delete deopt_context;
// Return value tells deoptimization stub to remove the given number of bytes
// from the stack.
arguments.SetReturn(Smi::Handle(Smi::New(deopt_arg_count * kWordSize)));
#else
UNREACHABLE();
#endif // !DART_PRECOMPILED_RUNTIME
}
DEFINE_RUNTIME_ENTRY(RewindPostDeopt, 0) {
#if !defined(DART_PRECOMPILED_RUNTIME)
#if !defined(PRODUCT)
isolate->debugger()->RewindPostDeopt();
#endif // !PRODUCT
#endif // !DART_PRECOMPILED_RUNTIME
UNREACHABLE();
}
double DartModulo(double left, double right) {
double remainder = fmod_ieee(left, right);
if (remainder == 0.0) {
// We explicitly switch to the positive 0.0 (just in case it was negative).
remainder = +0.0;
} else if (remainder < 0.0) {
if (right < 0) {
remainder -= right;
} else {
remainder += right;
}
}
return remainder;
}
// Update global type feedback recorded for a field recording the assignment
// of the given value.
// Arg0: Field object;
// Arg1: Value that is being stored.
DEFINE_RUNTIME_ENTRY(UpdateFieldCid, 2) {
#if !defined(DART_PRECOMPILED_RUNTIME)
const Field& field = Field::CheckedHandle(zone, arguments.ArgAt(0));
const Object& value = Object::Handle(arguments.ArgAt(1));
field.RecordStore(value);
#else
UNREACHABLE();
#endif
}
DEFINE_RUNTIME_ENTRY(InitInstanceField, 2) {
const Instance& instance = Instance::CheckedHandle(zone, arguments.ArgAt(0));
const Field& field = Field::CheckedHandle(zone, arguments.ArgAt(1));
Object& result = Object::Handle(zone, field.InitializeInstance(instance));
ThrowIfError(result);
result = instance.GetField(field);
ASSERT((result.raw() != Object::sentinel().raw()) &&
(result.raw() != Object::transition_sentinel().raw()));
arguments.SetReturn(result);
}
DEFINE_RUNTIME_ENTRY(InitStaticField, 1) {
const Field& field = Field::CheckedHandle(zone, arguments.ArgAt(0));
Object& result = Object::Handle(zone, field.InitializeStatic());
ThrowIfError(result);
result = field.StaticValue();
ASSERT((result.raw() != Object::sentinel().raw()) &&
(result.raw() != Object::transition_sentinel().raw()));
arguments.SetReturn(result);
}
DEFINE_RUNTIME_ENTRY(LateInitializationError, 1) {
const Field& field = Field::CheckedHandle(zone, arguments.ArgAt(0));
Exceptions::ThrowLateInitializationError(String::Handle(field.name()));
}
DEFINE_RUNTIME_ENTRY(NotLoaded, 0) {
// We could just use a trap instruction in the stub, but we get better stack
// traces when there is an exit frame.
FATAL("Not loaded");
}
// Use expected function signatures to help MSVC compiler resolve overloading.
typedef double (*UnaryMathCFunction)(double x);
typedef double (*BinaryMathCFunction)(double x, double y);
DEFINE_RAW_LEAF_RUNTIME_ENTRY(
LibcPow,
2,
true /* is_float */,
reinterpret_cast<RuntimeFunction>(static_cast<BinaryMathCFunction>(&pow)));
DEFINE_RAW_LEAF_RUNTIME_ENTRY(
DartModulo,
2,
true /* is_float */,
reinterpret_cast<RuntimeFunction>(
static_cast<BinaryMathCFunction>(&DartModulo)));
DEFINE_RAW_LEAF_RUNTIME_ENTRY(
LibcAtan2,
2,
true /* is_float */,
reinterpret_cast<RuntimeFunction>(
static_cast<BinaryMathCFunction>(&atan2_ieee)));
DEFINE_RAW_LEAF_RUNTIME_ENTRY(
LibcFloor,
1,
true /* is_float */,
reinterpret_cast<RuntimeFunction>(static_cast<UnaryMathCFunction>(&floor)));
DEFINE_RAW_LEAF_RUNTIME_ENTRY(
LibcCeil,
1,
true /* is_float */,
reinterpret_cast<RuntimeFunction>(static_cast<UnaryMathCFunction>(&ceil)));
DEFINE_RAW_LEAF_RUNTIME_ENTRY(
LibcTrunc,
1,
true /* is_float */,
reinterpret_cast<RuntimeFunction>(static_cast<UnaryMathCFunction>(&trunc)));
DEFINE_RAW_LEAF_RUNTIME_ENTRY(
LibcRound,
1,
true /* is_float */,
reinterpret_cast<RuntimeFunction>(static_cast<UnaryMathCFunction>(&round)));
DEFINE_RAW_LEAF_RUNTIME_ENTRY(
LibcCos,
1,
true /* is_float */,
reinterpret_cast<RuntimeFunction>(static_cast<UnaryMathCFunction>(&cos)));
DEFINE_RAW_LEAF_RUNTIME_ENTRY(
LibcSin,
1,
true /* is_float */,
reinterpret_cast<RuntimeFunction>(static_cast<UnaryMathCFunction>(&sin)));
DEFINE_RAW_LEAF_RUNTIME_ENTRY(
LibcAsin,
1,
true /* is_float */,
reinterpret_cast<RuntimeFunction>(static_cast<UnaryMathCFunction>(&asin)));
DEFINE_RAW_LEAF_RUNTIME_ENTRY(
LibcAcos,
1,
true /* is_float */,
reinterpret_cast<RuntimeFunction>(static_cast<UnaryMathCFunction>(&acos)));
DEFINE_RAW_LEAF_RUNTIME_ENTRY(
LibcTan,
1,
true /* is_float */,
reinterpret_cast<RuntimeFunction>(static_cast<UnaryMathCFunction>(&tan)));
DEFINE_RAW_LEAF_RUNTIME_ENTRY(
LibcAtan,
1,
true /* is_float */,
reinterpret_cast<RuntimeFunction>(static_cast<UnaryMathCFunction>(&atan)));
// Interpret a function call. Should be called only for non-jitted functions.
// argc indicates the number of arguments, including the type arguments.
// argv points to the first argument.
// If argc < 0, arguments are passed at decreasing memory addresses from argv.
extern "C" uword /*ObjectPtr*/ InterpretCall(uword /*FunctionPtr*/ function_in,
uword /*ArrayPtr*/ argdesc_in,
intptr_t argc,
ObjectPtr* argv,
Thread* thread) {
#if defined(DART_PRECOMPILED_RUNTIME)
UNREACHABLE();
#else
FunctionPtr function = static_cast<FunctionPtr>(function_in);
ArrayPtr argdesc = static_cast<ArrayPtr>(argdesc_in);
ASSERT(FLAG_enable_interpreter);
Interpreter* interpreter = Interpreter::Current();
#if defined(DEBUG)
uword exit_fp = thread->top_exit_frame_info();
ASSERT(exit_fp != 0);
ASSERT(thread == Thread::Current());
// Caller is InterpretCall stub called from generated code.
// We stay in "in generated code" execution state when interpreting code.
ASSERT(thread->execution_state() == Thread::kThreadInGenerated);
ASSERT(!Function::HasCode(function));
ASSERT(Function::HasBytecode(function));
ASSERT(interpreter != NULL);
#endif
// Tell MemorySanitizer 'argv' is initialized by generated code.
if (argc < 0) {
MSAN_UNPOISON(argv - argc, -argc * sizeof(ObjectPtr));
} else {
MSAN_UNPOISON(argv, argc * sizeof(ObjectPtr));
}
ObjectPtr result = interpreter->Call(function, argdesc, argc, argv, thread);
DEBUG_ASSERT(thread->top_exit_frame_info() == exit_fp);
if (IsErrorClassId(result->GetClassIdMayBeSmi())) {
// Must not leak handles in the caller's zone.
HANDLESCOPE(thread);
// Protect the result in a handle before transitioning, which may trigger
// GC.
const Error& error = Error::Handle(Error::RawCast(result));
// Propagating an error may cause allocation. Check if we need to block for
// a safepoint by switching to "in VM" execution state.
TransitionGeneratedToVM transition(thread);
Exceptions::PropagateError(error);
}
return static_cast<uword>(result);
#endif // defined(DART_PRECOMPILED_RUNTIME)
}
uword RuntimeEntry::InterpretCallEntry() {
uword entry = reinterpret_cast<uword>(InterpretCall);
#if defined(USING_SIMULATOR)
entry = Simulator::RedirectExternalReference(entry,
Simulator::kLeafRuntimeCall, 5);
#endif
return entry;
}
extern "C" void DFLRT_EnterSafepoint(NativeArguments __unusable_) {
CHECK_STACK_ALIGNMENT;
TRACE_RUNTIME_CALL("%s", "EnterSafepoint");
Thread* thread = Thread::Current();
ASSERT(thread->top_exit_frame_info() != 0);
ASSERT(thread->execution_state() == Thread::kThreadInNative);
thread->EnterSafepoint();
TRACE_RUNTIME_CALL("%s", "EnterSafepoint done");
}
DEFINE_RAW_LEAF_RUNTIME_ENTRY(EnterSafepoint, 0, false, &DFLRT_EnterSafepoint);
extern "C" void DFLRT_ExitSafepoint(NativeArguments __unusable_) {
CHECK_STACK_ALIGNMENT;
TRACE_RUNTIME_CALL("%s", "ExitSafepoint");
Thread* thread = Thread::Current();
ASSERT(thread->top_exit_frame_info() != 0);
ASSERT(thread->execution_state() == Thread::kThreadInVM);
thread->ExitSafepoint();
TRACE_RUNTIME_CALL("%s", "ExitSafepoint done");
}
DEFINE_RAW_LEAF_RUNTIME_ENTRY(ExitSafepoint, 0, false, &DFLRT_ExitSafepoint);
// Not registered as a runtime entry because we can't use Thread to look it up.
static Thread* GetThreadForNativeCallback(uword callback_id,
uword return_address) {
Thread* const thread = Thread::Current();
if (thread == nullptr) {
FATAL("Cannot invoke native callback outside an isolate.");
}
if (thread->no_callback_scope_depth() != 0) {
FATAL("Cannot invoke native callback when API callbacks are prohibited.");
}
if (!thread->IsMutatorThread()) {
FATAL("Native callbacks must be invoked on the mutator thread.");
}
// Set the execution state to VM while waiting for the safepoint to end.
// This isn't strictly necessary but enables tests to check that we're not
// in native code anymore. See tests/ffi/function_gc_test.dart for example.
thread->set_execution_state(Thread::kThreadInVM);
thread->ExitSafepoint();
thread->VerifyCallbackIsolate(callback_id, return_address);
return thread;
}
#if defined(HOST_OS_WINDOWS)
#pragma intrinsic(_ReturnAddress)
#endif
// This is called directly by NativeEntryInstr. At the moment we enter this
// routine, the caller is generated code in the Isolate heap. Therefore we check
// that the return address (caller) corresponds to the declared callback ID's
// code within this Isolate.
extern "C" Thread* DLRT_GetThreadForNativeCallback(uword callback_id) {
CHECK_STACK_ALIGNMENT;
TRACE_RUNTIME_CALL("GetThreadForNativeCallback %" Pd, callback_id);
#if defined(HOST_OS_WINDOWS)
void* return_address = _ReturnAddress();
#else
void* return_address = __builtin_return_address(0);
#endif
Thread* return_value = GetThreadForNativeCallback(
callback_id, reinterpret_cast<uword>(return_address));
TRACE_RUNTIME_CALL("GetThreadForNativeCallback returning %p", return_value);
return return_value;
}
// This is called by a native callback trampoline
// (see StubCodeCompiler::GenerateJITCallbackTrampolines). There is no need to
// check the return address because the trampoline will use the callback ID to
// look up the generated code. We still check that the callback ID is valid for
// this isolate.
extern "C" Thread* DLRT_GetThreadForNativeCallbackTrampoline(
uword callback_id) {
CHECK_STACK_ALIGNMENT;
return GetThreadForNativeCallback(callback_id, 0);
}
// This is called directly by EnterHandleScopeInstr.
extern "C" ApiLocalScope* DLRT_EnterHandleScope(Thread* thread) {
CHECK_STACK_ALIGNMENT;
TRACE_RUNTIME_CALL("EnterHandleScope %p", thread);
thread->EnterApiScope();
ApiLocalScope* return_value = thread->api_top_scope();
TRACE_RUNTIME_CALL("EnterHandleScope returning %p", return_value);
return return_value;
}
DEFINE_RAW_LEAF_RUNTIME_ENTRY(
EnterHandleScope,
1,
false /* is_float */,
reinterpret_cast<RuntimeFunction>(&DLRT_EnterHandleScope));
// This is called directly by ExitHandleScopeInstr.
extern "C" void DLRT_ExitHandleScope(Thread* thread) {
CHECK_STACK_ALIGNMENT;
TRACE_RUNTIME_CALL("ExitHandleScope %p", thread);
thread->ExitApiScope();
TRACE_RUNTIME_CALL("ExitHandleScope %s", "done");
}
DEFINE_RAW_LEAF_RUNTIME_ENTRY(
ExitHandleScope,
1,
false /* is_float */,
reinterpret_cast<RuntimeFunction>(&DLRT_ExitHandleScope));
// This is called directly by AllocateHandleInstr.
extern "C" LocalHandle* DLRT_AllocateHandle(ApiLocalScope* scope) {
CHECK_STACK_ALIGNMENT;
TRACE_RUNTIME_CALL("AllocateHandle %p", scope);
LocalHandle* return_value = scope->local_handles()->AllocateHandle();
TRACE_RUNTIME_CALL("AllocateHandle returning %p", return_value);
return return_value;
}
DEFINE_RAW_LEAF_RUNTIME_ENTRY(
AllocateHandle,
1,
false /* is_float */,
reinterpret_cast<RuntimeFunction>(&DLRT_AllocateHandle));
} // namespace dart
Reference in New Issue View Git Blame Copy Permalink