Files
Alexander Aprelev 5b0285866d Revert "Reland "[vm] Recognize int.trailingZeroBitCount/oneBitCount as graph-inlinable""
This reverts commit 415b040d6f.

Reason for revert: breaks riscv https://github.com/dart-lang/sdk/issues/63479

Original change's description:
> Reland "[vm] Recognize int.trailingZeroBitCount/oneBitCount as graph-inlinable"
>
> The previous attempt was reverted because it broke unoptimized JIT
> on ARM 32. This reland force-optimizes the two getters.
>
> Stacks on top of the int.{trailingZeroBitCount,oneBitCount} API CL
> (commit 754239b077). Both getters route through OTHER_RECOGNIZED_LIST
> when a hardware fast path is available; otherwise the newly added
> Dart bodies inline at call sites via vm:prefer-inline. The C++
> natives are removed.
>
> Backend codegen
> ---------------
> ARM64:     NEON CNT + UADDLV (popcount); RBIT + CLZ (ctz).
> ARM:       NEON CNT + VPADDL chain (popcount); RBIT + CLZ on the
>            register pair (ctz).
> x64:       popcntq when TargetCPUFeatures::popcnt_supported();
>            LoadImmediate(64) + rep_bsfq for ctz (decodes as tzcnt
>            on BMI1+, preserves dest on zero otherwise).
> RISC-V 64: cpop / ctz when RV_baseline includes Zbb.
>
> Per-arch availability is encapsulated in
> UnaryInt64OpInstr::IsSupported(Token::Kind).
>
> Apple M-series ARM64, AOT (us/iter, lower is better):
>   cardinality.swar              371
>   cardinality.accelerated       154    (2.4x)
>   forEachSetBit.swar          19031
>   forEachSetBit.accelerated    4988    (3.8x)
>   select.swar                   199
>   select.accelerated             77    (2.6x)
>   complementCardinality.swar    399
>   complementCardinality.accel   152    (2.6x)
>
> Work towards https://github.com/dart-lang/sdk/issues/6486 (popcount
> and ctz intrinsification).
>
> Work towards https://github.com/dart-lang/sdk/issues/1053 (efficient
> BitSet implementation).
>
> Fixes https://github.com/dart-lang/sdk/issues/52673
> Fixes https://github.com/dart-lang/sdk/issues/38346
> Fixes https://github.com/dart-lang/sdk/issues/63436
> Issue https://github.com/dart-lang/sdk/issues/10212
> Issue https://github.com/dart-lang/sdk/issues/5798
> TEST=tests/corelib/int_bit_count_test
>
> Cq-Include-Trybots: luci.dart.try:vm-linux-release-simarm-try,vm-ffi-qemu-linux-release-arm-try,vm-aot-linux-release-simarm_x64-try,vm-aot-linux-debug-simarm_x64-try,dart-sdk-linux-riscv64-try
> Change-Id: Ib812cbaec6e371b9720df7a543411f78e524cac1
> Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/506060
> Reviewed-by: Martin Kustermann <kustermann@google.com>
> Auto-Submit: Modestas Valauskas <valauskasmodestas@gmail.com>
> Reviewed-by: Slava Egorov <vegorov@google.com>
> Commit-Queue: Martin Kustermann <kustermann@google.com>

Cq-Include-Trybots: luci.dart.try:vm-linux-release-simarm-try,vm-ffi-qemu-linux-release-arm-try,vm-aot-linux-release-simarm_x64-try,vm-aot-linux-debug-simarm_x64-try,dart-sdk-linux-riscv64-try
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Change-Id: Iaf11d03d394fa615098bed8fcdea38ba40c7e45f
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/507520
Bot-Commit: rubber-stamper@appspot.gserviceaccount.com <rubber-stamper@appspot.gserviceaccount.com>
Commit-Queue: Alexander Aprelev <aam@google.com>
Reviewed-by: Kevin Moore <kevmoo@google.com>
2026-05-29 10:29:33 -07:00

311 lines
10 KiB
C++

// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "vm/compiler/backend/evaluator.h"
namespace dart {
static IntegerPtr BinaryIntegerEvaluateRaw(const Integer& left,
const Integer& right,
Token::Kind token_kind) {
switch (token_kind) {
case Token::kTRUNCDIV:
FALL_THROUGH;
case Token::kMOD:
// Check right value for zero.
if (right.Value() == 0) {
break; // Will throw.
}
FALL_THROUGH;
case Token::kADD:
FALL_THROUGH;
case Token::kSUB:
FALL_THROUGH;
case Token::kMUL:
return left.ArithmeticOp(token_kind, right, Heap::kOld);
case Token::kSHL:
FALL_THROUGH;
case Token::kSHR:
FALL_THROUGH;
case Token::kUSHR:
if (right.Value() >= 0) {
return left.ShiftOp(token_kind, right, Heap::kOld);
}
break;
case Token::kBIT_AND:
FALL_THROUGH;
case Token::kBIT_OR:
FALL_THROUGH;
case Token::kBIT_XOR:
return left.BitOp(token_kind, right, Heap::kOld);
case Token::kDIV:
break;
default:
UNREACHABLE();
}
return Integer::null();
}
static IntegerPtr UnaryIntegerEvaluateRaw(const Integer& value,
Token::Kind token_kind,
Zone* zone) {
switch (token_kind) {
case Token::kNEGATE:
return value.ArithmeticOp(Token::kMUL, Smi::Handle(zone, Smi::New(-1)),
Heap::kOld);
case Token::kBIT_NOT:
if (value.IsInteger()) {
return Integer::New(~value.Value(), Heap::kOld);
}
break;
default:
UNREACHABLE();
}
return Integer::null();
}
static IntegerPtr BitLengthEvaluateRaw(const Integer& value, Zone* zone) {
if (value.IsInteger()) {
return Integer::New(Utils::BitLength(value.Value()), Heap::kOld);
}
return Integer::null();
}
int64_t Evaluator::TruncateTo(int64_t v, Representation r) {
switch (r) {
case kTagged: {
const intptr_t kTruncateBits =
kBitsPerInt64 - (compiler::target::kSmiBits + 1 /*sign bit*/);
return Utils::ShiftLeftWithTruncation(v, kTruncateBits) >> kTruncateBits;
}
case kUnboxedInt32:
return Utils::ShiftLeftWithTruncation(v, kBitsPerInt32) >> kBitsPerInt32;
case kUnboxedUint32:
return v & kMaxUint32;
case kUnboxedInt64:
return v;
default:
UNREACHABLE();
}
}
IntegerPtr Evaluator::BinaryIntegerEvaluate(const Object& left,
const Object& right,
Token::Kind token_kind,
bool is_truncating,
Representation representation,
Thread* thread) {
if (!left.IsInteger() || !right.IsInteger()) {
return Integer::null();
}
Zone* zone = thread->zone();
const Integer& left_int = Integer::Cast(left);
const Integer& right_int = Integer::Cast(right);
Integer& result = Integer::Handle(
zone, BinaryIntegerEvaluateRaw(left_int, right_int, token_kind));
if (!result.IsNull()) {
if (is_truncating) {
const int64_t truncated = TruncateTo(result.Value(), representation);
result = Integer::New(truncated, Heap::kOld);
ASSERT(FlowGraph::IsConstantRepresentable(
result, representation, /*tagged_value_must_be_smi=*/true));
} else if (!FlowGraph::IsConstantRepresentable(
result, representation, /*tagged_value_must_be_smi=*/true)) {
// If this operation is not truncating it would deoptimize on overflow.
// Check that we match this behavior and don't produce a value that is
// larger than something this operation can produce. We could have
// specialized instructions that use this value under this assumption.
return Integer::null();
}
result ^= result.Canonicalize(thread);
}
return result.ptr();
}
IntegerPtr Evaluator::UnaryIntegerEvaluate(const Object& value,
Token::Kind token_kind,
Representation representation,
Thread* thread) {
if (!value.IsInteger()) {
return Integer::null();
}
Zone* zone = thread->zone();
const Integer& value_int = Integer::Cast(value);
Integer& result = Integer::Handle(
zone, UnaryIntegerEvaluateRaw(value_int, token_kind, zone));
if (!result.IsNull()) {
if (!FlowGraph::IsConstantRepresentable(
result, representation,
/*tagged_value_must_be_smi=*/true)) {
// If this operation is not truncating it would deoptimize on overflow.
// Check that we match this behavior and don't produce a value that is
// larger than something this operation can produce. We could have
// specialized instructions that use this value under this assumption.
return Integer::null();
}
result ^= result.Canonicalize(thread);
}
return result.ptr();
}
IntegerPtr Evaluator::BitLengthEvaluate(const Object& value,
Representation representation,
Thread* thread) {
if (!value.IsInteger()) {
return Integer::null();
}
Zone* zone = thread->zone();
const Integer& value_int = Integer::Cast(value);
Integer& result =
Integer::Handle(zone, BitLengthEvaluateRaw(value_int, zone));
if (!result.IsNull()) {
if (!FlowGraph::IsConstantRepresentable(
result, representation,
/*tagged_value_must_be_smi=*/true)) {
// If this operation is not truncating it would deoptimize on overflow.
// Check that we match this behavior and don't produce a value that is
// larger than something this operation can produce. We could have
// specialized instructions that use this value under this assumption.
return Integer::null();
}
result ^= result.Canonicalize(thread);
}
return result.ptr();
}
double Evaluator::EvaluateUnaryDoubleOp(const double value,
Token::Kind token_kind,
Representation representation) {
// The different set of operations for float32 and float64 is due to the
// different set of operations made available by dart:core.double and
// dart:typed_data.Float64x2 versus dart:typed_data.Float32x4.
if (representation == kUnboxedDouble) {
switch (token_kind) {
case Token::kABS:
return fabs(value);
case Token::kNEGATE:
return -value;
case Token::kSQRT:
return sqrt(value);
case Token::kSQUARE:
return value * value;
case Token::kTRUNCATE:
return trunc(value);
case Token::kFLOOR:
return floor(value);
case Token::kCEILING:
return ceil(value);
default:
UNREACHABLE();
}
} else {
ASSERT(representation == kUnboxedFloat);
switch (token_kind) {
case Token::kABS:
return fabsf(static_cast<float>(value));
case Token::kNEGATE:
return -static_cast<float>(value);
case Token::kRECIPROCAL:
return 1.0f / static_cast<float>(value);
case Token::kRECIPROCAL_SQRT:
return sqrtf(1.0f / static_cast<float>(value));
case Token::kSQRT:
return sqrtf(static_cast<float>(value));
case Token::kSQUARE:
return static_cast<float>(value) * static_cast<float>(value);
default:
UNREACHABLE();
}
}
}
double Evaluator::EvaluateBinaryDoubleOp(const double left,
const double right,
Token::Kind token_kind,
Representation representation) {
if (representation == kUnboxedDouble) {
switch (token_kind) {
case Token::kADD:
return left + right;
case Token::kSUB:
return left - right;
case Token::kMUL:
return left * right;
case Token::kDIV:
return Utils::DivideAllowZero(left, right);
case Token::kMIN:
return fmin(left, right);
case Token::kMAX:
return fmax(left, right);
default:
UNREACHABLE();
}
} else {
ASSERT(representation == kUnboxedFloat);
switch (token_kind) {
case Token::kADD:
return static_cast<float>(left) + static_cast<float>(right);
case Token::kSUB:
return static_cast<float>(left) - static_cast<float>(right);
case Token::kMUL:
return static_cast<float>(left) * static_cast<float>(right);
case Token::kDIV:
return Utils::DivideAllowZero(static_cast<float>(left),
static_cast<float>(right));
case Token::kMIN:
return fminf(static_cast<float>(left), static_cast<float>(right));
case Token::kMAX:
return fmaxf(static_cast<float>(left), static_cast<float>(right));
default:
UNREACHABLE();
}
}
}
bool Evaluator::ToIntegerConstant(Value* value, int64_t* result) {
if (!value->BindsToConstant()) {
UnboxInstr* unbox = value->definition()->AsUnbox();
if (unbox != nullptr) {
switch (unbox->representation()) {
case kUnboxedDouble:
case kUnboxedInt64:
return ToIntegerConstant(unbox->value(), result);
case kUnboxedUint32:
if (ToIntegerConstant(unbox->value(), result)) {
*result = Evaluator::TruncateTo(*result, kUnboxedUint32);
return true;
}
break;
// No need to handle Unbox<Int32>(Constant(C)) because it gets
// canonicalized to UnboxedConstant<Int32>(C).
case kUnboxedInt32:
default:
break;
}
}
return false;
}
const Object& constant = value->BoundConstant();
if (constant.IsDouble()) {
const Double& double_constant = Double::Cast(constant);
*result = Utils::SafeDoubleToInt<int64_t>(double_constant.value());
return (static_cast<double>(*result) == double_constant.value());
} else if (constant.IsInteger()) {
*result = Integer::Cast(constant).Value();
return true;
}
return false;
}
} // namespace dart