Files
sdk/runtime/lib/integers.cc
Alexander Aprelev 5b0285866d Revert "Reland "[vm] Recognize int.trailingZeroBitCount/oneBitCount as graph-inlinable""
This reverts commit 415b040d6f.

Reason for revert: breaks riscv https://github.com/dart-lang/sdk/issues/63479

Original change's description:
> Reland "[vm] Recognize int.trailingZeroBitCount/oneBitCount as graph-inlinable"
>
> The previous attempt was reverted because it broke unoptimized JIT
> on ARM 32. This reland force-optimizes the two getters.
>
> Stacks on top of the int.{trailingZeroBitCount,oneBitCount} API CL
> (commit 754239b077). Both getters route through OTHER_RECOGNIZED_LIST
> when a hardware fast path is available; otherwise the newly added
> Dart bodies inline at call sites via vm:prefer-inline. The C++
> natives are removed.
>
> Backend codegen
> ---------------
> ARM64:     NEON CNT + UADDLV (popcount); RBIT + CLZ (ctz).
> ARM:       NEON CNT + VPADDL chain (popcount); RBIT + CLZ on the
>            register pair (ctz).
> x64:       popcntq when TargetCPUFeatures::popcnt_supported();
>            LoadImmediate(64) + rep_bsfq for ctz (decodes as tzcnt
>            on BMI1+, preserves dest on zero otherwise).
> RISC-V 64: cpop / ctz when RV_baseline includes Zbb.
>
> Per-arch availability is encapsulated in
> UnaryInt64OpInstr::IsSupported(Token::Kind).
>
> Apple M-series ARM64, AOT (us/iter, lower is better):
>   cardinality.swar              371
>   cardinality.accelerated       154    (2.4x)
>   forEachSetBit.swar          19031
>   forEachSetBit.accelerated    4988    (3.8x)
>   select.swar                   199
>   select.accelerated             77    (2.6x)
>   complementCardinality.swar    399
>   complementCardinality.accel   152    (2.6x)
>
> Work towards https://github.com/dart-lang/sdk/issues/6486 (popcount
> and ctz intrinsification).
>
> Work towards https://github.com/dart-lang/sdk/issues/1053 (efficient
> BitSet implementation).
>
> Fixes https://github.com/dart-lang/sdk/issues/52673
> Fixes https://github.com/dart-lang/sdk/issues/38346
> Fixes https://github.com/dart-lang/sdk/issues/63436
> Issue https://github.com/dart-lang/sdk/issues/10212
> Issue https://github.com/dart-lang/sdk/issues/5798
> TEST=tests/corelib/int_bit_count_test
>
> Cq-Include-Trybots: luci.dart.try:vm-linux-release-simarm-try,vm-ffi-qemu-linux-release-arm-try,vm-aot-linux-release-simarm_x64-try,vm-aot-linux-debug-simarm_x64-try,dart-sdk-linux-riscv64-try
> Change-Id: Ib812cbaec6e371b9720df7a543411f78e524cac1
> Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/506060
> Reviewed-by: Martin Kustermann <kustermann@google.com>
> Auto-Submit: Modestas Valauskas <valauskasmodestas@gmail.com>
> Reviewed-by: Slava Egorov <vegorov@google.com>
> Commit-Queue: Martin Kustermann <kustermann@google.com>

Cq-Include-Trybots: luci.dart.try:vm-linux-release-simarm-try,vm-ffi-qemu-linux-release-arm-try,vm-aot-linux-release-simarm_x64-try,vm-aot-linux-debug-simarm_x64-try,dart-sdk-linux-riscv64-try
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Change-Id: Iaf11d03d394fa615098bed8fcdea38ba40c7e45f
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/507520
Bot-Commit: rubber-stamper@appspot.gserviceaccount.com <rubber-stamper@appspot.gserviceaccount.com>
Commit-Queue: Alexander Aprelev <aam@google.com>
Reviewed-by: Kevin Moore <kevmoo@google.com>
2026-05-29 10:29:33 -07:00

344 lines
12 KiB
C++

// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "lib/integers.h"
#include "vm/bootstrap_natives.h"
#include "include/dart_api.h"
#include "vm/dart_api_impl.h"
#include "vm/dart_entry.h"
#include "vm/exceptions.h"
#include "vm/isolate.h"
#include "vm/native_entry.h"
#include "vm/object.h"
#include "vm/object_store.h"
#include "vm/symbols.h"
namespace dart {
// Smi natives.
// Returns false if integer is in wrong representation, e.g., as is a Mint
// when it could have been a Smi.
static bool CheckInteger(const Integer& i) {
if (i.IsMint()) {
const Mint& mint = Mint::Cast(i);
return !Smi::IsValid(mint.Value());
}
return true;
}
DEFINE_NATIVE_ENTRY(Integer_bitAndFromInteger, 0, 2) {
const Integer& right =
Integer::CheckedHandle(zone, arguments->NativeArgAt(0));
GET_NON_NULL_NATIVE_ARGUMENT(Integer, left, arguments->NativeArgAt(1));
ASSERT(CheckInteger(right));
ASSERT(CheckInteger(left));
if (FLAG_trace_intrinsified_natives) {
OS::PrintErr("Integer_bitAndFromInteger %s & %s\n", right.ToCString(),
left.ToCString());
}
return left.BitOp(Token::kBIT_AND, right);
}
DEFINE_NATIVE_ENTRY(Integer_bitOrFromInteger, 0, 2) {
const Integer& right =
Integer::CheckedHandle(zone, arguments->NativeArgAt(0));
GET_NON_NULL_NATIVE_ARGUMENT(Integer, left, arguments->NativeArgAt(1));
ASSERT(CheckInteger(right));
ASSERT(CheckInteger(left));
if (FLAG_trace_intrinsified_natives) {
OS::PrintErr("Integer_bitOrFromInteger %s | %s\n", left.ToCString(),
right.ToCString());
}
return left.BitOp(Token::kBIT_OR, right);
}
DEFINE_NATIVE_ENTRY(Integer_bitXorFromInteger, 0, 2) {
const Integer& right =
Integer::CheckedHandle(zone, arguments->NativeArgAt(0));
GET_NON_NULL_NATIVE_ARGUMENT(Integer, left, arguments->NativeArgAt(1));
ASSERT(CheckInteger(right));
ASSERT(CheckInteger(left));
if (FLAG_trace_intrinsified_natives) {
OS::PrintErr("Integer_bitXorFromInteger %s ^ %s\n", left.ToCString(),
right.ToCString());
}
return left.BitOp(Token::kBIT_XOR, right);
}
DEFINE_NATIVE_ENTRY(Integer_addFromInteger, 0, 2) {
const Integer& right_int =
Integer::CheckedHandle(zone, arguments->NativeArgAt(0));
GET_NON_NULL_NATIVE_ARGUMENT(Integer, left_int, arguments->NativeArgAt(1));
ASSERT(CheckInteger(right_int));
ASSERT(CheckInteger(left_int));
if (FLAG_trace_intrinsified_natives) {
OS::PrintErr("Integer_addFromInteger %s + %s\n", left_int.ToCString(),
right_int.ToCString());
}
return left_int.ArithmeticOp(Token::kADD, right_int);
}
DEFINE_NATIVE_ENTRY(Integer_subFromInteger, 0, 2) {
const Integer& right_int =
Integer::CheckedHandle(zone, arguments->NativeArgAt(0));
GET_NON_NULL_NATIVE_ARGUMENT(Integer, left_int, arguments->NativeArgAt(1));
ASSERT(CheckInteger(right_int));
ASSERT(CheckInteger(left_int));
if (FLAG_trace_intrinsified_natives) {
OS::PrintErr("Integer_subFromInteger %s - %s\n", left_int.ToCString(),
right_int.ToCString());
}
return left_int.ArithmeticOp(Token::kSUB, right_int);
}
DEFINE_NATIVE_ENTRY(Integer_mulFromInteger, 0, 2) {
const Integer& right_int =
Integer::CheckedHandle(zone, arguments->NativeArgAt(0));
GET_NON_NULL_NATIVE_ARGUMENT(Integer, left_int, arguments->NativeArgAt(1));
ASSERT(CheckInteger(right_int));
ASSERT(CheckInteger(left_int));
if (FLAG_trace_intrinsified_natives) {
OS::PrintErr("Integer_mulFromInteger %s * %s\n", left_int.ToCString(),
right_int.ToCString());
}
return left_int.ArithmeticOp(Token::kMUL, right_int);
}
DEFINE_NATIVE_ENTRY(Integer_truncDivFromInteger, 0, 2) {
const Integer& right_int =
Integer::CheckedHandle(zone, arguments->NativeArgAt(0));
GET_NON_NULL_NATIVE_ARGUMENT(Integer, left_int, arguments->NativeArgAt(1));
ASSERT(CheckInteger(right_int));
ASSERT(CheckInteger(left_int));
ASSERT(right_int.Value() != 0);
return left_int.ArithmeticOp(Token::kTRUNCDIV, right_int);
}
DEFINE_NATIVE_ENTRY(Integer_moduloFromInteger, 0, 2) {
const Integer& right_int =
Integer::CheckedHandle(zone, arguments->NativeArgAt(0));
GET_NON_NULL_NATIVE_ARGUMENT(Integer, left_int, arguments->NativeArgAt(1));
ASSERT(CheckInteger(right_int));
ASSERT(CheckInteger(left_int));
if (FLAG_trace_intrinsified_natives) {
OS::PrintErr("Integer_moduloFromInteger %s mod %s\n", left_int.ToCString(),
right_int.ToCString());
}
if (right_int.Value() == 0) {
// Should have been caught before calling into runtime.
UNIMPLEMENTED();
}
return left_int.ArithmeticOp(Token::kMOD, right_int);
}
DEFINE_NATIVE_ENTRY(Integer_greaterThanFromInteger, 0, 2) {
const Integer& right =
Integer::CheckedHandle(zone, arguments->NativeArgAt(0));
GET_NON_NULL_NATIVE_ARGUMENT(Integer, left, arguments->NativeArgAt(1));
ASSERT(CheckInteger(right));
ASSERT(CheckInteger(left));
if (FLAG_trace_intrinsified_natives) {
OS::PrintErr("Integer_greaterThanFromInteger %s > %s\n", left.ToCString(),
right.ToCString());
}
return Bool::Get(left.CompareWith(right) == 1).ptr();
}
DEFINE_NATIVE_ENTRY(Integer_equalToInteger, 0, 2) {
const Integer& left = Integer::CheckedHandle(zone, arguments->NativeArgAt(0));
GET_NON_NULL_NATIVE_ARGUMENT(Integer, right, arguments->NativeArgAt(1));
ASSERT(CheckInteger(left));
ASSERT(CheckInteger(right));
if (FLAG_trace_intrinsified_natives) {
OS::PrintErr("Integer_equalToInteger %s == %s\n", left.ToCString(),
right.ToCString());
}
return Bool::Get(left.CompareWith(right) == 0).ptr();
}
static IntegerPtr ParseInteger(const String& value) {
// Used by Integer_fromEnvironment.
if (value.IsOneByteString()) {
// Quick conversion for unpadded integers in strings.
const intptr_t len = value.Length();
if (len > 0) {
const char* cstr = value.ToCString();
ASSERT(cstr != nullptr);
char* p_end = nullptr;
const int64_t int_value = strtoll(cstr, &p_end, 10);
if (p_end == (cstr + len)) {
if ((int_value != LLONG_MIN) && (int_value != LLONG_MAX)) {
return Integer::New(int_value);
}
}
}
}
return Integer::New(value);
}
DEFINE_NATIVE_ENTRY(Integer_fromEnvironment, 0, 2) {
GET_NON_NULL_NATIVE_ARGUMENT(String, name, arguments->NativeArgAt(0));
GET_NATIVE_ARGUMENT(Integer, default_value, arguments->NativeArgAt(1));
// Call the embedder to supply us with the environment.
const String& env_value =
String::Handle(Api::GetEnvironmentValue(thread, name));
if (!env_value.IsNull()) {
const Integer& result = Integer::Handle(ParseInteger(env_value));
if (!result.IsNull()) {
if (result.IsSmi()) {
return result.ptr();
}
return result.Canonicalize(thread);
}
}
return default_value.ptr();
}
static IntegerPtr ShiftOperationHelper(Token::Kind kind,
const Integer& value,
const Integer& amount) {
if (amount.Value() < 0) {
Exceptions::ThrowArgumentError(amount);
}
return value.ShiftOp(kind, amount, Heap::kNew);
}
DEFINE_NATIVE_ENTRY(Integer_shrFromInteger, 0, 2) {
const Integer& amount =
Integer::CheckedHandle(zone, arguments->NativeArgAt(0));
GET_NON_NULL_NATIVE_ARGUMENT(Integer, value, arguments->NativeArgAt(1));
ASSERT(CheckInteger(amount));
ASSERT(CheckInteger(value));
if (FLAG_trace_intrinsified_natives) {
OS::PrintErr("Integer_shrFromInteger: %s >> %s\n", value.ToCString(),
amount.ToCString());
}
return ShiftOperationHelper(Token::kSHR, value, amount);
}
DEFINE_NATIVE_ENTRY(Integer_ushrFromInteger, 0, 2) {
const Integer& amount =
Integer::CheckedHandle(zone, arguments->NativeArgAt(0));
GET_NON_NULL_NATIVE_ARGUMENT(Integer, value, arguments->NativeArgAt(1));
ASSERT(CheckInteger(amount));
ASSERT(CheckInteger(value));
if (FLAG_trace_intrinsified_natives) {
OS::PrintErr("Integer_ushrFromInteger: %s >>> %s\n", value.ToCString(),
amount.ToCString());
}
return ShiftOperationHelper(Token::kUSHR, value, amount);
}
DEFINE_NATIVE_ENTRY(Integer_shlFromInteger, 0, 2) {
const Integer& amount =
Integer::CheckedHandle(zone, arguments->NativeArgAt(0));
GET_NON_NULL_NATIVE_ARGUMENT(Integer, value, arguments->NativeArgAt(1));
ASSERT(CheckInteger(amount));
ASSERT(CheckInteger(value));
if (FLAG_trace_intrinsified_natives) {
OS::PrintErr("Integer_shlFromInteger: %s << %s\n", value.ToCString(),
amount.ToCString());
}
return ShiftOperationHelper(Token::kSHL, value, amount);
}
DEFINE_NATIVE_ENTRY(Smi_bitNegate, 0, 1) {
const Smi& operand = Smi::CheckedHandle(zone, arguments->NativeArgAt(0));
if (FLAG_trace_intrinsified_natives) {
OS::PrintErr("Smi_bitNegate: %s\n", operand.ToCString());
}
intptr_t result = ~operand.Value();
ASSERT(Smi::IsValid(result));
return Smi::New(result);
}
DEFINE_NATIVE_ENTRY(Smi_bitLength, 0, 1) {
const Smi& operand = Smi::CheckedHandle(zone, arguments->NativeArgAt(0));
if (FLAG_trace_intrinsified_natives) {
OS::PrintErr("Smi_bitLength: %s\n", operand.ToCString());
}
int64_t value = operand.Value();
intptr_t result = Utils::BitLength(value);
ASSERT(Smi::IsValid(result));
return Smi::New(result);
}
// Unified bit-count natives. Receiver is _IntegerImplementation, so the
// operand can be either Smi or Mint at runtime.
DEFINE_NATIVE_ENTRY(Integer_trailingZeroBitCount, 0, 1) {
const Integer& operand =
Integer::CheckedHandle(zone, arguments->NativeArgAt(0));
intptr_t result =
Utils::CountTrailingZeros64(static_cast<uint64_t>(operand.Value()));
ASSERT(Smi::IsValid(result));
return Smi::New(result);
}
DEFINE_NATIVE_ENTRY(Integer_oneBitCount, 0, 1) {
const Integer& operand =
Integer::CheckedHandle(zone, arguments->NativeArgAt(0));
intptr_t result =
Utils::CountOneBits64(static_cast<uint64_t>(operand.Value()));
ASSERT(Smi::IsValid(result));
return Smi::New(result);
}
// Should be kept in sync with il_*.cc EmitHashIntegerCodeSequence
uint32_t Multiply64Hash(int64_t ivalue) {
const uint64_t magic_constant = /*0x1b873593cc9e*/ 0x2d51;
uint64_t value = static_cast<uint64_t>(ivalue);
#if defined(ARCH_IS_64_BIT)
#ifdef _MSC_VER
__int64 hi = __umulh(value, magic_constant);
uint64_t lo = value * magic_constant;
uint64_t hash = lo ^ hi;
#else
const __int128 res = static_cast<__int128>(value) * magic_constant;
uint64_t hash = res ^ static_cast<int64_t>(res >> 64);
#endif
hash = hash ^ (hash >> 32);
#else
uint64_t prod_lo64 = value * magic_constant;
uint64_t value_lo32 = value & 0xffffffff;
uint64_t value_hi32 = value >> 32;
uint64_t carry = (((value_hi32 * magic_constant) & 0xffffffff) +
((value_lo32 * magic_constant) >> 32)) >>
32;
uint64_t prod_hi64 = ((value_hi32 * magic_constant) >> 32) + carry;
uint64_t hash = prod_hi64 ^ prod_lo64;
hash = hash ^ (hash >> 32);
#endif
return hash & 0x3fffffff;
}
// Mint natives.
DEFINE_NATIVE_ENTRY(Mint_bitNegate, 0, 1) {
const Mint& operand = Mint::CheckedHandle(zone, arguments->NativeArgAt(0));
ASSERT(CheckInteger(operand));
if (FLAG_trace_intrinsified_natives) {
OS::PrintErr("Mint_bitNegate: %s\n", operand.ToCString());
}
int64_t result = ~operand.Value();
return Integer::New(result);
}
DEFINE_NATIVE_ENTRY(Mint_bitLength, 0, 1) {
const Mint& operand = Mint::CheckedHandle(zone, arguments->NativeArgAt(0));
ASSERT(CheckInteger(operand));
if (FLAG_trace_intrinsified_natives) {
OS::PrintErr("Mint_bitLength: %s\n", operand.ToCString());
}
int64_t value = operand.Value();
intptr_t result = Utils::BitLength(value);
ASSERT(Smi::IsValid(result));
return Smi::New(result);
}
} // namespace dart