From 5b0285866d056e7406219f284c26c51712c8a6bd Mon Sep 17 00:00:00 2001 From: Alexander Aprelev Date: Fri, 29 May 2026 10:29:33 -0700 Subject: [PATCH] Revert "Reland "[vm] Recognize int.trailingZeroBitCount/oneBitCount as graph-inlinable"" This reverts commit 415b040d6fe730aea6d0db960ec5c1dbcc8a1d4f. Reason for revert: breaks riscv https://github.com/dart-lang/sdk/issues/63479 Original change's description: > Reland "[vm] Recognize int.trailingZeroBitCount/oneBitCount as graph-inlinable" > > The previous attempt was reverted because it broke unoptimized JIT > on ARM 32. This reland force-optimizes the two getters. > > Stacks on top of the int.{trailingZeroBitCount,oneBitCount} API CL > (commit 754239b077e). Both getters route through OTHER_RECOGNIZED_LIST > when a hardware fast path is available; otherwise the newly added > Dart bodies inline at call sites via vm:prefer-inline. The C++ > natives are removed. > > Backend codegen > --------------- > ARM64: NEON CNT + UADDLV (popcount); RBIT + CLZ (ctz). > ARM: NEON CNT + VPADDL chain (popcount); RBIT + CLZ on the > register pair (ctz). > x64: popcntq when TargetCPUFeatures::popcnt_supported(); > LoadImmediate(64) + rep_bsfq for ctz (decodes as tzcnt > on BMI1+, preserves dest on zero otherwise). > RISC-V 64: cpop / ctz when RV_baseline includes Zbb. > > Per-arch availability is encapsulated in > UnaryInt64OpInstr::IsSupported(Token::Kind). > > Apple M-series ARM64, AOT (us/iter, lower is better): > cardinality.swar 371 > cardinality.accelerated 154 (2.4x) > forEachSetBit.swar 19031 > forEachSetBit.accelerated 4988 (3.8x) > select.swar 199 > select.accelerated 77 (2.6x) > complementCardinality.swar 399 > complementCardinality.accel 152 (2.6x) > > Work towards https://github.com/dart-lang/sdk/issues/6486 (popcount > and ctz intrinsification). > > Work towards https://github.com/dart-lang/sdk/issues/1053 (efficient > BitSet implementation). > > Fixes https://github.com/dart-lang/sdk/issues/52673 > Fixes https://github.com/dart-lang/sdk/issues/38346 > Fixes https://github.com/dart-lang/sdk/issues/63436 > Issue https://github.com/dart-lang/sdk/issues/10212 > Issue https://github.com/dart-lang/sdk/issues/5798 > TEST=tests/corelib/int_bit_count_test > > Cq-Include-Trybots: luci.dart.try:vm-linux-release-simarm-try,vm-ffi-qemu-linux-release-arm-try,vm-aot-linux-release-simarm_x64-try,vm-aot-linux-debug-simarm_x64-try,dart-sdk-linux-riscv64-try > Change-Id: Ib812cbaec6e371b9720df7a543411f78e524cac1 > Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/506060 > Reviewed-by: Martin Kustermann > Auto-Submit: Modestas Valauskas > Reviewed-by: Slava Egorov > Commit-Queue: Martin Kustermann Cq-Include-Trybots: luci.dart.try:vm-linux-release-simarm-try,vm-ffi-qemu-linux-release-arm-try,vm-aot-linux-release-simarm_x64-try,vm-aot-linux-debug-simarm_x64-try,dart-sdk-linux-riscv64-try No-Presubmit: true No-Tree-Checks: true No-Try: true Change-Id: Iaf11d03d394fa615098bed8fcdea38ba40c7e45f Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/507520 Bot-Commit: rubber-stamper@appspot.gserviceaccount.com Commit-Queue: Alexander Aprelev Reviewed-by: Kevin Moore --- runtime/lib/integers.cc | 20 ++++++++ runtime/vm/bootstrap_natives.h | 2 + .../vm/compiler/assembler/assembler_arm.cc | 22 --------- runtime/vm/compiler/assembler/assembler_arm.h | 3 -- .../vm/compiler/assembler/assembler_arm64.h | 15 ------ .../assembler/assembler_arm64_test.cc | 43 ----------------- .../compiler/assembler/assembler_arm_test.cc | 41 ---------------- runtime/vm/compiler/assembler/assembler_x64.h | 1 - .../compiler/assembler/assembler_x64_test.cc | 25 ---------- .../vm/compiler/assembler/disassembler_arm.cc | 10 ---- .../compiler/assembler/disassembler_arm64.cc | 10 ---- .../vm/compiler/assembler/disassembler_x86.cc | 3 -- runtime/vm/compiler/backend/evaluator.cc | 12 ----- runtime/vm/compiler/backend/il.cc | 28 ----------- runtime/vm/compiler/backend/il.h | 8 +--- runtime/vm/compiler/backend/il_arm.cc | 25 ---------- runtime/vm/compiler/backend/il_arm64.cc | 13 ----- runtime/vm/compiler/backend/il_riscv.cc | 6 --- runtime/vm/compiler/backend/il_x64.cc | 27 +---------- runtime/vm/compiler/frontend/kernel_to_il.cc | 22 --------- runtime/vm/compiler/recognized_methods_list.h | 4 -- runtime/vm/object.cc | 2 - runtime/vm/simulator_arm.cc | 47 ------------------- runtime/vm/simulator_arm64.cc | 41 ---------------- runtime/vm/token.h | 4 -- sdk/lib/_internal/vm/lib/integers.dart | 19 ++------ 26 files changed, 29 insertions(+), 424 deletions(-) diff --git a/runtime/lib/integers.cc b/runtime/lib/integers.cc index 6b566e9a3b9..d055a6947c4 100644 --- a/runtime/lib/integers.cc +++ b/runtime/lib/integers.cc @@ -267,6 +267,26 @@ DEFINE_NATIVE_ENTRY(Smi_bitLength, 0, 1) { return Smi::New(result); } +// Unified bit-count natives. Receiver is _IntegerImplementation, so the +// operand can be either Smi or Mint at runtime. +DEFINE_NATIVE_ENTRY(Integer_trailingZeroBitCount, 0, 1) { + const Integer& operand = + Integer::CheckedHandle(zone, arguments->NativeArgAt(0)); + intptr_t result = + Utils::CountTrailingZeros64(static_cast(operand.Value())); + ASSERT(Smi::IsValid(result)); + return Smi::New(result); +} + +DEFINE_NATIVE_ENTRY(Integer_oneBitCount, 0, 1) { + const Integer& operand = + Integer::CheckedHandle(zone, arguments->NativeArgAt(0)); + intptr_t result = + Utils::CountOneBits64(static_cast(operand.Value())); + ASSERT(Smi::IsValid(result)); + return Smi::New(result); +} + // Should be kept in sync with il_*.cc EmitHashIntegerCodeSequence uint32_t Multiply64Hash(int64_t ivalue) { const uint64_t magic_constant = /*0x1b873593cc9e*/ 0x2d51; diff --git a/runtime/vm/bootstrap_natives.h b/runtime/vm/bootstrap_natives.h index 40756aab3e0..93d4d885f6e 100644 --- a/runtime/vm/bootstrap_natives.h +++ b/runtime/vm/bootstrap_natives.h @@ -65,6 +65,8 @@ namespace dart { V(SendPort_sendInternal_, 2) \ V(Smi_bitNegate, 1) \ V(Smi_bitLength, 1) \ + V(Integer_trailingZeroBitCount, 1) \ + V(Integer_oneBitCount, 1) \ V(SuspendState_instantiateClosureWithFutureTypeArgument, 2) \ V(Mint_bitNegate, 1) \ V(Mint_bitLength, 1) \ diff --git a/runtime/vm/compiler/assembler/assembler_arm.cc b/runtime/vm/compiler/assembler/assembler_arm.cc index 584cd29019b..aa42a571eff 100644 --- a/runtime/vm/compiler/assembler/assembler_arm.cc +++ b/runtime/vm/compiler/assembler/assembler_arm.cc @@ -1391,28 +1391,6 @@ void Assembler::vmvnq(QRegister qd, QRegister qm) { EmitSIMDqqq(B25 | B24 | B23 | B10 | B8 | B7, kWordPair, qd, Q0, qm); } -void Assembler::vcnt(DRegister dd, DRegister dm) { - EmitSIMDddd(B24 | B23 | B21 | B20 | B10 | B8, kByte, dd, D0, dm); -} - -void Assembler::vpaddlu(OperandSize sz, DRegister dd, DRegister dm) { - int32_t size_bits = 0; - switch (sz) { - case kByte: - size_bits = 0; - break; - case kTwoBytes: - size_bits = B18; - break; - case kFourBytes: - size_bits = B19; - break; - default: - UNREACHABLE(); - } - EmitSIMDddd(B24 | B23 | B21 | B20 | size_bits | B9 | B7, kByte, dd, D0, dm); -} - void Assembler::vminqs(QRegister qd, QRegister qn, QRegister qm) { EmitSIMDqqq(B21 | B11 | B10 | B9 | B8, kSWord, qd, qn, qm); } diff --git a/runtime/vm/compiler/assembler/assembler_arm.h b/runtime/vm/compiler/assembler/assembler_arm.h index bcd32ff791b..00bf7cd07e0 100644 --- a/runtime/vm/compiler/assembler/assembler_arm.h +++ b/runtime/vm/compiler/assembler/assembler_arm.h @@ -725,9 +725,6 @@ class Assembler : public AssemblerBase { void vandq(QRegister qd, QRegister qn, QRegister qm); void vmvnq(QRegister qd, QRegister qm); - void vcnt(DRegister dd, DRegister dm); - void vpaddlu(OperandSize sz, DRegister dd, DRegister dm); - void vceqqi(OperandSize sz, QRegister qd, QRegister qn, QRegister qm); void vceqqs(QRegister qd, QRegister qn, QRegister qm); void vcgeqi(OperandSize sz, QRegister qd, QRegister qn, QRegister qm); diff --git a/runtime/vm/compiler/assembler/assembler_arm64.h b/runtime/vm/compiler/assembler/assembler_arm64.h index 24fbb34b9ce..64e767a244a 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64.h +++ b/runtime/vm/compiler/assembler/assembler_arm64.h @@ -1476,21 +1476,6 @@ class Assembler : public AssemblerBase { void vrsqrtes(VRegister vd, VRegister vn) { EmitSIMDTwoRegOp(VRSQRTES, vd, vn); } - // CNT Vd.8B, Vn.8B: byte-wise population count. - void vcnt(VRegister vd, VRegister vn) { - const int32_t encoding = 0x0E205800 | - (static_cast(vn) << kVnShift) | - (static_cast(vd) << kVdShift); - Emit(encoding); - } - // UADDLV Hd, Vn.8B: sum across the 8 unsigned byte lanes of Vn into the - // 16-bit scalar in the low bits of Vd. - void vuaddlv(VRegister vd, VRegister vn) { - const int32_t encoding = 0x2E303800 | - (static_cast(vn) << kVnShift) | - (static_cast(vd) << kVdShift); - Emit(encoding); - } void vdupw(VRegister vd, Register rn) { const VRegister vn = static_cast(rn); EmitSIMDCopyOp(VDUPI, vd, vn, kFourBytes, 0, 0); diff --git a/runtime/vm/compiler/assembler/assembler_arm64_test.cc b/runtime/vm/compiler/assembler/assembler_arm64_test.cc index a87d80ae388..a979b64fa96 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64_test.cc +++ b/runtime/vm/compiler/assembler/assembler_arm64_test.cc @@ -2250,49 +2250,6 @@ ASSEMBLER_TEST_RUN(Rbit, test) { "ret\n"); } -ASSEMBLER_TEST_GENERATE(Vcnt8B, assembler) { - // 0x0F has popcount 4 in the low byte; all other bytes are zero. - __ LoadImmediate(R1, 0xF); - __ fmovdr(V0, R1); - __ vcnt(V0, V0); - __ vmovrd(R0, V0, 0); - __ ret(); -} - -ASSEMBLER_TEST_RUN(Vcnt8B, test) { - typedef int64_t (*Int64Return)() DART_UNUSED; - EXPECT_EQ(4, EXECUTE_TEST_CODE_INT64(Int64Return, test->entry())); - EXPECT_DISASSEMBLY( - "movz r1, #0xf\n" - "fmovdr v0, r1\n" - "vcnt v0, v0\n" - "vmovrd r0, v0[0]\n" - "ret\n"); -} - -ASSEMBLER_TEST_GENERATE(VcntUaddlv8B, assembler) { - // Low 16 bits set: bytes {0xFF, 0xFF, 0, 0, 0, 0, 0, 0}. - // vcnt -> {8, 8, 0, ...}; vuaddlv -> 16 in H[0]. - __ LoadImmediate(R1, 0xFFFF); - __ fmovdr(V0, R1); - __ vcnt(V0, V0); - __ vuaddlv(V0, V0); - __ fmovrs(R0, V0); - __ ret(); -} - -ASSEMBLER_TEST_RUN(VcntUaddlv8B, test) { - typedef int64_t (*Int64Return)() DART_UNUSED; - EXPECT_EQ(16, EXECUTE_TEST_CODE_INT64(Int64Return, test->entry())); - EXPECT_DISASSEMBLY( - "mov r1, 0xffff\n" - "fmovdr v0, r1\n" - "vcnt v0, v0\n" - "vuaddlv v0, v0\n" - "fmovrsw r0, v0\n" - "ret\n"); -} - // Comparisons, branching. ASSEMBLER_TEST_GENERATE(BranchALForward, assembler) { Label l; diff --git a/runtime/vm/compiler/assembler/assembler_arm_test.cc b/runtime/vm/compiler/assembler/assembler_arm_test.cc index b0c52072511..12f46fb5696 100644 --- a/runtime/vm/compiler/assembler/assembler_arm_test.cc +++ b/runtime/vm/compiler/assembler/assembler_arm_test.cc @@ -3015,47 +3015,6 @@ ASSEMBLER_TEST_RUN(Vmvnq, test) { } } -ASSEMBLER_TEST_GENERATE(Vcnt8B, assembler) { - if (TargetCPUFeatures::neon_supported()) { - __ LoadImmediate(R1, 0xF); - __ LoadImmediate(R2, 0); - __ vmovdrr(D0, R1, R2); - __ vcnt(D0, D0); - __ vmovrs(R0, S0); - } - __ Ret(); -} - -ASSEMBLER_TEST_RUN(Vcnt8B, test) { - EXPECT(test != nullptr); - if (TargetCPUFeatures::neon_supported()) { - typedef int (*Tst)() DART_UNUSED; - EXPECT_EQ(4, EXECUTE_TEST_CODE_INT32(Tst, test->entry())); - } -} - -ASSEMBLER_TEST_GENERATE(VcntVpaddlu8B, assembler) { - if (TargetCPUFeatures::neon_supported()) { - __ LoadImmediate(R1, 0xFFFF); - __ LoadImmediate(R2, 0); - __ vmovdrr(D0, R1, R2); - __ vcnt(D0, D0); - __ vpaddlu(kByte, D0, D0); - __ vpaddlu(kTwoBytes, D0, D0); - __ vpaddlu(kFourBytes, D0, D0); - __ vmovrs(R0, S0); - } - __ Ret(); -} - -ASSEMBLER_TEST_RUN(VcntVpaddlu8B, test) { - EXPECT(test != nullptr); - if (TargetCPUFeatures::neon_supported()) { - typedef int (*Tst)() DART_UNUSED; - EXPECT_EQ(16, EXECUTE_TEST_CODE_INT32(Tst, test->entry())); - } -} - ASSEMBLER_TEST_GENERATE(Vdupb, assembler) { if (TargetCPUFeatures::neon_supported()) { __ LoadImmediate(R0, 0x00000000); diff --git a/runtime/vm/compiler/assembler/assembler_x64.h b/runtime/vm/compiler/assembler/assembler_x64.h index 02af1a54563..ffa3dad8081 100644 --- a/runtime/vm/compiler/assembler/assembler_x64.h +++ b/runtime/vm/compiler/assembler/assembler_x64.h @@ -359,7 +359,6 @@ class Assembler : public AssemblerBase { REGULAR_INSTRUCTION(imul, 0xAF, 0x0F) REGULAR_INSTRUCTION(bsf, 0xBC, 0x0F) REGULAR_INSTRUCTION(bsr, 0xBD, 0x0F) - REGULAR_INSTRUCTION(rep_bsf, 0xBC, 0x0F, 0xF3) REGULAR_INSTRUCTION(popcnt, 0xB8, 0x0F, 0xF3) REGULAR_INSTRUCTION(lzcnt, 0xBD, 0x0F, 0xF3) #undef REGULAR_INSTRUCTION diff --git a/runtime/vm/compiler/assembler/assembler_x64_test.cc b/runtime/vm/compiler/assembler/assembler_x64_test.cc index 4c039b9589f..8c4e1ffdf1c 100644 --- a/runtime/vm/compiler/assembler/assembler_x64_test.cc +++ b/runtime/vm/compiler/assembler/assembler_x64_test.cc @@ -735,31 +735,6 @@ ASSEMBLER_TEST_RUN(Lzcnt, test) { "ret\n"); } -ASSEMBLER_TEST_GENERATE(RepBsf, assembler) { - __ movq(RCX, Immediate(0)); - __ LoadImmediate(RAX, Immediate(64)); - __ rep_bsfq(RAX, RCX); - __ movq(RCX, Immediate(0xFF00)); - __ LoadImmediate(RDX, Immediate(64)); - __ rep_bsfq(RDX, RCX); - __ addq(RAX, RDX); - __ ret(); -} - -ASSEMBLER_TEST_RUN(RepBsf, test) { - typedef int64_t (*RepBsfCode)(); - EXPECT_EQ(72, reinterpret_cast(test->entry())()); - EXPECT_DISASSEMBLY( - "movl rcx,0\n" - "movl rax,0x40\n" - "tzcntq rax,rcx\n" - "movl rcx,0xff00\n" - "movl rdx,0x40\n" - "tzcntq rdx,rcx\n" - "addq rax,rdx\n" - "ret\n"); -} - struct JumpAddress { uword filler1; uword filler2; diff --git a/runtime/vm/compiler/assembler/disassembler_arm.cc b/runtime/vm/compiler/assembler/disassembler_arm.cc index bc90216ecf6..57f39795b89 100644 --- a/runtime/vm/compiler/assembler/disassembler_arm.cc +++ b/runtime/vm/compiler/assembler/disassembler_arm.cc @@ -1356,16 +1356,6 @@ void ARMDecoder::DecodeSIMDDataProcessing(Instr* instr) { (instr->Bits(20, 2) == 3) && (instr->Bits(23, 5) == 7) && (instr->Bits(16, 4) == 0)) { Format(instr, "vmvnq 'qd, 'qm"); - } else if ((instr->Bits(8, 4) == 5) && (instr->Bit(4) == 0) && - (instr->Bit(6) == 0) && (instr->Bit(7) == 0) && - (instr->Bits(20, 2) == 3) && (instr->Bits(23, 2) == 3) && - (instr->Bits(16, 4) == 0)) { - Format(instr, "vcnt 'dd, 'dm"); - } else if ((instr->Bits(8, 4) == 2) && (instr->Bit(4) == 0) && - (instr->Bit(6) == 0) && (instr->Bit(7) == 1) && - (instr->Bits(20, 2) == 3) && (instr->Bits(23, 2) == 3) && - (instr->Bits(16, 2) == 0)) { - Format(instr, "vpaddlu 'dd, 'dm"); } else if ((instr->Bits(8, 4) == 15) && (instr->Bit(4) == 0) && (instr->Bits(20, 2) == 2) && (instr->Bits(23, 2) == 0)) { Format(instr, "vminqs 'qd, 'qn, 'qm"); diff --git a/runtime/vm/compiler/assembler/disassembler_arm64.cc b/runtime/vm/compiler/assembler/disassembler_arm64.cc index c738eff027e..7870927c6b2 100644 --- a/runtime/vm/compiler/assembler/disassembler_arm64.cc +++ b/runtime/vm/compiler/assembler/disassembler_arm64.cc @@ -1492,16 +1492,6 @@ void ARM64Decoder::DecodeSIMDTwoReg(Instr* instr) { } void ARM64Decoder::DecodeDPSimd1(Instr* instr) { - // CNT Vd.8B, Vn.8B (Q=0, U=0, size=00, opcode=00101). - if ((instr->InstructionBits() & 0xFFFFFC00) == 0x0E205800) { - Format(instr, "vcnt 'vd, 'vn"); - return; - } - // UADDLV Hd, Vn.8B (Q=0, U=1, size=00, across-lanes ADDLV form). - if ((instr->InstructionBits() & 0xFFFFFC00) == 0x2E303800) { - Format(instr, "vuaddlv 'vd, 'vn"); - return; - } if (instr->IsSIMDCopyOp()) { DecodeSIMDCopy(instr); } else if (instr->IsSIMDThreeSameOp()) { diff --git a/runtime/vm/compiler/assembler/disassembler_x86.cc b/runtime/vm/compiler/assembler/disassembler_x86.cc index f72aa4713b9..3d6671c2ccb 100644 --- a/runtime/vm/compiler/assembler/disassembler_x86.cc +++ b/runtime/vm/compiler/assembler/disassembler_x86.cc @@ -1478,9 +1478,6 @@ int DisassemblerX64::TwoByteOpcodeInstruction(uint8_t* data) { } else if (opcode == 0xB8) { // POPCNT. current += PrintOperands(mnemonic, REG_OPER_OP_ORDER, current); - } else if (opcode == 0xBC) { - // TZCNT (rep BSF encoding). - current += PrintOperands("tzcnt", REG_OPER_OP_ORDER, current); } else if (opcode == 0xBD) { // LZCNT (rep BSR encoding). current += PrintOperands("lzcnt", REG_OPER_OP_ORDER, current); diff --git a/runtime/vm/compiler/backend/evaluator.cc b/runtime/vm/compiler/backend/evaluator.cc index 61e617e604c..6bd7c3b8ddd 100644 --- a/runtime/vm/compiler/backend/evaluator.cc +++ b/runtime/vm/compiler/backend/evaluator.cc @@ -60,18 +60,6 @@ static IntegerPtr UnaryIntegerEvaluateRaw(const Integer& value, return Integer::New(~value.Value(), Heap::kOld); } break; - case Token::kPOPCNT: - if (value.IsInteger()) { - return Integer::New(Utils::CountOneBits64(value.Value()), Heap::kOld); - } - break; - case Token::kCTZ: - if (value.IsInteger()) { - const int64_t v = value.Value(); - return Integer::New(v == 0 ? 64 : Utils::CountTrailingZeros64(v), - Heap::kOld); - } - break; default: UNREACHABLE(); } diff --git a/runtime/vm/compiler/backend/il.cc b/runtime/vm/compiler/backend/il.cc index cfd9225a25e..5d1d3504535 100644 --- a/runtime/vm/compiler/backend/il.cc +++ b/runtime/vm/compiler/backend/il.cc @@ -2263,34 +2263,6 @@ Definition* DoubleTestOpInstr::Canonicalize(FlowGraph* flow_graph) { return HasUses() ? this : nullptr; } -bool UnaryInt64OpInstr::IsSupported(Token::Kind op_kind) { - switch (op_kind) { - case Token::kPOPCNT: -#if defined(TARGET_ARCH_ARM64) - return true; -#elif defined(TARGET_ARCH_ARM) - return TargetCPUFeatures::neon_supported(); -#elif defined(TARGET_ARCH_X64) - return TargetCPUFeatures::popcnt_supported(); -#elif defined(TARGET_ARCH_RISCV64) - return RV_baseline.Includes(RV_Zbb); -#else - return false; -#endif - case Token::kCTZ: -#if defined(TARGET_ARCH_ARM64) || defined(TARGET_ARCH_X64) || \ - defined(TARGET_ARCH_ARM) - return true; -#elif defined(TARGET_ARCH_RISCV64) - return RV_baseline.Includes(RV_Zbb); -#else - return false; -#endif - default: - return false; - } -} - UnaryIntegerOpInstr* UnaryIntegerOpInstr::Make(Representation representation, Token::Kind op_kind, Value* value, diff --git a/runtime/vm/compiler/backend/il.h b/runtime/vm/compiler/backend/il.h index 319c0108519..a98f99c4e82 100644 --- a/runtime/vm/compiler/backend/il.h +++ b/runtime/vm/compiler/backend/il.h @@ -9200,8 +9200,7 @@ class UnaryIntegerOpInstr : public TemplateDefinition<1, NoThrow, Pure> { public: UnaryIntegerOpInstr(Token::Kind op_kind, Value* value, intptr_t deopt_id) : TemplateDefinition(deopt_id), op_kind_(op_kind) { - ASSERT((op_kind == Token::kNEGATE) || (op_kind == Token::kBIT_NOT) || - (op_kind == Token::kPOPCNT) || (op_kind == Token::kCTZ)); + ASSERT((op_kind == Token::kNEGATE) || (op_kind == Token::kBIT_NOT)); SetInputAt(0, value); } @@ -9295,12 +9294,9 @@ class UnaryInt64OpInstr : public UnaryIntegerOpInstr { public: UnaryInt64OpInstr(Token::Kind op_kind, Value* value, intptr_t deopt_id) : UnaryIntegerOpInstr(op_kind, value, deopt_id) { - ASSERT(op_kind == Token::kBIT_NOT || op_kind == Token::kNEGATE || - op_kind == Token::kPOPCNT || op_kind == Token::kCTZ); + ASSERT(op_kind == Token::kBIT_NOT || op_kind == Token::kNEGATE); } - static bool IsSupported(Token::Kind op_kind); - virtual bool ComputeCanDeoptimize() const { return false; } virtual Representation representation() const { return kUnboxedInt64; } diff --git a/runtime/vm/compiler/backend/il_arm.cc b/runtime/vm/compiler/backend/il_arm.cc index 4c0d974bddf..7e64c0712f1 100644 --- a/runtime/vm/compiler/backend/il_arm.cc +++ b/runtime/vm/compiler/backend/il_arm.cc @@ -6742,31 +6742,6 @@ void UnaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ sbc(out_hi, out_hi, compiler::Operand(out_hi)); __ sub(out_hi, out_hi, compiler::Operand(left_hi)); break; - case Token::kPOPCNT: { - __ vmovdrr(DTMP, left_lo, left_hi); - __ vcnt(DTMP, DTMP); - __ vpaddlu(compiler::kByte, DTMP, DTMP); - __ vpaddlu(compiler::kTwoBytes, DTMP, DTMP); - __ vpaddlu(compiler::kFourBytes, DTMP, DTMP); - __ vmovrs(out_lo, EvenSRegisterOf(DTMP)); - __ mov(out_hi, compiler::Operand(0)); - break; - } - case Token::kCTZ: { - compiler::Label hi_path, done; - __ cmp(left_lo, compiler::Operand(0)); - __ b(&hi_path, EQ); - __ rbit(out_lo, left_lo); - __ clz(out_lo, out_lo); - __ b(&done); - __ Bind(&hi_path); - __ rbit(out_lo, left_hi); - __ clz(out_lo, out_lo); - __ add(out_lo, out_lo, compiler::Operand(32)); - __ Bind(&done); - __ mov(out_hi, compiler::Operand(0)); - break; - } default: UNREACHABLE(); } diff --git a/runtime/vm/compiler/backend/il_arm64.cc b/runtime/vm/compiler/backend/il_arm64.cc index cb64bcdddd0..d08b3d5534d 100644 --- a/runtime/vm/compiler/backend/il_arm64.cc +++ b/runtime/vm/compiler/backend/il_arm64.cc @@ -5822,19 +5822,6 @@ void UnaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) { case Token::kNEGATE: __ sub(out, ZR, compiler::Operand(left)); break; - case Token::kPOPCNT: { - __ fmovdr(VTMP, left); - __ vcnt(VTMP, VTMP); - __ vuaddlv(VTMP, VTMP); - __ fmovrs(out, VTMP); - break; - } - case Token::kCTZ: - // RBIT(0) = 0 and CLZ(0) = 64, so the zero-input case naturally - // returns the platform width (matching the Dart API contract). - __ rbit(out, left); - __ clz(out, out); - break; default: UNREACHABLE(); } diff --git a/runtime/vm/compiler/backend/il_riscv.cc b/runtime/vm/compiler/backend/il_riscv.cc index 2ecd412dc1a..b58f6a51caa 100644 --- a/runtime/vm/compiler/backend/il_riscv.cc +++ b/runtime/vm/compiler/backend/il_riscv.cc @@ -6302,12 +6302,6 @@ void UnaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) { case Token::kNEGATE: __ neg(out, left); break; - case Token::kPOPCNT: - __ cpop(out, left); - break; - case Token::kCTZ: - __ ctz(out, left); - break; default: UNREACHABLE(); } diff --git a/runtime/vm/compiler/backend/il_x64.cc b/runtime/vm/compiler/backend/il_x64.cc index 9f58380458b..25dfbcfe309 100644 --- a/runtime/vm/compiler/backend/il_x64.cc +++ b/runtime/vm/compiler/backend/il_x64.cc @@ -5976,14 +5976,11 @@ void BinaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) { LocationSummary* UnaryInt64OpInstr::MakeLocationSummary(Zone* zone, bool opt) const { const intptr_t kNumInputs = 1; - const intptr_t kNumTemps = (op_kind() == Token::kCTZ) ? 1 : 0; + const intptr_t kNumTemps = 0; LocationSummary* summary = new (zone) LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall); summary->set_in(0, Location::RequiresRegister()); summary->set_out(0, Location::SameAsFirstInput()); - if (op_kind() == Token::kCTZ) { - summary->set_temp(0, Location::RequiresRegister()); - } return summary; } @@ -5998,28 +5995,6 @@ void UnaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) { case Token::kNEGATE: __ negq(left); break; - case Token::kPOPCNT: - __ popcntq(out, left); - break; - case Token::kCTZ: { - // Intel Software Development Manual documents BSF behavior with zero - // input as undefined. However in reality all modern CPUs instead leave - // out unmodified - and compilers like LLVM rely on this behavior, so - // it makes sense for us to rely on it as well. This has an additional - // benefit of breaking false-data dependency on output register which - // can cause performance issues on some microarchitectures. - // - // |out| aliases |left| (SameAsFirstInput), so we stage through |tmp| - // to keep |left| readable as the BSF source. - const Register tmp = locs()->temp(0).reg(); - __ LoadImmediate(tmp, compiler::Immediate(64)); - // On CPUs supporting BMI2 extensions `rep bsf` will be interpreted as - // tzcnt, which is a faster instruction. On older CPUs rep-prefix will - // be ignored. - __ rep_bsfq(tmp, left); - __ movq(out, tmp); - break; - } default: UNREACHABLE(); } diff --git a/runtime/vm/compiler/frontend/kernel_to_il.cc b/runtime/vm/compiler/frontend/kernel_to_il.cc index cbf0494c69a..9b84f64398b 100644 --- a/runtime/vm/compiler/frontend/kernel_to_il.cc +++ b/runtime/vm/compiler/frontend/kernel_to_il.cc @@ -1119,10 +1119,6 @@ bool FlowGraphBuilder::IsRecognizedMethodForFlowGraph( STORE_NATIVE_FIELD_NO_BARRIER(CASE) #undef CASE return true; - case MethodRecognizer::kInteger_trailingZeroBitCount: - return UnaryInt64OpInstr::IsSupported(Token::kCTZ); - case MethodRecognizer::kInteger_oneBitCount: - return UnaryInt64OpInstr::IsSupported(Token::kPOPCNT); case MethodRecognizer::kDoubleToInteger: case MethodRecognizer::kDoubleMod: case MethodRecognizer::kDoubleRem: @@ -1775,24 +1771,6 @@ FlowGraph* FlowGraphBuilder::BuildGraphOfRecognizedMethod( body += BuildDoubleHashCode(); body += Box(kUnboxedInt64); } break; - case MethodRecognizer::kInteger_trailingZeroBitCount: - case MethodRecognizer::kInteger_oneBitCount: { - const auto op_kind = - kind == MethodRecognizer::kInteger_trailingZeroBitCount - ? Token::kCTZ - : Token::kPOPCNT; - if (UnaryInt64OpInstr::IsSupported(op_kind)) { - ASSERT_EQUAL(function.NumParameters(), 1); - body += LoadLocal(parsed_function_->RawParameterVariable(0)); - body += UnboxTruncate(kUnboxedInt64); - Value* value = Pop(); - UnaryInt64OpInstr* op = - new (Z) UnaryInt64OpInstr(op_kind, value, DeoptId::kNone); - Push(op); - body <<= op; - body += Box(kUnboxedInt64); - } - } break; case MethodRecognizer::kFfiAsExternalTypedDataInt8: case MethodRecognizer::kFfiAsExternalTypedDataInt16: case MethodRecognizer::kFfiAsExternalTypedDataInt32: diff --git a/runtime/vm/compiler/recognized_methods_list.h b/runtime/vm/compiler/recognized_methods_list.h index 5859c861b1e..bbcf6452795 100644 --- a/runtime/vm/compiler/recognized_methods_list.h +++ b/runtime/vm/compiler/recognized_methods_list.h @@ -76,10 +76,6 @@ namespace dart { V(CoreLibrary, _WeakReference, set:_target, WeakReference_setTarget, \ 0xc70c51ba) \ V(CoreLibrary, _Smi, get:hashCode, Smi_hashCode, 0x75c3b512) \ - V(CoreLibrary, _IntegerImplementation, get:trailingZeroBitCount, \ - Integer_trailingZeroBitCount, 0xea8b2826) \ - V(CoreLibrary, _IntegerImplementation, get:oneBitCount, \ - Integer_oneBitCount, 0x60c41821) \ V(CoreLibrary, _Mint, get:hashCode, Mint_hashCode, 0x75c3b512) \ V(CoreLibrary, _Double, get:hashCode, Double_hashCode, 0x75c3b8d3) \ V(CompactHashLibrary, _LinkedHashBase, get:_index, LinkedHashBase_getIndex, \ diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index ac646b09e4d..20e6d04c883 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -9383,8 +9383,6 @@ bool Function::RecognizedKindForceOptimize() const { case MethodRecognizer::kStringBaseCodeUnitAt: case MethodRecognizer::kUtf8DecoderScan: case MethodRecognizer::kDouble_hashCode: - case MethodRecognizer::kInteger_trailingZeroBitCount: - case MethodRecognizer::kInteger_oneBitCount: case MethodRecognizer::kTypedList_GetInt8: case MethodRecognizer::kTypedList_SetInt8: case MethodRecognizer::kTypedList_GetUint8: diff --git a/runtime/vm/simulator_arm.cc b/runtime/vm/simulator_arm.cc index 839e78618d2..62c076fa3aa 100644 --- a/runtime/vm/simulator_arm.cc +++ b/runtime/vm/simulator_arm.cc @@ -3409,53 +3409,6 @@ void Simulator::DecodeSIMDDataProcessing(Instr* instr) { } set_dregister_bits(dd, result); - } else if ((instr->Bits(8, 4) == 5) && (instr->Bit(7) == 0) && - (instr->Bit(5) == 0) && (instr->Bit(4) == 0) && - (instr->Bits(16, 4) == 0) && (instr->Bits(20, 2) == 3) && - (instr->Bits(23, 5) == 7)) { - // Format(instr, "vcnt.8 'dd, 'dm"); - DRegister dd = instr->DdField(); - DRegister dm = instr->DmField(); - uint64_t dm_value = static_cast(get_dregister_bits(dm)); - uint64_t result = 0; - uint8_t* in = reinterpret_cast(&dm_value); - uint8_t* out = reinterpret_cast(&result); - for (int i = 0; i < 8; i++) { - out[i] = Utils::CountOneBits32(in[i]); - } - set_dregister_bits(dd, static_cast(result)); - } else if ((instr->Bits(8, 4) == 2) && (instr->Bit(7) == 1) && - (instr->Bit(5) == 0) && (instr->Bit(4) == 0) && - (instr->Bits(16, 2) == 0) && (instr->Bits(20, 2) == 3) && - (instr->Bits(23, 5) == 7)) { - // Format(instr, "vpaddl.u 'dd, 'dm"); - DRegister dd = instr->DdField(); - DRegister dm = instr->DmField(); - const int size = instr->Bits(18, 2); - uint64_t dm_value = static_cast(get_dregister_bits(dm)); - uint64_t result = 0; - if (size == 0) { - uint8_t* in = reinterpret_cast(&dm_value); - uint16_t* out = reinterpret_cast(&result); - for (int i = 0; i < 4; i++) { - out[i] = static_cast(in[2 * i]) + - static_cast(in[2 * i + 1]); - } - } else if (size == 1) { - uint16_t* in = reinterpret_cast(&dm_value); - uint32_t* out = reinterpret_cast(&result); - for (int i = 0; i < 2; i++) { - out[i] = static_cast(in[2 * i]) + - static_cast(in[2 * i + 1]); - } - } else if (size == 2) { - uint32_t* in = reinterpret_cast(&dm_value); - result = static_cast(in[0]) + static_cast(in[1]); - } else { - UnimplementedInstruction(instr); - return; - } - set_dregister_bits(dd, static_cast(result)); } else { UnimplementedInstruction(instr); } diff --git a/runtime/vm/simulator_arm64.cc b/runtime/vm/simulator_arm64.cc index b44636f4a57..0d49f6bf6f2 100644 --- a/runtime/vm/simulator_arm64.cc +++ b/runtime/vm/simulator_arm64.cc @@ -3560,21 +3560,6 @@ void Simulator::DecodeSIMDTwoReg(Instr* instr) { const VRegister vd = instr->VdField(); const VRegister vn = instr->VnField(); - if ((U == 0) && (op == 5) && (sz == 0)) { - // Format(instr, "vcnt 'vd.{8,16}B, 'vn.{8,16}B"); - const int lanes = (Q == 1) ? 16 : 8; - int64_t vn_bits[2] = {get_vregisterd(vn, 0), get_vregisterd(vn, 1)}; - int64_t result[2] = {0, 0}; - uint8_t* in = reinterpret_cast(&vn_bits[0]); - uint8_t* out = reinterpret_cast(&result[0]); - for (int i = 0; i < lanes; i++) { - out[i] = Utils::CountOneBits32(in[i]); - } - set_vregisterd(vd, 0, result[0]); - set_vregisterd(vd, 1, result[1]); - return; - } - if (Q != 1) { UnimplementedInstruction(instr); return; @@ -3669,32 +3654,6 @@ void Simulator::DecodeSIMDTwoReg(Instr* instr) { } void Simulator::DecodeDPSimd1(Instr* instr) { - // UADDLV Hd, Vn. — Advanced SIMD across-vector unsigned add long. - // Encoding: 0 Q 1 01110 sz 11000 00011 10 Vn Vd. - if ((instr->Bits(24, 5) == 0xE) && (instr->Bit(29) == 1) && - (instr->Bits(17, 5) == 0x18) && (instr->Bits(12, 5) == 0x3) && - (instr->Bits(10, 2) == 0x2)) { - const int32_t Q = instr->Bit(30); - const int32_t sz = instr->Bits(22, 2); - const VRegister vd = instr->VdField(); - const VRegister vn = instr->VnField(); - if (sz == 0) { - // Format(instr, "vuaddlv 'hd, 'vn.{8,16}B"); - int64_t vn_bits[2] = {get_vregisterd(vn, 0), get_vregisterd(vn, 1)}; - uint8_t* in = reinterpret_cast(&vn_bits[0]); - const int lanes = (Q == 1) ? 16 : 8; - uint32_t sum = 0; - for (int i = 0; i < lanes; i++) { - sum += in[i]; - } - set_vregisterd(vd, 0, static_cast(sum)); - set_vregisterd(vd, 1, 0); - return; - } - UnimplementedInstruction(instr); - return; - } - if (instr->IsSIMDCopyOp()) { DecodeSIMDCopy(instr); } else if (instr->IsSIMDThreeSameOp()) { diff --git a/runtime/vm/token.h b/runtime/vm/token.h index 4afc9b46687..9df11d1b658 100644 --- a/runtime/vm/token.h +++ b/runtime/vm/token.h @@ -60,10 +60,6 @@ namespace dart { TOK(kBIT_AND, "&", 11, kNoAttribute) \ TOK(kBIT_NOT, "~", 0, kNoAttribute) \ \ - /* Bit-counting. */ \ - TOK(kPOPCNT, "@popcnt", 0, kNoAttribute) \ - TOK(kCTZ, "@ctz", 0, kNoAttribute) \ - \ /* Shift operators. */ \ TOK(kSHL, "<<", 12, kNoAttribute) \ TOK(kSHR, ">>", 12, kNoAttribute) \ diff --git a/sdk/lib/_internal/vm/lib/integers.dart b/sdk/lib/_internal/vm/lib/integers.dart index 2b58e6c21bb..80dcd34ad4b 100644 --- a/sdk/lib/_internal/vm/lib/integers.dart +++ b/sdk/lib/_internal/vm/lib/integers.dart @@ -163,24 +163,13 @@ abstract final class _IntegerImplementation implements int { @pragma("vm:external-name", "Integer_equalToInteger") external bool _equalToInteger(int other); - @pragma("vm:recognized", "other") - @pragma("vm:prefer-inline") @pragma("vm:exact-result-type", "dart:core#_Smi") - int get trailingZeroBitCount { - final int v = this; - return ((v - 1) & ~v).oneBitCount; - } + @pragma("vm:external-name", "Integer_trailingZeroBitCount") + external int get trailingZeroBitCount; - @pragma("vm:recognized", "other") - @pragma("vm:prefer-inline") @pragma("vm:exact-result-type", "dart:core#_Smi") - int get oneBitCount { - int v = this; - v = v - ((v >>> 1) & 0x5555555555555555); - v = (v & 0x3333333333333333) + ((v >>> 2) & 0x3333333333333333); - v = (v + (v >>> 4)) & 0x0F0F0F0F0F0F0F0F; - return ((v * 0x0101010101010101) >>> 56); - } + @pragma("vm:external-name", "Integer_oneBitCount") + external int get oneBitCount; int abs() { return this < 0 ? -this : this;