diff --git a/runtime/lib/integers.cc b/runtime/lib/integers.cc index d055a6947c4..6b566e9a3b9 100644 --- a/runtime/lib/integers.cc +++ b/runtime/lib/integers.cc @@ -267,26 +267,6 @@ DEFINE_NATIVE_ENTRY(Smi_bitLength, 0, 1) { return Smi::New(result); } -// Unified bit-count natives. Receiver is _IntegerImplementation, so the -// operand can be either Smi or Mint at runtime. -DEFINE_NATIVE_ENTRY(Integer_trailingZeroBitCount, 0, 1) { - const Integer& operand = - Integer::CheckedHandle(zone, arguments->NativeArgAt(0)); - intptr_t result = - Utils::CountTrailingZeros64(static_cast(operand.Value())); - ASSERT(Smi::IsValid(result)); - return Smi::New(result); -} - -DEFINE_NATIVE_ENTRY(Integer_oneBitCount, 0, 1) { - const Integer& operand = - Integer::CheckedHandle(zone, arguments->NativeArgAt(0)); - intptr_t result = - Utils::CountOneBits64(static_cast(operand.Value())); - ASSERT(Smi::IsValid(result)); - return Smi::New(result); -} - // Should be kept in sync with il_*.cc EmitHashIntegerCodeSequence uint32_t Multiply64Hash(int64_t ivalue) { const uint64_t magic_constant = /*0x1b873593cc9e*/ 0x2d51; diff --git a/runtime/vm/bootstrap_natives.h b/runtime/vm/bootstrap_natives.h index 551ecd2c67b..67dee2f39c2 100644 --- a/runtime/vm/bootstrap_natives.h +++ b/runtime/vm/bootstrap_natives.h @@ -65,8 +65,6 @@ namespace dart { V(SendPort_sendInternal_, 2) \ V(Smi_bitNegate, 1) \ V(Smi_bitLength, 1) \ - V(Integer_trailingZeroBitCount, 1) \ - V(Integer_oneBitCount, 1) \ V(SuspendState_instantiateClosureWithFutureTypeArgument, 2) \ V(Mint_bitNegate, 1) \ V(Mint_bitLength, 1) \ diff --git a/runtime/vm/compiler/assembler/assembler_arm.cc b/runtime/vm/compiler/assembler/assembler_arm.cc index aa42a571eff..584cd29019b 100644 --- a/runtime/vm/compiler/assembler/assembler_arm.cc +++ b/runtime/vm/compiler/assembler/assembler_arm.cc @@ -1391,6 +1391,28 @@ void Assembler::vmvnq(QRegister qd, QRegister qm) { EmitSIMDqqq(B25 | B24 | B23 | B10 | B8 | B7, kWordPair, qd, Q0, qm); } +void Assembler::vcnt(DRegister dd, DRegister dm) { + EmitSIMDddd(B24 | B23 | B21 | B20 | B10 | B8, kByte, dd, D0, dm); +} + +void Assembler::vpaddlu(OperandSize sz, DRegister dd, DRegister dm) { + int32_t size_bits = 0; + switch (sz) { + case kByte: + size_bits = 0; + break; + case kTwoBytes: + size_bits = B18; + break; + case kFourBytes: + size_bits = B19; + break; + default: + UNREACHABLE(); + } + EmitSIMDddd(B24 | B23 | B21 | B20 | size_bits | B9 | B7, kByte, dd, D0, dm); +} + void Assembler::vminqs(QRegister qd, QRegister qn, QRegister qm) { EmitSIMDqqq(B21 | B11 | B10 | B9 | B8, kSWord, qd, qn, qm); } diff --git a/runtime/vm/compiler/assembler/assembler_arm.h b/runtime/vm/compiler/assembler/assembler_arm.h index 00bf7cd07e0..bcd32ff791b 100644 --- a/runtime/vm/compiler/assembler/assembler_arm.h +++ b/runtime/vm/compiler/assembler/assembler_arm.h @@ -725,6 +725,9 @@ class Assembler : public AssemblerBase { void vandq(QRegister qd, QRegister qn, QRegister qm); void vmvnq(QRegister qd, QRegister qm); + void vcnt(DRegister dd, DRegister dm); + void vpaddlu(OperandSize sz, DRegister dd, DRegister dm); + void vceqqi(OperandSize sz, QRegister qd, QRegister qn, QRegister qm); void vceqqs(QRegister qd, QRegister qn, QRegister qm); void vcgeqi(OperandSize sz, QRegister qd, QRegister qn, QRegister qm); diff --git a/runtime/vm/compiler/assembler/assembler_arm64.h b/runtime/vm/compiler/assembler/assembler_arm64.h index 64e767a244a..24fbb34b9ce 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64.h +++ b/runtime/vm/compiler/assembler/assembler_arm64.h @@ -1476,6 +1476,21 @@ class Assembler : public AssemblerBase { void vrsqrtes(VRegister vd, VRegister vn) { EmitSIMDTwoRegOp(VRSQRTES, vd, vn); } + // CNT Vd.8B, Vn.8B: byte-wise population count. + void vcnt(VRegister vd, VRegister vn) { + const int32_t encoding = 0x0E205800 | + (static_cast(vn) << kVnShift) | + (static_cast(vd) << kVdShift); + Emit(encoding); + } + // UADDLV Hd, Vn.8B: sum across the 8 unsigned byte lanes of Vn into the + // 16-bit scalar in the low bits of Vd. + void vuaddlv(VRegister vd, VRegister vn) { + const int32_t encoding = 0x2E303800 | + (static_cast(vn) << kVnShift) | + (static_cast(vd) << kVdShift); + Emit(encoding); + } void vdupw(VRegister vd, Register rn) { const VRegister vn = static_cast(rn); EmitSIMDCopyOp(VDUPI, vd, vn, kFourBytes, 0, 0); diff --git a/runtime/vm/compiler/assembler/assembler_arm64_test.cc b/runtime/vm/compiler/assembler/assembler_arm64_test.cc index a979b64fa96..a87d80ae388 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64_test.cc +++ b/runtime/vm/compiler/assembler/assembler_arm64_test.cc @@ -2250,6 +2250,49 @@ ASSEMBLER_TEST_RUN(Rbit, test) { "ret\n"); } +ASSEMBLER_TEST_GENERATE(Vcnt8B, assembler) { + // 0x0F has popcount 4 in the low byte; all other bytes are zero. + __ LoadImmediate(R1, 0xF); + __ fmovdr(V0, R1); + __ vcnt(V0, V0); + __ vmovrd(R0, V0, 0); + __ ret(); +} + +ASSEMBLER_TEST_RUN(Vcnt8B, test) { + typedef int64_t (*Int64Return)() DART_UNUSED; + EXPECT_EQ(4, EXECUTE_TEST_CODE_INT64(Int64Return, test->entry())); + EXPECT_DISASSEMBLY( + "movz r1, #0xf\n" + "fmovdr v0, r1\n" + "vcnt v0, v0\n" + "vmovrd r0, v0[0]\n" + "ret\n"); +} + +ASSEMBLER_TEST_GENERATE(VcntUaddlv8B, assembler) { + // Low 16 bits set: bytes {0xFF, 0xFF, 0, 0, 0, 0, 0, 0}. + // vcnt -> {8, 8, 0, ...}; vuaddlv -> 16 in H[0]. + __ LoadImmediate(R1, 0xFFFF); + __ fmovdr(V0, R1); + __ vcnt(V0, V0); + __ vuaddlv(V0, V0); + __ fmovrs(R0, V0); + __ ret(); +} + +ASSEMBLER_TEST_RUN(VcntUaddlv8B, test) { + typedef int64_t (*Int64Return)() DART_UNUSED; + EXPECT_EQ(16, EXECUTE_TEST_CODE_INT64(Int64Return, test->entry())); + EXPECT_DISASSEMBLY( + "mov r1, 0xffff\n" + "fmovdr v0, r1\n" + "vcnt v0, v0\n" + "vuaddlv v0, v0\n" + "fmovrsw r0, v0\n" + "ret\n"); +} + // Comparisons, branching. ASSEMBLER_TEST_GENERATE(BranchALForward, assembler) { Label l; diff --git a/runtime/vm/compiler/assembler/assembler_arm_test.cc b/runtime/vm/compiler/assembler/assembler_arm_test.cc index 12f46fb5696..b0c52072511 100644 --- a/runtime/vm/compiler/assembler/assembler_arm_test.cc +++ b/runtime/vm/compiler/assembler/assembler_arm_test.cc @@ -3015,6 +3015,47 @@ ASSEMBLER_TEST_RUN(Vmvnq, test) { } } +ASSEMBLER_TEST_GENERATE(Vcnt8B, assembler) { + if (TargetCPUFeatures::neon_supported()) { + __ LoadImmediate(R1, 0xF); + __ LoadImmediate(R2, 0); + __ vmovdrr(D0, R1, R2); + __ vcnt(D0, D0); + __ vmovrs(R0, S0); + } + __ Ret(); +} + +ASSEMBLER_TEST_RUN(Vcnt8B, test) { + EXPECT(test != nullptr); + if (TargetCPUFeatures::neon_supported()) { + typedef int (*Tst)() DART_UNUSED; + EXPECT_EQ(4, EXECUTE_TEST_CODE_INT32(Tst, test->entry())); + } +} + +ASSEMBLER_TEST_GENERATE(VcntVpaddlu8B, assembler) { + if (TargetCPUFeatures::neon_supported()) { + __ LoadImmediate(R1, 0xFFFF); + __ LoadImmediate(R2, 0); + __ vmovdrr(D0, R1, R2); + __ vcnt(D0, D0); + __ vpaddlu(kByte, D0, D0); + __ vpaddlu(kTwoBytes, D0, D0); + __ vpaddlu(kFourBytes, D0, D0); + __ vmovrs(R0, S0); + } + __ Ret(); +} + +ASSEMBLER_TEST_RUN(VcntVpaddlu8B, test) { + EXPECT(test != nullptr); + if (TargetCPUFeatures::neon_supported()) { + typedef int (*Tst)() DART_UNUSED; + EXPECT_EQ(16, EXECUTE_TEST_CODE_INT32(Tst, test->entry())); + } +} + ASSEMBLER_TEST_GENERATE(Vdupb, assembler) { if (TargetCPUFeatures::neon_supported()) { __ LoadImmediate(R0, 0x00000000); diff --git a/runtime/vm/compiler/assembler/assembler_x64.h b/runtime/vm/compiler/assembler/assembler_x64.h index ffa3dad8081..02af1a54563 100644 --- a/runtime/vm/compiler/assembler/assembler_x64.h +++ b/runtime/vm/compiler/assembler/assembler_x64.h @@ -359,6 +359,7 @@ class Assembler : public AssemblerBase { REGULAR_INSTRUCTION(imul, 0xAF, 0x0F) REGULAR_INSTRUCTION(bsf, 0xBC, 0x0F) REGULAR_INSTRUCTION(bsr, 0xBD, 0x0F) + REGULAR_INSTRUCTION(rep_bsf, 0xBC, 0x0F, 0xF3) REGULAR_INSTRUCTION(popcnt, 0xB8, 0x0F, 0xF3) REGULAR_INSTRUCTION(lzcnt, 0xBD, 0x0F, 0xF3) #undef REGULAR_INSTRUCTION diff --git a/runtime/vm/compiler/assembler/assembler_x64_test.cc b/runtime/vm/compiler/assembler/assembler_x64_test.cc index 8c4e1ffdf1c..4c039b9589f 100644 --- a/runtime/vm/compiler/assembler/assembler_x64_test.cc +++ b/runtime/vm/compiler/assembler/assembler_x64_test.cc @@ -735,6 +735,31 @@ ASSEMBLER_TEST_RUN(Lzcnt, test) { "ret\n"); } +ASSEMBLER_TEST_GENERATE(RepBsf, assembler) { + __ movq(RCX, Immediate(0)); + __ LoadImmediate(RAX, Immediate(64)); + __ rep_bsfq(RAX, RCX); + __ movq(RCX, Immediate(0xFF00)); + __ LoadImmediate(RDX, Immediate(64)); + __ rep_bsfq(RDX, RCX); + __ addq(RAX, RDX); + __ ret(); +} + +ASSEMBLER_TEST_RUN(RepBsf, test) { + typedef int64_t (*RepBsfCode)(); + EXPECT_EQ(72, reinterpret_cast(test->entry())()); + EXPECT_DISASSEMBLY( + "movl rcx,0\n" + "movl rax,0x40\n" + "tzcntq rax,rcx\n" + "movl rcx,0xff00\n" + "movl rdx,0x40\n" + "tzcntq rdx,rcx\n" + "addq rax,rdx\n" + "ret\n"); +} + struct JumpAddress { uword filler1; uword filler2; diff --git a/runtime/vm/compiler/assembler/disassembler_arm.cc b/runtime/vm/compiler/assembler/disassembler_arm.cc index 57f39795b89..bc90216ecf6 100644 --- a/runtime/vm/compiler/assembler/disassembler_arm.cc +++ b/runtime/vm/compiler/assembler/disassembler_arm.cc @@ -1356,6 +1356,16 @@ void ARMDecoder::DecodeSIMDDataProcessing(Instr* instr) { (instr->Bits(20, 2) == 3) && (instr->Bits(23, 5) == 7) && (instr->Bits(16, 4) == 0)) { Format(instr, "vmvnq 'qd, 'qm"); + } else if ((instr->Bits(8, 4) == 5) && (instr->Bit(4) == 0) && + (instr->Bit(6) == 0) && (instr->Bit(7) == 0) && + (instr->Bits(20, 2) == 3) && (instr->Bits(23, 2) == 3) && + (instr->Bits(16, 4) == 0)) { + Format(instr, "vcnt 'dd, 'dm"); + } else if ((instr->Bits(8, 4) == 2) && (instr->Bit(4) == 0) && + (instr->Bit(6) == 0) && (instr->Bit(7) == 1) && + (instr->Bits(20, 2) == 3) && (instr->Bits(23, 2) == 3) && + (instr->Bits(16, 2) == 0)) { + Format(instr, "vpaddlu 'dd, 'dm"); } else if ((instr->Bits(8, 4) == 15) && (instr->Bit(4) == 0) && (instr->Bits(20, 2) == 2) && (instr->Bits(23, 2) == 0)) { Format(instr, "vminqs 'qd, 'qn, 'qm"); diff --git a/runtime/vm/compiler/assembler/disassembler_arm64.cc b/runtime/vm/compiler/assembler/disassembler_arm64.cc index 7870927c6b2..c738eff027e 100644 --- a/runtime/vm/compiler/assembler/disassembler_arm64.cc +++ b/runtime/vm/compiler/assembler/disassembler_arm64.cc @@ -1492,6 +1492,16 @@ void ARM64Decoder::DecodeSIMDTwoReg(Instr* instr) { } void ARM64Decoder::DecodeDPSimd1(Instr* instr) { + // CNT Vd.8B, Vn.8B (Q=0, U=0, size=00, opcode=00101). + if ((instr->InstructionBits() & 0xFFFFFC00) == 0x0E205800) { + Format(instr, "vcnt 'vd, 'vn"); + return; + } + // UADDLV Hd, Vn.8B (Q=0, U=1, size=00, across-lanes ADDLV form). + if ((instr->InstructionBits() & 0xFFFFFC00) == 0x2E303800) { + Format(instr, "vuaddlv 'vd, 'vn"); + return; + } if (instr->IsSIMDCopyOp()) { DecodeSIMDCopy(instr); } else if (instr->IsSIMDThreeSameOp()) { diff --git a/runtime/vm/compiler/assembler/disassembler_x86.cc b/runtime/vm/compiler/assembler/disassembler_x86.cc index 3d6671c2ccb..f72aa4713b9 100644 --- a/runtime/vm/compiler/assembler/disassembler_x86.cc +++ b/runtime/vm/compiler/assembler/disassembler_x86.cc @@ -1478,6 +1478,9 @@ int DisassemblerX64::TwoByteOpcodeInstruction(uint8_t* data) { } else if (opcode == 0xB8) { // POPCNT. current += PrintOperands(mnemonic, REG_OPER_OP_ORDER, current); + } else if (opcode == 0xBC) { + // TZCNT (rep BSF encoding). + current += PrintOperands("tzcnt", REG_OPER_OP_ORDER, current); } else if (opcode == 0xBD) { // LZCNT (rep BSR encoding). current += PrintOperands("lzcnt", REG_OPER_OP_ORDER, current); diff --git a/runtime/vm/compiler/backend/evaluator.cc b/runtime/vm/compiler/backend/evaluator.cc index 6bd7c3b8ddd..61e617e604c 100644 --- a/runtime/vm/compiler/backend/evaluator.cc +++ b/runtime/vm/compiler/backend/evaluator.cc @@ -60,6 +60,18 @@ static IntegerPtr UnaryIntegerEvaluateRaw(const Integer& value, return Integer::New(~value.Value(), Heap::kOld); } break; + case Token::kPOPCNT: + if (value.IsInteger()) { + return Integer::New(Utils::CountOneBits64(value.Value()), Heap::kOld); + } + break; + case Token::kCTZ: + if (value.IsInteger()) { + const int64_t v = value.Value(); + return Integer::New(v == 0 ? 64 : Utils::CountTrailingZeros64(v), + Heap::kOld); + } + break; default: UNREACHABLE(); } diff --git a/runtime/vm/compiler/backend/il.cc b/runtime/vm/compiler/backend/il.cc index 5d1d3504535..cfd9225a25e 100644 --- a/runtime/vm/compiler/backend/il.cc +++ b/runtime/vm/compiler/backend/il.cc @@ -2263,6 +2263,34 @@ Definition* DoubleTestOpInstr::Canonicalize(FlowGraph* flow_graph) { return HasUses() ? this : nullptr; } +bool UnaryInt64OpInstr::IsSupported(Token::Kind op_kind) { + switch (op_kind) { + case Token::kPOPCNT: +#if defined(TARGET_ARCH_ARM64) + return true; +#elif defined(TARGET_ARCH_ARM) + return TargetCPUFeatures::neon_supported(); +#elif defined(TARGET_ARCH_X64) + return TargetCPUFeatures::popcnt_supported(); +#elif defined(TARGET_ARCH_RISCV64) + return RV_baseline.Includes(RV_Zbb); +#else + return false; +#endif + case Token::kCTZ: +#if defined(TARGET_ARCH_ARM64) || defined(TARGET_ARCH_X64) || \ + defined(TARGET_ARCH_ARM) + return true; +#elif defined(TARGET_ARCH_RISCV64) + return RV_baseline.Includes(RV_Zbb); +#else + return false; +#endif + default: + return false; + } +} + UnaryIntegerOpInstr* UnaryIntegerOpInstr::Make(Representation representation, Token::Kind op_kind, Value* value, diff --git a/runtime/vm/compiler/backend/il.h b/runtime/vm/compiler/backend/il.h index a98f99c4e82..319c0108519 100644 --- a/runtime/vm/compiler/backend/il.h +++ b/runtime/vm/compiler/backend/il.h @@ -9200,7 +9200,8 @@ class UnaryIntegerOpInstr : public TemplateDefinition<1, NoThrow, Pure> { public: UnaryIntegerOpInstr(Token::Kind op_kind, Value* value, intptr_t deopt_id) : TemplateDefinition(deopt_id), op_kind_(op_kind) { - ASSERT((op_kind == Token::kNEGATE) || (op_kind == Token::kBIT_NOT)); + ASSERT((op_kind == Token::kNEGATE) || (op_kind == Token::kBIT_NOT) || + (op_kind == Token::kPOPCNT) || (op_kind == Token::kCTZ)); SetInputAt(0, value); } @@ -9294,9 +9295,12 @@ class UnaryInt64OpInstr : public UnaryIntegerOpInstr { public: UnaryInt64OpInstr(Token::Kind op_kind, Value* value, intptr_t deopt_id) : UnaryIntegerOpInstr(op_kind, value, deopt_id) { - ASSERT(op_kind == Token::kBIT_NOT || op_kind == Token::kNEGATE); + ASSERT(op_kind == Token::kBIT_NOT || op_kind == Token::kNEGATE || + op_kind == Token::kPOPCNT || op_kind == Token::kCTZ); } + static bool IsSupported(Token::Kind op_kind); + virtual bool ComputeCanDeoptimize() const { return false; } virtual Representation representation() const { return kUnboxedInt64; } diff --git a/runtime/vm/compiler/backend/il_arm.cc b/runtime/vm/compiler/backend/il_arm.cc index 7e64c0712f1..4c0d974bddf 100644 --- a/runtime/vm/compiler/backend/il_arm.cc +++ b/runtime/vm/compiler/backend/il_arm.cc @@ -6742,6 +6742,31 @@ void UnaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ sbc(out_hi, out_hi, compiler::Operand(out_hi)); __ sub(out_hi, out_hi, compiler::Operand(left_hi)); break; + case Token::kPOPCNT: { + __ vmovdrr(DTMP, left_lo, left_hi); + __ vcnt(DTMP, DTMP); + __ vpaddlu(compiler::kByte, DTMP, DTMP); + __ vpaddlu(compiler::kTwoBytes, DTMP, DTMP); + __ vpaddlu(compiler::kFourBytes, DTMP, DTMP); + __ vmovrs(out_lo, EvenSRegisterOf(DTMP)); + __ mov(out_hi, compiler::Operand(0)); + break; + } + case Token::kCTZ: { + compiler::Label hi_path, done; + __ cmp(left_lo, compiler::Operand(0)); + __ b(&hi_path, EQ); + __ rbit(out_lo, left_lo); + __ clz(out_lo, out_lo); + __ b(&done); + __ Bind(&hi_path); + __ rbit(out_lo, left_hi); + __ clz(out_lo, out_lo); + __ add(out_lo, out_lo, compiler::Operand(32)); + __ Bind(&done); + __ mov(out_hi, compiler::Operand(0)); + break; + } default: UNREACHABLE(); } diff --git a/runtime/vm/compiler/backend/il_arm64.cc b/runtime/vm/compiler/backend/il_arm64.cc index d08b3d5534d..cb64bcdddd0 100644 --- a/runtime/vm/compiler/backend/il_arm64.cc +++ b/runtime/vm/compiler/backend/il_arm64.cc @@ -5822,6 +5822,19 @@ void UnaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) { case Token::kNEGATE: __ sub(out, ZR, compiler::Operand(left)); break; + case Token::kPOPCNT: { + __ fmovdr(VTMP, left); + __ vcnt(VTMP, VTMP); + __ vuaddlv(VTMP, VTMP); + __ fmovrs(out, VTMP); + break; + } + case Token::kCTZ: + // RBIT(0) = 0 and CLZ(0) = 64, so the zero-input case naturally + // returns the platform width (matching the Dart API contract). + __ rbit(out, left); + __ clz(out, out); + break; default: UNREACHABLE(); } diff --git a/runtime/vm/compiler/backend/il_riscv.cc b/runtime/vm/compiler/backend/il_riscv.cc index b58f6a51caa..2ecd412dc1a 100644 --- a/runtime/vm/compiler/backend/il_riscv.cc +++ b/runtime/vm/compiler/backend/il_riscv.cc @@ -6302,6 +6302,12 @@ void UnaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) { case Token::kNEGATE: __ neg(out, left); break; + case Token::kPOPCNT: + __ cpop(out, left); + break; + case Token::kCTZ: + __ ctz(out, left); + break; default: UNREACHABLE(); } diff --git a/runtime/vm/compiler/backend/il_x64.cc b/runtime/vm/compiler/backend/il_x64.cc index 25dfbcfe309..9f58380458b 100644 --- a/runtime/vm/compiler/backend/il_x64.cc +++ b/runtime/vm/compiler/backend/il_x64.cc @@ -5976,11 +5976,14 @@ void BinaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) { LocationSummary* UnaryInt64OpInstr::MakeLocationSummary(Zone* zone, bool opt) const { const intptr_t kNumInputs = 1; - const intptr_t kNumTemps = 0; + const intptr_t kNumTemps = (op_kind() == Token::kCTZ) ? 1 : 0; LocationSummary* summary = new (zone) LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall); summary->set_in(0, Location::RequiresRegister()); summary->set_out(0, Location::SameAsFirstInput()); + if (op_kind() == Token::kCTZ) { + summary->set_temp(0, Location::RequiresRegister()); + } return summary; } @@ -5995,6 +5998,28 @@ void UnaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) { case Token::kNEGATE: __ negq(left); break; + case Token::kPOPCNT: + __ popcntq(out, left); + break; + case Token::kCTZ: { + // Intel Software Development Manual documents BSF behavior with zero + // input as undefined. However in reality all modern CPUs instead leave + // out unmodified - and compilers like LLVM rely on this behavior, so + // it makes sense for us to rely on it as well. This has an additional + // benefit of breaking false-data dependency on output register which + // can cause performance issues on some microarchitectures. + // + // |out| aliases |left| (SameAsFirstInput), so we stage through |tmp| + // to keep |left| readable as the BSF source. + const Register tmp = locs()->temp(0).reg(); + __ LoadImmediate(tmp, compiler::Immediate(64)); + // On CPUs supporting BMI2 extensions `rep bsf` will be interpreted as + // tzcnt, which is a faster instruction. On older CPUs rep-prefix will + // be ignored. + __ rep_bsfq(tmp, left); + __ movq(out, tmp); + break; + } default: UNREACHABLE(); } diff --git a/runtime/vm/compiler/frontend/kernel_to_il.cc b/runtime/vm/compiler/frontend/kernel_to_il.cc index 9b84f64398b..cbf0494c69a 100644 --- a/runtime/vm/compiler/frontend/kernel_to_il.cc +++ b/runtime/vm/compiler/frontend/kernel_to_il.cc @@ -1119,6 +1119,10 @@ bool FlowGraphBuilder::IsRecognizedMethodForFlowGraph( STORE_NATIVE_FIELD_NO_BARRIER(CASE) #undef CASE return true; + case MethodRecognizer::kInteger_trailingZeroBitCount: + return UnaryInt64OpInstr::IsSupported(Token::kCTZ); + case MethodRecognizer::kInteger_oneBitCount: + return UnaryInt64OpInstr::IsSupported(Token::kPOPCNT); case MethodRecognizer::kDoubleToInteger: case MethodRecognizer::kDoubleMod: case MethodRecognizer::kDoubleRem: @@ -1771,6 +1775,24 @@ FlowGraph* FlowGraphBuilder::BuildGraphOfRecognizedMethod( body += BuildDoubleHashCode(); body += Box(kUnboxedInt64); } break; + case MethodRecognizer::kInteger_trailingZeroBitCount: + case MethodRecognizer::kInteger_oneBitCount: { + const auto op_kind = + kind == MethodRecognizer::kInteger_trailingZeroBitCount + ? Token::kCTZ + : Token::kPOPCNT; + if (UnaryInt64OpInstr::IsSupported(op_kind)) { + ASSERT_EQUAL(function.NumParameters(), 1); + body += LoadLocal(parsed_function_->RawParameterVariable(0)); + body += UnboxTruncate(kUnboxedInt64); + Value* value = Pop(); + UnaryInt64OpInstr* op = + new (Z) UnaryInt64OpInstr(op_kind, value, DeoptId::kNone); + Push(op); + body <<= op; + body += Box(kUnboxedInt64); + } + } break; case MethodRecognizer::kFfiAsExternalTypedDataInt8: case MethodRecognizer::kFfiAsExternalTypedDataInt16: case MethodRecognizer::kFfiAsExternalTypedDataInt32: diff --git a/runtime/vm/compiler/recognized_methods_list.h b/runtime/vm/compiler/recognized_methods_list.h index bbcf6452795..5859c861b1e 100644 --- a/runtime/vm/compiler/recognized_methods_list.h +++ b/runtime/vm/compiler/recognized_methods_list.h @@ -76,6 +76,10 @@ namespace dart { V(CoreLibrary, _WeakReference, set:_target, WeakReference_setTarget, \ 0xc70c51ba) \ V(CoreLibrary, _Smi, get:hashCode, Smi_hashCode, 0x75c3b512) \ + V(CoreLibrary, _IntegerImplementation, get:trailingZeroBitCount, \ + Integer_trailingZeroBitCount, 0xea8b2826) \ + V(CoreLibrary, _IntegerImplementation, get:oneBitCount, \ + Integer_oneBitCount, 0x60c41821) \ V(CoreLibrary, _Mint, get:hashCode, Mint_hashCode, 0x75c3b512) \ V(CoreLibrary, _Double, get:hashCode, Double_hashCode, 0x75c3b8d3) \ V(CompactHashLibrary, _LinkedHashBase, get:_index, LinkedHashBase_getIndex, \ diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index 20e6d04c883..ac646b09e4d 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -9383,6 +9383,8 @@ bool Function::RecognizedKindForceOptimize() const { case MethodRecognizer::kStringBaseCodeUnitAt: case MethodRecognizer::kUtf8DecoderScan: case MethodRecognizer::kDouble_hashCode: + case MethodRecognizer::kInteger_trailingZeroBitCount: + case MethodRecognizer::kInteger_oneBitCount: case MethodRecognizer::kTypedList_GetInt8: case MethodRecognizer::kTypedList_SetInt8: case MethodRecognizer::kTypedList_GetUint8: diff --git a/runtime/vm/simulator_arm.cc b/runtime/vm/simulator_arm.cc index 62c076fa3aa..839e78618d2 100644 --- a/runtime/vm/simulator_arm.cc +++ b/runtime/vm/simulator_arm.cc @@ -3409,6 +3409,53 @@ void Simulator::DecodeSIMDDataProcessing(Instr* instr) { } set_dregister_bits(dd, result); + } else if ((instr->Bits(8, 4) == 5) && (instr->Bit(7) == 0) && + (instr->Bit(5) == 0) && (instr->Bit(4) == 0) && + (instr->Bits(16, 4) == 0) && (instr->Bits(20, 2) == 3) && + (instr->Bits(23, 5) == 7)) { + // Format(instr, "vcnt.8 'dd, 'dm"); + DRegister dd = instr->DdField(); + DRegister dm = instr->DmField(); + uint64_t dm_value = static_cast(get_dregister_bits(dm)); + uint64_t result = 0; + uint8_t* in = reinterpret_cast(&dm_value); + uint8_t* out = reinterpret_cast(&result); + for (int i = 0; i < 8; i++) { + out[i] = Utils::CountOneBits32(in[i]); + } + set_dregister_bits(dd, static_cast(result)); + } else if ((instr->Bits(8, 4) == 2) && (instr->Bit(7) == 1) && + (instr->Bit(5) == 0) && (instr->Bit(4) == 0) && + (instr->Bits(16, 2) == 0) && (instr->Bits(20, 2) == 3) && + (instr->Bits(23, 5) == 7)) { + // Format(instr, "vpaddl.u 'dd, 'dm"); + DRegister dd = instr->DdField(); + DRegister dm = instr->DmField(); + const int size = instr->Bits(18, 2); + uint64_t dm_value = static_cast(get_dregister_bits(dm)); + uint64_t result = 0; + if (size == 0) { + uint8_t* in = reinterpret_cast(&dm_value); + uint16_t* out = reinterpret_cast(&result); + for (int i = 0; i < 4; i++) { + out[i] = static_cast(in[2 * i]) + + static_cast(in[2 * i + 1]); + } + } else if (size == 1) { + uint16_t* in = reinterpret_cast(&dm_value); + uint32_t* out = reinterpret_cast(&result); + for (int i = 0; i < 2; i++) { + out[i] = static_cast(in[2 * i]) + + static_cast(in[2 * i + 1]); + } + } else if (size == 2) { + uint32_t* in = reinterpret_cast(&dm_value); + result = static_cast(in[0]) + static_cast(in[1]); + } else { + UnimplementedInstruction(instr); + return; + } + set_dregister_bits(dd, static_cast(result)); } else { UnimplementedInstruction(instr); } diff --git a/runtime/vm/simulator_arm64.cc b/runtime/vm/simulator_arm64.cc index 0d49f6bf6f2..b44636f4a57 100644 --- a/runtime/vm/simulator_arm64.cc +++ b/runtime/vm/simulator_arm64.cc @@ -3560,6 +3560,21 @@ void Simulator::DecodeSIMDTwoReg(Instr* instr) { const VRegister vd = instr->VdField(); const VRegister vn = instr->VnField(); + if ((U == 0) && (op == 5) && (sz == 0)) { + // Format(instr, "vcnt 'vd.{8,16}B, 'vn.{8,16}B"); + const int lanes = (Q == 1) ? 16 : 8; + int64_t vn_bits[2] = {get_vregisterd(vn, 0), get_vregisterd(vn, 1)}; + int64_t result[2] = {0, 0}; + uint8_t* in = reinterpret_cast(&vn_bits[0]); + uint8_t* out = reinterpret_cast(&result[0]); + for (int i = 0; i < lanes; i++) { + out[i] = Utils::CountOneBits32(in[i]); + } + set_vregisterd(vd, 0, result[0]); + set_vregisterd(vd, 1, result[1]); + return; + } + if (Q != 1) { UnimplementedInstruction(instr); return; @@ -3654,6 +3669,32 @@ void Simulator::DecodeSIMDTwoReg(Instr* instr) { } void Simulator::DecodeDPSimd1(Instr* instr) { + // UADDLV Hd, Vn. — Advanced SIMD across-vector unsigned add long. + // Encoding: 0 Q 1 01110 sz 11000 00011 10 Vn Vd. + if ((instr->Bits(24, 5) == 0xE) && (instr->Bit(29) == 1) && + (instr->Bits(17, 5) == 0x18) && (instr->Bits(12, 5) == 0x3) && + (instr->Bits(10, 2) == 0x2)) { + const int32_t Q = instr->Bit(30); + const int32_t sz = instr->Bits(22, 2); + const VRegister vd = instr->VdField(); + const VRegister vn = instr->VnField(); + if (sz == 0) { + // Format(instr, "vuaddlv 'hd, 'vn.{8,16}B"); + int64_t vn_bits[2] = {get_vregisterd(vn, 0), get_vregisterd(vn, 1)}; + uint8_t* in = reinterpret_cast(&vn_bits[0]); + const int lanes = (Q == 1) ? 16 : 8; + uint32_t sum = 0; + for (int i = 0; i < lanes; i++) { + sum += in[i]; + } + set_vregisterd(vd, 0, static_cast(sum)); + set_vregisterd(vd, 1, 0); + return; + } + UnimplementedInstruction(instr); + return; + } + if (instr->IsSIMDCopyOp()) { DecodeSIMDCopy(instr); } else if (instr->IsSIMDThreeSameOp()) { diff --git a/runtime/vm/token.h b/runtime/vm/token.h index 9df11d1b658..4afc9b46687 100644 --- a/runtime/vm/token.h +++ b/runtime/vm/token.h @@ -60,6 +60,10 @@ namespace dart { TOK(kBIT_AND, "&", 11, kNoAttribute) \ TOK(kBIT_NOT, "~", 0, kNoAttribute) \ \ + /* Bit-counting. */ \ + TOK(kPOPCNT, "@popcnt", 0, kNoAttribute) \ + TOK(kCTZ, "@ctz", 0, kNoAttribute) \ + \ /* Shift operators. */ \ TOK(kSHL, "<<", 12, kNoAttribute) \ TOK(kSHR, ">>", 12, kNoAttribute) \ diff --git a/sdk/lib/_internal/vm/lib/integers.dart b/sdk/lib/_internal/vm/lib/integers.dart index 80dcd34ad4b..2b58e6c21bb 100644 --- a/sdk/lib/_internal/vm/lib/integers.dart +++ b/sdk/lib/_internal/vm/lib/integers.dart @@ -163,13 +163,24 @@ abstract final class _IntegerImplementation implements int { @pragma("vm:external-name", "Integer_equalToInteger") external bool _equalToInteger(int other); + @pragma("vm:recognized", "other") + @pragma("vm:prefer-inline") @pragma("vm:exact-result-type", "dart:core#_Smi") - @pragma("vm:external-name", "Integer_trailingZeroBitCount") - external int get trailingZeroBitCount; + int get trailingZeroBitCount { + final int v = this; + return ((v - 1) & ~v).oneBitCount; + } + @pragma("vm:recognized", "other") + @pragma("vm:prefer-inline") @pragma("vm:exact-result-type", "dart:core#_Smi") - @pragma("vm:external-name", "Integer_oneBitCount") - external int get oneBitCount; + int get oneBitCount { + int v = this; + v = v - ((v >>> 1) & 0x5555555555555555); + v = (v & 0x3333333333333333) + ((v >>> 2) & 0x3333333333333333); + v = (v + (v >>> 4)) & 0x0F0F0F0F0F0F0F0F; + return ((v * 0x0101010101010101) >>> 56); + } int abs() { return this < 0 ? -this : this;