1// Copyright 2015, ARM Limited 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are met: 6// 7// * Redistributions of source code must retain the above copyright notice, 8// this list of conditions and the following disclaimer. 9// * Redistributions in binary form must reproduce the above copyright notice, 10// this list of conditions and the following disclaimer in the documentation 11// and/or other materials provided with the distribution. 12// * Neither the name of ARM Limited nor the names of its contributors may be 13// used to endorse or promote products derived from this software without 14// specific prior written permission. 15// 16// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND 17// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 20// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 27#include "vixl/a64/macro-assembler-a64.h" 28 29namespace vixl { 30 31 32void Pool::Release() { 33 if (--monitor_ == 0) { 34 // Ensure the pool has not been blocked for too long. 35 VIXL_ASSERT(masm_->CursorOffset() < checkpoint_); 36 } 37} 38 39 40void Pool::SetNextCheckpoint(ptrdiff_t checkpoint) { 41 masm_->checkpoint_ = std::min(masm_->checkpoint_, checkpoint); 42 checkpoint_ = checkpoint; 43} 44 45 46LiteralPool::LiteralPool(MacroAssembler* masm) 47 : Pool(masm), size_(0), first_use_(-1), 48 recommended_checkpoint_(kNoCheckpointRequired) { 49} 50 51 52LiteralPool::~LiteralPool() { 53 VIXL_ASSERT(IsEmpty()); 54 VIXL_ASSERT(!IsBlocked()); 55} 56 57 58void LiteralPool::Reset() { 59 std::vector<RawLiteral*>::iterator it, end; 60 for (it = entries_.begin(), end = entries_.end(); it != end; ++it) { 61 delete *it; 62 } 63 entries_.clear(); 64 size_ = 0; 65 first_use_ = -1; 66 Pool::Reset(); 67 recommended_checkpoint_ = kNoCheckpointRequired; 68} 69 70 71void LiteralPool::CheckEmitFor(size_t amount, EmitOption option) { 72 if (IsEmpty() || IsBlocked()) return; 73 74 ptrdiff_t distance = masm_->CursorOffset() + amount - first_use_; 75 if (distance >= kRecommendedLiteralPoolRange) { 76 Emit(option); 77 } 78} 79 80 81void LiteralPool::Emit(EmitOption option) { 82 // There is an issue if we are asked to emit a blocked or empty pool. 83 VIXL_ASSERT(!IsBlocked()); 84 VIXL_ASSERT(!IsEmpty()); 85 86 size_t pool_size = Size(); 87 size_t emit_size = pool_size; 88 if (option == kBranchRequired) emit_size += kInstructionSize; 89 Label end_of_pool; 90 91 VIXL_ASSERT(emit_size % kInstructionSize == 0); 92 InstructionAccurateScope guard(masm_, emit_size / kInstructionSize); 93 if (option == kBranchRequired) masm_->b(&end_of_pool); 94 95 // Marker indicating the size of the literal pool in 32-bit words. 96 VIXL_ASSERT((pool_size % kWRegSizeInBytes) == 0); 97 masm_->ldr(xzr, pool_size / kWRegSizeInBytes); 98 99 // Now populate the literal pool. 100 std::vector<RawLiteral*>::iterator it, end; 101 for (it = entries_.begin(), end = entries_.end(); it != end; ++it) { 102 VIXL_ASSERT((*it)->IsUsed()); 103 masm_->place(*it); 104 delete *it; 105 } 106 107 if (option == kBranchRequired) masm_->bind(&end_of_pool); 108 109 entries_.clear(); 110 Reset(); 111} 112 113 114RawLiteral* LiteralPool::AddEntry(RawLiteral* literal) { 115 if (IsEmpty()) { 116 first_use_ = masm_->CursorOffset(); 117 SetNextRecommendedCheckpoint(NextRecommendedCheckpoint()); 118 SetNextCheckpoint(first_use_ + Instruction::kLoadLiteralRange); 119 } else { 120 VIXL_ASSERT(masm_->CursorOffset() > first_use_); 121 } 122 123 entries_.push_back(literal); 124 size_ += literal->size(); 125 126 return literal; 127} 128 129 130void VeneerPool::Reset() { 131 Pool::Reset(); 132 unresolved_branches_.Reset(); 133} 134 135 136void VeneerPool::Release() { 137 if (--monitor_ == 0) { 138 VIXL_ASSERT(IsEmpty() || 139 masm_->CursorOffset() < unresolved_branches_.FirstLimit()); 140 } 141} 142 143 144void VeneerPool::RegisterUnresolvedBranch(ptrdiff_t branch_pos, 145 Label* label, 146 ImmBranchType branch_type) { 147 VIXL_ASSERT(!label->IsBound()); 148 BranchInfo branch_info = BranchInfo(branch_pos, label, branch_type); 149 unresolved_branches_.insert(branch_info); 150 UpdateNextCheckPoint(); 151 // TODO: In debug mode register the label with the assembler to make sure it 152 // is bound with masm Bind and not asm bind. 153} 154 155 156void VeneerPool::DeleteUnresolvedBranchInfoForLabel(Label* label) { 157 if (IsEmpty()) { 158 VIXL_ASSERT(checkpoint_ == kNoCheckpointRequired); 159 return; 160 } 161 162 if (label->IsLinked()) { 163 Label::LabelLinksIterator links_it(label); 164 for (; !links_it.Done(); links_it.Advance()) { 165 ptrdiff_t link_offset = *links_it.Current(); 166 Instruction* link = masm_->InstructionAt(link_offset); 167 168 // ADR instructions are not handled. 169 if (BranchTypeUsesVeneers(link->BranchType())) { 170 BranchInfo branch_info(link_offset, label, link->BranchType()); 171 unresolved_branches_.erase(branch_info); 172 } 173 } 174 } 175 176 UpdateNextCheckPoint(); 177} 178 179 180bool VeneerPool::ShouldEmitVeneer(int max_reachable_pc, size_t amount) { 181 ptrdiff_t offset = 182 kPoolNonVeneerCodeSize + amount + MaxSize() + OtherPoolsMaxSize(); 183 return (masm_->CursorOffset() + offset) > max_reachable_pc; 184} 185 186 187void VeneerPool::CheckEmitFor(size_t amount, EmitOption option) { 188 if (IsEmpty()) return; 189 190 VIXL_ASSERT(masm_->CursorOffset() < unresolved_branches_.FirstLimit()); 191 192 if (IsBlocked()) return; 193 194 if (ShouldEmitVeneers(amount)) { 195 Emit(option, amount); 196 } else { 197 UpdateNextCheckPoint(); 198 } 199} 200 201 202void VeneerPool::Emit(EmitOption option, size_t amount) { 203 // There is an issue if we are asked to emit a blocked or empty pool. 204 VIXL_ASSERT(!IsBlocked()); 205 VIXL_ASSERT(!IsEmpty()); 206 207 Label end; 208 if (option == kBranchRequired) { 209 InstructionAccurateScope scope(masm_, 1); 210 masm_->b(&end); 211 } 212 213 // We want to avoid generating veneer pools too often, so generate veneers for 214 // branches that don't immediately require a veneer but will soon go out of 215 // range. 216 static const size_t kVeneerEmissionMargin = 1 * KBytes; 217 218 for (BranchInfoSetIterator it(&unresolved_branches_); !it.Done();) { 219 BranchInfo* branch_info = it.Current(); 220 if (ShouldEmitVeneer(branch_info->max_reachable_pc_, 221 amount + kVeneerEmissionMargin)) { 222 InstructionAccurateScope scope(masm_, kVeneerCodeSize / kInstructionSize); 223 ptrdiff_t branch_pos = branch_info->pc_offset_; 224 Instruction* branch = masm_->InstructionAt(branch_pos); 225 Label* label = branch_info->label_; 226 227 // Patch the branch to point to the current position, and emit a branch 228 // to the label. 229 Instruction* veneer = masm_->GetCursorAddress<Instruction*>(); 230 branch->SetImmPCOffsetTarget(veneer); 231 masm_->b(label); 232 233 // Update the label. The branch patched does not point to it any longer. 234 label->DeleteLink(branch_pos); 235 236 it.DeleteCurrentAndAdvance(); 237 } else { 238 it.AdvanceToNextType(); 239 } 240 } 241 242 UpdateNextCheckPoint(); 243 244 masm_->bind(&end); 245} 246 247 248EmissionCheckScope::EmissionCheckScope(MacroAssembler* masm, size_t size) { 249 if (masm) { 250 masm->EnsureEmitFor(size); 251#ifdef VIXL_DEBUG 252 masm_ = masm; 253 masm->Bind(&start_); 254 size_ = size; 255 masm->AcquireBuffer(); 256#endif 257 } 258} 259 260 261EmissionCheckScope::~EmissionCheckScope() { 262#ifdef VIXL_DEBUG 263 if (masm_) { 264 masm_->ReleaseBuffer(); 265 VIXL_ASSERT(masm_->SizeOfCodeGeneratedSince(&start_) <= size_); 266 } 267#endif 268} 269 270 271MacroAssembler::MacroAssembler(size_t capacity, 272 PositionIndependentCodeOption pic) 273 : Assembler(capacity, pic), 274#ifdef VIXL_DEBUG 275 allow_macro_instructions_(true), 276#endif 277 sp_(sp), 278 tmp_list_(ip0, ip1), 279 fptmp_list_(d31), 280 literal_pool_(this), 281 veneer_pool_(this), 282 recommended_checkpoint_(Pool::kNoCheckpointRequired) { 283 checkpoint_ = NextCheckPoint(); 284} 285 286 287MacroAssembler::MacroAssembler(byte * buffer, 288 size_t capacity, 289 PositionIndependentCodeOption pic) 290 : Assembler(buffer, capacity, pic), 291#ifdef VIXL_DEBUG 292 allow_macro_instructions_(true), 293#endif 294 sp_(sp), 295 tmp_list_(ip0, ip1), 296 fptmp_list_(d31), 297 literal_pool_(this), 298 veneer_pool_(this), 299 recommended_checkpoint_(Pool::kNoCheckpointRequired) { 300 checkpoint_ = NextCheckPoint(); 301} 302 303 304MacroAssembler::~MacroAssembler() { 305} 306 307 308void MacroAssembler::Reset() { 309 Assembler::Reset(); 310 311 VIXL_ASSERT(!literal_pool_.IsBlocked()); 312 literal_pool_.Reset(); 313 veneer_pool_.Reset(); 314 315 checkpoint_ = NextCheckPoint(); 316} 317 318 319void MacroAssembler::FinalizeCode() { 320 if (!literal_pool_.IsEmpty()) literal_pool_.Emit(); 321 VIXL_ASSERT(veneer_pool_.IsEmpty()); 322 323 Assembler::FinalizeCode(); 324} 325 326 327void MacroAssembler::CheckEmitFor(size_t amount) { 328 ptrdiff_t offset = amount; 329 330 literal_pool_.CheckEmitFor(amount); 331 veneer_pool_.CheckEmitFor(amount); 332 // Ensure there's enough space for the emit, keep in mind the cursor will 333 // have moved if a pool was emitted. 334 if ((CursorOffset() + offset) > BufferEndOffset()) { 335 EnsureSpaceFor(amount); 336 } 337 338 checkpoint_ = NextCheckPoint(); 339} 340 341 342int MacroAssembler::MoveImmediateHelper(MacroAssembler* masm, 343 const Register &rd, 344 uint64_t imm) { 345 bool emit_code = (masm != NULL); 346 VIXL_ASSERT(is_uint32(imm) || is_int32(imm) || rd.Is64Bits()); 347 // The worst case for size is mov 64-bit immediate to sp: 348 // * up to 4 instructions to materialise the constant 349 // * 1 instruction to move to sp 350 MacroEmissionCheckScope guard(masm); 351 352 // Immediates on Aarch64 can be produced using an initial value, and zero to 353 // three move keep operations. 354 // 355 // Initial values can be generated with: 356 // 1. 64-bit move zero (movz). 357 // 2. 32-bit move inverted (movn). 358 // 3. 64-bit move inverted. 359 // 4. 32-bit orr immediate. 360 // 5. 64-bit orr immediate. 361 // Move-keep may then be used to modify each of the 16-bit half words. 362 // 363 // The code below supports all five initial value generators, and 364 // applying move-keep operations to move-zero and move-inverted initial 365 // values. 366 367 // Try to move the immediate in one instruction, and if that fails, switch to 368 // using multiple instructions. 369 if (OneInstrMoveImmediateHelper(masm, rd, imm)) { 370 return 1; 371 } else { 372 int instruction_count = 0; 373 unsigned reg_size = rd.size(); 374 375 // Generic immediate case. Imm will be represented by 376 // [imm3, imm2, imm1, imm0], where each imm is 16 bits. 377 // A move-zero or move-inverted is generated for the first non-zero or 378 // non-0xffff immX, and a move-keep for subsequent non-zero immX. 379 380 uint64_t ignored_halfword = 0; 381 bool invert_move = false; 382 // If the number of 0xffff halfwords is greater than the number of 0x0000 383 // halfwords, it's more efficient to use move-inverted. 384 if (CountClearHalfWords(~imm, reg_size) > 385 CountClearHalfWords(imm, reg_size)) { 386 ignored_halfword = 0xffff; 387 invert_move = true; 388 } 389 390 // Mov instructions can't move values into the stack pointer, so set up a 391 // temporary register, if needed. 392 UseScratchRegisterScope temps; 393 Register temp; 394 if (emit_code) { 395 temps.Open(masm); 396 temp = rd.IsSP() ? temps.AcquireSameSizeAs(rd) : rd; 397 } 398 399 // Iterate through the halfwords. Use movn/movz for the first non-ignored 400 // halfword, and movk for subsequent halfwords. 401 VIXL_ASSERT((reg_size % 16) == 0); 402 bool first_mov_done = false; 403 for (unsigned i = 0; i < (temp.size() / 16); i++) { 404 uint64_t imm16 = (imm >> (16 * i)) & 0xffff; 405 if (imm16 != ignored_halfword) { 406 if (!first_mov_done) { 407 if (invert_move) { 408 if (emit_code) masm->movn(temp, ~imm16 & 0xffff, 16 * i); 409 instruction_count++; 410 } else { 411 if (emit_code) masm->movz(temp, imm16, 16 * i); 412 instruction_count++; 413 } 414 first_mov_done = true; 415 } else { 416 // Construct a wider constant. 417 if (emit_code) masm->movk(temp, imm16, 16 * i); 418 instruction_count++; 419 } 420 } 421 } 422 423 VIXL_ASSERT(first_mov_done); 424 425 // Move the temporary if the original destination register was the stack 426 // pointer. 427 if (rd.IsSP()) { 428 if (emit_code) masm->mov(rd, temp); 429 instruction_count++; 430 } 431 return instruction_count; 432 } 433} 434 435 436bool MacroAssembler::OneInstrMoveImmediateHelper(MacroAssembler* masm, 437 const Register& dst, 438 int64_t imm) { 439 bool emit_code = masm != NULL; 440 unsigned n, imm_s, imm_r; 441 int reg_size = dst.size(); 442 443 if (IsImmMovz(imm, reg_size) && !dst.IsSP()) { 444 // Immediate can be represented in a move zero instruction. Movz can't write 445 // to the stack pointer. 446 if (emit_code) { 447 masm->movz(dst, imm); 448 } 449 return true; 450 } else if (IsImmMovn(imm, reg_size) && !dst.IsSP()) { 451 // Immediate can be represented in a move negative instruction. Movn can't 452 // write to the stack pointer. 453 if (emit_code) { 454 masm->movn(dst, dst.Is64Bits() ? ~imm : (~imm & kWRegMask)); 455 } 456 return true; 457 } else if (IsImmLogical(imm, reg_size, &n, &imm_s, &imm_r)) { 458 // Immediate can be represented in a logical orr instruction. 459 VIXL_ASSERT(!dst.IsZero()); 460 if (emit_code) { 461 masm->LogicalImmediate( 462 dst, AppropriateZeroRegFor(dst), n, imm_s, imm_r, ORR); 463 } 464 return true; 465 } 466 return false; 467} 468 469 470void MacroAssembler::B(Label* label, BranchType type, Register reg, int bit) { 471 VIXL_ASSERT((reg.Is(NoReg) || (type >= kBranchTypeFirstUsingReg)) && 472 ((bit == -1) || (type >= kBranchTypeFirstUsingBit))); 473 if (kBranchTypeFirstCondition <= type && type <= kBranchTypeLastCondition) { 474 B(static_cast<Condition>(type), label); 475 } else { 476 switch (type) { 477 case always: B(label); break; 478 case never: break; 479 case reg_zero: Cbz(reg, label); break; 480 case reg_not_zero: Cbnz(reg, label); break; 481 case reg_bit_clear: Tbz(reg, bit, label); break; 482 case reg_bit_set: Tbnz(reg, bit, label); break; 483 default: 484 VIXL_UNREACHABLE(); 485 } 486 } 487} 488 489 490void MacroAssembler::B(Label* label) { 491 SingleEmissionCheckScope guard(this); 492 b(label); 493} 494 495 496void MacroAssembler::B(Label* label, Condition cond) { 497 VIXL_ASSERT(allow_macro_instructions_); 498 VIXL_ASSERT((cond != al) && (cond != nv)); 499 EmissionCheckScope guard(this, 2 * kInstructionSize); 500 501 if (label->IsBound() && LabelIsOutOfRange(label, CondBranchType)) { 502 Label done; 503 b(&done, InvertCondition(cond)); 504 b(label); 505 bind(&done); 506 } else { 507 if (!label->IsBound()) { 508 veneer_pool_.RegisterUnresolvedBranch(CursorOffset(), 509 label, 510 CondBranchType); 511 } 512 b(label, cond); 513 } 514} 515 516 517void MacroAssembler::Cbnz(const Register& rt, Label* label) { 518 VIXL_ASSERT(allow_macro_instructions_); 519 VIXL_ASSERT(!rt.IsZero()); 520 EmissionCheckScope guard(this, 2 * kInstructionSize); 521 522 if (label->IsBound() && LabelIsOutOfRange(label, CondBranchType)) { 523 Label done; 524 cbz(rt, &done); 525 b(label); 526 bind(&done); 527 } else { 528 if (!label->IsBound()) { 529 veneer_pool_.RegisterUnresolvedBranch(CursorOffset(), 530 label, 531 CompareBranchType); 532 } 533 cbnz(rt, label); 534 } 535} 536 537 538void MacroAssembler::Cbz(const Register& rt, Label* label) { 539 VIXL_ASSERT(allow_macro_instructions_); 540 VIXL_ASSERT(!rt.IsZero()); 541 EmissionCheckScope guard(this, 2 * kInstructionSize); 542 543 if (label->IsBound() && LabelIsOutOfRange(label, CondBranchType)) { 544 Label done; 545 cbnz(rt, &done); 546 b(label); 547 bind(&done); 548 } else { 549 if (!label->IsBound()) { 550 veneer_pool_.RegisterUnresolvedBranch(CursorOffset(), 551 label, 552 CompareBranchType); 553 } 554 cbz(rt, label); 555 } 556} 557 558 559void MacroAssembler::Tbnz(const Register& rt, unsigned bit_pos, Label* label) { 560 VIXL_ASSERT(allow_macro_instructions_); 561 VIXL_ASSERT(!rt.IsZero()); 562 EmissionCheckScope guard(this, 2 * kInstructionSize); 563 564 if (label->IsBound() && LabelIsOutOfRange(label, TestBranchType)) { 565 Label done; 566 tbz(rt, bit_pos, &done); 567 b(label); 568 bind(&done); 569 } else { 570 if (!label->IsBound()) { 571 veneer_pool_.RegisterUnresolvedBranch(CursorOffset(), 572 label, 573 TestBranchType); 574 } 575 tbnz(rt, bit_pos, label); 576 } 577} 578 579 580void MacroAssembler::Tbz(const Register& rt, unsigned bit_pos, Label* label) { 581 VIXL_ASSERT(allow_macro_instructions_); 582 VIXL_ASSERT(!rt.IsZero()); 583 EmissionCheckScope guard(this, 2 * kInstructionSize); 584 585 if (label->IsBound() && LabelIsOutOfRange(label, TestBranchType)) { 586 Label done; 587 tbnz(rt, bit_pos, &done); 588 b(label); 589 bind(&done); 590 } else { 591 if (!label->IsBound()) { 592 veneer_pool_.RegisterUnresolvedBranch(CursorOffset(), 593 label, 594 TestBranchType); 595 } 596 tbz(rt, bit_pos, label); 597 } 598} 599 600 601void MacroAssembler::Bind(Label* label) { 602 VIXL_ASSERT(allow_macro_instructions_); 603 veneer_pool_.DeleteUnresolvedBranchInfoForLabel(label); 604 bind(label); 605} 606 607 608// Bind a label to a specified offset from the start of the buffer. 609void MacroAssembler::BindToOffset(Label* label, ptrdiff_t offset) { 610 VIXL_ASSERT(allow_macro_instructions_); 611 veneer_pool_.DeleteUnresolvedBranchInfoForLabel(label); 612 Assembler::BindToOffset(label, offset); 613} 614 615 616void MacroAssembler::And(const Register& rd, 617 const Register& rn, 618 const Operand& operand) { 619 VIXL_ASSERT(allow_macro_instructions_); 620 LogicalMacro(rd, rn, operand, AND); 621} 622 623 624void MacroAssembler::Ands(const Register& rd, 625 const Register& rn, 626 const Operand& operand) { 627 VIXL_ASSERT(allow_macro_instructions_); 628 LogicalMacro(rd, rn, operand, ANDS); 629} 630 631 632void MacroAssembler::Tst(const Register& rn, 633 const Operand& operand) { 634 VIXL_ASSERT(allow_macro_instructions_); 635 Ands(AppropriateZeroRegFor(rn), rn, operand); 636} 637 638 639void MacroAssembler::Bic(const Register& rd, 640 const Register& rn, 641 const Operand& operand) { 642 VIXL_ASSERT(allow_macro_instructions_); 643 LogicalMacro(rd, rn, operand, BIC); 644} 645 646 647void MacroAssembler::Bics(const Register& rd, 648 const Register& rn, 649 const Operand& operand) { 650 VIXL_ASSERT(allow_macro_instructions_); 651 LogicalMacro(rd, rn, operand, BICS); 652} 653 654 655void MacroAssembler::Orr(const Register& rd, 656 const Register& rn, 657 const Operand& operand) { 658 VIXL_ASSERT(allow_macro_instructions_); 659 LogicalMacro(rd, rn, operand, ORR); 660} 661 662 663void MacroAssembler::Orn(const Register& rd, 664 const Register& rn, 665 const Operand& operand) { 666 VIXL_ASSERT(allow_macro_instructions_); 667 LogicalMacro(rd, rn, operand, ORN); 668} 669 670 671void MacroAssembler::Eor(const Register& rd, 672 const Register& rn, 673 const Operand& operand) { 674 VIXL_ASSERT(allow_macro_instructions_); 675 LogicalMacro(rd, rn, operand, EOR); 676} 677 678 679void MacroAssembler::Eon(const Register& rd, 680 const Register& rn, 681 const Operand& operand) { 682 VIXL_ASSERT(allow_macro_instructions_); 683 LogicalMacro(rd, rn, operand, EON); 684} 685 686 687void MacroAssembler::LogicalMacro(const Register& rd, 688 const Register& rn, 689 const Operand& operand, 690 LogicalOp op) { 691 // The worst case for size is logical immediate to sp: 692 // * up to 4 instructions to materialise the constant 693 // * 1 instruction to do the operation 694 // * 1 instruction to move to sp 695 MacroEmissionCheckScope guard(this); 696 UseScratchRegisterScope temps(this); 697 698 if (operand.IsImmediate()) { 699 int64_t immediate = operand.immediate(); 700 unsigned reg_size = rd.size(); 701 702 // If the operation is NOT, invert the operation and immediate. 703 if ((op & NOT) == NOT) { 704 op = static_cast<LogicalOp>(op & ~NOT); 705 immediate = ~immediate; 706 } 707 708 // Ignore the top 32 bits of an immediate if we're moving to a W register. 709 if (rd.Is32Bits()) { 710 // Check that the top 32 bits are consistent. 711 VIXL_ASSERT(((immediate >> kWRegSize) == 0) || 712 ((immediate >> kWRegSize) == -1)); 713 immediate &= kWRegMask; 714 } 715 716 VIXL_ASSERT(rd.Is64Bits() || is_uint32(immediate)); 717 718 // Special cases for all set or all clear immediates. 719 if (immediate == 0) { 720 switch (op) { 721 case AND: 722 Mov(rd, 0); 723 return; 724 case ORR: 725 VIXL_FALLTHROUGH(); 726 case EOR: 727 Mov(rd, rn); 728 return; 729 case ANDS: 730 VIXL_FALLTHROUGH(); 731 case BICS: 732 break; 733 default: 734 VIXL_UNREACHABLE(); 735 } 736 } else if ((rd.Is64Bits() && (immediate == -1)) || 737 (rd.Is32Bits() && (immediate == 0xffffffff))) { 738 switch (op) { 739 case AND: 740 Mov(rd, rn); 741 return; 742 case ORR: 743 Mov(rd, immediate); 744 return; 745 case EOR: 746 Mvn(rd, rn); 747 return; 748 case ANDS: 749 VIXL_FALLTHROUGH(); 750 case BICS: 751 break; 752 default: 753 VIXL_UNREACHABLE(); 754 } 755 } 756 757 unsigned n, imm_s, imm_r; 758 if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) { 759 // Immediate can be encoded in the instruction. 760 LogicalImmediate(rd, rn, n, imm_s, imm_r, op); 761 } else { 762 // Immediate can't be encoded: synthesize using move immediate. 763 Register temp = temps.AcquireSameSizeAs(rn); 764 Operand imm_operand = MoveImmediateForShiftedOp(temp, immediate); 765 766 if (rd.Is(sp)) { 767 // If rd is the stack pointer we cannot use it as the destination 768 // register so we use the temp register as an intermediate again. 769 Logical(temp, rn, imm_operand, op); 770 Mov(sp, temp); 771 } else { 772 Logical(rd, rn, imm_operand, op); 773 } 774 } 775 } else if (operand.IsExtendedRegister()) { 776 VIXL_ASSERT(operand.reg().size() <= rd.size()); 777 // Add/sub extended supports shift <= 4. We want to support exactly the 778 // same modes here. 779 VIXL_ASSERT(operand.shift_amount() <= 4); 780 VIXL_ASSERT(operand.reg().Is64Bits() || 781 ((operand.extend() != UXTX) && (operand.extend() != SXTX))); 782 783 temps.Exclude(operand.reg()); 784 Register temp = temps.AcquireSameSizeAs(rn); 785 EmitExtendShift(temp, operand.reg(), operand.extend(), 786 operand.shift_amount()); 787 Logical(rd, rn, Operand(temp), op); 788 } else { 789 // The operand can be encoded in the instruction. 790 VIXL_ASSERT(operand.IsShiftedRegister()); 791 Logical(rd, rn, operand, op); 792 } 793} 794 795 796void MacroAssembler::Mov(const Register& rd, 797 const Operand& operand, 798 DiscardMoveMode discard_mode) { 799 VIXL_ASSERT(allow_macro_instructions_); 800 // The worst case for size is mov immediate with up to 4 instructions. 801 MacroEmissionCheckScope guard(this); 802 803 if (operand.IsImmediate()) { 804 // Call the macro assembler for generic immediates. 805 Mov(rd, operand.immediate()); 806 } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) { 807 // Emit a shift instruction if moving a shifted register. This operation 808 // could also be achieved using an orr instruction (like orn used by Mvn), 809 // but using a shift instruction makes the disassembly clearer. 810 EmitShift(rd, operand.reg(), operand.shift(), operand.shift_amount()); 811 } else if (operand.IsExtendedRegister()) { 812 // Emit an extend instruction if moving an extended register. This handles 813 // extend with post-shift operations, too. 814 EmitExtendShift(rd, operand.reg(), operand.extend(), 815 operand.shift_amount()); 816 } else { 817 // Otherwise, emit a register move only if the registers are distinct, or 818 // if they are not X registers. 819 // 820 // Note that mov(w0, w0) is not a no-op because it clears the top word of 821 // x0. A flag is provided (kDiscardForSameWReg) if a move between the same W 822 // registers is not required to clear the top word of the X register. In 823 // this case, the instruction is discarded. 824 // 825 // If the sp is an operand, add #0 is emitted, otherwise, orr #0. 826 if (!rd.Is(operand.reg()) || (rd.Is32Bits() && 827 (discard_mode == kDontDiscardForSameWReg))) { 828 mov(rd, operand.reg()); 829 } 830 } 831} 832 833 834void MacroAssembler::Movi16bitHelper(const VRegister& vd, uint64_t imm) { 835 VIXL_ASSERT(is_uint16(imm)); 836 int byte1 = (imm & 0xff); 837 int byte2 = ((imm >> 8) & 0xff); 838 if (byte1 == byte2) { 839 movi(vd.Is64Bits() ? vd.V8B() : vd.V16B(), byte1); 840 } else if (byte1 == 0) { 841 movi(vd, byte2, LSL, 8); 842 } else if (byte2 == 0) { 843 movi(vd, byte1); 844 } else if (byte1 == 0xff) { 845 mvni(vd, ~byte2 & 0xff, LSL, 8); 846 } else if (byte2 == 0xff) { 847 mvni(vd, ~byte1 & 0xff); 848 } else { 849 UseScratchRegisterScope temps(this); 850 Register temp = temps.AcquireW(); 851 movz(temp, imm); 852 dup(vd, temp); 853 } 854} 855 856 857void MacroAssembler::Movi32bitHelper(const VRegister& vd, uint64_t imm) { 858 VIXL_ASSERT(is_uint32(imm)); 859 860 uint8_t bytes[sizeof(imm)]; 861 memcpy(bytes, &imm, sizeof(imm)); 862 863 // All bytes are either 0x00 or 0xff. 864 { 865 bool all0orff = true; 866 for (int i = 0; i < 4; ++i) { 867 if ((bytes[i] != 0) && (bytes[i] != 0xff)) { 868 all0orff = false; 869 break; 870 } 871 } 872 873 if (all0orff == true) { 874 movi(vd.Is64Bits() ? vd.V1D() : vd.V2D(), ((imm << 32) | imm)); 875 return; 876 } 877 } 878 879 // Of the 4 bytes, only one byte is non-zero. 880 for (int i = 0; i < 4; i++) { 881 if ((imm & (0xff << (i * 8))) == imm) { 882 movi(vd, bytes[i], LSL, i * 8); 883 return; 884 } 885 } 886 887 // Of the 4 bytes, only one byte is not 0xff. 888 for (int i = 0; i < 4; i++) { 889 uint32_t mask = ~(0xff << (i * 8)); 890 if ((imm & mask) == mask) { 891 mvni(vd, ~bytes[i] & 0xff, LSL, i * 8); 892 return; 893 } 894 } 895 896 // Immediate is of the form 0x00MMFFFF. 897 if ((imm & 0xff00ffff) == 0x0000ffff) { 898 movi(vd, bytes[2], MSL, 16); 899 return; 900 } 901 902 // Immediate is of the form 0x0000MMFF. 903 if ((imm & 0xffff00ff) == 0x000000ff) { 904 movi(vd, bytes[1], MSL, 8); 905 return; 906 } 907 908 // Immediate is of the form 0xFFMM0000. 909 if ((imm & 0xff00ffff) == 0xff000000) { 910 mvni(vd, ~bytes[2] & 0xff, MSL, 16); 911 return; 912 } 913 // Immediate is of the form 0xFFFFMM00. 914 if ((imm & 0xffff00ff) == 0xffff0000) { 915 mvni(vd, ~bytes[1] & 0xff, MSL, 8); 916 return; 917 } 918 919 // Top and bottom 16-bits are equal. 920 if (((imm >> 16) & 0xffff) == (imm & 0xffff)) { 921 Movi16bitHelper(vd.Is64Bits() ? vd.V4H() : vd.V8H(), imm & 0xffff); 922 return; 923 } 924 925 // Default case. 926 { 927 UseScratchRegisterScope temps(this); 928 Register temp = temps.AcquireW(); 929 Mov(temp, imm); 930 dup(vd, temp); 931 } 932} 933 934 935void MacroAssembler::Movi64bitHelper(const VRegister& vd, uint64_t imm) { 936 // All bytes are either 0x00 or 0xff. 937 { 938 bool all0orff = true; 939 for (int i = 0; i < 8; ++i) { 940 int byteval = (imm >> (i * 8)) & 0xff; 941 if (byteval != 0 && byteval != 0xff) { 942 all0orff = false; 943 break; 944 } 945 } 946 if (all0orff == true) { 947 movi(vd, imm); 948 return; 949 } 950 } 951 952 // Top and bottom 32-bits are equal. 953 if (((imm >> 32) & 0xffffffff) == (imm & 0xffffffff)) { 954 Movi32bitHelper(vd.Is64Bits() ? vd.V2S() : vd.V4S(), imm & 0xffffffff); 955 return; 956 } 957 958 // Default case. 959 { 960 UseScratchRegisterScope temps(this); 961 Register temp = temps.AcquireX(); 962 Mov(temp, imm); 963 if (vd.Is1D()) { 964 mov(vd.D(), 0, temp); 965 } else { 966 dup(vd.V2D(), temp); 967 } 968 } 969} 970 971 972void MacroAssembler::Movi(const VRegister& vd, 973 uint64_t imm, 974 Shift shift, 975 int shift_amount) { 976 VIXL_ASSERT(allow_macro_instructions_); 977 MacroEmissionCheckScope guard(this); 978 if (shift_amount != 0 || shift != LSL) { 979 movi(vd, imm, shift, shift_amount); 980 } else if (vd.Is8B() || vd.Is16B()) { 981 // 8-bit immediate. 982 VIXL_ASSERT(is_uint8(imm)); 983 movi(vd, imm); 984 } else if (vd.Is4H() || vd.Is8H()) { 985 // 16-bit immediate. 986 Movi16bitHelper(vd, imm); 987 } else if (vd.Is2S() || vd.Is4S()) { 988 // 32-bit immediate. 989 Movi32bitHelper(vd, imm); 990 } else { 991 // 64-bit immediate. 992 Movi64bitHelper(vd, imm); 993 } 994} 995 996 997void MacroAssembler::Movi(const VRegister& vd, 998 uint64_t hi, 999 uint64_t lo) { 1000 // TODO: Move 128-bit values in a more efficient way. 1001 VIXL_ASSERT(vd.Is128Bits()); 1002 UseScratchRegisterScope temps(this); 1003 Movi(vd.V2D(), lo); 1004 Register temp = temps.AcquireX(); 1005 Mov(temp, hi); 1006 Ins(vd.V2D(), 1, temp); 1007} 1008 1009 1010void MacroAssembler::Mvn(const Register& rd, const Operand& operand) { 1011 VIXL_ASSERT(allow_macro_instructions_); 1012 // The worst case for size is mvn immediate with up to 4 instructions. 1013 MacroEmissionCheckScope guard(this); 1014 1015 if (operand.IsImmediate()) { 1016 // Call the macro assembler for generic immediates. 1017 Mvn(rd, operand.immediate()); 1018 } else if (operand.IsExtendedRegister()) { 1019 UseScratchRegisterScope temps(this); 1020 temps.Exclude(operand.reg()); 1021 1022 // Emit two instructions for the extend case. This differs from Mov, as 1023 // the extend and invert can't be achieved in one instruction. 1024 Register temp = temps.AcquireSameSizeAs(rd); 1025 EmitExtendShift(temp, operand.reg(), operand.extend(), 1026 operand.shift_amount()); 1027 mvn(rd, Operand(temp)); 1028 } else { 1029 // Otherwise, register and shifted register cases can be handled by the 1030 // assembler directly, using orn. 1031 mvn(rd, operand); 1032 } 1033} 1034 1035 1036void MacroAssembler::Mov(const Register& rd, uint64_t imm) { 1037 VIXL_ASSERT(allow_macro_instructions_); 1038 MoveImmediateHelper(this, rd, imm); 1039} 1040 1041 1042void MacroAssembler::Ccmp(const Register& rn, 1043 const Operand& operand, 1044 StatusFlags nzcv, 1045 Condition cond) { 1046 VIXL_ASSERT(allow_macro_instructions_); 1047 if (operand.IsImmediate() && (operand.immediate() < 0)) { 1048 ConditionalCompareMacro(rn, -operand.immediate(), nzcv, cond, CCMN); 1049 } else { 1050 ConditionalCompareMacro(rn, operand, nzcv, cond, CCMP); 1051 } 1052} 1053 1054 1055void MacroAssembler::Ccmn(const Register& rn, 1056 const Operand& operand, 1057 StatusFlags nzcv, 1058 Condition cond) { 1059 VIXL_ASSERT(allow_macro_instructions_); 1060 if (operand.IsImmediate() && (operand.immediate() < 0)) { 1061 ConditionalCompareMacro(rn, -operand.immediate(), nzcv, cond, CCMP); 1062 } else { 1063 ConditionalCompareMacro(rn, operand, nzcv, cond, CCMN); 1064 } 1065} 1066 1067 1068void MacroAssembler::ConditionalCompareMacro(const Register& rn, 1069 const Operand& operand, 1070 StatusFlags nzcv, 1071 Condition cond, 1072 ConditionalCompareOp op) { 1073 VIXL_ASSERT((cond != al) && (cond != nv)); 1074 // The worst case for size is ccmp immediate: 1075 // * up to 4 instructions to materialise the constant 1076 // * 1 instruction for ccmp 1077 MacroEmissionCheckScope guard(this); 1078 1079 if ((operand.IsShiftedRegister() && (operand.shift_amount() == 0)) || 1080 (operand.IsImmediate() && IsImmConditionalCompare(operand.immediate()))) { 1081 // The immediate can be encoded in the instruction, or the operand is an 1082 // unshifted register: call the assembler. 1083 ConditionalCompare(rn, operand, nzcv, cond, op); 1084 } else { 1085 UseScratchRegisterScope temps(this); 1086 // The operand isn't directly supported by the instruction: perform the 1087 // operation on a temporary register. 1088 Register temp = temps.AcquireSameSizeAs(rn); 1089 Mov(temp, operand); 1090 ConditionalCompare(rn, temp, nzcv, cond, op); 1091 } 1092} 1093 1094 1095void MacroAssembler::Csel(const Register& rd, 1096 const Register& rn, 1097 const Operand& operand, 1098 Condition cond) { 1099 VIXL_ASSERT(allow_macro_instructions_); 1100 VIXL_ASSERT(!rd.IsZero()); 1101 VIXL_ASSERT(!rn.IsZero()); 1102 VIXL_ASSERT((cond != al) && (cond != nv)); 1103 // The worst case for size is csel immediate: 1104 // * up to 4 instructions to materialise the constant 1105 // * 1 instruction for csel 1106 MacroEmissionCheckScope guard(this); 1107 1108 if (operand.IsImmediate()) { 1109 // Immediate argument. Handle special cases of 0, 1 and -1 using zero 1110 // register. 1111 int64_t imm = operand.immediate(); 1112 Register zr = AppropriateZeroRegFor(rn); 1113 if (imm == 0) { 1114 csel(rd, rn, zr, cond); 1115 } else if (imm == 1) { 1116 csinc(rd, rn, zr, cond); 1117 } else if (imm == -1) { 1118 csinv(rd, rn, zr, cond); 1119 } else { 1120 UseScratchRegisterScope temps(this); 1121 Register temp = temps.AcquireSameSizeAs(rn); 1122 Mov(temp, operand.immediate()); 1123 csel(rd, rn, temp, cond); 1124 } 1125 } else if (operand.IsShiftedRegister() && (operand.shift_amount() == 0)) { 1126 // Unshifted register argument. 1127 csel(rd, rn, operand.reg(), cond); 1128 } else { 1129 // All other arguments. 1130 UseScratchRegisterScope temps(this); 1131 Register temp = temps.AcquireSameSizeAs(rn); 1132 Mov(temp, operand); 1133 csel(rd, rn, temp, cond); 1134 } 1135} 1136 1137 1138void MacroAssembler::Add(const Register& rd, 1139 const Register& rn, 1140 const Operand& operand, 1141 FlagsUpdate S) { 1142 VIXL_ASSERT(allow_macro_instructions_); 1143 if (operand.IsImmediate() && (operand.immediate() < 0) && 1144 IsImmAddSub(-operand.immediate())) { 1145 AddSubMacro(rd, rn, -operand.immediate(), S, SUB); 1146 } else { 1147 AddSubMacro(rd, rn, operand, S, ADD); 1148 } 1149} 1150 1151 1152void MacroAssembler::Adds(const Register& rd, 1153 const Register& rn, 1154 const Operand& operand) { 1155 Add(rd, rn, operand, SetFlags); 1156} 1157 1158 1159void MacroAssembler::Sub(const Register& rd, 1160 const Register& rn, 1161 const Operand& operand, 1162 FlagsUpdate S) { 1163 VIXL_ASSERT(allow_macro_instructions_); 1164 if (operand.IsImmediate() && (operand.immediate() < 0) && 1165 IsImmAddSub(-operand.immediate())) { 1166 AddSubMacro(rd, rn, -operand.immediate(), S, ADD); 1167 } else { 1168 AddSubMacro(rd, rn, operand, S, SUB); 1169 } 1170} 1171 1172 1173void MacroAssembler::Subs(const Register& rd, 1174 const Register& rn, 1175 const Operand& operand) { 1176 Sub(rd, rn, operand, SetFlags); 1177} 1178 1179 1180void MacroAssembler::Cmn(const Register& rn, const Operand& operand) { 1181 VIXL_ASSERT(allow_macro_instructions_); 1182 Adds(AppropriateZeroRegFor(rn), rn, operand); 1183} 1184 1185 1186void MacroAssembler::Cmp(const Register& rn, const Operand& operand) { 1187 VIXL_ASSERT(allow_macro_instructions_); 1188 Subs(AppropriateZeroRegFor(rn), rn, operand); 1189} 1190 1191 1192void MacroAssembler::Fcmp(const FPRegister& fn, double value, 1193 FPTrapFlags trap) { 1194 VIXL_ASSERT(allow_macro_instructions_); 1195 // The worst case for size is: 1196 // * 1 to materialise the constant, using literal pool if necessary 1197 // * 1 instruction for fcmp{e} 1198 MacroEmissionCheckScope guard(this); 1199 if (value != 0.0) { 1200 UseScratchRegisterScope temps(this); 1201 FPRegister tmp = temps.AcquireSameSizeAs(fn); 1202 Fmov(tmp, value); 1203 FPCompareMacro(fn, tmp, trap); 1204 } else { 1205 FPCompareMacro(fn, value, trap); 1206 } 1207} 1208 1209 1210void MacroAssembler::Fcmpe(const FPRegister& fn, double value) { 1211 Fcmp(fn, value, EnableTrap); 1212} 1213 1214 1215void MacroAssembler::Fmov(VRegister vd, double imm) { 1216 VIXL_ASSERT(allow_macro_instructions_); 1217 // Floating point immediates are loaded through the literal pool. 1218 MacroEmissionCheckScope guard(this); 1219 1220 if (vd.Is1S() || vd.Is2S() || vd.Is4S()) { 1221 Fmov(vd, static_cast<float>(imm)); 1222 return; 1223 } 1224 1225 VIXL_ASSERT(vd.Is1D() || vd.Is2D()); 1226 if (IsImmFP64(imm)) { 1227 fmov(vd, imm); 1228 } else { 1229 uint64_t rawbits = double_to_rawbits(imm); 1230 if (vd.IsScalar()) { 1231 if (rawbits == 0) { 1232 fmov(vd, xzr); 1233 } else { 1234 RawLiteral* literal = literal_pool_.Add(imm); 1235 ldr(vd, literal); 1236 } 1237 } else { 1238 // TODO: consider NEON support for load literal. 1239 Movi(vd, rawbits); 1240 } 1241 } 1242} 1243 1244 1245void MacroAssembler::Fmov(VRegister vd, float imm) { 1246 VIXL_ASSERT(allow_macro_instructions_); 1247 // Floating point immediates are loaded through the literal pool. 1248 MacroEmissionCheckScope guard(this); 1249 1250 if (vd.Is1D() || vd.Is2D()) { 1251 Fmov(vd, static_cast<double>(imm)); 1252 return; 1253 } 1254 1255 VIXL_ASSERT(vd.Is1S() || vd.Is2S() || vd.Is4S()); 1256 if (IsImmFP32(imm)) { 1257 fmov(vd, imm); 1258 } else { 1259 uint32_t rawbits = float_to_rawbits(imm); 1260 if (vd.IsScalar()) { 1261 if (rawbits == 0) { 1262 fmov(vd, wzr); 1263 } else { 1264 RawLiteral* literal = literal_pool_.Add(imm); 1265 ldr(vd, literal); 1266 } 1267 } else { 1268 // TODO: consider NEON support for load literal. 1269 Movi(vd, rawbits); 1270 } 1271 } 1272} 1273 1274 1275 1276void MacroAssembler::Neg(const Register& rd, 1277 const Operand& operand) { 1278 VIXL_ASSERT(allow_macro_instructions_); 1279 if (operand.IsImmediate()) { 1280 Mov(rd, -operand.immediate()); 1281 } else { 1282 Sub(rd, AppropriateZeroRegFor(rd), operand); 1283 } 1284} 1285 1286 1287void MacroAssembler::Negs(const Register& rd, 1288 const Operand& operand) { 1289 VIXL_ASSERT(allow_macro_instructions_); 1290 Subs(rd, AppropriateZeroRegFor(rd), operand); 1291} 1292 1293 1294bool MacroAssembler::TryOneInstrMoveImmediate(const Register& dst, 1295 int64_t imm) { 1296 return OneInstrMoveImmediateHelper(this, dst, imm); 1297} 1298 1299 1300Operand MacroAssembler::MoveImmediateForShiftedOp(const Register& dst, 1301 int64_t imm) { 1302 int reg_size = dst.size(); 1303 1304 // Encode the immediate in a single move instruction, if possible. 1305 if (TryOneInstrMoveImmediate(dst, imm)) { 1306 // The move was successful; nothing to do here. 1307 } else { 1308 // Pre-shift the immediate to the least-significant bits of the register. 1309 int shift_low = CountTrailingZeros(imm, reg_size); 1310 int64_t imm_low = imm >> shift_low; 1311 1312 // Pre-shift the immediate to the most-significant bits of the register, 1313 // inserting set bits in the least-significant bits. 1314 int shift_high = CountLeadingZeros(imm, reg_size); 1315 int64_t imm_high = (imm << shift_high) | ((INT64_C(1) << shift_high) - 1); 1316 1317 if (TryOneInstrMoveImmediate(dst, imm_low)) { 1318 // The new immediate has been moved into the destination's low bits: 1319 // return a new leftward-shifting operand. 1320 return Operand(dst, LSL, shift_low); 1321 } else if (TryOneInstrMoveImmediate(dst, imm_high)) { 1322 // The new immediate has been moved into the destination's high bits: 1323 // return a new rightward-shifting operand. 1324 return Operand(dst, LSR, shift_high); 1325 } else { 1326 Mov(dst, imm); 1327 } 1328 } 1329 return Operand(dst); 1330} 1331 1332 1333void MacroAssembler::ComputeAddress(const Register& dst, 1334 const MemOperand& mem_op) { 1335 // We cannot handle pre-indexing or post-indexing. 1336 VIXL_ASSERT(mem_op.addrmode() == Offset); 1337 Register base = mem_op.base(); 1338 if (mem_op.IsImmediateOffset()) { 1339 Add(dst, base, mem_op.offset()); 1340 } else { 1341 VIXL_ASSERT(mem_op.IsRegisterOffset()); 1342 Register reg_offset = mem_op.regoffset(); 1343 Shift shift = mem_op.shift(); 1344 Extend extend = mem_op.extend(); 1345 if (shift == NO_SHIFT) { 1346 VIXL_ASSERT(extend != NO_EXTEND); 1347 Add(dst, base, Operand(reg_offset, extend, mem_op.shift_amount())); 1348 } else { 1349 VIXL_ASSERT(extend == NO_EXTEND); 1350 Add(dst, base, Operand(reg_offset, shift, mem_op.shift_amount())); 1351 } 1352 } 1353} 1354 1355 1356void MacroAssembler::AddSubMacro(const Register& rd, 1357 const Register& rn, 1358 const Operand& operand, 1359 FlagsUpdate S, 1360 AddSubOp op) { 1361 // Worst case is add/sub immediate: 1362 // * up to 4 instructions to materialise the constant 1363 // * 1 instruction for add/sub 1364 MacroEmissionCheckScope guard(this); 1365 1366 if (operand.IsZero() && rd.Is(rn) && rd.Is64Bits() && rn.Is64Bits() && 1367 (S == LeaveFlags)) { 1368 // The instruction would be a nop. Avoid generating useless code. 1369 return; 1370 } 1371 1372 if ((operand.IsImmediate() && !IsImmAddSub(operand.immediate())) || 1373 (rn.IsZero() && !operand.IsShiftedRegister()) || 1374 (operand.IsShiftedRegister() && (operand.shift() == ROR))) { 1375 UseScratchRegisterScope temps(this); 1376 Register temp = temps.AcquireSameSizeAs(rn); 1377 if (operand.IsImmediate()) { 1378 Operand imm_operand = 1379 MoveImmediateForShiftedOp(temp, operand.immediate()); 1380 AddSub(rd, rn, imm_operand, S, op); 1381 } else { 1382 Mov(temp, operand); 1383 AddSub(rd, rn, temp, S, op); 1384 } 1385 } else { 1386 AddSub(rd, rn, operand, S, op); 1387 } 1388} 1389 1390 1391void MacroAssembler::Adc(const Register& rd, 1392 const Register& rn, 1393 const Operand& operand) { 1394 VIXL_ASSERT(allow_macro_instructions_); 1395 AddSubWithCarryMacro(rd, rn, operand, LeaveFlags, ADC); 1396} 1397 1398 1399void MacroAssembler::Adcs(const Register& rd, 1400 const Register& rn, 1401 const Operand& operand) { 1402 VIXL_ASSERT(allow_macro_instructions_); 1403 AddSubWithCarryMacro(rd, rn, operand, SetFlags, ADC); 1404} 1405 1406 1407void MacroAssembler::Sbc(const Register& rd, 1408 const Register& rn, 1409 const Operand& operand) { 1410 VIXL_ASSERT(allow_macro_instructions_); 1411 AddSubWithCarryMacro(rd, rn, operand, LeaveFlags, SBC); 1412} 1413 1414 1415void MacroAssembler::Sbcs(const Register& rd, 1416 const Register& rn, 1417 const Operand& operand) { 1418 VIXL_ASSERT(allow_macro_instructions_); 1419 AddSubWithCarryMacro(rd, rn, operand, SetFlags, SBC); 1420} 1421 1422 1423void MacroAssembler::Ngc(const Register& rd, 1424 const Operand& operand) { 1425 VIXL_ASSERT(allow_macro_instructions_); 1426 Register zr = AppropriateZeroRegFor(rd); 1427 Sbc(rd, zr, operand); 1428} 1429 1430 1431void MacroAssembler::Ngcs(const Register& rd, 1432 const Operand& operand) { 1433 VIXL_ASSERT(allow_macro_instructions_); 1434 Register zr = AppropriateZeroRegFor(rd); 1435 Sbcs(rd, zr, operand); 1436} 1437 1438 1439void MacroAssembler::AddSubWithCarryMacro(const Register& rd, 1440 const Register& rn, 1441 const Operand& operand, 1442 FlagsUpdate S, 1443 AddSubWithCarryOp op) { 1444 VIXL_ASSERT(rd.size() == rn.size()); 1445 // Worst case is addc/subc immediate: 1446 // * up to 4 instructions to materialise the constant 1447 // * 1 instruction for add/sub 1448 MacroEmissionCheckScope guard(this); 1449 UseScratchRegisterScope temps(this); 1450 1451 if (operand.IsImmediate() || 1452 (operand.IsShiftedRegister() && (operand.shift() == ROR))) { 1453 // Add/sub with carry (immediate or ROR shifted register.) 1454 Register temp = temps.AcquireSameSizeAs(rn); 1455 Mov(temp, operand); 1456 AddSubWithCarry(rd, rn, Operand(temp), S, op); 1457 } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) { 1458 // Add/sub with carry (shifted register). 1459 VIXL_ASSERT(operand.reg().size() == rd.size()); 1460 VIXL_ASSERT(operand.shift() != ROR); 1461 VIXL_ASSERT(is_uintn(rd.size() == kXRegSize ? kXRegSizeLog2 : kWRegSizeLog2, 1462 operand.shift_amount())); 1463 temps.Exclude(operand.reg()); 1464 Register temp = temps.AcquireSameSizeAs(rn); 1465 EmitShift(temp, operand.reg(), operand.shift(), operand.shift_amount()); 1466 AddSubWithCarry(rd, rn, Operand(temp), S, op); 1467 } else if (operand.IsExtendedRegister()) { 1468 // Add/sub with carry (extended register). 1469 VIXL_ASSERT(operand.reg().size() <= rd.size()); 1470 // Add/sub extended supports a shift <= 4. We want to support exactly the 1471 // same modes. 1472 VIXL_ASSERT(operand.shift_amount() <= 4); 1473 VIXL_ASSERT(operand.reg().Is64Bits() || 1474 ((operand.extend() != UXTX) && (operand.extend() != SXTX))); 1475 temps.Exclude(operand.reg()); 1476 Register temp = temps.AcquireSameSizeAs(rn); 1477 EmitExtendShift(temp, operand.reg(), operand.extend(), 1478 operand.shift_amount()); 1479 AddSubWithCarry(rd, rn, Operand(temp), S, op); 1480 } else { 1481 // The addressing mode is directly supported by the instruction. 1482 AddSubWithCarry(rd, rn, operand, S, op); 1483 } 1484} 1485 1486 1487#define DEFINE_FUNCTION(FN, REGTYPE, REG, OP) \ 1488void MacroAssembler::FN(const REGTYPE REG, const MemOperand& addr) { \ 1489 VIXL_ASSERT(allow_macro_instructions_); \ 1490 LoadStoreMacro(REG, addr, OP); \ 1491} 1492LS_MACRO_LIST(DEFINE_FUNCTION) 1493#undef DEFINE_FUNCTION 1494 1495 1496void MacroAssembler::LoadStoreMacro(const CPURegister& rt, 1497 const MemOperand& addr, 1498 LoadStoreOp op) { 1499 // Worst case is ldr/str pre/post index: 1500 // * 1 instruction for ldr/str 1501 // * up to 4 instructions to materialise the constant 1502 // * 1 instruction to update the base 1503 MacroEmissionCheckScope guard(this); 1504 1505 int64_t offset = addr.offset(); 1506 unsigned access_size = CalcLSDataSize(op); 1507 1508 // Check if an immediate offset fits in the immediate field of the 1509 // appropriate instruction. If not, emit two instructions to perform 1510 // the operation. 1511 if (addr.IsImmediateOffset() && !IsImmLSScaled(offset, access_size) && 1512 !IsImmLSUnscaled(offset)) { 1513 // Immediate offset that can't be encoded using unsigned or unscaled 1514 // addressing modes. 1515 UseScratchRegisterScope temps(this); 1516 Register temp = temps.AcquireSameSizeAs(addr.base()); 1517 Mov(temp, addr.offset()); 1518 LoadStore(rt, MemOperand(addr.base(), temp), op); 1519 } else if (addr.IsPostIndex() && !IsImmLSUnscaled(offset)) { 1520 // Post-index beyond unscaled addressing range. 1521 LoadStore(rt, MemOperand(addr.base()), op); 1522 Add(addr.base(), addr.base(), Operand(offset)); 1523 } else if (addr.IsPreIndex() && !IsImmLSUnscaled(offset)) { 1524 // Pre-index beyond unscaled addressing range. 1525 Add(addr.base(), addr.base(), Operand(offset)); 1526 LoadStore(rt, MemOperand(addr.base()), op); 1527 } else { 1528 // Encodable in one load/store instruction. 1529 LoadStore(rt, addr, op); 1530 } 1531} 1532 1533 1534#define DEFINE_FUNCTION(FN, REGTYPE, REG, REG2, OP) \ 1535void MacroAssembler::FN(const REGTYPE REG, \ 1536 const REGTYPE REG2, \ 1537 const MemOperand& addr) { \ 1538 VIXL_ASSERT(allow_macro_instructions_); \ 1539 LoadStorePairMacro(REG, REG2, addr, OP); \ 1540} 1541LSPAIR_MACRO_LIST(DEFINE_FUNCTION) 1542#undef DEFINE_FUNCTION 1543 1544void MacroAssembler::LoadStorePairMacro(const CPURegister& rt, 1545 const CPURegister& rt2, 1546 const MemOperand& addr, 1547 LoadStorePairOp op) { 1548 // TODO(all): Should we support register offset for load-store-pair? 1549 VIXL_ASSERT(!addr.IsRegisterOffset()); 1550 // Worst case is ldp/stp immediate: 1551 // * 1 instruction for ldp/stp 1552 // * up to 4 instructions to materialise the constant 1553 // * 1 instruction to update the base 1554 MacroEmissionCheckScope guard(this); 1555 1556 int64_t offset = addr.offset(); 1557 unsigned access_size = CalcLSPairDataSize(op); 1558 1559 // Check if the offset fits in the immediate field of the appropriate 1560 // instruction. If not, emit two instructions to perform the operation. 1561 if (IsImmLSPair(offset, access_size)) { 1562 // Encodable in one load/store pair instruction. 1563 LoadStorePair(rt, rt2, addr, op); 1564 } else { 1565 Register base = addr.base(); 1566 if (addr.IsImmediateOffset()) { 1567 UseScratchRegisterScope temps(this); 1568 Register temp = temps.AcquireSameSizeAs(base); 1569 Add(temp, base, offset); 1570 LoadStorePair(rt, rt2, MemOperand(temp), op); 1571 } else if (addr.IsPostIndex()) { 1572 LoadStorePair(rt, rt2, MemOperand(base), op); 1573 Add(base, base, offset); 1574 } else { 1575 VIXL_ASSERT(addr.IsPreIndex()); 1576 Add(base, base, offset); 1577 LoadStorePair(rt, rt2, MemOperand(base), op); 1578 } 1579 } 1580} 1581 1582 1583void MacroAssembler::Prfm(PrefetchOperation op, const MemOperand& addr) { 1584 MacroEmissionCheckScope guard(this); 1585 1586 // There are no pre- or post-index modes for prfm. 1587 VIXL_ASSERT(addr.IsImmediateOffset() || addr.IsRegisterOffset()); 1588 1589 // The access size is implicitly 8 bytes for all prefetch operations. 1590 unsigned size = kXRegSizeInBytesLog2; 1591 1592 // Check if an immediate offset fits in the immediate field of the 1593 // appropriate instruction. If not, emit two instructions to perform 1594 // the operation. 1595 if (addr.IsImmediateOffset() && !IsImmLSScaled(addr.offset(), size) && 1596 !IsImmLSUnscaled(addr.offset())) { 1597 // Immediate offset that can't be encoded using unsigned or unscaled 1598 // addressing modes. 1599 UseScratchRegisterScope temps(this); 1600 Register temp = temps.AcquireSameSizeAs(addr.base()); 1601 Mov(temp, addr.offset()); 1602 Prefetch(op, MemOperand(addr.base(), temp)); 1603 } else { 1604 // Simple register-offsets are encodable in one instruction. 1605 Prefetch(op, addr); 1606 } 1607} 1608 1609 1610void MacroAssembler::Push(const CPURegister& src0, const CPURegister& src1, 1611 const CPURegister& src2, const CPURegister& src3) { 1612 VIXL_ASSERT(allow_macro_instructions_); 1613 VIXL_ASSERT(AreSameSizeAndType(src0, src1, src2, src3)); 1614 VIXL_ASSERT(src0.IsValid()); 1615 1616 int count = 1 + src1.IsValid() + src2.IsValid() + src3.IsValid(); 1617 int size = src0.SizeInBytes(); 1618 1619 PrepareForPush(count, size); 1620 PushHelper(count, size, src0, src1, src2, src3); 1621} 1622 1623 1624void MacroAssembler::Pop(const CPURegister& dst0, const CPURegister& dst1, 1625 const CPURegister& dst2, const CPURegister& dst3) { 1626 // It is not valid to pop into the same register more than once in one 1627 // instruction, not even into the zero register. 1628 VIXL_ASSERT(allow_macro_instructions_); 1629 VIXL_ASSERT(!AreAliased(dst0, dst1, dst2, dst3)); 1630 VIXL_ASSERT(AreSameSizeAndType(dst0, dst1, dst2, dst3)); 1631 VIXL_ASSERT(dst0.IsValid()); 1632 1633 int count = 1 + dst1.IsValid() + dst2.IsValid() + dst3.IsValid(); 1634 int size = dst0.SizeInBytes(); 1635 1636 PrepareForPop(count, size); 1637 PopHelper(count, size, dst0, dst1, dst2, dst3); 1638} 1639 1640 1641void MacroAssembler::PushCPURegList(CPURegList registers) { 1642 VIXL_ASSERT(!registers.Overlaps(*TmpList())); 1643 VIXL_ASSERT(!registers.Overlaps(*FPTmpList())); 1644 VIXL_ASSERT(allow_macro_instructions_); 1645 1646 int reg_size = registers.RegisterSizeInBytes(); 1647 PrepareForPush(registers.Count(), reg_size); 1648 1649 // Bump the stack pointer and store two registers at the bottom. 1650 int size = registers.TotalSizeInBytes(); 1651 const CPURegister& bottom_0 = registers.PopLowestIndex(); 1652 const CPURegister& bottom_1 = registers.PopLowestIndex(); 1653 if (bottom_0.IsValid() && bottom_1.IsValid()) { 1654 Stp(bottom_0, bottom_1, MemOperand(StackPointer(), -size, PreIndex)); 1655 } else if (bottom_0.IsValid()) { 1656 Str(bottom_0, MemOperand(StackPointer(), -size, PreIndex)); 1657 } 1658 1659 int offset = 2 * reg_size; 1660 while (!registers.IsEmpty()) { 1661 const CPURegister& src0 = registers.PopLowestIndex(); 1662 const CPURegister& src1 = registers.PopLowestIndex(); 1663 if (src1.IsValid()) { 1664 Stp(src0, src1, MemOperand(StackPointer(), offset)); 1665 } else { 1666 Str(src0, MemOperand(StackPointer(), offset)); 1667 } 1668 offset += 2 * reg_size; 1669 } 1670} 1671 1672 1673void MacroAssembler::PopCPURegList(CPURegList registers) { 1674 VIXL_ASSERT(!registers.Overlaps(*TmpList())); 1675 VIXL_ASSERT(!registers.Overlaps(*FPTmpList())); 1676 VIXL_ASSERT(allow_macro_instructions_); 1677 1678 int reg_size = registers.RegisterSizeInBytes(); 1679 PrepareForPop(registers.Count(), reg_size); 1680 1681 1682 int size = registers.TotalSizeInBytes(); 1683 const CPURegister& bottom_0 = registers.PopLowestIndex(); 1684 const CPURegister& bottom_1 = registers.PopLowestIndex(); 1685 1686 int offset = 2 * reg_size; 1687 while (!registers.IsEmpty()) { 1688 const CPURegister& dst0 = registers.PopLowestIndex(); 1689 const CPURegister& dst1 = registers.PopLowestIndex(); 1690 if (dst1.IsValid()) { 1691 Ldp(dst0, dst1, MemOperand(StackPointer(), offset)); 1692 } else { 1693 Ldr(dst0, MemOperand(StackPointer(), offset)); 1694 } 1695 offset += 2 * reg_size; 1696 } 1697 1698 // Load the two registers at the bottom and drop the stack pointer. 1699 if (bottom_0.IsValid() && bottom_1.IsValid()) { 1700 Ldp(bottom_0, bottom_1, MemOperand(StackPointer(), size, PostIndex)); 1701 } else if (bottom_0.IsValid()) { 1702 Ldr(bottom_0, MemOperand(StackPointer(), size, PostIndex)); 1703 } 1704} 1705 1706 1707void MacroAssembler::PushMultipleTimes(int count, Register src) { 1708 VIXL_ASSERT(allow_macro_instructions_); 1709 int size = src.SizeInBytes(); 1710 1711 PrepareForPush(count, size); 1712 // Push up to four registers at a time if possible because if the current 1713 // stack pointer is sp and the register size is 32, registers must be pushed 1714 // in blocks of four in order to maintain the 16-byte alignment for sp. 1715 while (count >= 4) { 1716 PushHelper(4, size, src, src, src, src); 1717 count -= 4; 1718 } 1719 if (count >= 2) { 1720 PushHelper(2, size, src, src, NoReg, NoReg); 1721 count -= 2; 1722 } 1723 if (count == 1) { 1724 PushHelper(1, size, src, NoReg, NoReg, NoReg); 1725 count -= 1; 1726 } 1727 VIXL_ASSERT(count == 0); 1728} 1729 1730 1731void MacroAssembler::PushHelper(int count, int size, 1732 const CPURegister& src0, 1733 const CPURegister& src1, 1734 const CPURegister& src2, 1735 const CPURegister& src3) { 1736 // Ensure that we don't unintentionally modify scratch or debug registers. 1737 // Worst case for size is 2 stp. 1738 InstructionAccurateScope scope(this, 2, 1739 InstructionAccurateScope::kMaximumSize); 1740 1741 VIXL_ASSERT(AreSameSizeAndType(src0, src1, src2, src3)); 1742 VIXL_ASSERT(size == src0.SizeInBytes()); 1743 1744 // When pushing multiple registers, the store order is chosen such that 1745 // Push(a, b) is equivalent to Push(a) followed by Push(b). 1746 switch (count) { 1747 case 1: 1748 VIXL_ASSERT(src1.IsNone() && src2.IsNone() && src3.IsNone()); 1749 str(src0, MemOperand(StackPointer(), -1 * size, PreIndex)); 1750 break; 1751 case 2: 1752 VIXL_ASSERT(src2.IsNone() && src3.IsNone()); 1753 stp(src1, src0, MemOperand(StackPointer(), -2 * size, PreIndex)); 1754 break; 1755 case 3: 1756 VIXL_ASSERT(src3.IsNone()); 1757 stp(src2, src1, MemOperand(StackPointer(), -3 * size, PreIndex)); 1758 str(src0, MemOperand(StackPointer(), 2 * size)); 1759 break; 1760 case 4: 1761 // Skip over 4 * size, then fill in the gap. This allows four W registers 1762 // to be pushed using sp, whilst maintaining 16-byte alignment for sp at 1763 // all times. 1764 stp(src3, src2, MemOperand(StackPointer(), -4 * size, PreIndex)); 1765 stp(src1, src0, MemOperand(StackPointer(), 2 * size)); 1766 break; 1767 default: 1768 VIXL_UNREACHABLE(); 1769 } 1770} 1771 1772 1773void MacroAssembler::PopHelper(int count, int size, 1774 const CPURegister& dst0, 1775 const CPURegister& dst1, 1776 const CPURegister& dst2, 1777 const CPURegister& dst3) { 1778 // Ensure that we don't unintentionally modify scratch or debug registers. 1779 // Worst case for size is 2 ldp. 1780 InstructionAccurateScope scope(this, 2, 1781 InstructionAccurateScope::kMaximumSize); 1782 1783 VIXL_ASSERT(AreSameSizeAndType(dst0, dst1, dst2, dst3)); 1784 VIXL_ASSERT(size == dst0.SizeInBytes()); 1785 1786 // When popping multiple registers, the load order is chosen such that 1787 // Pop(a, b) is equivalent to Pop(a) followed by Pop(b). 1788 switch (count) { 1789 case 1: 1790 VIXL_ASSERT(dst1.IsNone() && dst2.IsNone() && dst3.IsNone()); 1791 ldr(dst0, MemOperand(StackPointer(), 1 * size, PostIndex)); 1792 break; 1793 case 2: 1794 VIXL_ASSERT(dst2.IsNone() && dst3.IsNone()); 1795 ldp(dst0, dst1, MemOperand(StackPointer(), 2 * size, PostIndex)); 1796 break; 1797 case 3: 1798 VIXL_ASSERT(dst3.IsNone()); 1799 ldr(dst2, MemOperand(StackPointer(), 2 * size)); 1800 ldp(dst0, dst1, MemOperand(StackPointer(), 3 * size, PostIndex)); 1801 break; 1802 case 4: 1803 // Load the higher addresses first, then load the lower addresses and skip 1804 // the whole block in the second instruction. This allows four W registers 1805 // to be popped using sp, whilst maintaining 16-byte alignment for sp at 1806 // all times. 1807 ldp(dst2, dst3, MemOperand(StackPointer(), 2 * size)); 1808 ldp(dst0, dst1, MemOperand(StackPointer(), 4 * size, PostIndex)); 1809 break; 1810 default: 1811 VIXL_UNREACHABLE(); 1812 } 1813} 1814 1815 1816void MacroAssembler::PrepareForPush(int count, int size) { 1817 if (sp.Is(StackPointer())) { 1818 // If the current stack pointer is sp, then it must be aligned to 16 bytes 1819 // on entry and the total size of the specified registers must also be a 1820 // multiple of 16 bytes. 1821 VIXL_ASSERT((count * size) % 16 == 0); 1822 } else { 1823 // Even if the current stack pointer is not the system stack pointer (sp), 1824 // the system stack pointer will still be modified in order to comply with 1825 // ABI rules about accessing memory below the system stack pointer. 1826 BumpSystemStackPointer(count * size); 1827 } 1828} 1829 1830 1831void MacroAssembler::PrepareForPop(int count, int size) { 1832 USE(count); 1833 USE(size); 1834 if (sp.Is(StackPointer())) { 1835 // If the current stack pointer is sp, then it must be aligned to 16 bytes 1836 // on entry and the total size of the specified registers must also be a 1837 // multiple of 16 bytes. 1838 VIXL_ASSERT((count * size) % 16 == 0); 1839 } 1840} 1841 1842void MacroAssembler::Poke(const Register& src, const Operand& offset) { 1843 VIXL_ASSERT(allow_macro_instructions_); 1844 if (offset.IsImmediate()) { 1845 VIXL_ASSERT(offset.immediate() >= 0); 1846 } 1847 1848 Str(src, MemOperand(StackPointer(), offset)); 1849} 1850 1851 1852void MacroAssembler::Peek(const Register& dst, const Operand& offset) { 1853 VIXL_ASSERT(allow_macro_instructions_); 1854 if (offset.IsImmediate()) { 1855 VIXL_ASSERT(offset.immediate() >= 0); 1856 } 1857 1858 Ldr(dst, MemOperand(StackPointer(), offset)); 1859} 1860 1861 1862void MacroAssembler::Claim(const Operand& size) { 1863 VIXL_ASSERT(allow_macro_instructions_); 1864 1865 if (size.IsZero()) { 1866 return; 1867 } 1868 1869 if (size.IsImmediate()) { 1870 VIXL_ASSERT(size.immediate() > 0); 1871 if (sp.Is(StackPointer())) { 1872 VIXL_ASSERT((size.immediate() % 16) == 0); 1873 } 1874 } 1875 1876 if (!sp.Is(StackPointer())) { 1877 BumpSystemStackPointer(size); 1878 } 1879 1880 Sub(StackPointer(), StackPointer(), size); 1881} 1882 1883 1884void MacroAssembler::Drop(const Operand& size) { 1885 VIXL_ASSERT(allow_macro_instructions_); 1886 1887 if (size.IsZero()) { 1888 return; 1889 } 1890 1891 if (size.IsImmediate()) { 1892 VIXL_ASSERT(size.immediate() > 0); 1893 if (sp.Is(StackPointer())) { 1894 VIXL_ASSERT((size.immediate() % 16) == 0); 1895 } 1896 } 1897 1898 Add(StackPointer(), StackPointer(), size); 1899} 1900 1901 1902void MacroAssembler::PushCalleeSavedRegisters() { 1903 // Ensure that the macro-assembler doesn't use any scratch registers. 1904 // 10 stp will be emitted. 1905 // TODO(all): Should we use GetCalleeSaved and SavedFP. 1906 InstructionAccurateScope scope(this, 10); 1907 1908 // This method must not be called unless the current stack pointer is sp. 1909 VIXL_ASSERT(sp.Is(StackPointer())); 1910 1911 MemOperand tos(sp, -2 * kXRegSizeInBytes, PreIndex); 1912 1913 stp(x29, x30, tos); 1914 stp(x27, x28, tos); 1915 stp(x25, x26, tos); 1916 stp(x23, x24, tos); 1917 stp(x21, x22, tos); 1918 stp(x19, x20, tos); 1919 1920 stp(d14, d15, tos); 1921 stp(d12, d13, tos); 1922 stp(d10, d11, tos); 1923 stp(d8, d9, tos); 1924} 1925 1926 1927void MacroAssembler::PopCalleeSavedRegisters() { 1928 // Ensure that the macro-assembler doesn't use any scratch registers. 1929 // 10 ldp will be emitted. 1930 // TODO(all): Should we use GetCalleeSaved and SavedFP. 1931 InstructionAccurateScope scope(this, 10); 1932 1933 // This method must not be called unless the current stack pointer is sp. 1934 VIXL_ASSERT(sp.Is(StackPointer())); 1935 1936 MemOperand tos(sp, 2 * kXRegSizeInBytes, PostIndex); 1937 1938 ldp(d8, d9, tos); 1939 ldp(d10, d11, tos); 1940 ldp(d12, d13, tos); 1941 ldp(d14, d15, tos); 1942 1943 ldp(x19, x20, tos); 1944 ldp(x21, x22, tos); 1945 ldp(x23, x24, tos); 1946 ldp(x25, x26, tos); 1947 ldp(x27, x28, tos); 1948 ldp(x29, x30, tos); 1949} 1950 1951void MacroAssembler::LoadCPURegList(CPURegList registers, 1952 const MemOperand& src) { 1953 LoadStoreCPURegListHelper(kLoad, registers, src); 1954} 1955 1956void MacroAssembler::StoreCPURegList(CPURegList registers, 1957 const MemOperand& dst) { 1958 LoadStoreCPURegListHelper(kStore, registers, dst); 1959} 1960 1961 1962void MacroAssembler::LoadStoreCPURegListHelper(LoadStoreCPURegListAction op, 1963 CPURegList registers, 1964 const MemOperand& mem) { 1965 // We do not handle pre-indexing or post-indexing. 1966 VIXL_ASSERT(!(mem.IsPreIndex() || mem.IsPostIndex())); 1967 VIXL_ASSERT(!registers.Overlaps(tmp_list_)); 1968 VIXL_ASSERT(!registers.Overlaps(fptmp_list_)); 1969 VIXL_ASSERT(!registers.IncludesAliasOf(sp)); 1970 1971 UseScratchRegisterScope temps(this); 1972 1973 MemOperand loc = BaseMemOperandForLoadStoreCPURegList(registers, 1974 mem, 1975 &temps); 1976 1977 while (registers.Count() >= 2) { 1978 const CPURegister& dst0 = registers.PopLowestIndex(); 1979 const CPURegister& dst1 = registers.PopLowestIndex(); 1980 if (op == kStore) { 1981 Stp(dst0, dst1, loc); 1982 } else { 1983 VIXL_ASSERT(op == kLoad); 1984 Ldp(dst0, dst1, loc); 1985 } 1986 loc.AddOffset(2 * registers.RegisterSizeInBytes()); 1987 } 1988 if (!registers.IsEmpty()) { 1989 if (op == kStore) { 1990 Str(registers.PopLowestIndex(), loc); 1991 } else { 1992 VIXL_ASSERT(op == kLoad); 1993 Ldr(registers.PopLowestIndex(), loc); 1994 } 1995 } 1996} 1997 1998MemOperand MacroAssembler::BaseMemOperandForLoadStoreCPURegList( 1999 const CPURegList& registers, 2000 const MemOperand& mem, 2001 UseScratchRegisterScope* scratch_scope) { 2002 // If necessary, pre-compute the base address for the accesses. 2003 if (mem.IsRegisterOffset()) { 2004 Register reg_base = scratch_scope->AcquireX(); 2005 ComputeAddress(reg_base, mem); 2006 return MemOperand(reg_base); 2007 2008 } else if (mem.IsImmediateOffset()) { 2009 int reg_size = registers.RegisterSizeInBytes(); 2010 int total_size = registers.TotalSizeInBytes(); 2011 int64_t min_offset = mem.offset(); 2012 int64_t max_offset = mem.offset() + std::max(0, total_size - 2 * reg_size); 2013 if ((registers.Count() >= 2) && 2014 (!Assembler::IsImmLSPair(min_offset, WhichPowerOf2(reg_size)) || 2015 !Assembler::IsImmLSPair(max_offset, WhichPowerOf2(reg_size)))) { 2016 Register reg_base = scratch_scope->AcquireX(); 2017 ComputeAddress(reg_base, mem); 2018 return MemOperand(reg_base); 2019 } 2020 } 2021 2022 return mem; 2023} 2024 2025void MacroAssembler::BumpSystemStackPointer(const Operand& space) { 2026 VIXL_ASSERT(!sp.Is(StackPointer())); 2027 // TODO: Several callers rely on this not using scratch registers, so we use 2028 // the assembler directly here. However, this means that large immediate 2029 // values of 'space' cannot be handled. 2030 InstructionAccurateScope scope(this, 1); 2031 sub(sp, StackPointer(), space); 2032} 2033 2034 2035// TODO(all): Fix printf for NEON registers, and resolve whether we should be 2036// using FPRegister or VRegister here. 2037 2038// This is the main Printf implementation. All callee-saved registers are 2039// preserved, but NZCV and the caller-saved registers may be clobbered. 2040void MacroAssembler::PrintfNoPreserve(const char * format, 2041 const CPURegister& arg0, 2042 const CPURegister& arg1, 2043 const CPURegister& arg2, 2044 const CPURegister& arg3) { 2045 // We cannot handle a caller-saved stack pointer. It doesn't make much sense 2046 // in most cases anyway, so this restriction shouldn't be too serious. 2047 VIXL_ASSERT(!kCallerSaved.IncludesAliasOf(StackPointer())); 2048 2049 // The provided arguments, and their proper PCS registers. 2050 CPURegister args[kPrintfMaxArgCount] = {arg0, arg1, arg2, arg3}; 2051 CPURegister pcs[kPrintfMaxArgCount]; 2052 2053 int arg_count = kPrintfMaxArgCount; 2054 2055 // The PCS varargs registers for printf. Note that x0 is used for the printf 2056 // format string. 2057 static const CPURegList kPCSVarargs = 2058 CPURegList(CPURegister::kRegister, kXRegSize, 1, arg_count); 2059 static const CPURegList kPCSVarargsFP = 2060 CPURegList(CPURegister::kVRegister, kDRegSize, 0, arg_count - 1); 2061 2062 // We can use caller-saved registers as scratch values, except for the 2063 // arguments and the PCS registers where they might need to go. 2064 UseScratchRegisterScope temps(this); 2065 temps.Include(kCallerSaved); 2066 temps.Include(kCallerSavedV); 2067 temps.Exclude(kPCSVarargs); 2068 temps.Exclude(kPCSVarargsFP); 2069 temps.Exclude(arg0, arg1, arg2, arg3); 2070 2071 // Copies of the arg lists that we can iterate through. 2072 CPURegList pcs_varargs = kPCSVarargs; 2073 CPURegList pcs_varargs_fp = kPCSVarargsFP; 2074 2075 // Place the arguments. There are lots of clever tricks and optimizations we 2076 // could use here, but Printf is a debug tool so instead we just try to keep 2077 // it simple: Move each input that isn't already in the right place to a 2078 // scratch register, then move everything back. 2079 for (unsigned i = 0; i < kPrintfMaxArgCount; i++) { 2080 // Work out the proper PCS register for this argument. 2081 if (args[i].IsRegister()) { 2082 pcs[i] = pcs_varargs.PopLowestIndex().X(); 2083 // We might only need a W register here. We need to know the size of the 2084 // argument so we can properly encode it for the simulator call. 2085 if (args[i].Is32Bits()) pcs[i] = pcs[i].W(); 2086 } else if (args[i].IsVRegister()) { 2087 // In C, floats are always cast to doubles for varargs calls. 2088 pcs[i] = pcs_varargs_fp.PopLowestIndex().D(); 2089 } else { 2090 VIXL_ASSERT(args[i].IsNone()); 2091 arg_count = i; 2092 break; 2093 } 2094 2095 // If the argument is already in the right place, leave it where it is. 2096 if (args[i].Aliases(pcs[i])) continue; 2097 2098 // Otherwise, if the argument is in a PCS argument register, allocate an 2099 // appropriate scratch register and then move it out of the way. 2100 if (kPCSVarargs.IncludesAliasOf(args[i]) || 2101 kPCSVarargsFP.IncludesAliasOf(args[i])) { 2102 if (args[i].IsRegister()) { 2103 Register old_arg = Register(args[i]); 2104 Register new_arg = temps.AcquireSameSizeAs(old_arg); 2105 Mov(new_arg, old_arg); 2106 args[i] = new_arg; 2107 } else { 2108 FPRegister old_arg = FPRegister(args[i]); 2109 FPRegister new_arg = temps.AcquireSameSizeAs(old_arg); 2110 Fmov(new_arg, old_arg); 2111 args[i] = new_arg; 2112 } 2113 } 2114 } 2115 2116 // Do a second pass to move values into their final positions and perform any 2117 // conversions that may be required. 2118 for (int i = 0; i < arg_count; i++) { 2119 VIXL_ASSERT(pcs[i].type() == args[i].type()); 2120 if (pcs[i].IsRegister()) { 2121 Mov(Register(pcs[i]), Register(args[i]), kDiscardForSameWReg); 2122 } else { 2123 VIXL_ASSERT(pcs[i].IsVRegister()); 2124 if (pcs[i].size() == args[i].size()) { 2125 Fmov(FPRegister(pcs[i]), FPRegister(args[i])); 2126 } else { 2127 Fcvt(FPRegister(pcs[i]), FPRegister(args[i])); 2128 } 2129 } 2130 } 2131 2132 // Load the format string into x0, as per the procedure-call standard. 2133 // 2134 // To make the code as portable as possible, the format string is encoded 2135 // directly in the instruction stream. It might be cleaner to encode it in a 2136 // literal pool, but since Printf is usually used for debugging, it is 2137 // beneficial for it to be minimally dependent on other features. 2138 temps.Exclude(x0); 2139 Label format_address; 2140 Adr(x0, &format_address); 2141 2142 // Emit the format string directly in the instruction stream. 2143 { 2144 BlockPoolsScope scope(this); 2145 // Data emitted: 2146 // branch 2147 // strlen(format) + 1 (includes null termination) 2148 // padding to next instruction 2149 // unreachable 2150 EmissionCheckScope guard( 2151 this, 2152 AlignUp(strlen(format) + 1, kInstructionSize) + 2 * kInstructionSize); 2153 Label after_data; 2154 B(&after_data); 2155 Bind(&format_address); 2156 EmitString(format); 2157 Unreachable(); 2158 Bind(&after_data); 2159 } 2160 2161 // We don't pass any arguments on the stack, but we still need to align the C 2162 // stack pointer to a 16-byte boundary for PCS compliance. 2163 if (!sp.Is(StackPointer())) { 2164 Bic(sp, StackPointer(), 0xf); 2165 } 2166 2167 // Actually call printf. This part needs special handling for the simulator, 2168 // since the system printf function will use a different instruction set and 2169 // the procedure-call standard will not be compatible. 2170#ifdef USE_SIMULATOR 2171 { 2172 InstructionAccurateScope scope(this, kPrintfLength / kInstructionSize); 2173 hlt(kPrintfOpcode); 2174 dc32(arg_count); // kPrintfArgCountOffset 2175 2176 // Determine the argument pattern. 2177 uint32_t arg_pattern_list = 0; 2178 for (int i = 0; i < arg_count; i++) { 2179 uint32_t arg_pattern; 2180 if (pcs[i].IsRegister()) { 2181 arg_pattern = pcs[i].Is32Bits() ? kPrintfArgW : kPrintfArgX; 2182 } else { 2183 VIXL_ASSERT(pcs[i].Is64Bits()); 2184 arg_pattern = kPrintfArgD; 2185 } 2186 VIXL_ASSERT(arg_pattern < (1 << kPrintfArgPatternBits)); 2187 arg_pattern_list |= (arg_pattern << (kPrintfArgPatternBits * i)); 2188 } 2189 dc32(arg_pattern_list); // kPrintfArgPatternListOffset 2190 } 2191#else 2192 Register tmp = temps.AcquireX(); 2193 Mov(tmp, reinterpret_cast<uintptr_t>(printf)); 2194 Blr(tmp); 2195#endif 2196} 2197 2198 2199void MacroAssembler::Printf(const char * format, 2200 CPURegister arg0, 2201 CPURegister arg1, 2202 CPURegister arg2, 2203 CPURegister arg3) { 2204 // We can only print sp if it is the current stack pointer. 2205 if (!sp.Is(StackPointer())) { 2206 VIXL_ASSERT(!sp.Aliases(arg0)); 2207 VIXL_ASSERT(!sp.Aliases(arg1)); 2208 VIXL_ASSERT(!sp.Aliases(arg2)); 2209 VIXL_ASSERT(!sp.Aliases(arg3)); 2210 } 2211 2212 // Make sure that the macro assembler doesn't try to use any of our arguments 2213 // as scratch registers. 2214 UseScratchRegisterScope exclude_all(this); 2215 exclude_all.ExcludeAll(); 2216 2217 // Preserve all caller-saved registers as well as NZCV. 2218 // If sp is the stack pointer, PushCPURegList asserts that the size of each 2219 // list is a multiple of 16 bytes. 2220 PushCPURegList(kCallerSaved); 2221 PushCPURegList(kCallerSavedV); 2222 2223 { UseScratchRegisterScope temps(this); 2224 // We can use caller-saved registers as scratch values (except for argN). 2225 temps.Include(kCallerSaved); 2226 temps.Include(kCallerSavedV); 2227 temps.Exclude(arg0, arg1, arg2, arg3); 2228 2229 // If any of the arguments are the current stack pointer, allocate a new 2230 // register for them, and adjust the value to compensate for pushing the 2231 // caller-saved registers. 2232 bool arg0_sp = StackPointer().Aliases(arg0); 2233 bool arg1_sp = StackPointer().Aliases(arg1); 2234 bool arg2_sp = StackPointer().Aliases(arg2); 2235 bool arg3_sp = StackPointer().Aliases(arg3); 2236 if (arg0_sp || arg1_sp || arg2_sp || arg3_sp) { 2237 // Allocate a register to hold the original stack pointer value, to pass 2238 // to PrintfNoPreserve as an argument. 2239 Register arg_sp = temps.AcquireX(); 2240 Add(arg_sp, StackPointer(), 2241 kCallerSaved.TotalSizeInBytes() + kCallerSavedV.TotalSizeInBytes()); 2242 if (arg0_sp) arg0 = Register(arg_sp.code(), arg0.size()); 2243 if (arg1_sp) arg1 = Register(arg_sp.code(), arg1.size()); 2244 if (arg2_sp) arg2 = Register(arg_sp.code(), arg2.size()); 2245 if (arg3_sp) arg3 = Register(arg_sp.code(), arg3.size()); 2246 } 2247 2248 // Preserve NZCV. 2249 Register tmp = temps.AcquireX(); 2250 Mrs(tmp, NZCV); 2251 Push(tmp, xzr); 2252 temps.Release(tmp); 2253 2254 PrintfNoPreserve(format, arg0, arg1, arg2, arg3); 2255 2256 // Restore NZCV. 2257 tmp = temps.AcquireX(); 2258 Pop(xzr, tmp); 2259 Msr(NZCV, tmp); 2260 temps.Release(tmp); 2261 } 2262 2263 PopCPURegList(kCallerSavedV); 2264 PopCPURegList(kCallerSaved); 2265} 2266 2267void MacroAssembler::Trace(TraceParameters parameters, TraceCommand command) { 2268 VIXL_ASSERT(allow_macro_instructions_); 2269 2270#ifdef USE_SIMULATOR 2271 // The arguments to the trace pseudo instruction need to be contiguous in 2272 // memory, so make sure we don't try to emit a literal pool. 2273 InstructionAccurateScope scope(this, kTraceLength / kInstructionSize); 2274 2275 Label start; 2276 bind(&start); 2277 2278 // Refer to simulator-a64.h for a description of the marker and its 2279 // arguments. 2280 hlt(kTraceOpcode); 2281 2282 VIXL_ASSERT(SizeOfCodeGeneratedSince(&start) == kTraceParamsOffset); 2283 dc32(parameters); 2284 2285 VIXL_ASSERT(SizeOfCodeGeneratedSince(&start) == kTraceCommandOffset); 2286 dc32(command); 2287#else 2288 // Emit nothing on real hardware. 2289 USE(parameters); 2290 USE(command); 2291#endif 2292} 2293 2294 2295void MacroAssembler::Log(TraceParameters parameters) { 2296 VIXL_ASSERT(allow_macro_instructions_); 2297 2298#ifdef USE_SIMULATOR 2299 // The arguments to the log pseudo instruction need to be contiguous in 2300 // memory, so make sure we don't try to emit a literal pool. 2301 InstructionAccurateScope scope(this, kLogLength / kInstructionSize); 2302 2303 Label start; 2304 bind(&start); 2305 2306 // Refer to simulator-a64.h for a description of the marker and its 2307 // arguments. 2308 hlt(kLogOpcode); 2309 2310 VIXL_ASSERT(SizeOfCodeGeneratedSince(&start) == kLogParamsOffset); 2311 dc32(parameters); 2312#else 2313 // Emit nothing on real hardware. 2314 USE(parameters); 2315#endif 2316} 2317 2318 2319void MacroAssembler::EnableInstrumentation() { 2320 VIXL_ASSERT(!isprint(InstrumentStateEnable)); 2321 InstructionAccurateScope scope(this, 1); 2322 movn(xzr, InstrumentStateEnable); 2323} 2324 2325 2326void MacroAssembler::DisableInstrumentation() { 2327 VIXL_ASSERT(!isprint(InstrumentStateDisable)); 2328 InstructionAccurateScope scope(this, 1); 2329 movn(xzr, InstrumentStateDisable); 2330} 2331 2332 2333void MacroAssembler::AnnotateInstrumentation(const char* marker_name) { 2334 VIXL_ASSERT(strlen(marker_name) == 2); 2335 2336 // We allow only printable characters in the marker names. Unprintable 2337 // characters are reserved for controlling features of the instrumentation. 2338 VIXL_ASSERT(isprint(marker_name[0]) && isprint(marker_name[1])); 2339 2340 InstructionAccurateScope scope(this, 1); 2341 movn(xzr, (marker_name[1] << 8) | marker_name[0]); 2342} 2343 2344 2345void UseScratchRegisterScope::Open(MacroAssembler* masm) { 2346 VIXL_ASSERT(!initialised_); 2347 available_ = masm->TmpList(); 2348 availablefp_ = masm->FPTmpList(); 2349 old_available_ = available_->list(); 2350 old_availablefp_ = availablefp_->list(); 2351 VIXL_ASSERT(available_->type() == CPURegister::kRegister); 2352 VIXL_ASSERT(availablefp_->type() == CPURegister::kVRegister); 2353#ifdef VIXL_DEBUG 2354 initialised_ = true; 2355#endif 2356} 2357 2358 2359void UseScratchRegisterScope::Close() { 2360 if (available_) { 2361 available_->set_list(old_available_); 2362 available_ = NULL; 2363 } 2364 if (availablefp_) { 2365 availablefp_->set_list(old_availablefp_); 2366 availablefp_ = NULL; 2367 } 2368#ifdef VIXL_DEBUG 2369 initialised_ = false; 2370#endif 2371} 2372 2373 2374UseScratchRegisterScope::UseScratchRegisterScope(MacroAssembler* masm) { 2375#ifdef VIXL_DEBUG 2376 initialised_ = false; 2377#endif 2378 Open(masm); 2379} 2380 2381// This allows deferred (and optional) initialisation of the scope. 2382UseScratchRegisterScope::UseScratchRegisterScope() 2383 : available_(NULL), availablefp_(NULL), 2384 old_available_(0), old_availablefp_(0) { 2385#ifdef VIXL_DEBUG 2386 initialised_ = false; 2387#endif 2388} 2389 2390UseScratchRegisterScope::~UseScratchRegisterScope() { 2391 Close(); 2392} 2393 2394 2395bool UseScratchRegisterScope::IsAvailable(const CPURegister& reg) const { 2396 return available_->IncludesAliasOf(reg) || availablefp_->IncludesAliasOf(reg); 2397} 2398 2399 2400Register UseScratchRegisterScope::AcquireSameSizeAs(const Register& reg) { 2401 int code = AcquireNextAvailable(available_).code(); 2402 return Register(code, reg.size()); 2403} 2404 2405 2406FPRegister UseScratchRegisterScope::AcquireSameSizeAs(const FPRegister& reg) { 2407 int code = AcquireNextAvailable(availablefp_).code(); 2408 return FPRegister(code, reg.size()); 2409} 2410 2411 2412void UseScratchRegisterScope::Release(const CPURegister& reg) { 2413 VIXL_ASSERT(initialised_); 2414 if (reg.IsRegister()) { 2415 ReleaseByCode(available_, reg.code()); 2416 } else if (reg.IsFPRegister()) { 2417 ReleaseByCode(availablefp_, reg.code()); 2418 } else { 2419 VIXL_ASSERT(reg.IsNone()); 2420 } 2421} 2422 2423 2424void UseScratchRegisterScope::Include(const CPURegList& list) { 2425 VIXL_ASSERT(initialised_); 2426 if (list.type() == CPURegister::kRegister) { 2427 // Make sure that neither sp nor xzr are included the list. 2428 IncludeByRegList(available_, list.list() & ~(xzr.Bit() | sp.Bit())); 2429 } else { 2430 VIXL_ASSERT(list.type() == CPURegister::kVRegister); 2431 IncludeByRegList(availablefp_, list.list()); 2432 } 2433} 2434 2435 2436void UseScratchRegisterScope::Include(const Register& reg1, 2437 const Register& reg2, 2438 const Register& reg3, 2439 const Register& reg4) { 2440 VIXL_ASSERT(initialised_); 2441 RegList include = reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit(); 2442 // Make sure that neither sp nor xzr are included the list. 2443 include &= ~(xzr.Bit() | sp.Bit()); 2444 2445 IncludeByRegList(available_, include); 2446} 2447 2448 2449void UseScratchRegisterScope::Include(const FPRegister& reg1, 2450 const FPRegister& reg2, 2451 const FPRegister& reg3, 2452 const FPRegister& reg4) { 2453 RegList include = reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit(); 2454 IncludeByRegList(availablefp_, include); 2455} 2456 2457 2458void UseScratchRegisterScope::Exclude(const CPURegList& list) { 2459 if (list.type() == CPURegister::kRegister) { 2460 ExcludeByRegList(available_, list.list()); 2461 } else { 2462 VIXL_ASSERT(list.type() == CPURegister::kVRegister); 2463 ExcludeByRegList(availablefp_, list.list()); 2464 } 2465} 2466 2467 2468void UseScratchRegisterScope::Exclude(const Register& reg1, 2469 const Register& reg2, 2470 const Register& reg3, 2471 const Register& reg4) { 2472 RegList exclude = reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit(); 2473 ExcludeByRegList(available_, exclude); 2474} 2475 2476 2477void UseScratchRegisterScope::Exclude(const FPRegister& reg1, 2478 const FPRegister& reg2, 2479 const FPRegister& reg3, 2480 const FPRegister& reg4) { 2481 RegList excludefp = reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit(); 2482 ExcludeByRegList(availablefp_, excludefp); 2483} 2484 2485 2486void UseScratchRegisterScope::Exclude(const CPURegister& reg1, 2487 const CPURegister& reg2, 2488 const CPURegister& reg3, 2489 const CPURegister& reg4) { 2490 RegList exclude = 0; 2491 RegList excludefp = 0; 2492 2493 const CPURegister regs[] = {reg1, reg2, reg3, reg4}; 2494 2495 for (unsigned i = 0; i < (sizeof(regs) / sizeof(regs[0])); i++) { 2496 if (regs[i].IsRegister()) { 2497 exclude |= regs[i].Bit(); 2498 } else if (regs[i].IsFPRegister()) { 2499 excludefp |= regs[i].Bit(); 2500 } else { 2501 VIXL_ASSERT(regs[i].IsNone()); 2502 } 2503 } 2504 2505 ExcludeByRegList(available_, exclude); 2506 ExcludeByRegList(availablefp_, excludefp); 2507} 2508 2509 2510void UseScratchRegisterScope::ExcludeAll() { 2511 ExcludeByRegList(available_, available_->list()); 2512 ExcludeByRegList(availablefp_, availablefp_->list()); 2513} 2514 2515 2516CPURegister UseScratchRegisterScope::AcquireNextAvailable( 2517 CPURegList* available) { 2518 VIXL_CHECK(!available->IsEmpty()); 2519 CPURegister result = available->PopLowestIndex(); 2520 VIXL_ASSERT(!AreAliased(result, xzr, sp)); 2521 return result; 2522} 2523 2524 2525void UseScratchRegisterScope::ReleaseByCode(CPURegList* available, int code) { 2526 ReleaseByRegList(available, static_cast<RegList>(1) << code); 2527} 2528 2529 2530void UseScratchRegisterScope::ReleaseByRegList(CPURegList* available, 2531 RegList regs) { 2532 available->set_list(available->list() | regs); 2533} 2534 2535 2536void UseScratchRegisterScope::IncludeByRegList(CPURegList* available, 2537 RegList regs) { 2538 available->set_list(available->list() | regs); 2539} 2540 2541 2542void UseScratchRegisterScope::ExcludeByRegList(CPURegList* available, 2543 RegList exclude) { 2544 available->set_list(available->list() & ~exclude); 2545} 2546 2547} // namespace vixl 2548