1// Copyright 2013 the V8 project authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "src/compiler/code-generator.h" 6 7#include <limits> 8 9#include "src/compilation-info.h" 10#include "src/compiler/code-generator-impl.h" 11#include "src/compiler/gap-resolver.h" 12#include "src/compiler/node-matchers.h" 13#include "src/compiler/osr.h" 14#include "src/heap/heap-inl.h" 15#include "src/wasm/wasm-module.h" 16#include "src/x64/assembler-x64.h" 17#include "src/x64/macro-assembler-x64.h" 18 19namespace v8 { 20namespace internal { 21namespace compiler { 22 23#define __ masm()-> 24 25// Adds X64 specific methods for decoding operands. 26class X64OperandConverter : public InstructionOperandConverter { 27 public: 28 X64OperandConverter(CodeGenerator* gen, Instruction* instr) 29 : InstructionOperandConverter(gen, instr) {} 30 31 Immediate InputImmediate(size_t index) { 32 return ToImmediate(instr_->InputAt(index)); 33 } 34 35 Operand InputOperand(size_t index, int extra = 0) { 36 return ToOperand(instr_->InputAt(index), extra); 37 } 38 39 Operand OutputOperand() { return ToOperand(instr_->Output()); } 40 41 Immediate ToImmediate(InstructionOperand* operand) { 42 Constant constant = ToConstant(operand); 43 if (constant.type() == Constant::kFloat64) { 44 DCHECK_EQ(0, bit_cast<int64_t>(constant.ToFloat64())); 45 return Immediate(0); 46 } 47 if (RelocInfo::IsWasmReference(constant.rmode())) { 48 return Immediate(constant.ToInt32(), constant.rmode()); 49 } 50 return Immediate(constant.ToInt32()); 51 } 52 53 Operand ToOperand(InstructionOperand* op, int extra = 0) { 54 DCHECK(op->IsStackSlot() || op->IsFPStackSlot()); 55 return SlotToOperand(AllocatedOperand::cast(op)->index(), extra); 56 } 57 58 Operand SlotToOperand(int slot_index, int extra = 0) { 59 FrameOffset offset = frame_access_state()->GetFrameOffset(slot_index); 60 return Operand(offset.from_stack_pointer() ? rsp : rbp, 61 offset.offset() + extra); 62 } 63 64 static size_t NextOffset(size_t* offset) { 65 size_t i = *offset; 66 (*offset)++; 67 return i; 68 } 69 70 static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) { 71 STATIC_ASSERT(0 == static_cast<int>(times_1)); 72 STATIC_ASSERT(1 == static_cast<int>(times_2)); 73 STATIC_ASSERT(2 == static_cast<int>(times_4)); 74 STATIC_ASSERT(3 == static_cast<int>(times_8)); 75 int scale = static_cast<int>(mode - one); 76 DCHECK(scale >= 0 && scale < 4); 77 return static_cast<ScaleFactor>(scale); 78 } 79 80 Operand MemoryOperand(size_t* offset) { 81 AddressingMode mode = AddressingModeField::decode(instr_->opcode()); 82 switch (mode) { 83 case kMode_MR: { 84 Register base = InputRegister(NextOffset(offset)); 85 int32_t disp = 0; 86 return Operand(base, disp); 87 } 88 case kMode_MRI: { 89 Register base = InputRegister(NextOffset(offset)); 90 int32_t disp = InputInt32(NextOffset(offset)); 91 return Operand(base, disp); 92 } 93 case kMode_MR1: 94 case kMode_MR2: 95 case kMode_MR4: 96 case kMode_MR8: { 97 Register base = InputRegister(NextOffset(offset)); 98 Register index = InputRegister(NextOffset(offset)); 99 ScaleFactor scale = ScaleFor(kMode_MR1, mode); 100 int32_t disp = 0; 101 return Operand(base, index, scale, disp); 102 } 103 case kMode_MR1I: 104 case kMode_MR2I: 105 case kMode_MR4I: 106 case kMode_MR8I: { 107 Register base = InputRegister(NextOffset(offset)); 108 Register index = InputRegister(NextOffset(offset)); 109 ScaleFactor scale = ScaleFor(kMode_MR1I, mode); 110 int32_t disp = InputInt32(NextOffset(offset)); 111 return Operand(base, index, scale, disp); 112 } 113 case kMode_M1: { 114 Register base = InputRegister(NextOffset(offset)); 115 int32_t disp = 0; 116 return Operand(base, disp); 117 } 118 case kMode_M2: 119 UNREACHABLE(); // Should use kModeMR with more compact encoding instead 120 return Operand(no_reg, 0); 121 case kMode_M4: 122 case kMode_M8: { 123 Register index = InputRegister(NextOffset(offset)); 124 ScaleFactor scale = ScaleFor(kMode_M1, mode); 125 int32_t disp = 0; 126 return Operand(index, scale, disp); 127 } 128 case kMode_M1I: 129 case kMode_M2I: 130 case kMode_M4I: 131 case kMode_M8I: { 132 Register index = InputRegister(NextOffset(offset)); 133 ScaleFactor scale = ScaleFor(kMode_M1I, mode); 134 int32_t disp = InputInt32(NextOffset(offset)); 135 return Operand(index, scale, disp); 136 } 137 case kMode_Root: { 138 Register base = kRootRegister; 139 int32_t disp = InputInt32(NextOffset(offset)); 140 return Operand(base, disp); 141 } 142 case kMode_None: 143 UNREACHABLE(); 144 return Operand(no_reg, 0); 145 } 146 UNREACHABLE(); 147 return Operand(no_reg, 0); 148 } 149 150 Operand MemoryOperand(size_t first_input = 0) { 151 return MemoryOperand(&first_input); 152 } 153}; 154 155 156namespace { 157 158bool HasImmediateInput(Instruction* instr, size_t index) { 159 return instr->InputAt(index)->IsImmediate(); 160} 161 162 163class OutOfLineLoadZero final : public OutOfLineCode { 164 public: 165 OutOfLineLoadZero(CodeGenerator* gen, Register result) 166 : OutOfLineCode(gen), result_(result) {} 167 168 void Generate() final { __ xorl(result_, result_); } 169 170 private: 171 Register const result_; 172}; 173 174class OutOfLineLoadFloat32NaN final : public OutOfLineCode { 175 public: 176 OutOfLineLoadFloat32NaN(CodeGenerator* gen, XMMRegister result) 177 : OutOfLineCode(gen), result_(result) {} 178 179 void Generate() final { 180 __ Xorps(result_, result_); 181 __ Divss(result_, result_); 182 } 183 184 private: 185 XMMRegister const result_; 186}; 187 188class OutOfLineLoadFloat64NaN final : public OutOfLineCode { 189 public: 190 OutOfLineLoadFloat64NaN(CodeGenerator* gen, XMMRegister result) 191 : OutOfLineCode(gen), result_(result) {} 192 193 void Generate() final { 194 __ Xorpd(result_, result_); 195 __ Divsd(result_, result_); 196 } 197 198 private: 199 XMMRegister const result_; 200}; 201 202class OutOfLineTruncateDoubleToI final : public OutOfLineCode { 203 public: 204 OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result, 205 XMMRegister input, 206 UnwindingInfoWriter* unwinding_info_writer) 207 : OutOfLineCode(gen), 208 result_(result), 209 input_(input), 210 unwinding_info_writer_(unwinding_info_writer) {} 211 212 void Generate() final { 213 __ subp(rsp, Immediate(kDoubleSize)); 214 unwinding_info_writer_->MaybeIncreaseBaseOffsetAt(__ pc_offset(), 215 kDoubleSize); 216 __ Movsd(MemOperand(rsp, 0), input_); 217 __ SlowTruncateToI(result_, rsp, 0); 218 __ addp(rsp, Immediate(kDoubleSize)); 219 unwinding_info_writer_->MaybeIncreaseBaseOffsetAt(__ pc_offset(), 220 -kDoubleSize); 221 } 222 223 private: 224 Register const result_; 225 XMMRegister const input_; 226 UnwindingInfoWriter* const unwinding_info_writer_; 227}; 228 229 230class OutOfLineRecordWrite final : public OutOfLineCode { 231 public: 232 OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand operand, 233 Register value, Register scratch0, Register scratch1, 234 RecordWriteMode mode) 235 : OutOfLineCode(gen), 236 object_(object), 237 operand_(operand), 238 value_(value), 239 scratch0_(scratch0), 240 scratch1_(scratch1), 241 mode_(mode) {} 242 243 void Generate() final { 244 if (mode_ > RecordWriteMode::kValueIsPointer) { 245 __ JumpIfSmi(value_, exit()); 246 } 247 __ CheckPageFlag(value_, scratch0_, 248 MemoryChunk::kPointersToHereAreInterestingMask, zero, 249 exit()); 250 RememberedSetAction const remembered_set_action = 251 mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET 252 : OMIT_REMEMBERED_SET; 253 SaveFPRegsMode const save_fp_mode = 254 frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs; 255 RecordWriteStub stub(isolate(), object_, scratch0_, scratch1_, 256 remembered_set_action, save_fp_mode); 257 __ leap(scratch1_, operand_); 258 __ CallStub(&stub); 259 } 260 261 private: 262 Register const object_; 263 Operand const operand_; 264 Register const value_; 265 Register const scratch0_; 266 Register const scratch1_; 267 RecordWriteMode const mode_; 268}; 269 270class WasmOutOfLineTrap final : public OutOfLineCode { 271 public: 272 WasmOutOfLineTrap(CodeGenerator* gen, int pc, bool frame_elided, 273 int32_t position, Instruction* instr) 274 : OutOfLineCode(gen), 275 gen_(gen), 276 pc_(pc), 277 frame_elided_(frame_elided), 278 position_(position), 279 instr_(instr) {} 280 281 // TODO(eholk): Refactor this method to take the code generator as a 282 // parameter. 283 void Generate() final { 284 __ RecordProtectedInstructionLanding(pc_); 285 286 if (frame_elided_) { 287 __ EnterFrame(StackFrame::WASM_COMPILED); 288 } 289 290 wasm::TrapReason trap_id = wasm::kTrapMemOutOfBounds; 291 int trap_reason = wasm::WasmOpcodes::TrapReasonToMessageId(trap_id); 292 __ Push(Smi::FromInt(trap_reason)); 293 __ Push(Smi::FromInt(position_)); 294 __ Move(rsi, gen_->isolate()->native_context()); 295 __ CallRuntime(Runtime::kThrowWasmError); 296 297 if (instr_->reference_map() != nullptr) { 298 gen_->RecordSafepoint(instr_->reference_map(), Safepoint::kSimple, 0, 299 Safepoint::kNoLazyDeopt); 300 } 301 } 302 303 private: 304 CodeGenerator* gen_; 305 int pc_; 306 bool frame_elided_; 307 int32_t position_; 308 Instruction* instr_; 309}; 310 311void EmitOOLTrapIfNeeded(Zone* zone, CodeGenerator* codegen, 312 InstructionCode opcode, size_t input_count, 313 X64OperandConverter& i, int pc, Instruction* instr) { 314 const X64MemoryProtection protection = 315 static_cast<X64MemoryProtection>(MiscField::decode(opcode)); 316 if (protection == X64MemoryProtection::kProtected) { 317 const bool frame_elided = !codegen->frame_access_state()->has_frame(); 318 const int32_t position = i.InputInt32(input_count - 1); 319 new (zone) WasmOutOfLineTrap(codegen, pc, frame_elided, position, instr); 320 } 321} 322} // namespace 323 324 325#define ASSEMBLE_UNOP(asm_instr) \ 326 do { \ 327 if (instr->Output()->IsRegister()) { \ 328 __ asm_instr(i.OutputRegister()); \ 329 } else { \ 330 __ asm_instr(i.OutputOperand()); \ 331 } \ 332 } while (0) 333 334 335#define ASSEMBLE_BINOP(asm_instr) \ 336 do { \ 337 if (HasImmediateInput(instr, 1)) { \ 338 if (instr->InputAt(0)->IsRegister()) { \ 339 __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \ 340 } else { \ 341 __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \ 342 } \ 343 } else { \ 344 if (instr->InputAt(1)->IsRegister()) { \ 345 __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \ 346 } else { \ 347 __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \ 348 } \ 349 } \ 350 } while (0) 351 352#define ASSEMBLE_COMPARE(asm_instr) \ 353 do { \ 354 if (AddressingModeField::decode(instr->opcode()) != kMode_None) { \ 355 size_t index = 0; \ 356 Operand left = i.MemoryOperand(&index); \ 357 if (HasImmediateInput(instr, index)) { \ 358 __ asm_instr(left, i.InputImmediate(index)); \ 359 } else { \ 360 __ asm_instr(left, i.InputRegister(index)); \ 361 } \ 362 } else { \ 363 if (HasImmediateInput(instr, 1)) { \ 364 if (instr->InputAt(0)->IsRegister()) { \ 365 __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \ 366 } else { \ 367 __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \ 368 } \ 369 } else { \ 370 if (instr->InputAt(1)->IsRegister()) { \ 371 __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \ 372 } else { \ 373 __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \ 374 } \ 375 } \ 376 } \ 377 } while (0) 378 379#define ASSEMBLE_MULT(asm_instr) \ 380 do { \ 381 if (HasImmediateInput(instr, 1)) { \ 382 if (instr->InputAt(0)->IsRegister()) { \ 383 __ asm_instr(i.OutputRegister(), i.InputRegister(0), \ 384 i.InputImmediate(1)); \ 385 } else { \ 386 __ asm_instr(i.OutputRegister(), i.InputOperand(0), \ 387 i.InputImmediate(1)); \ 388 } \ 389 } else { \ 390 if (instr->InputAt(1)->IsRegister()) { \ 391 __ asm_instr(i.OutputRegister(), i.InputRegister(1)); \ 392 } else { \ 393 __ asm_instr(i.OutputRegister(), i.InputOperand(1)); \ 394 } \ 395 } \ 396 } while (0) 397 398 399#define ASSEMBLE_SHIFT(asm_instr, width) \ 400 do { \ 401 if (HasImmediateInput(instr, 1)) { \ 402 if (instr->Output()->IsRegister()) { \ 403 __ asm_instr(i.OutputRegister(), Immediate(i.InputInt##width(1))); \ 404 } else { \ 405 __ asm_instr(i.OutputOperand(), Immediate(i.InputInt##width(1))); \ 406 } \ 407 } else { \ 408 if (instr->Output()->IsRegister()) { \ 409 __ asm_instr##_cl(i.OutputRegister()); \ 410 } else { \ 411 __ asm_instr##_cl(i.OutputOperand()); \ 412 } \ 413 } \ 414 } while (0) 415 416 417#define ASSEMBLE_MOVX(asm_instr) \ 418 do { \ 419 if (instr->addressing_mode() != kMode_None) { \ 420 __ asm_instr(i.OutputRegister(), i.MemoryOperand()); \ 421 } else if (instr->InputAt(0)->IsRegister()) { \ 422 __ asm_instr(i.OutputRegister(), i.InputRegister(0)); \ 423 } else { \ 424 __ asm_instr(i.OutputRegister(), i.InputOperand(0)); \ 425 } \ 426 } while (0) 427 428#define ASSEMBLE_SSE_BINOP(asm_instr) \ 429 do { \ 430 if (instr->InputAt(1)->IsFPRegister()) { \ 431 __ asm_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); \ 432 } else { \ 433 __ asm_instr(i.InputDoubleRegister(0), i.InputOperand(1)); \ 434 } \ 435 } while (0) 436 437#define ASSEMBLE_SSE_UNOP(asm_instr) \ 438 do { \ 439 if (instr->InputAt(0)->IsFPRegister()) { \ 440 __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); \ 441 } else { \ 442 __ asm_instr(i.OutputDoubleRegister(), i.InputOperand(0)); \ 443 } \ 444 } while (0) 445 446#define ASSEMBLE_AVX_BINOP(asm_instr) \ 447 do { \ 448 CpuFeatureScope avx_scope(masm(), AVX); \ 449 if (instr->InputAt(1)->IsFPRegister()) { \ 450 __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \ 451 i.InputDoubleRegister(1)); \ 452 } else { \ 453 __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \ 454 i.InputOperand(1)); \ 455 } \ 456 } while (0) 457 458#define ASSEMBLE_CHECKED_LOAD_FLOAT(asm_instr, OutOfLineLoadNaN) \ 459 do { \ 460 auto result = i.OutputDoubleRegister(); \ 461 auto buffer = i.InputRegister(0); \ 462 auto index1 = i.InputRegister(1); \ 463 auto index2 = i.InputUint32(2); \ 464 OutOfLineCode* ool; \ 465 if (instr->InputAt(3)->IsRegister()) { \ 466 auto length = i.InputRegister(3); \ 467 DCHECK_EQ(0u, index2); \ 468 __ cmpl(index1, length); \ 469 ool = new (zone()) OutOfLineLoadNaN(this, result); \ 470 } else { \ 471 auto length = i.InputUint32(3); \ 472 RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \ 473 DCHECK_LE(index2, length); \ 474 __ cmpl(index1, Immediate(length - index2, rmode)); \ 475 class OutOfLineLoadFloat final : public OutOfLineCode { \ 476 public: \ 477 OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result, \ 478 Register buffer, Register index1, int32_t index2, \ 479 int32_t length, RelocInfo::Mode rmode) \ 480 : OutOfLineCode(gen), \ 481 result_(result), \ 482 buffer_(buffer), \ 483 index1_(index1), \ 484 index2_(index2), \ 485 length_(length), \ 486 rmode_(rmode) {} \ 487 \ 488 void Generate() final { \ 489 __ leal(kScratchRegister, Operand(index1_, index2_)); \ 490 __ Pcmpeqd(result_, result_); \ 491 __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \ 492 __ j(above_equal, exit()); \ 493 __ asm_instr(result_, \ 494 Operand(buffer_, kScratchRegister, times_1, 0)); \ 495 } \ 496 \ 497 private: \ 498 XMMRegister const result_; \ 499 Register const buffer_; \ 500 Register const index1_; \ 501 int32_t const index2_; \ 502 int32_t const length_; \ 503 RelocInfo::Mode rmode_; \ 504 }; \ 505 ool = new (zone()) OutOfLineLoadFloat(this, result, buffer, index1, \ 506 index2, length, rmode); \ 507 } \ 508 __ j(above_equal, ool->entry()); \ 509 __ asm_instr(result, Operand(buffer, index1, times_1, index2)); \ 510 __ bind(ool->exit()); \ 511 } while (false) 512 513#define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr) \ 514 do { \ 515 auto result = i.OutputRegister(); \ 516 auto buffer = i.InputRegister(0); \ 517 auto index1 = i.InputRegister(1); \ 518 auto index2 = i.InputUint32(2); \ 519 OutOfLineCode* ool; \ 520 if (instr->InputAt(3)->IsRegister()) { \ 521 auto length = i.InputRegister(3); \ 522 DCHECK_EQ(0u, index2); \ 523 __ cmpl(index1, length); \ 524 ool = new (zone()) OutOfLineLoadZero(this, result); \ 525 } else { \ 526 auto length = i.InputUint32(3); \ 527 RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \ 528 DCHECK_LE(index2, length); \ 529 __ cmpl(index1, Immediate(length - index2, rmode)); \ 530 class OutOfLineLoadInteger final : public OutOfLineCode { \ 531 public: \ 532 OutOfLineLoadInteger(CodeGenerator* gen, Register result, \ 533 Register buffer, Register index1, int32_t index2, \ 534 int32_t length, RelocInfo::Mode rmode) \ 535 : OutOfLineCode(gen), \ 536 result_(result), \ 537 buffer_(buffer), \ 538 index1_(index1), \ 539 index2_(index2), \ 540 length_(length), \ 541 rmode_(rmode) {} \ 542 \ 543 void Generate() final { \ 544 Label oob; \ 545 __ leal(kScratchRegister, Operand(index1_, index2_)); \ 546 __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \ 547 __ j(above_equal, &oob, Label::kNear); \ 548 __ asm_instr(result_, \ 549 Operand(buffer_, kScratchRegister, times_1, 0)); \ 550 __ jmp(exit()); \ 551 __ bind(&oob); \ 552 __ xorl(result_, result_); \ 553 } \ 554 \ 555 private: \ 556 Register const result_; \ 557 Register const buffer_; \ 558 Register const index1_; \ 559 int32_t const index2_; \ 560 int32_t const length_; \ 561 RelocInfo::Mode const rmode_; \ 562 }; \ 563 ool = new (zone()) OutOfLineLoadInteger(this, result, buffer, index1, \ 564 index2, length, rmode); \ 565 } \ 566 __ j(above_equal, ool->entry()); \ 567 __ asm_instr(result, Operand(buffer, index1, times_1, index2)); \ 568 __ bind(ool->exit()); \ 569 } while (false) 570 571#define ASSEMBLE_CHECKED_STORE_FLOAT(asm_instr) \ 572 do { \ 573 auto buffer = i.InputRegister(0); \ 574 auto index1 = i.InputRegister(1); \ 575 auto index2 = i.InputUint32(2); \ 576 auto value = i.InputDoubleRegister(4); \ 577 if (instr->InputAt(3)->IsRegister()) { \ 578 auto length = i.InputRegister(3); \ 579 DCHECK_EQ(0u, index2); \ 580 Label done; \ 581 __ cmpl(index1, length); \ 582 __ j(above_equal, &done, Label::kNear); \ 583 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \ 584 __ bind(&done); \ 585 } else { \ 586 auto length = i.InputUint32(3); \ 587 RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \ 588 DCHECK_LE(index2, length); \ 589 __ cmpl(index1, Immediate(length - index2, rmode)); \ 590 class OutOfLineStoreFloat final : public OutOfLineCode { \ 591 public: \ 592 OutOfLineStoreFloat(CodeGenerator* gen, Register buffer, \ 593 Register index1, int32_t index2, int32_t length, \ 594 XMMRegister value, RelocInfo::Mode rmode) \ 595 : OutOfLineCode(gen), \ 596 buffer_(buffer), \ 597 index1_(index1), \ 598 index2_(index2), \ 599 length_(length), \ 600 value_(value), \ 601 rmode_(rmode) {} \ 602 \ 603 void Generate() final { \ 604 __ leal(kScratchRegister, Operand(index1_, index2_)); \ 605 __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \ 606 __ j(above_equal, exit()); \ 607 __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0), \ 608 value_); \ 609 } \ 610 \ 611 private: \ 612 Register const buffer_; \ 613 Register const index1_; \ 614 int32_t const index2_; \ 615 int32_t const length_; \ 616 XMMRegister const value_; \ 617 RelocInfo::Mode rmode_; \ 618 }; \ 619 auto ool = new (zone()) OutOfLineStoreFloat( \ 620 this, buffer, index1, index2, length, value, rmode); \ 621 __ j(above_equal, ool->entry()); \ 622 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \ 623 __ bind(ool->exit()); \ 624 } \ 625 } while (false) 626 627#define ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Value) \ 628 do { \ 629 auto buffer = i.InputRegister(0); \ 630 auto index1 = i.InputRegister(1); \ 631 auto index2 = i.InputUint32(2); \ 632 if (instr->InputAt(3)->IsRegister()) { \ 633 auto length = i.InputRegister(3); \ 634 DCHECK_EQ(0u, index2); \ 635 Label done; \ 636 __ cmpl(index1, length); \ 637 __ j(above_equal, &done, Label::kNear); \ 638 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \ 639 __ bind(&done); \ 640 } else { \ 641 auto length = i.InputUint32(3); \ 642 RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \ 643 DCHECK_LE(index2, length); \ 644 __ cmpl(index1, Immediate(length - index2, rmode)); \ 645 class OutOfLineStoreInteger final : public OutOfLineCode { \ 646 public: \ 647 OutOfLineStoreInteger(CodeGenerator* gen, Register buffer, \ 648 Register index1, int32_t index2, int32_t length, \ 649 Value value, RelocInfo::Mode rmode) \ 650 : OutOfLineCode(gen), \ 651 buffer_(buffer), \ 652 index1_(index1), \ 653 index2_(index2), \ 654 length_(length), \ 655 value_(value), \ 656 rmode_(rmode) {} \ 657 \ 658 void Generate() final { \ 659 __ leal(kScratchRegister, Operand(index1_, index2_)); \ 660 __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \ 661 __ j(above_equal, exit()); \ 662 __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0), \ 663 value_); \ 664 } \ 665 \ 666 private: \ 667 Register const buffer_; \ 668 Register const index1_; \ 669 int32_t const index2_; \ 670 int32_t const length_; \ 671 Value const value_; \ 672 RelocInfo::Mode rmode_; \ 673 }; \ 674 auto ool = new (zone()) OutOfLineStoreInteger( \ 675 this, buffer, index1, index2, length, value, rmode); \ 676 __ j(above_equal, ool->entry()); \ 677 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \ 678 __ bind(ool->exit()); \ 679 } \ 680 } while (false) 681 682#define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr) \ 683 do { \ 684 if (instr->InputAt(4)->IsRegister()) { \ 685 Register value = i.InputRegister(4); \ 686 ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Register); \ 687 } else { \ 688 Immediate value = i.InputImmediate(4); \ 689 ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Immediate); \ 690 } \ 691 } while (false) 692 693#define ASSEMBLE_IEEE754_BINOP(name) \ 694 do { \ 695 __ PrepareCallCFunction(2); \ 696 __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \ 697 2); \ 698 } while (false) 699 700#define ASSEMBLE_IEEE754_UNOP(name) \ 701 do { \ 702 __ PrepareCallCFunction(1); \ 703 __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \ 704 1); \ 705 } while (false) 706 707void CodeGenerator::AssembleDeconstructFrame() { 708 unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset()); 709 __ movq(rsp, rbp); 710 __ popq(rbp); 711} 712 713void CodeGenerator::AssemblePrepareTailCall() { 714 if (frame_access_state()->has_frame()) { 715 __ movq(rbp, MemOperand(rbp, 0)); 716 } 717 frame_access_state()->SetFrameAccessToSP(); 718} 719 720void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg, 721 Register scratch1, 722 Register scratch2, 723 Register scratch3) { 724 DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3)); 725 Label done; 726 727 // Check if current frame is an arguments adaptor frame. 728 __ cmpp(Operand(rbp, CommonFrameConstants::kContextOrFrameTypeOffset), 729 Immediate(StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR))); 730 __ j(not_equal, &done, Label::kNear); 731 732 // Load arguments count from current arguments adaptor frame (note, it 733 // does not include receiver). 734 Register caller_args_count_reg = scratch1; 735 __ SmiToInteger32( 736 caller_args_count_reg, 737 Operand(rbp, ArgumentsAdaptorFrameConstants::kLengthOffset)); 738 739 ParameterCount callee_args_count(args_reg); 740 __ PrepareForTailCall(callee_args_count, caller_args_count_reg, scratch2, 741 scratch3, ReturnAddressState::kOnStack); 742 __ bind(&done); 743} 744 745namespace { 746 747void AdjustStackPointerForTailCall(MacroAssembler* masm, 748 FrameAccessState* state, 749 int new_slot_above_sp, 750 bool allow_shrinkage = true) { 751 int current_sp_offset = state->GetSPToFPSlotCount() + 752 StandardFrameConstants::kFixedSlotCountAboveFp; 753 int stack_slot_delta = new_slot_above_sp - current_sp_offset; 754 if (stack_slot_delta > 0) { 755 masm->subq(rsp, Immediate(stack_slot_delta * kPointerSize)); 756 state->IncreaseSPDelta(stack_slot_delta); 757 } else if (allow_shrinkage && stack_slot_delta < 0) { 758 masm->addq(rsp, Immediate(-stack_slot_delta * kPointerSize)); 759 state->IncreaseSPDelta(stack_slot_delta); 760 } 761} 762 763} // namespace 764 765void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr, 766 int first_unused_stack_slot) { 767 CodeGenerator::PushTypeFlags flags(kImmediatePush | kScalarPush); 768 ZoneVector<MoveOperands*> pushes(zone()); 769 GetPushCompatibleMoves(instr, flags, &pushes); 770 771 if (!pushes.empty() && 772 (LocationOperand::cast(pushes.back()->destination()).index() + 1 == 773 first_unused_stack_slot)) { 774 X64OperandConverter g(this, instr); 775 for (auto move : pushes) { 776 LocationOperand destination_location( 777 LocationOperand::cast(move->destination())); 778 InstructionOperand source(move->source()); 779 AdjustStackPointerForTailCall(masm(), frame_access_state(), 780 destination_location.index()); 781 if (source.IsStackSlot()) { 782 LocationOperand source_location(LocationOperand::cast(source)); 783 __ Push(g.SlotToOperand(source_location.index())); 784 } else if (source.IsRegister()) { 785 LocationOperand source_location(LocationOperand::cast(source)); 786 __ Push(source_location.GetRegister()); 787 } else if (source.IsImmediate()) { 788 __ Push(Immediate(ImmediateOperand::cast(source).inline_value())); 789 } else { 790 // Pushes of non-scalar data types is not supported. 791 UNIMPLEMENTED(); 792 } 793 frame_access_state()->IncreaseSPDelta(1); 794 move->Eliminate(); 795 } 796 } 797 AdjustStackPointerForTailCall(masm(), frame_access_state(), 798 first_unused_stack_slot, false); 799} 800 801void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr, 802 int first_unused_stack_slot) { 803 AdjustStackPointerForTailCall(masm(), frame_access_state(), 804 first_unused_stack_slot); 805} 806 807// Assembles an instruction after register allocation, producing machine code. 808CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( 809 Instruction* instr) { 810 X64OperandConverter i(this, instr); 811 InstructionCode opcode = instr->opcode(); 812 ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode); 813 switch (arch_opcode) { 814 case kArchCallCodeObject: { 815 EnsureSpaceForLazyDeopt(); 816 if (HasImmediateInput(instr, 0)) { 817 Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0)); 818 __ Call(code, RelocInfo::CODE_TARGET); 819 } else { 820 Register reg = i.InputRegister(0); 821 __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag)); 822 __ call(reg); 823 } 824 RecordCallPosition(instr); 825 frame_access_state()->ClearSPDelta(); 826 break; 827 } 828 case kArchTailCallCodeObjectFromJSFunction: 829 case kArchTailCallCodeObject: { 830 if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) { 831 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister, 832 i.TempRegister(0), i.TempRegister(1), 833 i.TempRegister(2)); 834 } 835 if (HasImmediateInput(instr, 0)) { 836 Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0)); 837 __ jmp(code, RelocInfo::CODE_TARGET); 838 } else { 839 Register reg = i.InputRegister(0); 840 __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag)); 841 __ jmp(reg); 842 } 843 unwinding_info_writer_.MarkBlockWillExit(); 844 frame_access_state()->ClearSPDelta(); 845 frame_access_state()->SetFrameAccessToDefault(); 846 break; 847 } 848 case kArchTailCallAddress: { 849 CHECK(!HasImmediateInput(instr, 0)); 850 Register reg = i.InputRegister(0); 851 __ jmp(reg); 852 unwinding_info_writer_.MarkBlockWillExit(); 853 frame_access_state()->ClearSPDelta(); 854 frame_access_state()->SetFrameAccessToDefault(); 855 break; 856 } 857 case kArchCallJSFunction: { 858 EnsureSpaceForLazyDeopt(); 859 Register func = i.InputRegister(0); 860 if (FLAG_debug_code) { 861 // Check the function's context matches the context argument. 862 __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset)); 863 __ Assert(equal, kWrongFunctionContext); 864 } 865 __ Call(FieldOperand(func, JSFunction::kCodeEntryOffset)); 866 frame_access_state()->ClearSPDelta(); 867 RecordCallPosition(instr); 868 break; 869 } 870 case kArchTailCallJSFunctionFromJSFunction: { 871 Register func = i.InputRegister(0); 872 if (FLAG_debug_code) { 873 // Check the function's context matches the context argument. 874 __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset)); 875 __ Assert(equal, kWrongFunctionContext); 876 } 877 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister, 878 i.TempRegister(0), i.TempRegister(1), 879 i.TempRegister(2)); 880 __ jmp(FieldOperand(func, JSFunction::kCodeEntryOffset)); 881 frame_access_state()->ClearSPDelta(); 882 frame_access_state()->SetFrameAccessToDefault(); 883 break; 884 } 885 case kArchPrepareCallCFunction: { 886 // Frame alignment requires using FP-relative frame addressing. 887 frame_access_state()->SetFrameAccessToFP(); 888 int const num_parameters = MiscField::decode(instr->opcode()); 889 __ PrepareCallCFunction(num_parameters); 890 break; 891 } 892 case kArchPrepareTailCall: 893 AssemblePrepareTailCall(); 894 break; 895 case kArchCallCFunction: { 896 int const num_parameters = MiscField::decode(instr->opcode()); 897 if (HasImmediateInput(instr, 0)) { 898 ExternalReference ref = i.InputExternalReference(0); 899 __ CallCFunction(ref, num_parameters); 900 } else { 901 Register func = i.InputRegister(0); 902 __ CallCFunction(func, num_parameters); 903 } 904 frame_access_state()->SetFrameAccessToDefault(); 905 frame_access_state()->ClearSPDelta(); 906 break; 907 } 908 case kArchJmp: 909 AssembleArchJump(i.InputRpo(0)); 910 break; 911 case kArchLookupSwitch: 912 AssembleArchLookupSwitch(instr); 913 break; 914 case kArchTableSwitch: 915 AssembleArchTableSwitch(instr); 916 break; 917 case kArchComment: { 918 Address comment_string = i.InputExternalReference(0).address(); 919 __ RecordComment(reinterpret_cast<const char*>(comment_string)); 920 break; 921 } 922 case kArchDebugBreak: 923 __ int3(); 924 break; 925 case kArchNop: 926 case kArchThrowTerminator: 927 // don't emit code for nops. 928 break; 929 case kArchDeoptimize: { 930 int deopt_state_id = 931 BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore()); 932 CodeGenResult result = 933 AssembleDeoptimizerCall(deopt_state_id, current_source_position_); 934 if (result != kSuccess) return result; 935 break; 936 } 937 case kArchRet: 938 AssembleReturn(instr->InputAt(0)); 939 break; 940 case kArchStackPointer: 941 __ movq(i.OutputRegister(), rsp); 942 break; 943 case kArchFramePointer: 944 __ movq(i.OutputRegister(), rbp); 945 break; 946 case kArchParentFramePointer: 947 if (frame_access_state()->has_frame()) { 948 __ movq(i.OutputRegister(), Operand(rbp, 0)); 949 } else { 950 __ movq(i.OutputRegister(), rbp); 951 } 952 break; 953 case kArchTruncateDoubleToI: { 954 auto result = i.OutputRegister(); 955 auto input = i.InputDoubleRegister(0); 956 auto ool = new (zone()) OutOfLineTruncateDoubleToI( 957 this, result, input, &unwinding_info_writer_); 958 // We use Cvttsd2siq instead of Cvttsd2si due to performance reasons. The 959 // use of Cvttsd2siq requires the movl below to avoid sign extension. 960 __ Cvttsd2siq(result, input); 961 __ cmpq(result, Immediate(1)); 962 __ j(overflow, ool->entry()); 963 __ bind(ool->exit()); 964 __ movl(result, result); 965 break; 966 } 967 case kArchStoreWithWriteBarrier: { 968 RecordWriteMode mode = 969 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode())); 970 Register object = i.InputRegister(0); 971 size_t index = 0; 972 Operand operand = i.MemoryOperand(&index); 973 Register value = i.InputRegister(index); 974 Register scratch0 = i.TempRegister(0); 975 Register scratch1 = i.TempRegister(1); 976 auto ool = new (zone()) OutOfLineRecordWrite(this, object, operand, value, 977 scratch0, scratch1, mode); 978 __ movp(operand, value); 979 __ CheckPageFlag(object, scratch0, 980 MemoryChunk::kPointersFromHereAreInterestingMask, 981 not_zero, ool->entry()); 982 __ bind(ool->exit()); 983 break; 984 } 985 case kArchStackSlot: { 986 FrameOffset offset = 987 frame_access_state()->GetFrameOffset(i.InputInt32(0)); 988 Register base; 989 if (offset.from_stack_pointer()) { 990 base = rsp; 991 } else { 992 base = rbp; 993 } 994 __ leaq(i.OutputRegister(), Operand(base, offset.offset())); 995 break; 996 } 997 case kIeee754Float64Acos: 998 ASSEMBLE_IEEE754_UNOP(acos); 999 break; 1000 case kIeee754Float64Acosh: 1001 ASSEMBLE_IEEE754_UNOP(acosh); 1002 break; 1003 case kIeee754Float64Asin: 1004 ASSEMBLE_IEEE754_UNOP(asin); 1005 break; 1006 case kIeee754Float64Asinh: 1007 ASSEMBLE_IEEE754_UNOP(asinh); 1008 break; 1009 case kIeee754Float64Atan: 1010 ASSEMBLE_IEEE754_UNOP(atan); 1011 break; 1012 case kIeee754Float64Atanh: 1013 ASSEMBLE_IEEE754_UNOP(atanh); 1014 break; 1015 case kIeee754Float64Atan2: 1016 ASSEMBLE_IEEE754_BINOP(atan2); 1017 break; 1018 case kIeee754Float64Cbrt: 1019 ASSEMBLE_IEEE754_UNOP(cbrt); 1020 break; 1021 case kIeee754Float64Cos: 1022 ASSEMBLE_IEEE754_UNOP(cos); 1023 break; 1024 case kIeee754Float64Cosh: 1025 ASSEMBLE_IEEE754_UNOP(cosh); 1026 break; 1027 case kIeee754Float64Exp: 1028 ASSEMBLE_IEEE754_UNOP(exp); 1029 break; 1030 case kIeee754Float64Expm1: 1031 ASSEMBLE_IEEE754_UNOP(expm1); 1032 break; 1033 case kIeee754Float64Log: 1034 ASSEMBLE_IEEE754_UNOP(log); 1035 break; 1036 case kIeee754Float64Log1p: 1037 ASSEMBLE_IEEE754_UNOP(log1p); 1038 break; 1039 case kIeee754Float64Log2: 1040 ASSEMBLE_IEEE754_UNOP(log2); 1041 break; 1042 case kIeee754Float64Log10: 1043 ASSEMBLE_IEEE754_UNOP(log10); 1044 break; 1045 case kIeee754Float64Pow: { 1046 // TODO(bmeurer): Improve integration of the stub. 1047 __ Movsd(xmm2, xmm0); 1048 MathPowStub stub(isolate(), MathPowStub::DOUBLE); 1049 __ CallStub(&stub); 1050 __ Movsd(xmm0, xmm3); 1051 break; 1052 } 1053 case kIeee754Float64Sin: 1054 ASSEMBLE_IEEE754_UNOP(sin); 1055 break; 1056 case kIeee754Float64Sinh: 1057 ASSEMBLE_IEEE754_UNOP(sinh); 1058 break; 1059 case kIeee754Float64Tan: 1060 ASSEMBLE_IEEE754_UNOP(tan); 1061 break; 1062 case kIeee754Float64Tanh: 1063 ASSEMBLE_IEEE754_UNOP(tanh); 1064 break; 1065 case kX64Add32: 1066 ASSEMBLE_BINOP(addl); 1067 break; 1068 case kX64Add: 1069 ASSEMBLE_BINOP(addq); 1070 break; 1071 case kX64Sub32: 1072 ASSEMBLE_BINOP(subl); 1073 break; 1074 case kX64Sub: 1075 ASSEMBLE_BINOP(subq); 1076 break; 1077 case kX64And32: 1078 ASSEMBLE_BINOP(andl); 1079 break; 1080 case kX64And: 1081 ASSEMBLE_BINOP(andq); 1082 break; 1083 case kX64Cmp8: 1084 ASSEMBLE_COMPARE(cmpb); 1085 break; 1086 case kX64Cmp16: 1087 ASSEMBLE_COMPARE(cmpw); 1088 break; 1089 case kX64Cmp32: 1090 ASSEMBLE_COMPARE(cmpl); 1091 break; 1092 case kX64Cmp: 1093 ASSEMBLE_COMPARE(cmpq); 1094 break; 1095 case kX64Test8: 1096 ASSEMBLE_COMPARE(testb); 1097 break; 1098 case kX64Test16: 1099 ASSEMBLE_COMPARE(testw); 1100 break; 1101 case kX64Test32: 1102 ASSEMBLE_COMPARE(testl); 1103 break; 1104 case kX64Test: 1105 ASSEMBLE_COMPARE(testq); 1106 break; 1107 case kX64Imul32: 1108 ASSEMBLE_MULT(imull); 1109 break; 1110 case kX64Imul: 1111 ASSEMBLE_MULT(imulq); 1112 break; 1113 case kX64ImulHigh32: 1114 if (instr->InputAt(1)->IsRegister()) { 1115 __ imull(i.InputRegister(1)); 1116 } else { 1117 __ imull(i.InputOperand(1)); 1118 } 1119 break; 1120 case kX64UmulHigh32: 1121 if (instr->InputAt(1)->IsRegister()) { 1122 __ mull(i.InputRegister(1)); 1123 } else { 1124 __ mull(i.InputOperand(1)); 1125 } 1126 break; 1127 case kX64Idiv32: 1128 __ cdq(); 1129 __ idivl(i.InputRegister(1)); 1130 break; 1131 case kX64Idiv: 1132 __ cqo(); 1133 __ idivq(i.InputRegister(1)); 1134 break; 1135 case kX64Udiv32: 1136 __ xorl(rdx, rdx); 1137 __ divl(i.InputRegister(1)); 1138 break; 1139 case kX64Udiv: 1140 __ xorq(rdx, rdx); 1141 __ divq(i.InputRegister(1)); 1142 break; 1143 case kX64Not: 1144 ASSEMBLE_UNOP(notq); 1145 break; 1146 case kX64Not32: 1147 ASSEMBLE_UNOP(notl); 1148 break; 1149 case kX64Neg: 1150 ASSEMBLE_UNOP(negq); 1151 break; 1152 case kX64Neg32: 1153 ASSEMBLE_UNOP(negl); 1154 break; 1155 case kX64Or32: 1156 ASSEMBLE_BINOP(orl); 1157 break; 1158 case kX64Or: 1159 ASSEMBLE_BINOP(orq); 1160 break; 1161 case kX64Xor32: 1162 ASSEMBLE_BINOP(xorl); 1163 break; 1164 case kX64Xor: 1165 ASSEMBLE_BINOP(xorq); 1166 break; 1167 case kX64Shl32: 1168 ASSEMBLE_SHIFT(shll, 5); 1169 break; 1170 case kX64Shl: 1171 ASSEMBLE_SHIFT(shlq, 6); 1172 break; 1173 case kX64Shr32: 1174 ASSEMBLE_SHIFT(shrl, 5); 1175 break; 1176 case kX64Shr: 1177 ASSEMBLE_SHIFT(shrq, 6); 1178 break; 1179 case kX64Sar32: 1180 ASSEMBLE_SHIFT(sarl, 5); 1181 break; 1182 case kX64Sar: 1183 ASSEMBLE_SHIFT(sarq, 6); 1184 break; 1185 case kX64Ror32: 1186 ASSEMBLE_SHIFT(rorl, 5); 1187 break; 1188 case kX64Ror: 1189 ASSEMBLE_SHIFT(rorq, 6); 1190 break; 1191 case kX64Lzcnt: 1192 if (instr->InputAt(0)->IsRegister()) { 1193 __ Lzcntq(i.OutputRegister(), i.InputRegister(0)); 1194 } else { 1195 __ Lzcntq(i.OutputRegister(), i.InputOperand(0)); 1196 } 1197 break; 1198 case kX64Lzcnt32: 1199 if (instr->InputAt(0)->IsRegister()) { 1200 __ Lzcntl(i.OutputRegister(), i.InputRegister(0)); 1201 } else { 1202 __ Lzcntl(i.OutputRegister(), i.InputOperand(0)); 1203 } 1204 break; 1205 case kX64Tzcnt: 1206 if (instr->InputAt(0)->IsRegister()) { 1207 __ Tzcntq(i.OutputRegister(), i.InputRegister(0)); 1208 } else { 1209 __ Tzcntq(i.OutputRegister(), i.InputOperand(0)); 1210 } 1211 break; 1212 case kX64Tzcnt32: 1213 if (instr->InputAt(0)->IsRegister()) { 1214 __ Tzcntl(i.OutputRegister(), i.InputRegister(0)); 1215 } else { 1216 __ Tzcntl(i.OutputRegister(), i.InputOperand(0)); 1217 } 1218 break; 1219 case kX64Popcnt: 1220 if (instr->InputAt(0)->IsRegister()) { 1221 __ Popcntq(i.OutputRegister(), i.InputRegister(0)); 1222 } else { 1223 __ Popcntq(i.OutputRegister(), i.InputOperand(0)); 1224 } 1225 break; 1226 case kX64Popcnt32: 1227 if (instr->InputAt(0)->IsRegister()) { 1228 __ Popcntl(i.OutputRegister(), i.InputRegister(0)); 1229 } else { 1230 __ Popcntl(i.OutputRegister(), i.InputOperand(0)); 1231 } 1232 break; 1233 case kSSEFloat32Cmp: 1234 ASSEMBLE_SSE_BINOP(Ucomiss); 1235 break; 1236 case kSSEFloat32Add: 1237 ASSEMBLE_SSE_BINOP(addss); 1238 break; 1239 case kSSEFloat32Sub: 1240 ASSEMBLE_SSE_BINOP(subss); 1241 break; 1242 case kSSEFloat32Mul: 1243 ASSEMBLE_SSE_BINOP(mulss); 1244 break; 1245 case kSSEFloat32Div: 1246 ASSEMBLE_SSE_BINOP(divss); 1247 // Don't delete this mov. It may improve performance on some CPUs, 1248 // when there is a (v)mulss depending on the result. 1249 __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister()); 1250 break; 1251 case kSSEFloat32Abs: { 1252 // TODO(bmeurer): Use RIP relative 128-bit constants. 1253 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); 1254 __ psrlq(kScratchDoubleReg, 33); 1255 __ andps(i.OutputDoubleRegister(), kScratchDoubleReg); 1256 break; 1257 } 1258 case kSSEFloat32Neg: { 1259 // TODO(bmeurer): Use RIP relative 128-bit constants. 1260 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); 1261 __ psllq(kScratchDoubleReg, 31); 1262 __ xorps(i.OutputDoubleRegister(), kScratchDoubleReg); 1263 break; 1264 } 1265 case kSSEFloat32Sqrt: 1266 ASSEMBLE_SSE_UNOP(sqrtss); 1267 break; 1268 case kSSEFloat32ToFloat64: 1269 ASSEMBLE_SSE_UNOP(Cvtss2sd); 1270 break; 1271 case kSSEFloat32Round: { 1272 CpuFeatureScope sse_scope(masm(), SSE4_1); 1273 RoundingMode const mode = 1274 static_cast<RoundingMode>(MiscField::decode(instr->opcode())); 1275 __ Roundss(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode); 1276 break; 1277 } 1278 case kSSEFloat32ToInt32: 1279 if (instr->InputAt(0)->IsFPRegister()) { 1280 __ Cvttss2si(i.OutputRegister(), i.InputDoubleRegister(0)); 1281 } else { 1282 __ Cvttss2si(i.OutputRegister(), i.InputOperand(0)); 1283 } 1284 break; 1285 case kSSEFloat32ToUint32: { 1286 if (instr->InputAt(0)->IsFPRegister()) { 1287 __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0)); 1288 } else { 1289 __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0)); 1290 } 1291 break; 1292 } 1293 case kSSEFloat64Cmp: 1294 ASSEMBLE_SSE_BINOP(Ucomisd); 1295 break; 1296 case kSSEFloat64Add: 1297 ASSEMBLE_SSE_BINOP(addsd); 1298 break; 1299 case kSSEFloat64Sub: 1300 ASSEMBLE_SSE_BINOP(subsd); 1301 break; 1302 case kSSEFloat64Mul: 1303 ASSEMBLE_SSE_BINOP(mulsd); 1304 break; 1305 case kSSEFloat64Div: 1306 ASSEMBLE_SSE_BINOP(divsd); 1307 // Don't delete this mov. It may improve performance on some CPUs, 1308 // when there is a (v)mulsd depending on the result. 1309 __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister()); 1310 break; 1311 case kSSEFloat64Mod: { 1312 __ subq(rsp, Immediate(kDoubleSize)); 1313 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), 1314 kDoubleSize); 1315 // Move values to st(0) and st(1). 1316 __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(1)); 1317 __ fld_d(Operand(rsp, 0)); 1318 __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0)); 1319 __ fld_d(Operand(rsp, 0)); 1320 // Loop while fprem isn't done. 1321 Label mod_loop; 1322 __ bind(&mod_loop); 1323 // This instructions traps on all kinds inputs, but we are assuming the 1324 // floating point control word is set to ignore them all. 1325 __ fprem(); 1326 // The following 2 instruction implicitly use rax. 1327 __ fnstsw_ax(); 1328 if (CpuFeatures::IsSupported(SAHF)) { 1329 CpuFeatureScope sahf_scope(masm(), SAHF); 1330 __ sahf(); 1331 } else { 1332 __ shrl(rax, Immediate(8)); 1333 __ andl(rax, Immediate(0xFF)); 1334 __ pushq(rax); 1335 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), 1336 kPointerSize); 1337 __ popfq(); 1338 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), 1339 -kPointerSize); 1340 } 1341 __ j(parity_even, &mod_loop); 1342 // Move output to stack and clean up. 1343 __ fstp(1); 1344 __ fstp_d(Operand(rsp, 0)); 1345 __ Movsd(i.OutputDoubleRegister(), Operand(rsp, 0)); 1346 __ addq(rsp, Immediate(kDoubleSize)); 1347 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), 1348 -kDoubleSize); 1349 break; 1350 } 1351 case kSSEFloat32Max: { 1352 Label compare_nan, compare_swap, done_compare; 1353 if (instr->InputAt(1)->IsFPRegister()) { 1354 __ Ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1355 } else { 1356 __ Ucomiss(i.InputDoubleRegister(0), i.InputOperand(1)); 1357 } 1358 auto ool = 1359 new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister()); 1360 __ j(parity_even, ool->entry()); 1361 __ j(above, &done_compare, Label::kNear); 1362 __ j(below, &compare_swap, Label::kNear); 1363 __ Movmskps(kScratchRegister, i.InputDoubleRegister(0)); 1364 __ testl(kScratchRegister, Immediate(1)); 1365 __ j(zero, &done_compare, Label::kNear); 1366 __ bind(&compare_swap); 1367 if (instr->InputAt(1)->IsFPRegister()) { 1368 __ Movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1369 } else { 1370 __ Movss(i.InputDoubleRegister(0), i.InputOperand(1)); 1371 } 1372 __ bind(&done_compare); 1373 __ bind(ool->exit()); 1374 break; 1375 } 1376 case kSSEFloat32Min: { 1377 Label compare_swap, done_compare; 1378 if (instr->InputAt(1)->IsFPRegister()) { 1379 __ Ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1380 } else { 1381 __ Ucomiss(i.InputDoubleRegister(0), i.InputOperand(1)); 1382 } 1383 auto ool = 1384 new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister()); 1385 __ j(parity_even, ool->entry()); 1386 __ j(below, &done_compare, Label::kNear); 1387 __ j(above, &compare_swap, Label::kNear); 1388 if (instr->InputAt(1)->IsFPRegister()) { 1389 __ Movmskps(kScratchRegister, i.InputDoubleRegister(1)); 1390 } else { 1391 __ Movss(kScratchDoubleReg, i.InputOperand(1)); 1392 __ Movmskps(kScratchRegister, kScratchDoubleReg); 1393 } 1394 __ testl(kScratchRegister, Immediate(1)); 1395 __ j(zero, &done_compare, Label::kNear); 1396 __ bind(&compare_swap); 1397 if (instr->InputAt(1)->IsFPRegister()) { 1398 __ Movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1399 } else { 1400 __ Movss(i.InputDoubleRegister(0), i.InputOperand(1)); 1401 } 1402 __ bind(&done_compare); 1403 __ bind(ool->exit()); 1404 break; 1405 } 1406 case kSSEFloat64Max: { 1407 Label compare_nan, compare_swap, done_compare; 1408 if (instr->InputAt(1)->IsFPRegister()) { 1409 __ Ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1410 } else { 1411 __ Ucomisd(i.InputDoubleRegister(0), i.InputOperand(1)); 1412 } 1413 auto ool = 1414 new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister()); 1415 __ j(parity_even, ool->entry()); 1416 __ j(above, &done_compare, Label::kNear); 1417 __ j(below, &compare_swap, Label::kNear); 1418 __ Movmskpd(kScratchRegister, i.InputDoubleRegister(0)); 1419 __ testl(kScratchRegister, Immediate(1)); 1420 __ j(zero, &done_compare, Label::kNear); 1421 __ bind(&compare_swap); 1422 if (instr->InputAt(1)->IsFPRegister()) { 1423 __ Movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1424 } else { 1425 __ Movsd(i.InputDoubleRegister(0), i.InputOperand(1)); 1426 } 1427 __ bind(&done_compare); 1428 __ bind(ool->exit()); 1429 break; 1430 } 1431 case kSSEFloat64Min: { 1432 Label compare_swap, done_compare; 1433 if (instr->InputAt(1)->IsFPRegister()) { 1434 __ Ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1435 } else { 1436 __ Ucomisd(i.InputDoubleRegister(0), i.InputOperand(1)); 1437 } 1438 auto ool = 1439 new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister()); 1440 __ j(parity_even, ool->entry()); 1441 __ j(below, &done_compare, Label::kNear); 1442 __ j(above, &compare_swap, Label::kNear); 1443 if (instr->InputAt(1)->IsFPRegister()) { 1444 __ Movmskpd(kScratchRegister, i.InputDoubleRegister(1)); 1445 } else { 1446 __ Movsd(kScratchDoubleReg, i.InputOperand(1)); 1447 __ Movmskpd(kScratchRegister, kScratchDoubleReg); 1448 } 1449 __ testl(kScratchRegister, Immediate(1)); 1450 __ j(zero, &done_compare, Label::kNear); 1451 __ bind(&compare_swap); 1452 if (instr->InputAt(1)->IsFPRegister()) { 1453 __ Movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1454 } else { 1455 __ Movsd(i.InputDoubleRegister(0), i.InputOperand(1)); 1456 } 1457 __ bind(&done_compare); 1458 __ bind(ool->exit()); 1459 break; 1460 } 1461 case kSSEFloat64Abs: { 1462 // TODO(bmeurer): Use RIP relative 128-bit constants. 1463 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); 1464 __ psrlq(kScratchDoubleReg, 1); 1465 __ andpd(i.OutputDoubleRegister(), kScratchDoubleReg); 1466 break; 1467 } 1468 case kSSEFloat64Neg: { 1469 // TODO(bmeurer): Use RIP relative 128-bit constants. 1470 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); 1471 __ psllq(kScratchDoubleReg, 63); 1472 __ xorpd(i.OutputDoubleRegister(), kScratchDoubleReg); 1473 break; 1474 } 1475 case kSSEFloat64Sqrt: 1476 ASSEMBLE_SSE_UNOP(Sqrtsd); 1477 break; 1478 case kSSEFloat64Round: { 1479 CpuFeatureScope sse_scope(masm(), SSE4_1); 1480 RoundingMode const mode = 1481 static_cast<RoundingMode>(MiscField::decode(instr->opcode())); 1482 __ Roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode); 1483 break; 1484 } 1485 case kSSEFloat64ToFloat32: 1486 ASSEMBLE_SSE_UNOP(Cvtsd2ss); 1487 break; 1488 case kSSEFloat64ToInt32: 1489 if (instr->InputAt(0)->IsFPRegister()) { 1490 __ Cvttsd2si(i.OutputRegister(), i.InputDoubleRegister(0)); 1491 } else { 1492 __ Cvttsd2si(i.OutputRegister(), i.InputOperand(0)); 1493 } 1494 break; 1495 case kSSEFloat64ToUint32: { 1496 if (instr->InputAt(0)->IsFPRegister()) { 1497 __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0)); 1498 } else { 1499 __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0)); 1500 } 1501 if (MiscField::decode(instr->opcode())) { 1502 __ AssertZeroExtended(i.OutputRegister()); 1503 } 1504 break; 1505 } 1506 case kSSEFloat32ToInt64: 1507 if (instr->InputAt(0)->IsFPRegister()) { 1508 __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0)); 1509 } else { 1510 __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0)); 1511 } 1512 if (instr->OutputCount() > 1) { 1513 __ Set(i.OutputRegister(1), 1); 1514 Label done; 1515 Label fail; 1516 __ Move(kScratchDoubleReg, static_cast<float>(INT64_MIN)); 1517 if (instr->InputAt(0)->IsFPRegister()) { 1518 __ Ucomiss(kScratchDoubleReg, i.InputDoubleRegister(0)); 1519 } else { 1520 __ Ucomiss(kScratchDoubleReg, i.InputOperand(0)); 1521 } 1522 // If the input is NaN, then the conversion fails. 1523 __ j(parity_even, &fail); 1524 // If the input is INT64_MIN, then the conversion succeeds. 1525 __ j(equal, &done); 1526 __ cmpq(i.OutputRegister(0), Immediate(1)); 1527 // If the conversion results in INT64_MIN, but the input was not 1528 // INT64_MIN, then the conversion fails. 1529 __ j(no_overflow, &done); 1530 __ bind(&fail); 1531 __ Set(i.OutputRegister(1), 0); 1532 __ bind(&done); 1533 } 1534 break; 1535 case kSSEFloat64ToInt64: 1536 if (instr->InputAt(0)->IsFPRegister()) { 1537 __ Cvttsd2siq(i.OutputRegister(0), i.InputDoubleRegister(0)); 1538 } else { 1539 __ Cvttsd2siq(i.OutputRegister(0), i.InputOperand(0)); 1540 } 1541 if (instr->OutputCount() > 1) { 1542 __ Set(i.OutputRegister(1), 1); 1543 Label done; 1544 Label fail; 1545 __ Move(kScratchDoubleReg, static_cast<double>(INT64_MIN)); 1546 if (instr->InputAt(0)->IsFPRegister()) { 1547 __ Ucomisd(kScratchDoubleReg, i.InputDoubleRegister(0)); 1548 } else { 1549 __ Ucomisd(kScratchDoubleReg, i.InputOperand(0)); 1550 } 1551 // If the input is NaN, then the conversion fails. 1552 __ j(parity_even, &fail); 1553 // If the input is INT64_MIN, then the conversion succeeds. 1554 __ j(equal, &done); 1555 __ cmpq(i.OutputRegister(0), Immediate(1)); 1556 // If the conversion results in INT64_MIN, but the input was not 1557 // INT64_MIN, then the conversion fails. 1558 __ j(no_overflow, &done); 1559 __ bind(&fail); 1560 __ Set(i.OutputRegister(1), 0); 1561 __ bind(&done); 1562 } 1563 break; 1564 case kSSEFloat32ToUint64: { 1565 Label done; 1566 Label success; 1567 if (instr->OutputCount() > 1) { 1568 __ Set(i.OutputRegister(1), 0); 1569 } 1570 // There does not exist a Float32ToUint64 instruction, so we have to use 1571 // the Float32ToInt64 instruction. 1572 if (instr->InputAt(0)->IsFPRegister()) { 1573 __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0)); 1574 } else { 1575 __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0)); 1576 } 1577 // Check if the result of the Float32ToInt64 conversion is positive, we 1578 // are already done. 1579 __ testq(i.OutputRegister(), i.OutputRegister()); 1580 __ j(positive, &success); 1581 // The result of the first conversion was negative, which means that the 1582 // input value was not within the positive int64 range. We subtract 2^64 1583 // and convert it again to see if it is within the uint64 range. 1584 __ Move(kScratchDoubleReg, -9223372036854775808.0f); 1585 if (instr->InputAt(0)->IsFPRegister()) { 1586 __ addss(kScratchDoubleReg, i.InputDoubleRegister(0)); 1587 } else { 1588 __ addss(kScratchDoubleReg, i.InputOperand(0)); 1589 } 1590 __ Cvttss2siq(i.OutputRegister(), kScratchDoubleReg); 1591 __ testq(i.OutputRegister(), i.OutputRegister()); 1592 // The only possible negative value here is 0x80000000000000000, which is 1593 // used on x64 to indicate an integer overflow. 1594 __ j(negative, &done); 1595 // The input value is within uint64 range and the second conversion worked 1596 // successfully, but we still have to undo the subtraction we did 1597 // earlier. 1598 __ Set(kScratchRegister, 0x8000000000000000); 1599 __ orq(i.OutputRegister(), kScratchRegister); 1600 __ bind(&success); 1601 if (instr->OutputCount() > 1) { 1602 __ Set(i.OutputRegister(1), 1); 1603 } 1604 __ bind(&done); 1605 break; 1606 } 1607 case kSSEFloat64ToUint64: { 1608 Label done; 1609 Label success; 1610 if (instr->OutputCount() > 1) { 1611 __ Set(i.OutputRegister(1), 0); 1612 } 1613 // There does not exist a Float64ToUint64 instruction, so we have to use 1614 // the Float64ToInt64 instruction. 1615 if (instr->InputAt(0)->IsFPRegister()) { 1616 __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0)); 1617 } else { 1618 __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0)); 1619 } 1620 // Check if the result of the Float64ToInt64 conversion is positive, we 1621 // are already done. 1622 __ testq(i.OutputRegister(), i.OutputRegister()); 1623 __ j(positive, &success); 1624 // The result of the first conversion was negative, which means that the 1625 // input value was not within the positive int64 range. We subtract 2^64 1626 // and convert it again to see if it is within the uint64 range. 1627 __ Move(kScratchDoubleReg, -9223372036854775808.0); 1628 if (instr->InputAt(0)->IsFPRegister()) { 1629 __ addsd(kScratchDoubleReg, i.InputDoubleRegister(0)); 1630 } else { 1631 __ addsd(kScratchDoubleReg, i.InputOperand(0)); 1632 } 1633 __ Cvttsd2siq(i.OutputRegister(), kScratchDoubleReg); 1634 __ testq(i.OutputRegister(), i.OutputRegister()); 1635 // The only possible negative value here is 0x80000000000000000, which is 1636 // used on x64 to indicate an integer overflow. 1637 __ j(negative, &done); 1638 // The input value is within uint64 range and the second conversion worked 1639 // successfully, but we still have to undo the subtraction we did 1640 // earlier. 1641 __ Set(kScratchRegister, 0x8000000000000000); 1642 __ orq(i.OutputRegister(), kScratchRegister); 1643 __ bind(&success); 1644 if (instr->OutputCount() > 1) { 1645 __ Set(i.OutputRegister(1), 1); 1646 } 1647 __ bind(&done); 1648 break; 1649 } 1650 case kSSEInt32ToFloat64: 1651 if (instr->InputAt(0)->IsRegister()) { 1652 __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputRegister(0)); 1653 } else { 1654 __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputOperand(0)); 1655 } 1656 break; 1657 case kSSEInt32ToFloat32: 1658 if (instr->InputAt(0)->IsRegister()) { 1659 __ Cvtlsi2ss(i.OutputDoubleRegister(), i.InputRegister(0)); 1660 } else { 1661 __ Cvtlsi2ss(i.OutputDoubleRegister(), i.InputOperand(0)); 1662 } 1663 break; 1664 case kSSEInt64ToFloat32: 1665 if (instr->InputAt(0)->IsRegister()) { 1666 __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputRegister(0)); 1667 } else { 1668 __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputOperand(0)); 1669 } 1670 break; 1671 case kSSEInt64ToFloat64: 1672 if (instr->InputAt(0)->IsRegister()) { 1673 __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputRegister(0)); 1674 } else { 1675 __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputOperand(0)); 1676 } 1677 break; 1678 case kSSEUint64ToFloat32: 1679 if (instr->InputAt(0)->IsRegister()) { 1680 __ movq(kScratchRegister, i.InputRegister(0)); 1681 } else { 1682 __ movq(kScratchRegister, i.InputOperand(0)); 1683 } 1684 __ Cvtqui2ss(i.OutputDoubleRegister(), kScratchRegister, 1685 i.TempRegister(0)); 1686 break; 1687 case kSSEUint64ToFloat64: 1688 if (instr->InputAt(0)->IsRegister()) { 1689 __ movq(kScratchRegister, i.InputRegister(0)); 1690 } else { 1691 __ movq(kScratchRegister, i.InputOperand(0)); 1692 } 1693 __ Cvtqui2sd(i.OutputDoubleRegister(), kScratchRegister, 1694 i.TempRegister(0)); 1695 break; 1696 case kSSEUint32ToFloat64: 1697 if (instr->InputAt(0)->IsRegister()) { 1698 __ movl(kScratchRegister, i.InputRegister(0)); 1699 } else { 1700 __ movl(kScratchRegister, i.InputOperand(0)); 1701 } 1702 __ Cvtqsi2sd(i.OutputDoubleRegister(), kScratchRegister); 1703 break; 1704 case kSSEUint32ToFloat32: 1705 if (instr->InputAt(0)->IsRegister()) { 1706 __ movl(kScratchRegister, i.InputRegister(0)); 1707 } else { 1708 __ movl(kScratchRegister, i.InputOperand(0)); 1709 } 1710 __ Cvtqsi2ss(i.OutputDoubleRegister(), kScratchRegister); 1711 break; 1712 case kSSEFloat64ExtractLowWord32: 1713 if (instr->InputAt(0)->IsFPStackSlot()) { 1714 __ movl(i.OutputRegister(), i.InputOperand(0)); 1715 } else { 1716 __ Movd(i.OutputRegister(), i.InputDoubleRegister(0)); 1717 } 1718 break; 1719 case kSSEFloat64ExtractHighWord32: 1720 if (instr->InputAt(0)->IsFPStackSlot()) { 1721 __ movl(i.OutputRegister(), i.InputOperand(0, kDoubleSize / 2)); 1722 } else { 1723 __ Pextrd(i.OutputRegister(), i.InputDoubleRegister(0), 1); 1724 } 1725 break; 1726 case kSSEFloat64InsertLowWord32: 1727 if (instr->InputAt(1)->IsRegister()) { 1728 __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 0); 1729 } else { 1730 __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 0); 1731 } 1732 break; 1733 case kSSEFloat64InsertHighWord32: 1734 if (instr->InputAt(1)->IsRegister()) { 1735 __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 1); 1736 } else { 1737 __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 1); 1738 } 1739 break; 1740 case kSSEFloat64LoadLowWord32: 1741 if (instr->InputAt(0)->IsRegister()) { 1742 __ Movd(i.OutputDoubleRegister(), i.InputRegister(0)); 1743 } else { 1744 __ Movd(i.OutputDoubleRegister(), i.InputOperand(0)); 1745 } 1746 break; 1747 case kAVXFloat32Cmp: { 1748 CpuFeatureScope avx_scope(masm(), AVX); 1749 if (instr->InputAt(1)->IsFPRegister()) { 1750 __ vucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1751 } else { 1752 __ vucomiss(i.InputDoubleRegister(0), i.InputOperand(1)); 1753 } 1754 break; 1755 } 1756 case kAVXFloat32Add: 1757 ASSEMBLE_AVX_BINOP(vaddss); 1758 break; 1759 case kAVXFloat32Sub: 1760 ASSEMBLE_AVX_BINOP(vsubss); 1761 break; 1762 case kAVXFloat32Mul: 1763 ASSEMBLE_AVX_BINOP(vmulss); 1764 break; 1765 case kAVXFloat32Div: 1766 ASSEMBLE_AVX_BINOP(vdivss); 1767 // Don't delete this mov. It may improve performance on some CPUs, 1768 // when there is a (v)mulss depending on the result. 1769 __ Movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister()); 1770 break; 1771 case kAVXFloat64Cmp: { 1772 CpuFeatureScope avx_scope(masm(), AVX); 1773 if (instr->InputAt(1)->IsFPRegister()) { 1774 __ vucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); 1775 } else { 1776 __ vucomisd(i.InputDoubleRegister(0), i.InputOperand(1)); 1777 } 1778 break; 1779 } 1780 case kAVXFloat64Add: 1781 ASSEMBLE_AVX_BINOP(vaddsd); 1782 break; 1783 case kAVXFloat64Sub: 1784 ASSEMBLE_AVX_BINOP(vsubsd); 1785 break; 1786 case kAVXFloat64Mul: 1787 ASSEMBLE_AVX_BINOP(vmulsd); 1788 break; 1789 case kAVXFloat64Div: 1790 ASSEMBLE_AVX_BINOP(vdivsd); 1791 // Don't delete this mov. It may improve performance on some CPUs, 1792 // when there is a (v)mulsd depending on the result. 1793 __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister()); 1794 break; 1795 case kAVXFloat32Abs: { 1796 // TODO(bmeurer): Use RIP relative 128-bit constants. 1797 CpuFeatureScope avx_scope(masm(), AVX); 1798 __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); 1799 __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 33); 1800 if (instr->InputAt(0)->IsFPRegister()) { 1801 __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg, 1802 i.InputDoubleRegister(0)); 1803 } else { 1804 __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg, 1805 i.InputOperand(0)); 1806 } 1807 break; 1808 } 1809 case kAVXFloat32Neg: { 1810 // TODO(bmeurer): Use RIP relative 128-bit constants. 1811 CpuFeatureScope avx_scope(masm(), AVX); 1812 __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); 1813 __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 31); 1814 if (instr->InputAt(0)->IsFPRegister()) { 1815 __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg, 1816 i.InputDoubleRegister(0)); 1817 } else { 1818 __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg, 1819 i.InputOperand(0)); 1820 } 1821 break; 1822 } 1823 case kAVXFloat64Abs: { 1824 // TODO(bmeurer): Use RIP relative 128-bit constants. 1825 CpuFeatureScope avx_scope(masm(), AVX); 1826 __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); 1827 __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 1); 1828 if (instr->InputAt(0)->IsFPRegister()) { 1829 __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg, 1830 i.InputDoubleRegister(0)); 1831 } else { 1832 __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg, 1833 i.InputOperand(0)); 1834 } 1835 break; 1836 } 1837 case kAVXFloat64Neg: { 1838 // TODO(bmeurer): Use RIP relative 128-bit constants. 1839 CpuFeatureScope avx_scope(masm(), AVX); 1840 __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); 1841 __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 63); 1842 if (instr->InputAt(0)->IsFPRegister()) { 1843 __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg, 1844 i.InputDoubleRegister(0)); 1845 } else { 1846 __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg, 1847 i.InputOperand(0)); 1848 } 1849 break; 1850 } 1851 case kSSEFloat64SilenceNaN: 1852 __ Xorpd(kScratchDoubleReg, kScratchDoubleReg); 1853 __ Subsd(i.InputDoubleRegister(0), kScratchDoubleReg); 1854 break; 1855 case kX64Movsxbl: 1856 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i, 1857 __ pc_offset(), instr); 1858 ASSEMBLE_MOVX(movsxbl); 1859 __ AssertZeroExtended(i.OutputRegister()); 1860 break; 1861 case kX64Movzxbl: 1862 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i, 1863 __ pc_offset(), instr); 1864 ASSEMBLE_MOVX(movzxbl); 1865 __ AssertZeroExtended(i.OutputRegister()); 1866 break; 1867 case kX64Movsxbq: 1868 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i, 1869 __ pc_offset(), instr); 1870 ASSEMBLE_MOVX(movsxbq); 1871 break; 1872 case kX64Movzxbq: 1873 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i, 1874 __ pc_offset(), instr); 1875 ASSEMBLE_MOVX(movzxbq); 1876 __ AssertZeroExtended(i.OutputRegister()); 1877 break; 1878 case kX64Movb: { 1879 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i, 1880 __ pc_offset(), instr); 1881 size_t index = 0; 1882 Operand operand = i.MemoryOperand(&index); 1883 if (HasImmediateInput(instr, index)) { 1884 __ movb(operand, Immediate(i.InputInt8(index))); 1885 } else { 1886 __ movb(operand, i.InputRegister(index)); 1887 } 1888 break; 1889 } 1890 case kX64Movsxwl: 1891 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i, 1892 __ pc_offset(), instr); 1893 ASSEMBLE_MOVX(movsxwl); 1894 __ AssertZeroExtended(i.OutputRegister()); 1895 break; 1896 case kX64Movzxwl: 1897 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i, 1898 __ pc_offset(), instr); 1899 ASSEMBLE_MOVX(movzxwl); 1900 __ AssertZeroExtended(i.OutputRegister()); 1901 break; 1902 case kX64Movsxwq: 1903 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i, 1904 __ pc_offset(), instr); 1905 ASSEMBLE_MOVX(movsxwq); 1906 break; 1907 case kX64Movzxwq: 1908 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i, 1909 __ pc_offset(), instr); 1910 ASSEMBLE_MOVX(movzxwq); 1911 __ AssertZeroExtended(i.OutputRegister()); 1912 break; 1913 case kX64Movw: { 1914 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i, 1915 __ pc_offset(), instr); 1916 size_t index = 0; 1917 Operand operand = i.MemoryOperand(&index); 1918 if (HasImmediateInput(instr, index)) { 1919 __ movw(operand, Immediate(i.InputInt16(index))); 1920 } else { 1921 __ movw(operand, i.InputRegister(index)); 1922 } 1923 break; 1924 } 1925 case kX64Movl: 1926 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i, 1927 __ pc_offset(), instr); 1928 if (instr->HasOutput()) { 1929 if (instr->addressing_mode() == kMode_None) { 1930 if (instr->InputAt(0)->IsRegister()) { 1931 __ movl(i.OutputRegister(), i.InputRegister(0)); 1932 } else { 1933 __ movl(i.OutputRegister(), i.InputOperand(0)); 1934 } 1935 } else { 1936 __ movl(i.OutputRegister(), i.MemoryOperand()); 1937 } 1938 __ AssertZeroExtended(i.OutputRegister()); 1939 } else { 1940 size_t index = 0; 1941 Operand operand = i.MemoryOperand(&index); 1942 if (HasImmediateInput(instr, index)) { 1943 __ movl(operand, i.InputImmediate(index)); 1944 } else { 1945 __ movl(operand, i.InputRegister(index)); 1946 } 1947 } 1948 break; 1949 case kX64Movsxlq: 1950 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i, 1951 __ pc_offset(), instr); 1952 ASSEMBLE_MOVX(movsxlq); 1953 break; 1954 case kX64Movq: 1955 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i, 1956 __ pc_offset(), instr); 1957 if (instr->HasOutput()) { 1958 __ movq(i.OutputRegister(), i.MemoryOperand()); 1959 } else { 1960 size_t index = 0; 1961 Operand operand = i.MemoryOperand(&index); 1962 if (HasImmediateInput(instr, index)) { 1963 __ movq(operand, i.InputImmediate(index)); 1964 } else { 1965 __ movq(operand, i.InputRegister(index)); 1966 } 1967 } 1968 break; 1969 case kX64Movss: 1970 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i, 1971 __ pc_offset(), instr); 1972 if (instr->HasOutput()) { 1973 __ movss(i.OutputDoubleRegister(), i.MemoryOperand()); 1974 } else { 1975 size_t index = 0; 1976 Operand operand = i.MemoryOperand(&index); 1977 __ movss(operand, i.InputDoubleRegister(index)); 1978 } 1979 break; 1980 case kX64Movsd: 1981 EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i, 1982 __ pc_offset(), instr); 1983 if (instr->HasOutput()) { 1984 __ Movsd(i.OutputDoubleRegister(), i.MemoryOperand()); 1985 } else { 1986 size_t index = 0; 1987 Operand operand = i.MemoryOperand(&index); 1988 __ Movsd(operand, i.InputDoubleRegister(index)); 1989 } 1990 break; 1991 case kX64BitcastFI: 1992 if (instr->InputAt(0)->IsFPStackSlot()) { 1993 __ movl(i.OutputRegister(), i.InputOperand(0)); 1994 } else { 1995 __ Movd(i.OutputRegister(), i.InputDoubleRegister(0)); 1996 } 1997 break; 1998 case kX64BitcastDL: 1999 if (instr->InputAt(0)->IsFPStackSlot()) { 2000 __ movq(i.OutputRegister(), i.InputOperand(0)); 2001 } else { 2002 __ Movq(i.OutputRegister(), i.InputDoubleRegister(0)); 2003 } 2004 break; 2005 case kX64BitcastIF: 2006 if (instr->InputAt(0)->IsRegister()) { 2007 __ Movd(i.OutputDoubleRegister(), i.InputRegister(0)); 2008 } else { 2009 __ movss(i.OutputDoubleRegister(), i.InputOperand(0)); 2010 } 2011 break; 2012 case kX64BitcastLD: 2013 if (instr->InputAt(0)->IsRegister()) { 2014 __ Movq(i.OutputDoubleRegister(), i.InputRegister(0)); 2015 } else { 2016 __ Movsd(i.OutputDoubleRegister(), i.InputOperand(0)); 2017 } 2018 break; 2019 case kX64Lea32: { 2020 AddressingMode mode = AddressingModeField::decode(instr->opcode()); 2021 // Shorten "leal" to "addl", "subl" or "shll" if the register allocation 2022 // and addressing mode just happens to work out. The "addl"/"subl" forms 2023 // in these cases are faster based on measurements. 2024 if (i.InputRegister(0).is(i.OutputRegister())) { 2025 if (mode == kMode_MRI) { 2026 int32_t constant_summand = i.InputInt32(1); 2027 if (constant_summand > 0) { 2028 __ addl(i.OutputRegister(), Immediate(constant_summand)); 2029 } else if (constant_summand < 0) { 2030 __ subl(i.OutputRegister(), Immediate(-constant_summand)); 2031 } 2032 } else if (mode == kMode_MR1) { 2033 if (i.InputRegister(1).is(i.OutputRegister())) { 2034 __ shll(i.OutputRegister(), Immediate(1)); 2035 } else { 2036 __ addl(i.OutputRegister(), i.InputRegister(1)); 2037 } 2038 } else if (mode == kMode_M2) { 2039 __ shll(i.OutputRegister(), Immediate(1)); 2040 } else if (mode == kMode_M4) { 2041 __ shll(i.OutputRegister(), Immediate(2)); 2042 } else if (mode == kMode_M8) { 2043 __ shll(i.OutputRegister(), Immediate(3)); 2044 } else { 2045 __ leal(i.OutputRegister(), i.MemoryOperand()); 2046 } 2047 } else if (mode == kMode_MR1 && 2048 i.InputRegister(1).is(i.OutputRegister())) { 2049 __ addl(i.OutputRegister(), i.InputRegister(0)); 2050 } else { 2051 __ leal(i.OutputRegister(), i.MemoryOperand()); 2052 } 2053 __ AssertZeroExtended(i.OutputRegister()); 2054 break; 2055 } 2056 case kX64Lea: { 2057 AddressingMode mode = AddressingModeField::decode(instr->opcode()); 2058 // Shorten "leaq" to "addq", "subq" or "shlq" if the register allocation 2059 // and addressing mode just happens to work out. The "addq"/"subq" forms 2060 // in these cases are faster based on measurements. 2061 if (i.InputRegister(0).is(i.OutputRegister())) { 2062 if (mode == kMode_MRI) { 2063 int32_t constant_summand = i.InputInt32(1); 2064 if (constant_summand > 0) { 2065 __ addq(i.OutputRegister(), Immediate(constant_summand)); 2066 } else if (constant_summand < 0) { 2067 __ subq(i.OutputRegister(), Immediate(-constant_summand)); 2068 } 2069 } else if (mode == kMode_MR1) { 2070 if (i.InputRegister(1).is(i.OutputRegister())) { 2071 __ shlq(i.OutputRegister(), Immediate(1)); 2072 } else { 2073 __ addq(i.OutputRegister(), i.InputRegister(1)); 2074 } 2075 } else if (mode == kMode_M2) { 2076 __ shlq(i.OutputRegister(), Immediate(1)); 2077 } else if (mode == kMode_M4) { 2078 __ shlq(i.OutputRegister(), Immediate(2)); 2079 } else if (mode == kMode_M8) { 2080 __ shlq(i.OutputRegister(), Immediate(3)); 2081 } else { 2082 __ leaq(i.OutputRegister(), i.MemoryOperand()); 2083 } 2084 } else if (mode == kMode_MR1 && 2085 i.InputRegister(1).is(i.OutputRegister())) { 2086 __ addq(i.OutputRegister(), i.InputRegister(0)); 2087 } else { 2088 __ leaq(i.OutputRegister(), i.MemoryOperand()); 2089 } 2090 break; 2091 } 2092 case kX64Dec32: 2093 __ decl(i.OutputRegister()); 2094 break; 2095 case kX64Inc32: 2096 __ incl(i.OutputRegister()); 2097 break; 2098 case kX64Push: 2099 if (AddressingModeField::decode(instr->opcode()) != kMode_None) { 2100 size_t index = 0; 2101 Operand operand = i.MemoryOperand(&index); 2102 __ pushq(operand); 2103 frame_access_state()->IncreaseSPDelta(1); 2104 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), 2105 kPointerSize); 2106 } else if (HasImmediateInput(instr, 0)) { 2107 __ pushq(i.InputImmediate(0)); 2108 frame_access_state()->IncreaseSPDelta(1); 2109 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), 2110 kPointerSize); 2111 } else if (instr->InputAt(0)->IsRegister()) { 2112 __ pushq(i.InputRegister(0)); 2113 frame_access_state()->IncreaseSPDelta(1); 2114 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), 2115 kPointerSize); 2116 } else if (instr->InputAt(0)->IsFPRegister()) { 2117 // TODO(titzer): use another machine instruction? 2118 __ subq(rsp, Immediate(kDoubleSize)); 2119 frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize); 2120 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), 2121 kDoubleSize); 2122 __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0)); 2123 } else { 2124 __ pushq(i.InputOperand(0)); 2125 frame_access_state()->IncreaseSPDelta(1); 2126 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), 2127 kPointerSize); 2128 } 2129 break; 2130 case kX64Poke: { 2131 int const slot = MiscField::decode(instr->opcode()); 2132 if (HasImmediateInput(instr, 0)) { 2133 __ movq(Operand(rsp, slot * kPointerSize), i.InputImmediate(0)); 2134 } else { 2135 __ movq(Operand(rsp, slot * kPointerSize), i.InputRegister(0)); 2136 } 2137 break; 2138 } 2139 case kX64Xchgb: { 2140 size_t index = 0; 2141 Operand operand = i.MemoryOperand(&index); 2142 __ xchgb(i.InputRegister(index), operand); 2143 break; 2144 } 2145 case kX64Xchgw: { 2146 size_t index = 0; 2147 Operand operand = i.MemoryOperand(&index); 2148 __ xchgw(i.InputRegister(index), operand); 2149 break; 2150 } 2151 case kX64Xchgl: { 2152 size_t index = 0; 2153 Operand operand = i.MemoryOperand(&index); 2154 __ xchgl(i.InputRegister(index), operand); 2155 break; 2156 } 2157 case kX64Int32x4Create: { 2158 CpuFeatureScope sse_scope(masm(), SSE4_1); 2159 XMMRegister dst = i.OutputSimd128Register(); 2160 __ Movd(dst, i.InputRegister(0)); 2161 __ shufps(dst, dst, 0x0); 2162 break; 2163 } 2164 case kX64Int32x4ExtractLane: { 2165 CpuFeatureScope sse_scope(masm(), SSE4_1); 2166 __ Pextrd(i.OutputRegister(), i.InputSimd128Register(0), i.InputInt8(1)); 2167 break; 2168 } 2169 case kX64Int32x4ReplaceLane: { 2170 CpuFeatureScope sse_scope(masm(), SSE4_1); 2171 if (instr->InputAt(2)->IsRegister()) { 2172 __ Pinsrd(i.OutputSimd128Register(), i.InputRegister(2), 2173 i.InputInt8(1)); 2174 } else { 2175 __ Pinsrd(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1)); 2176 } 2177 break; 2178 } 2179 case kX64Int32x4Add: { 2180 CpuFeatureScope sse_scope(masm(), SSE4_1); 2181 __ paddd(i.OutputSimd128Register(), i.InputSimd128Register(1)); 2182 break; 2183 } 2184 case kX64Int32x4Sub: { 2185 CpuFeatureScope sse_scope(masm(), SSE4_1); 2186 __ psubd(i.OutputSimd128Register(), i.InputSimd128Register(1)); 2187 break; 2188 } 2189 case kCheckedLoadInt8: 2190 ASSEMBLE_CHECKED_LOAD_INTEGER(movsxbl); 2191 break; 2192 case kCheckedLoadUint8: 2193 ASSEMBLE_CHECKED_LOAD_INTEGER(movzxbl); 2194 break; 2195 case kCheckedLoadInt16: 2196 ASSEMBLE_CHECKED_LOAD_INTEGER(movsxwl); 2197 break; 2198 case kCheckedLoadUint16: 2199 ASSEMBLE_CHECKED_LOAD_INTEGER(movzxwl); 2200 break; 2201 case kCheckedLoadWord32: 2202 ASSEMBLE_CHECKED_LOAD_INTEGER(movl); 2203 break; 2204 case kCheckedLoadWord64: 2205 ASSEMBLE_CHECKED_LOAD_INTEGER(movq); 2206 break; 2207 case kCheckedLoadFloat32: 2208 ASSEMBLE_CHECKED_LOAD_FLOAT(Movss, OutOfLineLoadFloat32NaN); 2209 break; 2210 case kCheckedLoadFloat64: 2211 ASSEMBLE_CHECKED_LOAD_FLOAT(Movsd, OutOfLineLoadFloat64NaN); 2212 break; 2213 case kCheckedStoreWord8: 2214 ASSEMBLE_CHECKED_STORE_INTEGER(movb); 2215 break; 2216 case kCheckedStoreWord16: 2217 ASSEMBLE_CHECKED_STORE_INTEGER(movw); 2218 break; 2219 case kCheckedStoreWord32: 2220 ASSEMBLE_CHECKED_STORE_INTEGER(movl); 2221 break; 2222 case kCheckedStoreWord64: 2223 ASSEMBLE_CHECKED_STORE_INTEGER(movq); 2224 break; 2225 case kCheckedStoreFloat32: 2226 ASSEMBLE_CHECKED_STORE_FLOAT(Movss); 2227 break; 2228 case kCheckedStoreFloat64: 2229 ASSEMBLE_CHECKED_STORE_FLOAT(Movsd); 2230 break; 2231 case kX64StackCheck: 2232 __ CompareRoot(rsp, Heap::kStackLimitRootIndex); 2233 break; 2234 case kAtomicLoadInt8: 2235 case kAtomicLoadUint8: 2236 case kAtomicLoadInt16: 2237 case kAtomicLoadUint16: 2238 case kAtomicLoadWord32: 2239 case kAtomicStoreWord8: 2240 case kAtomicStoreWord16: 2241 case kAtomicStoreWord32: 2242 UNREACHABLE(); // Won't be generated by instruction selector. 2243 break; 2244 } 2245 return kSuccess; 2246} // NOLINT(readability/fn_size) 2247 2248namespace { 2249 2250Condition FlagsConditionToCondition(FlagsCondition condition) { 2251 switch (condition) { 2252 case kUnorderedEqual: 2253 case kEqual: 2254 return equal; 2255 case kUnorderedNotEqual: 2256 case kNotEqual: 2257 return not_equal; 2258 case kSignedLessThan: 2259 return less; 2260 case kSignedGreaterThanOrEqual: 2261 return greater_equal; 2262 case kSignedLessThanOrEqual: 2263 return less_equal; 2264 case kSignedGreaterThan: 2265 return greater; 2266 case kUnsignedLessThan: 2267 return below; 2268 case kUnsignedGreaterThanOrEqual: 2269 return above_equal; 2270 case kUnsignedLessThanOrEqual: 2271 return below_equal; 2272 case kUnsignedGreaterThan: 2273 return above; 2274 case kOverflow: 2275 return overflow; 2276 case kNotOverflow: 2277 return no_overflow; 2278 default: 2279 break; 2280 } 2281 UNREACHABLE(); 2282 return no_condition; 2283} 2284 2285} // namespace 2286 2287// Assembles branches after this instruction. 2288void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) { 2289 Label::Distance flabel_distance = 2290 branch->fallthru ? Label::kNear : Label::kFar; 2291 Label* tlabel = branch->true_label; 2292 Label* flabel = branch->false_label; 2293 if (branch->condition == kUnorderedEqual) { 2294 __ j(parity_even, flabel, flabel_distance); 2295 } else if (branch->condition == kUnorderedNotEqual) { 2296 __ j(parity_even, tlabel); 2297 } 2298 __ j(FlagsConditionToCondition(branch->condition), tlabel); 2299 2300 if (!branch->fallthru) __ jmp(flabel, flabel_distance); 2301} 2302 2303 2304void CodeGenerator::AssembleArchJump(RpoNumber target) { 2305 if (!IsNextInAssemblyOrder(target)) __ jmp(GetLabel(target)); 2306} 2307 2308void CodeGenerator::AssembleArchTrap(Instruction* instr, 2309 FlagsCondition condition) { 2310 class OutOfLineTrap final : public OutOfLineCode { 2311 public: 2312 OutOfLineTrap(CodeGenerator* gen, bool frame_elided, Instruction* instr) 2313 : OutOfLineCode(gen), 2314 frame_elided_(frame_elided), 2315 instr_(instr), 2316 gen_(gen) {} 2317 2318 void Generate() final { 2319 X64OperandConverter i(gen_, instr_); 2320 2321 Builtins::Name trap_id = 2322 static_cast<Builtins::Name>(i.InputInt32(instr_->InputCount() - 1)); 2323 bool old_has_frame = __ has_frame(); 2324 if (frame_elided_) { 2325 __ set_has_frame(true); 2326 __ EnterFrame(StackFrame::WASM_COMPILED); 2327 } 2328 GenerateCallToTrap(trap_id); 2329 if (frame_elided_) { 2330 __ set_has_frame(old_has_frame); 2331 } 2332 } 2333 2334 private: 2335 void GenerateCallToTrap(Builtins::Name trap_id) { 2336 if (trap_id == Builtins::builtin_count) { 2337 // We cannot test calls to the runtime in cctest/test-run-wasm. 2338 // Therefore we emit a call to C here instead of a call to the runtime. 2339 __ PrepareCallCFunction(0); 2340 __ CallCFunction( 2341 ExternalReference::wasm_call_trap_callback_for_testing(isolate()), 2342 0); 2343 __ LeaveFrame(StackFrame::WASM_COMPILED); 2344 __ Ret(); 2345 } else { 2346 gen_->AssembleSourcePosition(instr_); 2347 __ Call(handle(isolate()->builtins()->builtin(trap_id), isolate()), 2348 RelocInfo::CODE_TARGET); 2349 ReferenceMap* reference_map = 2350 new (gen_->zone()) ReferenceMap(gen_->zone()); 2351 gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0, 2352 Safepoint::kNoLazyDeopt); 2353 if (FLAG_debug_code) { 2354 __ ud2(); 2355 } 2356 } 2357 } 2358 2359 bool frame_elided_; 2360 Instruction* instr_; 2361 CodeGenerator* gen_; 2362 }; 2363 bool frame_elided = !frame_access_state()->has_frame(); 2364 auto ool = new (zone()) OutOfLineTrap(this, frame_elided, instr); 2365 Label* tlabel = ool->entry(); 2366 Label end; 2367 if (condition == kUnorderedEqual) { 2368 __ j(parity_even, &end); 2369 } else if (condition == kUnorderedNotEqual) { 2370 __ j(parity_even, tlabel); 2371 } 2372 __ j(FlagsConditionToCondition(condition), tlabel); 2373 __ bind(&end); 2374} 2375 2376// Assembles boolean materializations after this instruction. 2377void CodeGenerator::AssembleArchBoolean(Instruction* instr, 2378 FlagsCondition condition) { 2379 X64OperandConverter i(this, instr); 2380 Label done; 2381 2382 // Materialize a full 64-bit 1 or 0 value. The result register is always the 2383 // last output of the instruction. 2384 Label check; 2385 DCHECK_NE(0u, instr->OutputCount()); 2386 Register reg = i.OutputRegister(instr->OutputCount() - 1); 2387 if (condition == kUnorderedEqual) { 2388 __ j(parity_odd, &check, Label::kNear); 2389 __ movl(reg, Immediate(0)); 2390 __ jmp(&done, Label::kNear); 2391 } else if (condition == kUnorderedNotEqual) { 2392 __ j(parity_odd, &check, Label::kNear); 2393 __ movl(reg, Immediate(1)); 2394 __ jmp(&done, Label::kNear); 2395 } 2396 __ bind(&check); 2397 __ setcc(FlagsConditionToCondition(condition), reg); 2398 __ movzxbl(reg, reg); 2399 __ bind(&done); 2400} 2401 2402 2403void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) { 2404 X64OperandConverter i(this, instr); 2405 Register input = i.InputRegister(0); 2406 for (size_t index = 2; index < instr->InputCount(); index += 2) { 2407 __ cmpl(input, Immediate(i.InputInt32(index + 0))); 2408 __ j(equal, GetLabel(i.InputRpo(index + 1))); 2409 } 2410 AssembleArchJump(i.InputRpo(1)); 2411} 2412 2413 2414void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) { 2415 X64OperandConverter i(this, instr); 2416 Register input = i.InputRegister(0); 2417 int32_t const case_count = static_cast<int32_t>(instr->InputCount() - 2); 2418 Label** cases = zone()->NewArray<Label*>(case_count); 2419 for (int32_t index = 0; index < case_count; ++index) { 2420 cases[index] = GetLabel(i.InputRpo(index + 2)); 2421 } 2422 Label* const table = AddJumpTable(cases, case_count); 2423 __ cmpl(input, Immediate(case_count)); 2424 __ j(above_equal, GetLabel(i.InputRpo(1))); 2425 __ leaq(kScratchRegister, Operand(table)); 2426 __ jmp(Operand(kScratchRegister, input, times_8, 0)); 2427} 2428 2429CodeGenerator::CodeGenResult CodeGenerator::AssembleDeoptimizerCall( 2430 int deoptimization_id, SourcePosition pos) { 2431 DeoptimizeKind deoptimization_kind = GetDeoptimizationKind(deoptimization_id); 2432 DeoptimizeReason deoptimization_reason = 2433 GetDeoptimizationReason(deoptimization_id); 2434 Deoptimizer::BailoutType bailout_type = 2435 deoptimization_kind == DeoptimizeKind::kSoft ? Deoptimizer::SOFT 2436 : Deoptimizer::EAGER; 2437 Address deopt_entry = Deoptimizer::GetDeoptimizationEntry( 2438 isolate(), deoptimization_id, bailout_type); 2439 if (deopt_entry == nullptr) return kTooManyDeoptimizationBailouts; 2440 __ RecordDeoptReason(deoptimization_reason, pos, deoptimization_id); 2441 __ call(deopt_entry, RelocInfo::RUNTIME_ENTRY); 2442 return kSuccess; 2443} 2444 2445 2446namespace { 2447 2448static const int kQuadWordSize = 16; 2449 2450} // namespace 2451 2452void CodeGenerator::FinishFrame(Frame* frame) { 2453 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor(); 2454 2455 const RegList saves_fp = descriptor->CalleeSavedFPRegisters(); 2456 if (saves_fp != 0) { 2457 frame->AlignSavedCalleeRegisterSlots(); 2458 if (saves_fp != 0) { // Save callee-saved XMM registers. 2459 const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp); 2460 frame->AllocateSavedCalleeRegisterSlots(saves_fp_count * 2461 (kQuadWordSize / kPointerSize)); 2462 } 2463 } 2464 const RegList saves = descriptor->CalleeSavedRegisters(); 2465 if (saves != 0) { // Save callee-saved registers. 2466 int count = 0; 2467 for (int i = Register::kNumRegisters - 1; i >= 0; i--) { 2468 if (((1 << i) & saves)) { 2469 ++count; 2470 } 2471 } 2472 frame->AllocateSavedCalleeRegisterSlots(count); 2473 } 2474} 2475 2476void CodeGenerator::AssembleConstructFrame() { 2477 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor(); 2478 if (frame_access_state()->has_frame()) { 2479 int pc_base = __ pc_offset(); 2480 2481 if (descriptor->IsCFunctionCall()) { 2482 __ pushq(rbp); 2483 __ movq(rbp, rsp); 2484 } else if (descriptor->IsJSFunctionCall()) { 2485 __ Prologue(this->info()->GeneratePreagedPrologue()); 2486 if (descriptor->PushArgumentCount()) { 2487 __ pushq(kJavaScriptCallArgCountRegister); 2488 } 2489 } else { 2490 __ StubPrologue(info()->GetOutputStackFrameType()); 2491 } 2492 2493 if (!descriptor->IsJSFunctionCall() || !info()->GeneratePreagedPrologue()) { 2494 unwinding_info_writer_.MarkFrameConstructed(pc_base); 2495 } 2496 } 2497 int shrink_slots = 2498 frame()->GetTotalFrameSlotCount() - descriptor->CalculateFixedFrameSize(); 2499 2500 if (info()->is_osr()) { 2501 // TurboFan OSR-compiled functions cannot be entered directly. 2502 __ Abort(kShouldNotDirectlyEnterOsrFunction); 2503 2504 // Unoptimized code jumps directly to this entrypoint while the unoptimized 2505 // frame is still on the stack. Optimized code uses OSR values directly from 2506 // the unoptimized frame. Thus, all that needs to be done is to allocate the 2507 // remaining stack slots. 2508 if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --"); 2509 osr_pc_offset_ = __ pc_offset(); 2510 shrink_slots -= static_cast<int>(OsrHelper(info()).UnoptimizedFrameSlots()); 2511 } 2512 2513 const RegList saves_fp = descriptor->CalleeSavedFPRegisters(); 2514 if (shrink_slots > 0) { 2515 __ subq(rsp, Immediate(shrink_slots * kPointerSize)); 2516 } 2517 2518 if (saves_fp != 0) { // Save callee-saved XMM registers. 2519 const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp); 2520 const int stack_size = saves_fp_count * kQuadWordSize; 2521 // Adjust the stack pointer. 2522 __ subp(rsp, Immediate(stack_size)); 2523 // Store the registers on the stack. 2524 int slot_idx = 0; 2525 for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) { 2526 if (!((1 << i) & saves_fp)) continue; 2527 __ movdqu(Operand(rsp, kQuadWordSize * slot_idx), 2528 XMMRegister::from_code(i)); 2529 slot_idx++; 2530 } 2531 } 2532 2533 const RegList saves = descriptor->CalleeSavedRegisters(); 2534 if (saves != 0) { // Save callee-saved registers. 2535 for (int i = Register::kNumRegisters - 1; i >= 0; i--) { 2536 if (!((1 << i) & saves)) continue; 2537 __ pushq(Register::from_code(i)); 2538 } 2539 } 2540} 2541 2542void CodeGenerator::AssembleReturn(InstructionOperand* pop) { 2543 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor(); 2544 2545 // Restore registers. 2546 const RegList saves = descriptor->CalleeSavedRegisters(); 2547 if (saves != 0) { 2548 for (int i = 0; i < Register::kNumRegisters; i++) { 2549 if (!((1 << i) & saves)) continue; 2550 __ popq(Register::from_code(i)); 2551 } 2552 } 2553 const RegList saves_fp = descriptor->CalleeSavedFPRegisters(); 2554 if (saves_fp != 0) { 2555 const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp); 2556 const int stack_size = saves_fp_count * kQuadWordSize; 2557 // Load the registers from the stack. 2558 int slot_idx = 0; 2559 for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) { 2560 if (!((1 << i) & saves_fp)) continue; 2561 __ movdqu(XMMRegister::from_code(i), 2562 Operand(rsp, kQuadWordSize * slot_idx)); 2563 slot_idx++; 2564 } 2565 // Adjust the stack pointer. 2566 __ addp(rsp, Immediate(stack_size)); 2567 } 2568 2569 unwinding_info_writer_.MarkBlockWillExit(); 2570 2571 // Might need rcx for scratch if pop_size is too big or if there is a variable 2572 // pop count. 2573 DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & rcx.bit()); 2574 DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & rdx.bit()); 2575 size_t pop_size = descriptor->StackParameterCount() * kPointerSize; 2576 X64OperandConverter g(this, nullptr); 2577 if (descriptor->IsCFunctionCall()) { 2578 AssembleDeconstructFrame(); 2579 } else if (frame_access_state()->has_frame()) { 2580 if (pop->IsImmediate() && g.ToConstant(pop).ToInt32() == 0) { 2581 // Canonicalize JSFunction return sites for now. 2582 if (return_label_.is_bound()) { 2583 __ jmp(&return_label_); 2584 return; 2585 } else { 2586 __ bind(&return_label_); 2587 AssembleDeconstructFrame(); 2588 } 2589 } else { 2590 AssembleDeconstructFrame(); 2591 } 2592 } 2593 2594 if (pop->IsImmediate()) { 2595 DCHECK_EQ(Constant::kInt32, g.ToConstant(pop).type()); 2596 pop_size += g.ToConstant(pop).ToInt32() * kPointerSize; 2597 CHECK_LT(pop_size, static_cast<size_t>(std::numeric_limits<int>::max())); 2598 __ Ret(static_cast<int>(pop_size), rcx); 2599 } else { 2600 Register pop_reg = g.ToRegister(pop); 2601 Register scratch_reg = pop_reg.is(rcx) ? rdx : rcx; 2602 __ popq(scratch_reg); 2603 __ leaq(rsp, Operand(rsp, pop_reg, times_8, static_cast<int>(pop_size))); 2604 __ jmp(scratch_reg); 2605 } 2606} 2607 2608 2609void CodeGenerator::AssembleMove(InstructionOperand* source, 2610 InstructionOperand* destination) { 2611 X64OperandConverter g(this, nullptr); 2612 // Dispatch on the source and destination operand kinds. Not all 2613 // combinations are possible. 2614 if (source->IsRegister()) { 2615 DCHECK(destination->IsRegister() || destination->IsStackSlot()); 2616 Register src = g.ToRegister(source); 2617 if (destination->IsRegister()) { 2618 __ movq(g.ToRegister(destination), src); 2619 } else { 2620 __ movq(g.ToOperand(destination), src); 2621 } 2622 } else if (source->IsStackSlot()) { 2623 DCHECK(destination->IsRegister() || destination->IsStackSlot()); 2624 Operand src = g.ToOperand(source); 2625 if (destination->IsRegister()) { 2626 Register dst = g.ToRegister(destination); 2627 __ movq(dst, src); 2628 } else { 2629 // Spill on demand to use a temporary register for memory-to-memory 2630 // moves. 2631 Register tmp = kScratchRegister; 2632 Operand dst = g.ToOperand(destination); 2633 __ movq(tmp, src); 2634 __ movq(dst, tmp); 2635 } 2636 } else if (source->IsConstant()) { 2637 ConstantOperand* constant_source = ConstantOperand::cast(source); 2638 Constant src = g.ToConstant(constant_source); 2639 if (destination->IsRegister() || destination->IsStackSlot()) { 2640 Register dst = destination->IsRegister() ? g.ToRegister(destination) 2641 : kScratchRegister; 2642 switch (src.type()) { 2643 case Constant::kInt32: { 2644 if (RelocInfo::IsWasmPtrReference(src.rmode())) { 2645 __ movq(dst, src.ToInt64(), src.rmode()); 2646 } else { 2647 // TODO(dcarney): don't need scratch in this case. 2648 int32_t value = src.ToInt32(); 2649 if (value == 0) { 2650 __ xorl(dst, dst); 2651 } else { 2652 if (RelocInfo::IsWasmSizeReference(src.rmode())) { 2653 __ movl(dst, Immediate(value, src.rmode())); 2654 } else { 2655 __ movl(dst, Immediate(value)); 2656 } 2657 } 2658 } 2659 break; 2660 } 2661 case Constant::kInt64: 2662 if (RelocInfo::IsWasmPtrReference(src.rmode())) { 2663 __ movq(dst, src.ToInt64(), src.rmode()); 2664 } else { 2665 DCHECK(!RelocInfo::IsWasmSizeReference(src.rmode())); 2666 __ Set(dst, src.ToInt64()); 2667 } 2668 break; 2669 case Constant::kFloat32: 2670 __ Move(dst, 2671 isolate()->factory()->NewNumber(src.ToFloat32(), TENURED)); 2672 break; 2673 case Constant::kFloat64: 2674 __ Move(dst, 2675 isolate()->factory()->NewNumber(src.ToFloat64(), TENURED)); 2676 break; 2677 case Constant::kExternalReference: 2678 __ Move(dst, src.ToExternalReference()); 2679 break; 2680 case Constant::kHeapObject: { 2681 Handle<HeapObject> src_object = src.ToHeapObject(); 2682 Heap::RootListIndex index; 2683 if (IsMaterializableFromRoot(src_object, &index)) { 2684 __ LoadRoot(dst, index); 2685 } else { 2686 __ Move(dst, src_object); 2687 } 2688 break; 2689 } 2690 case Constant::kRpoNumber: 2691 UNREACHABLE(); // TODO(dcarney): load of labels on x64. 2692 break; 2693 } 2694 if (destination->IsStackSlot()) { 2695 __ movq(g.ToOperand(destination), kScratchRegister); 2696 } 2697 } else if (src.type() == Constant::kFloat32) { 2698 // TODO(turbofan): Can we do better here? 2699 uint32_t src_const = bit_cast<uint32_t>(src.ToFloat32()); 2700 if (destination->IsFPRegister()) { 2701 __ Move(g.ToDoubleRegister(destination), src_const); 2702 } else { 2703 DCHECK(destination->IsFPStackSlot()); 2704 Operand dst = g.ToOperand(destination); 2705 __ movl(dst, Immediate(src_const)); 2706 } 2707 } else { 2708 DCHECK_EQ(Constant::kFloat64, src.type()); 2709 uint64_t src_const = bit_cast<uint64_t>(src.ToFloat64()); 2710 if (destination->IsFPRegister()) { 2711 __ Move(g.ToDoubleRegister(destination), src_const); 2712 } else { 2713 DCHECK(destination->IsFPStackSlot()); 2714 __ movq(kScratchRegister, src_const); 2715 __ movq(g.ToOperand(destination), kScratchRegister); 2716 } 2717 } 2718 } else if (source->IsFPRegister()) { 2719 XMMRegister src = g.ToDoubleRegister(source); 2720 if (destination->IsFPRegister()) { 2721 XMMRegister dst = g.ToDoubleRegister(destination); 2722 __ Movapd(dst, src); 2723 } else { 2724 DCHECK(destination->IsFPStackSlot()); 2725 Operand dst = g.ToOperand(destination); 2726 MachineRepresentation rep = 2727 LocationOperand::cast(source)->representation(); 2728 if (rep != MachineRepresentation::kSimd128) { 2729 __ Movsd(dst, src); 2730 } else { 2731 __ Movups(dst, src); 2732 } 2733 } 2734 } else if (source->IsFPStackSlot()) { 2735 DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot()); 2736 Operand src = g.ToOperand(source); 2737 MachineRepresentation rep = LocationOperand::cast(source)->representation(); 2738 if (destination->IsFPRegister()) { 2739 XMMRegister dst = g.ToDoubleRegister(destination); 2740 if (rep != MachineRepresentation::kSimd128) { 2741 __ Movsd(dst, src); 2742 } else { 2743 __ Movups(dst, src); 2744 } 2745 } else { 2746 Operand dst = g.ToOperand(destination); 2747 if (rep != MachineRepresentation::kSimd128) { 2748 __ Movsd(kScratchDoubleReg, src); 2749 __ Movsd(dst, kScratchDoubleReg); 2750 } else { 2751 __ Movups(kScratchDoubleReg, src); 2752 __ Movups(dst, kScratchDoubleReg); 2753 } 2754 } 2755 } else { 2756 UNREACHABLE(); 2757 } 2758} 2759 2760 2761void CodeGenerator::AssembleSwap(InstructionOperand* source, 2762 InstructionOperand* destination) { 2763 X64OperandConverter g(this, nullptr); 2764 // Dispatch on the source and destination operand kinds. Not all 2765 // combinations are possible. 2766 if (source->IsRegister() && destination->IsRegister()) { 2767 // Register-register. 2768 Register src = g.ToRegister(source); 2769 Register dst = g.ToRegister(destination); 2770 __ movq(kScratchRegister, src); 2771 __ movq(src, dst); 2772 __ movq(dst, kScratchRegister); 2773 } else if (source->IsRegister() && destination->IsStackSlot()) { 2774 Register src = g.ToRegister(source); 2775 __ pushq(src); 2776 frame_access_state()->IncreaseSPDelta(1); 2777 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), 2778 kPointerSize); 2779 Operand dst = g.ToOperand(destination); 2780 __ movq(src, dst); 2781 frame_access_state()->IncreaseSPDelta(-1); 2782 dst = g.ToOperand(destination); 2783 __ popq(dst); 2784 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), 2785 -kPointerSize); 2786 } else if ((source->IsStackSlot() && destination->IsStackSlot()) || 2787 (source->IsFPStackSlot() && destination->IsFPStackSlot())) { 2788 // Memory-memory. 2789 Operand src = g.ToOperand(source); 2790 Operand dst = g.ToOperand(destination); 2791 MachineRepresentation rep = LocationOperand::cast(source)->representation(); 2792 if (rep != MachineRepresentation::kSimd128) { 2793 Register tmp = kScratchRegister; 2794 __ movq(tmp, dst); 2795 __ pushq(src); 2796 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), 2797 kPointerSize); 2798 frame_access_state()->IncreaseSPDelta(1); 2799 src = g.ToOperand(source); 2800 __ movq(src, tmp); 2801 frame_access_state()->IncreaseSPDelta(-1); 2802 dst = g.ToOperand(destination); 2803 __ popq(dst); 2804 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), 2805 -kPointerSize); 2806 } else { 2807 // Use the XOR trick to swap without a temporary. 2808 __ Movups(kScratchDoubleReg, src); 2809 __ Xorps(kScratchDoubleReg, dst); // scratch contains src ^ dst. 2810 __ Movups(src, kScratchDoubleReg); 2811 __ Xorps(kScratchDoubleReg, dst); // scratch contains src. 2812 __ Movups(dst, kScratchDoubleReg); 2813 __ Xorps(kScratchDoubleReg, src); // scratch contains dst. 2814 __ Movups(src, kScratchDoubleReg); 2815 } 2816 } else if (source->IsFPRegister() && destination->IsFPRegister()) { 2817 // XMM register-register swap. 2818 XMMRegister src = g.ToDoubleRegister(source); 2819 XMMRegister dst = g.ToDoubleRegister(destination); 2820 __ Movapd(kScratchDoubleReg, src); 2821 __ Movapd(src, dst); 2822 __ Movapd(dst, kScratchDoubleReg); 2823 } else if (source->IsFPRegister() && destination->IsFPStackSlot()) { 2824 // XMM register-memory swap. 2825 XMMRegister src = g.ToDoubleRegister(source); 2826 Operand dst = g.ToOperand(destination); 2827 MachineRepresentation rep = LocationOperand::cast(source)->representation(); 2828 if (rep != MachineRepresentation::kSimd128) { 2829 __ Movsd(kScratchDoubleReg, src); 2830 __ Movsd(src, dst); 2831 __ Movsd(dst, kScratchDoubleReg); 2832 } else { 2833 __ Movups(kScratchDoubleReg, src); 2834 __ Movups(src, dst); 2835 __ Movups(dst, kScratchDoubleReg); 2836 } 2837 } else { 2838 // No other combinations are possible. 2839 UNREACHABLE(); 2840 } 2841} 2842 2843 2844void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) { 2845 for (size_t index = 0; index < target_count; ++index) { 2846 __ dq(targets[index]); 2847 } 2848} 2849 2850 2851void CodeGenerator::EnsureSpaceForLazyDeopt() { 2852 if (!info()->ShouldEnsureSpaceForLazyDeopt()) { 2853 return; 2854 } 2855 2856 int space_needed = Deoptimizer::patch_size(); 2857 // Ensure that we have enough space after the previous lazy-bailout 2858 // instruction for patching the code here. 2859 int current_pc = __ pc_offset(); 2860 if (current_pc < last_lazy_deopt_pc_ + space_needed) { 2861 int padding_size = last_lazy_deopt_pc_ + space_needed - current_pc; 2862 __ Nop(padding_size); 2863 } 2864} 2865 2866#undef __ 2867 2868} // namespace compiler 2869} // namespace internal 2870} // namespace v8 2871