assembler_x86_64.cc revision 3fd0e6a86430bf060c7eb391c1378394c4a2c574
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "assembler_x86_64.h" 18 19#include "base/casts.h" 20#include "entrypoints/quick/quick_entrypoints.h" 21#include "memory_region.h" 22#include "thread.h" 23 24namespace art { 25namespace x86_64 { 26 27std::ostream& operator<<(std::ostream& os, const CpuRegister& reg) { 28 return os << reg.AsRegister(); 29} 30 31std::ostream& operator<<(std::ostream& os, const XmmRegister& reg) { 32 return os << reg.AsFloatRegister(); 33} 34 35std::ostream& operator<<(std::ostream& os, const X87Register& reg) { 36 return os << "ST" << static_cast<int>(reg); 37} 38 39void X86_64Assembler::call(CpuRegister reg) { 40 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 41 EmitOptionalRex32(reg); 42 EmitUint8(0xFF); 43 EmitRegisterOperand(2, reg.LowBits()); 44} 45 46 47void X86_64Assembler::call(const Address& address) { 48 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 49 EmitOptionalRex32(address); 50 EmitUint8(0xFF); 51 EmitOperand(2, address); 52} 53 54 55void X86_64Assembler::call(Label* label) { 56 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 57 EmitUint8(0xE8); 58 static const int kSize = 5; 59 // Offset by one because we already have emitted the opcode. 60 EmitLabel(label, kSize - 1); 61} 62 63void X86_64Assembler::pushq(CpuRegister reg) { 64 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 65 EmitOptionalRex32(reg); 66 EmitUint8(0x50 + reg.LowBits()); 67} 68 69 70void X86_64Assembler::pushq(const Address& address) { 71 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 72 EmitOptionalRex32(address); 73 EmitUint8(0xFF); 74 EmitOperand(6, address); 75} 76 77 78void X86_64Assembler::pushq(const Immediate& imm) { 79 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 80 CHECK(imm.is_int32()); // pushq only supports 32b immediate. 81 if (imm.is_int8()) { 82 EmitUint8(0x6A); 83 EmitUint8(imm.value() & 0xFF); 84 } else { 85 EmitUint8(0x68); 86 EmitImmediate(imm); 87 } 88} 89 90 91void X86_64Assembler::popq(CpuRegister reg) { 92 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 93 EmitOptionalRex32(reg); 94 EmitUint8(0x58 + reg.LowBits()); 95} 96 97 98void X86_64Assembler::popq(const Address& address) { 99 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 100 EmitOptionalRex32(address); 101 EmitUint8(0x8F); 102 EmitOperand(0, address); 103} 104 105 106void X86_64Assembler::movq(CpuRegister dst, const Immediate& imm) { 107 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 108 if (imm.is_int32()) { 109 // 32 bit. Note: sign-extends. 110 EmitRex64(dst); 111 EmitUint8(0xC7); 112 EmitRegisterOperand(0, dst.LowBits()); 113 EmitInt32(static_cast<int32_t>(imm.value())); 114 } else { 115 EmitRex64(dst); 116 EmitUint8(0xB8 + dst.LowBits()); 117 EmitInt64(imm.value()); 118 } 119} 120 121 122void X86_64Assembler::movl(CpuRegister dst, const Immediate& imm) { 123 CHECK(imm.is_int32()); 124 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 125 EmitOptionalRex32(dst); 126 EmitUint8(0xB8 + dst.LowBits()); 127 EmitImmediate(imm); 128} 129 130 131void X86_64Assembler::movq(const Address& dst, const Immediate& imm) { 132 CHECK(imm.is_int32()); 133 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 134 EmitRex64(dst); 135 EmitUint8(0xC7); 136 EmitOperand(0, dst); 137 EmitImmediate(imm); 138} 139 140 141void X86_64Assembler::movq(CpuRegister dst, CpuRegister src) { 142 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 143 // 0x89 is movq r/m64 <- r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 144 EmitRex64(src, dst); 145 EmitUint8(0x89); 146 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 147} 148 149 150void X86_64Assembler::movl(CpuRegister dst, CpuRegister src) { 151 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 152 EmitOptionalRex32(dst, src); 153 EmitUint8(0x8B); 154 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 155} 156 157 158void X86_64Assembler::movq(CpuRegister dst, const Address& src) { 159 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 160 EmitRex64(dst, src); 161 EmitUint8(0x8B); 162 EmitOperand(dst.LowBits(), src); 163} 164 165 166void X86_64Assembler::movl(CpuRegister dst, const Address& src) { 167 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 168 EmitOptionalRex32(dst, src); 169 EmitUint8(0x8B); 170 EmitOperand(dst.LowBits(), src); 171} 172 173 174void X86_64Assembler::movq(const Address& dst, CpuRegister src) { 175 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 176 EmitRex64(src, dst); 177 EmitUint8(0x89); 178 EmitOperand(src.LowBits(), dst); 179} 180 181 182void X86_64Assembler::movl(const Address& dst, CpuRegister src) { 183 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 184 EmitOptionalRex32(src, dst); 185 EmitUint8(0x89); 186 EmitOperand(src.LowBits(), dst); 187} 188 189void X86_64Assembler::movl(const Address& dst, const Immediate& imm) { 190 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 191 EmitOptionalRex32(dst); 192 EmitUint8(0xC7); 193 EmitOperand(0, dst); 194 EmitImmediate(imm); 195} 196 197void X86_64Assembler::movntl(const Address& dst, CpuRegister src) { 198 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 199 EmitOptionalRex32(src, dst); 200 EmitUint8(0x0F); 201 EmitUint8(0xC3); 202 EmitOperand(src.LowBits(), dst); 203} 204 205void X86_64Assembler::movntq(const Address& dst, CpuRegister src) { 206 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 207 EmitRex64(src, dst); 208 EmitUint8(0x0F); 209 EmitUint8(0xC3); 210 EmitOperand(src.LowBits(), dst); 211} 212 213void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src) { 214 cmov(c, dst, src, true); 215} 216 217void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src, bool is64bit) { 218 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 219 EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex()); 220 EmitUint8(0x0F); 221 EmitUint8(0x40 + c); 222 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 223} 224 225 226void X86_64Assembler::movzxb(CpuRegister dst, CpuRegister src) { 227 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 228 EmitOptionalByteRegNormalizingRex32(dst, src); 229 EmitUint8(0x0F); 230 EmitUint8(0xB6); 231 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 232} 233 234 235void X86_64Assembler::movzxb(CpuRegister dst, const Address& src) { 236 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 237 // Byte register is only in the source register form, so we don't use 238 // EmitOptionalByteRegNormalizingRex32(dst, src); 239 EmitOptionalRex32(dst, src); 240 EmitUint8(0x0F); 241 EmitUint8(0xB6); 242 EmitOperand(dst.LowBits(), src); 243} 244 245 246void X86_64Assembler::movsxb(CpuRegister dst, CpuRegister src) { 247 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 248 EmitOptionalByteRegNormalizingRex32(dst, src); 249 EmitUint8(0x0F); 250 EmitUint8(0xBE); 251 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 252} 253 254 255void X86_64Assembler::movsxb(CpuRegister dst, const Address& src) { 256 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 257 // Byte register is only in the source register form, so we don't use 258 // EmitOptionalByteRegNormalizingRex32(dst, src); 259 EmitOptionalRex32(dst, src); 260 EmitUint8(0x0F); 261 EmitUint8(0xBE); 262 EmitOperand(dst.LowBits(), src); 263} 264 265 266void X86_64Assembler::movb(CpuRegister /*dst*/, const Address& /*src*/) { 267 LOG(FATAL) << "Use movzxb or movsxb instead."; 268} 269 270 271void X86_64Assembler::movb(const Address& dst, CpuRegister src) { 272 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 273 EmitOptionalByteRegNormalizingRex32(src, dst); 274 EmitUint8(0x88); 275 EmitOperand(src.LowBits(), dst); 276} 277 278 279void X86_64Assembler::movb(const Address& dst, const Immediate& imm) { 280 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 281 EmitOptionalRex32(dst); 282 EmitUint8(0xC6); 283 EmitOperand(Register::RAX, dst); 284 CHECK(imm.is_int8()); 285 EmitUint8(imm.value() & 0xFF); 286} 287 288 289void X86_64Assembler::movzxw(CpuRegister dst, CpuRegister src) { 290 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 291 EmitOptionalRex32(dst, src); 292 EmitUint8(0x0F); 293 EmitUint8(0xB7); 294 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 295} 296 297 298void X86_64Assembler::movzxw(CpuRegister dst, const Address& src) { 299 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 300 EmitOptionalRex32(dst, src); 301 EmitUint8(0x0F); 302 EmitUint8(0xB7); 303 EmitOperand(dst.LowBits(), src); 304} 305 306 307void X86_64Assembler::movsxw(CpuRegister dst, CpuRegister src) { 308 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 309 EmitOptionalRex32(dst, src); 310 EmitUint8(0x0F); 311 EmitUint8(0xBF); 312 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 313} 314 315 316void X86_64Assembler::movsxw(CpuRegister dst, const Address& src) { 317 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 318 EmitOptionalRex32(dst, src); 319 EmitUint8(0x0F); 320 EmitUint8(0xBF); 321 EmitOperand(dst.LowBits(), src); 322} 323 324 325void X86_64Assembler::movw(CpuRegister /*dst*/, const Address& /*src*/) { 326 LOG(FATAL) << "Use movzxw or movsxw instead."; 327} 328 329 330void X86_64Assembler::movw(const Address& dst, CpuRegister src) { 331 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 332 EmitOperandSizeOverride(); 333 EmitOptionalRex32(src, dst); 334 EmitUint8(0x89); 335 EmitOperand(src.LowBits(), dst); 336} 337 338 339void X86_64Assembler::movw(const Address& dst, const Immediate& imm) { 340 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 341 EmitOperandSizeOverride(); 342 EmitOptionalRex32(dst); 343 EmitUint8(0xC7); 344 EmitOperand(Register::RAX, dst); 345 CHECK(imm.is_uint16() || imm.is_int16()); 346 EmitUint8(imm.value() & 0xFF); 347 EmitUint8(imm.value() >> 8); 348} 349 350 351void X86_64Assembler::leaq(CpuRegister dst, const Address& src) { 352 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 353 EmitRex64(dst, src); 354 EmitUint8(0x8D); 355 EmitOperand(dst.LowBits(), src); 356} 357 358 359void X86_64Assembler::leal(CpuRegister dst, const Address& src) { 360 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 361 EmitOptionalRex32(dst, src); 362 EmitUint8(0x8D); 363 EmitOperand(dst.LowBits(), src); 364} 365 366 367void X86_64Assembler::movaps(XmmRegister dst, XmmRegister src) { 368 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 369 EmitOptionalRex32(dst, src); 370 EmitUint8(0x0F); 371 EmitUint8(0x28); 372 EmitXmmRegisterOperand(dst.LowBits(), src); 373} 374 375 376void X86_64Assembler::movss(XmmRegister dst, const Address& src) { 377 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 378 EmitUint8(0xF3); 379 EmitOptionalRex32(dst, src); 380 EmitUint8(0x0F); 381 EmitUint8(0x10); 382 EmitOperand(dst.LowBits(), src); 383} 384 385 386void X86_64Assembler::movss(const Address& dst, XmmRegister src) { 387 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 388 EmitUint8(0xF3); 389 EmitOptionalRex32(src, dst); 390 EmitUint8(0x0F); 391 EmitUint8(0x11); 392 EmitOperand(src.LowBits(), dst); 393} 394 395 396void X86_64Assembler::movss(XmmRegister dst, XmmRegister src) { 397 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 398 EmitUint8(0xF3); 399 EmitOptionalRex32(src, dst); // Movss is MR encoding instead of the usual RM. 400 EmitUint8(0x0F); 401 EmitUint8(0x11); 402 EmitXmmRegisterOperand(src.LowBits(), dst); 403} 404 405 406void X86_64Assembler::movsxd(CpuRegister dst, CpuRegister src) { 407 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 408 EmitRex64(dst, src); 409 EmitUint8(0x63); 410 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 411} 412 413 414void X86_64Assembler::movsxd(CpuRegister dst, const Address& src) { 415 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 416 EmitRex64(dst, src); 417 EmitUint8(0x63); 418 EmitOperand(dst.LowBits(), src); 419} 420 421 422void X86_64Assembler::movd(XmmRegister dst, CpuRegister src) { 423 movd(dst, src, true); 424} 425 426void X86_64Assembler::movd(CpuRegister dst, XmmRegister src) { 427 movd(dst, src, true); 428} 429 430void X86_64Assembler::movd(XmmRegister dst, CpuRegister src, bool is64bit) { 431 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 432 EmitUint8(0x66); 433 EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex()); 434 EmitUint8(0x0F); 435 EmitUint8(0x6E); 436 EmitOperand(dst.LowBits(), Operand(src)); 437} 438 439void X86_64Assembler::movd(CpuRegister dst, XmmRegister src, bool is64bit) { 440 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 441 EmitUint8(0x66); 442 EmitOptionalRex(false, is64bit, src.NeedsRex(), false, dst.NeedsRex()); 443 EmitUint8(0x0F); 444 EmitUint8(0x7E); 445 EmitOperand(src.LowBits(), Operand(dst)); 446} 447 448 449void X86_64Assembler::addss(XmmRegister dst, XmmRegister src) { 450 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 451 EmitUint8(0xF3); 452 EmitOptionalRex32(dst, src); 453 EmitUint8(0x0F); 454 EmitUint8(0x58); 455 EmitXmmRegisterOperand(dst.LowBits(), src); 456} 457 458 459void X86_64Assembler::addss(XmmRegister dst, const Address& src) { 460 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 461 EmitUint8(0xF3); 462 EmitOptionalRex32(dst, src); 463 EmitUint8(0x0F); 464 EmitUint8(0x58); 465 EmitOperand(dst.LowBits(), src); 466} 467 468 469void X86_64Assembler::subss(XmmRegister dst, XmmRegister src) { 470 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 471 EmitUint8(0xF3); 472 EmitOptionalRex32(dst, src); 473 EmitUint8(0x0F); 474 EmitUint8(0x5C); 475 EmitXmmRegisterOperand(dst.LowBits(), src); 476} 477 478 479void X86_64Assembler::subss(XmmRegister dst, const Address& src) { 480 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 481 EmitUint8(0xF3); 482 EmitOptionalRex32(dst, src); 483 EmitUint8(0x0F); 484 EmitUint8(0x5C); 485 EmitOperand(dst.LowBits(), src); 486} 487 488 489void X86_64Assembler::mulss(XmmRegister dst, XmmRegister src) { 490 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 491 EmitUint8(0xF3); 492 EmitOptionalRex32(dst, src); 493 EmitUint8(0x0F); 494 EmitUint8(0x59); 495 EmitXmmRegisterOperand(dst.LowBits(), src); 496} 497 498 499void X86_64Assembler::mulss(XmmRegister dst, const Address& src) { 500 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 501 EmitUint8(0xF3); 502 EmitOptionalRex32(dst, src); 503 EmitUint8(0x0F); 504 EmitUint8(0x59); 505 EmitOperand(dst.LowBits(), src); 506} 507 508 509void X86_64Assembler::divss(XmmRegister dst, XmmRegister src) { 510 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 511 EmitUint8(0xF3); 512 EmitOptionalRex32(dst, src); 513 EmitUint8(0x0F); 514 EmitUint8(0x5E); 515 EmitXmmRegisterOperand(dst.LowBits(), src); 516} 517 518 519void X86_64Assembler::divss(XmmRegister dst, const Address& src) { 520 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 521 EmitUint8(0xF3); 522 EmitOptionalRex32(dst, src); 523 EmitUint8(0x0F); 524 EmitUint8(0x5E); 525 EmitOperand(dst.LowBits(), src); 526} 527 528 529void X86_64Assembler::flds(const Address& src) { 530 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 531 EmitUint8(0xD9); 532 EmitOperand(0, src); 533} 534 535 536void X86_64Assembler::fsts(const Address& dst) { 537 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 538 EmitUint8(0xD9); 539 EmitOperand(2, dst); 540} 541 542 543void X86_64Assembler::fstps(const Address& dst) { 544 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 545 EmitUint8(0xD9); 546 EmitOperand(3, dst); 547} 548 549 550void X86_64Assembler::movsd(XmmRegister dst, const Address& src) { 551 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 552 EmitUint8(0xF2); 553 EmitOptionalRex32(dst, src); 554 EmitUint8(0x0F); 555 EmitUint8(0x10); 556 EmitOperand(dst.LowBits(), src); 557} 558 559 560void X86_64Assembler::movsd(const Address& dst, XmmRegister src) { 561 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 562 EmitUint8(0xF2); 563 EmitOptionalRex32(src, dst); 564 EmitUint8(0x0F); 565 EmitUint8(0x11); 566 EmitOperand(src.LowBits(), dst); 567} 568 569 570void X86_64Assembler::movsd(XmmRegister dst, XmmRegister src) { 571 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 572 EmitUint8(0xF2); 573 EmitOptionalRex32(src, dst); // Movsd is MR encoding instead of the usual RM. 574 EmitUint8(0x0F); 575 EmitUint8(0x11); 576 EmitXmmRegisterOperand(src.LowBits(), dst); 577} 578 579 580void X86_64Assembler::addsd(XmmRegister dst, XmmRegister src) { 581 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 582 EmitUint8(0xF2); 583 EmitOptionalRex32(dst, src); 584 EmitUint8(0x0F); 585 EmitUint8(0x58); 586 EmitXmmRegisterOperand(dst.LowBits(), src); 587} 588 589 590void X86_64Assembler::addsd(XmmRegister dst, const Address& src) { 591 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 592 EmitUint8(0xF2); 593 EmitOptionalRex32(dst, src); 594 EmitUint8(0x0F); 595 EmitUint8(0x58); 596 EmitOperand(dst.LowBits(), src); 597} 598 599 600void X86_64Assembler::subsd(XmmRegister dst, XmmRegister src) { 601 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 602 EmitUint8(0xF2); 603 EmitOptionalRex32(dst, src); 604 EmitUint8(0x0F); 605 EmitUint8(0x5C); 606 EmitXmmRegisterOperand(dst.LowBits(), src); 607} 608 609 610void X86_64Assembler::subsd(XmmRegister dst, const Address& src) { 611 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 612 EmitUint8(0xF2); 613 EmitOptionalRex32(dst, src); 614 EmitUint8(0x0F); 615 EmitUint8(0x5C); 616 EmitOperand(dst.LowBits(), src); 617} 618 619 620void X86_64Assembler::mulsd(XmmRegister dst, XmmRegister src) { 621 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 622 EmitUint8(0xF2); 623 EmitOptionalRex32(dst, src); 624 EmitUint8(0x0F); 625 EmitUint8(0x59); 626 EmitXmmRegisterOperand(dst.LowBits(), src); 627} 628 629 630void X86_64Assembler::mulsd(XmmRegister dst, const Address& src) { 631 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 632 EmitUint8(0xF2); 633 EmitOptionalRex32(dst, src); 634 EmitUint8(0x0F); 635 EmitUint8(0x59); 636 EmitOperand(dst.LowBits(), src); 637} 638 639 640void X86_64Assembler::divsd(XmmRegister dst, XmmRegister src) { 641 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 642 EmitUint8(0xF2); 643 EmitOptionalRex32(dst, src); 644 EmitUint8(0x0F); 645 EmitUint8(0x5E); 646 EmitXmmRegisterOperand(dst.LowBits(), src); 647} 648 649 650void X86_64Assembler::divsd(XmmRegister dst, const Address& src) { 651 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 652 EmitUint8(0xF2); 653 EmitOptionalRex32(dst, src); 654 EmitUint8(0x0F); 655 EmitUint8(0x5E); 656 EmitOperand(dst.LowBits(), src); 657} 658 659 660void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) { 661 cvtsi2ss(dst, src, false); 662} 663 664 665void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src, bool is64bit) { 666 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 667 EmitUint8(0xF3); 668 if (is64bit) { 669 // Emit a REX.W prefix if the operand size is 64 bits. 670 EmitRex64(dst, src); 671 } else { 672 EmitOptionalRex32(dst, src); 673 } 674 EmitUint8(0x0F); 675 EmitUint8(0x2A); 676 EmitOperand(dst.LowBits(), Operand(src)); 677} 678 679 680void X86_64Assembler::cvtsi2ss(XmmRegister dst, const Address& src, bool is64bit) { 681 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 682 EmitUint8(0xF3); 683 if (is64bit) { 684 // Emit a REX.W prefix if the operand size is 64 bits. 685 EmitRex64(dst, src); 686 } else { 687 EmitOptionalRex32(dst, src); 688 } 689 EmitUint8(0x0F); 690 EmitUint8(0x2A); 691 EmitOperand(dst.LowBits(), src); 692} 693 694 695void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src) { 696 cvtsi2sd(dst, src, false); 697} 698 699 700void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src, bool is64bit) { 701 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 702 EmitUint8(0xF2); 703 if (is64bit) { 704 // Emit a REX.W prefix if the operand size is 64 bits. 705 EmitRex64(dst, src); 706 } else { 707 EmitOptionalRex32(dst, src); 708 } 709 EmitUint8(0x0F); 710 EmitUint8(0x2A); 711 EmitOperand(dst.LowBits(), Operand(src)); 712} 713 714 715void X86_64Assembler::cvtsi2sd(XmmRegister dst, const Address& src, bool is64bit) { 716 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 717 EmitUint8(0xF2); 718 if (is64bit) { 719 // Emit a REX.W prefix if the operand size is 64 bits. 720 EmitRex64(dst, src); 721 } else { 722 EmitOptionalRex32(dst, src); 723 } 724 EmitUint8(0x0F); 725 EmitUint8(0x2A); 726 EmitOperand(dst.LowBits(), src); 727} 728 729 730void X86_64Assembler::cvtss2si(CpuRegister dst, XmmRegister src) { 731 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 732 EmitUint8(0xF3); 733 EmitOptionalRex32(dst, src); 734 EmitUint8(0x0F); 735 EmitUint8(0x2D); 736 EmitXmmRegisterOperand(dst.LowBits(), src); 737} 738 739 740void X86_64Assembler::cvtss2sd(XmmRegister dst, XmmRegister src) { 741 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 742 EmitUint8(0xF3); 743 EmitOptionalRex32(dst, src); 744 EmitUint8(0x0F); 745 EmitUint8(0x5A); 746 EmitXmmRegisterOperand(dst.LowBits(), src); 747} 748 749 750void X86_64Assembler::cvtss2sd(XmmRegister dst, const Address& src) { 751 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 752 EmitUint8(0xF3); 753 EmitOptionalRex32(dst, src); 754 EmitUint8(0x0F); 755 EmitUint8(0x5A); 756 EmitOperand(dst.LowBits(), src); 757} 758 759 760void X86_64Assembler::cvtsd2si(CpuRegister dst, XmmRegister src) { 761 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 762 EmitUint8(0xF2); 763 EmitOptionalRex32(dst, src); 764 EmitUint8(0x0F); 765 EmitUint8(0x2D); 766 EmitXmmRegisterOperand(dst.LowBits(), src); 767} 768 769 770void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src) { 771 cvttss2si(dst, src, false); 772} 773 774 775void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src, bool is64bit) { 776 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 777 EmitUint8(0xF3); 778 if (is64bit) { 779 // Emit a REX.W prefix if the operand size is 64 bits. 780 EmitRex64(dst, src); 781 } else { 782 EmitOptionalRex32(dst, src); 783 } 784 EmitUint8(0x0F); 785 EmitUint8(0x2C); 786 EmitXmmRegisterOperand(dst.LowBits(), src); 787} 788 789 790void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src) { 791 cvttsd2si(dst, src, false); 792} 793 794 795void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src, bool is64bit) { 796 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 797 EmitUint8(0xF2); 798 if (is64bit) { 799 // Emit a REX.W prefix if the operand size is 64 bits. 800 EmitRex64(dst, src); 801 } else { 802 EmitOptionalRex32(dst, src); 803 } 804 EmitUint8(0x0F); 805 EmitUint8(0x2C); 806 EmitXmmRegisterOperand(dst.LowBits(), src); 807} 808 809 810void X86_64Assembler::cvtsd2ss(XmmRegister dst, XmmRegister src) { 811 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 812 EmitUint8(0xF2); 813 EmitOptionalRex32(dst, src); 814 EmitUint8(0x0F); 815 EmitUint8(0x5A); 816 EmitXmmRegisterOperand(dst.LowBits(), src); 817} 818 819 820void X86_64Assembler::cvtsd2ss(XmmRegister dst, const Address& src) { 821 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 822 EmitUint8(0xF2); 823 EmitOptionalRex32(dst, src); 824 EmitUint8(0x0F); 825 EmitUint8(0x5A); 826 EmitOperand(dst.LowBits(), src); 827} 828 829 830void X86_64Assembler::cvtdq2pd(XmmRegister dst, XmmRegister src) { 831 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 832 EmitUint8(0xF3); 833 EmitOptionalRex32(dst, src); 834 EmitUint8(0x0F); 835 EmitUint8(0xE6); 836 EmitXmmRegisterOperand(dst.LowBits(), src); 837} 838 839 840void X86_64Assembler::comiss(XmmRegister a, XmmRegister b) { 841 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 842 EmitOptionalRex32(a, b); 843 EmitUint8(0x0F); 844 EmitUint8(0x2F); 845 EmitXmmRegisterOperand(a.LowBits(), b); 846} 847 848 849void X86_64Assembler::comiss(XmmRegister a, const Address& b) { 850 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 851 EmitOptionalRex32(a, b); 852 EmitUint8(0x0F); 853 EmitUint8(0x2F); 854 EmitOperand(a.LowBits(), b); 855} 856 857 858void X86_64Assembler::comisd(XmmRegister a, XmmRegister b) { 859 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 860 EmitUint8(0x66); 861 EmitOptionalRex32(a, b); 862 EmitUint8(0x0F); 863 EmitUint8(0x2F); 864 EmitXmmRegisterOperand(a.LowBits(), b); 865} 866 867 868void X86_64Assembler::comisd(XmmRegister a, const Address& b) { 869 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 870 EmitUint8(0x66); 871 EmitOptionalRex32(a, b); 872 EmitUint8(0x0F); 873 EmitUint8(0x2F); 874 EmitOperand(a.LowBits(), b); 875} 876 877 878void X86_64Assembler::ucomiss(XmmRegister a, XmmRegister b) { 879 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 880 EmitOptionalRex32(a, b); 881 EmitUint8(0x0F); 882 EmitUint8(0x2E); 883 EmitXmmRegisterOperand(a.LowBits(), b); 884} 885 886 887void X86_64Assembler::ucomiss(XmmRegister a, const Address& b) { 888 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 889 EmitOptionalRex32(a, b); 890 EmitUint8(0x0F); 891 EmitUint8(0x2E); 892 EmitOperand(a.LowBits(), b); 893} 894 895 896void X86_64Assembler::ucomisd(XmmRegister a, XmmRegister b) { 897 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 898 EmitUint8(0x66); 899 EmitOptionalRex32(a, b); 900 EmitUint8(0x0F); 901 EmitUint8(0x2E); 902 EmitXmmRegisterOperand(a.LowBits(), b); 903} 904 905 906void X86_64Assembler::ucomisd(XmmRegister a, const Address& b) { 907 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 908 EmitUint8(0x66); 909 EmitOptionalRex32(a, b); 910 EmitUint8(0x0F); 911 EmitUint8(0x2E); 912 EmitOperand(a.LowBits(), b); 913} 914 915 916void X86_64Assembler::roundsd(XmmRegister dst, XmmRegister src, const Immediate& imm) { 917 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 918 EmitUint8(0x66); 919 EmitOptionalRex32(dst, src); 920 EmitUint8(0x0F); 921 EmitUint8(0x3A); 922 EmitUint8(0x0B); 923 EmitXmmRegisterOperand(dst.LowBits(), src); 924 EmitUint8(imm.value()); 925} 926 927 928void X86_64Assembler::roundss(XmmRegister dst, XmmRegister src, const Immediate& imm) { 929 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 930 EmitUint8(0x66); 931 EmitOptionalRex32(dst, src); 932 EmitUint8(0x0F); 933 EmitUint8(0x3A); 934 EmitUint8(0x0A); 935 EmitXmmRegisterOperand(dst.LowBits(), src); 936 EmitUint8(imm.value()); 937} 938 939 940void X86_64Assembler::sqrtsd(XmmRegister dst, XmmRegister src) { 941 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 942 EmitUint8(0xF2); 943 EmitOptionalRex32(dst, src); 944 EmitUint8(0x0F); 945 EmitUint8(0x51); 946 EmitXmmRegisterOperand(dst.LowBits(), src); 947} 948 949 950void X86_64Assembler::sqrtss(XmmRegister dst, XmmRegister src) { 951 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 952 EmitUint8(0xF3); 953 EmitOptionalRex32(dst, src); 954 EmitUint8(0x0F); 955 EmitUint8(0x51); 956 EmitXmmRegisterOperand(dst.LowBits(), src); 957} 958 959 960void X86_64Assembler::xorpd(XmmRegister dst, const Address& src) { 961 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 962 EmitUint8(0x66); 963 EmitOptionalRex32(dst, src); 964 EmitUint8(0x0F); 965 EmitUint8(0x57); 966 EmitOperand(dst.LowBits(), src); 967} 968 969 970void X86_64Assembler::xorpd(XmmRegister dst, XmmRegister src) { 971 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 972 EmitUint8(0x66); 973 EmitOptionalRex32(dst, src); 974 EmitUint8(0x0F); 975 EmitUint8(0x57); 976 EmitXmmRegisterOperand(dst.LowBits(), src); 977} 978 979 980void X86_64Assembler::xorps(XmmRegister dst, const Address& src) { 981 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 982 EmitOptionalRex32(dst, src); 983 EmitUint8(0x0F); 984 EmitUint8(0x57); 985 EmitOperand(dst.LowBits(), src); 986} 987 988 989void X86_64Assembler::xorps(XmmRegister dst, XmmRegister src) { 990 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 991 EmitOptionalRex32(dst, src); 992 EmitUint8(0x0F); 993 EmitUint8(0x57); 994 EmitXmmRegisterOperand(dst.LowBits(), src); 995} 996 997 998void X86_64Assembler::andpd(XmmRegister dst, const Address& src) { 999 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1000 EmitUint8(0x66); 1001 EmitOptionalRex32(dst, src); 1002 EmitUint8(0x0F); 1003 EmitUint8(0x54); 1004 EmitOperand(dst.LowBits(), src); 1005} 1006 1007void X86_64Assembler::andpd(XmmRegister dst, XmmRegister src) { 1008 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1009 EmitUint8(0x66); 1010 EmitOptionalRex32(dst, src); 1011 EmitUint8(0x0F); 1012 EmitUint8(0x54); 1013 EmitXmmRegisterOperand(dst.LowBits(), src); 1014} 1015 1016void X86_64Assembler::andps(XmmRegister dst, XmmRegister src) { 1017 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1018 EmitOptionalRex32(dst, src); 1019 EmitUint8(0x0F); 1020 EmitUint8(0x54); 1021 EmitXmmRegisterOperand(dst.LowBits(), src); 1022} 1023 1024void X86_64Assembler::orpd(XmmRegister dst, XmmRegister src) { 1025 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1026 EmitUint8(0x66); 1027 EmitOptionalRex32(dst, src); 1028 EmitUint8(0x0F); 1029 EmitUint8(0x56); 1030 EmitXmmRegisterOperand(dst.LowBits(), src); 1031} 1032 1033void X86_64Assembler::orps(XmmRegister dst, XmmRegister src) { 1034 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1035 EmitOptionalRex32(dst, src); 1036 EmitUint8(0x0F); 1037 EmitUint8(0x56); 1038 EmitXmmRegisterOperand(dst.LowBits(), src); 1039} 1040 1041void X86_64Assembler::fldl(const Address& src) { 1042 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1043 EmitUint8(0xDD); 1044 EmitOperand(0, src); 1045} 1046 1047 1048void X86_64Assembler::fstl(const Address& dst) { 1049 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1050 EmitUint8(0xDD); 1051 EmitOperand(2, dst); 1052} 1053 1054 1055void X86_64Assembler::fstpl(const Address& dst) { 1056 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1057 EmitUint8(0xDD); 1058 EmitOperand(3, dst); 1059} 1060 1061 1062void X86_64Assembler::fstsw() { 1063 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1064 EmitUint8(0x9B); 1065 EmitUint8(0xDF); 1066 EmitUint8(0xE0); 1067} 1068 1069 1070void X86_64Assembler::fnstcw(const Address& dst) { 1071 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1072 EmitUint8(0xD9); 1073 EmitOperand(7, dst); 1074} 1075 1076 1077void X86_64Assembler::fldcw(const Address& src) { 1078 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1079 EmitUint8(0xD9); 1080 EmitOperand(5, src); 1081} 1082 1083 1084void X86_64Assembler::fistpl(const Address& dst) { 1085 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1086 EmitUint8(0xDF); 1087 EmitOperand(7, dst); 1088} 1089 1090 1091void X86_64Assembler::fistps(const Address& dst) { 1092 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1093 EmitUint8(0xDB); 1094 EmitOperand(3, dst); 1095} 1096 1097 1098void X86_64Assembler::fildl(const Address& src) { 1099 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1100 EmitUint8(0xDF); 1101 EmitOperand(5, src); 1102} 1103 1104 1105void X86_64Assembler::filds(const Address& src) { 1106 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1107 EmitUint8(0xDB); 1108 EmitOperand(0, src); 1109} 1110 1111 1112void X86_64Assembler::fincstp() { 1113 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1114 EmitUint8(0xD9); 1115 EmitUint8(0xF7); 1116} 1117 1118 1119void X86_64Assembler::ffree(const Immediate& index) { 1120 CHECK_LT(index.value(), 7); 1121 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1122 EmitUint8(0xDD); 1123 EmitUint8(0xC0 + index.value()); 1124} 1125 1126 1127void X86_64Assembler::fsin() { 1128 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1129 EmitUint8(0xD9); 1130 EmitUint8(0xFE); 1131} 1132 1133 1134void X86_64Assembler::fcos() { 1135 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1136 EmitUint8(0xD9); 1137 EmitUint8(0xFF); 1138} 1139 1140 1141void X86_64Assembler::fptan() { 1142 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1143 EmitUint8(0xD9); 1144 EmitUint8(0xF2); 1145} 1146 1147void X86_64Assembler::fucompp() { 1148 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1149 EmitUint8(0xDA); 1150 EmitUint8(0xE9); 1151} 1152 1153 1154void X86_64Assembler::fprem() { 1155 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1156 EmitUint8(0xD9); 1157 EmitUint8(0xF8); 1158} 1159 1160 1161void X86_64Assembler::xchgl(CpuRegister dst, CpuRegister src) { 1162 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1163 // There is a short version for rax. 1164 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't 1165 // work. 1166 const bool src_rax = src.AsRegister() == RAX; 1167 const bool dst_rax = dst.AsRegister() == RAX; 1168 if (src_rax || dst_rax) { 1169 EmitOptionalRex32(src_rax ? dst : src); 1170 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits())); 1171 return; 1172 } 1173 1174 // General case. 1175 EmitOptionalRex32(src, dst); 1176 EmitUint8(0x87); 1177 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1178} 1179 1180 1181void X86_64Assembler::xchgq(CpuRegister dst, CpuRegister src) { 1182 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1183 // There is a short version for rax. 1184 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't 1185 // work. 1186 const bool src_rax = src.AsRegister() == RAX; 1187 const bool dst_rax = dst.AsRegister() == RAX; 1188 if (src_rax || dst_rax) { 1189 // If src == target, emit a nop instead. 1190 if (src_rax && dst_rax) { 1191 EmitUint8(0x90); 1192 } else { 1193 EmitRex64(src_rax ? dst : src); 1194 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits())); 1195 } 1196 return; 1197 } 1198 1199 // General case. 1200 EmitRex64(src, dst); 1201 EmitUint8(0x87); 1202 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1203} 1204 1205 1206void X86_64Assembler::xchgl(CpuRegister reg, const Address& address) { 1207 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1208 EmitOptionalRex32(reg, address); 1209 EmitUint8(0x87); 1210 EmitOperand(reg.LowBits(), address); 1211} 1212 1213 1214void X86_64Assembler::cmpw(const Address& address, const Immediate& imm) { 1215 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1216 EmitOptionalRex32(address); 1217 EmitUint8(0x66); 1218 EmitComplex(7, address, imm); 1219} 1220 1221 1222void X86_64Assembler::cmpl(CpuRegister reg, const Immediate& imm) { 1223 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1224 EmitOptionalRex32(reg); 1225 EmitComplex(7, Operand(reg), imm); 1226} 1227 1228 1229void X86_64Assembler::cmpl(CpuRegister reg0, CpuRegister reg1) { 1230 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1231 EmitOptionalRex32(reg0, reg1); 1232 EmitUint8(0x3B); 1233 EmitOperand(reg0.LowBits(), Operand(reg1)); 1234} 1235 1236 1237void X86_64Assembler::cmpl(CpuRegister reg, const Address& address) { 1238 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1239 EmitOptionalRex32(reg, address); 1240 EmitUint8(0x3B); 1241 EmitOperand(reg.LowBits(), address); 1242} 1243 1244 1245void X86_64Assembler::cmpl(const Address& address, CpuRegister reg) { 1246 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1247 EmitOptionalRex32(reg, address); 1248 EmitUint8(0x39); 1249 EmitOperand(reg.LowBits(), address); 1250} 1251 1252 1253void X86_64Assembler::cmpl(const Address& address, const Immediate& imm) { 1254 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1255 EmitOptionalRex32(address); 1256 EmitComplex(7, address, imm); 1257} 1258 1259 1260void X86_64Assembler::cmpq(CpuRegister reg0, CpuRegister reg1) { 1261 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1262 EmitRex64(reg0, reg1); 1263 EmitUint8(0x3B); 1264 EmitOperand(reg0.LowBits(), Operand(reg1)); 1265} 1266 1267 1268void X86_64Assembler::cmpq(CpuRegister reg, const Immediate& imm) { 1269 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1270 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 1271 EmitRex64(reg); 1272 EmitComplex(7, Operand(reg), imm); 1273} 1274 1275 1276void X86_64Assembler::cmpq(CpuRegister reg, const Address& address) { 1277 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1278 EmitRex64(reg, address); 1279 EmitUint8(0x3B); 1280 EmitOperand(reg.LowBits(), address); 1281} 1282 1283 1284void X86_64Assembler::cmpq(const Address& address, const Immediate& imm) { 1285 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 1286 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1287 EmitRex64(address); 1288 EmitComplex(7, address, imm); 1289} 1290 1291 1292void X86_64Assembler::addl(CpuRegister dst, CpuRegister src) { 1293 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1294 EmitOptionalRex32(dst, src); 1295 EmitUint8(0x03); 1296 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1297} 1298 1299 1300void X86_64Assembler::addl(CpuRegister reg, const Address& address) { 1301 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1302 EmitOptionalRex32(reg, address); 1303 EmitUint8(0x03); 1304 EmitOperand(reg.LowBits(), address); 1305} 1306 1307 1308void X86_64Assembler::testl(CpuRegister reg1, CpuRegister reg2) { 1309 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1310 EmitOptionalRex32(reg1, reg2); 1311 EmitUint8(0x85); 1312 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 1313} 1314 1315 1316void X86_64Assembler::testl(CpuRegister reg, const Address& address) { 1317 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1318 EmitOptionalRex32(reg, address); 1319 EmitUint8(0x85); 1320 EmitOperand(reg.LowBits(), address); 1321} 1322 1323 1324void X86_64Assembler::testl(CpuRegister reg, const Immediate& immediate) { 1325 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1326 // For registers that have a byte variant (RAX, RBX, RCX, and RDX) 1327 // we only test the byte CpuRegister to keep the encoding short. 1328 if (immediate.is_uint8() && reg.AsRegister() < 4) { 1329 // Use zero-extended 8-bit immediate. 1330 if (reg.AsRegister() == RAX) { 1331 EmitUint8(0xA8); 1332 } else { 1333 EmitUint8(0xF6); 1334 EmitUint8(0xC0 + reg.AsRegister()); 1335 } 1336 EmitUint8(immediate.value() & 0xFF); 1337 } else if (reg.AsRegister() == RAX) { 1338 // Use short form if the destination is RAX. 1339 EmitUint8(0xA9); 1340 EmitImmediate(immediate); 1341 } else { 1342 EmitOptionalRex32(reg); 1343 EmitUint8(0xF7); 1344 EmitOperand(0, Operand(reg)); 1345 EmitImmediate(immediate); 1346 } 1347} 1348 1349 1350void X86_64Assembler::testq(CpuRegister reg1, CpuRegister reg2) { 1351 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1352 EmitRex64(reg1, reg2); 1353 EmitUint8(0x85); 1354 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 1355} 1356 1357 1358void X86_64Assembler::testq(CpuRegister reg, const Address& address) { 1359 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1360 EmitRex64(reg, address); 1361 EmitUint8(0x85); 1362 EmitOperand(reg.LowBits(), address); 1363} 1364 1365 1366void X86_64Assembler::andl(CpuRegister dst, CpuRegister src) { 1367 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1368 EmitOptionalRex32(dst, src); 1369 EmitUint8(0x23); 1370 EmitOperand(dst.LowBits(), Operand(src)); 1371} 1372 1373 1374void X86_64Assembler::andl(CpuRegister reg, const Address& address) { 1375 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1376 EmitOptionalRex32(reg, address); 1377 EmitUint8(0x23); 1378 EmitOperand(reg.LowBits(), address); 1379} 1380 1381 1382void X86_64Assembler::andl(CpuRegister dst, const Immediate& imm) { 1383 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1384 EmitOptionalRex32(dst); 1385 EmitComplex(4, Operand(dst), imm); 1386} 1387 1388 1389void X86_64Assembler::andq(CpuRegister reg, const Immediate& imm) { 1390 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1391 CHECK(imm.is_int32()); // andq only supports 32b immediate. 1392 EmitRex64(reg); 1393 EmitComplex(4, Operand(reg), imm); 1394} 1395 1396 1397void X86_64Assembler::andq(CpuRegister dst, CpuRegister src) { 1398 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1399 EmitRex64(dst, src); 1400 EmitUint8(0x23); 1401 EmitOperand(dst.LowBits(), Operand(src)); 1402} 1403 1404 1405void X86_64Assembler::andq(CpuRegister dst, const Address& src) { 1406 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1407 EmitRex64(dst, src); 1408 EmitUint8(0x23); 1409 EmitOperand(dst.LowBits(), src); 1410} 1411 1412 1413void X86_64Assembler::orl(CpuRegister dst, CpuRegister src) { 1414 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1415 EmitOptionalRex32(dst, src); 1416 EmitUint8(0x0B); 1417 EmitOperand(dst.LowBits(), Operand(src)); 1418} 1419 1420 1421void X86_64Assembler::orl(CpuRegister reg, const Address& address) { 1422 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1423 EmitOptionalRex32(reg, address); 1424 EmitUint8(0x0B); 1425 EmitOperand(reg.LowBits(), address); 1426} 1427 1428 1429void X86_64Assembler::orl(CpuRegister dst, const Immediate& imm) { 1430 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1431 EmitOptionalRex32(dst); 1432 EmitComplex(1, Operand(dst), imm); 1433} 1434 1435 1436void X86_64Assembler::orq(CpuRegister dst, const Immediate& imm) { 1437 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1438 CHECK(imm.is_int32()); // orq only supports 32b immediate. 1439 EmitRex64(dst); 1440 EmitComplex(1, Operand(dst), imm); 1441} 1442 1443 1444void X86_64Assembler::orq(CpuRegister dst, CpuRegister src) { 1445 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1446 EmitRex64(dst, src); 1447 EmitUint8(0x0B); 1448 EmitOperand(dst.LowBits(), Operand(src)); 1449} 1450 1451 1452void X86_64Assembler::orq(CpuRegister dst, const Address& src) { 1453 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1454 EmitRex64(dst, src); 1455 EmitUint8(0x0B); 1456 EmitOperand(dst.LowBits(), src); 1457} 1458 1459 1460void X86_64Assembler::xorl(CpuRegister dst, CpuRegister src) { 1461 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1462 EmitOptionalRex32(dst, src); 1463 EmitUint8(0x33); 1464 EmitOperand(dst.LowBits(), Operand(src)); 1465} 1466 1467 1468void X86_64Assembler::xorl(CpuRegister reg, const Address& address) { 1469 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1470 EmitOptionalRex32(reg, address); 1471 EmitUint8(0x33); 1472 EmitOperand(reg.LowBits(), address); 1473} 1474 1475 1476void X86_64Assembler::xorl(CpuRegister dst, const Immediate& imm) { 1477 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1478 EmitOptionalRex32(dst); 1479 EmitComplex(6, Operand(dst), imm); 1480} 1481 1482 1483void X86_64Assembler::xorq(CpuRegister dst, CpuRegister src) { 1484 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1485 EmitRex64(dst, src); 1486 EmitUint8(0x33); 1487 EmitOperand(dst.LowBits(), Operand(src)); 1488} 1489 1490 1491void X86_64Assembler::xorq(CpuRegister dst, const Immediate& imm) { 1492 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1493 CHECK(imm.is_int32()); // xorq only supports 32b immediate. 1494 EmitRex64(dst); 1495 EmitComplex(6, Operand(dst), imm); 1496} 1497 1498void X86_64Assembler::xorq(CpuRegister dst, const Address& src) { 1499 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1500 EmitRex64(dst, src); 1501 EmitUint8(0x33); 1502 EmitOperand(dst.LowBits(), src); 1503} 1504 1505 1506#if 0 1507void X86_64Assembler::rex(bool force, bool w, Register* r, Register* x, Register* b) { 1508 // REX.WRXB 1509 // W - 64-bit operand 1510 // R - MODRM.reg 1511 // X - SIB.index 1512 // B - MODRM.rm/SIB.base 1513 uint8_t rex = force ? 0x40 : 0; 1514 if (w) { 1515 rex |= 0x48; // REX.W000 1516 } 1517 if (r != nullptr && *r >= Register::R8 && *r < Register::kNumberOfCpuRegisters) { 1518 rex |= 0x44; // REX.0R00 1519 *r = static_cast<Register>(*r - 8); 1520 } 1521 if (x != nullptr && *x >= Register::R8 && *x < Register::kNumberOfCpuRegisters) { 1522 rex |= 0x42; // REX.00X0 1523 *x = static_cast<Register>(*x - 8); 1524 } 1525 if (b != nullptr && *b >= Register::R8 && *b < Register::kNumberOfCpuRegisters) { 1526 rex |= 0x41; // REX.000B 1527 *b = static_cast<Register>(*b - 8); 1528 } 1529 if (rex != 0) { 1530 EmitUint8(rex); 1531 } 1532} 1533 1534void X86_64Assembler::rex_reg_mem(bool force, bool w, Register* dst, const Address& mem) { 1535 // REX.WRXB 1536 // W - 64-bit operand 1537 // R - MODRM.reg 1538 // X - SIB.index 1539 // B - MODRM.rm/SIB.base 1540 uint8_t rex = mem->rex(); 1541 if (force) { 1542 rex |= 0x40; // REX.0000 1543 } 1544 if (w) { 1545 rex |= 0x48; // REX.W000 1546 } 1547 if (dst != nullptr && *dst >= Register::R8 && *dst < Register::kNumberOfCpuRegisters) { 1548 rex |= 0x44; // REX.0R00 1549 *dst = static_cast<Register>(*dst - 8); 1550 } 1551 if (rex != 0) { 1552 EmitUint8(rex); 1553 } 1554} 1555 1556void rex_mem_reg(bool force, bool w, Address* mem, Register* src); 1557#endif 1558 1559void X86_64Assembler::addl(CpuRegister reg, const Immediate& imm) { 1560 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1561 EmitOptionalRex32(reg); 1562 EmitComplex(0, Operand(reg), imm); 1563} 1564 1565 1566void X86_64Assembler::addq(CpuRegister reg, const Immediate& imm) { 1567 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1568 CHECK(imm.is_int32()); // addq only supports 32b immediate. 1569 EmitRex64(reg); 1570 EmitComplex(0, Operand(reg), imm); 1571} 1572 1573 1574void X86_64Assembler::addq(CpuRegister dst, const Address& address) { 1575 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1576 EmitRex64(dst, address); 1577 EmitUint8(0x03); 1578 EmitOperand(dst.LowBits(), address); 1579} 1580 1581 1582void X86_64Assembler::addq(CpuRegister dst, CpuRegister src) { 1583 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1584 // 0x01 is addq r/m64 <- r/m64 + r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 1585 EmitRex64(src, dst); 1586 EmitUint8(0x01); 1587 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1588} 1589 1590 1591void X86_64Assembler::addl(const Address& address, CpuRegister reg) { 1592 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1593 EmitOptionalRex32(reg, address); 1594 EmitUint8(0x01); 1595 EmitOperand(reg.LowBits(), address); 1596} 1597 1598 1599void X86_64Assembler::addl(const Address& address, const Immediate& imm) { 1600 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1601 EmitOptionalRex32(address); 1602 EmitComplex(0, address, imm); 1603} 1604 1605 1606void X86_64Assembler::subl(CpuRegister dst, CpuRegister src) { 1607 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1608 EmitOptionalRex32(dst, src); 1609 EmitUint8(0x2B); 1610 EmitOperand(dst.LowBits(), Operand(src)); 1611} 1612 1613 1614void X86_64Assembler::subl(CpuRegister reg, const Immediate& imm) { 1615 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1616 EmitOptionalRex32(reg); 1617 EmitComplex(5, Operand(reg), imm); 1618} 1619 1620 1621void X86_64Assembler::subq(CpuRegister reg, const Immediate& imm) { 1622 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1623 CHECK(imm.is_int32()); // subq only supports 32b immediate. 1624 EmitRex64(reg); 1625 EmitComplex(5, Operand(reg), imm); 1626} 1627 1628 1629void X86_64Assembler::subq(CpuRegister dst, CpuRegister src) { 1630 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1631 EmitRex64(dst, src); 1632 EmitUint8(0x2B); 1633 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1634} 1635 1636 1637void X86_64Assembler::subq(CpuRegister reg, const Address& address) { 1638 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1639 EmitRex64(reg, address); 1640 EmitUint8(0x2B); 1641 EmitOperand(reg.LowBits() & 7, address); 1642} 1643 1644 1645void X86_64Assembler::subl(CpuRegister reg, const Address& address) { 1646 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1647 EmitOptionalRex32(reg, address); 1648 EmitUint8(0x2B); 1649 EmitOperand(reg.LowBits(), address); 1650} 1651 1652 1653void X86_64Assembler::cdq() { 1654 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1655 EmitUint8(0x99); 1656} 1657 1658 1659void X86_64Assembler::cqo() { 1660 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1661 EmitRex64(); 1662 EmitUint8(0x99); 1663} 1664 1665 1666void X86_64Assembler::idivl(CpuRegister reg) { 1667 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1668 EmitOptionalRex32(reg); 1669 EmitUint8(0xF7); 1670 EmitUint8(0xF8 | reg.LowBits()); 1671} 1672 1673 1674void X86_64Assembler::idivq(CpuRegister reg) { 1675 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1676 EmitRex64(reg); 1677 EmitUint8(0xF7); 1678 EmitUint8(0xF8 | reg.LowBits()); 1679} 1680 1681 1682void X86_64Assembler::imull(CpuRegister dst, CpuRegister src) { 1683 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1684 EmitOptionalRex32(dst, src); 1685 EmitUint8(0x0F); 1686 EmitUint8(0xAF); 1687 EmitOperand(dst.LowBits(), Operand(src)); 1688} 1689 1690void X86_64Assembler::imull(CpuRegister dst, CpuRegister src, const Immediate& imm) { 1691 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1692 CHECK(imm.is_int32()); // imull only supports 32b immediate. 1693 1694 EmitOptionalRex32(dst, src); 1695 1696 // See whether imm can be represented as a sign-extended 8bit value. 1697 int32_t v32 = static_cast<int32_t>(imm.value()); 1698 if (IsInt<8>(v32)) { 1699 // Sign-extension works. 1700 EmitUint8(0x6B); 1701 EmitOperand(dst.LowBits(), Operand(src)); 1702 EmitUint8(static_cast<uint8_t>(v32 & 0xFF)); 1703 } else { 1704 // Not representable, use full immediate. 1705 EmitUint8(0x69); 1706 EmitOperand(dst.LowBits(), Operand(src)); 1707 EmitImmediate(imm); 1708 } 1709} 1710 1711 1712void X86_64Assembler::imull(CpuRegister reg, const Immediate& imm) { 1713 imull(reg, reg, imm); 1714} 1715 1716 1717void X86_64Assembler::imull(CpuRegister reg, const Address& address) { 1718 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1719 EmitOptionalRex32(reg, address); 1720 EmitUint8(0x0F); 1721 EmitUint8(0xAF); 1722 EmitOperand(reg.LowBits(), address); 1723} 1724 1725 1726void X86_64Assembler::imulq(CpuRegister dst, CpuRegister src) { 1727 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1728 EmitRex64(dst, src); 1729 EmitUint8(0x0F); 1730 EmitUint8(0xAF); 1731 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1732} 1733 1734 1735void X86_64Assembler::imulq(CpuRegister reg, const Immediate& imm) { 1736 imulq(reg, reg, imm); 1737} 1738 1739void X86_64Assembler::imulq(CpuRegister dst, CpuRegister reg, const Immediate& imm) { 1740 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1741 CHECK(imm.is_int32()); // imulq only supports 32b immediate. 1742 1743 EmitRex64(dst, reg); 1744 1745 // See whether imm can be represented as a sign-extended 8bit value. 1746 int64_t v64 = imm.value(); 1747 if (IsInt<8>(v64)) { 1748 // Sign-extension works. 1749 EmitUint8(0x6B); 1750 EmitOperand(dst.LowBits(), Operand(reg)); 1751 EmitUint8(static_cast<uint8_t>(v64 & 0xFF)); 1752 } else { 1753 // Not representable, use full immediate. 1754 EmitUint8(0x69); 1755 EmitOperand(dst.LowBits(), Operand(reg)); 1756 EmitImmediate(imm); 1757 } 1758} 1759 1760void X86_64Assembler::imulq(CpuRegister reg, const Address& address) { 1761 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1762 EmitRex64(reg, address); 1763 EmitUint8(0x0F); 1764 EmitUint8(0xAF); 1765 EmitOperand(reg.LowBits(), address); 1766} 1767 1768 1769void X86_64Assembler::imull(CpuRegister reg) { 1770 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1771 EmitOptionalRex32(reg); 1772 EmitUint8(0xF7); 1773 EmitOperand(5, Operand(reg)); 1774} 1775 1776 1777void X86_64Assembler::imulq(CpuRegister reg) { 1778 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1779 EmitRex64(reg); 1780 EmitUint8(0xF7); 1781 EmitOperand(5, Operand(reg)); 1782} 1783 1784 1785void X86_64Assembler::imull(const Address& address) { 1786 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1787 EmitOptionalRex32(address); 1788 EmitUint8(0xF7); 1789 EmitOperand(5, address); 1790} 1791 1792 1793void X86_64Assembler::mull(CpuRegister reg) { 1794 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1795 EmitOptionalRex32(reg); 1796 EmitUint8(0xF7); 1797 EmitOperand(4, Operand(reg)); 1798} 1799 1800 1801void X86_64Assembler::mull(const Address& address) { 1802 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1803 EmitOptionalRex32(address); 1804 EmitUint8(0xF7); 1805 EmitOperand(4, address); 1806} 1807 1808 1809void X86_64Assembler::shll(CpuRegister reg, const Immediate& imm) { 1810 EmitGenericShift(false, 4, reg, imm); 1811} 1812 1813 1814void X86_64Assembler::shlq(CpuRegister reg, const Immediate& imm) { 1815 EmitGenericShift(true, 4, reg, imm); 1816} 1817 1818 1819void X86_64Assembler::shll(CpuRegister operand, CpuRegister shifter) { 1820 EmitGenericShift(false, 4, operand, shifter); 1821} 1822 1823 1824void X86_64Assembler::shlq(CpuRegister operand, CpuRegister shifter) { 1825 EmitGenericShift(true, 4, operand, shifter); 1826} 1827 1828 1829void X86_64Assembler::shrl(CpuRegister reg, const Immediate& imm) { 1830 EmitGenericShift(false, 5, reg, imm); 1831} 1832 1833 1834void X86_64Assembler::shrq(CpuRegister reg, const Immediate& imm) { 1835 EmitGenericShift(true, 5, reg, imm); 1836} 1837 1838 1839void X86_64Assembler::shrl(CpuRegister operand, CpuRegister shifter) { 1840 EmitGenericShift(false, 5, operand, shifter); 1841} 1842 1843 1844void X86_64Assembler::shrq(CpuRegister operand, CpuRegister shifter) { 1845 EmitGenericShift(true, 5, operand, shifter); 1846} 1847 1848 1849void X86_64Assembler::sarl(CpuRegister reg, const Immediate& imm) { 1850 EmitGenericShift(false, 7, reg, imm); 1851} 1852 1853 1854void X86_64Assembler::sarl(CpuRegister operand, CpuRegister shifter) { 1855 EmitGenericShift(false, 7, operand, shifter); 1856} 1857 1858 1859void X86_64Assembler::sarq(CpuRegister reg, const Immediate& imm) { 1860 EmitGenericShift(true, 7, reg, imm); 1861} 1862 1863 1864void X86_64Assembler::sarq(CpuRegister operand, CpuRegister shifter) { 1865 EmitGenericShift(true, 7, operand, shifter); 1866} 1867 1868 1869void X86_64Assembler::negl(CpuRegister reg) { 1870 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1871 EmitOptionalRex32(reg); 1872 EmitUint8(0xF7); 1873 EmitOperand(3, Operand(reg)); 1874} 1875 1876 1877void X86_64Assembler::negq(CpuRegister reg) { 1878 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1879 EmitRex64(reg); 1880 EmitUint8(0xF7); 1881 EmitOperand(3, Operand(reg)); 1882} 1883 1884 1885void X86_64Assembler::notl(CpuRegister reg) { 1886 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1887 EmitOptionalRex32(reg); 1888 EmitUint8(0xF7); 1889 EmitUint8(0xD0 | reg.LowBits()); 1890} 1891 1892 1893void X86_64Assembler::notq(CpuRegister reg) { 1894 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1895 EmitRex64(reg); 1896 EmitUint8(0xF7); 1897 EmitOperand(2, Operand(reg)); 1898} 1899 1900 1901void X86_64Assembler::enter(const Immediate& imm) { 1902 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1903 EmitUint8(0xC8); 1904 CHECK(imm.is_uint16()) << imm.value(); 1905 EmitUint8(imm.value() & 0xFF); 1906 EmitUint8((imm.value() >> 8) & 0xFF); 1907 EmitUint8(0x00); 1908} 1909 1910 1911void X86_64Assembler::leave() { 1912 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1913 EmitUint8(0xC9); 1914} 1915 1916 1917void X86_64Assembler::ret() { 1918 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1919 EmitUint8(0xC3); 1920} 1921 1922 1923void X86_64Assembler::ret(const Immediate& imm) { 1924 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1925 EmitUint8(0xC2); 1926 CHECK(imm.is_uint16()); 1927 EmitUint8(imm.value() & 0xFF); 1928 EmitUint8((imm.value() >> 8) & 0xFF); 1929} 1930 1931 1932 1933void X86_64Assembler::nop() { 1934 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1935 EmitUint8(0x90); 1936} 1937 1938 1939void X86_64Assembler::int3() { 1940 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1941 EmitUint8(0xCC); 1942} 1943 1944 1945void X86_64Assembler::hlt() { 1946 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1947 EmitUint8(0xF4); 1948} 1949 1950 1951void X86_64Assembler::j(Condition condition, Label* label) { 1952 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1953 if (label->IsBound()) { 1954 static const int kShortSize = 2; 1955 static const int kLongSize = 6; 1956 int offset = label->Position() - buffer_.Size(); 1957 CHECK_LE(offset, 0); 1958 if (IsInt<8>(offset - kShortSize)) { 1959 EmitUint8(0x70 + condition); 1960 EmitUint8((offset - kShortSize) & 0xFF); 1961 } else { 1962 EmitUint8(0x0F); 1963 EmitUint8(0x80 + condition); 1964 EmitInt32(offset - kLongSize); 1965 } 1966 } else { 1967 EmitUint8(0x0F); 1968 EmitUint8(0x80 + condition); 1969 EmitLabelLink(label); 1970 } 1971} 1972 1973 1974void X86_64Assembler::jmp(CpuRegister reg) { 1975 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1976 EmitOptionalRex32(reg); 1977 EmitUint8(0xFF); 1978 EmitRegisterOperand(4, reg.LowBits()); 1979} 1980 1981void X86_64Assembler::jmp(const Address& address) { 1982 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1983 EmitOptionalRex32(address); 1984 EmitUint8(0xFF); 1985 EmitOperand(4, address); 1986} 1987 1988void X86_64Assembler::jmp(Label* label) { 1989 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1990 if (label->IsBound()) { 1991 static const int kShortSize = 2; 1992 static const int kLongSize = 5; 1993 int offset = label->Position() - buffer_.Size(); 1994 CHECK_LE(offset, 0); 1995 if (IsInt<8>(offset - kShortSize)) { 1996 EmitUint8(0xEB); 1997 EmitUint8((offset - kShortSize) & 0xFF); 1998 } else { 1999 EmitUint8(0xE9); 2000 EmitInt32(offset - kLongSize); 2001 } 2002 } else { 2003 EmitUint8(0xE9); 2004 EmitLabelLink(label); 2005 } 2006} 2007 2008 2009X86_64Assembler* X86_64Assembler::lock() { 2010 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2011 EmitUint8(0xF0); 2012 return this; 2013} 2014 2015 2016void X86_64Assembler::cmpxchgl(const Address& address, CpuRegister reg) { 2017 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2018 EmitOptionalRex32(reg, address); 2019 EmitUint8(0x0F); 2020 EmitUint8(0xB1); 2021 EmitOperand(reg.LowBits(), address); 2022} 2023 2024 2025void X86_64Assembler::cmpxchgq(const Address& address, CpuRegister reg) { 2026 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2027 EmitRex64(reg, address); 2028 EmitUint8(0x0F); 2029 EmitUint8(0xB1); 2030 EmitOperand(reg.LowBits(), address); 2031} 2032 2033 2034void X86_64Assembler::mfence() { 2035 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2036 EmitUint8(0x0F); 2037 EmitUint8(0xAE); 2038 EmitUint8(0xF0); 2039} 2040 2041 2042X86_64Assembler* X86_64Assembler::gs() { 2043 // TODO: gs is a prefix and not an instruction 2044 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2045 EmitUint8(0x65); 2046 return this; 2047} 2048 2049 2050void X86_64Assembler::AddImmediate(CpuRegister reg, const Immediate& imm) { 2051 int value = imm.value(); 2052 if (value != 0) { 2053 if (value > 0) { 2054 addl(reg, imm); 2055 } else { 2056 subl(reg, Immediate(value)); 2057 } 2058 } 2059} 2060 2061 2062void X86_64Assembler::setcc(Condition condition, CpuRegister dst) { 2063 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2064 // RSP, RBP, RDI, RSI need rex prefix (else the pattern encodes ah/bh/ch/dh). 2065 if (dst.NeedsRex() || dst.AsRegister() > 3) { 2066 EmitOptionalRex(true, false, false, false, dst.NeedsRex()); 2067 } 2068 EmitUint8(0x0F); 2069 EmitUint8(0x90 + condition); 2070 EmitUint8(0xC0 + dst.LowBits()); 2071} 2072 2073void X86_64Assembler::bswapl(CpuRegister dst) { 2074 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2075 EmitOptionalRex(false, false, false, false, dst.NeedsRex()); 2076 EmitUint8(0x0F); 2077 EmitUint8(0xC8 + dst.LowBits()); 2078} 2079 2080void X86_64Assembler::bswapq(CpuRegister dst) { 2081 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2082 EmitOptionalRex(false, true, false, false, dst.NeedsRex()); 2083 EmitUint8(0x0F); 2084 EmitUint8(0xC8 + dst.LowBits()); 2085} 2086 2087 2088void X86_64Assembler::repne_scasw() { 2089 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2090 EmitUint8(0x66); 2091 EmitUint8(0xF2); 2092 EmitUint8(0xAF); 2093} 2094 2095 2096void X86_64Assembler::repe_cmpsw() { 2097 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2098 EmitUint8(0x66); 2099 EmitUint8(0xF3); 2100 EmitUint8(0xA7); 2101} 2102 2103 2104void X86_64Assembler::repe_cmpsl() { 2105 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2106 EmitUint8(0xF3); 2107 EmitUint8(0xA7); 2108} 2109 2110 2111void X86_64Assembler::repe_cmpsq() { 2112 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2113 EmitUint8(0xF3); 2114 EmitRex64(); 2115 EmitUint8(0xA7); 2116} 2117 2118 2119void X86_64Assembler::LoadDoubleConstant(XmmRegister dst, double value) { 2120 // TODO: Need to have a code constants table. 2121 int64_t constant = bit_cast<int64_t, double>(value); 2122 pushq(Immediate(High32Bits(constant))); 2123 pushq(Immediate(Low32Bits(constant))); 2124 movsd(dst, Address(CpuRegister(RSP), 0)); 2125 addq(CpuRegister(RSP), Immediate(2 * sizeof(intptr_t))); 2126} 2127 2128 2129void X86_64Assembler::Align(int alignment, int offset) { 2130 CHECK(IsPowerOfTwo(alignment)); 2131 // Emit nop instruction until the real position is aligned. 2132 while (((offset + buffer_.GetPosition()) & (alignment-1)) != 0) { 2133 nop(); 2134 } 2135} 2136 2137 2138void X86_64Assembler::Bind(Label* label) { 2139 int bound = buffer_.Size(); 2140 CHECK(!label->IsBound()); // Labels can only be bound once. 2141 while (label->IsLinked()) { 2142 int position = label->LinkPosition(); 2143 int next = buffer_.Load<int32_t>(position); 2144 buffer_.Store<int32_t>(position, bound - (position + 4)); 2145 label->position_ = next; 2146 } 2147 label->BindTo(bound); 2148} 2149 2150 2151void X86_64Assembler::EmitOperand(uint8_t reg_or_opcode, const Operand& operand) { 2152 CHECK_GE(reg_or_opcode, 0); 2153 CHECK_LT(reg_or_opcode, 8); 2154 const int length = operand.length_; 2155 CHECK_GT(length, 0); 2156 // Emit the ModRM byte updated with the given reg value. 2157 CHECK_EQ(operand.encoding_[0] & 0x38, 0); 2158 EmitUint8(operand.encoding_[0] + (reg_or_opcode << 3)); 2159 // Emit the rest of the encoded operand. 2160 for (int i = 1; i < length; i++) { 2161 EmitUint8(operand.encoding_[i]); 2162 } 2163 AssemblerFixup* fixup = operand.GetFixup(); 2164 if (fixup != nullptr) { 2165 EmitFixup(fixup); 2166 } 2167} 2168 2169 2170void X86_64Assembler::EmitImmediate(const Immediate& imm) { 2171 if (imm.is_int32()) { 2172 EmitInt32(static_cast<int32_t>(imm.value())); 2173 } else { 2174 EmitInt64(imm.value()); 2175 } 2176} 2177 2178 2179void X86_64Assembler::EmitComplex(uint8_t reg_or_opcode, 2180 const Operand& operand, 2181 const Immediate& immediate) { 2182 CHECK_GE(reg_or_opcode, 0); 2183 CHECK_LT(reg_or_opcode, 8); 2184 if (immediate.is_int8()) { 2185 // Use sign-extended 8-bit immediate. 2186 EmitUint8(0x83); 2187 EmitOperand(reg_or_opcode, operand); 2188 EmitUint8(immediate.value() & 0xFF); 2189 } else if (operand.IsRegister(CpuRegister(RAX))) { 2190 // Use short form if the destination is eax. 2191 EmitUint8(0x05 + (reg_or_opcode << 3)); 2192 EmitImmediate(immediate); 2193 } else { 2194 EmitUint8(0x81); 2195 EmitOperand(reg_or_opcode, operand); 2196 EmitImmediate(immediate); 2197 } 2198} 2199 2200 2201void X86_64Assembler::EmitLabel(Label* label, int instruction_size) { 2202 if (label->IsBound()) { 2203 int offset = label->Position() - buffer_.Size(); 2204 CHECK_LE(offset, 0); 2205 EmitInt32(offset - instruction_size); 2206 } else { 2207 EmitLabelLink(label); 2208 } 2209} 2210 2211 2212void X86_64Assembler::EmitLabelLink(Label* label) { 2213 CHECK(!label->IsBound()); 2214 int position = buffer_.Size(); 2215 EmitInt32(label->position_); 2216 label->LinkTo(position); 2217} 2218 2219 2220void X86_64Assembler::EmitGenericShift(bool wide, 2221 int reg_or_opcode, 2222 CpuRegister reg, 2223 const Immediate& imm) { 2224 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2225 CHECK(imm.is_int8()); 2226 if (wide) { 2227 EmitRex64(reg); 2228 } else { 2229 EmitOptionalRex32(reg); 2230 } 2231 if (imm.value() == 1) { 2232 EmitUint8(0xD1); 2233 EmitOperand(reg_or_opcode, Operand(reg)); 2234 } else { 2235 EmitUint8(0xC1); 2236 EmitOperand(reg_or_opcode, Operand(reg)); 2237 EmitUint8(imm.value() & 0xFF); 2238 } 2239} 2240 2241 2242void X86_64Assembler::EmitGenericShift(bool wide, 2243 int reg_or_opcode, 2244 CpuRegister operand, 2245 CpuRegister shifter) { 2246 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2247 CHECK_EQ(shifter.AsRegister(), RCX); 2248 if (wide) { 2249 EmitRex64(operand); 2250 } else { 2251 EmitOptionalRex32(operand); 2252 } 2253 EmitUint8(0xD3); 2254 EmitOperand(reg_or_opcode, Operand(operand)); 2255} 2256 2257void X86_64Assembler::EmitOptionalRex(bool force, bool w, bool r, bool x, bool b) { 2258 // REX.WRXB 2259 // W - 64-bit operand 2260 // R - MODRM.reg 2261 // X - SIB.index 2262 // B - MODRM.rm/SIB.base 2263 uint8_t rex = force ? 0x40 : 0; 2264 if (w) { 2265 rex |= 0x48; // REX.W000 2266 } 2267 if (r) { 2268 rex |= 0x44; // REX.0R00 2269 } 2270 if (x) { 2271 rex |= 0x42; // REX.00X0 2272 } 2273 if (b) { 2274 rex |= 0x41; // REX.000B 2275 } 2276 if (rex != 0) { 2277 EmitUint8(rex); 2278 } 2279} 2280 2281void X86_64Assembler::EmitOptionalRex32(CpuRegister reg) { 2282 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 2283} 2284 2285void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, CpuRegister src) { 2286 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2287} 2288 2289void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, XmmRegister src) { 2290 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2291} 2292 2293void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, XmmRegister src) { 2294 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2295} 2296 2297void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, CpuRegister src) { 2298 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2299} 2300 2301void X86_64Assembler::EmitOptionalRex32(const Operand& operand) { 2302 uint8_t rex = operand.rex(); 2303 if (rex != 0) { 2304 EmitUint8(rex); 2305 } 2306} 2307 2308void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, const Operand& operand) { 2309 uint8_t rex = operand.rex(); 2310 if (dst.NeedsRex()) { 2311 rex |= 0x44; // REX.0R00 2312 } 2313 if (rex != 0) { 2314 EmitUint8(rex); 2315 } 2316} 2317 2318void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, const Operand& operand) { 2319 uint8_t rex = operand.rex(); 2320 if (dst.NeedsRex()) { 2321 rex |= 0x44; // REX.0R00 2322 } 2323 if (rex != 0) { 2324 EmitUint8(rex); 2325 } 2326} 2327 2328void X86_64Assembler::EmitRex64() { 2329 EmitOptionalRex(false, true, false, false, false); 2330} 2331 2332void X86_64Assembler::EmitRex64(CpuRegister reg) { 2333 EmitOptionalRex(false, true, false, false, reg.NeedsRex()); 2334} 2335 2336void X86_64Assembler::EmitRex64(const Operand& operand) { 2337 uint8_t rex = operand.rex(); 2338 rex |= 0x48; // REX.W000 2339 EmitUint8(rex); 2340} 2341 2342void X86_64Assembler::EmitRex64(CpuRegister dst, CpuRegister src) { 2343 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2344} 2345 2346void X86_64Assembler::EmitRex64(XmmRegister dst, CpuRegister src) { 2347 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2348} 2349 2350void X86_64Assembler::EmitRex64(CpuRegister dst, XmmRegister src) { 2351 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2352} 2353 2354void X86_64Assembler::EmitRex64(CpuRegister dst, const Operand& operand) { 2355 uint8_t rex = 0x48 | operand.rex(); // REX.W000 2356 if (dst.NeedsRex()) { 2357 rex |= 0x44; // REX.0R00 2358 } 2359 EmitUint8(rex); 2360} 2361 2362void X86_64Assembler::EmitRex64(XmmRegister dst, const Operand& operand) { 2363 uint8_t rex = 0x48 | operand.rex(); // REX.W000 2364 if (dst.NeedsRex()) { 2365 rex |= 0x44; // REX.0R00 2366 } 2367 EmitUint8(rex); 2368} 2369 2370void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, CpuRegister src) { 2371 // For src, SPL, BPL, SIL, DIL need the rex prefix. 2372 bool force = src.AsRegister() > 3; 2373 EmitOptionalRex(force, false, dst.NeedsRex(), false, src.NeedsRex()); 2374} 2375 2376void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand) { 2377 uint8_t rex = operand.rex(); 2378 // For dst, SPL, BPL, SIL, DIL need the rex prefix. 2379 bool force = dst.AsRegister() > 3; 2380 if (force) { 2381 rex |= 0x40; // REX.0000 2382 } 2383 if (dst.NeedsRex()) { 2384 rex |= 0x44; // REX.0R00 2385 } 2386 if (rex != 0) { 2387 EmitUint8(rex); 2388 } 2389} 2390 2391static dwarf::Reg DWARFReg(Register reg) { 2392 return dwarf::Reg::X86_64Core(static_cast<int>(reg)); 2393} 2394static dwarf::Reg DWARFReg(FloatRegister reg) { 2395 return dwarf::Reg::X86_64Fp(static_cast<int>(reg)); 2396} 2397 2398constexpr size_t kFramePointerSize = 8; 2399 2400void X86_64Assembler::BuildFrame(size_t frame_size, ManagedRegister method_reg, 2401 const std::vector<ManagedRegister>& spill_regs, 2402 const ManagedRegisterEntrySpills& entry_spills) { 2403 DCHECK_EQ(buffer_.Size(), 0U); // Nothing emitted yet. 2404 cfi_.SetCurrentCFAOffset(8); // Return address on stack. 2405 CHECK_ALIGNED(frame_size, kStackAlignment); 2406 int gpr_count = 0; 2407 for (int i = spill_regs.size() - 1; i >= 0; --i) { 2408 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2409 if (spill.IsCpuRegister()) { 2410 pushq(spill.AsCpuRegister()); 2411 gpr_count++; 2412 cfi_.AdjustCFAOffset(kFramePointerSize); 2413 cfi_.RelOffset(DWARFReg(spill.AsCpuRegister().AsRegister()), 0); 2414 } 2415 } 2416 // return address then method on stack. 2417 int64_t rest_of_frame = static_cast<int64_t>(frame_size) 2418 - (gpr_count * kFramePointerSize) 2419 - kFramePointerSize /*return address*/; 2420 subq(CpuRegister(RSP), Immediate(rest_of_frame)); 2421 cfi_.AdjustCFAOffset(rest_of_frame); 2422 2423 // spill xmms 2424 int64_t offset = rest_of_frame; 2425 for (int i = spill_regs.size() - 1; i >= 0; --i) { 2426 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2427 if (spill.IsXmmRegister()) { 2428 offset -= sizeof(double); 2429 movsd(Address(CpuRegister(RSP), offset), spill.AsXmmRegister()); 2430 cfi_.RelOffset(DWARFReg(spill.AsXmmRegister().AsFloatRegister()), offset); 2431 } 2432 } 2433 2434 DCHECK_EQ(kX86_64PointerSize, kFramePointerSize); 2435 2436 movq(Address(CpuRegister(RSP), 0), method_reg.AsX86_64().AsCpuRegister()); 2437 2438 for (size_t i = 0; i < entry_spills.size(); ++i) { 2439 ManagedRegisterSpill spill = entry_spills.at(i); 2440 if (spill.AsX86_64().IsCpuRegister()) { 2441 if (spill.getSize() == 8) { 2442 movq(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), 2443 spill.AsX86_64().AsCpuRegister()); 2444 } else { 2445 CHECK_EQ(spill.getSize(), 4); 2446 movl(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsCpuRegister()); 2447 } 2448 } else { 2449 if (spill.getSize() == 8) { 2450 movsd(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 2451 } else { 2452 CHECK_EQ(spill.getSize(), 4); 2453 movss(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 2454 } 2455 } 2456 } 2457} 2458 2459void X86_64Assembler::RemoveFrame(size_t frame_size, 2460 const std::vector<ManagedRegister>& spill_regs) { 2461 CHECK_ALIGNED(frame_size, kStackAlignment); 2462 cfi_.RememberState(); 2463 int gpr_count = 0; 2464 // unspill xmms 2465 int64_t offset = static_cast<int64_t>(frame_size) - (spill_regs.size() * kFramePointerSize) - 2 * kFramePointerSize; 2466 for (size_t i = 0; i < spill_regs.size(); ++i) { 2467 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2468 if (spill.IsXmmRegister()) { 2469 offset += sizeof(double); 2470 movsd(spill.AsXmmRegister(), Address(CpuRegister(RSP), offset)); 2471 cfi_.Restore(DWARFReg(spill.AsXmmRegister().AsFloatRegister())); 2472 } else { 2473 gpr_count++; 2474 } 2475 } 2476 int adjust = static_cast<int>(frame_size) - (gpr_count * kFramePointerSize) - kFramePointerSize; 2477 addq(CpuRegister(RSP), Immediate(adjust)); 2478 cfi_.AdjustCFAOffset(-adjust); 2479 for (size_t i = 0; i < spill_regs.size(); ++i) { 2480 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2481 if (spill.IsCpuRegister()) { 2482 popq(spill.AsCpuRegister()); 2483 cfi_.AdjustCFAOffset(-static_cast<int>(kFramePointerSize)); 2484 cfi_.Restore(DWARFReg(spill.AsCpuRegister().AsRegister())); 2485 } 2486 } 2487 ret(); 2488 // The CFI should be restored for any code that follows the exit block. 2489 cfi_.RestoreState(); 2490 cfi_.DefCFAOffset(frame_size); 2491} 2492 2493void X86_64Assembler::IncreaseFrameSize(size_t adjust) { 2494 CHECK_ALIGNED(adjust, kStackAlignment); 2495 addq(CpuRegister(RSP), Immediate(-static_cast<int64_t>(adjust))); 2496 cfi_.AdjustCFAOffset(adjust); 2497} 2498 2499void X86_64Assembler::DecreaseFrameSize(size_t adjust) { 2500 CHECK_ALIGNED(adjust, kStackAlignment); 2501 addq(CpuRegister(RSP), Immediate(adjust)); 2502 cfi_.AdjustCFAOffset(-adjust); 2503} 2504 2505void X86_64Assembler::Store(FrameOffset offs, ManagedRegister msrc, size_t size) { 2506 X86_64ManagedRegister src = msrc.AsX86_64(); 2507 if (src.IsNoRegister()) { 2508 CHECK_EQ(0u, size); 2509 } else if (src.IsCpuRegister()) { 2510 if (size == 4) { 2511 CHECK_EQ(4u, size); 2512 movl(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 2513 } else { 2514 CHECK_EQ(8u, size); 2515 movq(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 2516 } 2517 } else if (src.IsRegisterPair()) { 2518 CHECK_EQ(0u, size); 2519 movq(Address(CpuRegister(RSP), offs), src.AsRegisterPairLow()); 2520 movq(Address(CpuRegister(RSP), FrameOffset(offs.Int32Value()+4)), 2521 src.AsRegisterPairHigh()); 2522 } else if (src.IsX87Register()) { 2523 if (size == 4) { 2524 fstps(Address(CpuRegister(RSP), offs)); 2525 } else { 2526 fstpl(Address(CpuRegister(RSP), offs)); 2527 } 2528 } else { 2529 CHECK(src.IsXmmRegister()); 2530 if (size == 4) { 2531 movss(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 2532 } else { 2533 movsd(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 2534 } 2535 } 2536} 2537 2538void X86_64Assembler::StoreRef(FrameOffset dest, ManagedRegister msrc) { 2539 X86_64ManagedRegister src = msrc.AsX86_64(); 2540 CHECK(src.IsCpuRegister()); 2541 movl(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 2542} 2543 2544void X86_64Assembler::StoreRawPtr(FrameOffset dest, ManagedRegister msrc) { 2545 X86_64ManagedRegister src = msrc.AsX86_64(); 2546 CHECK(src.IsCpuRegister()); 2547 movq(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 2548} 2549 2550void X86_64Assembler::StoreImmediateToFrame(FrameOffset dest, uint32_t imm, 2551 ManagedRegister) { 2552 movl(Address(CpuRegister(RSP), dest), Immediate(imm)); // TODO(64) movq? 2553} 2554 2555void X86_64Assembler::StoreImmediateToThread64(ThreadOffset<8> dest, uint32_t imm, 2556 ManagedRegister) { 2557 gs()->movl(Address::Absolute(dest, true), Immediate(imm)); // TODO(64) movq? 2558} 2559 2560void X86_64Assembler::StoreStackOffsetToThread64(ThreadOffset<8> thr_offs, 2561 FrameOffset fr_offs, 2562 ManagedRegister mscratch) { 2563 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2564 CHECK(scratch.IsCpuRegister()); 2565 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), fr_offs)); 2566 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2567} 2568 2569void X86_64Assembler::StoreStackPointerToThread64(ThreadOffset<8> thr_offs) { 2570 gs()->movq(Address::Absolute(thr_offs, true), CpuRegister(RSP)); 2571} 2572 2573void X86_64Assembler::StoreSpanning(FrameOffset /*dst*/, ManagedRegister /*src*/, 2574 FrameOffset /*in_off*/, ManagedRegister /*scratch*/) { 2575 UNIMPLEMENTED(FATAL); // this case only currently exists for ARM 2576} 2577 2578void X86_64Assembler::Load(ManagedRegister mdest, FrameOffset src, size_t size) { 2579 X86_64ManagedRegister dest = mdest.AsX86_64(); 2580 if (dest.IsNoRegister()) { 2581 CHECK_EQ(0u, size); 2582 } else if (dest.IsCpuRegister()) { 2583 if (size == 4) { 2584 CHECK_EQ(4u, size); 2585 movl(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2586 } else { 2587 CHECK_EQ(8u, size); 2588 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2589 } 2590 } else if (dest.IsRegisterPair()) { 2591 CHECK_EQ(0u, size); 2592 movq(dest.AsRegisterPairLow(), Address(CpuRegister(RSP), src)); 2593 movq(dest.AsRegisterPairHigh(), Address(CpuRegister(RSP), FrameOffset(src.Int32Value()+4))); 2594 } else if (dest.IsX87Register()) { 2595 if (size == 4) { 2596 flds(Address(CpuRegister(RSP), src)); 2597 } else { 2598 fldl(Address(CpuRegister(RSP), src)); 2599 } 2600 } else { 2601 CHECK(dest.IsXmmRegister()); 2602 if (size == 4) { 2603 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2604 } else { 2605 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2606 } 2607 } 2608} 2609 2610void X86_64Assembler::LoadFromThread64(ManagedRegister mdest, ThreadOffset<8> src, size_t size) { 2611 X86_64ManagedRegister dest = mdest.AsX86_64(); 2612 if (dest.IsNoRegister()) { 2613 CHECK_EQ(0u, size); 2614 } else if (dest.IsCpuRegister()) { 2615 CHECK_EQ(4u, size); 2616 gs()->movl(dest.AsCpuRegister(), Address::Absolute(src, true)); 2617 } else if (dest.IsRegisterPair()) { 2618 CHECK_EQ(8u, size); 2619 gs()->movq(dest.AsRegisterPairLow(), Address::Absolute(src, true)); 2620 } else if (dest.IsX87Register()) { 2621 if (size == 4) { 2622 gs()->flds(Address::Absolute(src, true)); 2623 } else { 2624 gs()->fldl(Address::Absolute(src, true)); 2625 } 2626 } else { 2627 CHECK(dest.IsXmmRegister()); 2628 if (size == 4) { 2629 gs()->movss(dest.AsXmmRegister(), Address::Absolute(src, true)); 2630 } else { 2631 gs()->movsd(dest.AsXmmRegister(), Address::Absolute(src, true)); 2632 } 2633 } 2634} 2635 2636void X86_64Assembler::LoadRef(ManagedRegister mdest, FrameOffset src) { 2637 X86_64ManagedRegister dest = mdest.AsX86_64(); 2638 CHECK(dest.IsCpuRegister()); 2639 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2640} 2641 2642void X86_64Assembler::LoadRef(ManagedRegister mdest, ManagedRegister base, MemberOffset offs, 2643 bool unpoison_reference) { 2644 X86_64ManagedRegister dest = mdest.AsX86_64(); 2645 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2646 movl(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2647 if (unpoison_reference) { 2648 MaybeUnpoisonHeapReference(dest.AsCpuRegister()); 2649 } 2650} 2651 2652void X86_64Assembler::LoadRawPtr(ManagedRegister mdest, ManagedRegister base, 2653 Offset offs) { 2654 X86_64ManagedRegister dest = mdest.AsX86_64(); 2655 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2656 movq(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2657} 2658 2659void X86_64Assembler::LoadRawPtrFromThread64(ManagedRegister mdest, ThreadOffset<8> offs) { 2660 X86_64ManagedRegister dest = mdest.AsX86_64(); 2661 CHECK(dest.IsCpuRegister()); 2662 gs()->movq(dest.AsCpuRegister(), Address::Absolute(offs, true)); 2663} 2664 2665void X86_64Assembler::SignExtend(ManagedRegister mreg, size_t size) { 2666 X86_64ManagedRegister reg = mreg.AsX86_64(); 2667 CHECK(size == 1 || size == 2) << size; 2668 CHECK(reg.IsCpuRegister()) << reg; 2669 if (size == 1) { 2670 movsxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2671 } else { 2672 movsxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2673 } 2674} 2675 2676void X86_64Assembler::ZeroExtend(ManagedRegister mreg, size_t size) { 2677 X86_64ManagedRegister reg = mreg.AsX86_64(); 2678 CHECK(size == 1 || size == 2) << size; 2679 CHECK(reg.IsCpuRegister()) << reg; 2680 if (size == 1) { 2681 movzxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2682 } else { 2683 movzxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2684 } 2685} 2686 2687void X86_64Assembler::Move(ManagedRegister mdest, ManagedRegister msrc, size_t size) { 2688 X86_64ManagedRegister dest = mdest.AsX86_64(); 2689 X86_64ManagedRegister src = msrc.AsX86_64(); 2690 if (!dest.Equals(src)) { 2691 if (dest.IsCpuRegister() && src.IsCpuRegister()) { 2692 movq(dest.AsCpuRegister(), src.AsCpuRegister()); 2693 } else if (src.IsX87Register() && dest.IsXmmRegister()) { 2694 // Pass via stack and pop X87 register 2695 subl(CpuRegister(RSP), Immediate(16)); 2696 if (size == 4) { 2697 CHECK_EQ(src.AsX87Register(), ST0); 2698 fstps(Address(CpuRegister(RSP), 0)); 2699 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2700 } else { 2701 CHECK_EQ(src.AsX87Register(), ST0); 2702 fstpl(Address(CpuRegister(RSP), 0)); 2703 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2704 } 2705 addq(CpuRegister(RSP), Immediate(16)); 2706 } else { 2707 // TODO: x87, SSE 2708 UNIMPLEMENTED(FATAL) << ": Move " << dest << ", " << src; 2709 } 2710 } 2711} 2712 2713void X86_64Assembler::CopyRef(FrameOffset dest, FrameOffset src, ManagedRegister mscratch) { 2714 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2715 CHECK(scratch.IsCpuRegister()); 2716 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2717 movl(Address(CpuRegister(RSP), dest), scratch.AsCpuRegister()); 2718} 2719 2720void X86_64Assembler::CopyRawPtrFromThread64(FrameOffset fr_offs, 2721 ThreadOffset<8> thr_offs, 2722 ManagedRegister mscratch) { 2723 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2724 CHECK(scratch.IsCpuRegister()); 2725 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(thr_offs, true)); 2726 Store(fr_offs, scratch, 8); 2727} 2728 2729void X86_64Assembler::CopyRawPtrToThread64(ThreadOffset<8> thr_offs, 2730 FrameOffset fr_offs, 2731 ManagedRegister mscratch) { 2732 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2733 CHECK(scratch.IsCpuRegister()); 2734 Load(scratch, fr_offs, 8); 2735 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2736} 2737 2738void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src, ManagedRegister mscratch, 2739 size_t size) { 2740 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2741 if (scratch.IsCpuRegister() && size == 8) { 2742 Load(scratch, src, 4); 2743 Store(dest, scratch, 4); 2744 Load(scratch, FrameOffset(src.Int32Value() + 4), 4); 2745 Store(FrameOffset(dest.Int32Value() + 4), scratch, 4); 2746 } else { 2747 Load(scratch, src, size); 2748 Store(dest, scratch, size); 2749 } 2750} 2751 2752void X86_64Assembler::Copy(FrameOffset /*dst*/, ManagedRegister /*src_base*/, Offset /*src_offset*/, 2753 ManagedRegister /*scratch*/, size_t /*size*/) { 2754 UNIMPLEMENTED(FATAL); 2755} 2756 2757void X86_64Assembler::Copy(ManagedRegister dest_base, Offset dest_offset, FrameOffset src, 2758 ManagedRegister scratch, size_t size) { 2759 CHECK(scratch.IsNoRegister()); 2760 CHECK_EQ(size, 4u); 2761 pushq(Address(CpuRegister(RSP), src)); 2762 popq(Address(dest_base.AsX86_64().AsCpuRegister(), dest_offset)); 2763} 2764 2765void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src_base, Offset src_offset, 2766 ManagedRegister mscratch, size_t size) { 2767 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2768 CHECK_EQ(size, 4u); 2769 movq(scratch, Address(CpuRegister(RSP), src_base)); 2770 movq(scratch, Address(scratch, src_offset)); 2771 movq(Address(CpuRegister(RSP), dest), scratch); 2772} 2773 2774void X86_64Assembler::Copy(ManagedRegister dest, Offset dest_offset, 2775 ManagedRegister src, Offset src_offset, 2776 ManagedRegister scratch, size_t size) { 2777 CHECK_EQ(size, 4u); 2778 CHECK(scratch.IsNoRegister()); 2779 pushq(Address(src.AsX86_64().AsCpuRegister(), src_offset)); 2780 popq(Address(dest.AsX86_64().AsCpuRegister(), dest_offset)); 2781} 2782 2783void X86_64Assembler::Copy(FrameOffset dest, Offset dest_offset, FrameOffset src, Offset src_offset, 2784 ManagedRegister mscratch, size_t size) { 2785 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2786 CHECK_EQ(size, 4u); 2787 CHECK_EQ(dest.Int32Value(), src.Int32Value()); 2788 movq(scratch, Address(CpuRegister(RSP), src)); 2789 pushq(Address(scratch, src_offset)); 2790 popq(Address(scratch, dest_offset)); 2791} 2792 2793void X86_64Assembler::MemoryBarrier(ManagedRegister) { 2794 mfence(); 2795} 2796 2797void X86_64Assembler::CreateHandleScopeEntry(ManagedRegister mout_reg, 2798 FrameOffset handle_scope_offset, 2799 ManagedRegister min_reg, bool null_allowed) { 2800 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2801 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2802 if (in_reg.IsNoRegister()) { // TODO(64): && null_allowed 2803 // Use out_reg as indicator of null. 2804 in_reg = out_reg; 2805 // TODO: movzwl 2806 movl(in_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2807 } 2808 CHECK(in_reg.IsCpuRegister()); 2809 CHECK(out_reg.IsCpuRegister()); 2810 VerifyObject(in_reg, null_allowed); 2811 if (null_allowed) { 2812 Label null_arg; 2813 if (!out_reg.Equals(in_reg)) { 2814 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2815 } 2816 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2817 j(kZero, &null_arg); 2818 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2819 Bind(&null_arg); 2820 } else { 2821 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2822 } 2823} 2824 2825void X86_64Assembler::CreateHandleScopeEntry(FrameOffset out_off, 2826 FrameOffset handle_scope_offset, 2827 ManagedRegister mscratch, 2828 bool null_allowed) { 2829 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2830 CHECK(scratch.IsCpuRegister()); 2831 if (null_allowed) { 2832 Label null_arg; 2833 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2834 testl(scratch.AsCpuRegister(), scratch.AsCpuRegister()); 2835 j(kZero, &null_arg); 2836 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2837 Bind(&null_arg); 2838 } else { 2839 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2840 } 2841 Store(out_off, scratch, 8); 2842} 2843 2844// Given a handle scope entry, load the associated reference. 2845void X86_64Assembler::LoadReferenceFromHandleScope(ManagedRegister mout_reg, 2846 ManagedRegister min_reg) { 2847 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2848 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2849 CHECK(out_reg.IsCpuRegister()); 2850 CHECK(in_reg.IsCpuRegister()); 2851 Label null_arg; 2852 if (!out_reg.Equals(in_reg)) { 2853 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2854 } 2855 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2856 j(kZero, &null_arg); 2857 movq(out_reg.AsCpuRegister(), Address(in_reg.AsCpuRegister(), 0)); 2858 Bind(&null_arg); 2859} 2860 2861void X86_64Assembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) { 2862 // TODO: not validating references 2863} 2864 2865void X86_64Assembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) { 2866 // TODO: not validating references 2867} 2868 2869void X86_64Assembler::Call(ManagedRegister mbase, Offset offset, ManagedRegister) { 2870 X86_64ManagedRegister base = mbase.AsX86_64(); 2871 CHECK(base.IsCpuRegister()); 2872 call(Address(base.AsCpuRegister(), offset.Int32Value())); 2873 // TODO: place reference map on call 2874} 2875 2876void X86_64Assembler::Call(FrameOffset base, Offset offset, ManagedRegister mscratch) { 2877 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2878 movq(scratch, Address(CpuRegister(RSP), base)); 2879 call(Address(scratch, offset)); 2880} 2881 2882void X86_64Assembler::CallFromThread64(ThreadOffset<8> offset, ManagedRegister /*mscratch*/) { 2883 gs()->call(Address::Absolute(offset, true)); 2884} 2885 2886void X86_64Assembler::GetCurrentThread(ManagedRegister tr) { 2887 gs()->movq(tr.AsX86_64().AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2888} 2889 2890void X86_64Assembler::GetCurrentThread(FrameOffset offset, ManagedRegister mscratch) { 2891 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2892 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2893 movq(Address(CpuRegister(RSP), offset), scratch.AsCpuRegister()); 2894} 2895 2896// Slowpath entered when Thread::Current()->_exception is non-null 2897class X86_64ExceptionSlowPath FINAL : public SlowPath { 2898 public: 2899 explicit X86_64ExceptionSlowPath(size_t stack_adjust) : stack_adjust_(stack_adjust) {} 2900 virtual void Emit(Assembler *sp_asm) OVERRIDE; 2901 private: 2902 const size_t stack_adjust_; 2903}; 2904 2905void X86_64Assembler::ExceptionPoll(ManagedRegister /*scratch*/, size_t stack_adjust) { 2906 X86_64ExceptionSlowPath* slow = new X86_64ExceptionSlowPath(stack_adjust); 2907 buffer_.EnqueueSlowPath(slow); 2908 gs()->cmpl(Address::Absolute(Thread::ExceptionOffset<8>(), true), Immediate(0)); 2909 j(kNotEqual, slow->Entry()); 2910} 2911 2912void X86_64ExceptionSlowPath::Emit(Assembler *sasm) { 2913 X86_64Assembler* sp_asm = down_cast<X86_64Assembler*>(sasm); 2914#define __ sp_asm-> 2915 __ Bind(&entry_); 2916 // Note: the return value is dead 2917 if (stack_adjust_ != 0) { // Fix up the frame. 2918 __ DecreaseFrameSize(stack_adjust_); 2919 } 2920 // Pass exception as argument in RDI 2921 __ gs()->movq(CpuRegister(RDI), Address::Absolute(Thread::ExceptionOffset<8>(), true)); 2922 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(8, pDeliverException), true)); 2923 // this call should never return 2924 __ int3(); 2925#undef __ 2926} 2927 2928void X86_64Assembler::AddConstantArea() { 2929 const std::vector<int32_t>& area = constant_area_.GetBuffer(); 2930 for (size_t i = 0, e = area.size(); i < e; i++) { 2931 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2932 EmitInt32(area[i]); 2933 } 2934} 2935 2936int ConstantArea::AddInt32(int32_t v) { 2937 for (size_t i = 0, e = buffer_.size(); i < e; i++) { 2938 if (v == buffer_[i]) { 2939 return i * elem_size_; 2940 } 2941 } 2942 2943 // Didn't match anything. 2944 int result = buffer_.size() * elem_size_; 2945 buffer_.push_back(v); 2946 return result; 2947} 2948 2949int ConstantArea::AddInt64(int64_t v) { 2950 int32_t v_low = v; 2951 int32_t v_high = v >> 32; 2952 if (buffer_.size() > 1) { 2953 // Ensure we don't pass the end of the buffer. 2954 for (size_t i = 0, e = buffer_.size() - 1; i < e; i++) { 2955 if (v_low == buffer_[i] && v_high == buffer_[i + 1]) { 2956 return i * elem_size_; 2957 } 2958 } 2959 } 2960 2961 // Didn't match anything. 2962 int result = buffer_.size() * elem_size_; 2963 buffer_.push_back(v_low); 2964 buffer_.push_back(v_high); 2965 return result; 2966} 2967 2968int ConstantArea::AddDouble(double v) { 2969 // Treat the value as a 64-bit integer value. 2970 return AddInt64(bit_cast<int64_t, double>(v)); 2971} 2972 2973int ConstantArea::AddFloat(float v) { 2974 // Treat the value as a 32-bit integer value. 2975 return AddInt32(bit_cast<int32_t, float>(v)); 2976} 2977 2978} // namespace x86_64 2979} // namespace art 2980