assembler_x86_64.cc revision 40741f394b2737e503f2c08be0ae9dd490fb106b
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "assembler_x86_64.h" 18 19#include "base/casts.h" 20#include "entrypoints/quick/quick_entrypoints.h" 21#include "memory_region.h" 22#include "thread.h" 23 24namespace art { 25namespace x86_64 { 26 27std::ostream& operator<<(std::ostream& os, const CpuRegister& reg) { 28 return os << reg.AsRegister(); 29} 30 31std::ostream& operator<<(std::ostream& os, const XmmRegister& reg) { 32 return os << reg.AsFloatRegister(); 33} 34 35std::ostream& operator<<(std::ostream& os, const X87Register& reg) { 36 return os << "ST" << static_cast<int>(reg); 37} 38 39void X86_64Assembler::call(CpuRegister reg) { 40 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 41 EmitOptionalRex32(reg); 42 EmitUint8(0xFF); 43 EmitRegisterOperand(2, reg.LowBits()); 44} 45 46 47void X86_64Assembler::call(const Address& address) { 48 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 49 EmitOptionalRex32(address); 50 EmitUint8(0xFF); 51 EmitOperand(2, address); 52} 53 54 55void X86_64Assembler::call(Label* label) { 56 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 57 EmitUint8(0xE8); 58 static const int kSize = 5; 59 // Offset by one because we already have emitted the opcode. 60 EmitLabel(label, kSize - 1); 61} 62 63void X86_64Assembler::pushq(CpuRegister reg) { 64 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 65 EmitOptionalRex32(reg); 66 EmitUint8(0x50 + reg.LowBits()); 67} 68 69 70void X86_64Assembler::pushq(const Address& address) { 71 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 72 EmitOptionalRex32(address); 73 EmitUint8(0xFF); 74 EmitOperand(6, address); 75} 76 77 78void X86_64Assembler::pushq(const Immediate& imm) { 79 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 80 CHECK(imm.is_int32()); // pushq only supports 32b immediate. 81 if (imm.is_int8()) { 82 EmitUint8(0x6A); 83 EmitUint8(imm.value() & 0xFF); 84 } else { 85 EmitUint8(0x68); 86 EmitImmediate(imm); 87 } 88} 89 90 91void X86_64Assembler::popq(CpuRegister reg) { 92 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 93 EmitOptionalRex32(reg); 94 EmitUint8(0x58 + reg.LowBits()); 95} 96 97 98void X86_64Assembler::popq(const Address& address) { 99 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 100 EmitOptionalRex32(address); 101 EmitUint8(0x8F); 102 EmitOperand(0, address); 103} 104 105 106void X86_64Assembler::movq(CpuRegister dst, const Immediate& imm) { 107 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 108 if (imm.is_int32()) { 109 // 32 bit. Note: sign-extends. 110 EmitRex64(dst); 111 EmitUint8(0xC7); 112 EmitRegisterOperand(0, dst.LowBits()); 113 EmitInt32(static_cast<int32_t>(imm.value())); 114 } else { 115 EmitRex64(dst); 116 EmitUint8(0xB8 + dst.LowBits()); 117 EmitInt64(imm.value()); 118 } 119} 120 121 122void X86_64Assembler::movl(CpuRegister dst, const Immediate& imm) { 123 CHECK(imm.is_int32()); 124 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 125 EmitOptionalRex32(dst); 126 EmitUint8(0xB8 + dst.LowBits()); 127 EmitImmediate(imm); 128} 129 130 131void X86_64Assembler::movq(const Address& dst, const Immediate& imm) { 132 CHECK(imm.is_int32()); 133 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 134 EmitRex64(dst); 135 EmitUint8(0xC7); 136 EmitOperand(0, dst); 137 EmitImmediate(imm); 138} 139 140 141void X86_64Assembler::movq(CpuRegister dst, CpuRegister src) { 142 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 143 // 0x89 is movq r/m64 <- r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 144 EmitRex64(src, dst); 145 EmitUint8(0x89); 146 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 147} 148 149 150void X86_64Assembler::movl(CpuRegister dst, CpuRegister src) { 151 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 152 EmitOptionalRex32(dst, src); 153 EmitUint8(0x8B); 154 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 155} 156 157 158void X86_64Assembler::movq(CpuRegister dst, const Address& src) { 159 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 160 EmitRex64(dst, src); 161 EmitUint8(0x8B); 162 EmitOperand(dst.LowBits(), src); 163} 164 165 166void X86_64Assembler::movl(CpuRegister dst, const Address& src) { 167 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 168 EmitOptionalRex32(dst, src); 169 EmitUint8(0x8B); 170 EmitOperand(dst.LowBits(), src); 171} 172 173 174void X86_64Assembler::movq(const Address& dst, CpuRegister src) { 175 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 176 EmitRex64(src, dst); 177 EmitUint8(0x89); 178 EmitOperand(src.LowBits(), dst); 179} 180 181 182void X86_64Assembler::movl(const Address& dst, CpuRegister src) { 183 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 184 EmitOptionalRex32(src, dst); 185 EmitUint8(0x89); 186 EmitOperand(src.LowBits(), dst); 187} 188 189void X86_64Assembler::movl(const Address& dst, const Immediate& imm) { 190 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 191 EmitOptionalRex32(dst); 192 EmitUint8(0xC7); 193 EmitOperand(0, dst); 194 EmitImmediate(imm); 195} 196 197 198void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src) { 199 cmov(c, dst, src, true); 200} 201 202void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src, bool is64bit) { 203 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 204 EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex()); 205 EmitUint8(0x0F); 206 EmitUint8(0x40 + c); 207 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 208} 209 210 211void X86_64Assembler::movzxb(CpuRegister dst, CpuRegister src) { 212 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 213 EmitOptionalByteRegNormalizingRex32(dst, src); 214 EmitUint8(0x0F); 215 EmitUint8(0xB6); 216 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 217} 218 219 220void X86_64Assembler::movzxb(CpuRegister dst, const Address& src) { 221 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 222 // Byte register is only in the source register form, so we don't use 223 // EmitOptionalByteRegNormalizingRex32(dst, src); 224 EmitOptionalRex32(dst, src); 225 EmitUint8(0x0F); 226 EmitUint8(0xB6); 227 EmitOperand(dst.LowBits(), src); 228} 229 230 231void X86_64Assembler::movsxb(CpuRegister dst, CpuRegister src) { 232 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 233 EmitOptionalByteRegNormalizingRex32(dst, src); 234 EmitUint8(0x0F); 235 EmitUint8(0xBE); 236 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 237} 238 239 240void X86_64Assembler::movsxb(CpuRegister dst, const Address& src) { 241 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 242 // Byte register is only in the source register form, so we don't use 243 // EmitOptionalByteRegNormalizingRex32(dst, src); 244 EmitOptionalRex32(dst, src); 245 EmitUint8(0x0F); 246 EmitUint8(0xBE); 247 EmitOperand(dst.LowBits(), src); 248} 249 250 251void X86_64Assembler::movb(CpuRegister /*dst*/, const Address& /*src*/) { 252 LOG(FATAL) << "Use movzxb or movsxb instead."; 253} 254 255 256void X86_64Assembler::movb(const Address& dst, CpuRegister src) { 257 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 258 EmitOptionalByteRegNormalizingRex32(src, dst); 259 EmitUint8(0x88); 260 EmitOperand(src.LowBits(), dst); 261} 262 263 264void X86_64Assembler::movb(const Address& dst, const Immediate& imm) { 265 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 266 EmitOptionalRex32(dst); 267 EmitUint8(0xC6); 268 EmitOperand(Register::RAX, dst); 269 CHECK(imm.is_int8()); 270 EmitUint8(imm.value() & 0xFF); 271} 272 273 274void X86_64Assembler::movzxw(CpuRegister dst, CpuRegister src) { 275 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 276 EmitOptionalRex32(dst, src); 277 EmitUint8(0x0F); 278 EmitUint8(0xB7); 279 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 280} 281 282 283void X86_64Assembler::movzxw(CpuRegister dst, const Address& src) { 284 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 285 EmitOptionalRex32(dst, src); 286 EmitUint8(0x0F); 287 EmitUint8(0xB7); 288 EmitOperand(dst.LowBits(), src); 289} 290 291 292void X86_64Assembler::movsxw(CpuRegister dst, CpuRegister src) { 293 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 294 EmitOptionalRex32(dst, src); 295 EmitUint8(0x0F); 296 EmitUint8(0xBF); 297 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 298} 299 300 301void X86_64Assembler::movsxw(CpuRegister dst, const Address& src) { 302 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 303 EmitOptionalRex32(dst, src); 304 EmitUint8(0x0F); 305 EmitUint8(0xBF); 306 EmitOperand(dst.LowBits(), src); 307} 308 309 310void X86_64Assembler::movw(CpuRegister /*dst*/, const Address& /*src*/) { 311 LOG(FATAL) << "Use movzxw or movsxw instead."; 312} 313 314 315void X86_64Assembler::movw(const Address& dst, CpuRegister src) { 316 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 317 EmitOperandSizeOverride(); 318 EmitOptionalRex32(src, dst); 319 EmitUint8(0x89); 320 EmitOperand(src.LowBits(), dst); 321} 322 323 324void X86_64Assembler::movw(const Address& dst, const Immediate& imm) { 325 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 326 EmitOperandSizeOverride(); 327 EmitOptionalRex32(dst); 328 EmitUint8(0xC7); 329 EmitOperand(Register::RAX, dst); 330 CHECK(imm.is_uint16() || imm.is_int16()); 331 EmitUint8(imm.value() & 0xFF); 332 EmitUint8(imm.value() >> 8); 333} 334 335 336void X86_64Assembler::leaq(CpuRegister dst, const Address& src) { 337 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 338 EmitRex64(dst, src); 339 EmitUint8(0x8D); 340 EmitOperand(dst.LowBits(), src); 341} 342 343 344void X86_64Assembler::leal(CpuRegister dst, const Address& src) { 345 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 346 EmitOptionalRex32(dst, src); 347 EmitUint8(0x8D); 348 EmitOperand(dst.LowBits(), src); 349} 350 351 352void X86_64Assembler::movaps(XmmRegister dst, XmmRegister src) { 353 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 354 EmitOptionalRex32(dst, src); 355 EmitUint8(0x0F); 356 EmitUint8(0x28); 357 EmitXmmRegisterOperand(dst.LowBits(), src); 358} 359 360 361void X86_64Assembler::movss(XmmRegister dst, const Address& src) { 362 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 363 EmitUint8(0xF3); 364 EmitOptionalRex32(dst, src); 365 EmitUint8(0x0F); 366 EmitUint8(0x10); 367 EmitOperand(dst.LowBits(), src); 368} 369 370 371void X86_64Assembler::movss(const Address& dst, XmmRegister src) { 372 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 373 EmitUint8(0xF3); 374 EmitOptionalRex32(src, dst); 375 EmitUint8(0x0F); 376 EmitUint8(0x11); 377 EmitOperand(src.LowBits(), dst); 378} 379 380 381void X86_64Assembler::movss(XmmRegister dst, XmmRegister src) { 382 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 383 EmitUint8(0xF3); 384 EmitOptionalRex32(src, dst); // Movss is MR encoding instead of the usual RM. 385 EmitUint8(0x0F); 386 EmitUint8(0x11); 387 EmitXmmRegisterOperand(src.LowBits(), dst); 388} 389 390 391void X86_64Assembler::movsxd(CpuRegister dst, CpuRegister src) { 392 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 393 EmitRex64(dst, src); 394 EmitUint8(0x63); 395 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 396} 397 398 399void X86_64Assembler::movsxd(CpuRegister dst, const Address& src) { 400 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 401 EmitRex64(dst); 402 EmitUint8(0x63); 403 EmitOperand(dst.LowBits(), src); 404} 405 406 407void X86_64Assembler::movd(XmmRegister dst, CpuRegister src) { 408 movd(dst, src, true); 409} 410 411void X86_64Assembler::movd(CpuRegister dst, XmmRegister src) { 412 movd(dst, src, true); 413} 414 415void X86_64Assembler::movd(XmmRegister dst, CpuRegister src, bool is64bit) { 416 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 417 EmitUint8(0x66); 418 EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex()); 419 EmitUint8(0x0F); 420 EmitUint8(0x6E); 421 EmitOperand(dst.LowBits(), Operand(src)); 422} 423 424void X86_64Assembler::movd(CpuRegister dst, XmmRegister src, bool is64bit) { 425 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 426 EmitUint8(0x66); 427 EmitOptionalRex(false, is64bit, src.NeedsRex(), false, dst.NeedsRex()); 428 EmitUint8(0x0F); 429 EmitUint8(0x7E); 430 EmitOperand(src.LowBits(), Operand(dst)); 431} 432 433 434void X86_64Assembler::addss(XmmRegister dst, XmmRegister src) { 435 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 436 EmitUint8(0xF3); 437 EmitOptionalRex32(dst, src); 438 EmitUint8(0x0F); 439 EmitUint8(0x58); 440 EmitXmmRegisterOperand(dst.LowBits(), src); 441} 442 443 444void X86_64Assembler::addss(XmmRegister dst, const Address& src) { 445 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 446 EmitUint8(0xF3); 447 EmitOptionalRex32(dst, src); 448 EmitUint8(0x0F); 449 EmitUint8(0x58); 450 EmitOperand(dst.LowBits(), src); 451} 452 453 454void X86_64Assembler::subss(XmmRegister dst, XmmRegister src) { 455 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 456 EmitUint8(0xF3); 457 EmitOptionalRex32(dst, src); 458 EmitUint8(0x0F); 459 EmitUint8(0x5C); 460 EmitXmmRegisterOperand(dst.LowBits(), src); 461} 462 463 464void X86_64Assembler::subss(XmmRegister dst, const Address& src) { 465 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 466 EmitUint8(0xF3); 467 EmitOptionalRex32(dst, src); 468 EmitUint8(0x0F); 469 EmitUint8(0x5C); 470 EmitOperand(dst.LowBits(), src); 471} 472 473 474void X86_64Assembler::mulss(XmmRegister dst, XmmRegister src) { 475 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 476 EmitUint8(0xF3); 477 EmitOptionalRex32(dst, src); 478 EmitUint8(0x0F); 479 EmitUint8(0x59); 480 EmitXmmRegisterOperand(dst.LowBits(), src); 481} 482 483 484void X86_64Assembler::mulss(XmmRegister dst, const Address& src) { 485 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 486 EmitUint8(0xF3); 487 EmitOptionalRex32(dst, src); 488 EmitUint8(0x0F); 489 EmitUint8(0x59); 490 EmitOperand(dst.LowBits(), src); 491} 492 493 494void X86_64Assembler::divss(XmmRegister dst, XmmRegister src) { 495 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 496 EmitUint8(0xF3); 497 EmitOptionalRex32(dst, src); 498 EmitUint8(0x0F); 499 EmitUint8(0x5E); 500 EmitXmmRegisterOperand(dst.LowBits(), src); 501} 502 503 504void X86_64Assembler::divss(XmmRegister dst, const Address& src) { 505 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 506 EmitUint8(0xF3); 507 EmitOptionalRex32(dst, src); 508 EmitUint8(0x0F); 509 EmitUint8(0x5E); 510 EmitOperand(dst.LowBits(), src); 511} 512 513 514void X86_64Assembler::flds(const Address& src) { 515 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 516 EmitUint8(0xD9); 517 EmitOperand(0, src); 518} 519 520 521void X86_64Assembler::fsts(const Address& dst) { 522 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 523 EmitUint8(0xD9); 524 EmitOperand(2, dst); 525} 526 527 528void X86_64Assembler::fstps(const Address& dst) { 529 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 530 EmitUint8(0xD9); 531 EmitOperand(3, dst); 532} 533 534 535void X86_64Assembler::movsd(XmmRegister dst, const Address& src) { 536 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 537 EmitUint8(0xF2); 538 EmitOptionalRex32(dst, src); 539 EmitUint8(0x0F); 540 EmitUint8(0x10); 541 EmitOperand(dst.LowBits(), src); 542} 543 544 545void X86_64Assembler::movsd(const Address& dst, XmmRegister src) { 546 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 547 EmitUint8(0xF2); 548 EmitOptionalRex32(src, dst); 549 EmitUint8(0x0F); 550 EmitUint8(0x11); 551 EmitOperand(src.LowBits(), dst); 552} 553 554 555void X86_64Assembler::movsd(XmmRegister dst, XmmRegister src) { 556 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 557 EmitUint8(0xF2); 558 EmitOptionalRex32(src, dst); // Movsd is MR encoding instead of the usual RM. 559 EmitUint8(0x0F); 560 EmitUint8(0x11); 561 EmitXmmRegisterOperand(src.LowBits(), dst); 562} 563 564 565void X86_64Assembler::addsd(XmmRegister dst, XmmRegister src) { 566 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 567 EmitUint8(0xF2); 568 EmitOptionalRex32(dst, src); 569 EmitUint8(0x0F); 570 EmitUint8(0x58); 571 EmitXmmRegisterOperand(dst.LowBits(), src); 572} 573 574 575void X86_64Assembler::addsd(XmmRegister dst, const Address& src) { 576 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 577 EmitUint8(0xF2); 578 EmitOptionalRex32(dst, src); 579 EmitUint8(0x0F); 580 EmitUint8(0x58); 581 EmitOperand(dst.LowBits(), src); 582} 583 584 585void X86_64Assembler::subsd(XmmRegister dst, XmmRegister src) { 586 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 587 EmitUint8(0xF2); 588 EmitOptionalRex32(dst, src); 589 EmitUint8(0x0F); 590 EmitUint8(0x5C); 591 EmitXmmRegisterOperand(dst.LowBits(), src); 592} 593 594 595void X86_64Assembler::subsd(XmmRegister dst, const Address& src) { 596 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 597 EmitUint8(0xF2); 598 EmitOptionalRex32(dst, src); 599 EmitUint8(0x0F); 600 EmitUint8(0x5C); 601 EmitOperand(dst.LowBits(), src); 602} 603 604 605void X86_64Assembler::mulsd(XmmRegister dst, XmmRegister src) { 606 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 607 EmitUint8(0xF2); 608 EmitOptionalRex32(dst, src); 609 EmitUint8(0x0F); 610 EmitUint8(0x59); 611 EmitXmmRegisterOperand(dst.LowBits(), src); 612} 613 614 615void X86_64Assembler::mulsd(XmmRegister dst, const Address& src) { 616 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 617 EmitUint8(0xF2); 618 EmitOptionalRex32(dst, src); 619 EmitUint8(0x0F); 620 EmitUint8(0x59); 621 EmitOperand(dst.LowBits(), src); 622} 623 624 625void X86_64Assembler::divsd(XmmRegister dst, XmmRegister src) { 626 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 627 EmitUint8(0xF2); 628 EmitOptionalRex32(dst, src); 629 EmitUint8(0x0F); 630 EmitUint8(0x5E); 631 EmitXmmRegisterOperand(dst.LowBits(), src); 632} 633 634 635void X86_64Assembler::divsd(XmmRegister dst, const Address& src) { 636 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 637 EmitUint8(0xF2); 638 EmitOptionalRex32(dst, src); 639 EmitUint8(0x0F); 640 EmitUint8(0x5E); 641 EmitOperand(dst.LowBits(), src); 642} 643 644 645void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) { 646 cvtsi2ss(dst, src, false); 647} 648 649 650void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src, bool is64bit) { 651 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 652 EmitUint8(0xF3); 653 if (is64bit) { 654 // Emit a REX.W prefix if the operand size is 64 bits. 655 EmitRex64(dst, src); 656 } else { 657 EmitOptionalRex32(dst, src); 658 } 659 EmitUint8(0x0F); 660 EmitUint8(0x2A); 661 EmitOperand(dst.LowBits(), Operand(src)); 662} 663 664 665void X86_64Assembler::cvtsi2ss(XmmRegister dst, const Address& src, bool is64bit) { 666 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 667 EmitUint8(0xF3); 668 if (is64bit) { 669 // Emit a REX.W prefix if the operand size is 64 bits. 670 EmitRex64(dst, src); 671 } else { 672 EmitOptionalRex32(dst, src); 673 } 674 EmitUint8(0x0F); 675 EmitUint8(0x2A); 676 EmitOperand(dst.LowBits(), src); 677} 678 679 680void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src) { 681 cvtsi2sd(dst, src, false); 682} 683 684 685void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src, bool is64bit) { 686 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 687 EmitUint8(0xF2); 688 if (is64bit) { 689 // Emit a REX.W prefix if the operand size is 64 bits. 690 EmitRex64(dst, src); 691 } else { 692 EmitOptionalRex32(dst, src); 693 } 694 EmitUint8(0x0F); 695 EmitUint8(0x2A); 696 EmitOperand(dst.LowBits(), Operand(src)); 697} 698 699 700void X86_64Assembler::cvtsi2sd(XmmRegister dst, const Address& src, bool is64bit) { 701 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 702 EmitUint8(0xF2); 703 if (is64bit) { 704 // Emit a REX.W prefix if the operand size is 64 bits. 705 EmitRex64(dst, src); 706 } else { 707 EmitOptionalRex32(dst, src); 708 } 709 EmitUint8(0x0F); 710 EmitUint8(0x2A); 711 EmitOperand(dst.LowBits(), src); 712} 713 714 715void X86_64Assembler::cvtss2si(CpuRegister dst, XmmRegister src) { 716 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 717 EmitUint8(0xF3); 718 EmitOptionalRex32(dst, src); 719 EmitUint8(0x0F); 720 EmitUint8(0x2D); 721 EmitXmmRegisterOperand(dst.LowBits(), src); 722} 723 724 725void X86_64Assembler::cvtss2sd(XmmRegister dst, XmmRegister src) { 726 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 727 EmitUint8(0xF3); 728 EmitOptionalRex32(dst, src); 729 EmitUint8(0x0F); 730 EmitUint8(0x5A); 731 EmitXmmRegisterOperand(dst.LowBits(), src); 732} 733 734 735void X86_64Assembler::cvtss2sd(XmmRegister dst, const Address& src) { 736 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 737 EmitUint8(0xF3); 738 EmitOptionalRex32(dst, src); 739 EmitUint8(0x0F); 740 EmitUint8(0x5A); 741 EmitOperand(dst.LowBits(), src); 742} 743 744 745void X86_64Assembler::cvtsd2si(CpuRegister dst, XmmRegister src) { 746 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 747 EmitUint8(0xF2); 748 EmitOptionalRex32(dst, src); 749 EmitUint8(0x0F); 750 EmitUint8(0x2D); 751 EmitXmmRegisterOperand(dst.LowBits(), src); 752} 753 754 755void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src) { 756 cvttss2si(dst, src, false); 757} 758 759 760void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src, bool is64bit) { 761 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 762 EmitUint8(0xF3); 763 if (is64bit) { 764 // Emit a REX.W prefix if the operand size is 64 bits. 765 EmitRex64(dst, src); 766 } else { 767 EmitOptionalRex32(dst, src); 768 } 769 EmitUint8(0x0F); 770 EmitUint8(0x2C); 771 EmitXmmRegisterOperand(dst.LowBits(), src); 772} 773 774 775void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src) { 776 cvttsd2si(dst, src, false); 777} 778 779 780void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src, bool is64bit) { 781 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 782 EmitUint8(0xF2); 783 if (is64bit) { 784 // Emit a REX.W prefix if the operand size is 64 bits. 785 EmitRex64(dst, src); 786 } else { 787 EmitOptionalRex32(dst, src); 788 } 789 EmitUint8(0x0F); 790 EmitUint8(0x2C); 791 EmitXmmRegisterOperand(dst.LowBits(), src); 792} 793 794 795void X86_64Assembler::cvtsd2ss(XmmRegister dst, XmmRegister src) { 796 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 797 EmitUint8(0xF2); 798 EmitOptionalRex32(dst, src); 799 EmitUint8(0x0F); 800 EmitUint8(0x5A); 801 EmitXmmRegisterOperand(dst.LowBits(), src); 802} 803 804 805void X86_64Assembler::cvtsd2ss(XmmRegister dst, const Address& src) { 806 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 807 EmitUint8(0xF2); 808 EmitOptionalRex32(dst, src); 809 EmitUint8(0x0F); 810 EmitUint8(0x5A); 811 EmitOperand(dst.LowBits(), src); 812} 813 814 815void X86_64Assembler::cvtdq2pd(XmmRegister dst, XmmRegister src) { 816 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 817 EmitUint8(0xF3); 818 EmitOptionalRex32(dst, src); 819 EmitUint8(0x0F); 820 EmitUint8(0xE6); 821 EmitXmmRegisterOperand(dst.LowBits(), src); 822} 823 824 825void X86_64Assembler::comiss(XmmRegister a, XmmRegister b) { 826 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 827 EmitOptionalRex32(a, b); 828 EmitUint8(0x0F); 829 EmitUint8(0x2F); 830 EmitXmmRegisterOperand(a.LowBits(), b); 831} 832 833 834void X86_64Assembler::comiss(XmmRegister a, const Address& b) { 835 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 836 EmitOptionalRex32(a, b); 837 EmitUint8(0x0F); 838 EmitUint8(0x2F); 839 EmitOperand(a.LowBits(), b); 840} 841 842 843void X86_64Assembler::comisd(XmmRegister a, XmmRegister b) { 844 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 845 EmitUint8(0x66); 846 EmitOptionalRex32(a, b); 847 EmitUint8(0x0F); 848 EmitUint8(0x2F); 849 EmitXmmRegisterOperand(a.LowBits(), b); 850} 851 852 853void X86_64Assembler::comisd(XmmRegister a, const Address& b) { 854 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 855 EmitUint8(0x66); 856 EmitOptionalRex32(a, b); 857 EmitUint8(0x0F); 858 EmitUint8(0x2F); 859 EmitOperand(a.LowBits(), b); 860} 861 862 863void X86_64Assembler::ucomiss(XmmRegister a, XmmRegister b) { 864 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 865 EmitOptionalRex32(a, b); 866 EmitUint8(0x0F); 867 EmitUint8(0x2E); 868 EmitXmmRegisterOperand(a.LowBits(), b); 869} 870 871 872void X86_64Assembler::ucomiss(XmmRegister a, const Address& b) { 873 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 874 EmitOptionalRex32(a, b); 875 EmitUint8(0x0F); 876 EmitUint8(0x2E); 877 EmitOperand(a.LowBits(), b); 878} 879 880 881void X86_64Assembler::ucomisd(XmmRegister a, XmmRegister b) { 882 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 883 EmitUint8(0x66); 884 EmitOptionalRex32(a, b); 885 EmitUint8(0x0F); 886 EmitUint8(0x2E); 887 EmitXmmRegisterOperand(a.LowBits(), b); 888} 889 890 891void X86_64Assembler::ucomisd(XmmRegister a, const Address& b) { 892 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 893 EmitUint8(0x66); 894 EmitOptionalRex32(a, b); 895 EmitUint8(0x0F); 896 EmitUint8(0x2E); 897 EmitOperand(a.LowBits(), b); 898} 899 900 901void X86_64Assembler::roundsd(XmmRegister dst, XmmRegister src, const Immediate& imm) { 902 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 903 EmitUint8(0x66); 904 EmitOptionalRex32(dst, src); 905 EmitUint8(0x0F); 906 EmitUint8(0x3A); 907 EmitUint8(0x0B); 908 EmitXmmRegisterOperand(dst.LowBits(), src); 909 EmitUint8(imm.value()); 910} 911 912 913void X86_64Assembler::roundss(XmmRegister dst, XmmRegister src, const Immediate& imm) { 914 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 915 EmitUint8(0x66); 916 EmitOptionalRex32(dst, src); 917 EmitUint8(0x0F); 918 EmitUint8(0x3A); 919 EmitUint8(0x0A); 920 EmitXmmRegisterOperand(dst.LowBits(), src); 921 EmitUint8(imm.value()); 922} 923 924 925void X86_64Assembler::sqrtsd(XmmRegister dst, XmmRegister src) { 926 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 927 EmitUint8(0xF2); 928 EmitOptionalRex32(dst, src); 929 EmitUint8(0x0F); 930 EmitUint8(0x51); 931 EmitXmmRegisterOperand(dst.LowBits(), src); 932} 933 934 935void X86_64Assembler::sqrtss(XmmRegister dst, XmmRegister src) { 936 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 937 EmitUint8(0xF3); 938 EmitOptionalRex32(dst, src); 939 EmitUint8(0x0F); 940 EmitUint8(0x51); 941 EmitXmmRegisterOperand(dst.LowBits(), src); 942} 943 944 945void X86_64Assembler::xorpd(XmmRegister dst, const Address& src) { 946 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 947 EmitUint8(0x66); 948 EmitOptionalRex32(dst, src); 949 EmitUint8(0x0F); 950 EmitUint8(0x57); 951 EmitOperand(dst.LowBits(), src); 952} 953 954 955void X86_64Assembler::xorpd(XmmRegister dst, XmmRegister src) { 956 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 957 EmitUint8(0x66); 958 EmitOptionalRex32(dst, src); 959 EmitUint8(0x0F); 960 EmitUint8(0x57); 961 EmitXmmRegisterOperand(dst.LowBits(), src); 962} 963 964 965void X86_64Assembler::xorps(XmmRegister dst, const Address& src) { 966 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 967 EmitOptionalRex32(dst, src); 968 EmitUint8(0x0F); 969 EmitUint8(0x57); 970 EmitOperand(dst.LowBits(), src); 971} 972 973 974void X86_64Assembler::xorps(XmmRegister dst, XmmRegister src) { 975 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 976 EmitOptionalRex32(dst, src); 977 EmitUint8(0x0F); 978 EmitUint8(0x57); 979 EmitXmmRegisterOperand(dst.LowBits(), src); 980} 981 982 983void X86_64Assembler::andpd(XmmRegister dst, const Address& src) { 984 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 985 EmitUint8(0x66); 986 EmitOptionalRex32(dst, src); 987 EmitUint8(0x0F); 988 EmitUint8(0x54); 989 EmitOperand(dst.LowBits(), src); 990} 991 992void X86_64Assembler::andpd(XmmRegister dst, XmmRegister src) { 993 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 994 EmitUint8(0x66); 995 EmitOptionalRex32(dst, src); 996 EmitUint8(0x0F); 997 EmitUint8(0x54); 998 EmitXmmRegisterOperand(dst.LowBits(), src); 999} 1000 1001void X86_64Assembler::andps(XmmRegister dst, XmmRegister src) { 1002 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1003 EmitOptionalRex32(dst, src); 1004 EmitUint8(0x0F); 1005 EmitUint8(0x54); 1006 EmitXmmRegisterOperand(dst.LowBits(), src); 1007} 1008 1009void X86_64Assembler::orpd(XmmRegister dst, XmmRegister src) { 1010 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1011 EmitUint8(0x66); 1012 EmitOptionalRex32(dst, src); 1013 EmitUint8(0x0F); 1014 EmitUint8(0x56); 1015 EmitXmmRegisterOperand(dst.LowBits(), src); 1016} 1017 1018void X86_64Assembler::orps(XmmRegister dst, XmmRegister src) { 1019 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1020 EmitOptionalRex32(dst, src); 1021 EmitUint8(0x0F); 1022 EmitUint8(0x56); 1023 EmitXmmRegisterOperand(dst.LowBits(), src); 1024} 1025 1026void X86_64Assembler::fldl(const Address& src) { 1027 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1028 EmitUint8(0xDD); 1029 EmitOperand(0, src); 1030} 1031 1032 1033void X86_64Assembler::fstl(const Address& dst) { 1034 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1035 EmitUint8(0xDD); 1036 EmitOperand(2, dst); 1037} 1038 1039 1040void X86_64Assembler::fstpl(const Address& dst) { 1041 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1042 EmitUint8(0xDD); 1043 EmitOperand(3, dst); 1044} 1045 1046 1047void X86_64Assembler::fstsw() { 1048 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1049 EmitUint8(0x9B); 1050 EmitUint8(0xDF); 1051 EmitUint8(0xE0); 1052} 1053 1054 1055void X86_64Assembler::fnstcw(const Address& dst) { 1056 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1057 EmitUint8(0xD9); 1058 EmitOperand(7, dst); 1059} 1060 1061 1062void X86_64Assembler::fldcw(const Address& src) { 1063 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1064 EmitUint8(0xD9); 1065 EmitOperand(5, src); 1066} 1067 1068 1069void X86_64Assembler::fistpl(const Address& dst) { 1070 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1071 EmitUint8(0xDF); 1072 EmitOperand(7, dst); 1073} 1074 1075 1076void X86_64Assembler::fistps(const Address& dst) { 1077 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1078 EmitUint8(0xDB); 1079 EmitOperand(3, dst); 1080} 1081 1082 1083void X86_64Assembler::fildl(const Address& src) { 1084 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1085 EmitUint8(0xDF); 1086 EmitOperand(5, src); 1087} 1088 1089 1090void X86_64Assembler::filds(const Address& src) { 1091 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1092 EmitUint8(0xDB); 1093 EmitOperand(0, src); 1094} 1095 1096 1097void X86_64Assembler::fincstp() { 1098 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1099 EmitUint8(0xD9); 1100 EmitUint8(0xF7); 1101} 1102 1103 1104void X86_64Assembler::ffree(const Immediate& index) { 1105 CHECK_LT(index.value(), 7); 1106 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1107 EmitUint8(0xDD); 1108 EmitUint8(0xC0 + index.value()); 1109} 1110 1111 1112void X86_64Assembler::fsin() { 1113 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1114 EmitUint8(0xD9); 1115 EmitUint8(0xFE); 1116} 1117 1118 1119void X86_64Assembler::fcos() { 1120 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1121 EmitUint8(0xD9); 1122 EmitUint8(0xFF); 1123} 1124 1125 1126void X86_64Assembler::fptan() { 1127 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1128 EmitUint8(0xD9); 1129 EmitUint8(0xF2); 1130} 1131 1132void X86_64Assembler::fucompp() { 1133 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1134 EmitUint8(0xDA); 1135 EmitUint8(0xE9); 1136} 1137 1138 1139void X86_64Assembler::fprem() { 1140 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1141 EmitUint8(0xD9); 1142 EmitUint8(0xF8); 1143} 1144 1145 1146void X86_64Assembler::xchgl(CpuRegister dst, CpuRegister src) { 1147 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1148 // There is a short version for rax. 1149 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't 1150 // work. 1151 const bool src_rax = src.AsRegister() == RAX; 1152 const bool dst_rax = dst.AsRegister() == RAX; 1153 if (src_rax || dst_rax) { 1154 EmitOptionalRex32(src_rax ? dst : src); 1155 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits())); 1156 return; 1157 } 1158 1159 // General case. 1160 EmitOptionalRex32(src, dst); 1161 EmitUint8(0x87); 1162 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1163} 1164 1165 1166void X86_64Assembler::xchgq(CpuRegister dst, CpuRegister src) { 1167 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1168 // There is a short version for rax. 1169 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't 1170 // work. 1171 const bool src_rax = src.AsRegister() == RAX; 1172 const bool dst_rax = dst.AsRegister() == RAX; 1173 if (src_rax || dst_rax) { 1174 // If src == target, emit a nop instead. 1175 if (src_rax && dst_rax) { 1176 EmitUint8(0x90); 1177 } else { 1178 EmitRex64(src_rax ? dst : src); 1179 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits())); 1180 } 1181 return; 1182 } 1183 1184 // General case. 1185 EmitRex64(src, dst); 1186 EmitUint8(0x87); 1187 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1188} 1189 1190 1191void X86_64Assembler::xchgl(CpuRegister reg, const Address& address) { 1192 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1193 EmitOptionalRex32(reg, address); 1194 EmitUint8(0x87); 1195 EmitOperand(reg.LowBits(), address); 1196} 1197 1198 1199void X86_64Assembler::cmpw(const Address& address, const Immediate& imm) { 1200 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1201 EmitOptionalRex32(address); 1202 EmitUint8(0x66); 1203 EmitComplex(7, address, imm); 1204} 1205 1206 1207void X86_64Assembler::cmpl(CpuRegister reg, const Immediate& imm) { 1208 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1209 EmitOptionalRex32(reg); 1210 EmitComplex(7, Operand(reg), imm); 1211} 1212 1213 1214void X86_64Assembler::cmpl(CpuRegister reg0, CpuRegister reg1) { 1215 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1216 EmitOptionalRex32(reg0, reg1); 1217 EmitUint8(0x3B); 1218 EmitOperand(reg0.LowBits(), Operand(reg1)); 1219} 1220 1221 1222void X86_64Assembler::cmpl(CpuRegister reg, const Address& address) { 1223 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1224 EmitOptionalRex32(reg, address); 1225 EmitUint8(0x3B); 1226 EmitOperand(reg.LowBits(), address); 1227} 1228 1229 1230void X86_64Assembler::cmpl(const Address& address, CpuRegister reg) { 1231 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1232 EmitOptionalRex32(reg, address); 1233 EmitUint8(0x39); 1234 EmitOperand(reg.LowBits(), address); 1235} 1236 1237 1238void X86_64Assembler::cmpl(const Address& address, const Immediate& imm) { 1239 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1240 EmitOptionalRex32(address); 1241 EmitComplex(7, address, imm); 1242} 1243 1244 1245void X86_64Assembler::cmpq(CpuRegister reg0, CpuRegister reg1) { 1246 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1247 EmitRex64(reg0, reg1); 1248 EmitUint8(0x3B); 1249 EmitOperand(reg0.LowBits(), Operand(reg1)); 1250} 1251 1252 1253void X86_64Assembler::cmpq(CpuRegister reg, const Immediate& imm) { 1254 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1255 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 1256 EmitRex64(reg); 1257 EmitComplex(7, Operand(reg), imm); 1258} 1259 1260 1261void X86_64Assembler::cmpq(CpuRegister reg, const Address& address) { 1262 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1263 EmitRex64(reg, address); 1264 EmitUint8(0x3B); 1265 EmitOperand(reg.LowBits(), address); 1266} 1267 1268 1269void X86_64Assembler::cmpq(const Address& address, const Immediate& imm) { 1270 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 1271 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1272 EmitRex64(address); 1273 EmitComplex(7, address, imm); 1274} 1275 1276 1277void X86_64Assembler::addl(CpuRegister dst, CpuRegister src) { 1278 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1279 EmitOptionalRex32(dst, src); 1280 EmitUint8(0x03); 1281 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1282} 1283 1284 1285void X86_64Assembler::addl(CpuRegister reg, const Address& address) { 1286 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1287 EmitOptionalRex32(reg, address); 1288 EmitUint8(0x03); 1289 EmitOperand(reg.LowBits(), address); 1290} 1291 1292 1293void X86_64Assembler::testl(CpuRegister reg1, CpuRegister reg2) { 1294 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1295 EmitOptionalRex32(reg1, reg2); 1296 EmitUint8(0x85); 1297 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 1298} 1299 1300 1301void X86_64Assembler::testl(CpuRegister reg, const Address& address) { 1302 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1303 EmitOptionalRex32(reg, address); 1304 EmitUint8(0x85); 1305 EmitOperand(reg.LowBits(), address); 1306} 1307 1308 1309void X86_64Assembler::testl(CpuRegister reg, const Immediate& immediate) { 1310 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1311 // For registers that have a byte variant (RAX, RBX, RCX, and RDX) 1312 // we only test the byte CpuRegister to keep the encoding short. 1313 if (immediate.is_uint8() && reg.AsRegister() < 4) { 1314 // Use zero-extended 8-bit immediate. 1315 if (reg.AsRegister() == RAX) { 1316 EmitUint8(0xA8); 1317 } else { 1318 EmitUint8(0xF6); 1319 EmitUint8(0xC0 + reg.AsRegister()); 1320 } 1321 EmitUint8(immediate.value() & 0xFF); 1322 } else if (reg.AsRegister() == RAX) { 1323 // Use short form if the destination is RAX. 1324 EmitUint8(0xA9); 1325 EmitImmediate(immediate); 1326 } else { 1327 EmitOptionalRex32(reg); 1328 EmitUint8(0xF7); 1329 EmitOperand(0, Operand(reg)); 1330 EmitImmediate(immediate); 1331 } 1332} 1333 1334 1335void X86_64Assembler::testq(CpuRegister reg1, CpuRegister reg2) { 1336 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1337 EmitRex64(reg1, reg2); 1338 EmitUint8(0x85); 1339 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 1340} 1341 1342 1343void X86_64Assembler::testq(CpuRegister reg, const Address& address) { 1344 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1345 EmitRex64(reg); 1346 EmitUint8(0x85); 1347 EmitOperand(reg.LowBits(), address); 1348} 1349 1350 1351void X86_64Assembler::andl(CpuRegister dst, CpuRegister src) { 1352 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1353 EmitOptionalRex32(dst, src); 1354 EmitUint8(0x23); 1355 EmitOperand(dst.LowBits(), Operand(src)); 1356} 1357 1358 1359void X86_64Assembler::andl(CpuRegister reg, const Address& address) { 1360 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1361 EmitOptionalRex32(reg, address); 1362 EmitUint8(0x23); 1363 EmitOperand(reg.LowBits(), address); 1364} 1365 1366 1367void X86_64Assembler::andl(CpuRegister dst, const Immediate& imm) { 1368 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1369 EmitOptionalRex32(dst); 1370 EmitComplex(4, Operand(dst), imm); 1371} 1372 1373 1374void X86_64Assembler::andq(CpuRegister reg, const Immediate& imm) { 1375 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1376 CHECK(imm.is_int32()); // andq only supports 32b immediate. 1377 EmitRex64(reg); 1378 EmitComplex(4, Operand(reg), imm); 1379} 1380 1381 1382void X86_64Assembler::andq(CpuRegister dst, CpuRegister src) { 1383 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1384 EmitRex64(dst, src); 1385 EmitUint8(0x23); 1386 EmitOperand(dst.LowBits(), Operand(src)); 1387} 1388 1389 1390void X86_64Assembler::andq(CpuRegister dst, const Address& src) { 1391 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1392 EmitRex64(dst, src); 1393 EmitUint8(0x23); 1394 EmitOperand(dst.LowBits(), src); 1395} 1396 1397 1398void X86_64Assembler::orl(CpuRegister dst, CpuRegister src) { 1399 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1400 EmitOptionalRex32(dst, src); 1401 EmitUint8(0x0B); 1402 EmitOperand(dst.LowBits(), Operand(src)); 1403} 1404 1405 1406void X86_64Assembler::orl(CpuRegister reg, const Address& address) { 1407 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1408 EmitOptionalRex32(reg, address); 1409 EmitUint8(0x0B); 1410 EmitOperand(reg.LowBits(), address); 1411} 1412 1413 1414void X86_64Assembler::orl(CpuRegister dst, const Immediate& imm) { 1415 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1416 EmitOptionalRex32(dst); 1417 EmitComplex(1, Operand(dst), imm); 1418} 1419 1420 1421void X86_64Assembler::orq(CpuRegister dst, const Immediate& imm) { 1422 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1423 CHECK(imm.is_int32()); // orq only supports 32b immediate. 1424 EmitRex64(dst); 1425 EmitComplex(1, Operand(dst), imm); 1426} 1427 1428 1429void X86_64Assembler::orq(CpuRegister dst, CpuRegister src) { 1430 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1431 EmitRex64(dst, src); 1432 EmitUint8(0x0B); 1433 EmitOperand(dst.LowBits(), Operand(src)); 1434} 1435 1436 1437void X86_64Assembler::orq(CpuRegister dst, const Address& src) { 1438 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1439 EmitRex64(dst, src); 1440 EmitUint8(0x0B); 1441 EmitOperand(dst.LowBits(), src); 1442} 1443 1444 1445void X86_64Assembler::xorl(CpuRegister dst, CpuRegister src) { 1446 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1447 EmitOptionalRex32(dst, src); 1448 EmitUint8(0x33); 1449 EmitOperand(dst.LowBits(), Operand(src)); 1450} 1451 1452 1453void X86_64Assembler::xorl(CpuRegister reg, const Address& address) { 1454 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1455 EmitOptionalRex32(reg, address); 1456 EmitUint8(0x33); 1457 EmitOperand(reg.LowBits(), address); 1458} 1459 1460 1461void X86_64Assembler::xorl(CpuRegister dst, const Immediate& imm) { 1462 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1463 EmitOptionalRex32(dst); 1464 EmitComplex(6, Operand(dst), imm); 1465} 1466 1467 1468void X86_64Assembler::xorq(CpuRegister dst, CpuRegister src) { 1469 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1470 EmitRex64(dst, src); 1471 EmitUint8(0x33); 1472 EmitOperand(dst.LowBits(), Operand(src)); 1473} 1474 1475 1476void X86_64Assembler::xorq(CpuRegister dst, const Immediate& imm) { 1477 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1478 CHECK(imm.is_int32()); // xorq only supports 32b immediate. 1479 EmitRex64(dst); 1480 EmitComplex(6, Operand(dst), imm); 1481} 1482 1483void X86_64Assembler::xorq(CpuRegister dst, const Address& src) { 1484 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1485 EmitRex64(dst, src); 1486 EmitUint8(0x33); 1487 EmitOperand(dst.LowBits(), src); 1488} 1489 1490 1491#if 0 1492void X86_64Assembler::rex(bool force, bool w, Register* r, Register* x, Register* b) { 1493 // REX.WRXB 1494 // W - 64-bit operand 1495 // R - MODRM.reg 1496 // X - SIB.index 1497 // B - MODRM.rm/SIB.base 1498 uint8_t rex = force ? 0x40 : 0; 1499 if (w) { 1500 rex |= 0x48; // REX.W000 1501 } 1502 if (r != nullptr && *r >= Register::R8 && *r < Register::kNumberOfCpuRegisters) { 1503 rex |= 0x44; // REX.0R00 1504 *r = static_cast<Register>(*r - 8); 1505 } 1506 if (x != nullptr && *x >= Register::R8 && *x < Register::kNumberOfCpuRegisters) { 1507 rex |= 0x42; // REX.00X0 1508 *x = static_cast<Register>(*x - 8); 1509 } 1510 if (b != nullptr && *b >= Register::R8 && *b < Register::kNumberOfCpuRegisters) { 1511 rex |= 0x41; // REX.000B 1512 *b = static_cast<Register>(*b - 8); 1513 } 1514 if (rex != 0) { 1515 EmitUint8(rex); 1516 } 1517} 1518 1519void X86_64Assembler::rex_reg_mem(bool force, bool w, Register* dst, const Address& mem) { 1520 // REX.WRXB 1521 // W - 64-bit operand 1522 // R - MODRM.reg 1523 // X - SIB.index 1524 // B - MODRM.rm/SIB.base 1525 uint8_t rex = mem->rex(); 1526 if (force) { 1527 rex |= 0x40; // REX.0000 1528 } 1529 if (w) { 1530 rex |= 0x48; // REX.W000 1531 } 1532 if (dst != nullptr && *dst >= Register::R8 && *dst < Register::kNumberOfCpuRegisters) { 1533 rex |= 0x44; // REX.0R00 1534 *dst = static_cast<Register>(*dst - 8); 1535 } 1536 if (rex != 0) { 1537 EmitUint8(rex); 1538 } 1539} 1540 1541void rex_mem_reg(bool force, bool w, Address* mem, Register* src); 1542#endif 1543 1544void X86_64Assembler::addl(CpuRegister reg, const Immediate& imm) { 1545 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1546 EmitOptionalRex32(reg); 1547 EmitComplex(0, Operand(reg), imm); 1548} 1549 1550 1551void X86_64Assembler::addq(CpuRegister reg, const Immediate& imm) { 1552 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1553 CHECK(imm.is_int32()); // addq only supports 32b immediate. 1554 EmitRex64(reg); 1555 EmitComplex(0, Operand(reg), imm); 1556} 1557 1558 1559void X86_64Assembler::addq(CpuRegister dst, const Address& address) { 1560 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1561 EmitRex64(dst); 1562 EmitUint8(0x03); 1563 EmitOperand(dst.LowBits(), address); 1564} 1565 1566 1567void X86_64Assembler::addq(CpuRegister dst, CpuRegister src) { 1568 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1569 // 0x01 is addq r/m64 <- r/m64 + r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 1570 EmitRex64(src, dst); 1571 EmitUint8(0x01); 1572 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1573} 1574 1575 1576void X86_64Assembler::addl(const Address& address, CpuRegister reg) { 1577 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1578 EmitOptionalRex32(reg, address); 1579 EmitUint8(0x01); 1580 EmitOperand(reg.LowBits(), address); 1581} 1582 1583 1584void X86_64Assembler::addl(const Address& address, const Immediate& imm) { 1585 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1586 EmitOptionalRex32(address); 1587 EmitComplex(0, address, imm); 1588} 1589 1590 1591void X86_64Assembler::subl(CpuRegister dst, CpuRegister src) { 1592 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1593 EmitOptionalRex32(dst, src); 1594 EmitUint8(0x2B); 1595 EmitOperand(dst.LowBits(), Operand(src)); 1596} 1597 1598 1599void X86_64Assembler::subl(CpuRegister reg, const Immediate& imm) { 1600 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1601 EmitOptionalRex32(reg); 1602 EmitComplex(5, Operand(reg), imm); 1603} 1604 1605 1606void X86_64Assembler::subq(CpuRegister reg, const Immediate& imm) { 1607 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1608 CHECK(imm.is_int32()); // subq only supports 32b immediate. 1609 EmitRex64(reg); 1610 EmitComplex(5, Operand(reg), imm); 1611} 1612 1613 1614void X86_64Assembler::subq(CpuRegister dst, CpuRegister src) { 1615 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1616 EmitRex64(dst, src); 1617 EmitUint8(0x2B); 1618 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1619} 1620 1621 1622void X86_64Assembler::subq(CpuRegister reg, const Address& address) { 1623 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1624 EmitRex64(reg); 1625 EmitUint8(0x2B); 1626 EmitOperand(reg.LowBits() & 7, address); 1627} 1628 1629 1630void X86_64Assembler::subl(CpuRegister reg, const Address& address) { 1631 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1632 EmitOptionalRex32(reg, address); 1633 EmitUint8(0x2B); 1634 EmitOperand(reg.LowBits(), address); 1635} 1636 1637 1638void X86_64Assembler::cdq() { 1639 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1640 EmitUint8(0x99); 1641} 1642 1643 1644void X86_64Assembler::cqo() { 1645 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1646 EmitRex64(); 1647 EmitUint8(0x99); 1648} 1649 1650 1651void X86_64Assembler::idivl(CpuRegister reg) { 1652 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1653 EmitOptionalRex32(reg); 1654 EmitUint8(0xF7); 1655 EmitUint8(0xF8 | reg.LowBits()); 1656} 1657 1658 1659void X86_64Assembler::idivq(CpuRegister reg) { 1660 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1661 EmitRex64(reg); 1662 EmitUint8(0xF7); 1663 EmitUint8(0xF8 | reg.LowBits()); 1664} 1665 1666 1667void X86_64Assembler::imull(CpuRegister dst, CpuRegister src) { 1668 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1669 EmitOptionalRex32(dst, src); 1670 EmitUint8(0x0F); 1671 EmitUint8(0xAF); 1672 EmitOperand(dst.LowBits(), Operand(src)); 1673} 1674 1675void X86_64Assembler::imull(CpuRegister reg, const Immediate& imm) { 1676 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1677 CHECK(imm.is_int32()); // imull only supports 32b immediate. 1678 1679 EmitOptionalRex32(reg, reg); 1680 1681 // See whether imm can be represented as a sign-extended 8bit value. 1682 int32_t v32 = static_cast<int32_t>(imm.value()); 1683 if (IsInt<8>(v32)) { 1684 // Sign-extension works. 1685 EmitUint8(0x6B); 1686 EmitOperand(reg.LowBits(), Operand(reg)); 1687 EmitUint8(static_cast<uint8_t>(v32 & 0xFF)); 1688 } else { 1689 // Not representable, use full immediate. 1690 EmitUint8(0x69); 1691 EmitOperand(reg.LowBits(), Operand(reg)); 1692 EmitImmediate(imm); 1693 } 1694} 1695 1696 1697void X86_64Assembler::imull(CpuRegister reg, const Address& address) { 1698 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1699 EmitOptionalRex32(reg, address); 1700 EmitUint8(0x0F); 1701 EmitUint8(0xAF); 1702 EmitOperand(reg.LowBits(), address); 1703} 1704 1705 1706void X86_64Assembler::imulq(CpuRegister dst, CpuRegister src) { 1707 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1708 EmitRex64(dst, src); 1709 EmitUint8(0x0F); 1710 EmitUint8(0xAF); 1711 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1712} 1713 1714 1715void X86_64Assembler::imulq(CpuRegister reg, const Immediate& imm) { 1716 imulq(reg, reg, imm); 1717} 1718 1719void X86_64Assembler::imulq(CpuRegister dst, CpuRegister reg, const Immediate& imm) { 1720 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1721 CHECK(imm.is_int32()); // imulq only supports 32b immediate. 1722 1723 EmitRex64(dst, reg); 1724 1725 // See whether imm can be represented as a sign-extended 8bit value. 1726 int64_t v64 = imm.value(); 1727 if (IsInt<8>(v64)) { 1728 // Sign-extension works. 1729 EmitUint8(0x6B); 1730 EmitOperand(dst.LowBits(), Operand(reg)); 1731 EmitUint8(static_cast<uint8_t>(v64 & 0xFF)); 1732 } else { 1733 // Not representable, use full immediate. 1734 EmitUint8(0x69); 1735 EmitOperand(dst.LowBits(), Operand(reg)); 1736 EmitImmediate(imm); 1737 } 1738} 1739 1740void X86_64Assembler::imulq(CpuRegister reg, const Address& address) { 1741 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1742 EmitRex64(reg, address); 1743 EmitUint8(0x0F); 1744 EmitUint8(0xAF); 1745 EmitOperand(reg.LowBits(), address); 1746} 1747 1748 1749void X86_64Assembler::imull(CpuRegister reg) { 1750 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1751 EmitOptionalRex32(reg); 1752 EmitUint8(0xF7); 1753 EmitOperand(5, Operand(reg)); 1754} 1755 1756 1757void X86_64Assembler::imulq(CpuRegister reg) { 1758 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1759 EmitRex64(reg); 1760 EmitUint8(0xF7); 1761 EmitOperand(5, Operand(reg)); 1762} 1763 1764 1765void X86_64Assembler::imull(const Address& address) { 1766 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1767 EmitOptionalRex32(address); 1768 EmitUint8(0xF7); 1769 EmitOperand(5, address); 1770} 1771 1772 1773void X86_64Assembler::mull(CpuRegister reg) { 1774 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1775 EmitOptionalRex32(reg); 1776 EmitUint8(0xF7); 1777 EmitOperand(4, Operand(reg)); 1778} 1779 1780 1781void X86_64Assembler::mull(const Address& address) { 1782 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1783 EmitOptionalRex32(address); 1784 EmitUint8(0xF7); 1785 EmitOperand(4, address); 1786} 1787 1788 1789void X86_64Assembler::shll(CpuRegister reg, const Immediate& imm) { 1790 EmitGenericShift(false, 4, reg, imm); 1791} 1792 1793 1794void X86_64Assembler::shlq(CpuRegister reg, const Immediate& imm) { 1795 EmitGenericShift(true, 4, reg, imm); 1796} 1797 1798 1799void X86_64Assembler::shll(CpuRegister operand, CpuRegister shifter) { 1800 EmitGenericShift(false, 4, operand, shifter); 1801} 1802 1803 1804void X86_64Assembler::shlq(CpuRegister operand, CpuRegister shifter) { 1805 EmitGenericShift(true, 4, operand, shifter); 1806} 1807 1808 1809void X86_64Assembler::shrl(CpuRegister reg, const Immediate& imm) { 1810 EmitGenericShift(false, 5, reg, imm); 1811} 1812 1813 1814void X86_64Assembler::shrq(CpuRegister reg, const Immediate& imm) { 1815 EmitGenericShift(true, 5, reg, imm); 1816} 1817 1818 1819void X86_64Assembler::shrl(CpuRegister operand, CpuRegister shifter) { 1820 EmitGenericShift(false, 5, operand, shifter); 1821} 1822 1823 1824void X86_64Assembler::shrq(CpuRegister operand, CpuRegister shifter) { 1825 EmitGenericShift(true, 5, operand, shifter); 1826} 1827 1828 1829void X86_64Assembler::sarl(CpuRegister reg, const Immediate& imm) { 1830 EmitGenericShift(false, 7, reg, imm); 1831} 1832 1833 1834void X86_64Assembler::sarl(CpuRegister operand, CpuRegister shifter) { 1835 EmitGenericShift(false, 7, operand, shifter); 1836} 1837 1838 1839void X86_64Assembler::sarq(CpuRegister reg, const Immediate& imm) { 1840 EmitGenericShift(true, 7, reg, imm); 1841} 1842 1843 1844void X86_64Assembler::sarq(CpuRegister operand, CpuRegister shifter) { 1845 EmitGenericShift(true, 7, operand, shifter); 1846} 1847 1848 1849void X86_64Assembler::negl(CpuRegister reg) { 1850 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1851 EmitOptionalRex32(reg); 1852 EmitUint8(0xF7); 1853 EmitOperand(3, Operand(reg)); 1854} 1855 1856 1857void X86_64Assembler::negq(CpuRegister reg) { 1858 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1859 EmitRex64(reg); 1860 EmitUint8(0xF7); 1861 EmitOperand(3, Operand(reg)); 1862} 1863 1864 1865void X86_64Assembler::notl(CpuRegister reg) { 1866 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1867 EmitOptionalRex32(reg); 1868 EmitUint8(0xF7); 1869 EmitUint8(0xD0 | reg.LowBits()); 1870} 1871 1872 1873void X86_64Assembler::notq(CpuRegister reg) { 1874 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1875 EmitRex64(reg); 1876 EmitUint8(0xF7); 1877 EmitOperand(2, Operand(reg)); 1878} 1879 1880 1881void X86_64Assembler::enter(const Immediate& imm) { 1882 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1883 EmitUint8(0xC8); 1884 CHECK(imm.is_uint16()) << imm.value(); 1885 EmitUint8(imm.value() & 0xFF); 1886 EmitUint8((imm.value() >> 8) & 0xFF); 1887 EmitUint8(0x00); 1888} 1889 1890 1891void X86_64Assembler::leave() { 1892 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1893 EmitUint8(0xC9); 1894} 1895 1896 1897void X86_64Assembler::ret() { 1898 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1899 EmitUint8(0xC3); 1900} 1901 1902 1903void X86_64Assembler::ret(const Immediate& imm) { 1904 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1905 EmitUint8(0xC2); 1906 CHECK(imm.is_uint16()); 1907 EmitUint8(imm.value() & 0xFF); 1908 EmitUint8((imm.value() >> 8) & 0xFF); 1909} 1910 1911 1912 1913void X86_64Assembler::nop() { 1914 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1915 EmitUint8(0x90); 1916} 1917 1918 1919void X86_64Assembler::int3() { 1920 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1921 EmitUint8(0xCC); 1922} 1923 1924 1925void X86_64Assembler::hlt() { 1926 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1927 EmitUint8(0xF4); 1928} 1929 1930 1931void X86_64Assembler::j(Condition condition, Label* label) { 1932 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1933 if (label->IsBound()) { 1934 static const int kShortSize = 2; 1935 static const int kLongSize = 6; 1936 int offset = label->Position() - buffer_.Size(); 1937 CHECK_LE(offset, 0); 1938 if (IsInt<8>(offset - kShortSize)) { 1939 EmitUint8(0x70 + condition); 1940 EmitUint8((offset - kShortSize) & 0xFF); 1941 } else { 1942 EmitUint8(0x0F); 1943 EmitUint8(0x80 + condition); 1944 EmitInt32(offset - kLongSize); 1945 } 1946 } else { 1947 EmitUint8(0x0F); 1948 EmitUint8(0x80 + condition); 1949 EmitLabelLink(label); 1950 } 1951} 1952 1953 1954void X86_64Assembler::jmp(CpuRegister reg) { 1955 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1956 EmitOptionalRex32(reg); 1957 EmitUint8(0xFF); 1958 EmitRegisterOperand(4, reg.LowBits()); 1959} 1960 1961void X86_64Assembler::jmp(const Address& address) { 1962 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1963 EmitOptionalRex32(address); 1964 EmitUint8(0xFF); 1965 EmitOperand(4, address); 1966} 1967 1968void X86_64Assembler::jmp(Label* label) { 1969 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1970 if (label->IsBound()) { 1971 static const int kShortSize = 2; 1972 static const int kLongSize = 5; 1973 int offset = label->Position() - buffer_.Size(); 1974 CHECK_LE(offset, 0); 1975 if (IsInt<8>(offset - kShortSize)) { 1976 EmitUint8(0xEB); 1977 EmitUint8((offset - kShortSize) & 0xFF); 1978 } else { 1979 EmitUint8(0xE9); 1980 EmitInt32(offset - kLongSize); 1981 } 1982 } else { 1983 EmitUint8(0xE9); 1984 EmitLabelLink(label); 1985 } 1986} 1987 1988 1989X86_64Assembler* X86_64Assembler::lock() { 1990 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1991 EmitUint8(0xF0); 1992 return this; 1993} 1994 1995 1996void X86_64Assembler::cmpxchgl(const Address& address, CpuRegister reg) { 1997 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1998 EmitOptionalRex32(reg, address); 1999 EmitUint8(0x0F); 2000 EmitUint8(0xB1); 2001 EmitOperand(reg.LowBits(), address); 2002} 2003 2004 2005void X86_64Assembler::cmpxchgq(const Address& address, CpuRegister reg) { 2006 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2007 EmitRex64(reg, address); 2008 EmitUint8(0x0F); 2009 EmitUint8(0xB1); 2010 EmitOperand(reg.LowBits(), address); 2011} 2012 2013 2014void X86_64Assembler::mfence() { 2015 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2016 EmitUint8(0x0F); 2017 EmitUint8(0xAE); 2018 EmitUint8(0xF0); 2019} 2020 2021 2022X86_64Assembler* X86_64Assembler::gs() { 2023 // TODO: gs is a prefix and not an instruction 2024 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2025 EmitUint8(0x65); 2026 return this; 2027} 2028 2029 2030void X86_64Assembler::AddImmediate(CpuRegister reg, const Immediate& imm) { 2031 int value = imm.value(); 2032 if (value != 0) { 2033 if (value > 0) { 2034 addl(reg, imm); 2035 } else { 2036 subl(reg, Immediate(value)); 2037 } 2038 } 2039} 2040 2041 2042void X86_64Assembler::setcc(Condition condition, CpuRegister dst) { 2043 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2044 // RSP, RBP, RDI, RSI need rex prefix (else the pattern encodes ah/bh/ch/dh). 2045 if (dst.NeedsRex() || dst.AsRegister() > 3) { 2046 EmitOptionalRex(true, false, false, false, dst.NeedsRex()); 2047 } 2048 EmitUint8(0x0F); 2049 EmitUint8(0x90 + condition); 2050 EmitUint8(0xC0 + dst.LowBits()); 2051} 2052 2053void X86_64Assembler::bswapl(CpuRegister dst) { 2054 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2055 EmitOptionalRex(false, false, false, false, dst.NeedsRex()); 2056 EmitUint8(0x0F); 2057 EmitUint8(0xC8 + dst.LowBits()); 2058} 2059 2060void X86_64Assembler::bswapq(CpuRegister dst) { 2061 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2062 EmitOptionalRex(false, true, false, false, dst.NeedsRex()); 2063 EmitUint8(0x0F); 2064 EmitUint8(0xC8 + dst.LowBits()); 2065} 2066 2067 2068void X86_64Assembler::LoadDoubleConstant(XmmRegister dst, double value) { 2069 // TODO: Need to have a code constants table. 2070 int64_t constant = bit_cast<int64_t, double>(value); 2071 pushq(Immediate(High32Bits(constant))); 2072 pushq(Immediate(Low32Bits(constant))); 2073 movsd(dst, Address(CpuRegister(RSP), 0)); 2074 addq(CpuRegister(RSP), Immediate(2 * sizeof(intptr_t))); 2075} 2076 2077 2078void X86_64Assembler::Align(int alignment, int offset) { 2079 CHECK(IsPowerOfTwo(alignment)); 2080 // Emit nop instruction until the real position is aligned. 2081 while (((offset + buffer_.GetPosition()) & (alignment-1)) != 0) { 2082 nop(); 2083 } 2084} 2085 2086 2087void X86_64Assembler::Bind(Label* label) { 2088 int bound = buffer_.Size(); 2089 CHECK(!label->IsBound()); // Labels can only be bound once. 2090 while (label->IsLinked()) { 2091 int position = label->LinkPosition(); 2092 int next = buffer_.Load<int32_t>(position); 2093 buffer_.Store<int32_t>(position, bound - (position + 4)); 2094 label->position_ = next; 2095 } 2096 label->BindTo(bound); 2097} 2098 2099 2100void X86_64Assembler::EmitOperand(uint8_t reg_or_opcode, const Operand& operand) { 2101 CHECK_GE(reg_or_opcode, 0); 2102 CHECK_LT(reg_or_opcode, 8); 2103 const int length = operand.length_; 2104 CHECK_GT(length, 0); 2105 // Emit the ModRM byte updated with the given reg value. 2106 CHECK_EQ(operand.encoding_[0] & 0x38, 0); 2107 EmitUint8(operand.encoding_[0] + (reg_or_opcode << 3)); 2108 // Emit the rest of the encoded operand. 2109 for (int i = 1; i < length; i++) { 2110 EmitUint8(operand.encoding_[i]); 2111 } 2112 AssemblerFixup* fixup = operand.GetFixup(); 2113 if (fixup != nullptr) { 2114 EmitFixup(fixup); 2115 } 2116} 2117 2118 2119void X86_64Assembler::EmitImmediate(const Immediate& imm) { 2120 if (imm.is_int32()) { 2121 EmitInt32(static_cast<int32_t>(imm.value())); 2122 } else { 2123 EmitInt64(imm.value()); 2124 } 2125} 2126 2127 2128void X86_64Assembler::EmitComplex(uint8_t reg_or_opcode, 2129 const Operand& operand, 2130 const Immediate& immediate) { 2131 CHECK_GE(reg_or_opcode, 0); 2132 CHECK_LT(reg_or_opcode, 8); 2133 if (immediate.is_int8()) { 2134 // Use sign-extended 8-bit immediate. 2135 EmitUint8(0x83); 2136 EmitOperand(reg_or_opcode, operand); 2137 EmitUint8(immediate.value() & 0xFF); 2138 } else if (operand.IsRegister(CpuRegister(RAX))) { 2139 // Use short form if the destination is eax. 2140 EmitUint8(0x05 + (reg_or_opcode << 3)); 2141 EmitImmediate(immediate); 2142 } else { 2143 EmitUint8(0x81); 2144 EmitOperand(reg_or_opcode, operand); 2145 EmitImmediate(immediate); 2146 } 2147} 2148 2149 2150void X86_64Assembler::EmitLabel(Label* label, int instruction_size) { 2151 if (label->IsBound()) { 2152 int offset = label->Position() - buffer_.Size(); 2153 CHECK_LE(offset, 0); 2154 EmitInt32(offset - instruction_size); 2155 } else { 2156 EmitLabelLink(label); 2157 } 2158} 2159 2160 2161void X86_64Assembler::EmitLabelLink(Label* label) { 2162 CHECK(!label->IsBound()); 2163 int position = buffer_.Size(); 2164 EmitInt32(label->position_); 2165 label->LinkTo(position); 2166} 2167 2168 2169void X86_64Assembler::EmitGenericShift(bool wide, 2170 int reg_or_opcode, 2171 CpuRegister reg, 2172 const Immediate& imm) { 2173 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2174 CHECK(imm.is_int8()); 2175 if (wide) { 2176 EmitRex64(reg); 2177 } else { 2178 EmitOptionalRex32(reg); 2179 } 2180 if (imm.value() == 1) { 2181 EmitUint8(0xD1); 2182 EmitOperand(reg_or_opcode, Operand(reg)); 2183 } else { 2184 EmitUint8(0xC1); 2185 EmitOperand(reg_or_opcode, Operand(reg)); 2186 EmitUint8(imm.value() & 0xFF); 2187 } 2188} 2189 2190 2191void X86_64Assembler::EmitGenericShift(bool wide, 2192 int reg_or_opcode, 2193 CpuRegister operand, 2194 CpuRegister shifter) { 2195 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2196 CHECK_EQ(shifter.AsRegister(), RCX); 2197 if (wide) { 2198 EmitRex64(operand); 2199 } else { 2200 EmitOptionalRex32(operand); 2201 } 2202 EmitUint8(0xD3); 2203 EmitOperand(reg_or_opcode, Operand(operand)); 2204} 2205 2206void X86_64Assembler::EmitOptionalRex(bool force, bool w, bool r, bool x, bool b) { 2207 // REX.WRXB 2208 // W - 64-bit operand 2209 // R - MODRM.reg 2210 // X - SIB.index 2211 // B - MODRM.rm/SIB.base 2212 uint8_t rex = force ? 0x40 : 0; 2213 if (w) { 2214 rex |= 0x48; // REX.W000 2215 } 2216 if (r) { 2217 rex |= 0x44; // REX.0R00 2218 } 2219 if (x) { 2220 rex |= 0x42; // REX.00X0 2221 } 2222 if (b) { 2223 rex |= 0x41; // REX.000B 2224 } 2225 if (rex != 0) { 2226 EmitUint8(rex); 2227 } 2228} 2229 2230void X86_64Assembler::EmitOptionalRex32(CpuRegister reg) { 2231 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 2232} 2233 2234void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, CpuRegister src) { 2235 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2236} 2237 2238void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, XmmRegister src) { 2239 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2240} 2241 2242void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, XmmRegister src) { 2243 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2244} 2245 2246void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, CpuRegister src) { 2247 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2248} 2249 2250void X86_64Assembler::EmitOptionalRex32(const Operand& operand) { 2251 uint8_t rex = operand.rex(); 2252 if (rex != 0) { 2253 EmitUint8(rex); 2254 } 2255} 2256 2257void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, const Operand& operand) { 2258 uint8_t rex = operand.rex(); 2259 if (dst.NeedsRex()) { 2260 rex |= 0x44; // REX.0R00 2261 } 2262 if (rex != 0) { 2263 EmitUint8(rex); 2264 } 2265} 2266 2267void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, const Operand& operand) { 2268 uint8_t rex = operand.rex(); 2269 if (dst.NeedsRex()) { 2270 rex |= 0x44; // REX.0R00 2271 } 2272 if (rex != 0) { 2273 EmitUint8(rex); 2274 } 2275} 2276 2277void X86_64Assembler::EmitRex64() { 2278 EmitOptionalRex(false, true, false, false, false); 2279} 2280 2281void X86_64Assembler::EmitRex64(CpuRegister reg) { 2282 EmitOptionalRex(false, true, false, false, reg.NeedsRex()); 2283} 2284 2285void X86_64Assembler::EmitRex64(const Operand& operand) { 2286 uint8_t rex = operand.rex(); 2287 rex |= 0x48; // REX.W000 2288 EmitUint8(rex); 2289} 2290 2291void X86_64Assembler::EmitRex64(CpuRegister dst, CpuRegister src) { 2292 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2293} 2294 2295void X86_64Assembler::EmitRex64(XmmRegister dst, CpuRegister src) { 2296 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2297} 2298 2299void X86_64Assembler::EmitRex64(CpuRegister dst, XmmRegister src) { 2300 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2301} 2302 2303void X86_64Assembler::EmitRex64(CpuRegister dst, const Operand& operand) { 2304 uint8_t rex = 0x48 | operand.rex(); // REX.W000 2305 if (dst.NeedsRex()) { 2306 rex |= 0x44; // REX.0R00 2307 } 2308 EmitUint8(rex); 2309} 2310 2311void X86_64Assembler::EmitRex64(XmmRegister dst, const Operand& operand) { 2312 uint8_t rex = 0x48 | operand.rex(); // REX.W000 2313 if (dst.NeedsRex()) { 2314 rex |= 0x44; // REX.0R00 2315 } 2316 EmitUint8(rex); 2317} 2318 2319void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, CpuRegister src) { 2320 // For src, SPL, BPL, SIL, DIL need the rex prefix. 2321 bool force = src.AsRegister() > 3; 2322 EmitOptionalRex(force, false, dst.NeedsRex(), false, src.NeedsRex()); 2323} 2324 2325void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand) { 2326 uint8_t rex = operand.rex(); 2327 // For dst, SPL, BPL, SIL, DIL need the rex prefix. 2328 bool force = dst.AsRegister() > 3; 2329 if (force) { 2330 rex |= 0x40; // REX.0000 2331 } 2332 if (dst.NeedsRex()) { 2333 rex |= 0x44; // REX.0R00 2334 } 2335 if (rex != 0) { 2336 EmitUint8(rex); 2337 } 2338} 2339 2340static dwarf::Reg DWARFReg(Register reg) { 2341 return dwarf::Reg::X86_64Core(static_cast<int>(reg)); 2342} 2343static dwarf::Reg DWARFReg(FloatRegister reg) { 2344 return dwarf::Reg::X86_64Fp(static_cast<int>(reg)); 2345} 2346 2347constexpr size_t kFramePointerSize = 8; 2348 2349void X86_64Assembler::BuildFrame(size_t frame_size, ManagedRegister method_reg, 2350 const std::vector<ManagedRegister>& spill_regs, 2351 const ManagedRegisterEntrySpills& entry_spills) { 2352 DCHECK_EQ(buffer_.Size(), 0U); // Nothing emitted yet. 2353 cfi_.SetCurrentCFAOffset(8); // Return address on stack. 2354 CHECK_ALIGNED(frame_size, kStackAlignment); 2355 int gpr_count = 0; 2356 for (int i = spill_regs.size() - 1; i >= 0; --i) { 2357 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2358 if (spill.IsCpuRegister()) { 2359 pushq(spill.AsCpuRegister()); 2360 gpr_count++; 2361 cfi_.AdjustCFAOffset(kFramePointerSize); 2362 cfi_.RelOffset(DWARFReg(spill.AsCpuRegister().AsRegister()), 0); 2363 } 2364 } 2365 // return address then method on stack. 2366 int64_t rest_of_frame = static_cast<int64_t>(frame_size) 2367 - (gpr_count * kFramePointerSize) 2368 - kFramePointerSize /*return address*/; 2369 subq(CpuRegister(RSP), Immediate(rest_of_frame)); 2370 cfi_.AdjustCFAOffset(rest_of_frame); 2371 2372 // spill xmms 2373 int64_t offset = rest_of_frame; 2374 for (int i = spill_regs.size() - 1; i >= 0; --i) { 2375 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2376 if (spill.IsXmmRegister()) { 2377 offset -= sizeof(double); 2378 movsd(Address(CpuRegister(RSP), offset), spill.AsXmmRegister()); 2379 cfi_.RelOffset(DWARFReg(spill.AsXmmRegister().AsFloatRegister()), offset); 2380 } 2381 } 2382 2383 DCHECK_EQ(4U, sizeof(StackReference<mirror::ArtMethod>)); 2384 2385 movl(Address(CpuRegister(RSP), 0), method_reg.AsX86_64().AsCpuRegister()); 2386 2387 for (size_t i = 0; i < entry_spills.size(); ++i) { 2388 ManagedRegisterSpill spill = entry_spills.at(i); 2389 if (spill.AsX86_64().IsCpuRegister()) { 2390 if (spill.getSize() == 8) { 2391 movq(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), 2392 spill.AsX86_64().AsCpuRegister()); 2393 } else { 2394 CHECK_EQ(spill.getSize(), 4); 2395 movl(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsCpuRegister()); 2396 } 2397 } else { 2398 if (spill.getSize() == 8) { 2399 movsd(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 2400 } else { 2401 CHECK_EQ(spill.getSize(), 4); 2402 movss(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 2403 } 2404 } 2405 } 2406} 2407 2408void X86_64Assembler::RemoveFrame(size_t frame_size, 2409 const std::vector<ManagedRegister>& spill_regs) { 2410 CHECK_ALIGNED(frame_size, kStackAlignment); 2411 cfi_.RememberState(); 2412 int gpr_count = 0; 2413 // unspill xmms 2414 int64_t offset = static_cast<int64_t>(frame_size) - (spill_regs.size() * kFramePointerSize) - 2 * kFramePointerSize; 2415 for (size_t i = 0; i < spill_regs.size(); ++i) { 2416 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2417 if (spill.IsXmmRegister()) { 2418 offset += sizeof(double); 2419 movsd(spill.AsXmmRegister(), Address(CpuRegister(RSP), offset)); 2420 cfi_.Restore(DWARFReg(spill.AsXmmRegister().AsFloatRegister())); 2421 } else { 2422 gpr_count++; 2423 } 2424 } 2425 int adjust = static_cast<int>(frame_size) - (gpr_count * kFramePointerSize) - kFramePointerSize; 2426 addq(CpuRegister(RSP), Immediate(adjust)); 2427 cfi_.AdjustCFAOffset(-adjust); 2428 for (size_t i = 0; i < spill_regs.size(); ++i) { 2429 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2430 if (spill.IsCpuRegister()) { 2431 popq(spill.AsCpuRegister()); 2432 cfi_.AdjustCFAOffset(-static_cast<int>(kFramePointerSize)); 2433 cfi_.Restore(DWARFReg(spill.AsCpuRegister().AsRegister())); 2434 } 2435 } 2436 ret(); 2437 // The CFI should be restored for any code that follows the exit block. 2438 cfi_.RestoreState(); 2439 cfi_.DefCFAOffset(frame_size); 2440} 2441 2442void X86_64Assembler::IncreaseFrameSize(size_t adjust) { 2443 CHECK_ALIGNED(adjust, kStackAlignment); 2444 addq(CpuRegister(RSP), Immediate(-static_cast<int64_t>(adjust))); 2445 cfi_.AdjustCFAOffset(adjust); 2446} 2447 2448void X86_64Assembler::DecreaseFrameSize(size_t adjust) { 2449 CHECK_ALIGNED(adjust, kStackAlignment); 2450 addq(CpuRegister(RSP), Immediate(adjust)); 2451 cfi_.AdjustCFAOffset(-adjust); 2452} 2453 2454void X86_64Assembler::Store(FrameOffset offs, ManagedRegister msrc, size_t size) { 2455 X86_64ManagedRegister src = msrc.AsX86_64(); 2456 if (src.IsNoRegister()) { 2457 CHECK_EQ(0u, size); 2458 } else if (src.IsCpuRegister()) { 2459 if (size == 4) { 2460 CHECK_EQ(4u, size); 2461 movl(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 2462 } else { 2463 CHECK_EQ(8u, size); 2464 movq(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 2465 } 2466 } else if (src.IsRegisterPair()) { 2467 CHECK_EQ(0u, size); 2468 movq(Address(CpuRegister(RSP), offs), src.AsRegisterPairLow()); 2469 movq(Address(CpuRegister(RSP), FrameOffset(offs.Int32Value()+4)), 2470 src.AsRegisterPairHigh()); 2471 } else if (src.IsX87Register()) { 2472 if (size == 4) { 2473 fstps(Address(CpuRegister(RSP), offs)); 2474 } else { 2475 fstpl(Address(CpuRegister(RSP), offs)); 2476 } 2477 } else { 2478 CHECK(src.IsXmmRegister()); 2479 if (size == 4) { 2480 movss(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 2481 } else { 2482 movsd(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 2483 } 2484 } 2485} 2486 2487void X86_64Assembler::StoreRef(FrameOffset dest, ManagedRegister msrc) { 2488 X86_64ManagedRegister src = msrc.AsX86_64(); 2489 CHECK(src.IsCpuRegister()); 2490 movl(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 2491} 2492 2493void X86_64Assembler::StoreRawPtr(FrameOffset dest, ManagedRegister msrc) { 2494 X86_64ManagedRegister src = msrc.AsX86_64(); 2495 CHECK(src.IsCpuRegister()); 2496 movq(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 2497} 2498 2499void X86_64Assembler::StoreImmediateToFrame(FrameOffset dest, uint32_t imm, 2500 ManagedRegister) { 2501 movl(Address(CpuRegister(RSP), dest), Immediate(imm)); // TODO(64) movq? 2502} 2503 2504void X86_64Assembler::StoreImmediateToThread64(ThreadOffset<8> dest, uint32_t imm, 2505 ManagedRegister) { 2506 gs()->movl(Address::Absolute(dest, true), Immediate(imm)); // TODO(64) movq? 2507} 2508 2509void X86_64Assembler::StoreStackOffsetToThread64(ThreadOffset<8> thr_offs, 2510 FrameOffset fr_offs, 2511 ManagedRegister mscratch) { 2512 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2513 CHECK(scratch.IsCpuRegister()); 2514 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), fr_offs)); 2515 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2516} 2517 2518void X86_64Assembler::StoreStackPointerToThread64(ThreadOffset<8> thr_offs) { 2519 gs()->movq(Address::Absolute(thr_offs, true), CpuRegister(RSP)); 2520} 2521 2522void X86_64Assembler::StoreSpanning(FrameOffset /*dst*/, ManagedRegister /*src*/, 2523 FrameOffset /*in_off*/, ManagedRegister /*scratch*/) { 2524 UNIMPLEMENTED(FATAL); // this case only currently exists for ARM 2525} 2526 2527void X86_64Assembler::Load(ManagedRegister mdest, FrameOffset src, size_t size) { 2528 X86_64ManagedRegister dest = mdest.AsX86_64(); 2529 if (dest.IsNoRegister()) { 2530 CHECK_EQ(0u, size); 2531 } else if (dest.IsCpuRegister()) { 2532 if (size == 4) { 2533 CHECK_EQ(4u, size); 2534 movl(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2535 } else { 2536 CHECK_EQ(8u, size); 2537 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2538 } 2539 } else if (dest.IsRegisterPair()) { 2540 CHECK_EQ(0u, size); 2541 movq(dest.AsRegisterPairLow(), Address(CpuRegister(RSP), src)); 2542 movq(dest.AsRegisterPairHigh(), Address(CpuRegister(RSP), FrameOffset(src.Int32Value()+4))); 2543 } else if (dest.IsX87Register()) { 2544 if (size == 4) { 2545 flds(Address(CpuRegister(RSP), src)); 2546 } else { 2547 fldl(Address(CpuRegister(RSP), src)); 2548 } 2549 } else { 2550 CHECK(dest.IsXmmRegister()); 2551 if (size == 4) { 2552 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2553 } else { 2554 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2555 } 2556 } 2557} 2558 2559void X86_64Assembler::LoadFromThread64(ManagedRegister mdest, ThreadOffset<8> src, size_t size) { 2560 X86_64ManagedRegister dest = mdest.AsX86_64(); 2561 if (dest.IsNoRegister()) { 2562 CHECK_EQ(0u, size); 2563 } else if (dest.IsCpuRegister()) { 2564 CHECK_EQ(4u, size); 2565 gs()->movl(dest.AsCpuRegister(), Address::Absolute(src, true)); 2566 } else if (dest.IsRegisterPair()) { 2567 CHECK_EQ(8u, size); 2568 gs()->movq(dest.AsRegisterPairLow(), Address::Absolute(src, true)); 2569 } else if (dest.IsX87Register()) { 2570 if (size == 4) { 2571 gs()->flds(Address::Absolute(src, true)); 2572 } else { 2573 gs()->fldl(Address::Absolute(src, true)); 2574 } 2575 } else { 2576 CHECK(dest.IsXmmRegister()); 2577 if (size == 4) { 2578 gs()->movss(dest.AsXmmRegister(), Address::Absolute(src, true)); 2579 } else { 2580 gs()->movsd(dest.AsXmmRegister(), Address::Absolute(src, true)); 2581 } 2582 } 2583} 2584 2585void X86_64Assembler::LoadRef(ManagedRegister mdest, FrameOffset src) { 2586 X86_64ManagedRegister dest = mdest.AsX86_64(); 2587 CHECK(dest.IsCpuRegister()); 2588 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2589} 2590 2591void X86_64Assembler::LoadRef(ManagedRegister mdest, ManagedRegister base, 2592 MemberOffset offs) { 2593 X86_64ManagedRegister dest = mdest.AsX86_64(); 2594 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2595 movl(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2596 if (kPoisonHeapReferences) { 2597 negl(dest.AsCpuRegister()); 2598 } 2599} 2600 2601void X86_64Assembler::LoadRawPtr(ManagedRegister mdest, ManagedRegister base, 2602 Offset offs) { 2603 X86_64ManagedRegister dest = mdest.AsX86_64(); 2604 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2605 movq(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2606} 2607 2608void X86_64Assembler::LoadRawPtrFromThread64(ManagedRegister mdest, ThreadOffset<8> offs) { 2609 X86_64ManagedRegister dest = mdest.AsX86_64(); 2610 CHECK(dest.IsCpuRegister()); 2611 gs()->movq(dest.AsCpuRegister(), Address::Absolute(offs, true)); 2612} 2613 2614void X86_64Assembler::SignExtend(ManagedRegister mreg, size_t size) { 2615 X86_64ManagedRegister reg = mreg.AsX86_64(); 2616 CHECK(size == 1 || size == 2) << size; 2617 CHECK(reg.IsCpuRegister()) << reg; 2618 if (size == 1) { 2619 movsxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2620 } else { 2621 movsxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2622 } 2623} 2624 2625void X86_64Assembler::ZeroExtend(ManagedRegister mreg, size_t size) { 2626 X86_64ManagedRegister reg = mreg.AsX86_64(); 2627 CHECK(size == 1 || size == 2) << size; 2628 CHECK(reg.IsCpuRegister()) << reg; 2629 if (size == 1) { 2630 movzxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2631 } else { 2632 movzxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2633 } 2634} 2635 2636void X86_64Assembler::Move(ManagedRegister mdest, ManagedRegister msrc, size_t size) { 2637 X86_64ManagedRegister dest = mdest.AsX86_64(); 2638 X86_64ManagedRegister src = msrc.AsX86_64(); 2639 if (!dest.Equals(src)) { 2640 if (dest.IsCpuRegister() && src.IsCpuRegister()) { 2641 movq(dest.AsCpuRegister(), src.AsCpuRegister()); 2642 } else if (src.IsX87Register() && dest.IsXmmRegister()) { 2643 // Pass via stack and pop X87 register 2644 subl(CpuRegister(RSP), Immediate(16)); 2645 if (size == 4) { 2646 CHECK_EQ(src.AsX87Register(), ST0); 2647 fstps(Address(CpuRegister(RSP), 0)); 2648 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2649 } else { 2650 CHECK_EQ(src.AsX87Register(), ST0); 2651 fstpl(Address(CpuRegister(RSP), 0)); 2652 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2653 } 2654 addq(CpuRegister(RSP), Immediate(16)); 2655 } else { 2656 // TODO: x87, SSE 2657 UNIMPLEMENTED(FATAL) << ": Move " << dest << ", " << src; 2658 } 2659 } 2660} 2661 2662void X86_64Assembler::CopyRef(FrameOffset dest, FrameOffset src, 2663 ManagedRegister mscratch) { 2664 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2665 CHECK(scratch.IsCpuRegister()); 2666 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2667 movl(Address(CpuRegister(RSP), dest), scratch.AsCpuRegister()); 2668} 2669 2670void X86_64Assembler::CopyRawPtrFromThread64(FrameOffset fr_offs, 2671 ThreadOffset<8> thr_offs, 2672 ManagedRegister mscratch) { 2673 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2674 CHECK(scratch.IsCpuRegister()); 2675 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(thr_offs, true)); 2676 Store(fr_offs, scratch, 8); 2677} 2678 2679void X86_64Assembler::CopyRawPtrToThread64(ThreadOffset<8> thr_offs, 2680 FrameOffset fr_offs, 2681 ManagedRegister mscratch) { 2682 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2683 CHECK(scratch.IsCpuRegister()); 2684 Load(scratch, fr_offs, 8); 2685 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2686} 2687 2688void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src, 2689 ManagedRegister mscratch, 2690 size_t size) { 2691 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2692 if (scratch.IsCpuRegister() && size == 8) { 2693 Load(scratch, src, 4); 2694 Store(dest, scratch, 4); 2695 Load(scratch, FrameOffset(src.Int32Value() + 4), 4); 2696 Store(FrameOffset(dest.Int32Value() + 4), scratch, 4); 2697 } else { 2698 Load(scratch, src, size); 2699 Store(dest, scratch, size); 2700 } 2701} 2702 2703void X86_64Assembler::Copy(FrameOffset /*dst*/, ManagedRegister /*src_base*/, Offset /*src_offset*/, 2704 ManagedRegister /*scratch*/, size_t /*size*/) { 2705 UNIMPLEMENTED(FATAL); 2706} 2707 2708void X86_64Assembler::Copy(ManagedRegister dest_base, Offset dest_offset, FrameOffset src, 2709 ManagedRegister scratch, size_t size) { 2710 CHECK(scratch.IsNoRegister()); 2711 CHECK_EQ(size, 4u); 2712 pushq(Address(CpuRegister(RSP), src)); 2713 popq(Address(dest_base.AsX86_64().AsCpuRegister(), dest_offset)); 2714} 2715 2716void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src_base, Offset src_offset, 2717 ManagedRegister mscratch, size_t size) { 2718 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2719 CHECK_EQ(size, 4u); 2720 movq(scratch, Address(CpuRegister(RSP), src_base)); 2721 movq(scratch, Address(scratch, src_offset)); 2722 movq(Address(CpuRegister(RSP), dest), scratch); 2723} 2724 2725void X86_64Assembler::Copy(ManagedRegister dest, Offset dest_offset, 2726 ManagedRegister src, Offset src_offset, 2727 ManagedRegister scratch, size_t size) { 2728 CHECK_EQ(size, 4u); 2729 CHECK(scratch.IsNoRegister()); 2730 pushq(Address(src.AsX86_64().AsCpuRegister(), src_offset)); 2731 popq(Address(dest.AsX86_64().AsCpuRegister(), dest_offset)); 2732} 2733 2734void X86_64Assembler::Copy(FrameOffset dest, Offset dest_offset, FrameOffset src, Offset src_offset, 2735 ManagedRegister mscratch, size_t size) { 2736 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2737 CHECK_EQ(size, 4u); 2738 CHECK_EQ(dest.Int32Value(), src.Int32Value()); 2739 movq(scratch, Address(CpuRegister(RSP), src)); 2740 pushq(Address(scratch, src_offset)); 2741 popq(Address(scratch, dest_offset)); 2742} 2743 2744void X86_64Assembler::MemoryBarrier(ManagedRegister) { 2745 mfence(); 2746} 2747 2748void X86_64Assembler::CreateHandleScopeEntry(ManagedRegister mout_reg, 2749 FrameOffset handle_scope_offset, 2750 ManagedRegister min_reg, bool null_allowed) { 2751 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2752 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2753 if (in_reg.IsNoRegister()) { // TODO(64): && null_allowed 2754 // Use out_reg as indicator of NULL 2755 in_reg = out_reg; 2756 // TODO: movzwl 2757 movl(in_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2758 } 2759 CHECK(in_reg.IsCpuRegister()); 2760 CHECK(out_reg.IsCpuRegister()); 2761 VerifyObject(in_reg, null_allowed); 2762 if (null_allowed) { 2763 Label null_arg; 2764 if (!out_reg.Equals(in_reg)) { 2765 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2766 } 2767 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2768 j(kZero, &null_arg); 2769 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2770 Bind(&null_arg); 2771 } else { 2772 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2773 } 2774} 2775 2776void X86_64Assembler::CreateHandleScopeEntry(FrameOffset out_off, 2777 FrameOffset handle_scope_offset, 2778 ManagedRegister mscratch, 2779 bool null_allowed) { 2780 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2781 CHECK(scratch.IsCpuRegister()); 2782 if (null_allowed) { 2783 Label null_arg; 2784 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2785 testl(scratch.AsCpuRegister(), scratch.AsCpuRegister()); 2786 j(kZero, &null_arg); 2787 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2788 Bind(&null_arg); 2789 } else { 2790 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2791 } 2792 Store(out_off, scratch, 8); 2793} 2794 2795// Given a handle scope entry, load the associated reference. 2796void X86_64Assembler::LoadReferenceFromHandleScope(ManagedRegister mout_reg, 2797 ManagedRegister min_reg) { 2798 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2799 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2800 CHECK(out_reg.IsCpuRegister()); 2801 CHECK(in_reg.IsCpuRegister()); 2802 Label null_arg; 2803 if (!out_reg.Equals(in_reg)) { 2804 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2805 } 2806 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2807 j(kZero, &null_arg); 2808 movq(out_reg.AsCpuRegister(), Address(in_reg.AsCpuRegister(), 0)); 2809 Bind(&null_arg); 2810} 2811 2812void X86_64Assembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) { 2813 // TODO: not validating references 2814} 2815 2816void X86_64Assembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) { 2817 // TODO: not validating references 2818} 2819 2820void X86_64Assembler::Call(ManagedRegister mbase, Offset offset, ManagedRegister) { 2821 X86_64ManagedRegister base = mbase.AsX86_64(); 2822 CHECK(base.IsCpuRegister()); 2823 call(Address(base.AsCpuRegister(), offset.Int32Value())); 2824 // TODO: place reference map on call 2825} 2826 2827void X86_64Assembler::Call(FrameOffset base, Offset offset, ManagedRegister mscratch) { 2828 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2829 movl(scratch, Address(CpuRegister(RSP), base)); 2830 call(Address(scratch, offset)); 2831} 2832 2833void X86_64Assembler::CallFromThread64(ThreadOffset<8> offset, ManagedRegister /*mscratch*/) { 2834 gs()->call(Address::Absolute(offset, true)); 2835} 2836 2837void X86_64Assembler::GetCurrentThread(ManagedRegister tr) { 2838 gs()->movq(tr.AsX86_64().AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2839} 2840 2841void X86_64Assembler::GetCurrentThread(FrameOffset offset, ManagedRegister mscratch) { 2842 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2843 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2844 movq(Address(CpuRegister(RSP), offset), scratch.AsCpuRegister()); 2845} 2846 2847// Slowpath entered when Thread::Current()->_exception is non-null 2848class X86_64ExceptionSlowPath FINAL : public SlowPath { 2849 public: 2850 explicit X86_64ExceptionSlowPath(size_t stack_adjust) : stack_adjust_(stack_adjust) {} 2851 virtual void Emit(Assembler *sp_asm) OVERRIDE; 2852 private: 2853 const size_t stack_adjust_; 2854}; 2855 2856void X86_64Assembler::ExceptionPoll(ManagedRegister /*scratch*/, size_t stack_adjust) { 2857 X86_64ExceptionSlowPath* slow = new X86_64ExceptionSlowPath(stack_adjust); 2858 buffer_.EnqueueSlowPath(slow); 2859 gs()->cmpl(Address::Absolute(Thread::ExceptionOffset<8>(), true), Immediate(0)); 2860 j(kNotEqual, slow->Entry()); 2861} 2862 2863void X86_64ExceptionSlowPath::Emit(Assembler *sasm) { 2864 X86_64Assembler* sp_asm = down_cast<X86_64Assembler*>(sasm); 2865#define __ sp_asm-> 2866 __ Bind(&entry_); 2867 // Note: the return value is dead 2868 if (stack_adjust_ != 0) { // Fix up the frame. 2869 __ DecreaseFrameSize(stack_adjust_); 2870 } 2871 // Pass exception as argument in RDI 2872 __ gs()->movq(CpuRegister(RDI), Address::Absolute(Thread::ExceptionOffset<8>(), true)); 2873 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(8, pDeliverException), true)); 2874 // this call should never return 2875 __ int3(); 2876#undef __ 2877} 2878 2879void X86_64Assembler::AddConstantArea() { 2880 const std::vector<int32_t>& area = constant_area_.GetBuffer(); 2881 for (size_t i = 0, e = area.size(); i < e; i++) { 2882 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2883 EmitInt32(area[i]); 2884 } 2885} 2886 2887int ConstantArea::AddInt32(int32_t v) { 2888 for (size_t i = 0, e = buffer_.size(); i < e; i++) { 2889 if (v == buffer_[i]) { 2890 return i * elem_size_; 2891 } 2892 } 2893 2894 // Didn't match anything. 2895 int result = buffer_.size() * elem_size_; 2896 buffer_.push_back(v); 2897 return result; 2898} 2899 2900int ConstantArea::AddInt64(int64_t v) { 2901 int32_t v_low = v; 2902 int32_t v_high = v >> 32; 2903 if (buffer_.size() > 1) { 2904 // Ensure we don't pass the end of the buffer. 2905 for (size_t i = 0, e = buffer_.size() - 1; i < e; i++) { 2906 if (v_low == buffer_[i] && v_high == buffer_[i + 1]) { 2907 return i * elem_size_; 2908 } 2909 } 2910 } 2911 2912 // Didn't match anything. 2913 int result = buffer_.size() * elem_size_; 2914 buffer_.push_back(v_low); 2915 buffer_.push_back(v_high); 2916 return result; 2917} 2918 2919int ConstantArea::AddDouble(double v) { 2920 // Treat the value as a 64-bit integer value. 2921 return AddInt64(bit_cast<int64_t, double>(v)); 2922} 2923 2924int ConstantArea::AddFloat(float v) { 2925 // Treat the value as a 32-bit integer value. 2926 return AddInt32(bit_cast<int32_t, float>(v)); 2927} 2928 2929} // namespace x86_64 2930} // namespace art 2931