assembler_x86_64.cc revision 0f88e87085b7cf6544dadff3f555773966a6853e
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "assembler_x86_64.h" 18 19#include "base/casts.h" 20#include "entrypoints/quick/quick_entrypoints.h" 21#include "memory_region.h" 22#include "thread.h" 23#include "utils/dwarf_cfi.h" 24 25namespace art { 26namespace x86_64 { 27 28std::ostream& operator<<(std::ostream& os, const CpuRegister& reg) { 29 return os << reg.AsRegister(); 30} 31 32std::ostream& operator<<(std::ostream& os, const XmmRegister& reg) { 33 return os << reg.AsFloatRegister(); 34} 35 36std::ostream& operator<<(std::ostream& os, const X87Register& reg) { 37 return os << "ST" << static_cast<int>(reg); 38} 39 40void X86_64Assembler::call(CpuRegister reg) { 41 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 42 EmitOptionalRex32(reg); 43 EmitUint8(0xFF); 44 EmitRegisterOperand(2, reg.LowBits()); 45} 46 47 48void X86_64Assembler::call(const Address& address) { 49 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 50 EmitOptionalRex32(address); 51 EmitUint8(0xFF); 52 EmitOperand(2, address); 53} 54 55 56void X86_64Assembler::call(Label* label) { 57 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 58 EmitUint8(0xE8); 59 static const int kSize = 5; 60 // Offset by one because we already have emitted the opcode. 61 EmitLabel(label, kSize - 1); 62} 63 64void X86_64Assembler::pushq(CpuRegister reg) { 65 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 66 EmitOptionalRex32(reg); 67 EmitUint8(0x50 + reg.LowBits()); 68} 69 70 71void X86_64Assembler::pushq(const Address& address) { 72 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 73 EmitOptionalRex32(address); 74 EmitUint8(0xFF); 75 EmitOperand(6, address); 76} 77 78 79void X86_64Assembler::pushq(const Immediate& imm) { 80 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 81 CHECK(imm.is_int32()); // pushq only supports 32b immediate. 82 if (imm.is_int8()) { 83 EmitUint8(0x6A); 84 EmitUint8(imm.value() & 0xFF); 85 } else { 86 EmitUint8(0x68); 87 EmitImmediate(imm); 88 } 89} 90 91 92void X86_64Assembler::popq(CpuRegister reg) { 93 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 94 EmitOptionalRex32(reg); 95 EmitUint8(0x58 + reg.LowBits()); 96} 97 98 99void X86_64Assembler::popq(const Address& address) { 100 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 101 EmitOptionalRex32(address); 102 EmitUint8(0x8F); 103 EmitOperand(0, address); 104} 105 106 107void X86_64Assembler::movq(CpuRegister dst, const Immediate& imm) { 108 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 109 if (imm.is_int32()) { 110 // 32 bit. Note: sign-extends. 111 EmitRex64(dst); 112 EmitUint8(0xC7); 113 EmitRegisterOperand(0, dst.LowBits()); 114 EmitInt32(static_cast<int32_t>(imm.value())); 115 } else { 116 EmitRex64(dst); 117 EmitUint8(0xB8 + dst.LowBits()); 118 EmitInt64(imm.value()); 119 } 120} 121 122 123void X86_64Assembler::movl(CpuRegister dst, const Immediate& imm) { 124 CHECK(imm.is_int32()); 125 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 126 EmitOptionalRex32(dst); 127 EmitUint8(0xB8 + dst.LowBits()); 128 EmitImmediate(imm); 129} 130 131 132void X86_64Assembler::movq(CpuRegister dst, CpuRegister src) { 133 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 134 // 0x89 is movq r/m64 <- r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 135 EmitRex64(src, dst); 136 EmitUint8(0x89); 137 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 138} 139 140 141void X86_64Assembler::movl(CpuRegister dst, CpuRegister src) { 142 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 143 EmitOptionalRex32(dst, src); 144 EmitUint8(0x8B); 145 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 146} 147 148 149void X86_64Assembler::movq(CpuRegister dst, const Address& src) { 150 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 151 EmitRex64(dst, src); 152 EmitUint8(0x8B); 153 EmitOperand(dst.LowBits(), src); 154} 155 156 157void X86_64Assembler::movl(CpuRegister dst, const Address& src) { 158 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 159 EmitOptionalRex32(dst, src); 160 EmitUint8(0x8B); 161 EmitOperand(dst.LowBits(), src); 162} 163 164 165void X86_64Assembler::movq(const Address& dst, CpuRegister src) { 166 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 167 EmitRex64(src, dst); 168 EmitUint8(0x89); 169 EmitOperand(src.LowBits(), dst); 170} 171 172 173void X86_64Assembler::movl(const Address& dst, CpuRegister src) { 174 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 175 EmitOptionalRex32(src, dst); 176 EmitUint8(0x89); 177 EmitOperand(src.LowBits(), dst); 178} 179 180void X86_64Assembler::movl(const Address& dst, const Immediate& imm) { 181 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 182 EmitOptionalRex32(dst); 183 EmitUint8(0xC7); 184 EmitOperand(0, dst); 185 EmitImmediate(imm); 186} 187 188 189void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src) { 190 cmov(c, dst, src, true); 191} 192 193void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src, bool is64bit) { 194 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 195 EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex()); 196 EmitUint8(0x0F); 197 EmitUint8(0x40 + c); 198 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 199} 200 201 202void X86_64Assembler::movzxb(CpuRegister dst, CpuRegister src) { 203 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 204 EmitOptionalByteRegNormalizingRex32(dst, src); 205 EmitUint8(0x0F); 206 EmitUint8(0xB6); 207 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 208} 209 210 211void X86_64Assembler::movzxb(CpuRegister dst, const Address& src) { 212 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 213 EmitOptionalByteRegNormalizingRex32(dst, src); 214 EmitUint8(0x0F); 215 EmitUint8(0xB6); 216 EmitOperand(dst.LowBits(), src); 217} 218 219 220void X86_64Assembler::movsxb(CpuRegister dst, CpuRegister src) { 221 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 222 EmitOptionalByteRegNormalizingRex32(dst, src); 223 EmitUint8(0x0F); 224 EmitUint8(0xBE); 225 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 226} 227 228 229void X86_64Assembler::movsxb(CpuRegister dst, const Address& src) { 230 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 231 EmitOptionalByteRegNormalizingRex32(dst, src); 232 EmitUint8(0x0F); 233 EmitUint8(0xBE); 234 EmitOperand(dst.LowBits(), src); 235} 236 237 238void X86_64Assembler::movb(CpuRegister /*dst*/, const Address& /*src*/) { 239 LOG(FATAL) << "Use movzxb or movsxb instead."; 240} 241 242 243void X86_64Assembler::movb(const Address& dst, CpuRegister src) { 244 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 245 EmitOptionalByteRegNormalizingRex32(src, dst); 246 EmitUint8(0x88); 247 EmitOperand(src.LowBits(), dst); 248} 249 250 251void X86_64Assembler::movb(const Address& dst, const Immediate& imm) { 252 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 253 EmitOptionalRex32(dst); 254 EmitUint8(0xC6); 255 EmitOperand(Register::RAX, dst); 256 CHECK(imm.is_int8()); 257 EmitUint8(imm.value() & 0xFF); 258} 259 260 261void X86_64Assembler::movzxw(CpuRegister dst, CpuRegister src) { 262 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 263 EmitOptionalRex32(dst, src); 264 EmitUint8(0x0F); 265 EmitUint8(0xB7); 266 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 267} 268 269 270void X86_64Assembler::movzxw(CpuRegister dst, const Address& src) { 271 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 272 EmitOptionalRex32(dst, src); 273 EmitUint8(0x0F); 274 EmitUint8(0xB7); 275 EmitOperand(dst.LowBits(), src); 276} 277 278 279void X86_64Assembler::movsxw(CpuRegister dst, CpuRegister src) { 280 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 281 EmitOptionalRex32(dst, src); 282 EmitUint8(0x0F); 283 EmitUint8(0xBF); 284 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 285} 286 287 288void X86_64Assembler::movsxw(CpuRegister dst, const Address& src) { 289 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 290 EmitOptionalRex32(dst, src); 291 EmitUint8(0x0F); 292 EmitUint8(0xBF); 293 EmitOperand(dst.LowBits(), src); 294} 295 296 297void X86_64Assembler::movw(CpuRegister /*dst*/, const Address& /*src*/) { 298 LOG(FATAL) << "Use movzxw or movsxw instead."; 299} 300 301 302void X86_64Assembler::movw(const Address& dst, CpuRegister src) { 303 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 304 EmitOperandSizeOverride(); 305 EmitOptionalRex32(src, dst); 306 EmitUint8(0x89); 307 EmitOperand(src.LowBits(), dst); 308} 309 310 311void X86_64Assembler::movw(const Address& dst, const Immediate& imm) { 312 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 313 EmitOperandSizeOverride(); 314 EmitOptionalRex32(dst); 315 EmitUint8(0xC7); 316 EmitOperand(Register::RAX, dst); 317 CHECK(imm.is_uint16() || imm.is_int16()); 318 EmitUint8(imm.value() & 0xFF); 319 EmitUint8(imm.value() >> 8); 320} 321 322 323void X86_64Assembler::leaq(CpuRegister dst, const Address& src) { 324 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 325 EmitRex64(dst, src); 326 EmitUint8(0x8D); 327 EmitOperand(dst.LowBits(), src); 328} 329 330 331void X86_64Assembler::leal(CpuRegister dst, const Address& src) { 332 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 333 EmitOptionalRex32(dst, src); 334 EmitUint8(0x8D); 335 EmitOperand(dst.LowBits(), src); 336} 337 338 339void X86_64Assembler::movaps(XmmRegister dst, XmmRegister src) { 340 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 341 EmitOptionalRex32(dst, src); 342 EmitUint8(0x0F); 343 EmitUint8(0x28); 344 EmitXmmRegisterOperand(dst.LowBits(), src); 345} 346 347 348void X86_64Assembler::movss(XmmRegister dst, const Address& src) { 349 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 350 EmitUint8(0xF3); 351 EmitOptionalRex32(dst, src); 352 EmitUint8(0x0F); 353 EmitUint8(0x10); 354 EmitOperand(dst.LowBits(), src); 355} 356 357 358void X86_64Assembler::movss(const Address& dst, XmmRegister src) { 359 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 360 EmitUint8(0xF3); 361 EmitOptionalRex32(src, dst); 362 EmitUint8(0x0F); 363 EmitUint8(0x11); 364 EmitOperand(src.LowBits(), dst); 365} 366 367 368void X86_64Assembler::movss(XmmRegister dst, XmmRegister src) { 369 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 370 EmitUint8(0xF3); 371 EmitOptionalRex32(src, dst); // Movss is MR encoding instead of the usual RM. 372 EmitUint8(0x0F); 373 EmitUint8(0x11); 374 EmitXmmRegisterOperand(src.LowBits(), dst); 375} 376 377 378void X86_64Assembler::movsxd(CpuRegister dst, CpuRegister src) { 379 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 380 EmitRex64(dst, src); 381 EmitUint8(0x63); 382 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 383} 384 385 386void X86_64Assembler::movsxd(CpuRegister dst, const Address& src) { 387 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 388 EmitRex64(dst); 389 EmitUint8(0x63); 390 EmitOperand(dst.LowBits(), src); 391} 392 393 394void X86_64Assembler::movd(XmmRegister dst, CpuRegister src) { 395 movd(dst, src, true); 396} 397 398void X86_64Assembler::movd(CpuRegister dst, XmmRegister src) { 399 movd(dst, src, true); 400} 401 402void X86_64Assembler::movd(XmmRegister dst, CpuRegister src, bool is64bit) { 403 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 404 EmitUint8(0x66); 405 EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex()); 406 EmitUint8(0x0F); 407 EmitUint8(0x6E); 408 EmitOperand(dst.LowBits(), Operand(src)); 409} 410 411void X86_64Assembler::movd(CpuRegister dst, XmmRegister src, bool is64bit) { 412 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 413 EmitUint8(0x66); 414 EmitOptionalRex(false, is64bit, src.NeedsRex(), false, dst.NeedsRex()); 415 EmitUint8(0x0F); 416 EmitUint8(0x7E); 417 EmitOperand(src.LowBits(), Operand(dst)); 418} 419 420 421void X86_64Assembler::addss(XmmRegister dst, XmmRegister src) { 422 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 423 EmitUint8(0xF3); 424 EmitOptionalRex32(dst, src); 425 EmitUint8(0x0F); 426 EmitUint8(0x58); 427 EmitXmmRegisterOperand(dst.LowBits(), src); 428} 429 430 431void X86_64Assembler::addss(XmmRegister dst, const Address& src) { 432 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 433 EmitUint8(0xF3); 434 EmitOptionalRex32(dst, src); 435 EmitUint8(0x0F); 436 EmitUint8(0x58); 437 EmitOperand(dst.LowBits(), src); 438} 439 440 441void X86_64Assembler::subss(XmmRegister dst, XmmRegister src) { 442 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 443 EmitUint8(0xF3); 444 EmitOptionalRex32(dst, src); 445 EmitUint8(0x0F); 446 EmitUint8(0x5C); 447 EmitXmmRegisterOperand(dst.LowBits(), src); 448} 449 450 451void X86_64Assembler::subss(XmmRegister dst, const Address& src) { 452 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 453 EmitUint8(0xF3); 454 EmitOptionalRex32(dst, src); 455 EmitUint8(0x0F); 456 EmitUint8(0x5C); 457 EmitOperand(dst.LowBits(), src); 458} 459 460 461void X86_64Assembler::mulss(XmmRegister dst, XmmRegister src) { 462 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 463 EmitUint8(0xF3); 464 EmitOptionalRex32(dst, src); 465 EmitUint8(0x0F); 466 EmitUint8(0x59); 467 EmitXmmRegisterOperand(dst.LowBits(), src); 468} 469 470 471void X86_64Assembler::mulss(XmmRegister dst, const Address& src) { 472 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 473 EmitUint8(0xF3); 474 EmitOptionalRex32(dst, src); 475 EmitUint8(0x0F); 476 EmitUint8(0x59); 477 EmitOperand(dst.LowBits(), src); 478} 479 480 481void X86_64Assembler::divss(XmmRegister dst, XmmRegister src) { 482 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 483 EmitUint8(0xF3); 484 EmitOptionalRex32(dst, src); 485 EmitUint8(0x0F); 486 EmitUint8(0x5E); 487 EmitXmmRegisterOperand(dst.LowBits(), src); 488} 489 490 491void X86_64Assembler::divss(XmmRegister dst, const Address& src) { 492 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 493 EmitUint8(0xF3); 494 EmitOptionalRex32(dst, src); 495 EmitUint8(0x0F); 496 EmitUint8(0x5E); 497 EmitOperand(dst.LowBits(), src); 498} 499 500 501void X86_64Assembler::flds(const Address& src) { 502 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 503 EmitUint8(0xD9); 504 EmitOperand(0, src); 505} 506 507 508void X86_64Assembler::fsts(const Address& dst) { 509 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 510 EmitUint8(0xD9); 511 EmitOperand(2, dst); 512} 513 514 515void X86_64Assembler::fstps(const Address& dst) { 516 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 517 EmitUint8(0xD9); 518 EmitOperand(3, dst); 519} 520 521 522void X86_64Assembler::movsd(XmmRegister dst, const Address& src) { 523 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 524 EmitUint8(0xF2); 525 EmitOptionalRex32(dst, src); 526 EmitUint8(0x0F); 527 EmitUint8(0x10); 528 EmitOperand(dst.LowBits(), src); 529} 530 531 532void X86_64Assembler::movsd(const Address& dst, XmmRegister src) { 533 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 534 EmitUint8(0xF2); 535 EmitOptionalRex32(src, dst); 536 EmitUint8(0x0F); 537 EmitUint8(0x11); 538 EmitOperand(src.LowBits(), dst); 539} 540 541 542void X86_64Assembler::movsd(XmmRegister dst, XmmRegister src) { 543 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 544 EmitUint8(0xF2); 545 EmitOptionalRex32(src, dst); // Movsd is MR encoding instead of the usual RM. 546 EmitUint8(0x0F); 547 EmitUint8(0x11); 548 EmitXmmRegisterOperand(src.LowBits(), dst); 549} 550 551 552void X86_64Assembler::addsd(XmmRegister dst, XmmRegister src) { 553 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 554 EmitUint8(0xF2); 555 EmitOptionalRex32(dst, src); 556 EmitUint8(0x0F); 557 EmitUint8(0x58); 558 EmitXmmRegisterOperand(dst.LowBits(), src); 559} 560 561 562void X86_64Assembler::addsd(XmmRegister dst, const Address& src) { 563 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 564 EmitUint8(0xF2); 565 EmitOptionalRex32(dst, src); 566 EmitUint8(0x0F); 567 EmitUint8(0x58); 568 EmitOperand(dst.LowBits(), src); 569} 570 571 572void X86_64Assembler::subsd(XmmRegister dst, XmmRegister src) { 573 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 574 EmitUint8(0xF2); 575 EmitOptionalRex32(dst, src); 576 EmitUint8(0x0F); 577 EmitUint8(0x5C); 578 EmitXmmRegisterOperand(dst.LowBits(), src); 579} 580 581 582void X86_64Assembler::subsd(XmmRegister dst, const Address& src) { 583 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 584 EmitUint8(0xF2); 585 EmitOptionalRex32(dst, src); 586 EmitUint8(0x0F); 587 EmitUint8(0x5C); 588 EmitOperand(dst.LowBits(), src); 589} 590 591 592void X86_64Assembler::mulsd(XmmRegister dst, XmmRegister src) { 593 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 594 EmitUint8(0xF2); 595 EmitOptionalRex32(dst, src); 596 EmitUint8(0x0F); 597 EmitUint8(0x59); 598 EmitXmmRegisterOperand(dst.LowBits(), src); 599} 600 601 602void X86_64Assembler::mulsd(XmmRegister dst, const Address& src) { 603 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 604 EmitUint8(0xF2); 605 EmitOptionalRex32(dst, src); 606 EmitUint8(0x0F); 607 EmitUint8(0x59); 608 EmitOperand(dst.LowBits(), src); 609} 610 611 612void X86_64Assembler::divsd(XmmRegister dst, XmmRegister src) { 613 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 614 EmitUint8(0xF2); 615 EmitOptionalRex32(dst, src); 616 EmitUint8(0x0F); 617 EmitUint8(0x5E); 618 EmitXmmRegisterOperand(dst.LowBits(), src); 619} 620 621 622void X86_64Assembler::divsd(XmmRegister dst, const Address& src) { 623 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 624 EmitUint8(0xF2); 625 EmitOptionalRex32(dst, src); 626 EmitUint8(0x0F); 627 EmitUint8(0x5E); 628 EmitOperand(dst.LowBits(), src); 629} 630 631 632void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) { 633 cvtsi2ss(dst, src, false); 634} 635 636 637void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src, bool is64bit) { 638 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 639 EmitUint8(0xF3); 640 if (is64bit) { 641 // Emit a REX.W prefix if the operand size is 64 bits. 642 EmitRex64(dst, src); 643 } else { 644 EmitOptionalRex32(dst, src); 645 } 646 EmitUint8(0x0F); 647 EmitUint8(0x2A); 648 EmitOperand(dst.LowBits(), Operand(src)); 649} 650 651 652void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src) { 653 cvtsi2sd(dst, src, false); 654} 655 656 657void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src, bool is64bit) { 658 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 659 EmitUint8(0xF2); 660 if (is64bit) { 661 // Emit a REX.W prefix if the operand size is 64 bits. 662 EmitRex64(dst, src); 663 } else { 664 EmitOptionalRex32(dst, src); 665 } 666 EmitUint8(0x0F); 667 EmitUint8(0x2A); 668 EmitOperand(dst.LowBits(), Operand(src)); 669} 670 671 672void X86_64Assembler::cvtss2si(CpuRegister dst, XmmRegister src) { 673 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 674 EmitUint8(0xF3); 675 EmitOptionalRex32(dst, src); 676 EmitUint8(0x0F); 677 EmitUint8(0x2D); 678 EmitXmmRegisterOperand(dst.LowBits(), src); 679} 680 681 682void X86_64Assembler::cvtss2sd(XmmRegister dst, XmmRegister src) { 683 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 684 EmitUint8(0xF3); 685 EmitOptionalRex32(dst, src); 686 EmitUint8(0x0F); 687 EmitUint8(0x5A); 688 EmitXmmRegisterOperand(dst.LowBits(), src); 689} 690 691 692void X86_64Assembler::cvtsd2si(CpuRegister dst, XmmRegister src) { 693 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 694 EmitUint8(0xF2); 695 EmitOptionalRex32(dst, src); 696 EmitUint8(0x0F); 697 EmitUint8(0x2D); 698 EmitXmmRegisterOperand(dst.LowBits(), src); 699} 700 701 702void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src) { 703 cvttss2si(dst, src, false); 704} 705 706 707void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src, bool is64bit) { 708 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 709 EmitUint8(0xF3); 710 if (is64bit) { 711 // Emit a REX.W prefix if the operand size is 64 bits. 712 EmitRex64(dst, src); 713 } else { 714 EmitOptionalRex32(dst, src); 715 } 716 EmitUint8(0x0F); 717 EmitUint8(0x2C); 718 EmitXmmRegisterOperand(dst.LowBits(), src); 719} 720 721 722void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src) { 723 cvttsd2si(dst, src, false); 724} 725 726 727void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src, bool is64bit) { 728 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 729 EmitUint8(0xF2); 730 if (is64bit) { 731 // Emit a REX.W prefix if the operand size is 64 bits. 732 EmitRex64(dst, src); 733 } else { 734 EmitOptionalRex32(dst, src); 735 } 736 EmitUint8(0x0F); 737 EmitUint8(0x2C); 738 EmitXmmRegisterOperand(dst.LowBits(), src); 739} 740 741 742void X86_64Assembler::cvtsd2ss(XmmRegister dst, XmmRegister src) { 743 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 744 EmitUint8(0xF2); 745 EmitOptionalRex32(dst, src); 746 EmitUint8(0x0F); 747 EmitUint8(0x5A); 748 EmitXmmRegisterOperand(dst.LowBits(), src); 749} 750 751 752void X86_64Assembler::cvtdq2pd(XmmRegister dst, XmmRegister src) { 753 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 754 EmitUint8(0xF3); 755 EmitOptionalRex32(dst, src); 756 EmitUint8(0x0F); 757 EmitUint8(0xE6); 758 EmitXmmRegisterOperand(dst.LowBits(), src); 759} 760 761 762void X86_64Assembler::comiss(XmmRegister a, XmmRegister b) { 763 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 764 EmitOptionalRex32(a, b); 765 EmitUint8(0x0F); 766 EmitUint8(0x2F); 767 EmitXmmRegisterOperand(a.LowBits(), b); 768} 769 770 771void X86_64Assembler::comisd(XmmRegister a, XmmRegister b) { 772 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 773 EmitUint8(0x66); 774 EmitOptionalRex32(a, b); 775 EmitUint8(0x0F); 776 EmitUint8(0x2F); 777 EmitXmmRegisterOperand(a.LowBits(), b); 778} 779 780void X86_64Assembler::ucomiss(XmmRegister a, XmmRegister b) { 781 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 782 EmitOptionalRex32(a, b); 783 EmitUint8(0x0F); 784 EmitUint8(0x2E); 785 EmitXmmRegisterOperand(a.LowBits(), b); 786} 787 788 789void X86_64Assembler::ucomisd(XmmRegister a, XmmRegister b) { 790 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 791 EmitUint8(0x66); 792 EmitOptionalRex32(a, b); 793 EmitUint8(0x0F); 794 EmitUint8(0x2E); 795 EmitXmmRegisterOperand(a.LowBits(), b); 796} 797 798 799void X86_64Assembler::sqrtsd(XmmRegister dst, XmmRegister src) { 800 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 801 EmitUint8(0xF2); 802 EmitOptionalRex32(dst, src); 803 EmitUint8(0x0F); 804 EmitUint8(0x51); 805 EmitXmmRegisterOperand(dst.LowBits(), src); 806} 807 808 809void X86_64Assembler::sqrtss(XmmRegister dst, XmmRegister src) { 810 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 811 EmitUint8(0xF3); 812 EmitOptionalRex32(dst, src); 813 EmitUint8(0x0F); 814 EmitUint8(0x51); 815 EmitXmmRegisterOperand(dst.LowBits(), src); 816} 817 818 819void X86_64Assembler::xorpd(XmmRegister dst, const Address& src) { 820 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 821 EmitUint8(0x66); 822 EmitOptionalRex32(dst, src); 823 EmitUint8(0x0F); 824 EmitUint8(0x57); 825 EmitOperand(dst.LowBits(), src); 826} 827 828 829void X86_64Assembler::xorpd(XmmRegister dst, XmmRegister src) { 830 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 831 EmitUint8(0x66); 832 EmitOptionalRex32(dst, src); 833 EmitUint8(0x0F); 834 EmitUint8(0x57); 835 EmitXmmRegisterOperand(dst.LowBits(), src); 836} 837 838 839void X86_64Assembler::xorps(XmmRegister dst, const Address& src) { 840 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 841 EmitOptionalRex32(dst, src); 842 EmitUint8(0x0F); 843 EmitUint8(0x57); 844 EmitOperand(dst.LowBits(), src); 845} 846 847 848void X86_64Assembler::xorps(XmmRegister dst, XmmRegister src) { 849 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 850 EmitOptionalRex32(dst, src); 851 EmitUint8(0x0F); 852 EmitUint8(0x57); 853 EmitXmmRegisterOperand(dst.LowBits(), src); 854} 855 856 857void X86_64Assembler::andpd(XmmRegister dst, const Address& src) { 858 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 859 EmitUint8(0x66); 860 EmitOptionalRex32(dst, src); 861 EmitUint8(0x0F); 862 EmitUint8(0x54); 863 EmitOperand(dst.LowBits(), src); 864} 865 866void X86_64Assembler::andpd(XmmRegister dst, XmmRegister src) { 867 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 868 EmitUint8(0x66); 869 EmitOptionalRex32(dst, src); 870 EmitUint8(0x0F); 871 EmitUint8(0x54); 872 EmitXmmRegisterOperand(dst.LowBits(), src); 873} 874 875void X86_64Assembler::andps(XmmRegister dst, XmmRegister src) { 876 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 877 EmitOptionalRex32(dst, src); 878 EmitUint8(0x0F); 879 EmitUint8(0x54); 880 EmitXmmRegisterOperand(dst.LowBits(), src); 881} 882 883void X86_64Assembler::orpd(XmmRegister dst, XmmRegister src) { 884 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 885 EmitUint8(0x66); 886 EmitOptionalRex32(dst, src); 887 EmitUint8(0x0F); 888 EmitUint8(0x56); 889 EmitXmmRegisterOperand(dst.LowBits(), src); 890} 891 892void X86_64Assembler::orps(XmmRegister dst, XmmRegister src) { 893 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 894 EmitOptionalRex32(dst, src); 895 EmitUint8(0x0F); 896 EmitUint8(0x56); 897 EmitXmmRegisterOperand(dst.LowBits(), src); 898} 899 900void X86_64Assembler::fldl(const Address& src) { 901 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 902 EmitUint8(0xDD); 903 EmitOperand(0, src); 904} 905 906 907void X86_64Assembler::fstl(const Address& dst) { 908 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 909 EmitUint8(0xDD); 910 EmitOperand(2, dst); 911} 912 913 914void X86_64Assembler::fstpl(const Address& dst) { 915 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 916 EmitUint8(0xDD); 917 EmitOperand(3, dst); 918} 919 920 921void X86_64Assembler::fstsw() { 922 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 923 EmitUint8(0x9B); 924 EmitUint8(0xDF); 925 EmitUint8(0xE0); 926} 927 928 929void X86_64Assembler::fnstcw(const Address& dst) { 930 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 931 EmitUint8(0xD9); 932 EmitOperand(7, dst); 933} 934 935 936void X86_64Assembler::fldcw(const Address& src) { 937 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 938 EmitUint8(0xD9); 939 EmitOperand(5, src); 940} 941 942 943void X86_64Assembler::fistpl(const Address& dst) { 944 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 945 EmitUint8(0xDF); 946 EmitOperand(7, dst); 947} 948 949 950void X86_64Assembler::fistps(const Address& dst) { 951 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 952 EmitUint8(0xDB); 953 EmitOperand(3, dst); 954} 955 956 957void X86_64Assembler::fildl(const Address& src) { 958 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 959 EmitUint8(0xDF); 960 EmitOperand(5, src); 961} 962 963 964void X86_64Assembler::fincstp() { 965 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 966 EmitUint8(0xD9); 967 EmitUint8(0xF7); 968} 969 970 971void X86_64Assembler::ffree(const Immediate& index) { 972 CHECK_LT(index.value(), 7); 973 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 974 EmitUint8(0xDD); 975 EmitUint8(0xC0 + index.value()); 976} 977 978 979void X86_64Assembler::fsin() { 980 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 981 EmitUint8(0xD9); 982 EmitUint8(0xFE); 983} 984 985 986void X86_64Assembler::fcos() { 987 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 988 EmitUint8(0xD9); 989 EmitUint8(0xFF); 990} 991 992 993void X86_64Assembler::fptan() { 994 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 995 EmitUint8(0xD9); 996 EmitUint8(0xF2); 997} 998 999void X86_64Assembler::fucompp() { 1000 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1001 EmitUint8(0xDA); 1002 EmitUint8(0xE9); 1003} 1004 1005 1006void X86_64Assembler::fprem() { 1007 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1008 EmitUint8(0xD9); 1009 EmitUint8(0xF8); 1010} 1011 1012 1013void X86_64Assembler::xchgl(CpuRegister dst, CpuRegister src) { 1014 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1015 // There is a short version for rax. 1016 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't 1017 // work. 1018 const bool src_rax = src.AsRegister() == RAX; 1019 const bool dst_rax = dst.AsRegister() == RAX; 1020 if (src_rax || dst_rax) { 1021 EmitOptionalRex32(src_rax ? dst : src); 1022 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits())); 1023 return; 1024 } 1025 1026 // General case. 1027 EmitOptionalRex32(src, dst); 1028 EmitUint8(0x87); 1029 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1030} 1031 1032 1033void X86_64Assembler::xchgq(CpuRegister dst, CpuRegister src) { 1034 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1035 // There is a short version for rax. 1036 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't 1037 // work. 1038 const bool src_rax = src.AsRegister() == RAX; 1039 const bool dst_rax = dst.AsRegister() == RAX; 1040 if (src_rax || dst_rax) { 1041 // If src == target, emit a nop instead. 1042 if (src_rax && dst_rax) { 1043 EmitUint8(0x90); 1044 } else { 1045 EmitRex64(src_rax ? dst : src); 1046 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits())); 1047 } 1048 return; 1049 } 1050 1051 // General case. 1052 EmitRex64(src, dst); 1053 EmitUint8(0x87); 1054 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1055} 1056 1057 1058void X86_64Assembler::xchgl(CpuRegister reg, const Address& address) { 1059 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1060 EmitOptionalRex32(reg, address); 1061 EmitUint8(0x87); 1062 EmitOperand(reg.LowBits(), address); 1063} 1064 1065 1066void X86_64Assembler::cmpw(const Address& address, const Immediate& imm) { 1067 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1068 EmitOptionalRex32(address); 1069 EmitUint8(0x66); 1070 EmitComplex(7, address, imm); 1071} 1072 1073 1074void X86_64Assembler::cmpl(CpuRegister reg, const Immediate& imm) { 1075 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1076 EmitOptionalRex32(reg); 1077 EmitComplex(7, Operand(reg), imm); 1078} 1079 1080 1081void X86_64Assembler::cmpl(CpuRegister reg0, CpuRegister reg1) { 1082 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1083 EmitOptionalRex32(reg0, reg1); 1084 EmitUint8(0x3B); 1085 EmitOperand(reg0.LowBits(), Operand(reg1)); 1086} 1087 1088 1089void X86_64Assembler::cmpl(CpuRegister reg, const Address& address) { 1090 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1091 EmitOptionalRex32(reg, address); 1092 EmitUint8(0x3B); 1093 EmitOperand(reg.LowBits(), address); 1094} 1095 1096 1097void X86_64Assembler::cmpl(const Address& address, CpuRegister reg) { 1098 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1099 EmitOptionalRex32(reg, address); 1100 EmitUint8(0x39); 1101 EmitOperand(reg.LowBits(), address); 1102} 1103 1104 1105void X86_64Assembler::cmpl(const Address& address, const Immediate& imm) { 1106 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1107 EmitOptionalRex32(address); 1108 EmitComplex(7, address, imm); 1109} 1110 1111 1112void X86_64Assembler::cmpq(CpuRegister reg0, CpuRegister reg1) { 1113 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1114 EmitRex64(reg0, reg1); 1115 EmitUint8(0x3B); 1116 EmitOperand(reg0.LowBits(), Operand(reg1)); 1117} 1118 1119 1120void X86_64Assembler::cmpq(CpuRegister reg, const Immediate& imm) { 1121 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1122 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 1123 EmitRex64(reg); 1124 EmitComplex(7, Operand(reg), imm); 1125} 1126 1127 1128void X86_64Assembler::cmpq(CpuRegister reg, const Address& address) { 1129 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1130 EmitRex64(reg); 1131 EmitUint8(0x3B); 1132 EmitOperand(reg.LowBits(), address); 1133} 1134 1135 1136void X86_64Assembler::cmpq(const Address& address, const Immediate& imm) { 1137 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 1138 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1139 EmitRex64(address); 1140 EmitComplex(7, address, imm); 1141} 1142 1143 1144void X86_64Assembler::addl(CpuRegister dst, CpuRegister src) { 1145 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1146 EmitOptionalRex32(dst, src); 1147 EmitUint8(0x03); 1148 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1149} 1150 1151 1152void X86_64Assembler::addl(CpuRegister reg, const Address& address) { 1153 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1154 EmitOptionalRex32(reg, address); 1155 EmitUint8(0x03); 1156 EmitOperand(reg.LowBits(), address); 1157} 1158 1159 1160void X86_64Assembler::testl(CpuRegister reg1, CpuRegister reg2) { 1161 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1162 EmitOptionalRex32(reg1, reg2); 1163 EmitUint8(0x85); 1164 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 1165} 1166 1167 1168void X86_64Assembler::testl(CpuRegister reg, const Address& address) { 1169 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1170 EmitOptionalRex32(reg, address); 1171 EmitUint8(0x85); 1172 EmitOperand(reg.LowBits(), address); 1173} 1174 1175 1176void X86_64Assembler::testl(CpuRegister reg, const Immediate& immediate) { 1177 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1178 // For registers that have a byte variant (RAX, RBX, RCX, and RDX) 1179 // we only test the byte CpuRegister to keep the encoding short. 1180 if (immediate.is_uint8() && reg.AsRegister() < 4) { 1181 // Use zero-extended 8-bit immediate. 1182 if (reg.AsRegister() == RAX) { 1183 EmitUint8(0xA8); 1184 } else { 1185 EmitUint8(0xF6); 1186 EmitUint8(0xC0 + reg.AsRegister()); 1187 } 1188 EmitUint8(immediate.value() & 0xFF); 1189 } else if (reg.AsRegister() == RAX) { 1190 // Use short form if the destination is RAX. 1191 EmitUint8(0xA9); 1192 EmitImmediate(immediate); 1193 } else { 1194 EmitOptionalRex32(reg); 1195 EmitUint8(0xF7); 1196 EmitOperand(0, Operand(reg)); 1197 EmitImmediate(immediate); 1198 } 1199} 1200 1201 1202void X86_64Assembler::testq(CpuRegister reg1, CpuRegister reg2) { 1203 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1204 EmitRex64(reg1, reg2); 1205 EmitUint8(0x85); 1206 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 1207} 1208 1209 1210void X86_64Assembler::testq(CpuRegister reg, const Address& address) { 1211 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1212 EmitRex64(reg); 1213 EmitUint8(0x85); 1214 EmitOperand(reg.LowBits(), address); 1215} 1216 1217 1218void X86_64Assembler::andl(CpuRegister dst, CpuRegister src) { 1219 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1220 EmitOptionalRex32(dst, src); 1221 EmitUint8(0x23); 1222 EmitOperand(dst.LowBits(), Operand(src)); 1223} 1224 1225 1226void X86_64Assembler::andl(CpuRegister reg, const Address& address) { 1227 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1228 EmitOptionalRex32(reg, address); 1229 EmitUint8(0x23); 1230 EmitOperand(reg.LowBits(), address); 1231} 1232 1233 1234void X86_64Assembler::andl(CpuRegister dst, const Immediate& imm) { 1235 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1236 EmitOptionalRex32(dst); 1237 EmitComplex(4, Operand(dst), imm); 1238} 1239 1240 1241void X86_64Assembler::andq(CpuRegister reg, const Immediate& imm) { 1242 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1243 CHECK(imm.is_int32()); // andq only supports 32b immediate. 1244 EmitRex64(reg); 1245 EmitComplex(4, Operand(reg), imm); 1246} 1247 1248 1249void X86_64Assembler::andq(CpuRegister dst, CpuRegister src) { 1250 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1251 EmitRex64(dst, src); 1252 EmitUint8(0x23); 1253 EmitOperand(dst.LowBits(), Operand(src)); 1254} 1255 1256 1257void X86_64Assembler::orl(CpuRegister dst, CpuRegister src) { 1258 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1259 EmitOptionalRex32(dst, src); 1260 EmitUint8(0x0B); 1261 EmitOperand(dst.LowBits(), Operand(src)); 1262} 1263 1264 1265void X86_64Assembler::orl(CpuRegister reg, const Address& address) { 1266 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1267 EmitOptionalRex32(reg, address); 1268 EmitUint8(0x0B); 1269 EmitOperand(reg.LowBits(), address); 1270} 1271 1272 1273void X86_64Assembler::orl(CpuRegister dst, const Immediate& imm) { 1274 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1275 EmitOptionalRex32(dst); 1276 EmitComplex(1, Operand(dst), imm); 1277} 1278 1279 1280void X86_64Assembler::orq(CpuRegister dst, const Immediate& imm) { 1281 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1282 CHECK(imm.is_int32()); // orq only supports 32b immediate. 1283 EmitRex64(dst); 1284 EmitComplex(1, Operand(dst), imm); 1285} 1286 1287 1288void X86_64Assembler::orq(CpuRegister dst, CpuRegister src) { 1289 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1290 EmitRex64(dst, src); 1291 EmitUint8(0x0B); 1292 EmitOperand(dst.LowBits(), Operand(src)); 1293} 1294 1295 1296void X86_64Assembler::xorl(CpuRegister dst, CpuRegister src) { 1297 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1298 EmitOptionalRex32(dst, src); 1299 EmitUint8(0x33); 1300 EmitOperand(dst.LowBits(), Operand(src)); 1301} 1302 1303 1304void X86_64Assembler::xorl(CpuRegister reg, const Address& address) { 1305 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1306 EmitOptionalRex32(reg, address); 1307 EmitUint8(0x33); 1308 EmitOperand(reg.LowBits(), address); 1309} 1310 1311 1312void X86_64Assembler::xorl(CpuRegister dst, const Immediate& imm) { 1313 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1314 EmitOptionalRex32(dst); 1315 EmitComplex(6, Operand(dst), imm); 1316} 1317 1318 1319void X86_64Assembler::xorq(CpuRegister dst, CpuRegister src) { 1320 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1321 EmitRex64(dst, src); 1322 EmitUint8(0x33); 1323 EmitOperand(dst.LowBits(), Operand(src)); 1324} 1325 1326 1327void X86_64Assembler::xorq(CpuRegister dst, const Immediate& imm) { 1328 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1329 CHECK(imm.is_int32()); // xorq only supports 32b immediate. 1330 EmitRex64(dst); 1331 EmitComplex(6, Operand(dst), imm); 1332} 1333 1334#if 0 1335void X86_64Assembler::rex(bool force, bool w, Register* r, Register* x, Register* b) { 1336 // REX.WRXB 1337 // W - 64-bit operand 1338 // R - MODRM.reg 1339 // X - SIB.index 1340 // B - MODRM.rm/SIB.base 1341 uint8_t rex = force ? 0x40 : 0; 1342 if (w) { 1343 rex |= 0x48; // REX.W000 1344 } 1345 if (r != nullptr && *r >= Register::R8 && *r < Register::kNumberOfCpuRegisters) { 1346 rex |= 0x44; // REX.0R00 1347 *r = static_cast<Register>(*r - 8); 1348 } 1349 if (x != nullptr && *x >= Register::R8 && *x < Register::kNumberOfCpuRegisters) { 1350 rex |= 0x42; // REX.00X0 1351 *x = static_cast<Register>(*x - 8); 1352 } 1353 if (b != nullptr && *b >= Register::R8 && *b < Register::kNumberOfCpuRegisters) { 1354 rex |= 0x41; // REX.000B 1355 *b = static_cast<Register>(*b - 8); 1356 } 1357 if (rex != 0) { 1358 EmitUint8(rex); 1359 } 1360} 1361 1362void X86_64Assembler::rex_reg_mem(bool force, bool w, Register* dst, const Address& mem) { 1363 // REX.WRXB 1364 // W - 64-bit operand 1365 // R - MODRM.reg 1366 // X - SIB.index 1367 // B - MODRM.rm/SIB.base 1368 uint8_t rex = mem->rex(); 1369 if (force) { 1370 rex |= 0x40; // REX.0000 1371 } 1372 if (w) { 1373 rex |= 0x48; // REX.W000 1374 } 1375 if (dst != nullptr && *dst >= Register::R8 && *dst < Register::kNumberOfCpuRegisters) { 1376 rex |= 0x44; // REX.0R00 1377 *dst = static_cast<Register>(*dst - 8); 1378 } 1379 if (rex != 0) { 1380 EmitUint8(rex); 1381 } 1382} 1383 1384void rex_mem_reg(bool force, bool w, Address* mem, Register* src); 1385#endif 1386 1387void X86_64Assembler::addl(CpuRegister reg, const Immediate& imm) { 1388 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1389 EmitOptionalRex32(reg); 1390 EmitComplex(0, Operand(reg), imm); 1391} 1392 1393 1394void X86_64Assembler::addq(CpuRegister reg, const Immediate& imm) { 1395 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1396 CHECK(imm.is_int32()); // addq only supports 32b immediate. 1397 EmitRex64(reg); 1398 EmitComplex(0, Operand(reg), imm); 1399} 1400 1401 1402void X86_64Assembler::addq(CpuRegister dst, const Address& address) { 1403 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1404 EmitRex64(dst); 1405 EmitUint8(0x03); 1406 EmitOperand(dst.LowBits(), address); 1407} 1408 1409 1410void X86_64Assembler::addq(CpuRegister dst, CpuRegister src) { 1411 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1412 // 0x01 is addq r/m64 <- r/m64 + r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 1413 EmitRex64(src, dst); 1414 EmitUint8(0x01); 1415 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1416} 1417 1418 1419void X86_64Assembler::addl(const Address& address, CpuRegister reg) { 1420 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1421 EmitOptionalRex32(reg, address); 1422 EmitUint8(0x01); 1423 EmitOperand(reg.LowBits(), address); 1424} 1425 1426 1427void X86_64Assembler::addl(const Address& address, const Immediate& imm) { 1428 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1429 EmitOptionalRex32(address); 1430 EmitComplex(0, address, imm); 1431} 1432 1433 1434void X86_64Assembler::subl(CpuRegister dst, CpuRegister src) { 1435 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1436 EmitOptionalRex32(dst, src); 1437 EmitUint8(0x2B); 1438 EmitOperand(dst.LowBits(), Operand(src)); 1439} 1440 1441 1442void X86_64Assembler::subl(CpuRegister reg, const Immediate& imm) { 1443 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1444 EmitOptionalRex32(reg); 1445 EmitComplex(5, Operand(reg), imm); 1446} 1447 1448 1449void X86_64Assembler::subq(CpuRegister reg, const Immediate& imm) { 1450 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1451 CHECK(imm.is_int32()); // subq only supports 32b immediate. 1452 EmitRex64(reg); 1453 EmitComplex(5, Operand(reg), imm); 1454} 1455 1456 1457void X86_64Assembler::subq(CpuRegister dst, CpuRegister src) { 1458 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1459 EmitRex64(dst, src); 1460 EmitUint8(0x2B); 1461 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1462} 1463 1464 1465void X86_64Assembler::subq(CpuRegister reg, const Address& address) { 1466 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1467 EmitRex64(reg); 1468 EmitUint8(0x2B); 1469 EmitOperand(reg.LowBits() & 7, address); 1470} 1471 1472 1473void X86_64Assembler::subl(CpuRegister reg, const Address& address) { 1474 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1475 EmitOptionalRex32(reg, address); 1476 EmitUint8(0x2B); 1477 EmitOperand(reg.LowBits(), address); 1478} 1479 1480 1481void X86_64Assembler::cdq() { 1482 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1483 EmitUint8(0x99); 1484} 1485 1486 1487void X86_64Assembler::cqo() { 1488 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1489 EmitRex64(); 1490 EmitUint8(0x99); 1491} 1492 1493 1494void X86_64Assembler::idivl(CpuRegister reg) { 1495 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1496 EmitOptionalRex32(reg); 1497 EmitUint8(0xF7); 1498 EmitUint8(0xF8 | reg.LowBits()); 1499} 1500 1501 1502void X86_64Assembler::idivq(CpuRegister reg) { 1503 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1504 EmitRex64(reg); 1505 EmitUint8(0xF7); 1506 EmitUint8(0xF8 | reg.LowBits()); 1507} 1508 1509 1510void X86_64Assembler::imull(CpuRegister dst, CpuRegister src) { 1511 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1512 EmitOptionalRex32(dst, src); 1513 EmitUint8(0x0F); 1514 EmitUint8(0xAF); 1515 EmitOperand(dst.LowBits(), Operand(src)); 1516} 1517 1518void X86_64Assembler::imull(CpuRegister reg, const Immediate& imm) { 1519 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1520 CHECK(imm.is_int32()); // imull only supports 32b immediate. 1521 1522 EmitOptionalRex32(reg, reg); 1523 1524 // See whether imm can be represented as a sign-extended 8bit value. 1525 int32_t v32 = static_cast<int32_t>(imm.value()); 1526 if (IsInt<8>(v32)) { 1527 // Sign-extension works. 1528 EmitUint8(0x6B); 1529 EmitOperand(reg.LowBits(), Operand(reg)); 1530 EmitUint8(static_cast<uint8_t>(v32 & 0xFF)); 1531 } else { 1532 // Not representable, use full immediate. 1533 EmitUint8(0x69); 1534 EmitOperand(reg.LowBits(), Operand(reg)); 1535 EmitImmediate(imm); 1536 } 1537} 1538 1539 1540void X86_64Assembler::imull(CpuRegister reg, const Address& address) { 1541 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1542 EmitOptionalRex32(reg, address); 1543 EmitUint8(0x0F); 1544 EmitUint8(0xAF); 1545 EmitOperand(reg.LowBits(), address); 1546} 1547 1548 1549void X86_64Assembler::imulq(CpuRegister dst, CpuRegister src) { 1550 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1551 EmitRex64(dst, src); 1552 EmitUint8(0x0F); 1553 EmitUint8(0xAF); 1554 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1555} 1556 1557 1558void X86_64Assembler::imulq(CpuRegister reg, const Immediate& imm) { 1559 imulq(reg, reg, imm); 1560} 1561 1562void X86_64Assembler::imulq(CpuRegister dst, CpuRegister reg, const Immediate& imm) { 1563 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1564 CHECK(imm.is_int32()); // imulq only supports 32b immediate. 1565 1566 EmitRex64(dst, reg); 1567 1568 // See whether imm can be represented as a sign-extended 8bit value. 1569 int64_t v64 = imm.value(); 1570 if (IsInt<8>(v64)) { 1571 // Sign-extension works. 1572 EmitUint8(0x6B); 1573 EmitOperand(dst.LowBits(), Operand(reg)); 1574 EmitUint8(static_cast<uint8_t>(v64 & 0xFF)); 1575 } else { 1576 // Not representable, use full immediate. 1577 EmitUint8(0x69); 1578 EmitOperand(dst.LowBits(), Operand(reg)); 1579 EmitImmediate(imm); 1580 } 1581} 1582 1583void X86_64Assembler::imulq(CpuRegister reg, const Address& address) { 1584 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1585 EmitRex64(reg, address); 1586 EmitUint8(0x0F); 1587 EmitUint8(0xAF); 1588 EmitOperand(reg.LowBits(), address); 1589} 1590 1591 1592void X86_64Assembler::imull(CpuRegister reg) { 1593 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1594 EmitOptionalRex32(reg); 1595 EmitUint8(0xF7); 1596 EmitOperand(5, Operand(reg)); 1597} 1598 1599 1600void X86_64Assembler::imulq(CpuRegister reg) { 1601 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1602 EmitRex64(reg); 1603 EmitUint8(0xF7); 1604 EmitOperand(5, Operand(reg)); 1605} 1606 1607 1608void X86_64Assembler::imull(const Address& address) { 1609 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1610 EmitOptionalRex32(address); 1611 EmitUint8(0xF7); 1612 EmitOperand(5, address); 1613} 1614 1615 1616void X86_64Assembler::mull(CpuRegister reg) { 1617 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1618 EmitOptionalRex32(reg); 1619 EmitUint8(0xF7); 1620 EmitOperand(4, Operand(reg)); 1621} 1622 1623 1624void X86_64Assembler::mull(const Address& address) { 1625 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1626 EmitOptionalRex32(address); 1627 EmitUint8(0xF7); 1628 EmitOperand(4, address); 1629} 1630 1631 1632void X86_64Assembler::shll(CpuRegister reg, const Immediate& imm) { 1633 EmitGenericShift(false, 4, reg, imm); 1634} 1635 1636 1637void X86_64Assembler::shlq(CpuRegister reg, const Immediate& imm) { 1638 EmitGenericShift(true, 4, reg, imm); 1639} 1640 1641 1642void X86_64Assembler::shll(CpuRegister operand, CpuRegister shifter) { 1643 EmitGenericShift(false, 4, operand, shifter); 1644} 1645 1646 1647void X86_64Assembler::shlq(CpuRegister operand, CpuRegister shifter) { 1648 EmitGenericShift(true, 4, operand, shifter); 1649} 1650 1651 1652void X86_64Assembler::shrl(CpuRegister reg, const Immediate& imm) { 1653 EmitGenericShift(false, 5, reg, imm); 1654} 1655 1656 1657void X86_64Assembler::shrq(CpuRegister reg, const Immediate& imm) { 1658 EmitGenericShift(true, 5, reg, imm); 1659} 1660 1661 1662void X86_64Assembler::shrl(CpuRegister operand, CpuRegister shifter) { 1663 EmitGenericShift(false, 5, operand, shifter); 1664} 1665 1666 1667void X86_64Assembler::shrq(CpuRegister operand, CpuRegister shifter) { 1668 EmitGenericShift(true, 5, operand, shifter); 1669} 1670 1671 1672void X86_64Assembler::sarl(CpuRegister reg, const Immediate& imm) { 1673 EmitGenericShift(false, 7, reg, imm); 1674} 1675 1676 1677void X86_64Assembler::sarl(CpuRegister operand, CpuRegister shifter) { 1678 EmitGenericShift(false, 7, operand, shifter); 1679} 1680 1681 1682void X86_64Assembler::sarq(CpuRegister reg, const Immediate& imm) { 1683 EmitGenericShift(true, 7, reg, imm); 1684} 1685 1686 1687void X86_64Assembler::sarq(CpuRegister operand, CpuRegister shifter) { 1688 EmitGenericShift(true, 7, operand, shifter); 1689} 1690 1691 1692void X86_64Assembler::negl(CpuRegister reg) { 1693 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1694 EmitOptionalRex32(reg); 1695 EmitUint8(0xF7); 1696 EmitOperand(3, Operand(reg)); 1697} 1698 1699 1700void X86_64Assembler::negq(CpuRegister reg) { 1701 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1702 EmitRex64(reg); 1703 EmitUint8(0xF7); 1704 EmitOperand(3, Operand(reg)); 1705} 1706 1707 1708void X86_64Assembler::notl(CpuRegister reg) { 1709 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1710 EmitOptionalRex32(reg); 1711 EmitUint8(0xF7); 1712 EmitUint8(0xD0 | reg.LowBits()); 1713} 1714 1715 1716void X86_64Assembler::notq(CpuRegister reg) { 1717 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1718 EmitRex64(reg); 1719 EmitUint8(0xF7); 1720 EmitOperand(2, Operand(reg)); 1721} 1722 1723 1724void X86_64Assembler::enter(const Immediate& imm) { 1725 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1726 EmitUint8(0xC8); 1727 CHECK(imm.is_uint16()) << imm.value(); 1728 EmitUint8(imm.value() & 0xFF); 1729 EmitUint8((imm.value() >> 8) & 0xFF); 1730 EmitUint8(0x00); 1731} 1732 1733 1734void X86_64Assembler::leave() { 1735 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1736 EmitUint8(0xC9); 1737} 1738 1739 1740void X86_64Assembler::ret() { 1741 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1742 EmitUint8(0xC3); 1743} 1744 1745 1746void X86_64Assembler::ret(const Immediate& imm) { 1747 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1748 EmitUint8(0xC2); 1749 CHECK(imm.is_uint16()); 1750 EmitUint8(imm.value() & 0xFF); 1751 EmitUint8((imm.value() >> 8) & 0xFF); 1752} 1753 1754 1755 1756void X86_64Assembler::nop() { 1757 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1758 EmitUint8(0x90); 1759} 1760 1761 1762void X86_64Assembler::int3() { 1763 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1764 EmitUint8(0xCC); 1765} 1766 1767 1768void X86_64Assembler::hlt() { 1769 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1770 EmitUint8(0xF4); 1771} 1772 1773 1774void X86_64Assembler::j(Condition condition, Label* label) { 1775 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1776 if (label->IsBound()) { 1777 static const int kShortSize = 2; 1778 static const int kLongSize = 6; 1779 int offset = label->Position() - buffer_.Size(); 1780 CHECK_LE(offset, 0); 1781 if (IsInt<8>(offset - kShortSize)) { 1782 EmitUint8(0x70 + condition); 1783 EmitUint8((offset - kShortSize) & 0xFF); 1784 } else { 1785 EmitUint8(0x0F); 1786 EmitUint8(0x80 + condition); 1787 EmitInt32(offset - kLongSize); 1788 } 1789 } else { 1790 EmitUint8(0x0F); 1791 EmitUint8(0x80 + condition); 1792 EmitLabelLink(label); 1793 } 1794} 1795 1796 1797void X86_64Assembler::jmp(CpuRegister reg) { 1798 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1799 EmitOptionalRex32(reg); 1800 EmitUint8(0xFF); 1801 EmitRegisterOperand(4, reg.LowBits()); 1802} 1803 1804void X86_64Assembler::jmp(const Address& address) { 1805 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1806 EmitOptionalRex32(address); 1807 EmitUint8(0xFF); 1808 EmitOperand(4, address); 1809} 1810 1811void X86_64Assembler::jmp(Label* label) { 1812 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1813 if (label->IsBound()) { 1814 static const int kShortSize = 2; 1815 static const int kLongSize = 5; 1816 int offset = label->Position() - buffer_.Size(); 1817 CHECK_LE(offset, 0); 1818 if (IsInt<8>(offset - kShortSize)) { 1819 EmitUint8(0xEB); 1820 EmitUint8((offset - kShortSize) & 0xFF); 1821 } else { 1822 EmitUint8(0xE9); 1823 EmitInt32(offset - kLongSize); 1824 } 1825 } else { 1826 EmitUint8(0xE9); 1827 EmitLabelLink(label); 1828 } 1829} 1830 1831 1832X86_64Assembler* X86_64Assembler::lock() { 1833 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1834 EmitUint8(0xF0); 1835 return this; 1836} 1837 1838 1839void X86_64Assembler::cmpxchgl(const Address& address, CpuRegister reg) { 1840 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1841 EmitUint8(0x0F); 1842 EmitUint8(0xB1); 1843 EmitOperand(reg.LowBits(), address); 1844} 1845 1846void X86_64Assembler::mfence() { 1847 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1848 EmitUint8(0x0F); 1849 EmitUint8(0xAE); 1850 EmitUint8(0xF0); 1851} 1852 1853 1854X86_64Assembler* X86_64Assembler::gs() { 1855 // TODO: gs is a prefix and not an instruction 1856 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1857 EmitUint8(0x65); 1858 return this; 1859} 1860 1861 1862void X86_64Assembler::AddImmediate(CpuRegister reg, const Immediate& imm) { 1863 int value = imm.value(); 1864 if (value != 0) { 1865 if (value > 0) { 1866 addl(reg, imm); 1867 } else { 1868 subl(reg, Immediate(value)); 1869 } 1870 } 1871} 1872 1873 1874void X86_64Assembler::setcc(Condition condition, CpuRegister dst) { 1875 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1876 // RSP, RBP, RDI, RSI need rex prefix (else the pattern encodes ah/bh/ch/dh). 1877 if (dst.NeedsRex() || dst.AsRegister() > 3) { 1878 EmitOptionalRex(true, false, false, false, dst.NeedsRex()); 1879 } 1880 EmitUint8(0x0F); 1881 EmitUint8(0x90 + condition); 1882 EmitUint8(0xC0 + dst.LowBits()); 1883} 1884 1885void X86_64Assembler::bswapl(CpuRegister dst) { 1886 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1887 EmitOptionalRex(false, false, false, false, dst.NeedsRex()); 1888 EmitUint8(0x0F); 1889 EmitUint8(0xC8 + dst.LowBits()); 1890} 1891 1892void X86_64Assembler::bswapq(CpuRegister dst) { 1893 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1894 EmitOptionalRex(false, true, false, false, dst.NeedsRex()); 1895 EmitUint8(0x0F); 1896 EmitUint8(0xC8 + dst.LowBits()); 1897} 1898 1899 1900void X86_64Assembler::LoadDoubleConstant(XmmRegister dst, double value) { 1901 // TODO: Need to have a code constants table. 1902 int64_t constant = bit_cast<int64_t, double>(value); 1903 pushq(Immediate(High32Bits(constant))); 1904 pushq(Immediate(Low32Bits(constant))); 1905 movsd(dst, Address(CpuRegister(RSP), 0)); 1906 addq(CpuRegister(RSP), Immediate(2 * sizeof(intptr_t))); 1907} 1908 1909 1910void X86_64Assembler::Align(int alignment, int offset) { 1911 CHECK(IsPowerOfTwo(alignment)); 1912 // Emit nop instruction until the real position is aligned. 1913 while (((offset + buffer_.GetPosition()) & (alignment-1)) != 0) { 1914 nop(); 1915 } 1916} 1917 1918 1919void X86_64Assembler::Bind(Label* label) { 1920 int bound = buffer_.Size(); 1921 CHECK(!label->IsBound()); // Labels can only be bound once. 1922 while (label->IsLinked()) { 1923 int position = label->LinkPosition(); 1924 int next = buffer_.Load<int32_t>(position); 1925 buffer_.Store<int32_t>(position, bound - (position + 4)); 1926 label->position_ = next; 1927 } 1928 label->BindTo(bound); 1929} 1930 1931 1932void X86_64Assembler::EmitOperand(uint8_t reg_or_opcode, const Operand& operand) { 1933 CHECK_GE(reg_or_opcode, 0); 1934 CHECK_LT(reg_or_opcode, 8); 1935 const int length = operand.length_; 1936 CHECK_GT(length, 0); 1937 // Emit the ModRM byte updated with the given reg value. 1938 CHECK_EQ(operand.encoding_[0] & 0x38, 0); 1939 EmitUint8(operand.encoding_[0] + (reg_or_opcode << 3)); 1940 // Emit the rest of the encoded operand. 1941 for (int i = 1; i < length; i++) { 1942 EmitUint8(operand.encoding_[i]); 1943 } 1944} 1945 1946 1947void X86_64Assembler::EmitImmediate(const Immediate& imm) { 1948 if (imm.is_int32()) { 1949 EmitInt32(static_cast<int32_t>(imm.value())); 1950 } else { 1951 EmitInt64(imm.value()); 1952 } 1953} 1954 1955 1956void X86_64Assembler::EmitComplex(uint8_t reg_or_opcode, 1957 const Operand& operand, 1958 const Immediate& immediate) { 1959 CHECK_GE(reg_or_opcode, 0); 1960 CHECK_LT(reg_or_opcode, 8); 1961 if (immediate.is_int8()) { 1962 // Use sign-extended 8-bit immediate. 1963 EmitUint8(0x83); 1964 EmitOperand(reg_or_opcode, operand); 1965 EmitUint8(immediate.value() & 0xFF); 1966 } else if (operand.IsRegister(CpuRegister(RAX))) { 1967 // Use short form if the destination is eax. 1968 EmitUint8(0x05 + (reg_or_opcode << 3)); 1969 EmitImmediate(immediate); 1970 } else { 1971 EmitUint8(0x81); 1972 EmitOperand(reg_or_opcode, operand); 1973 EmitImmediate(immediate); 1974 } 1975} 1976 1977 1978void X86_64Assembler::EmitLabel(Label* label, int instruction_size) { 1979 if (label->IsBound()) { 1980 int offset = label->Position() - buffer_.Size(); 1981 CHECK_LE(offset, 0); 1982 EmitInt32(offset - instruction_size); 1983 } else { 1984 EmitLabelLink(label); 1985 } 1986} 1987 1988 1989void X86_64Assembler::EmitLabelLink(Label* label) { 1990 CHECK(!label->IsBound()); 1991 int position = buffer_.Size(); 1992 EmitInt32(label->position_); 1993 label->LinkTo(position); 1994} 1995 1996 1997void X86_64Assembler::EmitGenericShift(bool wide, 1998 int reg_or_opcode, 1999 CpuRegister reg, 2000 const Immediate& imm) { 2001 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2002 CHECK(imm.is_int8()); 2003 if (wide) { 2004 EmitRex64(reg); 2005 } else { 2006 EmitOptionalRex32(reg); 2007 } 2008 if (imm.value() == 1) { 2009 EmitUint8(0xD1); 2010 EmitOperand(reg_or_opcode, Operand(reg)); 2011 } else { 2012 EmitUint8(0xC1); 2013 EmitOperand(reg_or_opcode, Operand(reg)); 2014 EmitUint8(imm.value() & 0xFF); 2015 } 2016} 2017 2018 2019void X86_64Assembler::EmitGenericShift(bool wide, 2020 int reg_or_opcode, 2021 CpuRegister operand, 2022 CpuRegister shifter) { 2023 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2024 CHECK_EQ(shifter.AsRegister(), RCX); 2025 if (wide) { 2026 EmitRex64(operand); 2027 } else { 2028 EmitOptionalRex32(operand); 2029 } 2030 EmitUint8(0xD3); 2031 EmitOperand(reg_or_opcode, Operand(operand)); 2032} 2033 2034void X86_64Assembler::EmitOptionalRex(bool force, bool w, bool r, bool x, bool b) { 2035 // REX.WRXB 2036 // W - 64-bit operand 2037 // R - MODRM.reg 2038 // X - SIB.index 2039 // B - MODRM.rm/SIB.base 2040 uint8_t rex = force ? 0x40 : 0; 2041 if (w) { 2042 rex |= 0x48; // REX.W000 2043 } 2044 if (r) { 2045 rex |= 0x44; // REX.0R00 2046 } 2047 if (x) { 2048 rex |= 0x42; // REX.00X0 2049 } 2050 if (b) { 2051 rex |= 0x41; // REX.000B 2052 } 2053 if (rex != 0) { 2054 EmitUint8(rex); 2055 } 2056} 2057 2058void X86_64Assembler::EmitOptionalRex32(CpuRegister reg) { 2059 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 2060} 2061 2062void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, CpuRegister src) { 2063 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2064} 2065 2066void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, XmmRegister src) { 2067 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2068} 2069 2070void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, XmmRegister src) { 2071 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2072} 2073 2074void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, CpuRegister src) { 2075 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2076} 2077 2078void X86_64Assembler::EmitOptionalRex32(const Operand& operand) { 2079 uint8_t rex = operand.rex(); 2080 if (rex != 0) { 2081 EmitUint8(rex); 2082 } 2083} 2084 2085void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, const Operand& operand) { 2086 uint8_t rex = operand.rex(); 2087 if (dst.NeedsRex()) { 2088 rex |= 0x44; // REX.0R00 2089 } 2090 if (rex != 0) { 2091 EmitUint8(rex); 2092 } 2093} 2094 2095void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, const Operand& operand) { 2096 uint8_t rex = operand.rex(); 2097 if (dst.NeedsRex()) { 2098 rex |= 0x44; // REX.0R00 2099 } 2100 if (rex != 0) { 2101 EmitUint8(rex); 2102 } 2103} 2104 2105void X86_64Assembler::EmitRex64() { 2106 EmitOptionalRex(false, true, false, false, false); 2107} 2108 2109void X86_64Assembler::EmitRex64(CpuRegister reg) { 2110 EmitOptionalRex(false, true, false, false, reg.NeedsRex()); 2111} 2112 2113void X86_64Assembler::EmitRex64(const Operand& operand) { 2114 uint8_t rex = operand.rex(); 2115 rex |= 0x48; // REX.W000 2116 EmitUint8(rex); 2117} 2118 2119void X86_64Assembler::EmitRex64(CpuRegister dst, CpuRegister src) { 2120 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2121} 2122 2123void X86_64Assembler::EmitRex64(XmmRegister dst, CpuRegister src) { 2124 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2125} 2126 2127void X86_64Assembler::EmitRex64(CpuRegister dst, XmmRegister src) { 2128 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2129} 2130 2131void X86_64Assembler::EmitRex64(CpuRegister dst, const Operand& operand) { 2132 uint8_t rex = 0x48 | operand.rex(); // REX.W000 2133 if (dst.NeedsRex()) { 2134 rex |= 0x44; // REX.0R00 2135 } 2136 if (rex != 0) { 2137 EmitUint8(rex); 2138 } 2139} 2140 2141void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, CpuRegister src) { 2142 EmitOptionalRex(true, false, dst.NeedsRex(), false, src.NeedsRex()); 2143} 2144 2145void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand) { 2146 uint8_t rex = 0x40 | operand.rex(); // REX.0000 2147 if (dst.NeedsRex()) { 2148 rex |= 0x44; // REX.0R00 2149 } 2150 if (rex != 0) { 2151 EmitUint8(rex); 2152 } 2153} 2154 2155void X86_64Assembler::InitializeFrameDescriptionEntry() { 2156 WriteFDEHeader(&cfi_info_, true /* is_64bit */); 2157} 2158 2159void X86_64Assembler::FinalizeFrameDescriptionEntry() { 2160 WriteFDEAddressRange(&cfi_info_, buffer_.Size(), true /* is_64bit */); 2161 PadCFI(&cfi_info_); 2162 WriteCFILength(&cfi_info_, true /* is_64bit */); 2163} 2164 2165constexpr size_t kFramePointerSize = 8; 2166 2167void X86_64Assembler::BuildFrame(size_t frame_size, ManagedRegister method_reg, 2168 const std::vector<ManagedRegister>& spill_regs, 2169 const ManagedRegisterEntrySpills& entry_spills) { 2170 cfi_cfa_offset_ = kFramePointerSize; // Only return address on stack 2171 cfi_pc_ = buffer_.Size(); // Nothing emitted yet 2172 DCHECK_EQ(cfi_pc_, 0U); 2173 2174 uint32_t reg_offset = 1; 2175 CHECK_ALIGNED(frame_size, kStackAlignment); 2176 int gpr_count = 0; 2177 for (int i = spill_regs.size() - 1; i >= 0; --i) { 2178 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2179 if (spill.IsCpuRegister()) { 2180 pushq(spill.AsCpuRegister()); 2181 gpr_count++; 2182 2183 // DW_CFA_advance_loc 2184 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 2185 cfi_pc_ = buffer_.Size(); 2186 // DW_CFA_def_cfa_offset 2187 cfi_cfa_offset_ += kFramePointerSize; 2188 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 2189 // DW_CFA_offset reg offset 2190 reg_offset++; 2191 DW_CFA_offset(&cfi_info_, spill.DWARFRegId(), reg_offset); 2192 } 2193 } 2194 // return address then method on stack 2195 int64_t rest_of_frame = static_cast<int64_t>(frame_size) 2196 - (gpr_count * kFramePointerSize) 2197 - kFramePointerSize /*return address*/; 2198 subq(CpuRegister(RSP), Immediate(rest_of_frame)); 2199 // DW_CFA_advance_loc 2200 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 2201 cfi_pc_ = buffer_.Size(); 2202 // DW_CFA_def_cfa_offset 2203 cfi_cfa_offset_ += rest_of_frame; 2204 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 2205 2206 // spill xmms 2207 int64_t offset = rest_of_frame; 2208 for (int i = spill_regs.size() - 1; i >= 0; --i) { 2209 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2210 if (spill.IsXmmRegister()) { 2211 offset -= sizeof(double); 2212 movsd(Address(CpuRegister(RSP), offset), spill.AsXmmRegister()); 2213 } 2214 } 2215 2216 DCHECK_EQ(4U, sizeof(StackReference<mirror::ArtMethod>)); 2217 2218 movl(Address(CpuRegister(RSP), 0), method_reg.AsX86_64().AsCpuRegister()); 2219 2220 for (size_t i = 0; i < entry_spills.size(); ++i) { 2221 ManagedRegisterSpill spill = entry_spills.at(i); 2222 if (spill.AsX86_64().IsCpuRegister()) { 2223 if (spill.getSize() == 8) { 2224 movq(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), 2225 spill.AsX86_64().AsCpuRegister()); 2226 } else { 2227 CHECK_EQ(spill.getSize(), 4); 2228 movl(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsCpuRegister()); 2229 } 2230 } else { 2231 if (spill.getSize() == 8) { 2232 movsd(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 2233 } else { 2234 CHECK_EQ(spill.getSize(), 4); 2235 movss(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 2236 } 2237 } 2238 } 2239} 2240 2241void X86_64Assembler::RemoveFrame(size_t frame_size, 2242 const std::vector<ManagedRegister>& spill_regs) { 2243 CHECK_ALIGNED(frame_size, kStackAlignment); 2244 int gpr_count = 0; 2245 // unspill xmms 2246 int64_t offset = static_cast<int64_t>(frame_size) - (spill_regs.size() * kFramePointerSize) - 2 * kFramePointerSize; 2247 for (size_t i = 0; i < spill_regs.size(); ++i) { 2248 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2249 if (spill.IsXmmRegister()) { 2250 offset += sizeof(double); 2251 movsd(spill.AsXmmRegister(), Address(CpuRegister(RSP), offset)); 2252 } else { 2253 gpr_count++; 2254 } 2255 } 2256 addq(CpuRegister(RSP), Immediate(static_cast<int64_t>(frame_size) - (gpr_count * kFramePointerSize) - kFramePointerSize)); 2257 for (size_t i = 0; i < spill_regs.size(); ++i) { 2258 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2259 if (spill.IsCpuRegister()) { 2260 popq(spill.AsCpuRegister()); 2261 } 2262 } 2263 ret(); 2264} 2265 2266void X86_64Assembler::IncreaseFrameSize(size_t adjust) { 2267 CHECK_ALIGNED(adjust, kStackAlignment); 2268 addq(CpuRegister(RSP), Immediate(-static_cast<int64_t>(adjust))); 2269 // DW_CFA_advance_loc 2270 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 2271 cfi_pc_ = buffer_.Size(); 2272 // DW_CFA_def_cfa_offset 2273 cfi_cfa_offset_ += adjust; 2274 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 2275} 2276 2277void X86_64Assembler::DecreaseFrameSize(size_t adjust) { 2278 CHECK_ALIGNED(adjust, kStackAlignment); 2279 addq(CpuRegister(RSP), Immediate(adjust)); 2280} 2281 2282void X86_64Assembler::Store(FrameOffset offs, ManagedRegister msrc, size_t size) { 2283 X86_64ManagedRegister src = msrc.AsX86_64(); 2284 if (src.IsNoRegister()) { 2285 CHECK_EQ(0u, size); 2286 } else if (src.IsCpuRegister()) { 2287 if (size == 4) { 2288 CHECK_EQ(4u, size); 2289 movl(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 2290 } else { 2291 CHECK_EQ(8u, size); 2292 movq(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 2293 } 2294 } else if (src.IsRegisterPair()) { 2295 CHECK_EQ(0u, size); 2296 movq(Address(CpuRegister(RSP), offs), src.AsRegisterPairLow()); 2297 movq(Address(CpuRegister(RSP), FrameOffset(offs.Int32Value()+4)), 2298 src.AsRegisterPairHigh()); 2299 } else if (src.IsX87Register()) { 2300 if (size == 4) { 2301 fstps(Address(CpuRegister(RSP), offs)); 2302 } else { 2303 fstpl(Address(CpuRegister(RSP), offs)); 2304 } 2305 } else { 2306 CHECK(src.IsXmmRegister()); 2307 if (size == 4) { 2308 movss(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 2309 } else { 2310 movsd(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 2311 } 2312 } 2313} 2314 2315void X86_64Assembler::StoreRef(FrameOffset dest, ManagedRegister msrc) { 2316 X86_64ManagedRegister src = msrc.AsX86_64(); 2317 CHECK(src.IsCpuRegister()); 2318 movl(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 2319} 2320 2321void X86_64Assembler::StoreRawPtr(FrameOffset dest, ManagedRegister msrc) { 2322 X86_64ManagedRegister src = msrc.AsX86_64(); 2323 CHECK(src.IsCpuRegister()); 2324 movq(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 2325} 2326 2327void X86_64Assembler::StoreImmediateToFrame(FrameOffset dest, uint32_t imm, 2328 ManagedRegister) { 2329 movl(Address(CpuRegister(RSP), dest), Immediate(imm)); // TODO(64) movq? 2330} 2331 2332void X86_64Assembler::StoreImmediateToThread64(ThreadOffset<8> dest, uint32_t imm, 2333 ManagedRegister) { 2334 gs()->movl(Address::Absolute(dest, true), Immediate(imm)); // TODO(64) movq? 2335} 2336 2337void X86_64Assembler::StoreStackOffsetToThread64(ThreadOffset<8> thr_offs, 2338 FrameOffset fr_offs, 2339 ManagedRegister mscratch) { 2340 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2341 CHECK(scratch.IsCpuRegister()); 2342 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), fr_offs)); 2343 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2344} 2345 2346void X86_64Assembler::StoreStackPointerToThread64(ThreadOffset<8> thr_offs) { 2347 gs()->movq(Address::Absolute(thr_offs, true), CpuRegister(RSP)); 2348} 2349 2350void X86_64Assembler::StoreSpanning(FrameOffset /*dst*/, ManagedRegister /*src*/, 2351 FrameOffset /*in_off*/, ManagedRegister /*scratch*/) { 2352 UNIMPLEMENTED(FATAL); // this case only currently exists for ARM 2353} 2354 2355void X86_64Assembler::Load(ManagedRegister mdest, FrameOffset src, size_t size) { 2356 X86_64ManagedRegister dest = mdest.AsX86_64(); 2357 if (dest.IsNoRegister()) { 2358 CHECK_EQ(0u, size); 2359 } else if (dest.IsCpuRegister()) { 2360 if (size == 4) { 2361 CHECK_EQ(4u, size); 2362 movl(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2363 } else { 2364 CHECK_EQ(8u, size); 2365 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2366 } 2367 } else if (dest.IsRegisterPair()) { 2368 CHECK_EQ(0u, size); 2369 movq(dest.AsRegisterPairLow(), Address(CpuRegister(RSP), src)); 2370 movq(dest.AsRegisterPairHigh(), Address(CpuRegister(RSP), FrameOffset(src.Int32Value()+4))); 2371 } else if (dest.IsX87Register()) { 2372 if (size == 4) { 2373 flds(Address(CpuRegister(RSP), src)); 2374 } else { 2375 fldl(Address(CpuRegister(RSP), src)); 2376 } 2377 } else { 2378 CHECK(dest.IsXmmRegister()); 2379 if (size == 4) { 2380 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2381 } else { 2382 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2383 } 2384 } 2385} 2386 2387void X86_64Assembler::LoadFromThread64(ManagedRegister mdest, ThreadOffset<8> src, size_t size) { 2388 X86_64ManagedRegister dest = mdest.AsX86_64(); 2389 if (dest.IsNoRegister()) { 2390 CHECK_EQ(0u, size); 2391 } else if (dest.IsCpuRegister()) { 2392 CHECK_EQ(4u, size); 2393 gs()->movl(dest.AsCpuRegister(), Address::Absolute(src, true)); 2394 } else if (dest.IsRegisterPair()) { 2395 CHECK_EQ(8u, size); 2396 gs()->movq(dest.AsRegisterPairLow(), Address::Absolute(src, true)); 2397 } else if (dest.IsX87Register()) { 2398 if (size == 4) { 2399 gs()->flds(Address::Absolute(src, true)); 2400 } else { 2401 gs()->fldl(Address::Absolute(src, true)); 2402 } 2403 } else { 2404 CHECK(dest.IsXmmRegister()); 2405 if (size == 4) { 2406 gs()->movss(dest.AsXmmRegister(), Address::Absolute(src, true)); 2407 } else { 2408 gs()->movsd(dest.AsXmmRegister(), Address::Absolute(src, true)); 2409 } 2410 } 2411} 2412 2413void X86_64Assembler::LoadRef(ManagedRegister mdest, FrameOffset src) { 2414 X86_64ManagedRegister dest = mdest.AsX86_64(); 2415 CHECK(dest.IsCpuRegister()); 2416 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2417} 2418 2419void X86_64Assembler::LoadRef(ManagedRegister mdest, ManagedRegister base, 2420 MemberOffset offs) { 2421 X86_64ManagedRegister dest = mdest.AsX86_64(); 2422 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2423 movl(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2424 if (kPoisonHeapReferences) { 2425 negl(dest.AsCpuRegister()); 2426 } 2427} 2428 2429void X86_64Assembler::LoadRawPtr(ManagedRegister mdest, ManagedRegister base, 2430 Offset offs) { 2431 X86_64ManagedRegister dest = mdest.AsX86_64(); 2432 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2433 movq(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2434} 2435 2436void X86_64Assembler::LoadRawPtrFromThread64(ManagedRegister mdest, ThreadOffset<8> offs) { 2437 X86_64ManagedRegister dest = mdest.AsX86_64(); 2438 CHECK(dest.IsCpuRegister()); 2439 gs()->movq(dest.AsCpuRegister(), Address::Absolute(offs, true)); 2440} 2441 2442void X86_64Assembler::SignExtend(ManagedRegister mreg, size_t size) { 2443 X86_64ManagedRegister reg = mreg.AsX86_64(); 2444 CHECK(size == 1 || size == 2) << size; 2445 CHECK(reg.IsCpuRegister()) << reg; 2446 if (size == 1) { 2447 movsxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2448 } else { 2449 movsxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2450 } 2451} 2452 2453void X86_64Assembler::ZeroExtend(ManagedRegister mreg, size_t size) { 2454 X86_64ManagedRegister reg = mreg.AsX86_64(); 2455 CHECK(size == 1 || size == 2) << size; 2456 CHECK(reg.IsCpuRegister()) << reg; 2457 if (size == 1) { 2458 movzxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2459 } else { 2460 movzxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2461 } 2462} 2463 2464void X86_64Assembler::Move(ManagedRegister mdest, ManagedRegister msrc, size_t size) { 2465 X86_64ManagedRegister dest = mdest.AsX86_64(); 2466 X86_64ManagedRegister src = msrc.AsX86_64(); 2467 if (!dest.Equals(src)) { 2468 if (dest.IsCpuRegister() && src.IsCpuRegister()) { 2469 movq(dest.AsCpuRegister(), src.AsCpuRegister()); 2470 } else if (src.IsX87Register() && dest.IsXmmRegister()) { 2471 // Pass via stack and pop X87 register 2472 subl(CpuRegister(RSP), Immediate(16)); 2473 if (size == 4) { 2474 CHECK_EQ(src.AsX87Register(), ST0); 2475 fstps(Address(CpuRegister(RSP), 0)); 2476 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2477 } else { 2478 CHECK_EQ(src.AsX87Register(), ST0); 2479 fstpl(Address(CpuRegister(RSP), 0)); 2480 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2481 } 2482 addq(CpuRegister(RSP), Immediate(16)); 2483 } else { 2484 // TODO: x87, SSE 2485 UNIMPLEMENTED(FATAL) << ": Move " << dest << ", " << src; 2486 } 2487 } 2488} 2489 2490void X86_64Assembler::CopyRef(FrameOffset dest, FrameOffset src, 2491 ManagedRegister mscratch) { 2492 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2493 CHECK(scratch.IsCpuRegister()); 2494 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2495 movl(Address(CpuRegister(RSP), dest), scratch.AsCpuRegister()); 2496} 2497 2498void X86_64Assembler::CopyRawPtrFromThread64(FrameOffset fr_offs, 2499 ThreadOffset<8> thr_offs, 2500 ManagedRegister mscratch) { 2501 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2502 CHECK(scratch.IsCpuRegister()); 2503 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(thr_offs, true)); 2504 Store(fr_offs, scratch, 8); 2505} 2506 2507void X86_64Assembler::CopyRawPtrToThread64(ThreadOffset<8> thr_offs, 2508 FrameOffset fr_offs, 2509 ManagedRegister mscratch) { 2510 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2511 CHECK(scratch.IsCpuRegister()); 2512 Load(scratch, fr_offs, 8); 2513 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2514} 2515 2516void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src, 2517 ManagedRegister mscratch, 2518 size_t size) { 2519 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2520 if (scratch.IsCpuRegister() && size == 8) { 2521 Load(scratch, src, 4); 2522 Store(dest, scratch, 4); 2523 Load(scratch, FrameOffset(src.Int32Value() + 4), 4); 2524 Store(FrameOffset(dest.Int32Value() + 4), scratch, 4); 2525 } else { 2526 Load(scratch, src, size); 2527 Store(dest, scratch, size); 2528 } 2529} 2530 2531void X86_64Assembler::Copy(FrameOffset /*dst*/, ManagedRegister /*src_base*/, Offset /*src_offset*/, 2532 ManagedRegister /*scratch*/, size_t /*size*/) { 2533 UNIMPLEMENTED(FATAL); 2534} 2535 2536void X86_64Assembler::Copy(ManagedRegister dest_base, Offset dest_offset, FrameOffset src, 2537 ManagedRegister scratch, size_t size) { 2538 CHECK(scratch.IsNoRegister()); 2539 CHECK_EQ(size, 4u); 2540 pushq(Address(CpuRegister(RSP), src)); 2541 popq(Address(dest_base.AsX86_64().AsCpuRegister(), dest_offset)); 2542} 2543 2544void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src_base, Offset src_offset, 2545 ManagedRegister mscratch, size_t size) { 2546 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2547 CHECK_EQ(size, 4u); 2548 movq(scratch, Address(CpuRegister(RSP), src_base)); 2549 movq(scratch, Address(scratch, src_offset)); 2550 movq(Address(CpuRegister(RSP), dest), scratch); 2551} 2552 2553void X86_64Assembler::Copy(ManagedRegister dest, Offset dest_offset, 2554 ManagedRegister src, Offset src_offset, 2555 ManagedRegister scratch, size_t size) { 2556 CHECK_EQ(size, 4u); 2557 CHECK(scratch.IsNoRegister()); 2558 pushq(Address(src.AsX86_64().AsCpuRegister(), src_offset)); 2559 popq(Address(dest.AsX86_64().AsCpuRegister(), dest_offset)); 2560} 2561 2562void X86_64Assembler::Copy(FrameOffset dest, Offset dest_offset, FrameOffset src, Offset src_offset, 2563 ManagedRegister mscratch, size_t size) { 2564 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2565 CHECK_EQ(size, 4u); 2566 CHECK_EQ(dest.Int32Value(), src.Int32Value()); 2567 movq(scratch, Address(CpuRegister(RSP), src)); 2568 pushq(Address(scratch, src_offset)); 2569 popq(Address(scratch, dest_offset)); 2570} 2571 2572void X86_64Assembler::MemoryBarrier(ManagedRegister) { 2573 mfence(); 2574} 2575 2576void X86_64Assembler::CreateHandleScopeEntry(ManagedRegister mout_reg, 2577 FrameOffset handle_scope_offset, 2578 ManagedRegister min_reg, bool null_allowed) { 2579 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2580 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2581 if (in_reg.IsNoRegister()) { // TODO(64): && null_allowed 2582 // Use out_reg as indicator of NULL 2583 in_reg = out_reg; 2584 // TODO: movzwl 2585 movl(in_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2586 } 2587 CHECK(in_reg.IsCpuRegister()); 2588 CHECK(out_reg.IsCpuRegister()); 2589 VerifyObject(in_reg, null_allowed); 2590 if (null_allowed) { 2591 Label null_arg; 2592 if (!out_reg.Equals(in_reg)) { 2593 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2594 } 2595 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2596 j(kZero, &null_arg); 2597 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2598 Bind(&null_arg); 2599 } else { 2600 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2601 } 2602} 2603 2604void X86_64Assembler::CreateHandleScopeEntry(FrameOffset out_off, 2605 FrameOffset handle_scope_offset, 2606 ManagedRegister mscratch, 2607 bool null_allowed) { 2608 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2609 CHECK(scratch.IsCpuRegister()); 2610 if (null_allowed) { 2611 Label null_arg; 2612 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2613 testl(scratch.AsCpuRegister(), scratch.AsCpuRegister()); 2614 j(kZero, &null_arg); 2615 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2616 Bind(&null_arg); 2617 } else { 2618 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2619 } 2620 Store(out_off, scratch, 8); 2621} 2622 2623// Given a handle scope entry, load the associated reference. 2624void X86_64Assembler::LoadReferenceFromHandleScope(ManagedRegister mout_reg, 2625 ManagedRegister min_reg) { 2626 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2627 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2628 CHECK(out_reg.IsCpuRegister()); 2629 CHECK(in_reg.IsCpuRegister()); 2630 Label null_arg; 2631 if (!out_reg.Equals(in_reg)) { 2632 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2633 } 2634 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2635 j(kZero, &null_arg); 2636 movq(out_reg.AsCpuRegister(), Address(in_reg.AsCpuRegister(), 0)); 2637 Bind(&null_arg); 2638} 2639 2640void X86_64Assembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) { 2641 // TODO: not validating references 2642} 2643 2644void X86_64Assembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) { 2645 // TODO: not validating references 2646} 2647 2648void X86_64Assembler::Call(ManagedRegister mbase, Offset offset, ManagedRegister) { 2649 X86_64ManagedRegister base = mbase.AsX86_64(); 2650 CHECK(base.IsCpuRegister()); 2651 call(Address(base.AsCpuRegister(), offset.Int32Value())); 2652 // TODO: place reference map on call 2653} 2654 2655void X86_64Assembler::Call(FrameOffset base, Offset offset, ManagedRegister mscratch) { 2656 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2657 movl(scratch, Address(CpuRegister(RSP), base)); 2658 call(Address(scratch, offset)); 2659} 2660 2661void X86_64Assembler::CallFromThread64(ThreadOffset<8> offset, ManagedRegister /*mscratch*/) { 2662 gs()->call(Address::Absolute(offset, true)); 2663} 2664 2665void X86_64Assembler::GetCurrentThread(ManagedRegister tr) { 2666 gs()->movq(tr.AsX86_64().AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2667} 2668 2669void X86_64Assembler::GetCurrentThread(FrameOffset offset, ManagedRegister mscratch) { 2670 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2671 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2672 movq(Address(CpuRegister(RSP), offset), scratch.AsCpuRegister()); 2673} 2674 2675// Slowpath entered when Thread::Current()->_exception is non-null 2676class X86_64ExceptionSlowPath FINAL : public SlowPath { 2677 public: 2678 explicit X86_64ExceptionSlowPath(size_t stack_adjust) : stack_adjust_(stack_adjust) {} 2679 virtual void Emit(Assembler *sp_asm) OVERRIDE; 2680 private: 2681 const size_t stack_adjust_; 2682}; 2683 2684void X86_64Assembler::ExceptionPoll(ManagedRegister /*scratch*/, size_t stack_adjust) { 2685 X86_64ExceptionSlowPath* slow = new X86_64ExceptionSlowPath(stack_adjust); 2686 buffer_.EnqueueSlowPath(slow); 2687 gs()->cmpl(Address::Absolute(Thread::ExceptionOffset<8>(), true), Immediate(0)); 2688 j(kNotEqual, slow->Entry()); 2689} 2690 2691void X86_64ExceptionSlowPath::Emit(Assembler *sasm) { 2692 X86_64Assembler* sp_asm = down_cast<X86_64Assembler*>(sasm); 2693#define __ sp_asm-> 2694 __ Bind(&entry_); 2695 // Note: the return value is dead 2696 if (stack_adjust_ != 0) { // Fix up the frame. 2697 __ DecreaseFrameSize(stack_adjust_); 2698 } 2699 // Pass exception as argument in RDI 2700 __ gs()->movq(CpuRegister(RDI), Address::Absolute(Thread::ExceptionOffset<8>(), true)); 2701 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(8, pDeliverException), true)); 2702 // this call should never return 2703 __ int3(); 2704#undef __ 2705} 2706 2707} // namespace x86_64 2708} // namespace art 2709