assembler_x86_64.cc revision fb8d279bc011b31d0765dc7ca59afea324fd0d0c
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "assembler_x86_64.h" 18 19#include "base/casts.h" 20#include "entrypoints/quick/quick_entrypoints.h" 21#include "memory_region.h" 22#include "thread.h" 23#include "utils/dwarf_cfi.h" 24 25namespace art { 26namespace x86_64 { 27 28std::ostream& operator<<(std::ostream& os, const CpuRegister& reg) { 29 return os << reg.AsRegister(); 30} 31 32std::ostream& operator<<(std::ostream& os, const XmmRegister& reg) { 33 return os << reg.AsFloatRegister(); 34} 35 36std::ostream& operator<<(std::ostream& os, const X87Register& reg) { 37 return os << "ST" << static_cast<int>(reg); 38} 39 40void X86_64Assembler::call(CpuRegister reg) { 41 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 42 EmitOptionalRex32(reg); 43 EmitUint8(0xFF); 44 EmitRegisterOperand(2, reg.LowBits()); 45} 46 47 48void X86_64Assembler::call(const Address& address) { 49 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 50 EmitOptionalRex32(address); 51 EmitUint8(0xFF); 52 EmitOperand(2, address); 53} 54 55 56void X86_64Assembler::call(Label* label) { 57 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 58 EmitUint8(0xE8); 59 static const int kSize = 5; 60 // Offset by one because we already have emitted the opcode. 61 EmitLabel(label, kSize - 1); 62} 63 64void X86_64Assembler::pushq(CpuRegister reg) { 65 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 66 EmitOptionalRex32(reg); 67 EmitUint8(0x50 + reg.LowBits()); 68} 69 70 71void X86_64Assembler::pushq(const Address& address) { 72 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 73 EmitOptionalRex32(address); 74 EmitUint8(0xFF); 75 EmitOperand(6, address); 76} 77 78 79void X86_64Assembler::pushq(const Immediate& imm) { 80 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 81 CHECK(imm.is_int32()); // pushq only supports 32b immediate. 82 if (imm.is_int8()) { 83 EmitUint8(0x6A); 84 EmitUint8(imm.value() & 0xFF); 85 } else { 86 EmitUint8(0x68); 87 EmitImmediate(imm); 88 } 89} 90 91 92void X86_64Assembler::popq(CpuRegister reg) { 93 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 94 EmitOptionalRex32(reg); 95 EmitUint8(0x58 + reg.LowBits()); 96} 97 98 99void X86_64Assembler::popq(const Address& address) { 100 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 101 EmitOptionalRex32(address); 102 EmitUint8(0x8F); 103 EmitOperand(0, address); 104} 105 106 107void X86_64Assembler::movq(CpuRegister dst, const Immediate& imm) { 108 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 109 if (imm.is_int32()) { 110 // 32 bit. Note: sign-extends. 111 EmitRex64(dst); 112 EmitUint8(0xC7); 113 EmitRegisterOperand(0, dst.LowBits()); 114 EmitInt32(static_cast<int32_t>(imm.value())); 115 } else { 116 EmitRex64(dst); 117 EmitUint8(0xB8 + dst.LowBits()); 118 EmitInt64(imm.value()); 119 } 120} 121 122 123void X86_64Assembler::movl(CpuRegister dst, const Immediate& imm) { 124 CHECK(imm.is_int32()); 125 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 126 EmitOptionalRex32(dst); 127 EmitUint8(0xB8 + dst.LowBits()); 128 EmitImmediate(imm); 129} 130 131 132void X86_64Assembler::movq(CpuRegister dst, CpuRegister src) { 133 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 134 // 0x89 is movq r/m64 <- r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 135 EmitRex64(src, dst); 136 EmitUint8(0x89); 137 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 138} 139 140 141void X86_64Assembler::movl(CpuRegister dst, CpuRegister src) { 142 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 143 EmitOptionalRex32(dst, src); 144 EmitUint8(0x8B); 145 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 146} 147 148 149void X86_64Assembler::movq(CpuRegister dst, const Address& src) { 150 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 151 EmitRex64(dst, src); 152 EmitUint8(0x8B); 153 EmitOperand(dst.LowBits(), src); 154} 155 156 157void X86_64Assembler::movl(CpuRegister dst, const Address& src) { 158 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 159 EmitOptionalRex32(dst, src); 160 EmitUint8(0x8B); 161 EmitOperand(dst.LowBits(), src); 162} 163 164 165void X86_64Assembler::movq(const Address& dst, CpuRegister src) { 166 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 167 EmitRex64(src, dst); 168 EmitUint8(0x89); 169 EmitOperand(src.LowBits(), dst); 170} 171 172 173void X86_64Assembler::movl(const Address& dst, CpuRegister src) { 174 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 175 EmitOptionalRex32(src, dst); 176 EmitUint8(0x89); 177 EmitOperand(src.LowBits(), dst); 178} 179 180void X86_64Assembler::movl(const Address& dst, const Immediate& imm) { 181 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 182 EmitOptionalRex32(dst); 183 EmitUint8(0xC7); 184 EmitOperand(0, dst); 185 EmitImmediate(imm); 186} 187 188 189void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src) { 190 cmov(c, dst, src, true); 191} 192 193void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src, bool is64bit) { 194 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 195 EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex()); 196 EmitUint8(0x0F); 197 EmitUint8(0x40 + c); 198 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 199} 200 201 202void X86_64Assembler::movzxb(CpuRegister dst, CpuRegister src) { 203 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 204 EmitOptionalByteRegNormalizingRex32(dst, src); 205 EmitUint8(0x0F); 206 EmitUint8(0xB6); 207 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 208} 209 210 211void X86_64Assembler::movzxb(CpuRegister dst, const Address& src) { 212 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 213 EmitOptionalByteRegNormalizingRex32(dst, src); 214 EmitUint8(0x0F); 215 EmitUint8(0xB6); 216 EmitOperand(dst.LowBits(), src); 217} 218 219 220void X86_64Assembler::movsxb(CpuRegister dst, CpuRegister src) { 221 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 222 EmitOptionalByteRegNormalizingRex32(dst, src); 223 EmitUint8(0x0F); 224 EmitUint8(0xBE); 225 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 226} 227 228 229void X86_64Assembler::movsxb(CpuRegister dst, const Address& src) { 230 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 231 EmitOptionalByteRegNormalizingRex32(dst, src); 232 EmitUint8(0x0F); 233 EmitUint8(0xBE); 234 EmitOperand(dst.LowBits(), src); 235} 236 237 238void X86_64Assembler::movb(CpuRegister /*dst*/, const Address& /*src*/) { 239 LOG(FATAL) << "Use movzxb or movsxb instead."; 240} 241 242 243void X86_64Assembler::movb(const Address& dst, CpuRegister src) { 244 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 245 EmitOptionalByteRegNormalizingRex32(src, dst); 246 EmitUint8(0x88); 247 EmitOperand(src.LowBits(), dst); 248} 249 250 251void X86_64Assembler::movb(const Address& dst, const Immediate& imm) { 252 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 253 EmitOptionalRex32(dst); 254 EmitUint8(0xC6); 255 EmitOperand(Register::RAX, dst); 256 CHECK(imm.is_int8()); 257 EmitUint8(imm.value() & 0xFF); 258} 259 260 261void X86_64Assembler::movzxw(CpuRegister dst, CpuRegister src) { 262 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 263 EmitOptionalRex32(dst, src); 264 EmitUint8(0x0F); 265 EmitUint8(0xB7); 266 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 267} 268 269 270void X86_64Assembler::movzxw(CpuRegister dst, const Address& src) { 271 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 272 EmitOptionalRex32(dst, src); 273 EmitUint8(0x0F); 274 EmitUint8(0xB7); 275 EmitOperand(dst.LowBits(), src); 276} 277 278 279void X86_64Assembler::movsxw(CpuRegister dst, CpuRegister src) { 280 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 281 EmitOptionalRex32(dst, src); 282 EmitUint8(0x0F); 283 EmitUint8(0xBF); 284 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 285} 286 287 288void X86_64Assembler::movsxw(CpuRegister dst, const Address& src) { 289 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 290 EmitOptionalRex32(dst, src); 291 EmitUint8(0x0F); 292 EmitUint8(0xBF); 293 EmitOperand(dst.LowBits(), src); 294} 295 296 297void X86_64Assembler::movw(CpuRegister /*dst*/, const Address& /*src*/) { 298 LOG(FATAL) << "Use movzxw or movsxw instead."; 299} 300 301 302void X86_64Assembler::movw(const Address& dst, CpuRegister src) { 303 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 304 EmitOperandSizeOverride(); 305 EmitOptionalRex32(src, dst); 306 EmitUint8(0x89); 307 EmitOperand(src.LowBits(), dst); 308} 309 310 311void X86_64Assembler::movw(const Address& dst, const Immediate& imm) { 312 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 313 EmitOperandSizeOverride(); 314 EmitOptionalRex32(dst); 315 EmitUint8(0xC7); 316 EmitOperand(Register::RAX, dst); 317 CHECK(imm.is_uint16() || imm.is_int16()); 318 EmitUint8(imm.value() & 0xFF); 319 EmitUint8(imm.value() >> 8); 320} 321 322 323void X86_64Assembler::leaq(CpuRegister dst, const Address& src) { 324 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 325 EmitRex64(dst, src); 326 EmitUint8(0x8D); 327 EmitOperand(dst.LowBits(), src); 328} 329 330 331void X86_64Assembler::leal(CpuRegister dst, const Address& src) { 332 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 333 EmitOptionalRex32(dst, src); 334 EmitUint8(0x8D); 335 EmitOperand(dst.LowBits(), src); 336} 337 338 339void X86_64Assembler::movaps(XmmRegister dst, XmmRegister src) { 340 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 341 EmitOptionalRex32(dst, src); 342 EmitUint8(0x0F); 343 EmitUint8(0x28); 344 EmitXmmRegisterOperand(dst.LowBits(), src); 345} 346 347 348void X86_64Assembler::movss(XmmRegister dst, const Address& src) { 349 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 350 EmitUint8(0xF3); 351 EmitOptionalRex32(dst, src); 352 EmitUint8(0x0F); 353 EmitUint8(0x10); 354 EmitOperand(dst.LowBits(), src); 355} 356 357 358void X86_64Assembler::movss(const Address& dst, XmmRegister src) { 359 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 360 EmitUint8(0xF3); 361 EmitOptionalRex32(src, dst); 362 EmitUint8(0x0F); 363 EmitUint8(0x11); 364 EmitOperand(src.LowBits(), dst); 365} 366 367 368void X86_64Assembler::movss(XmmRegister dst, XmmRegister src) { 369 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 370 EmitUint8(0xF3); 371 EmitOptionalRex32(src, dst); // Movss is MR encoding instead of the usual RM. 372 EmitUint8(0x0F); 373 EmitUint8(0x11); 374 EmitXmmRegisterOperand(src.LowBits(), dst); 375} 376 377 378void X86_64Assembler::movsxd(CpuRegister dst, CpuRegister src) { 379 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 380 EmitRex64(dst, src); 381 EmitUint8(0x63); 382 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 383} 384 385 386void X86_64Assembler::movsxd(CpuRegister dst, const Address& src) { 387 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 388 EmitRex64(dst); 389 EmitUint8(0x63); 390 EmitOperand(dst.LowBits(), src); 391} 392 393 394void X86_64Assembler::movd(XmmRegister dst, CpuRegister src) { 395 movd(dst, src, true); 396} 397 398void X86_64Assembler::movd(CpuRegister dst, XmmRegister src) { 399 movd(dst, src, true); 400} 401 402void X86_64Assembler::movd(XmmRegister dst, CpuRegister src, bool is64bit) { 403 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 404 EmitUint8(0x66); 405 EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex()); 406 EmitUint8(0x0F); 407 EmitUint8(0x6E); 408 EmitOperand(dst.LowBits(), Operand(src)); 409} 410 411void X86_64Assembler::movd(CpuRegister dst, XmmRegister src, bool is64bit) { 412 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 413 EmitUint8(0x66); 414 EmitOptionalRex(false, is64bit, src.NeedsRex(), false, dst.NeedsRex()); 415 EmitUint8(0x0F); 416 EmitUint8(0x7E); 417 EmitOperand(src.LowBits(), Operand(dst)); 418} 419 420 421void X86_64Assembler::addss(XmmRegister dst, XmmRegister src) { 422 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 423 EmitUint8(0xF3); 424 EmitOptionalRex32(dst, src); 425 EmitUint8(0x0F); 426 EmitUint8(0x58); 427 EmitXmmRegisterOperand(dst.LowBits(), src); 428} 429 430 431void X86_64Assembler::addss(XmmRegister dst, const Address& src) { 432 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 433 EmitUint8(0xF3); 434 EmitOptionalRex32(dst, src); 435 EmitUint8(0x0F); 436 EmitUint8(0x58); 437 EmitOperand(dst.LowBits(), src); 438} 439 440 441void X86_64Assembler::subss(XmmRegister dst, XmmRegister src) { 442 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 443 EmitUint8(0xF3); 444 EmitOptionalRex32(dst, src); 445 EmitUint8(0x0F); 446 EmitUint8(0x5C); 447 EmitXmmRegisterOperand(dst.LowBits(), src); 448} 449 450 451void X86_64Assembler::subss(XmmRegister dst, const Address& src) { 452 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 453 EmitUint8(0xF3); 454 EmitOptionalRex32(dst, src); 455 EmitUint8(0x0F); 456 EmitUint8(0x5C); 457 EmitOperand(dst.LowBits(), src); 458} 459 460 461void X86_64Assembler::mulss(XmmRegister dst, XmmRegister src) { 462 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 463 EmitUint8(0xF3); 464 EmitOptionalRex32(dst, src); 465 EmitUint8(0x0F); 466 EmitUint8(0x59); 467 EmitXmmRegisterOperand(dst.LowBits(), src); 468} 469 470 471void X86_64Assembler::mulss(XmmRegister dst, const Address& src) { 472 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 473 EmitUint8(0xF3); 474 EmitOptionalRex32(dst, src); 475 EmitUint8(0x0F); 476 EmitUint8(0x59); 477 EmitOperand(dst.LowBits(), src); 478} 479 480 481void X86_64Assembler::divss(XmmRegister dst, XmmRegister src) { 482 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 483 EmitUint8(0xF3); 484 EmitOptionalRex32(dst, src); 485 EmitUint8(0x0F); 486 EmitUint8(0x5E); 487 EmitXmmRegisterOperand(dst.LowBits(), src); 488} 489 490 491void X86_64Assembler::divss(XmmRegister dst, const Address& src) { 492 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 493 EmitUint8(0xF3); 494 EmitOptionalRex32(dst, src); 495 EmitUint8(0x0F); 496 EmitUint8(0x5E); 497 EmitOperand(dst.LowBits(), src); 498} 499 500 501void X86_64Assembler::flds(const Address& src) { 502 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 503 EmitUint8(0xD9); 504 EmitOperand(0, src); 505} 506 507 508void X86_64Assembler::fsts(const Address& dst) { 509 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 510 EmitUint8(0xD9); 511 EmitOperand(2, dst); 512} 513 514 515void X86_64Assembler::fstps(const Address& dst) { 516 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 517 EmitUint8(0xD9); 518 EmitOperand(3, dst); 519} 520 521 522void X86_64Assembler::movsd(XmmRegister dst, const Address& src) { 523 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 524 EmitUint8(0xF2); 525 EmitOptionalRex32(dst, src); 526 EmitUint8(0x0F); 527 EmitUint8(0x10); 528 EmitOperand(dst.LowBits(), src); 529} 530 531 532void X86_64Assembler::movsd(const Address& dst, XmmRegister src) { 533 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 534 EmitUint8(0xF2); 535 EmitOptionalRex32(src, dst); 536 EmitUint8(0x0F); 537 EmitUint8(0x11); 538 EmitOperand(src.LowBits(), dst); 539} 540 541 542void X86_64Assembler::movsd(XmmRegister dst, XmmRegister src) { 543 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 544 EmitUint8(0xF2); 545 EmitOptionalRex32(src, dst); // Movsd is MR encoding instead of the usual RM. 546 EmitUint8(0x0F); 547 EmitUint8(0x11); 548 EmitXmmRegisterOperand(src.LowBits(), dst); 549} 550 551 552void X86_64Assembler::addsd(XmmRegister dst, XmmRegister src) { 553 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 554 EmitUint8(0xF2); 555 EmitOptionalRex32(dst, src); 556 EmitUint8(0x0F); 557 EmitUint8(0x58); 558 EmitXmmRegisterOperand(dst.LowBits(), src); 559} 560 561 562void X86_64Assembler::addsd(XmmRegister dst, const Address& src) { 563 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 564 EmitUint8(0xF2); 565 EmitOptionalRex32(dst, src); 566 EmitUint8(0x0F); 567 EmitUint8(0x58); 568 EmitOperand(dst.LowBits(), src); 569} 570 571 572void X86_64Assembler::subsd(XmmRegister dst, XmmRegister src) { 573 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 574 EmitUint8(0xF2); 575 EmitOptionalRex32(dst, src); 576 EmitUint8(0x0F); 577 EmitUint8(0x5C); 578 EmitXmmRegisterOperand(dst.LowBits(), src); 579} 580 581 582void X86_64Assembler::subsd(XmmRegister dst, const Address& src) { 583 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 584 EmitUint8(0xF2); 585 EmitOptionalRex32(dst, src); 586 EmitUint8(0x0F); 587 EmitUint8(0x5C); 588 EmitOperand(dst.LowBits(), src); 589} 590 591 592void X86_64Assembler::mulsd(XmmRegister dst, XmmRegister src) { 593 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 594 EmitUint8(0xF2); 595 EmitOptionalRex32(dst, src); 596 EmitUint8(0x0F); 597 EmitUint8(0x59); 598 EmitXmmRegisterOperand(dst.LowBits(), src); 599} 600 601 602void X86_64Assembler::mulsd(XmmRegister dst, const Address& src) { 603 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 604 EmitUint8(0xF2); 605 EmitOptionalRex32(dst, src); 606 EmitUint8(0x0F); 607 EmitUint8(0x59); 608 EmitOperand(dst.LowBits(), src); 609} 610 611 612void X86_64Assembler::divsd(XmmRegister dst, XmmRegister src) { 613 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 614 EmitUint8(0xF2); 615 EmitOptionalRex32(dst, src); 616 EmitUint8(0x0F); 617 EmitUint8(0x5E); 618 EmitXmmRegisterOperand(dst.LowBits(), src); 619} 620 621 622void X86_64Assembler::divsd(XmmRegister dst, const Address& src) { 623 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 624 EmitUint8(0xF2); 625 EmitOptionalRex32(dst, src); 626 EmitUint8(0x0F); 627 EmitUint8(0x5E); 628 EmitOperand(dst.LowBits(), src); 629} 630 631 632void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) { 633 cvtsi2ss(dst, src, false); 634} 635 636 637void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src, bool is64bit) { 638 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 639 EmitUint8(0xF3); 640 if (is64bit) { 641 // Emit a REX.W prefix if the operand size is 64 bits. 642 EmitRex64(dst, src); 643 } else { 644 EmitOptionalRex32(dst, src); 645 } 646 EmitUint8(0x0F); 647 EmitUint8(0x2A); 648 EmitOperand(dst.LowBits(), Operand(src)); 649} 650 651 652void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src) { 653 cvtsi2sd(dst, src, false); 654} 655 656 657void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src, bool is64bit) { 658 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 659 EmitUint8(0xF2); 660 if (is64bit) { 661 // Emit a REX.W prefix if the operand size is 64 bits. 662 EmitRex64(dst, src); 663 } else { 664 EmitOptionalRex32(dst, src); 665 } 666 EmitUint8(0x0F); 667 EmitUint8(0x2A); 668 EmitOperand(dst.LowBits(), Operand(src)); 669} 670 671 672void X86_64Assembler::cvtss2si(CpuRegister dst, XmmRegister src) { 673 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 674 EmitUint8(0xF3); 675 EmitOptionalRex32(dst, src); 676 EmitUint8(0x0F); 677 EmitUint8(0x2D); 678 EmitXmmRegisterOperand(dst.LowBits(), src); 679} 680 681 682void X86_64Assembler::cvtss2sd(XmmRegister dst, XmmRegister src) { 683 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 684 EmitUint8(0xF3); 685 EmitOptionalRex32(dst, src); 686 EmitUint8(0x0F); 687 EmitUint8(0x5A); 688 EmitXmmRegisterOperand(dst.LowBits(), src); 689} 690 691 692void X86_64Assembler::cvtsd2si(CpuRegister dst, XmmRegister src) { 693 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 694 EmitUint8(0xF2); 695 EmitOptionalRex32(dst, src); 696 EmitUint8(0x0F); 697 EmitUint8(0x2D); 698 EmitXmmRegisterOperand(dst.LowBits(), src); 699} 700 701 702void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src) { 703 cvttss2si(dst, src, false); 704} 705 706 707void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src, bool is64bit) { 708 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 709 EmitUint8(0xF3); 710 if (is64bit) { 711 // Emit a REX.W prefix if the operand size is 64 bits. 712 EmitRex64(dst, src); 713 } else { 714 EmitOptionalRex32(dst, src); 715 } 716 EmitUint8(0x0F); 717 EmitUint8(0x2C); 718 EmitXmmRegisterOperand(dst.LowBits(), src); 719} 720 721 722void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src) { 723 cvttsd2si(dst, src, false); 724} 725 726 727void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src, bool is64bit) { 728 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 729 EmitUint8(0xF2); 730 if (is64bit) { 731 // Emit a REX.W prefix if the operand size is 64 bits. 732 EmitRex64(dst, src); 733 } else { 734 EmitOptionalRex32(dst, src); 735 } 736 EmitUint8(0x0F); 737 EmitUint8(0x2C); 738 EmitXmmRegisterOperand(dst.LowBits(), src); 739} 740 741 742void X86_64Assembler::cvtsd2ss(XmmRegister dst, XmmRegister src) { 743 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 744 EmitUint8(0xF2); 745 EmitOptionalRex32(dst, src); 746 EmitUint8(0x0F); 747 EmitUint8(0x5A); 748 EmitXmmRegisterOperand(dst.LowBits(), src); 749} 750 751 752void X86_64Assembler::cvtdq2pd(XmmRegister dst, XmmRegister src) { 753 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 754 EmitUint8(0xF3); 755 EmitOptionalRex32(dst, src); 756 EmitUint8(0x0F); 757 EmitUint8(0xE6); 758 EmitXmmRegisterOperand(dst.LowBits(), src); 759} 760 761 762void X86_64Assembler::comiss(XmmRegister a, XmmRegister b) { 763 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 764 EmitOptionalRex32(a, b); 765 EmitUint8(0x0F); 766 EmitUint8(0x2F); 767 EmitXmmRegisterOperand(a.LowBits(), b); 768} 769 770 771void X86_64Assembler::comisd(XmmRegister a, XmmRegister b) { 772 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 773 EmitUint8(0x66); 774 EmitOptionalRex32(a, b); 775 EmitUint8(0x0F); 776 EmitUint8(0x2F); 777 EmitXmmRegisterOperand(a.LowBits(), b); 778} 779 780void X86_64Assembler::ucomiss(XmmRegister a, XmmRegister b) { 781 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 782 EmitOptionalRex32(a, b); 783 EmitUint8(0x0F); 784 EmitUint8(0x2E); 785 EmitXmmRegisterOperand(a.LowBits(), b); 786} 787 788 789void X86_64Assembler::ucomisd(XmmRegister a, XmmRegister b) { 790 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 791 EmitUint8(0x66); 792 EmitOptionalRex32(a, b); 793 EmitUint8(0x0F); 794 EmitUint8(0x2E); 795 EmitXmmRegisterOperand(a.LowBits(), b); 796} 797 798 799void X86_64Assembler::roundsd(XmmRegister dst, XmmRegister src, const Immediate& imm) { 800 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 801 EmitUint8(0x66); 802 EmitOptionalRex32(dst, src); 803 EmitUint8(0x0F); 804 EmitUint8(0x3A); 805 EmitUint8(0x0B); 806 EmitXmmRegisterOperand(dst.LowBits(), src); 807 EmitUint8(imm.value()); 808} 809 810 811void X86_64Assembler::roundss(XmmRegister dst, XmmRegister src, const Immediate& imm) { 812 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 813 EmitUint8(0x66); 814 EmitOptionalRex32(dst, src); 815 EmitUint8(0x0F); 816 EmitUint8(0x3A); 817 EmitUint8(0x0A); 818 EmitXmmRegisterOperand(dst.LowBits(), src); 819 EmitUint8(imm.value()); 820} 821 822 823void X86_64Assembler::sqrtsd(XmmRegister dst, XmmRegister src) { 824 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 825 EmitUint8(0xF2); 826 EmitOptionalRex32(dst, src); 827 EmitUint8(0x0F); 828 EmitUint8(0x51); 829 EmitXmmRegisterOperand(dst.LowBits(), src); 830} 831 832 833void X86_64Assembler::sqrtss(XmmRegister dst, XmmRegister src) { 834 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 835 EmitUint8(0xF3); 836 EmitOptionalRex32(dst, src); 837 EmitUint8(0x0F); 838 EmitUint8(0x51); 839 EmitXmmRegisterOperand(dst.LowBits(), src); 840} 841 842 843void X86_64Assembler::xorpd(XmmRegister dst, const Address& src) { 844 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 845 EmitUint8(0x66); 846 EmitOptionalRex32(dst, src); 847 EmitUint8(0x0F); 848 EmitUint8(0x57); 849 EmitOperand(dst.LowBits(), src); 850} 851 852 853void X86_64Assembler::xorpd(XmmRegister dst, XmmRegister src) { 854 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 855 EmitUint8(0x66); 856 EmitOptionalRex32(dst, src); 857 EmitUint8(0x0F); 858 EmitUint8(0x57); 859 EmitXmmRegisterOperand(dst.LowBits(), src); 860} 861 862 863void X86_64Assembler::xorps(XmmRegister dst, const Address& src) { 864 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 865 EmitOptionalRex32(dst, src); 866 EmitUint8(0x0F); 867 EmitUint8(0x57); 868 EmitOperand(dst.LowBits(), src); 869} 870 871 872void X86_64Assembler::xorps(XmmRegister dst, XmmRegister src) { 873 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 874 EmitOptionalRex32(dst, src); 875 EmitUint8(0x0F); 876 EmitUint8(0x57); 877 EmitXmmRegisterOperand(dst.LowBits(), src); 878} 879 880 881void X86_64Assembler::andpd(XmmRegister dst, const Address& src) { 882 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 883 EmitUint8(0x66); 884 EmitOptionalRex32(dst, src); 885 EmitUint8(0x0F); 886 EmitUint8(0x54); 887 EmitOperand(dst.LowBits(), src); 888} 889 890void X86_64Assembler::andpd(XmmRegister dst, XmmRegister src) { 891 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 892 EmitUint8(0x66); 893 EmitOptionalRex32(dst, src); 894 EmitUint8(0x0F); 895 EmitUint8(0x54); 896 EmitXmmRegisterOperand(dst.LowBits(), src); 897} 898 899void X86_64Assembler::andps(XmmRegister dst, XmmRegister src) { 900 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 901 EmitOptionalRex32(dst, src); 902 EmitUint8(0x0F); 903 EmitUint8(0x54); 904 EmitXmmRegisterOperand(dst.LowBits(), src); 905} 906 907void X86_64Assembler::orpd(XmmRegister dst, XmmRegister src) { 908 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 909 EmitUint8(0x66); 910 EmitOptionalRex32(dst, src); 911 EmitUint8(0x0F); 912 EmitUint8(0x56); 913 EmitXmmRegisterOperand(dst.LowBits(), src); 914} 915 916void X86_64Assembler::orps(XmmRegister dst, XmmRegister src) { 917 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 918 EmitOptionalRex32(dst, src); 919 EmitUint8(0x0F); 920 EmitUint8(0x56); 921 EmitXmmRegisterOperand(dst.LowBits(), src); 922} 923 924void X86_64Assembler::fldl(const Address& src) { 925 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 926 EmitUint8(0xDD); 927 EmitOperand(0, src); 928} 929 930 931void X86_64Assembler::fstl(const Address& dst) { 932 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 933 EmitUint8(0xDD); 934 EmitOperand(2, dst); 935} 936 937 938void X86_64Assembler::fstpl(const Address& dst) { 939 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 940 EmitUint8(0xDD); 941 EmitOperand(3, dst); 942} 943 944 945void X86_64Assembler::fstsw() { 946 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 947 EmitUint8(0x9B); 948 EmitUint8(0xDF); 949 EmitUint8(0xE0); 950} 951 952 953void X86_64Assembler::fnstcw(const Address& dst) { 954 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 955 EmitUint8(0xD9); 956 EmitOperand(7, dst); 957} 958 959 960void X86_64Assembler::fldcw(const Address& src) { 961 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 962 EmitUint8(0xD9); 963 EmitOperand(5, src); 964} 965 966 967void X86_64Assembler::fistpl(const Address& dst) { 968 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 969 EmitUint8(0xDF); 970 EmitOperand(7, dst); 971} 972 973 974void X86_64Assembler::fistps(const Address& dst) { 975 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 976 EmitUint8(0xDB); 977 EmitOperand(3, dst); 978} 979 980 981void X86_64Assembler::fildl(const Address& src) { 982 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 983 EmitUint8(0xDF); 984 EmitOperand(5, src); 985} 986 987 988void X86_64Assembler::fincstp() { 989 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 990 EmitUint8(0xD9); 991 EmitUint8(0xF7); 992} 993 994 995void X86_64Assembler::ffree(const Immediate& index) { 996 CHECK_LT(index.value(), 7); 997 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 998 EmitUint8(0xDD); 999 EmitUint8(0xC0 + index.value()); 1000} 1001 1002 1003void X86_64Assembler::fsin() { 1004 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1005 EmitUint8(0xD9); 1006 EmitUint8(0xFE); 1007} 1008 1009 1010void X86_64Assembler::fcos() { 1011 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1012 EmitUint8(0xD9); 1013 EmitUint8(0xFF); 1014} 1015 1016 1017void X86_64Assembler::fptan() { 1018 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1019 EmitUint8(0xD9); 1020 EmitUint8(0xF2); 1021} 1022 1023void X86_64Assembler::fucompp() { 1024 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1025 EmitUint8(0xDA); 1026 EmitUint8(0xE9); 1027} 1028 1029 1030void X86_64Assembler::fprem() { 1031 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1032 EmitUint8(0xD9); 1033 EmitUint8(0xF8); 1034} 1035 1036 1037void X86_64Assembler::xchgl(CpuRegister dst, CpuRegister src) { 1038 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1039 // There is a short version for rax. 1040 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't 1041 // work. 1042 const bool src_rax = src.AsRegister() == RAX; 1043 const bool dst_rax = dst.AsRegister() == RAX; 1044 if (src_rax || dst_rax) { 1045 EmitOptionalRex32(src_rax ? dst : src); 1046 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits())); 1047 return; 1048 } 1049 1050 // General case. 1051 EmitOptionalRex32(src, dst); 1052 EmitUint8(0x87); 1053 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1054} 1055 1056 1057void X86_64Assembler::xchgq(CpuRegister dst, CpuRegister src) { 1058 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1059 // There is a short version for rax. 1060 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't 1061 // work. 1062 const bool src_rax = src.AsRegister() == RAX; 1063 const bool dst_rax = dst.AsRegister() == RAX; 1064 if (src_rax || dst_rax) { 1065 // If src == target, emit a nop instead. 1066 if (src_rax && dst_rax) { 1067 EmitUint8(0x90); 1068 } else { 1069 EmitRex64(src_rax ? dst : src); 1070 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits())); 1071 } 1072 return; 1073 } 1074 1075 // General case. 1076 EmitRex64(src, dst); 1077 EmitUint8(0x87); 1078 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1079} 1080 1081 1082void X86_64Assembler::xchgl(CpuRegister reg, const Address& address) { 1083 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1084 EmitOptionalRex32(reg, address); 1085 EmitUint8(0x87); 1086 EmitOperand(reg.LowBits(), address); 1087} 1088 1089 1090void X86_64Assembler::cmpw(const Address& address, const Immediate& imm) { 1091 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1092 EmitOptionalRex32(address); 1093 EmitUint8(0x66); 1094 EmitComplex(7, address, imm); 1095} 1096 1097 1098void X86_64Assembler::cmpl(CpuRegister reg, const Immediate& imm) { 1099 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1100 EmitOptionalRex32(reg); 1101 EmitComplex(7, Operand(reg), imm); 1102} 1103 1104 1105void X86_64Assembler::cmpl(CpuRegister reg0, CpuRegister reg1) { 1106 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1107 EmitOptionalRex32(reg0, reg1); 1108 EmitUint8(0x3B); 1109 EmitOperand(reg0.LowBits(), Operand(reg1)); 1110} 1111 1112 1113void X86_64Assembler::cmpl(CpuRegister reg, const Address& address) { 1114 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1115 EmitOptionalRex32(reg, address); 1116 EmitUint8(0x3B); 1117 EmitOperand(reg.LowBits(), address); 1118} 1119 1120 1121void X86_64Assembler::cmpl(const Address& address, CpuRegister reg) { 1122 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1123 EmitOptionalRex32(reg, address); 1124 EmitUint8(0x39); 1125 EmitOperand(reg.LowBits(), address); 1126} 1127 1128 1129void X86_64Assembler::cmpl(const Address& address, const Immediate& imm) { 1130 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1131 EmitOptionalRex32(address); 1132 EmitComplex(7, address, imm); 1133} 1134 1135 1136void X86_64Assembler::cmpq(CpuRegister reg0, CpuRegister reg1) { 1137 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1138 EmitRex64(reg0, reg1); 1139 EmitUint8(0x3B); 1140 EmitOperand(reg0.LowBits(), Operand(reg1)); 1141} 1142 1143 1144void X86_64Assembler::cmpq(CpuRegister reg, const Immediate& imm) { 1145 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1146 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 1147 EmitRex64(reg); 1148 EmitComplex(7, Operand(reg), imm); 1149} 1150 1151 1152void X86_64Assembler::cmpq(CpuRegister reg, const Address& address) { 1153 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1154 EmitRex64(reg); 1155 EmitUint8(0x3B); 1156 EmitOperand(reg.LowBits(), address); 1157} 1158 1159 1160void X86_64Assembler::cmpq(const Address& address, const Immediate& imm) { 1161 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 1162 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1163 EmitRex64(address); 1164 EmitComplex(7, address, imm); 1165} 1166 1167 1168void X86_64Assembler::addl(CpuRegister dst, CpuRegister src) { 1169 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1170 EmitOptionalRex32(dst, src); 1171 EmitUint8(0x03); 1172 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1173} 1174 1175 1176void X86_64Assembler::addl(CpuRegister reg, const Address& address) { 1177 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1178 EmitOptionalRex32(reg, address); 1179 EmitUint8(0x03); 1180 EmitOperand(reg.LowBits(), address); 1181} 1182 1183 1184void X86_64Assembler::testl(CpuRegister reg1, CpuRegister reg2) { 1185 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1186 EmitOptionalRex32(reg1, reg2); 1187 EmitUint8(0x85); 1188 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 1189} 1190 1191 1192void X86_64Assembler::testl(CpuRegister reg, const Address& address) { 1193 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1194 EmitOptionalRex32(reg, address); 1195 EmitUint8(0x85); 1196 EmitOperand(reg.LowBits(), address); 1197} 1198 1199 1200void X86_64Assembler::testl(CpuRegister reg, const Immediate& immediate) { 1201 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1202 // For registers that have a byte variant (RAX, RBX, RCX, and RDX) 1203 // we only test the byte CpuRegister to keep the encoding short. 1204 if (immediate.is_uint8() && reg.AsRegister() < 4) { 1205 // Use zero-extended 8-bit immediate. 1206 if (reg.AsRegister() == RAX) { 1207 EmitUint8(0xA8); 1208 } else { 1209 EmitUint8(0xF6); 1210 EmitUint8(0xC0 + reg.AsRegister()); 1211 } 1212 EmitUint8(immediate.value() & 0xFF); 1213 } else if (reg.AsRegister() == RAX) { 1214 // Use short form if the destination is RAX. 1215 EmitUint8(0xA9); 1216 EmitImmediate(immediate); 1217 } else { 1218 EmitOptionalRex32(reg); 1219 EmitUint8(0xF7); 1220 EmitOperand(0, Operand(reg)); 1221 EmitImmediate(immediate); 1222 } 1223} 1224 1225 1226void X86_64Assembler::testq(CpuRegister reg1, CpuRegister reg2) { 1227 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1228 EmitRex64(reg1, reg2); 1229 EmitUint8(0x85); 1230 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 1231} 1232 1233 1234void X86_64Assembler::testq(CpuRegister reg, const Address& address) { 1235 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1236 EmitRex64(reg); 1237 EmitUint8(0x85); 1238 EmitOperand(reg.LowBits(), address); 1239} 1240 1241 1242void X86_64Assembler::andl(CpuRegister dst, CpuRegister src) { 1243 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1244 EmitOptionalRex32(dst, src); 1245 EmitUint8(0x23); 1246 EmitOperand(dst.LowBits(), Operand(src)); 1247} 1248 1249 1250void X86_64Assembler::andl(CpuRegister reg, const Address& address) { 1251 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1252 EmitOptionalRex32(reg, address); 1253 EmitUint8(0x23); 1254 EmitOperand(reg.LowBits(), address); 1255} 1256 1257 1258void X86_64Assembler::andl(CpuRegister dst, const Immediate& imm) { 1259 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1260 EmitOptionalRex32(dst); 1261 EmitComplex(4, Operand(dst), imm); 1262} 1263 1264 1265void X86_64Assembler::andq(CpuRegister reg, const Immediate& imm) { 1266 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1267 CHECK(imm.is_int32()); // andq only supports 32b immediate. 1268 EmitRex64(reg); 1269 EmitComplex(4, Operand(reg), imm); 1270} 1271 1272 1273void X86_64Assembler::andq(CpuRegister dst, CpuRegister src) { 1274 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1275 EmitRex64(dst, src); 1276 EmitUint8(0x23); 1277 EmitOperand(dst.LowBits(), Operand(src)); 1278} 1279 1280 1281void X86_64Assembler::orl(CpuRegister dst, CpuRegister src) { 1282 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1283 EmitOptionalRex32(dst, src); 1284 EmitUint8(0x0B); 1285 EmitOperand(dst.LowBits(), Operand(src)); 1286} 1287 1288 1289void X86_64Assembler::orl(CpuRegister reg, const Address& address) { 1290 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1291 EmitOptionalRex32(reg, address); 1292 EmitUint8(0x0B); 1293 EmitOperand(reg.LowBits(), address); 1294} 1295 1296 1297void X86_64Assembler::orl(CpuRegister dst, const Immediate& imm) { 1298 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1299 EmitOptionalRex32(dst); 1300 EmitComplex(1, Operand(dst), imm); 1301} 1302 1303 1304void X86_64Assembler::orq(CpuRegister dst, const Immediate& imm) { 1305 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1306 CHECK(imm.is_int32()); // orq only supports 32b immediate. 1307 EmitRex64(dst); 1308 EmitComplex(1, Operand(dst), imm); 1309} 1310 1311 1312void X86_64Assembler::orq(CpuRegister dst, CpuRegister src) { 1313 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1314 EmitRex64(dst, src); 1315 EmitUint8(0x0B); 1316 EmitOperand(dst.LowBits(), Operand(src)); 1317} 1318 1319 1320void X86_64Assembler::xorl(CpuRegister dst, CpuRegister src) { 1321 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1322 EmitOptionalRex32(dst, src); 1323 EmitUint8(0x33); 1324 EmitOperand(dst.LowBits(), Operand(src)); 1325} 1326 1327 1328void X86_64Assembler::xorl(CpuRegister reg, const Address& address) { 1329 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1330 EmitOptionalRex32(reg, address); 1331 EmitUint8(0x33); 1332 EmitOperand(reg.LowBits(), address); 1333} 1334 1335 1336void X86_64Assembler::xorl(CpuRegister dst, const Immediate& imm) { 1337 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1338 EmitOptionalRex32(dst); 1339 EmitComplex(6, Operand(dst), imm); 1340} 1341 1342 1343void X86_64Assembler::xorq(CpuRegister dst, CpuRegister src) { 1344 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1345 EmitRex64(dst, src); 1346 EmitUint8(0x33); 1347 EmitOperand(dst.LowBits(), Operand(src)); 1348} 1349 1350 1351void X86_64Assembler::xorq(CpuRegister dst, const Immediate& imm) { 1352 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1353 CHECK(imm.is_int32()); // xorq only supports 32b immediate. 1354 EmitRex64(dst); 1355 EmitComplex(6, Operand(dst), imm); 1356} 1357 1358#if 0 1359void X86_64Assembler::rex(bool force, bool w, Register* r, Register* x, Register* b) { 1360 // REX.WRXB 1361 // W - 64-bit operand 1362 // R - MODRM.reg 1363 // X - SIB.index 1364 // B - MODRM.rm/SIB.base 1365 uint8_t rex = force ? 0x40 : 0; 1366 if (w) { 1367 rex |= 0x48; // REX.W000 1368 } 1369 if (r != nullptr && *r >= Register::R8 && *r < Register::kNumberOfCpuRegisters) { 1370 rex |= 0x44; // REX.0R00 1371 *r = static_cast<Register>(*r - 8); 1372 } 1373 if (x != nullptr && *x >= Register::R8 && *x < Register::kNumberOfCpuRegisters) { 1374 rex |= 0x42; // REX.00X0 1375 *x = static_cast<Register>(*x - 8); 1376 } 1377 if (b != nullptr && *b >= Register::R8 && *b < Register::kNumberOfCpuRegisters) { 1378 rex |= 0x41; // REX.000B 1379 *b = static_cast<Register>(*b - 8); 1380 } 1381 if (rex != 0) { 1382 EmitUint8(rex); 1383 } 1384} 1385 1386void X86_64Assembler::rex_reg_mem(bool force, bool w, Register* dst, const Address& mem) { 1387 // REX.WRXB 1388 // W - 64-bit operand 1389 // R - MODRM.reg 1390 // X - SIB.index 1391 // B - MODRM.rm/SIB.base 1392 uint8_t rex = mem->rex(); 1393 if (force) { 1394 rex |= 0x40; // REX.0000 1395 } 1396 if (w) { 1397 rex |= 0x48; // REX.W000 1398 } 1399 if (dst != nullptr && *dst >= Register::R8 && *dst < Register::kNumberOfCpuRegisters) { 1400 rex |= 0x44; // REX.0R00 1401 *dst = static_cast<Register>(*dst - 8); 1402 } 1403 if (rex != 0) { 1404 EmitUint8(rex); 1405 } 1406} 1407 1408void rex_mem_reg(bool force, bool w, Address* mem, Register* src); 1409#endif 1410 1411void X86_64Assembler::addl(CpuRegister reg, const Immediate& imm) { 1412 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1413 EmitOptionalRex32(reg); 1414 EmitComplex(0, Operand(reg), imm); 1415} 1416 1417 1418void X86_64Assembler::addq(CpuRegister reg, const Immediate& imm) { 1419 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1420 CHECK(imm.is_int32()); // addq only supports 32b immediate. 1421 EmitRex64(reg); 1422 EmitComplex(0, Operand(reg), imm); 1423} 1424 1425 1426void X86_64Assembler::addq(CpuRegister dst, const Address& address) { 1427 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1428 EmitRex64(dst); 1429 EmitUint8(0x03); 1430 EmitOperand(dst.LowBits(), address); 1431} 1432 1433 1434void X86_64Assembler::addq(CpuRegister dst, CpuRegister src) { 1435 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1436 // 0x01 is addq r/m64 <- r/m64 + r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 1437 EmitRex64(src, dst); 1438 EmitUint8(0x01); 1439 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1440} 1441 1442 1443void X86_64Assembler::addl(const Address& address, CpuRegister reg) { 1444 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1445 EmitOptionalRex32(reg, address); 1446 EmitUint8(0x01); 1447 EmitOperand(reg.LowBits(), address); 1448} 1449 1450 1451void X86_64Assembler::addl(const Address& address, const Immediate& imm) { 1452 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1453 EmitOptionalRex32(address); 1454 EmitComplex(0, address, imm); 1455} 1456 1457 1458void X86_64Assembler::subl(CpuRegister dst, CpuRegister src) { 1459 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1460 EmitOptionalRex32(dst, src); 1461 EmitUint8(0x2B); 1462 EmitOperand(dst.LowBits(), Operand(src)); 1463} 1464 1465 1466void X86_64Assembler::subl(CpuRegister reg, const Immediate& imm) { 1467 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1468 EmitOptionalRex32(reg); 1469 EmitComplex(5, Operand(reg), imm); 1470} 1471 1472 1473void X86_64Assembler::subq(CpuRegister reg, const Immediate& imm) { 1474 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1475 CHECK(imm.is_int32()); // subq only supports 32b immediate. 1476 EmitRex64(reg); 1477 EmitComplex(5, Operand(reg), imm); 1478} 1479 1480 1481void X86_64Assembler::subq(CpuRegister dst, CpuRegister src) { 1482 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1483 EmitRex64(dst, src); 1484 EmitUint8(0x2B); 1485 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1486} 1487 1488 1489void X86_64Assembler::subq(CpuRegister reg, const Address& address) { 1490 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1491 EmitRex64(reg); 1492 EmitUint8(0x2B); 1493 EmitOperand(reg.LowBits() & 7, address); 1494} 1495 1496 1497void X86_64Assembler::subl(CpuRegister reg, const Address& address) { 1498 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1499 EmitOptionalRex32(reg, address); 1500 EmitUint8(0x2B); 1501 EmitOperand(reg.LowBits(), address); 1502} 1503 1504 1505void X86_64Assembler::cdq() { 1506 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1507 EmitUint8(0x99); 1508} 1509 1510 1511void X86_64Assembler::cqo() { 1512 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1513 EmitRex64(); 1514 EmitUint8(0x99); 1515} 1516 1517 1518void X86_64Assembler::idivl(CpuRegister reg) { 1519 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1520 EmitOptionalRex32(reg); 1521 EmitUint8(0xF7); 1522 EmitUint8(0xF8 | reg.LowBits()); 1523} 1524 1525 1526void X86_64Assembler::idivq(CpuRegister reg) { 1527 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1528 EmitRex64(reg); 1529 EmitUint8(0xF7); 1530 EmitUint8(0xF8 | reg.LowBits()); 1531} 1532 1533 1534void X86_64Assembler::imull(CpuRegister dst, CpuRegister src) { 1535 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1536 EmitOptionalRex32(dst, src); 1537 EmitUint8(0x0F); 1538 EmitUint8(0xAF); 1539 EmitOperand(dst.LowBits(), Operand(src)); 1540} 1541 1542void X86_64Assembler::imull(CpuRegister reg, const Immediate& imm) { 1543 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1544 CHECK(imm.is_int32()); // imull only supports 32b immediate. 1545 1546 EmitOptionalRex32(reg, reg); 1547 1548 // See whether imm can be represented as a sign-extended 8bit value. 1549 int32_t v32 = static_cast<int32_t>(imm.value()); 1550 if (IsInt<8>(v32)) { 1551 // Sign-extension works. 1552 EmitUint8(0x6B); 1553 EmitOperand(reg.LowBits(), Operand(reg)); 1554 EmitUint8(static_cast<uint8_t>(v32 & 0xFF)); 1555 } else { 1556 // Not representable, use full immediate. 1557 EmitUint8(0x69); 1558 EmitOperand(reg.LowBits(), Operand(reg)); 1559 EmitImmediate(imm); 1560 } 1561} 1562 1563 1564void X86_64Assembler::imull(CpuRegister reg, const Address& address) { 1565 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1566 EmitOptionalRex32(reg, address); 1567 EmitUint8(0x0F); 1568 EmitUint8(0xAF); 1569 EmitOperand(reg.LowBits(), address); 1570} 1571 1572 1573void X86_64Assembler::imulq(CpuRegister dst, CpuRegister src) { 1574 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1575 EmitRex64(dst, src); 1576 EmitUint8(0x0F); 1577 EmitUint8(0xAF); 1578 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1579} 1580 1581 1582void X86_64Assembler::imulq(CpuRegister reg, const Immediate& imm) { 1583 imulq(reg, reg, imm); 1584} 1585 1586void X86_64Assembler::imulq(CpuRegister dst, CpuRegister reg, const Immediate& imm) { 1587 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1588 CHECK(imm.is_int32()); // imulq only supports 32b immediate. 1589 1590 EmitRex64(dst, reg); 1591 1592 // See whether imm can be represented as a sign-extended 8bit value. 1593 int64_t v64 = imm.value(); 1594 if (IsInt<8>(v64)) { 1595 // Sign-extension works. 1596 EmitUint8(0x6B); 1597 EmitOperand(dst.LowBits(), Operand(reg)); 1598 EmitUint8(static_cast<uint8_t>(v64 & 0xFF)); 1599 } else { 1600 // Not representable, use full immediate. 1601 EmitUint8(0x69); 1602 EmitOperand(dst.LowBits(), Operand(reg)); 1603 EmitImmediate(imm); 1604 } 1605} 1606 1607void X86_64Assembler::imulq(CpuRegister reg, const Address& address) { 1608 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1609 EmitRex64(reg, address); 1610 EmitUint8(0x0F); 1611 EmitUint8(0xAF); 1612 EmitOperand(reg.LowBits(), address); 1613} 1614 1615 1616void X86_64Assembler::imull(CpuRegister reg) { 1617 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1618 EmitOptionalRex32(reg); 1619 EmitUint8(0xF7); 1620 EmitOperand(5, Operand(reg)); 1621} 1622 1623 1624void X86_64Assembler::imull(const Address& address) { 1625 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1626 EmitOptionalRex32(address); 1627 EmitUint8(0xF7); 1628 EmitOperand(5, address); 1629} 1630 1631 1632void X86_64Assembler::mull(CpuRegister reg) { 1633 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1634 EmitOptionalRex32(reg); 1635 EmitUint8(0xF7); 1636 EmitOperand(4, Operand(reg)); 1637} 1638 1639 1640void X86_64Assembler::mull(const Address& address) { 1641 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1642 EmitOptionalRex32(address); 1643 EmitUint8(0xF7); 1644 EmitOperand(4, address); 1645} 1646 1647 1648void X86_64Assembler::shll(CpuRegister reg, const Immediate& imm) { 1649 EmitGenericShift(false, 4, reg, imm); 1650} 1651 1652 1653void X86_64Assembler::shlq(CpuRegister reg, const Immediate& imm) { 1654 EmitGenericShift(true, 4, reg, imm); 1655} 1656 1657 1658void X86_64Assembler::shll(CpuRegister operand, CpuRegister shifter) { 1659 EmitGenericShift(false, 4, operand, shifter); 1660} 1661 1662 1663void X86_64Assembler::shlq(CpuRegister operand, CpuRegister shifter) { 1664 EmitGenericShift(true, 4, operand, shifter); 1665} 1666 1667 1668void X86_64Assembler::shrl(CpuRegister reg, const Immediate& imm) { 1669 EmitGenericShift(false, 5, reg, imm); 1670} 1671 1672 1673void X86_64Assembler::shrq(CpuRegister reg, const Immediate& imm) { 1674 EmitGenericShift(true, 5, reg, imm); 1675} 1676 1677 1678void X86_64Assembler::shrl(CpuRegister operand, CpuRegister shifter) { 1679 EmitGenericShift(false, 5, operand, shifter); 1680} 1681 1682 1683void X86_64Assembler::shrq(CpuRegister operand, CpuRegister shifter) { 1684 EmitGenericShift(true, 5, operand, shifter); 1685} 1686 1687 1688void X86_64Assembler::sarl(CpuRegister reg, const Immediate& imm) { 1689 EmitGenericShift(false, 7, reg, imm); 1690} 1691 1692 1693void X86_64Assembler::sarl(CpuRegister operand, CpuRegister shifter) { 1694 EmitGenericShift(false, 7, operand, shifter); 1695} 1696 1697 1698void X86_64Assembler::sarq(CpuRegister reg, const Immediate& imm) { 1699 EmitGenericShift(true, 7, reg, imm); 1700} 1701 1702 1703void X86_64Assembler::sarq(CpuRegister operand, CpuRegister shifter) { 1704 EmitGenericShift(true, 7, operand, shifter); 1705} 1706 1707 1708void X86_64Assembler::negl(CpuRegister reg) { 1709 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1710 EmitOptionalRex32(reg); 1711 EmitUint8(0xF7); 1712 EmitOperand(3, Operand(reg)); 1713} 1714 1715 1716void X86_64Assembler::negq(CpuRegister reg) { 1717 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1718 EmitRex64(reg); 1719 EmitUint8(0xF7); 1720 EmitOperand(3, Operand(reg)); 1721} 1722 1723 1724void X86_64Assembler::notl(CpuRegister reg) { 1725 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1726 EmitOptionalRex32(reg); 1727 EmitUint8(0xF7); 1728 EmitUint8(0xD0 | reg.LowBits()); 1729} 1730 1731 1732void X86_64Assembler::notq(CpuRegister reg) { 1733 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1734 EmitRex64(reg); 1735 EmitUint8(0xF7); 1736 EmitOperand(2, Operand(reg)); 1737} 1738 1739 1740void X86_64Assembler::enter(const Immediate& imm) { 1741 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1742 EmitUint8(0xC8); 1743 CHECK(imm.is_uint16()) << imm.value(); 1744 EmitUint8(imm.value() & 0xFF); 1745 EmitUint8((imm.value() >> 8) & 0xFF); 1746 EmitUint8(0x00); 1747} 1748 1749 1750void X86_64Assembler::leave() { 1751 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1752 EmitUint8(0xC9); 1753} 1754 1755 1756void X86_64Assembler::ret() { 1757 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1758 EmitUint8(0xC3); 1759} 1760 1761 1762void X86_64Assembler::ret(const Immediate& imm) { 1763 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1764 EmitUint8(0xC2); 1765 CHECK(imm.is_uint16()); 1766 EmitUint8(imm.value() & 0xFF); 1767 EmitUint8((imm.value() >> 8) & 0xFF); 1768} 1769 1770 1771 1772void X86_64Assembler::nop() { 1773 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1774 EmitUint8(0x90); 1775} 1776 1777 1778void X86_64Assembler::int3() { 1779 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1780 EmitUint8(0xCC); 1781} 1782 1783 1784void X86_64Assembler::hlt() { 1785 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1786 EmitUint8(0xF4); 1787} 1788 1789 1790void X86_64Assembler::j(Condition condition, Label* label) { 1791 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1792 if (label->IsBound()) { 1793 static const int kShortSize = 2; 1794 static const int kLongSize = 6; 1795 int offset = label->Position() - buffer_.Size(); 1796 CHECK_LE(offset, 0); 1797 if (IsInt<8>(offset - kShortSize)) { 1798 EmitUint8(0x70 + condition); 1799 EmitUint8((offset - kShortSize) & 0xFF); 1800 } else { 1801 EmitUint8(0x0F); 1802 EmitUint8(0x80 + condition); 1803 EmitInt32(offset - kLongSize); 1804 } 1805 } else { 1806 EmitUint8(0x0F); 1807 EmitUint8(0x80 + condition); 1808 EmitLabelLink(label); 1809 } 1810} 1811 1812 1813void X86_64Assembler::jmp(CpuRegister reg) { 1814 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1815 EmitOptionalRex32(reg); 1816 EmitUint8(0xFF); 1817 EmitRegisterOperand(4, reg.LowBits()); 1818} 1819 1820void X86_64Assembler::jmp(const Address& address) { 1821 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1822 EmitOptionalRex32(address); 1823 EmitUint8(0xFF); 1824 EmitOperand(4, address); 1825} 1826 1827void X86_64Assembler::jmp(Label* label) { 1828 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1829 if (label->IsBound()) { 1830 static const int kShortSize = 2; 1831 static const int kLongSize = 5; 1832 int offset = label->Position() - buffer_.Size(); 1833 CHECK_LE(offset, 0); 1834 if (IsInt<8>(offset - kShortSize)) { 1835 EmitUint8(0xEB); 1836 EmitUint8((offset - kShortSize) & 0xFF); 1837 } else { 1838 EmitUint8(0xE9); 1839 EmitInt32(offset - kLongSize); 1840 } 1841 } else { 1842 EmitUint8(0xE9); 1843 EmitLabelLink(label); 1844 } 1845} 1846 1847 1848X86_64Assembler* X86_64Assembler::lock() { 1849 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1850 EmitUint8(0xF0); 1851 return this; 1852} 1853 1854 1855void X86_64Assembler::cmpxchgl(const Address& address, CpuRegister reg) { 1856 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1857 EmitUint8(0x0F); 1858 EmitUint8(0xB1); 1859 EmitOperand(reg.LowBits(), address); 1860} 1861 1862void X86_64Assembler::mfence() { 1863 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1864 EmitUint8(0x0F); 1865 EmitUint8(0xAE); 1866 EmitUint8(0xF0); 1867} 1868 1869 1870X86_64Assembler* X86_64Assembler::gs() { 1871 // TODO: gs is a prefix and not an instruction 1872 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1873 EmitUint8(0x65); 1874 return this; 1875} 1876 1877 1878void X86_64Assembler::AddImmediate(CpuRegister reg, const Immediate& imm) { 1879 int value = imm.value(); 1880 if (value != 0) { 1881 if (value > 0) { 1882 addl(reg, imm); 1883 } else { 1884 subl(reg, Immediate(value)); 1885 } 1886 } 1887} 1888 1889 1890void X86_64Assembler::setcc(Condition condition, CpuRegister dst) { 1891 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1892 // RSP, RBP, RDI, RSI need rex prefix (else the pattern encodes ah/bh/ch/dh). 1893 if (dst.NeedsRex() || dst.AsRegister() > 3) { 1894 EmitOptionalRex(true, false, false, false, dst.NeedsRex()); 1895 } 1896 EmitUint8(0x0F); 1897 EmitUint8(0x90 + condition); 1898 EmitUint8(0xC0 + dst.LowBits()); 1899} 1900 1901void X86_64Assembler::bswapl(CpuRegister dst) { 1902 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1903 EmitOptionalRex(false, false, false, false, dst.NeedsRex()); 1904 EmitUint8(0x0F); 1905 EmitUint8(0xC8 + dst.LowBits()); 1906} 1907 1908void X86_64Assembler::bswapq(CpuRegister dst) { 1909 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1910 EmitOptionalRex(false, true, false, false, dst.NeedsRex()); 1911 EmitUint8(0x0F); 1912 EmitUint8(0xC8 + dst.LowBits()); 1913} 1914 1915 1916void X86_64Assembler::LoadDoubleConstant(XmmRegister dst, double value) { 1917 // TODO: Need to have a code constants table. 1918 int64_t constant = bit_cast<int64_t, double>(value); 1919 pushq(Immediate(High32Bits(constant))); 1920 pushq(Immediate(Low32Bits(constant))); 1921 movsd(dst, Address(CpuRegister(RSP), 0)); 1922 addq(CpuRegister(RSP), Immediate(2 * sizeof(intptr_t))); 1923} 1924 1925 1926void X86_64Assembler::Align(int alignment, int offset) { 1927 CHECK(IsPowerOfTwo(alignment)); 1928 // Emit nop instruction until the real position is aligned. 1929 while (((offset + buffer_.GetPosition()) & (alignment-1)) != 0) { 1930 nop(); 1931 } 1932} 1933 1934 1935void X86_64Assembler::Bind(Label* label) { 1936 int bound = buffer_.Size(); 1937 CHECK(!label->IsBound()); // Labels can only be bound once. 1938 while (label->IsLinked()) { 1939 int position = label->LinkPosition(); 1940 int next = buffer_.Load<int32_t>(position); 1941 buffer_.Store<int32_t>(position, bound - (position + 4)); 1942 label->position_ = next; 1943 } 1944 label->BindTo(bound); 1945} 1946 1947 1948void X86_64Assembler::EmitOperand(uint8_t reg_or_opcode, const Operand& operand) { 1949 CHECK_GE(reg_or_opcode, 0); 1950 CHECK_LT(reg_or_opcode, 8); 1951 const int length = operand.length_; 1952 CHECK_GT(length, 0); 1953 // Emit the ModRM byte updated with the given reg value. 1954 CHECK_EQ(operand.encoding_[0] & 0x38, 0); 1955 EmitUint8(operand.encoding_[0] + (reg_or_opcode << 3)); 1956 // Emit the rest of the encoded operand. 1957 for (int i = 1; i < length; i++) { 1958 EmitUint8(operand.encoding_[i]); 1959 } 1960} 1961 1962 1963void X86_64Assembler::EmitImmediate(const Immediate& imm) { 1964 if (imm.is_int32()) { 1965 EmitInt32(static_cast<int32_t>(imm.value())); 1966 } else { 1967 EmitInt64(imm.value()); 1968 } 1969} 1970 1971 1972void X86_64Assembler::EmitComplex(uint8_t reg_or_opcode, 1973 const Operand& operand, 1974 const Immediate& immediate) { 1975 CHECK_GE(reg_or_opcode, 0); 1976 CHECK_LT(reg_or_opcode, 8); 1977 if (immediate.is_int8()) { 1978 // Use sign-extended 8-bit immediate. 1979 EmitUint8(0x83); 1980 EmitOperand(reg_or_opcode, operand); 1981 EmitUint8(immediate.value() & 0xFF); 1982 } else if (operand.IsRegister(CpuRegister(RAX))) { 1983 // Use short form if the destination is eax. 1984 EmitUint8(0x05 + (reg_or_opcode << 3)); 1985 EmitImmediate(immediate); 1986 } else { 1987 EmitUint8(0x81); 1988 EmitOperand(reg_or_opcode, operand); 1989 EmitImmediate(immediate); 1990 } 1991} 1992 1993 1994void X86_64Assembler::EmitLabel(Label* label, int instruction_size) { 1995 if (label->IsBound()) { 1996 int offset = label->Position() - buffer_.Size(); 1997 CHECK_LE(offset, 0); 1998 EmitInt32(offset - instruction_size); 1999 } else { 2000 EmitLabelLink(label); 2001 } 2002} 2003 2004 2005void X86_64Assembler::EmitLabelLink(Label* label) { 2006 CHECK(!label->IsBound()); 2007 int position = buffer_.Size(); 2008 EmitInt32(label->position_); 2009 label->LinkTo(position); 2010} 2011 2012 2013void X86_64Assembler::EmitGenericShift(bool wide, 2014 int reg_or_opcode, 2015 CpuRegister reg, 2016 const Immediate& imm) { 2017 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2018 CHECK(imm.is_int8()); 2019 if (wide) { 2020 EmitRex64(reg); 2021 } else { 2022 EmitOptionalRex32(reg); 2023 } 2024 if (imm.value() == 1) { 2025 EmitUint8(0xD1); 2026 EmitOperand(reg_or_opcode, Operand(reg)); 2027 } else { 2028 EmitUint8(0xC1); 2029 EmitOperand(reg_or_opcode, Operand(reg)); 2030 EmitUint8(imm.value() & 0xFF); 2031 } 2032} 2033 2034 2035void X86_64Assembler::EmitGenericShift(bool wide, 2036 int reg_or_opcode, 2037 CpuRegister operand, 2038 CpuRegister shifter) { 2039 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2040 CHECK_EQ(shifter.AsRegister(), RCX); 2041 if (wide) { 2042 EmitRex64(operand); 2043 } else { 2044 EmitOptionalRex32(operand); 2045 } 2046 EmitUint8(0xD3); 2047 EmitOperand(reg_or_opcode, Operand(operand)); 2048} 2049 2050void X86_64Assembler::EmitOptionalRex(bool force, bool w, bool r, bool x, bool b) { 2051 // REX.WRXB 2052 // W - 64-bit operand 2053 // R - MODRM.reg 2054 // X - SIB.index 2055 // B - MODRM.rm/SIB.base 2056 uint8_t rex = force ? 0x40 : 0; 2057 if (w) { 2058 rex |= 0x48; // REX.W000 2059 } 2060 if (r) { 2061 rex |= 0x44; // REX.0R00 2062 } 2063 if (x) { 2064 rex |= 0x42; // REX.00X0 2065 } 2066 if (b) { 2067 rex |= 0x41; // REX.000B 2068 } 2069 if (rex != 0) { 2070 EmitUint8(rex); 2071 } 2072} 2073 2074void X86_64Assembler::EmitOptionalRex32(CpuRegister reg) { 2075 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 2076} 2077 2078void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, CpuRegister src) { 2079 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2080} 2081 2082void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, XmmRegister src) { 2083 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2084} 2085 2086void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, XmmRegister src) { 2087 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2088} 2089 2090void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, CpuRegister src) { 2091 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2092} 2093 2094void X86_64Assembler::EmitOptionalRex32(const Operand& operand) { 2095 uint8_t rex = operand.rex(); 2096 if (rex != 0) { 2097 EmitUint8(rex); 2098 } 2099} 2100 2101void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, const Operand& operand) { 2102 uint8_t rex = operand.rex(); 2103 if (dst.NeedsRex()) { 2104 rex |= 0x44; // REX.0R00 2105 } 2106 if (rex != 0) { 2107 EmitUint8(rex); 2108 } 2109} 2110 2111void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, const Operand& operand) { 2112 uint8_t rex = operand.rex(); 2113 if (dst.NeedsRex()) { 2114 rex |= 0x44; // REX.0R00 2115 } 2116 if (rex != 0) { 2117 EmitUint8(rex); 2118 } 2119} 2120 2121void X86_64Assembler::EmitRex64() { 2122 EmitOptionalRex(false, true, false, false, false); 2123} 2124 2125void X86_64Assembler::EmitRex64(CpuRegister reg) { 2126 EmitOptionalRex(false, true, false, false, reg.NeedsRex()); 2127} 2128 2129void X86_64Assembler::EmitRex64(const Operand& operand) { 2130 uint8_t rex = operand.rex(); 2131 rex |= 0x48; // REX.W000 2132 EmitUint8(rex); 2133} 2134 2135void X86_64Assembler::EmitRex64(CpuRegister dst, CpuRegister src) { 2136 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2137} 2138 2139void X86_64Assembler::EmitRex64(XmmRegister dst, CpuRegister src) { 2140 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2141} 2142 2143void X86_64Assembler::EmitRex64(CpuRegister dst, XmmRegister src) { 2144 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2145} 2146 2147void X86_64Assembler::EmitRex64(CpuRegister dst, const Operand& operand) { 2148 uint8_t rex = 0x48 | operand.rex(); // REX.W000 2149 if (dst.NeedsRex()) { 2150 rex |= 0x44; // REX.0R00 2151 } 2152 if (rex != 0) { 2153 EmitUint8(rex); 2154 } 2155} 2156 2157void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, CpuRegister src) { 2158 EmitOptionalRex(true, false, dst.NeedsRex(), false, src.NeedsRex()); 2159} 2160 2161void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand) { 2162 uint8_t rex = 0x40 | operand.rex(); // REX.0000 2163 if (dst.NeedsRex()) { 2164 rex |= 0x44; // REX.0R00 2165 } 2166 if (rex != 0) { 2167 EmitUint8(rex); 2168 } 2169} 2170 2171void X86_64Assembler::InitializeFrameDescriptionEntry() { 2172 WriteFDEHeader(&cfi_info_, true /* is_64bit */); 2173} 2174 2175void X86_64Assembler::FinalizeFrameDescriptionEntry() { 2176 WriteFDEAddressRange(&cfi_info_, buffer_.Size(), true /* is_64bit */); 2177 PadCFI(&cfi_info_); 2178 WriteCFILength(&cfi_info_, true /* is_64bit */); 2179} 2180 2181constexpr size_t kFramePointerSize = 8; 2182 2183void X86_64Assembler::BuildFrame(size_t frame_size, ManagedRegister method_reg, 2184 const std::vector<ManagedRegister>& spill_regs, 2185 const ManagedRegisterEntrySpills& entry_spills) { 2186 cfi_cfa_offset_ = kFramePointerSize; // Only return address on stack 2187 cfi_pc_ = buffer_.Size(); // Nothing emitted yet 2188 DCHECK_EQ(cfi_pc_, 0U); 2189 2190 uint32_t reg_offset = 1; 2191 CHECK_ALIGNED(frame_size, kStackAlignment); 2192 int gpr_count = 0; 2193 for (int i = spill_regs.size() - 1; i >= 0; --i) { 2194 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2195 if (spill.IsCpuRegister()) { 2196 pushq(spill.AsCpuRegister()); 2197 gpr_count++; 2198 2199 // DW_CFA_advance_loc 2200 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 2201 cfi_pc_ = buffer_.Size(); 2202 // DW_CFA_def_cfa_offset 2203 cfi_cfa_offset_ += kFramePointerSize; 2204 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 2205 // DW_CFA_offset reg offset 2206 reg_offset++; 2207 DW_CFA_offset(&cfi_info_, spill.DWARFRegId(), reg_offset); 2208 } 2209 } 2210 // return address then method on stack 2211 int64_t rest_of_frame = static_cast<int64_t>(frame_size) 2212 - (gpr_count * kFramePointerSize) 2213 - kFramePointerSize /*return address*/; 2214 subq(CpuRegister(RSP), Immediate(rest_of_frame)); 2215 // DW_CFA_advance_loc 2216 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 2217 cfi_pc_ = buffer_.Size(); 2218 // DW_CFA_def_cfa_offset 2219 cfi_cfa_offset_ += rest_of_frame; 2220 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 2221 2222 // spill xmms 2223 int64_t offset = rest_of_frame; 2224 for (int i = spill_regs.size() - 1; i >= 0; --i) { 2225 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2226 if (spill.IsXmmRegister()) { 2227 offset -= sizeof(double); 2228 movsd(Address(CpuRegister(RSP), offset), spill.AsXmmRegister()); 2229 } 2230 } 2231 2232 DCHECK_EQ(4U, sizeof(StackReference<mirror::ArtMethod>)); 2233 2234 movl(Address(CpuRegister(RSP), 0), method_reg.AsX86_64().AsCpuRegister()); 2235 2236 for (size_t i = 0; i < entry_spills.size(); ++i) { 2237 ManagedRegisterSpill spill = entry_spills.at(i); 2238 if (spill.AsX86_64().IsCpuRegister()) { 2239 if (spill.getSize() == 8) { 2240 movq(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), 2241 spill.AsX86_64().AsCpuRegister()); 2242 } else { 2243 CHECK_EQ(spill.getSize(), 4); 2244 movl(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsCpuRegister()); 2245 } 2246 } else { 2247 if (spill.getSize() == 8) { 2248 movsd(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 2249 } else { 2250 CHECK_EQ(spill.getSize(), 4); 2251 movss(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 2252 } 2253 } 2254 } 2255} 2256 2257void X86_64Assembler::RemoveFrame(size_t frame_size, 2258 const std::vector<ManagedRegister>& spill_regs) { 2259 CHECK_ALIGNED(frame_size, kStackAlignment); 2260 int gpr_count = 0; 2261 // unspill xmms 2262 int64_t offset = static_cast<int64_t>(frame_size) - (spill_regs.size() * kFramePointerSize) - 2 * kFramePointerSize; 2263 for (size_t i = 0; i < spill_regs.size(); ++i) { 2264 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2265 if (spill.IsXmmRegister()) { 2266 offset += sizeof(double); 2267 movsd(spill.AsXmmRegister(), Address(CpuRegister(RSP), offset)); 2268 } else { 2269 gpr_count++; 2270 } 2271 } 2272 addq(CpuRegister(RSP), Immediate(static_cast<int64_t>(frame_size) - (gpr_count * kFramePointerSize) - kFramePointerSize)); 2273 for (size_t i = 0; i < spill_regs.size(); ++i) { 2274 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2275 if (spill.IsCpuRegister()) { 2276 popq(spill.AsCpuRegister()); 2277 } 2278 } 2279 ret(); 2280} 2281 2282void X86_64Assembler::IncreaseFrameSize(size_t adjust) { 2283 CHECK_ALIGNED(adjust, kStackAlignment); 2284 addq(CpuRegister(RSP), Immediate(-static_cast<int64_t>(adjust))); 2285 // DW_CFA_advance_loc 2286 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 2287 cfi_pc_ = buffer_.Size(); 2288 // DW_CFA_def_cfa_offset 2289 cfi_cfa_offset_ += adjust; 2290 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 2291} 2292 2293void X86_64Assembler::DecreaseFrameSize(size_t adjust) { 2294 CHECK_ALIGNED(adjust, kStackAlignment); 2295 addq(CpuRegister(RSP), Immediate(adjust)); 2296} 2297 2298void X86_64Assembler::Store(FrameOffset offs, ManagedRegister msrc, size_t size) { 2299 X86_64ManagedRegister src = msrc.AsX86_64(); 2300 if (src.IsNoRegister()) { 2301 CHECK_EQ(0u, size); 2302 } else if (src.IsCpuRegister()) { 2303 if (size == 4) { 2304 CHECK_EQ(4u, size); 2305 movl(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 2306 } else { 2307 CHECK_EQ(8u, size); 2308 movq(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 2309 } 2310 } else if (src.IsRegisterPair()) { 2311 CHECK_EQ(0u, size); 2312 movq(Address(CpuRegister(RSP), offs), src.AsRegisterPairLow()); 2313 movq(Address(CpuRegister(RSP), FrameOffset(offs.Int32Value()+4)), 2314 src.AsRegisterPairHigh()); 2315 } else if (src.IsX87Register()) { 2316 if (size == 4) { 2317 fstps(Address(CpuRegister(RSP), offs)); 2318 } else { 2319 fstpl(Address(CpuRegister(RSP), offs)); 2320 } 2321 } else { 2322 CHECK(src.IsXmmRegister()); 2323 if (size == 4) { 2324 movss(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 2325 } else { 2326 movsd(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 2327 } 2328 } 2329} 2330 2331void X86_64Assembler::StoreRef(FrameOffset dest, ManagedRegister msrc) { 2332 X86_64ManagedRegister src = msrc.AsX86_64(); 2333 CHECK(src.IsCpuRegister()); 2334 movl(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 2335} 2336 2337void X86_64Assembler::StoreRawPtr(FrameOffset dest, ManagedRegister msrc) { 2338 X86_64ManagedRegister src = msrc.AsX86_64(); 2339 CHECK(src.IsCpuRegister()); 2340 movq(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 2341} 2342 2343void X86_64Assembler::StoreImmediateToFrame(FrameOffset dest, uint32_t imm, 2344 ManagedRegister) { 2345 movl(Address(CpuRegister(RSP), dest), Immediate(imm)); // TODO(64) movq? 2346} 2347 2348void X86_64Assembler::StoreImmediateToThread64(ThreadOffset<8> dest, uint32_t imm, 2349 ManagedRegister) { 2350 gs()->movl(Address::Absolute(dest, true), Immediate(imm)); // TODO(64) movq? 2351} 2352 2353void X86_64Assembler::StoreStackOffsetToThread64(ThreadOffset<8> thr_offs, 2354 FrameOffset fr_offs, 2355 ManagedRegister mscratch) { 2356 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2357 CHECK(scratch.IsCpuRegister()); 2358 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), fr_offs)); 2359 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2360} 2361 2362void X86_64Assembler::StoreStackPointerToThread64(ThreadOffset<8> thr_offs) { 2363 gs()->movq(Address::Absolute(thr_offs, true), CpuRegister(RSP)); 2364} 2365 2366void X86_64Assembler::StoreSpanning(FrameOffset /*dst*/, ManagedRegister /*src*/, 2367 FrameOffset /*in_off*/, ManagedRegister /*scratch*/) { 2368 UNIMPLEMENTED(FATAL); // this case only currently exists for ARM 2369} 2370 2371void X86_64Assembler::Load(ManagedRegister mdest, FrameOffset src, size_t size) { 2372 X86_64ManagedRegister dest = mdest.AsX86_64(); 2373 if (dest.IsNoRegister()) { 2374 CHECK_EQ(0u, size); 2375 } else if (dest.IsCpuRegister()) { 2376 if (size == 4) { 2377 CHECK_EQ(4u, size); 2378 movl(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2379 } else { 2380 CHECK_EQ(8u, size); 2381 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2382 } 2383 } else if (dest.IsRegisterPair()) { 2384 CHECK_EQ(0u, size); 2385 movq(dest.AsRegisterPairLow(), Address(CpuRegister(RSP), src)); 2386 movq(dest.AsRegisterPairHigh(), Address(CpuRegister(RSP), FrameOffset(src.Int32Value()+4))); 2387 } else if (dest.IsX87Register()) { 2388 if (size == 4) { 2389 flds(Address(CpuRegister(RSP), src)); 2390 } else { 2391 fldl(Address(CpuRegister(RSP), src)); 2392 } 2393 } else { 2394 CHECK(dest.IsXmmRegister()); 2395 if (size == 4) { 2396 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2397 } else { 2398 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2399 } 2400 } 2401} 2402 2403void X86_64Assembler::LoadFromThread64(ManagedRegister mdest, ThreadOffset<8> src, size_t size) { 2404 X86_64ManagedRegister dest = mdest.AsX86_64(); 2405 if (dest.IsNoRegister()) { 2406 CHECK_EQ(0u, size); 2407 } else if (dest.IsCpuRegister()) { 2408 CHECK_EQ(4u, size); 2409 gs()->movl(dest.AsCpuRegister(), Address::Absolute(src, true)); 2410 } else if (dest.IsRegisterPair()) { 2411 CHECK_EQ(8u, size); 2412 gs()->movq(dest.AsRegisterPairLow(), Address::Absolute(src, true)); 2413 } else if (dest.IsX87Register()) { 2414 if (size == 4) { 2415 gs()->flds(Address::Absolute(src, true)); 2416 } else { 2417 gs()->fldl(Address::Absolute(src, true)); 2418 } 2419 } else { 2420 CHECK(dest.IsXmmRegister()); 2421 if (size == 4) { 2422 gs()->movss(dest.AsXmmRegister(), Address::Absolute(src, true)); 2423 } else { 2424 gs()->movsd(dest.AsXmmRegister(), Address::Absolute(src, true)); 2425 } 2426 } 2427} 2428 2429void X86_64Assembler::LoadRef(ManagedRegister mdest, FrameOffset src) { 2430 X86_64ManagedRegister dest = mdest.AsX86_64(); 2431 CHECK(dest.IsCpuRegister()); 2432 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2433} 2434 2435void X86_64Assembler::LoadRef(ManagedRegister mdest, ManagedRegister base, 2436 MemberOffset offs) { 2437 X86_64ManagedRegister dest = mdest.AsX86_64(); 2438 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2439 movl(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2440 if (kPoisonHeapReferences) { 2441 negl(dest.AsCpuRegister()); 2442 } 2443} 2444 2445void X86_64Assembler::LoadRawPtr(ManagedRegister mdest, ManagedRegister base, 2446 Offset offs) { 2447 X86_64ManagedRegister dest = mdest.AsX86_64(); 2448 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2449 movq(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2450} 2451 2452void X86_64Assembler::LoadRawPtrFromThread64(ManagedRegister mdest, ThreadOffset<8> offs) { 2453 X86_64ManagedRegister dest = mdest.AsX86_64(); 2454 CHECK(dest.IsCpuRegister()); 2455 gs()->movq(dest.AsCpuRegister(), Address::Absolute(offs, true)); 2456} 2457 2458void X86_64Assembler::SignExtend(ManagedRegister mreg, size_t size) { 2459 X86_64ManagedRegister reg = mreg.AsX86_64(); 2460 CHECK(size == 1 || size == 2) << size; 2461 CHECK(reg.IsCpuRegister()) << reg; 2462 if (size == 1) { 2463 movsxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2464 } else { 2465 movsxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2466 } 2467} 2468 2469void X86_64Assembler::ZeroExtend(ManagedRegister mreg, size_t size) { 2470 X86_64ManagedRegister reg = mreg.AsX86_64(); 2471 CHECK(size == 1 || size == 2) << size; 2472 CHECK(reg.IsCpuRegister()) << reg; 2473 if (size == 1) { 2474 movzxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2475 } else { 2476 movzxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2477 } 2478} 2479 2480void X86_64Assembler::Move(ManagedRegister mdest, ManagedRegister msrc, size_t size) { 2481 X86_64ManagedRegister dest = mdest.AsX86_64(); 2482 X86_64ManagedRegister src = msrc.AsX86_64(); 2483 if (!dest.Equals(src)) { 2484 if (dest.IsCpuRegister() && src.IsCpuRegister()) { 2485 movq(dest.AsCpuRegister(), src.AsCpuRegister()); 2486 } else if (src.IsX87Register() && dest.IsXmmRegister()) { 2487 // Pass via stack and pop X87 register 2488 subl(CpuRegister(RSP), Immediate(16)); 2489 if (size == 4) { 2490 CHECK_EQ(src.AsX87Register(), ST0); 2491 fstps(Address(CpuRegister(RSP), 0)); 2492 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2493 } else { 2494 CHECK_EQ(src.AsX87Register(), ST0); 2495 fstpl(Address(CpuRegister(RSP), 0)); 2496 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2497 } 2498 addq(CpuRegister(RSP), Immediate(16)); 2499 } else { 2500 // TODO: x87, SSE 2501 UNIMPLEMENTED(FATAL) << ": Move " << dest << ", " << src; 2502 } 2503 } 2504} 2505 2506void X86_64Assembler::CopyRef(FrameOffset dest, FrameOffset src, 2507 ManagedRegister mscratch) { 2508 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2509 CHECK(scratch.IsCpuRegister()); 2510 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2511 movl(Address(CpuRegister(RSP), dest), scratch.AsCpuRegister()); 2512} 2513 2514void X86_64Assembler::CopyRawPtrFromThread64(FrameOffset fr_offs, 2515 ThreadOffset<8> thr_offs, 2516 ManagedRegister mscratch) { 2517 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2518 CHECK(scratch.IsCpuRegister()); 2519 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(thr_offs, true)); 2520 Store(fr_offs, scratch, 8); 2521} 2522 2523void X86_64Assembler::CopyRawPtrToThread64(ThreadOffset<8> thr_offs, 2524 FrameOffset fr_offs, 2525 ManagedRegister mscratch) { 2526 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2527 CHECK(scratch.IsCpuRegister()); 2528 Load(scratch, fr_offs, 8); 2529 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2530} 2531 2532void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src, 2533 ManagedRegister mscratch, 2534 size_t size) { 2535 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2536 if (scratch.IsCpuRegister() && size == 8) { 2537 Load(scratch, src, 4); 2538 Store(dest, scratch, 4); 2539 Load(scratch, FrameOffset(src.Int32Value() + 4), 4); 2540 Store(FrameOffset(dest.Int32Value() + 4), scratch, 4); 2541 } else { 2542 Load(scratch, src, size); 2543 Store(dest, scratch, size); 2544 } 2545} 2546 2547void X86_64Assembler::Copy(FrameOffset /*dst*/, ManagedRegister /*src_base*/, Offset /*src_offset*/, 2548 ManagedRegister /*scratch*/, size_t /*size*/) { 2549 UNIMPLEMENTED(FATAL); 2550} 2551 2552void X86_64Assembler::Copy(ManagedRegister dest_base, Offset dest_offset, FrameOffset src, 2553 ManagedRegister scratch, size_t size) { 2554 CHECK(scratch.IsNoRegister()); 2555 CHECK_EQ(size, 4u); 2556 pushq(Address(CpuRegister(RSP), src)); 2557 popq(Address(dest_base.AsX86_64().AsCpuRegister(), dest_offset)); 2558} 2559 2560void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src_base, Offset src_offset, 2561 ManagedRegister mscratch, size_t size) { 2562 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2563 CHECK_EQ(size, 4u); 2564 movq(scratch, Address(CpuRegister(RSP), src_base)); 2565 movq(scratch, Address(scratch, src_offset)); 2566 movq(Address(CpuRegister(RSP), dest), scratch); 2567} 2568 2569void X86_64Assembler::Copy(ManagedRegister dest, Offset dest_offset, 2570 ManagedRegister src, Offset src_offset, 2571 ManagedRegister scratch, size_t size) { 2572 CHECK_EQ(size, 4u); 2573 CHECK(scratch.IsNoRegister()); 2574 pushq(Address(src.AsX86_64().AsCpuRegister(), src_offset)); 2575 popq(Address(dest.AsX86_64().AsCpuRegister(), dest_offset)); 2576} 2577 2578void X86_64Assembler::Copy(FrameOffset dest, Offset dest_offset, FrameOffset src, Offset src_offset, 2579 ManagedRegister mscratch, size_t size) { 2580 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2581 CHECK_EQ(size, 4u); 2582 CHECK_EQ(dest.Int32Value(), src.Int32Value()); 2583 movq(scratch, Address(CpuRegister(RSP), src)); 2584 pushq(Address(scratch, src_offset)); 2585 popq(Address(scratch, dest_offset)); 2586} 2587 2588void X86_64Assembler::MemoryBarrier(ManagedRegister) { 2589 mfence(); 2590} 2591 2592void X86_64Assembler::CreateHandleScopeEntry(ManagedRegister mout_reg, 2593 FrameOffset handle_scope_offset, 2594 ManagedRegister min_reg, bool null_allowed) { 2595 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2596 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2597 if (in_reg.IsNoRegister()) { // TODO(64): && null_allowed 2598 // Use out_reg as indicator of NULL 2599 in_reg = out_reg; 2600 // TODO: movzwl 2601 movl(in_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2602 } 2603 CHECK(in_reg.IsCpuRegister()); 2604 CHECK(out_reg.IsCpuRegister()); 2605 VerifyObject(in_reg, null_allowed); 2606 if (null_allowed) { 2607 Label null_arg; 2608 if (!out_reg.Equals(in_reg)) { 2609 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2610 } 2611 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2612 j(kZero, &null_arg); 2613 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2614 Bind(&null_arg); 2615 } else { 2616 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2617 } 2618} 2619 2620void X86_64Assembler::CreateHandleScopeEntry(FrameOffset out_off, 2621 FrameOffset handle_scope_offset, 2622 ManagedRegister mscratch, 2623 bool null_allowed) { 2624 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2625 CHECK(scratch.IsCpuRegister()); 2626 if (null_allowed) { 2627 Label null_arg; 2628 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2629 testl(scratch.AsCpuRegister(), scratch.AsCpuRegister()); 2630 j(kZero, &null_arg); 2631 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2632 Bind(&null_arg); 2633 } else { 2634 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2635 } 2636 Store(out_off, scratch, 8); 2637} 2638 2639// Given a handle scope entry, load the associated reference. 2640void X86_64Assembler::LoadReferenceFromHandleScope(ManagedRegister mout_reg, 2641 ManagedRegister min_reg) { 2642 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2643 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2644 CHECK(out_reg.IsCpuRegister()); 2645 CHECK(in_reg.IsCpuRegister()); 2646 Label null_arg; 2647 if (!out_reg.Equals(in_reg)) { 2648 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2649 } 2650 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2651 j(kZero, &null_arg); 2652 movq(out_reg.AsCpuRegister(), Address(in_reg.AsCpuRegister(), 0)); 2653 Bind(&null_arg); 2654} 2655 2656void X86_64Assembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) { 2657 // TODO: not validating references 2658} 2659 2660void X86_64Assembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) { 2661 // TODO: not validating references 2662} 2663 2664void X86_64Assembler::Call(ManagedRegister mbase, Offset offset, ManagedRegister) { 2665 X86_64ManagedRegister base = mbase.AsX86_64(); 2666 CHECK(base.IsCpuRegister()); 2667 call(Address(base.AsCpuRegister(), offset.Int32Value())); 2668 // TODO: place reference map on call 2669} 2670 2671void X86_64Assembler::Call(FrameOffset base, Offset offset, ManagedRegister mscratch) { 2672 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2673 movl(scratch, Address(CpuRegister(RSP), base)); 2674 call(Address(scratch, offset)); 2675} 2676 2677void X86_64Assembler::CallFromThread64(ThreadOffset<8> offset, ManagedRegister /*mscratch*/) { 2678 gs()->call(Address::Absolute(offset, true)); 2679} 2680 2681void X86_64Assembler::GetCurrentThread(ManagedRegister tr) { 2682 gs()->movq(tr.AsX86_64().AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2683} 2684 2685void X86_64Assembler::GetCurrentThread(FrameOffset offset, ManagedRegister mscratch) { 2686 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2687 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2688 movq(Address(CpuRegister(RSP), offset), scratch.AsCpuRegister()); 2689} 2690 2691// Slowpath entered when Thread::Current()->_exception is non-null 2692class X86_64ExceptionSlowPath FINAL : public SlowPath { 2693 public: 2694 explicit X86_64ExceptionSlowPath(size_t stack_adjust) : stack_adjust_(stack_adjust) {} 2695 virtual void Emit(Assembler *sp_asm) OVERRIDE; 2696 private: 2697 const size_t stack_adjust_; 2698}; 2699 2700void X86_64Assembler::ExceptionPoll(ManagedRegister /*scratch*/, size_t stack_adjust) { 2701 X86_64ExceptionSlowPath* slow = new X86_64ExceptionSlowPath(stack_adjust); 2702 buffer_.EnqueueSlowPath(slow); 2703 gs()->cmpl(Address::Absolute(Thread::ExceptionOffset<8>(), true), Immediate(0)); 2704 j(kNotEqual, slow->Entry()); 2705} 2706 2707void X86_64ExceptionSlowPath::Emit(Assembler *sasm) { 2708 X86_64Assembler* sp_asm = down_cast<X86_64Assembler*>(sasm); 2709#define __ sp_asm-> 2710 __ Bind(&entry_); 2711 // Note: the return value is dead 2712 if (stack_adjust_ != 0) { // Fix up the frame. 2713 __ DecreaseFrameSize(stack_adjust_); 2714 } 2715 // Pass exception as argument in RDI 2716 __ gs()->movq(CpuRegister(RDI), Address::Absolute(Thread::ExceptionOffset<8>(), true)); 2717 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(8, pDeliverException), true)); 2718 // this call should never return 2719 __ int3(); 2720#undef __ 2721} 2722 2723} // namespace x86_64 2724} // namespace art 2725