assembler_x86_64.cc revision 4c0b61f506644bb6b647be05d02c5fb45b9ceb48
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "assembler_x86_64.h" 18 19#include "base/casts.h" 20#include "entrypoints/quick/quick_entrypoints.h" 21#include "memory_region.h" 22#include "thread.h" 23#include "utils/dwarf_cfi.h" 24 25namespace art { 26namespace x86_64 { 27 28std::ostream& operator<<(std::ostream& os, const CpuRegister& reg) { 29 return os << reg.AsRegister(); 30} 31 32std::ostream& operator<<(std::ostream& os, const XmmRegister& reg) { 33 return os << reg.AsFloatRegister(); 34} 35 36std::ostream& operator<<(std::ostream& os, const X87Register& reg) { 37 return os << "ST" << static_cast<int>(reg); 38} 39 40void X86_64Assembler::call(CpuRegister reg) { 41 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 42 EmitOptionalRex32(reg); 43 EmitUint8(0xFF); 44 EmitRegisterOperand(2, reg.LowBits()); 45} 46 47 48void X86_64Assembler::call(const Address& address) { 49 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 50 EmitOptionalRex32(address); 51 EmitUint8(0xFF); 52 EmitOperand(2, address); 53} 54 55 56void X86_64Assembler::call(Label* label) { 57 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 58 EmitUint8(0xE8); 59 static const int kSize = 5; 60 EmitLabel(label, kSize); 61} 62 63void X86_64Assembler::pushq(CpuRegister reg) { 64 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 65 EmitOptionalRex32(reg); 66 EmitUint8(0x50 + reg.LowBits()); 67} 68 69 70void X86_64Assembler::pushq(const Address& address) { 71 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 72 EmitOptionalRex32(address); 73 EmitUint8(0xFF); 74 EmitOperand(6, address); 75} 76 77 78void X86_64Assembler::pushq(const Immediate& imm) { 79 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 80 CHECK(imm.is_int32()); // pushq only supports 32b immediate. 81 if (imm.is_int8()) { 82 EmitUint8(0x6A); 83 EmitUint8(imm.value() & 0xFF); 84 } else { 85 EmitUint8(0x68); 86 EmitImmediate(imm); 87 } 88} 89 90 91void X86_64Assembler::popq(CpuRegister reg) { 92 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 93 EmitOptionalRex32(reg); 94 EmitUint8(0x58 + reg.LowBits()); 95} 96 97 98void X86_64Assembler::popq(const Address& address) { 99 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 100 EmitOptionalRex32(address); 101 EmitUint8(0x8F); 102 EmitOperand(0, address); 103} 104 105 106void X86_64Assembler::movq(CpuRegister dst, const Immediate& imm) { 107 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 108 if (imm.is_int32()) { 109 // 32 bit. Note: sign-extends. 110 EmitRex64(dst); 111 EmitUint8(0xC7); 112 EmitRegisterOperand(0, dst.LowBits()); 113 EmitInt32(static_cast<int32_t>(imm.value())); 114 } else { 115 EmitRex64(dst); 116 EmitUint8(0xB8 + dst.LowBits()); 117 EmitInt64(imm.value()); 118 } 119} 120 121 122void X86_64Assembler::movl(CpuRegister dst, const Immediate& imm) { 123 CHECK(imm.is_int32()); 124 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 125 EmitOptionalRex32(dst); 126 EmitUint8(0xB8 + dst.LowBits()); 127 EmitImmediate(imm); 128} 129 130 131void X86_64Assembler::movq(CpuRegister dst, CpuRegister src) { 132 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 133 // 0x89 is movq r/m64 <- r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 134 EmitRex64(src, dst); 135 EmitUint8(0x89); 136 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 137} 138 139 140void X86_64Assembler::movl(CpuRegister dst, CpuRegister src) { 141 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 142 EmitOptionalRex32(dst, src); 143 EmitUint8(0x8B); 144 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 145} 146 147 148void X86_64Assembler::movq(CpuRegister dst, const Address& src) { 149 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 150 EmitRex64(dst, src); 151 EmitUint8(0x8B); 152 EmitOperand(dst.LowBits(), src); 153} 154 155 156void X86_64Assembler::movl(CpuRegister dst, const Address& src) { 157 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 158 EmitOptionalRex32(dst, src); 159 EmitUint8(0x8B); 160 EmitOperand(dst.LowBits(), src); 161} 162 163 164void X86_64Assembler::movq(const Address& dst, CpuRegister src) { 165 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 166 EmitRex64(src, dst); 167 EmitUint8(0x89); 168 EmitOperand(src.LowBits(), dst); 169} 170 171 172void X86_64Assembler::movl(const Address& dst, CpuRegister src) { 173 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 174 EmitOptionalRex32(src, dst); 175 EmitUint8(0x89); 176 EmitOperand(src.LowBits(), dst); 177} 178 179void X86_64Assembler::movl(const Address& dst, const Immediate& imm) { 180 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 181 EmitOptionalRex32(dst); 182 EmitUint8(0xC7); 183 EmitOperand(0, dst); 184 EmitImmediate(imm); 185} 186 187void X86_64Assembler::movzxb(CpuRegister dst, CpuRegister src) { 188 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 189 EmitOptionalByteRegNormalizingRex32(dst, src); 190 EmitUint8(0x0F); 191 EmitUint8(0xB6); 192 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 193} 194 195 196void X86_64Assembler::movzxb(CpuRegister dst, const Address& src) { 197 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 198 EmitOptionalByteRegNormalizingRex32(dst, src); 199 EmitUint8(0x0F); 200 EmitUint8(0xB6); 201 EmitOperand(dst.LowBits(), src); 202} 203 204 205void X86_64Assembler::movsxb(CpuRegister dst, CpuRegister src) { 206 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 207 EmitOptionalByteRegNormalizingRex32(dst, src); 208 EmitUint8(0x0F); 209 EmitUint8(0xBE); 210 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 211} 212 213 214void X86_64Assembler::movsxb(CpuRegister dst, const Address& src) { 215 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 216 EmitOptionalByteRegNormalizingRex32(dst, src); 217 EmitUint8(0x0F); 218 EmitUint8(0xBE); 219 EmitOperand(dst.LowBits(), src); 220} 221 222 223void X86_64Assembler::movb(CpuRegister /*dst*/, const Address& /*src*/) { 224 LOG(FATAL) << "Use movzxb or movsxb instead."; 225} 226 227 228void X86_64Assembler::movb(const Address& dst, CpuRegister src) { 229 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 230 EmitOptionalByteRegNormalizingRex32(src, dst); 231 EmitUint8(0x88); 232 EmitOperand(src.LowBits(), dst); 233} 234 235 236void X86_64Assembler::movb(const Address& dst, const Immediate& imm) { 237 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 238 EmitOptionalRex32(dst); 239 EmitUint8(0xC6); 240 EmitOperand(Register::RAX, dst); 241 CHECK(imm.is_int8()); 242 EmitUint8(imm.value() & 0xFF); 243} 244 245 246void X86_64Assembler::movzxw(CpuRegister dst, CpuRegister src) { 247 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 248 EmitOptionalRex32(dst, src); 249 EmitUint8(0x0F); 250 EmitUint8(0xB7); 251 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 252} 253 254 255void X86_64Assembler::movzxw(CpuRegister dst, const Address& src) { 256 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 257 EmitOptionalRex32(dst, src); 258 EmitUint8(0x0F); 259 EmitUint8(0xB7); 260 EmitOperand(dst.LowBits(), src); 261} 262 263 264void X86_64Assembler::movsxw(CpuRegister dst, CpuRegister src) { 265 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 266 EmitOptionalRex32(dst, src); 267 EmitUint8(0x0F); 268 EmitUint8(0xBF); 269 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 270} 271 272 273void X86_64Assembler::movsxw(CpuRegister dst, const Address& src) { 274 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 275 EmitOptionalRex32(dst, src); 276 EmitUint8(0x0F); 277 EmitUint8(0xBF); 278 EmitOperand(dst.LowBits(), src); 279} 280 281 282void X86_64Assembler::movw(CpuRegister /*dst*/, const Address& /*src*/) { 283 LOG(FATAL) << "Use movzxw or movsxw instead."; 284} 285 286 287void X86_64Assembler::movw(const Address& dst, CpuRegister src) { 288 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 289 EmitOperandSizeOverride(); 290 EmitOptionalRex32(src, dst); 291 EmitUint8(0x89); 292 EmitOperand(src.LowBits(), dst); 293} 294 295 296void X86_64Assembler::movw(const Address& dst, const Immediate& imm) { 297 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 298 EmitOperandSizeOverride(); 299 EmitOptionalRex32(dst); 300 EmitUint8(0xC7); 301 EmitOperand(Register::RAX, dst); 302 CHECK(imm.is_uint16() || imm.is_int16()); 303 EmitUint8(imm.value() & 0xFF); 304 EmitUint8(imm.value() >> 8); 305} 306 307 308void X86_64Assembler::leaq(CpuRegister dst, const Address& src) { 309 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 310 EmitRex64(dst, src); 311 EmitUint8(0x8D); 312 EmitOperand(dst.LowBits(), src); 313} 314 315 316void X86_64Assembler::movaps(XmmRegister dst, XmmRegister src) { 317 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 318 EmitOptionalRex32(dst, src); 319 EmitUint8(0x0F); 320 EmitUint8(0x28); 321 EmitXmmRegisterOperand(dst.LowBits(), src); 322} 323 324 325void X86_64Assembler::movss(XmmRegister dst, const Address& src) { 326 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 327 EmitUint8(0xF3); 328 EmitOptionalRex32(dst, src); 329 EmitUint8(0x0F); 330 EmitUint8(0x10); 331 EmitOperand(dst.LowBits(), src); 332} 333 334 335void X86_64Assembler::movss(const Address& dst, XmmRegister src) { 336 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 337 EmitUint8(0xF3); 338 EmitOptionalRex32(src, dst); 339 EmitUint8(0x0F); 340 EmitUint8(0x11); 341 EmitOperand(src.LowBits(), dst); 342} 343 344 345void X86_64Assembler::movss(XmmRegister dst, XmmRegister src) { 346 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 347 EmitUint8(0xF3); 348 EmitOptionalRex32(src, dst); // Movss is MR encoding instead of the usual RM. 349 EmitUint8(0x0F); 350 EmitUint8(0x11); 351 EmitXmmRegisterOperand(src.LowBits(), dst); 352} 353 354 355void X86_64Assembler::movsxd(CpuRegister dst, CpuRegister src) { 356 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 357 EmitRex64(dst, src); 358 EmitUint8(0x63); 359 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 360} 361 362 363void X86_64Assembler::movsxd(CpuRegister dst, const Address& src) { 364 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 365 EmitRex64(dst); 366 EmitUint8(0x63); 367 EmitOperand(dst.LowBits(), src); 368} 369 370 371void X86_64Assembler::movd(XmmRegister dst, CpuRegister src) { 372 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 373 EmitUint8(0x66); 374 EmitRex64(dst, src); 375 EmitUint8(0x0F); 376 EmitUint8(0x6E); 377 EmitOperand(dst.LowBits(), Operand(src)); 378} 379 380 381void X86_64Assembler::movd(CpuRegister dst, XmmRegister src) { 382 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 383 EmitUint8(0x66); 384 EmitRex64(src, dst); 385 EmitUint8(0x0F); 386 EmitUint8(0x7E); 387 EmitOperand(src.LowBits(), Operand(dst)); 388} 389 390 391void X86_64Assembler::addss(XmmRegister dst, XmmRegister src) { 392 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 393 EmitUint8(0xF3); 394 EmitOptionalRex32(dst, src); 395 EmitUint8(0x0F); 396 EmitUint8(0x58); 397 EmitXmmRegisterOperand(dst.LowBits(), src); 398} 399 400 401void X86_64Assembler::addss(XmmRegister dst, const Address& src) { 402 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 403 EmitUint8(0xF3); 404 EmitOptionalRex32(dst, src); 405 EmitUint8(0x0F); 406 EmitUint8(0x58); 407 EmitOperand(dst.LowBits(), src); 408} 409 410 411void X86_64Assembler::subss(XmmRegister dst, XmmRegister src) { 412 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 413 EmitUint8(0xF3); 414 EmitOptionalRex32(dst, src); 415 EmitUint8(0x0F); 416 EmitUint8(0x5C); 417 EmitXmmRegisterOperand(dst.LowBits(), src); 418} 419 420 421void X86_64Assembler::subss(XmmRegister dst, const Address& src) { 422 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 423 EmitUint8(0xF3); 424 EmitOptionalRex32(dst, src); 425 EmitUint8(0x0F); 426 EmitUint8(0x5C); 427 EmitOperand(dst.LowBits(), src); 428} 429 430 431void X86_64Assembler::mulss(XmmRegister dst, XmmRegister src) { 432 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 433 EmitUint8(0xF3); 434 EmitOptionalRex32(dst, src); 435 EmitUint8(0x0F); 436 EmitUint8(0x59); 437 EmitXmmRegisterOperand(dst.LowBits(), src); 438} 439 440 441void X86_64Assembler::mulss(XmmRegister dst, const Address& src) { 442 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 443 EmitUint8(0xF3); 444 EmitOptionalRex32(dst, src); 445 EmitUint8(0x0F); 446 EmitUint8(0x59); 447 EmitOperand(dst.LowBits(), src); 448} 449 450 451void X86_64Assembler::divss(XmmRegister dst, XmmRegister src) { 452 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 453 EmitUint8(0xF3); 454 EmitOptionalRex32(dst, src); 455 EmitUint8(0x0F); 456 EmitUint8(0x5E); 457 EmitXmmRegisterOperand(dst.LowBits(), src); 458} 459 460 461void X86_64Assembler::divss(XmmRegister dst, const Address& src) { 462 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 463 EmitUint8(0xF3); 464 EmitOptionalRex32(dst, src); 465 EmitUint8(0x0F); 466 EmitUint8(0x5E); 467 EmitOperand(dst.LowBits(), src); 468} 469 470 471void X86_64Assembler::flds(const Address& src) { 472 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 473 EmitUint8(0xD9); 474 EmitOperand(0, src); 475} 476 477 478void X86_64Assembler::fstps(const Address& dst) { 479 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 480 EmitUint8(0xD9); 481 EmitOperand(3, dst); 482} 483 484 485void X86_64Assembler::movsd(XmmRegister dst, const Address& src) { 486 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 487 EmitUint8(0xF2); 488 EmitOptionalRex32(dst, src); 489 EmitUint8(0x0F); 490 EmitUint8(0x10); 491 EmitOperand(dst.LowBits(), src); 492} 493 494 495void X86_64Assembler::movsd(const Address& dst, XmmRegister src) { 496 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 497 EmitUint8(0xF2); 498 EmitOptionalRex32(src, dst); 499 EmitUint8(0x0F); 500 EmitUint8(0x11); 501 EmitOperand(src.LowBits(), dst); 502} 503 504 505void X86_64Assembler::movsd(XmmRegister dst, XmmRegister src) { 506 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 507 EmitUint8(0xF2); 508 EmitOptionalRex32(src, dst); // Movsd is MR encoding instead of the usual RM. 509 EmitUint8(0x0F); 510 EmitUint8(0x11); 511 EmitXmmRegisterOperand(src.LowBits(), dst); 512} 513 514 515void X86_64Assembler::addsd(XmmRegister dst, XmmRegister src) { 516 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 517 EmitUint8(0xF2); 518 EmitOptionalRex32(dst, src); 519 EmitUint8(0x0F); 520 EmitUint8(0x58); 521 EmitXmmRegisterOperand(dst.LowBits(), src); 522} 523 524 525void X86_64Assembler::addsd(XmmRegister dst, const Address& src) { 526 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 527 EmitUint8(0xF2); 528 EmitOptionalRex32(dst, src); 529 EmitUint8(0x0F); 530 EmitUint8(0x58); 531 EmitOperand(dst.LowBits(), src); 532} 533 534 535void X86_64Assembler::subsd(XmmRegister dst, XmmRegister src) { 536 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 537 EmitUint8(0xF2); 538 EmitOptionalRex32(dst, src); 539 EmitUint8(0x0F); 540 EmitUint8(0x5C); 541 EmitXmmRegisterOperand(dst.LowBits(), src); 542} 543 544 545void X86_64Assembler::subsd(XmmRegister dst, const Address& src) { 546 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 547 EmitUint8(0xF2); 548 EmitOptionalRex32(dst, src); 549 EmitUint8(0x0F); 550 EmitUint8(0x5C); 551 EmitOperand(dst.LowBits(), src); 552} 553 554 555void X86_64Assembler::mulsd(XmmRegister dst, XmmRegister src) { 556 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 557 EmitUint8(0xF2); 558 EmitOptionalRex32(dst, src); 559 EmitUint8(0x0F); 560 EmitUint8(0x59); 561 EmitXmmRegisterOperand(dst.LowBits(), src); 562} 563 564 565void X86_64Assembler::mulsd(XmmRegister dst, const Address& src) { 566 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 567 EmitUint8(0xF2); 568 EmitOptionalRex32(dst, src); 569 EmitUint8(0x0F); 570 EmitUint8(0x59); 571 EmitOperand(dst.LowBits(), src); 572} 573 574 575void X86_64Assembler::divsd(XmmRegister dst, XmmRegister src) { 576 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 577 EmitUint8(0xF2); 578 EmitOptionalRex32(dst, src); 579 EmitUint8(0x0F); 580 EmitUint8(0x5E); 581 EmitXmmRegisterOperand(dst.LowBits(), src); 582} 583 584 585void X86_64Assembler::divsd(XmmRegister dst, const Address& src) { 586 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 587 EmitUint8(0xF2); 588 EmitOptionalRex32(dst, src); 589 EmitUint8(0x0F); 590 EmitUint8(0x5E); 591 EmitOperand(dst.LowBits(), src); 592} 593 594 595void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) { 596 cvtsi2ss(dst, src, false); 597} 598 599 600void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src, bool is64bit) { 601 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 602 EmitUint8(0xF3); 603 if (is64bit) { 604 // Emit a REX.W prefix if the operand size is 64 bits. 605 EmitRex64(dst, src); 606 } else { 607 EmitOptionalRex32(dst, src); 608 } 609 EmitUint8(0x0F); 610 EmitUint8(0x2A); 611 EmitOperand(dst.LowBits(), Operand(src)); 612} 613 614 615void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src) { 616 cvtsi2sd(dst, src, false); 617} 618 619 620void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src, bool is64bit) { 621 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 622 EmitUint8(0xF2); 623 if (is64bit) { 624 // Emit a REX.W prefix if the operand size is 64 bits. 625 EmitRex64(dst, src); 626 } else { 627 EmitOptionalRex32(dst, src); 628 } 629 EmitUint8(0x0F); 630 EmitUint8(0x2A); 631 EmitOperand(dst.LowBits(), Operand(src)); 632} 633 634 635void X86_64Assembler::cvtss2si(CpuRegister dst, XmmRegister src) { 636 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 637 EmitUint8(0xF3); 638 EmitOptionalRex32(dst, src); 639 EmitUint8(0x0F); 640 EmitUint8(0x2D); 641 EmitXmmRegisterOperand(dst.LowBits(), src); 642} 643 644 645void X86_64Assembler::cvtss2sd(XmmRegister dst, XmmRegister src) { 646 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 647 EmitUint8(0xF3); 648 EmitOptionalRex32(dst, src); 649 EmitUint8(0x0F); 650 EmitUint8(0x5A); 651 EmitXmmRegisterOperand(dst.LowBits(), src); 652} 653 654 655void X86_64Assembler::cvtsd2si(CpuRegister dst, XmmRegister src) { 656 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 657 EmitUint8(0xF2); 658 EmitOptionalRex32(dst, src); 659 EmitUint8(0x0F); 660 EmitUint8(0x2D); 661 EmitXmmRegisterOperand(dst.LowBits(), src); 662} 663 664 665void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src) { 666 cvttss2si(dst, src, false); 667} 668 669 670void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src, bool is64bit) { 671 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 672 EmitUint8(0xF3); 673 if (is64bit) { 674 // Emit a REX.W prefix if the operand size is 64 bits. 675 EmitRex64(dst, src); 676 } else { 677 EmitOptionalRex32(dst, src); 678 } 679 EmitUint8(0x0F); 680 EmitUint8(0x2C); 681 EmitXmmRegisterOperand(dst.LowBits(), src); 682} 683 684 685void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src) { 686 cvttsd2si(dst, src, false); 687} 688 689 690void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src, bool is64bit) { 691 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 692 EmitUint8(0xF2); 693 if (is64bit) { 694 // Emit a REX.W prefix if the operand size is 64 bits. 695 EmitRex64(dst, src); 696 } else { 697 EmitOptionalRex32(dst, src); 698 } 699 EmitUint8(0x0F); 700 EmitUint8(0x2C); 701 EmitXmmRegisterOperand(dst.LowBits(), src); 702} 703 704 705void X86_64Assembler::cvtsd2ss(XmmRegister dst, XmmRegister src) { 706 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 707 EmitUint8(0xF2); 708 EmitOptionalRex32(dst, src); 709 EmitUint8(0x0F); 710 EmitUint8(0x5A); 711 EmitXmmRegisterOperand(dst.LowBits(), src); 712} 713 714 715void X86_64Assembler::cvtdq2pd(XmmRegister dst, XmmRegister src) { 716 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 717 EmitUint8(0xF3); 718 EmitOptionalRex32(dst, src); 719 EmitUint8(0x0F); 720 EmitUint8(0xE6); 721 EmitXmmRegisterOperand(dst.LowBits(), src); 722} 723 724 725void X86_64Assembler::comiss(XmmRegister a, XmmRegister b) { 726 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 727 EmitOptionalRex32(a, b); 728 EmitUint8(0x0F); 729 EmitUint8(0x2F); 730 EmitXmmRegisterOperand(a.LowBits(), b); 731} 732 733 734void X86_64Assembler::comisd(XmmRegister a, XmmRegister b) { 735 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 736 EmitUint8(0x66); 737 EmitOptionalRex32(a, b); 738 EmitUint8(0x0F); 739 EmitUint8(0x2F); 740 EmitXmmRegisterOperand(a.LowBits(), b); 741} 742 743void X86_64Assembler::ucomiss(XmmRegister a, XmmRegister b) { 744 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 745 EmitOptionalRex32(a, b); 746 EmitUint8(0x0F); 747 EmitUint8(0x2E); 748 EmitXmmRegisterOperand(a.LowBits(), b); 749} 750 751 752void X86_64Assembler::ucomisd(XmmRegister a, XmmRegister b) { 753 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 754 EmitUint8(0x66); 755 EmitOptionalRex32(a, b); 756 EmitUint8(0x0F); 757 EmitUint8(0x2E); 758 EmitXmmRegisterOperand(a.LowBits(), b); 759} 760 761 762void X86_64Assembler::sqrtsd(XmmRegister dst, XmmRegister src) { 763 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 764 EmitUint8(0xF2); 765 EmitOptionalRex32(dst, src); 766 EmitUint8(0x0F); 767 EmitUint8(0x51); 768 EmitXmmRegisterOperand(dst.LowBits(), src); 769} 770 771 772void X86_64Assembler::sqrtss(XmmRegister dst, XmmRegister src) { 773 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 774 EmitUint8(0xF3); 775 EmitOptionalRex32(dst, src); 776 EmitUint8(0x0F); 777 EmitUint8(0x51); 778 EmitXmmRegisterOperand(dst.LowBits(), src); 779} 780 781 782void X86_64Assembler::xorpd(XmmRegister dst, const Address& src) { 783 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 784 EmitUint8(0x66); 785 EmitOptionalRex32(dst, src); 786 EmitUint8(0x0F); 787 EmitUint8(0x57); 788 EmitOperand(dst.LowBits(), src); 789} 790 791 792void X86_64Assembler::xorpd(XmmRegister dst, XmmRegister src) { 793 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 794 EmitUint8(0x66); 795 EmitOptionalRex32(dst, src); 796 EmitUint8(0x0F); 797 EmitUint8(0x57); 798 EmitXmmRegisterOperand(dst.LowBits(), src); 799} 800 801 802void X86_64Assembler::xorps(XmmRegister dst, const Address& src) { 803 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 804 EmitOptionalRex32(dst, src); 805 EmitUint8(0x0F); 806 EmitUint8(0x57); 807 EmitOperand(dst.LowBits(), src); 808} 809 810 811void X86_64Assembler::xorps(XmmRegister dst, XmmRegister src) { 812 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 813 EmitOptionalRex32(dst, src); 814 EmitUint8(0x0F); 815 EmitUint8(0x57); 816 EmitXmmRegisterOperand(dst.LowBits(), src); 817} 818 819 820void X86_64Assembler::andpd(XmmRegister dst, const Address& src) { 821 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 822 EmitUint8(0x66); 823 EmitOptionalRex32(dst, src); 824 EmitUint8(0x0F); 825 EmitUint8(0x54); 826 EmitOperand(dst.LowBits(), src); 827} 828 829 830void X86_64Assembler::fldl(const Address& src) { 831 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 832 EmitUint8(0xDD); 833 EmitOperand(0, src); 834} 835 836 837void X86_64Assembler::fstpl(const Address& dst) { 838 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 839 EmitUint8(0xDD); 840 EmitOperand(3, dst); 841} 842 843 844void X86_64Assembler::fnstcw(const Address& dst) { 845 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 846 EmitUint8(0xD9); 847 EmitOperand(7, dst); 848} 849 850 851void X86_64Assembler::fldcw(const Address& src) { 852 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 853 EmitUint8(0xD9); 854 EmitOperand(5, src); 855} 856 857 858void X86_64Assembler::fistpl(const Address& dst) { 859 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 860 EmitUint8(0xDF); 861 EmitOperand(7, dst); 862} 863 864 865void X86_64Assembler::fistps(const Address& dst) { 866 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 867 EmitUint8(0xDB); 868 EmitOperand(3, dst); 869} 870 871 872void X86_64Assembler::fildl(const Address& src) { 873 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 874 EmitUint8(0xDF); 875 EmitOperand(5, src); 876} 877 878 879void X86_64Assembler::fincstp() { 880 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 881 EmitUint8(0xD9); 882 EmitUint8(0xF7); 883} 884 885 886void X86_64Assembler::ffree(const Immediate& index) { 887 CHECK_LT(index.value(), 7); 888 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 889 EmitUint8(0xDD); 890 EmitUint8(0xC0 + index.value()); 891} 892 893 894void X86_64Assembler::fsin() { 895 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 896 EmitUint8(0xD9); 897 EmitUint8(0xFE); 898} 899 900 901void X86_64Assembler::fcos() { 902 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 903 EmitUint8(0xD9); 904 EmitUint8(0xFF); 905} 906 907 908void X86_64Assembler::fptan() { 909 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 910 EmitUint8(0xD9); 911 EmitUint8(0xF2); 912} 913 914 915void X86_64Assembler::xchgl(CpuRegister dst, CpuRegister src) { 916 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 917 // There is a short version for rax. 918 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't 919 // work. 920 const bool src_rax = src.AsRegister() == RAX; 921 const bool dst_rax = dst.AsRegister() == RAX; 922 if (src_rax || dst_rax) { 923 EmitOptionalRex32(src_rax ? dst : src); 924 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits())); 925 return; 926 } 927 928 // General case. 929 EmitOptionalRex32(src, dst); 930 EmitUint8(0x87); 931 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 932} 933 934 935void X86_64Assembler::xchgq(CpuRegister dst, CpuRegister src) { 936 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 937 // There is a short version for rax. 938 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't 939 // work. 940 const bool src_rax = src.AsRegister() == RAX; 941 const bool dst_rax = dst.AsRegister() == RAX; 942 if (src_rax || dst_rax) { 943 // If src == target, emit a nop instead. 944 if (src_rax && dst_rax) { 945 EmitUint8(0x90); 946 } else { 947 EmitRex64(src_rax ? dst : src); 948 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits())); 949 } 950 return; 951 } 952 953 // General case. 954 EmitRex64(src, dst); 955 EmitUint8(0x87); 956 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 957} 958 959 960void X86_64Assembler::xchgl(CpuRegister reg, const Address& address) { 961 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 962 EmitOptionalRex32(reg, address); 963 EmitUint8(0x87); 964 EmitOperand(reg.LowBits(), address); 965} 966 967 968void X86_64Assembler::cmpw(const Address& address, const Immediate& imm) { 969 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 970 EmitOptionalRex32(address); 971 EmitUint8(0x66); 972 EmitComplex(7, address, imm); 973} 974 975 976void X86_64Assembler::cmpl(CpuRegister reg, const Immediate& imm) { 977 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 978 EmitOptionalRex32(reg); 979 EmitComplex(7, Operand(reg), imm); 980} 981 982 983void X86_64Assembler::cmpl(CpuRegister reg0, CpuRegister reg1) { 984 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 985 EmitOptionalRex32(reg0, reg1); 986 EmitUint8(0x3B); 987 EmitOperand(reg0.LowBits(), Operand(reg1)); 988} 989 990 991void X86_64Assembler::cmpl(CpuRegister reg, const Address& address) { 992 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 993 EmitOptionalRex32(reg, address); 994 EmitUint8(0x3B); 995 EmitOperand(reg.LowBits(), address); 996} 997 998 999void X86_64Assembler::cmpl(const Address& address, CpuRegister reg) { 1000 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1001 EmitOptionalRex32(reg, address); 1002 EmitUint8(0x39); 1003 EmitOperand(reg.LowBits(), address); 1004} 1005 1006 1007void X86_64Assembler::cmpl(const Address& address, const Immediate& imm) { 1008 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1009 EmitOptionalRex32(address); 1010 EmitComplex(7, address, imm); 1011} 1012 1013 1014void X86_64Assembler::cmpq(CpuRegister reg0, CpuRegister reg1) { 1015 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1016 EmitRex64(reg0, reg1); 1017 EmitUint8(0x3B); 1018 EmitOperand(reg0.LowBits(), Operand(reg1)); 1019} 1020 1021 1022void X86_64Assembler::cmpq(CpuRegister reg, const Immediate& imm) { 1023 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1024 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 1025 EmitRex64(reg); 1026 EmitComplex(7, Operand(reg), imm); 1027} 1028 1029 1030void X86_64Assembler::cmpq(CpuRegister reg, const Address& address) { 1031 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1032 EmitRex64(reg); 1033 EmitUint8(0x3B); 1034 EmitOperand(reg.LowBits(), address); 1035} 1036 1037 1038void X86_64Assembler::cmpq(const Address& address, const Immediate& imm) { 1039 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 1040 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1041 EmitRex64(address); 1042 EmitComplex(7, address, imm); 1043} 1044 1045 1046void X86_64Assembler::addl(CpuRegister dst, CpuRegister src) { 1047 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1048 EmitOptionalRex32(dst, src); 1049 EmitUint8(0x03); 1050 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1051} 1052 1053 1054void X86_64Assembler::addl(CpuRegister reg, const Address& address) { 1055 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1056 EmitOptionalRex32(reg, address); 1057 EmitUint8(0x03); 1058 EmitOperand(reg.LowBits(), address); 1059} 1060 1061 1062void X86_64Assembler::testl(CpuRegister reg1, CpuRegister reg2) { 1063 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1064 EmitOptionalRex32(reg1, reg2); 1065 EmitUint8(0x85); 1066 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 1067} 1068 1069 1070void X86_64Assembler::testl(CpuRegister reg, const Immediate& immediate) { 1071 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1072 // For registers that have a byte variant (RAX, RBX, RCX, and RDX) 1073 // we only test the byte CpuRegister to keep the encoding short. 1074 if (immediate.is_uint8() && reg.AsRegister() < 4) { 1075 // Use zero-extended 8-bit immediate. 1076 if (reg.AsRegister() == RAX) { 1077 EmitUint8(0xA8); 1078 } else { 1079 EmitUint8(0xF6); 1080 EmitUint8(0xC0 + reg.AsRegister()); 1081 } 1082 EmitUint8(immediate.value() & 0xFF); 1083 } else if (reg.AsRegister() == RAX) { 1084 // Use short form if the destination is RAX. 1085 EmitUint8(0xA9); 1086 EmitImmediate(immediate); 1087 } else { 1088 EmitOptionalRex32(reg); 1089 EmitUint8(0xF7); 1090 EmitOperand(0, Operand(reg)); 1091 EmitImmediate(immediate); 1092 } 1093} 1094 1095 1096void X86_64Assembler::testq(CpuRegister reg1, CpuRegister reg2) { 1097 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1098 EmitRex64(reg1, reg2); 1099 EmitUint8(0x85); 1100 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 1101} 1102 1103 1104void X86_64Assembler::testq(CpuRegister reg, const Address& address) { 1105 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1106 EmitRex64(reg); 1107 EmitUint8(0x85); 1108 EmitOperand(reg.LowBits(), address); 1109} 1110 1111 1112void X86_64Assembler::andl(CpuRegister dst, CpuRegister src) { 1113 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1114 EmitOptionalRex32(dst, src); 1115 EmitUint8(0x23); 1116 EmitOperand(dst.LowBits(), Operand(src)); 1117} 1118 1119 1120void X86_64Assembler::andl(CpuRegister reg, const Address& address) { 1121 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1122 EmitOptionalRex32(reg, address); 1123 EmitUint8(0x23); 1124 EmitOperand(reg.LowBits(), address); 1125} 1126 1127 1128void X86_64Assembler::andl(CpuRegister dst, const Immediate& imm) { 1129 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1130 EmitOptionalRex32(dst); 1131 EmitComplex(4, Operand(dst), imm); 1132} 1133 1134 1135void X86_64Assembler::andq(CpuRegister reg, const Immediate& imm) { 1136 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1137 CHECK(imm.is_int32()); // andq only supports 32b immediate. 1138 EmitRex64(reg); 1139 EmitComplex(4, Operand(reg), imm); 1140} 1141 1142 1143void X86_64Assembler::andq(CpuRegister dst, CpuRegister src) { 1144 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1145 EmitRex64(dst, src); 1146 EmitUint8(0x23); 1147 EmitOperand(dst.LowBits(), Operand(src)); 1148} 1149 1150 1151void X86_64Assembler::orl(CpuRegister dst, CpuRegister src) { 1152 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1153 EmitOptionalRex32(dst, src); 1154 EmitUint8(0x0B); 1155 EmitOperand(dst.LowBits(), Operand(src)); 1156} 1157 1158 1159void X86_64Assembler::orl(CpuRegister reg, const Address& address) { 1160 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1161 EmitOptionalRex32(reg, address); 1162 EmitUint8(0x0B); 1163 EmitOperand(reg.LowBits(), address); 1164} 1165 1166 1167void X86_64Assembler::orl(CpuRegister dst, const Immediate& imm) { 1168 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1169 EmitOptionalRex32(dst); 1170 EmitComplex(1, Operand(dst), imm); 1171} 1172 1173 1174void X86_64Assembler::orq(CpuRegister dst, CpuRegister src) { 1175 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1176 EmitRex64(dst, src); 1177 EmitUint8(0x0B); 1178 EmitOperand(dst.LowBits(), Operand(src)); 1179} 1180 1181 1182void X86_64Assembler::xorl(CpuRegister dst, CpuRegister src) { 1183 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1184 EmitOptionalRex32(dst, src); 1185 EmitUint8(0x33); 1186 EmitOperand(dst.LowBits(), Operand(src)); 1187} 1188 1189 1190void X86_64Assembler::xorl(CpuRegister reg, const Address& address) { 1191 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1192 EmitOptionalRex32(reg, address); 1193 EmitUint8(0x33); 1194 EmitOperand(reg.LowBits(), address); 1195} 1196 1197 1198void X86_64Assembler::xorl(CpuRegister dst, const Immediate& imm) { 1199 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1200 EmitOptionalRex32(dst); 1201 EmitComplex(6, Operand(dst), imm); 1202} 1203 1204 1205void X86_64Assembler::xorq(CpuRegister dst, CpuRegister src) { 1206 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1207 EmitRex64(dst, src); 1208 EmitUint8(0x33); 1209 EmitOperand(dst.LowBits(), Operand(src)); 1210} 1211 1212 1213void X86_64Assembler::xorq(CpuRegister dst, const Immediate& imm) { 1214 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1215 CHECK(imm.is_int32()); // xorq only supports 32b immediate. 1216 EmitRex64(dst); 1217 EmitComplex(6, Operand(dst), imm); 1218} 1219 1220#if 0 1221void X86_64Assembler::rex(bool force, bool w, Register* r, Register* x, Register* b) { 1222 // REX.WRXB 1223 // W - 64-bit operand 1224 // R - MODRM.reg 1225 // X - SIB.index 1226 // B - MODRM.rm/SIB.base 1227 uint8_t rex = force ? 0x40 : 0; 1228 if (w) { 1229 rex |= 0x48; // REX.W000 1230 } 1231 if (r != nullptr && *r >= Register::R8 && *r < Register::kNumberOfCpuRegisters) { 1232 rex |= 0x44; // REX.0R00 1233 *r = static_cast<Register>(*r - 8); 1234 } 1235 if (x != nullptr && *x >= Register::R8 && *x < Register::kNumberOfCpuRegisters) { 1236 rex |= 0x42; // REX.00X0 1237 *x = static_cast<Register>(*x - 8); 1238 } 1239 if (b != nullptr && *b >= Register::R8 && *b < Register::kNumberOfCpuRegisters) { 1240 rex |= 0x41; // REX.000B 1241 *b = static_cast<Register>(*b - 8); 1242 } 1243 if (rex != 0) { 1244 EmitUint8(rex); 1245 } 1246} 1247 1248void X86_64Assembler::rex_reg_mem(bool force, bool w, Register* dst, const Address& mem) { 1249 // REX.WRXB 1250 // W - 64-bit operand 1251 // R - MODRM.reg 1252 // X - SIB.index 1253 // B - MODRM.rm/SIB.base 1254 uint8_t rex = mem->rex(); 1255 if (force) { 1256 rex |= 0x40; // REX.0000 1257 } 1258 if (w) { 1259 rex |= 0x48; // REX.W000 1260 } 1261 if (dst != nullptr && *dst >= Register::R8 && *dst < Register::kNumberOfCpuRegisters) { 1262 rex |= 0x44; // REX.0R00 1263 *dst = static_cast<Register>(*dst - 8); 1264 } 1265 if (rex != 0) { 1266 EmitUint8(rex); 1267 } 1268} 1269 1270void rex_mem_reg(bool force, bool w, Address* mem, Register* src); 1271#endif 1272 1273void X86_64Assembler::addl(CpuRegister reg, const Immediate& imm) { 1274 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1275 EmitOptionalRex32(reg); 1276 EmitComplex(0, Operand(reg), imm); 1277} 1278 1279 1280void X86_64Assembler::addq(CpuRegister reg, const Immediate& imm) { 1281 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1282 CHECK(imm.is_int32()); // addq only supports 32b immediate. 1283 EmitRex64(reg); 1284 EmitComplex(0, Operand(reg), imm); 1285} 1286 1287 1288void X86_64Assembler::addq(CpuRegister dst, const Address& address) { 1289 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1290 EmitRex64(dst); 1291 EmitUint8(0x03); 1292 EmitOperand(dst.LowBits(), address); 1293} 1294 1295 1296void X86_64Assembler::addq(CpuRegister dst, CpuRegister src) { 1297 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1298 // 0x01 is addq r/m64 <- r/m64 + r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 1299 EmitRex64(src, dst); 1300 EmitUint8(0x01); 1301 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1302} 1303 1304 1305void X86_64Assembler::addl(const Address& address, CpuRegister reg) { 1306 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1307 EmitOptionalRex32(reg, address); 1308 EmitUint8(0x01); 1309 EmitOperand(reg.LowBits(), address); 1310} 1311 1312 1313void X86_64Assembler::addl(const Address& address, const Immediate& imm) { 1314 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1315 EmitOptionalRex32(address); 1316 EmitComplex(0, address, imm); 1317} 1318 1319 1320void X86_64Assembler::subl(CpuRegister dst, CpuRegister src) { 1321 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1322 EmitOptionalRex32(dst, src); 1323 EmitUint8(0x2B); 1324 EmitOperand(dst.LowBits(), Operand(src)); 1325} 1326 1327 1328void X86_64Assembler::subl(CpuRegister reg, const Immediate& imm) { 1329 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1330 EmitOptionalRex32(reg); 1331 EmitComplex(5, Operand(reg), imm); 1332} 1333 1334 1335void X86_64Assembler::subq(CpuRegister reg, const Immediate& imm) { 1336 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1337 CHECK(imm.is_int32()); // subq only supports 32b immediate. 1338 EmitRex64(reg); 1339 EmitComplex(5, Operand(reg), imm); 1340} 1341 1342 1343void X86_64Assembler::subq(CpuRegister dst, CpuRegister src) { 1344 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1345 EmitRex64(dst, src); 1346 EmitUint8(0x2B); 1347 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1348} 1349 1350 1351void X86_64Assembler::subq(CpuRegister reg, const Address& address) { 1352 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1353 EmitRex64(reg); 1354 EmitUint8(0x2B); 1355 EmitOperand(reg.LowBits() & 7, address); 1356} 1357 1358 1359void X86_64Assembler::subl(CpuRegister reg, const Address& address) { 1360 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1361 EmitOptionalRex32(reg, address); 1362 EmitUint8(0x2B); 1363 EmitOperand(reg.LowBits(), address); 1364} 1365 1366 1367void X86_64Assembler::cdq() { 1368 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1369 EmitUint8(0x99); 1370} 1371 1372 1373void X86_64Assembler::cqo() { 1374 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1375 EmitRex64(); 1376 EmitUint8(0x99); 1377} 1378 1379 1380void X86_64Assembler::idivl(CpuRegister reg) { 1381 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1382 EmitOptionalRex32(reg); 1383 EmitUint8(0xF7); 1384 EmitUint8(0xF8 | reg.LowBits()); 1385} 1386 1387 1388void X86_64Assembler::idivq(CpuRegister reg) { 1389 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1390 EmitRex64(reg); 1391 EmitUint8(0xF7); 1392 EmitUint8(0xF8 | reg.LowBits()); 1393} 1394 1395 1396void X86_64Assembler::imull(CpuRegister dst, CpuRegister src) { 1397 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1398 EmitOptionalRex32(dst, src); 1399 EmitUint8(0x0F); 1400 EmitUint8(0xAF); 1401 EmitOperand(dst.LowBits(), Operand(src)); 1402} 1403 1404void X86_64Assembler::imull(CpuRegister reg, const Immediate& imm) { 1405 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1406 CHECK(imm.is_int32()); // imull only supports 32b immediate. 1407 1408 EmitOptionalRex32(reg, reg); 1409 1410 // See whether imm can be represented as a sign-extended 8bit value. 1411 int32_t v32 = static_cast<int32_t>(imm.value()); 1412 if (IsInt32(8, v32)) { 1413 // Sign-extension works. 1414 EmitUint8(0x6B); 1415 EmitOperand(reg.LowBits(), Operand(reg)); 1416 EmitUint8(static_cast<uint8_t>(v32 & 0xFF)); 1417 } else { 1418 // Not representable, use full immediate. 1419 EmitUint8(0x69); 1420 EmitOperand(reg.LowBits(), Operand(reg)); 1421 EmitImmediate(imm); 1422 } 1423} 1424 1425 1426void X86_64Assembler::imull(CpuRegister reg, const Address& address) { 1427 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1428 EmitOptionalRex32(reg, address); 1429 EmitUint8(0x0F); 1430 EmitUint8(0xAF); 1431 EmitOperand(reg.LowBits(), address); 1432} 1433 1434 1435void X86_64Assembler::imulq(CpuRegister dst, CpuRegister src) { 1436 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1437 EmitRex64(dst, src); 1438 EmitUint8(0x0F); 1439 EmitUint8(0xAF); 1440 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1441} 1442 1443 1444void X86_64Assembler::imulq(CpuRegister reg, const Immediate& imm) { 1445 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1446 CHECK(imm.is_int32()); // imulq only supports 32b immediate. 1447 1448 EmitRex64(reg, reg); 1449 1450 // See whether imm can be represented as a sign-extended 8bit value. 1451 int64_t v64 = imm.value(); 1452 if (IsInt64(8, v64)) { 1453 // Sign-extension works. 1454 EmitUint8(0x6B); 1455 EmitOperand(reg.LowBits(), Operand(reg)); 1456 EmitUint8(static_cast<uint8_t>(v64 & 0xFF)); 1457 } else { 1458 // Not representable, use full immediate. 1459 EmitUint8(0x69); 1460 EmitOperand(reg.LowBits(), Operand(reg)); 1461 EmitImmediate(imm); 1462 } 1463} 1464 1465 1466void X86_64Assembler::imulq(CpuRegister reg, const Address& address) { 1467 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1468 EmitRex64(reg, address); 1469 EmitUint8(0x0F); 1470 EmitUint8(0xAF); 1471 EmitOperand(reg.LowBits(), address); 1472} 1473 1474 1475void X86_64Assembler::imull(CpuRegister reg) { 1476 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1477 EmitOptionalRex32(reg); 1478 EmitUint8(0xF7); 1479 EmitOperand(5, Operand(reg)); 1480} 1481 1482 1483void X86_64Assembler::imull(const Address& address) { 1484 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1485 EmitOptionalRex32(address); 1486 EmitUint8(0xF7); 1487 EmitOperand(5, address); 1488} 1489 1490 1491void X86_64Assembler::mull(CpuRegister reg) { 1492 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1493 EmitOptionalRex32(reg); 1494 EmitUint8(0xF7); 1495 EmitOperand(4, Operand(reg)); 1496} 1497 1498 1499void X86_64Assembler::mull(const Address& address) { 1500 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1501 EmitOptionalRex32(address); 1502 EmitUint8(0xF7); 1503 EmitOperand(4, address); 1504} 1505 1506 1507void X86_64Assembler::shll(CpuRegister reg, const Immediate& imm) { 1508 EmitGenericShift(false, 4, reg, imm); 1509} 1510 1511 1512void X86_64Assembler::shlq(CpuRegister reg, const Immediate& imm) { 1513 EmitGenericShift(true, 4, reg, imm); 1514} 1515 1516 1517void X86_64Assembler::shll(CpuRegister operand, CpuRegister shifter) { 1518 EmitGenericShift(false, 4, operand, shifter); 1519} 1520 1521 1522void X86_64Assembler::shlq(CpuRegister operand, CpuRegister shifter) { 1523 EmitGenericShift(true, 4, operand, shifter); 1524} 1525 1526 1527void X86_64Assembler::shrl(CpuRegister reg, const Immediate& imm) { 1528 EmitGenericShift(false, 5, reg, imm); 1529} 1530 1531 1532void X86_64Assembler::shrq(CpuRegister reg, const Immediate& imm) { 1533 EmitGenericShift(true, 5, reg, imm); 1534} 1535 1536 1537void X86_64Assembler::shrl(CpuRegister operand, CpuRegister shifter) { 1538 EmitGenericShift(false, 5, operand, shifter); 1539} 1540 1541 1542void X86_64Assembler::shrq(CpuRegister operand, CpuRegister shifter) { 1543 EmitGenericShift(true, 5, operand, shifter); 1544} 1545 1546 1547void X86_64Assembler::sarl(CpuRegister reg, const Immediate& imm) { 1548 EmitGenericShift(false, 7, reg, imm); 1549} 1550 1551 1552void X86_64Assembler::sarl(CpuRegister operand, CpuRegister shifter) { 1553 EmitGenericShift(false, 7, operand, shifter); 1554} 1555 1556 1557void X86_64Assembler::sarq(CpuRegister reg, const Immediate& imm) { 1558 EmitGenericShift(true, 7, reg, imm); 1559} 1560 1561 1562void X86_64Assembler::sarq(CpuRegister operand, CpuRegister shifter) { 1563 EmitGenericShift(true, 7, operand, shifter); 1564} 1565 1566 1567void X86_64Assembler::negl(CpuRegister reg) { 1568 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1569 EmitOptionalRex32(reg); 1570 EmitUint8(0xF7); 1571 EmitOperand(3, Operand(reg)); 1572} 1573 1574 1575void X86_64Assembler::negq(CpuRegister reg) { 1576 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1577 EmitRex64(reg); 1578 EmitUint8(0xF7); 1579 EmitOperand(3, Operand(reg)); 1580} 1581 1582 1583void X86_64Assembler::notl(CpuRegister reg) { 1584 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1585 EmitOptionalRex32(reg); 1586 EmitUint8(0xF7); 1587 EmitUint8(0xD0 | reg.LowBits()); 1588} 1589 1590 1591void X86_64Assembler::notq(CpuRegister reg) { 1592 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1593 EmitRex64(reg); 1594 EmitUint8(0xF7); 1595 EmitOperand(2, Operand(reg)); 1596} 1597 1598 1599void X86_64Assembler::enter(const Immediate& imm) { 1600 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1601 EmitUint8(0xC8); 1602 CHECK(imm.is_uint16()); 1603 EmitUint8(imm.value() & 0xFF); 1604 EmitUint8((imm.value() >> 8) & 0xFF); 1605 EmitUint8(0x00); 1606} 1607 1608 1609void X86_64Assembler::leave() { 1610 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1611 EmitUint8(0xC9); 1612} 1613 1614 1615void X86_64Assembler::ret() { 1616 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1617 EmitUint8(0xC3); 1618} 1619 1620 1621void X86_64Assembler::ret(const Immediate& imm) { 1622 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1623 EmitUint8(0xC2); 1624 CHECK(imm.is_uint16()); 1625 EmitUint8(imm.value() & 0xFF); 1626 EmitUint8((imm.value() >> 8) & 0xFF); 1627} 1628 1629 1630 1631void X86_64Assembler::nop() { 1632 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1633 EmitUint8(0x90); 1634} 1635 1636 1637void X86_64Assembler::int3() { 1638 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1639 EmitUint8(0xCC); 1640} 1641 1642 1643void X86_64Assembler::hlt() { 1644 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1645 EmitUint8(0xF4); 1646} 1647 1648 1649void X86_64Assembler::j(Condition condition, Label* label) { 1650 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1651 if (label->IsBound()) { 1652 static const int kShortSize = 2; 1653 static const int kLongSize = 6; 1654 int offset = label->Position() - buffer_.Size(); 1655 CHECK_LE(offset, 0); 1656 if (IsInt(8, offset - kShortSize)) { 1657 EmitUint8(0x70 + condition); 1658 EmitUint8((offset - kShortSize) & 0xFF); 1659 } else { 1660 EmitUint8(0x0F); 1661 EmitUint8(0x80 + condition); 1662 EmitInt32(offset - kLongSize); 1663 } 1664 } else { 1665 EmitUint8(0x0F); 1666 EmitUint8(0x80 + condition); 1667 EmitLabelLink(label); 1668 } 1669} 1670 1671 1672void X86_64Assembler::jmp(CpuRegister reg) { 1673 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1674 EmitOptionalRex32(reg); 1675 EmitUint8(0xFF); 1676 EmitRegisterOperand(4, reg.LowBits()); 1677} 1678 1679void X86_64Assembler::jmp(const Address& address) { 1680 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1681 EmitOptionalRex32(address); 1682 EmitUint8(0xFF); 1683 EmitOperand(4, address); 1684} 1685 1686void X86_64Assembler::jmp(Label* label) { 1687 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1688 if (label->IsBound()) { 1689 static const int kShortSize = 2; 1690 static const int kLongSize = 5; 1691 int offset = label->Position() - buffer_.Size(); 1692 CHECK_LE(offset, 0); 1693 if (IsInt(8, offset - kShortSize)) { 1694 EmitUint8(0xEB); 1695 EmitUint8((offset - kShortSize) & 0xFF); 1696 } else { 1697 EmitUint8(0xE9); 1698 EmitInt32(offset - kLongSize); 1699 } 1700 } else { 1701 EmitUint8(0xE9); 1702 EmitLabelLink(label); 1703 } 1704} 1705 1706 1707X86_64Assembler* X86_64Assembler::lock() { 1708 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1709 EmitUint8(0xF0); 1710 return this; 1711} 1712 1713 1714void X86_64Assembler::cmpxchgl(const Address& address, CpuRegister reg) { 1715 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1716 EmitUint8(0x0F); 1717 EmitUint8(0xB1); 1718 EmitOperand(reg.LowBits(), address); 1719} 1720 1721void X86_64Assembler::mfence() { 1722 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1723 EmitUint8(0x0F); 1724 EmitUint8(0xAE); 1725 EmitUint8(0xF0); 1726} 1727 1728 1729X86_64Assembler* X86_64Assembler::gs() { 1730 // TODO: gs is a prefix and not an instruction 1731 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1732 EmitUint8(0x65); 1733 return this; 1734} 1735 1736 1737void X86_64Assembler::AddImmediate(CpuRegister reg, const Immediate& imm) { 1738 int value = imm.value(); 1739 if (value != 0) { 1740 if (value > 0) { 1741 addl(reg, imm); 1742 } else { 1743 subl(reg, Immediate(value)); 1744 } 1745 } 1746} 1747 1748 1749void X86_64Assembler::setcc(Condition condition, CpuRegister dst) { 1750 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1751 // RSP, RBP, RDI, RSI need rex prefix (else the pattern encodes ah/bh/ch/dh). 1752 if (dst.NeedsRex() || dst.AsRegister() > 3) { 1753 EmitOptionalRex(true, false, false, false, dst.NeedsRex()); 1754 } 1755 EmitUint8(0x0F); 1756 EmitUint8(0x90 + condition); 1757 EmitUint8(0xC0 + dst.LowBits()); 1758} 1759 1760 1761void X86_64Assembler::LoadDoubleConstant(XmmRegister dst, double value) { 1762 // TODO: Need to have a code constants table. 1763 int64_t constant = bit_cast<int64_t, double>(value); 1764 pushq(Immediate(High32Bits(constant))); 1765 pushq(Immediate(Low32Bits(constant))); 1766 movsd(dst, Address(CpuRegister(RSP), 0)); 1767 addq(CpuRegister(RSP), Immediate(2 * sizeof(intptr_t))); 1768} 1769 1770 1771void X86_64Assembler::FloatNegate(XmmRegister f) { 1772 static const struct { 1773 uint32_t a; 1774 uint32_t b; 1775 uint32_t c; 1776 uint32_t d; 1777 } float_negate_constant __attribute__((aligned(16))) = 1778 { 0x80000000, 0x00000000, 0x80000000, 0x00000000 }; 1779 xorps(f, Address::Absolute(reinterpret_cast<uintptr_t>(&float_negate_constant))); 1780} 1781 1782 1783void X86_64Assembler::DoubleNegate(XmmRegister d) { 1784 static const struct { 1785 uint64_t a; 1786 uint64_t b; 1787 } double_negate_constant __attribute__((aligned(16))) = 1788 {0x8000000000000000LL, 0x8000000000000000LL}; 1789 xorpd(d, Address::Absolute(reinterpret_cast<uintptr_t>(&double_negate_constant))); 1790} 1791 1792 1793void X86_64Assembler::DoubleAbs(XmmRegister reg) { 1794 static const struct { 1795 uint64_t a; 1796 uint64_t b; 1797 } double_abs_constant __attribute__((aligned(16))) = 1798 {0x7FFFFFFFFFFFFFFFLL, 0x7FFFFFFFFFFFFFFFLL}; 1799 andpd(reg, Address::Absolute(reinterpret_cast<uintptr_t>(&double_abs_constant))); 1800} 1801 1802 1803void X86_64Assembler::Align(int alignment, int offset) { 1804 CHECK(IsPowerOfTwo(alignment)); 1805 // Emit nop instruction until the real position is aligned. 1806 while (((offset + buffer_.GetPosition()) & (alignment-1)) != 0) { 1807 nop(); 1808 } 1809} 1810 1811 1812void X86_64Assembler::Bind(Label* label) { 1813 int bound = buffer_.Size(); 1814 CHECK(!label->IsBound()); // Labels can only be bound once. 1815 while (label->IsLinked()) { 1816 int position = label->LinkPosition(); 1817 int next = buffer_.Load<int32_t>(position); 1818 buffer_.Store<int32_t>(position, bound - (position + 4)); 1819 label->position_ = next; 1820 } 1821 label->BindTo(bound); 1822} 1823 1824 1825void X86_64Assembler::EmitOperand(uint8_t reg_or_opcode, const Operand& operand) { 1826 CHECK_GE(reg_or_opcode, 0); 1827 CHECK_LT(reg_or_opcode, 8); 1828 const int length = operand.length_; 1829 CHECK_GT(length, 0); 1830 // Emit the ModRM byte updated with the given reg value. 1831 CHECK_EQ(operand.encoding_[0] & 0x38, 0); 1832 EmitUint8(operand.encoding_[0] + (reg_or_opcode << 3)); 1833 // Emit the rest of the encoded operand. 1834 for (int i = 1; i < length; i++) { 1835 EmitUint8(operand.encoding_[i]); 1836 } 1837} 1838 1839 1840void X86_64Assembler::EmitImmediate(const Immediate& imm) { 1841 if (imm.is_int32()) { 1842 EmitInt32(static_cast<int32_t>(imm.value())); 1843 } else { 1844 EmitInt64(imm.value()); 1845 } 1846} 1847 1848 1849void X86_64Assembler::EmitComplex(uint8_t reg_or_opcode, 1850 const Operand& operand, 1851 const Immediate& immediate) { 1852 CHECK_GE(reg_or_opcode, 0); 1853 CHECK_LT(reg_or_opcode, 8); 1854 if (immediate.is_int8()) { 1855 // Use sign-extended 8-bit immediate. 1856 EmitUint8(0x83); 1857 EmitOperand(reg_or_opcode, operand); 1858 EmitUint8(immediate.value() & 0xFF); 1859 } else if (operand.IsRegister(CpuRegister(RAX))) { 1860 // Use short form if the destination is eax. 1861 EmitUint8(0x05 + (reg_or_opcode << 3)); 1862 EmitImmediate(immediate); 1863 } else { 1864 EmitUint8(0x81); 1865 EmitOperand(reg_or_opcode, operand); 1866 EmitImmediate(immediate); 1867 } 1868} 1869 1870 1871void X86_64Assembler::EmitLabel(Label* label, int instruction_size) { 1872 if (label->IsBound()) { 1873 int offset = label->Position() - buffer_.Size(); 1874 CHECK_LE(offset, 0); 1875 EmitInt32(offset - instruction_size); 1876 } else { 1877 EmitLabelLink(label); 1878 } 1879} 1880 1881 1882void X86_64Assembler::EmitLabelLink(Label* label) { 1883 CHECK(!label->IsBound()); 1884 int position = buffer_.Size(); 1885 EmitInt32(label->position_); 1886 label->LinkTo(position); 1887} 1888 1889 1890void X86_64Assembler::EmitGenericShift(bool wide, 1891 int reg_or_opcode, 1892 CpuRegister reg, 1893 const Immediate& imm) { 1894 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1895 CHECK(imm.is_int8()); 1896 if (wide) { 1897 EmitRex64(reg); 1898 } else { 1899 EmitOptionalRex32(reg); 1900 } 1901 if (imm.value() == 1) { 1902 EmitUint8(0xD1); 1903 EmitOperand(reg_or_opcode, Operand(reg)); 1904 } else { 1905 EmitUint8(0xC1); 1906 EmitOperand(reg_or_opcode, Operand(reg)); 1907 EmitUint8(imm.value() & 0xFF); 1908 } 1909} 1910 1911 1912void X86_64Assembler::EmitGenericShift(bool wide, 1913 int reg_or_opcode, 1914 CpuRegister operand, 1915 CpuRegister shifter) { 1916 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1917 CHECK_EQ(shifter.AsRegister(), RCX); 1918 if (wide) { 1919 EmitRex64(operand); 1920 } else { 1921 EmitOptionalRex32(operand); 1922 } 1923 EmitUint8(0xD3); 1924 EmitOperand(reg_or_opcode, Operand(operand)); 1925} 1926 1927void X86_64Assembler::EmitOptionalRex(bool force, bool w, bool r, bool x, bool b) { 1928 // REX.WRXB 1929 // W - 64-bit operand 1930 // R - MODRM.reg 1931 // X - SIB.index 1932 // B - MODRM.rm/SIB.base 1933 uint8_t rex = force ? 0x40 : 0; 1934 if (w) { 1935 rex |= 0x48; // REX.W000 1936 } 1937 if (r) { 1938 rex |= 0x44; // REX.0R00 1939 } 1940 if (x) { 1941 rex |= 0x42; // REX.00X0 1942 } 1943 if (b) { 1944 rex |= 0x41; // REX.000B 1945 } 1946 if (rex != 0) { 1947 EmitUint8(rex); 1948 } 1949} 1950 1951void X86_64Assembler::EmitOptionalRex32(CpuRegister reg) { 1952 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 1953} 1954 1955void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, CpuRegister src) { 1956 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1957} 1958 1959void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, XmmRegister src) { 1960 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1961} 1962 1963void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, XmmRegister src) { 1964 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1965} 1966 1967void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, CpuRegister src) { 1968 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1969} 1970 1971void X86_64Assembler::EmitOptionalRex32(const Operand& operand) { 1972 uint8_t rex = operand.rex(); 1973 if (rex != 0) { 1974 EmitUint8(rex); 1975 } 1976} 1977 1978void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, const Operand& operand) { 1979 uint8_t rex = operand.rex(); 1980 if (dst.NeedsRex()) { 1981 rex |= 0x44; // REX.0R00 1982 } 1983 if (rex != 0) { 1984 EmitUint8(rex); 1985 } 1986} 1987 1988void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, const Operand& operand) { 1989 uint8_t rex = operand.rex(); 1990 if (dst.NeedsRex()) { 1991 rex |= 0x44; // REX.0R00 1992 } 1993 if (rex != 0) { 1994 EmitUint8(rex); 1995 } 1996} 1997 1998void X86_64Assembler::EmitRex64() { 1999 EmitOptionalRex(false, true, false, false, false); 2000} 2001 2002void X86_64Assembler::EmitRex64(CpuRegister reg) { 2003 EmitOptionalRex(false, true, false, false, reg.NeedsRex()); 2004} 2005 2006void X86_64Assembler::EmitRex64(const Operand& operand) { 2007 uint8_t rex = operand.rex(); 2008 rex |= 0x48; // REX.W000 2009 EmitUint8(rex); 2010} 2011 2012void X86_64Assembler::EmitRex64(CpuRegister dst, CpuRegister src) { 2013 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2014} 2015 2016void X86_64Assembler::EmitRex64(XmmRegister dst, CpuRegister src) { 2017 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2018} 2019 2020void X86_64Assembler::EmitRex64(CpuRegister dst, XmmRegister src) { 2021 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2022} 2023 2024void X86_64Assembler::EmitRex64(CpuRegister dst, const Operand& operand) { 2025 uint8_t rex = 0x48 | operand.rex(); // REX.W000 2026 if (dst.NeedsRex()) { 2027 rex |= 0x44; // REX.0R00 2028 } 2029 if (rex != 0) { 2030 EmitUint8(rex); 2031 } 2032} 2033 2034void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, CpuRegister src) { 2035 EmitOptionalRex(true, false, dst.NeedsRex(), false, src.NeedsRex()); 2036} 2037 2038void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand) { 2039 uint8_t rex = 0x40 | operand.rex(); // REX.0000 2040 if (dst.NeedsRex()) { 2041 rex |= 0x44; // REX.0R00 2042 } 2043 if (rex != 0) { 2044 EmitUint8(rex); 2045 } 2046} 2047 2048void X86_64Assembler::InitializeFrameDescriptionEntry() { 2049 WriteFDEHeader(&cfi_info_, true /* is_64bit */); 2050} 2051 2052void X86_64Assembler::FinalizeFrameDescriptionEntry() { 2053 WriteFDEAddressRange(&cfi_info_, buffer_.Size(), true /* is_64bit */); 2054 PadCFI(&cfi_info_); 2055 WriteCFILength(&cfi_info_, true /* is_64bit */); 2056} 2057 2058constexpr size_t kFramePointerSize = 8; 2059 2060void X86_64Assembler::BuildFrame(size_t frame_size, ManagedRegister method_reg, 2061 const std::vector<ManagedRegister>& spill_regs, 2062 const ManagedRegisterEntrySpills& entry_spills) { 2063 cfi_cfa_offset_ = kFramePointerSize; // Only return address on stack 2064 cfi_pc_ = buffer_.Size(); // Nothing emitted yet 2065 DCHECK_EQ(cfi_pc_, 0U); 2066 2067 uint32_t reg_offset = 1; 2068 CHECK_ALIGNED(frame_size, kStackAlignment); 2069 int gpr_count = 0; 2070 for (int i = spill_regs.size() - 1; i >= 0; --i) { 2071 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2072 if (spill.IsCpuRegister()) { 2073 pushq(spill.AsCpuRegister()); 2074 gpr_count++; 2075 2076 // DW_CFA_advance_loc 2077 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 2078 cfi_pc_ = buffer_.Size(); 2079 // DW_CFA_def_cfa_offset 2080 cfi_cfa_offset_ += kFramePointerSize; 2081 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 2082 // DW_CFA_offset reg offset 2083 reg_offset++; 2084 DW_CFA_offset(&cfi_info_, spill.DWARFRegId(), reg_offset); 2085 } 2086 } 2087 // return address then method on stack 2088 int64_t rest_of_frame = static_cast<int64_t>(frame_size) 2089 - (gpr_count * kFramePointerSize) 2090 - kFramePointerSize /*return address*/; 2091 subq(CpuRegister(RSP), Immediate(rest_of_frame)); 2092 // DW_CFA_advance_loc 2093 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 2094 cfi_pc_ = buffer_.Size(); 2095 // DW_CFA_def_cfa_offset 2096 cfi_cfa_offset_ += rest_of_frame; 2097 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 2098 2099 // spill xmms 2100 int64_t offset = rest_of_frame; 2101 for (int i = spill_regs.size() - 1; i >= 0; --i) { 2102 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2103 if (spill.IsXmmRegister()) { 2104 offset -= sizeof(double); 2105 movsd(Address(CpuRegister(RSP), offset), spill.AsXmmRegister()); 2106 } 2107 } 2108 2109 DCHECK_EQ(4U, sizeof(StackReference<mirror::ArtMethod>)); 2110 2111 movl(Address(CpuRegister(RSP), 0), method_reg.AsX86_64().AsCpuRegister()); 2112 2113 for (size_t i = 0; i < entry_spills.size(); ++i) { 2114 ManagedRegisterSpill spill = entry_spills.at(i); 2115 if (spill.AsX86_64().IsCpuRegister()) { 2116 if (spill.getSize() == 8) { 2117 movq(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), 2118 spill.AsX86_64().AsCpuRegister()); 2119 } else { 2120 CHECK_EQ(spill.getSize(), 4); 2121 movl(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsCpuRegister()); 2122 } 2123 } else { 2124 if (spill.getSize() == 8) { 2125 movsd(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 2126 } else { 2127 CHECK_EQ(spill.getSize(), 4); 2128 movss(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 2129 } 2130 } 2131 } 2132} 2133 2134void X86_64Assembler::RemoveFrame(size_t frame_size, 2135 const std::vector<ManagedRegister>& spill_regs) { 2136 CHECK_ALIGNED(frame_size, kStackAlignment); 2137 int gpr_count = 0; 2138 // unspill xmms 2139 int64_t offset = static_cast<int64_t>(frame_size) - (spill_regs.size() * kFramePointerSize) - 2 * kFramePointerSize; 2140 for (size_t i = 0; i < spill_regs.size(); ++i) { 2141 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2142 if (spill.IsXmmRegister()) { 2143 offset += sizeof(double); 2144 movsd(spill.AsXmmRegister(), Address(CpuRegister(RSP), offset)); 2145 } else { 2146 gpr_count++; 2147 } 2148 } 2149 addq(CpuRegister(RSP), Immediate(static_cast<int64_t>(frame_size) - (gpr_count * kFramePointerSize) - kFramePointerSize)); 2150 for (size_t i = 0; i < spill_regs.size(); ++i) { 2151 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2152 if (spill.IsCpuRegister()) { 2153 popq(spill.AsCpuRegister()); 2154 } 2155 } 2156 ret(); 2157} 2158 2159void X86_64Assembler::IncreaseFrameSize(size_t adjust) { 2160 CHECK_ALIGNED(adjust, kStackAlignment); 2161 addq(CpuRegister(RSP), Immediate(-static_cast<int64_t>(adjust))); 2162 // DW_CFA_advance_loc 2163 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 2164 cfi_pc_ = buffer_.Size(); 2165 // DW_CFA_def_cfa_offset 2166 cfi_cfa_offset_ += adjust; 2167 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 2168} 2169 2170void X86_64Assembler::DecreaseFrameSize(size_t adjust) { 2171 CHECK_ALIGNED(adjust, kStackAlignment); 2172 addq(CpuRegister(RSP), Immediate(adjust)); 2173} 2174 2175void X86_64Assembler::Store(FrameOffset offs, ManagedRegister msrc, size_t size) { 2176 X86_64ManagedRegister src = msrc.AsX86_64(); 2177 if (src.IsNoRegister()) { 2178 CHECK_EQ(0u, size); 2179 } else if (src.IsCpuRegister()) { 2180 if (size == 4) { 2181 CHECK_EQ(4u, size); 2182 movl(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 2183 } else { 2184 CHECK_EQ(8u, size); 2185 movq(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 2186 } 2187 } else if (src.IsRegisterPair()) { 2188 CHECK_EQ(0u, size); 2189 movq(Address(CpuRegister(RSP), offs), src.AsRegisterPairLow()); 2190 movq(Address(CpuRegister(RSP), FrameOffset(offs.Int32Value()+4)), 2191 src.AsRegisterPairHigh()); 2192 } else if (src.IsX87Register()) { 2193 if (size == 4) { 2194 fstps(Address(CpuRegister(RSP), offs)); 2195 } else { 2196 fstpl(Address(CpuRegister(RSP), offs)); 2197 } 2198 } else { 2199 CHECK(src.IsXmmRegister()); 2200 if (size == 4) { 2201 movss(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 2202 } else { 2203 movsd(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 2204 } 2205 } 2206} 2207 2208void X86_64Assembler::StoreRef(FrameOffset dest, ManagedRegister msrc) { 2209 X86_64ManagedRegister src = msrc.AsX86_64(); 2210 CHECK(src.IsCpuRegister()); 2211 movl(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 2212} 2213 2214void X86_64Assembler::StoreRawPtr(FrameOffset dest, ManagedRegister msrc) { 2215 X86_64ManagedRegister src = msrc.AsX86_64(); 2216 CHECK(src.IsCpuRegister()); 2217 movq(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 2218} 2219 2220void X86_64Assembler::StoreImmediateToFrame(FrameOffset dest, uint32_t imm, 2221 ManagedRegister) { 2222 movl(Address(CpuRegister(RSP), dest), Immediate(imm)); // TODO(64) movq? 2223} 2224 2225void X86_64Assembler::StoreImmediateToThread64(ThreadOffset<8> dest, uint32_t imm, 2226 ManagedRegister) { 2227 gs()->movl(Address::Absolute(dest, true), Immediate(imm)); // TODO(64) movq? 2228} 2229 2230void X86_64Assembler::StoreStackOffsetToThread64(ThreadOffset<8> thr_offs, 2231 FrameOffset fr_offs, 2232 ManagedRegister mscratch) { 2233 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2234 CHECK(scratch.IsCpuRegister()); 2235 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), fr_offs)); 2236 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2237} 2238 2239void X86_64Assembler::StoreStackPointerToThread64(ThreadOffset<8> thr_offs) { 2240 gs()->movq(Address::Absolute(thr_offs, true), CpuRegister(RSP)); 2241} 2242 2243void X86_64Assembler::StoreSpanning(FrameOffset /*dst*/, ManagedRegister /*src*/, 2244 FrameOffset /*in_off*/, ManagedRegister /*scratch*/) { 2245 UNIMPLEMENTED(FATAL); // this case only currently exists for ARM 2246} 2247 2248void X86_64Assembler::Load(ManagedRegister mdest, FrameOffset src, size_t size) { 2249 X86_64ManagedRegister dest = mdest.AsX86_64(); 2250 if (dest.IsNoRegister()) { 2251 CHECK_EQ(0u, size); 2252 } else if (dest.IsCpuRegister()) { 2253 if (size == 4) { 2254 CHECK_EQ(4u, size); 2255 movl(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2256 } else { 2257 CHECK_EQ(8u, size); 2258 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2259 } 2260 } else if (dest.IsRegisterPair()) { 2261 CHECK_EQ(0u, size); 2262 movq(dest.AsRegisterPairLow(), Address(CpuRegister(RSP), src)); 2263 movq(dest.AsRegisterPairHigh(), Address(CpuRegister(RSP), FrameOffset(src.Int32Value()+4))); 2264 } else if (dest.IsX87Register()) { 2265 if (size == 4) { 2266 flds(Address(CpuRegister(RSP), src)); 2267 } else { 2268 fldl(Address(CpuRegister(RSP), src)); 2269 } 2270 } else { 2271 CHECK(dest.IsXmmRegister()); 2272 if (size == 4) { 2273 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2274 } else { 2275 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2276 } 2277 } 2278} 2279 2280void X86_64Assembler::LoadFromThread64(ManagedRegister mdest, ThreadOffset<8> src, size_t size) { 2281 X86_64ManagedRegister dest = mdest.AsX86_64(); 2282 if (dest.IsNoRegister()) { 2283 CHECK_EQ(0u, size); 2284 } else if (dest.IsCpuRegister()) { 2285 CHECK_EQ(4u, size); 2286 gs()->movl(dest.AsCpuRegister(), Address::Absolute(src, true)); 2287 } else if (dest.IsRegisterPair()) { 2288 CHECK_EQ(8u, size); 2289 gs()->movq(dest.AsRegisterPairLow(), Address::Absolute(src, true)); 2290 } else if (dest.IsX87Register()) { 2291 if (size == 4) { 2292 gs()->flds(Address::Absolute(src, true)); 2293 } else { 2294 gs()->fldl(Address::Absolute(src, true)); 2295 } 2296 } else { 2297 CHECK(dest.IsXmmRegister()); 2298 if (size == 4) { 2299 gs()->movss(dest.AsXmmRegister(), Address::Absolute(src, true)); 2300 } else { 2301 gs()->movsd(dest.AsXmmRegister(), Address::Absolute(src, true)); 2302 } 2303 } 2304} 2305 2306void X86_64Assembler::LoadRef(ManagedRegister mdest, FrameOffset src) { 2307 X86_64ManagedRegister dest = mdest.AsX86_64(); 2308 CHECK(dest.IsCpuRegister()); 2309 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2310} 2311 2312void X86_64Assembler::LoadRef(ManagedRegister mdest, ManagedRegister base, 2313 MemberOffset offs) { 2314 X86_64ManagedRegister dest = mdest.AsX86_64(); 2315 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2316 movl(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2317 if (kPoisonHeapReferences) { 2318 negl(dest.AsCpuRegister()); 2319 } 2320} 2321 2322void X86_64Assembler::LoadRawPtr(ManagedRegister mdest, ManagedRegister base, 2323 Offset offs) { 2324 X86_64ManagedRegister dest = mdest.AsX86_64(); 2325 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2326 movq(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2327} 2328 2329void X86_64Assembler::LoadRawPtrFromThread64(ManagedRegister mdest, ThreadOffset<8> offs) { 2330 X86_64ManagedRegister dest = mdest.AsX86_64(); 2331 CHECK(dest.IsCpuRegister()); 2332 gs()->movq(dest.AsCpuRegister(), Address::Absolute(offs, true)); 2333} 2334 2335void X86_64Assembler::SignExtend(ManagedRegister mreg, size_t size) { 2336 X86_64ManagedRegister reg = mreg.AsX86_64(); 2337 CHECK(size == 1 || size == 2) << size; 2338 CHECK(reg.IsCpuRegister()) << reg; 2339 if (size == 1) { 2340 movsxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2341 } else { 2342 movsxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2343 } 2344} 2345 2346void X86_64Assembler::ZeroExtend(ManagedRegister mreg, size_t size) { 2347 X86_64ManagedRegister reg = mreg.AsX86_64(); 2348 CHECK(size == 1 || size == 2) << size; 2349 CHECK(reg.IsCpuRegister()) << reg; 2350 if (size == 1) { 2351 movzxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2352 } else { 2353 movzxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2354 } 2355} 2356 2357void X86_64Assembler::Move(ManagedRegister mdest, ManagedRegister msrc, size_t size) { 2358 X86_64ManagedRegister dest = mdest.AsX86_64(); 2359 X86_64ManagedRegister src = msrc.AsX86_64(); 2360 if (!dest.Equals(src)) { 2361 if (dest.IsCpuRegister() && src.IsCpuRegister()) { 2362 movq(dest.AsCpuRegister(), src.AsCpuRegister()); 2363 } else if (src.IsX87Register() && dest.IsXmmRegister()) { 2364 // Pass via stack and pop X87 register 2365 subl(CpuRegister(RSP), Immediate(16)); 2366 if (size == 4) { 2367 CHECK_EQ(src.AsX87Register(), ST0); 2368 fstps(Address(CpuRegister(RSP), 0)); 2369 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2370 } else { 2371 CHECK_EQ(src.AsX87Register(), ST0); 2372 fstpl(Address(CpuRegister(RSP), 0)); 2373 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2374 } 2375 addq(CpuRegister(RSP), Immediate(16)); 2376 } else { 2377 // TODO: x87, SSE 2378 UNIMPLEMENTED(FATAL) << ": Move " << dest << ", " << src; 2379 } 2380 } 2381} 2382 2383void X86_64Assembler::CopyRef(FrameOffset dest, FrameOffset src, 2384 ManagedRegister mscratch) { 2385 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2386 CHECK(scratch.IsCpuRegister()); 2387 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2388 movl(Address(CpuRegister(RSP), dest), scratch.AsCpuRegister()); 2389} 2390 2391void X86_64Assembler::CopyRawPtrFromThread64(FrameOffset fr_offs, 2392 ThreadOffset<8> thr_offs, 2393 ManagedRegister mscratch) { 2394 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2395 CHECK(scratch.IsCpuRegister()); 2396 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(thr_offs, true)); 2397 Store(fr_offs, scratch, 8); 2398} 2399 2400void X86_64Assembler::CopyRawPtrToThread64(ThreadOffset<8> thr_offs, 2401 FrameOffset fr_offs, 2402 ManagedRegister mscratch) { 2403 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2404 CHECK(scratch.IsCpuRegister()); 2405 Load(scratch, fr_offs, 8); 2406 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2407} 2408 2409void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src, 2410 ManagedRegister mscratch, 2411 size_t size) { 2412 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2413 if (scratch.IsCpuRegister() && size == 8) { 2414 Load(scratch, src, 4); 2415 Store(dest, scratch, 4); 2416 Load(scratch, FrameOffset(src.Int32Value() + 4), 4); 2417 Store(FrameOffset(dest.Int32Value() + 4), scratch, 4); 2418 } else { 2419 Load(scratch, src, size); 2420 Store(dest, scratch, size); 2421 } 2422} 2423 2424void X86_64Assembler::Copy(FrameOffset /*dst*/, ManagedRegister /*src_base*/, Offset /*src_offset*/, 2425 ManagedRegister /*scratch*/, size_t /*size*/) { 2426 UNIMPLEMENTED(FATAL); 2427} 2428 2429void X86_64Assembler::Copy(ManagedRegister dest_base, Offset dest_offset, FrameOffset src, 2430 ManagedRegister scratch, size_t size) { 2431 CHECK(scratch.IsNoRegister()); 2432 CHECK_EQ(size, 4u); 2433 pushq(Address(CpuRegister(RSP), src)); 2434 popq(Address(dest_base.AsX86_64().AsCpuRegister(), dest_offset)); 2435} 2436 2437void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src_base, Offset src_offset, 2438 ManagedRegister mscratch, size_t size) { 2439 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2440 CHECK_EQ(size, 4u); 2441 movq(scratch, Address(CpuRegister(RSP), src_base)); 2442 movq(scratch, Address(scratch, src_offset)); 2443 movq(Address(CpuRegister(RSP), dest), scratch); 2444} 2445 2446void X86_64Assembler::Copy(ManagedRegister dest, Offset dest_offset, 2447 ManagedRegister src, Offset src_offset, 2448 ManagedRegister scratch, size_t size) { 2449 CHECK_EQ(size, 4u); 2450 CHECK(scratch.IsNoRegister()); 2451 pushq(Address(src.AsX86_64().AsCpuRegister(), src_offset)); 2452 popq(Address(dest.AsX86_64().AsCpuRegister(), dest_offset)); 2453} 2454 2455void X86_64Assembler::Copy(FrameOffset dest, Offset dest_offset, FrameOffset src, Offset src_offset, 2456 ManagedRegister mscratch, size_t size) { 2457 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2458 CHECK_EQ(size, 4u); 2459 CHECK_EQ(dest.Int32Value(), src.Int32Value()); 2460 movq(scratch, Address(CpuRegister(RSP), src)); 2461 pushq(Address(scratch, src_offset)); 2462 popq(Address(scratch, dest_offset)); 2463} 2464 2465void X86_64Assembler::MemoryBarrier(ManagedRegister) { 2466 mfence(); 2467} 2468 2469void X86_64Assembler::CreateHandleScopeEntry(ManagedRegister mout_reg, 2470 FrameOffset handle_scope_offset, 2471 ManagedRegister min_reg, bool null_allowed) { 2472 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2473 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2474 if (in_reg.IsNoRegister()) { // TODO(64): && null_allowed 2475 // Use out_reg as indicator of NULL 2476 in_reg = out_reg; 2477 // TODO: movzwl 2478 movl(in_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2479 } 2480 CHECK(in_reg.IsCpuRegister()); 2481 CHECK(out_reg.IsCpuRegister()); 2482 VerifyObject(in_reg, null_allowed); 2483 if (null_allowed) { 2484 Label null_arg; 2485 if (!out_reg.Equals(in_reg)) { 2486 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2487 } 2488 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2489 j(kZero, &null_arg); 2490 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2491 Bind(&null_arg); 2492 } else { 2493 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2494 } 2495} 2496 2497void X86_64Assembler::CreateHandleScopeEntry(FrameOffset out_off, 2498 FrameOffset handle_scope_offset, 2499 ManagedRegister mscratch, 2500 bool null_allowed) { 2501 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2502 CHECK(scratch.IsCpuRegister()); 2503 if (null_allowed) { 2504 Label null_arg; 2505 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2506 testl(scratch.AsCpuRegister(), scratch.AsCpuRegister()); 2507 j(kZero, &null_arg); 2508 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2509 Bind(&null_arg); 2510 } else { 2511 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2512 } 2513 Store(out_off, scratch, 8); 2514} 2515 2516// Given a handle scope entry, load the associated reference. 2517void X86_64Assembler::LoadReferenceFromHandleScope(ManagedRegister mout_reg, 2518 ManagedRegister min_reg) { 2519 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2520 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2521 CHECK(out_reg.IsCpuRegister()); 2522 CHECK(in_reg.IsCpuRegister()); 2523 Label null_arg; 2524 if (!out_reg.Equals(in_reg)) { 2525 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2526 } 2527 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2528 j(kZero, &null_arg); 2529 movq(out_reg.AsCpuRegister(), Address(in_reg.AsCpuRegister(), 0)); 2530 Bind(&null_arg); 2531} 2532 2533void X86_64Assembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) { 2534 // TODO: not validating references 2535} 2536 2537void X86_64Assembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) { 2538 // TODO: not validating references 2539} 2540 2541void X86_64Assembler::Call(ManagedRegister mbase, Offset offset, ManagedRegister) { 2542 X86_64ManagedRegister base = mbase.AsX86_64(); 2543 CHECK(base.IsCpuRegister()); 2544 call(Address(base.AsCpuRegister(), offset.Int32Value())); 2545 // TODO: place reference map on call 2546} 2547 2548void X86_64Assembler::Call(FrameOffset base, Offset offset, ManagedRegister mscratch) { 2549 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2550 movl(scratch, Address(CpuRegister(RSP), base)); 2551 call(Address(scratch, offset)); 2552} 2553 2554void X86_64Assembler::CallFromThread64(ThreadOffset<8> offset, ManagedRegister /*mscratch*/) { 2555 gs()->call(Address::Absolute(offset, true)); 2556} 2557 2558void X86_64Assembler::GetCurrentThread(ManagedRegister tr) { 2559 gs()->movq(tr.AsX86_64().AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2560} 2561 2562void X86_64Assembler::GetCurrentThread(FrameOffset offset, ManagedRegister mscratch) { 2563 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2564 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2565 movq(Address(CpuRegister(RSP), offset), scratch.AsCpuRegister()); 2566} 2567 2568// Slowpath entered when Thread::Current()->_exception is non-null 2569class X86_64ExceptionSlowPath FINAL : public SlowPath { 2570 public: 2571 explicit X86_64ExceptionSlowPath(size_t stack_adjust) : stack_adjust_(stack_adjust) {} 2572 virtual void Emit(Assembler *sp_asm) OVERRIDE; 2573 private: 2574 const size_t stack_adjust_; 2575}; 2576 2577void X86_64Assembler::ExceptionPoll(ManagedRegister /*scratch*/, size_t stack_adjust) { 2578 X86_64ExceptionSlowPath* slow = new X86_64ExceptionSlowPath(stack_adjust); 2579 buffer_.EnqueueSlowPath(slow); 2580 gs()->cmpl(Address::Absolute(Thread::ExceptionOffset<8>(), true), Immediate(0)); 2581 j(kNotEqual, slow->Entry()); 2582} 2583 2584void X86_64ExceptionSlowPath::Emit(Assembler *sasm) { 2585 X86_64Assembler* sp_asm = down_cast<X86_64Assembler*>(sasm); 2586#define __ sp_asm-> 2587 __ Bind(&entry_); 2588 // Note: the return value is dead 2589 if (stack_adjust_ != 0) { // Fix up the frame. 2590 __ DecreaseFrameSize(stack_adjust_); 2591 } 2592 // Pass exception as argument in RDI 2593 __ gs()->movq(CpuRegister(RDI), Address::Absolute(Thread::ExceptionOffset<8>(), true)); 2594 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(8, pDeliverException), true)); 2595 // this call should never return 2596 __ int3(); 2597#undef __ 2598} 2599 2600} // namespace x86_64 2601} // namespace art 2602