assembler_x86_64.cc revision 9aec02fc5df5518c16f1e5a9b6cb198a192db973
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "assembler_x86_64.h" 18 19#include "base/casts.h" 20#include "entrypoints/quick/quick_entrypoints.h" 21#include "memory_region.h" 22#include "thread.h" 23#include "utils/dwarf_cfi.h" 24 25namespace art { 26namespace x86_64 { 27 28std::ostream& operator<<(std::ostream& os, const CpuRegister& reg) { 29 return os << reg.AsRegister(); 30} 31 32std::ostream& operator<<(std::ostream& os, const XmmRegister& reg) { 33 return os << reg.AsFloatRegister(); 34} 35 36std::ostream& operator<<(std::ostream& os, const X87Register& reg) { 37 return os << "ST" << static_cast<int>(reg); 38} 39 40void X86_64Assembler::call(CpuRegister reg) { 41 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 42 EmitOptionalRex32(reg); 43 EmitUint8(0xFF); 44 EmitRegisterOperand(2, reg.LowBits()); 45} 46 47 48void X86_64Assembler::call(const Address& address) { 49 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 50 EmitOptionalRex32(address); 51 EmitUint8(0xFF); 52 EmitOperand(2, address); 53} 54 55 56void X86_64Assembler::call(Label* label) { 57 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 58 EmitUint8(0xE8); 59 static const int kSize = 5; 60 EmitLabel(label, kSize); 61} 62 63void X86_64Assembler::pushq(CpuRegister reg) { 64 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 65 EmitOptionalRex32(reg); 66 EmitUint8(0x50 + reg.LowBits()); 67} 68 69 70void X86_64Assembler::pushq(const Address& address) { 71 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 72 EmitOptionalRex32(address); 73 EmitUint8(0xFF); 74 EmitOperand(6, address); 75} 76 77 78void X86_64Assembler::pushq(const Immediate& imm) { 79 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 80 CHECK(imm.is_int32()); // pushq only supports 32b immediate. 81 if (imm.is_int8()) { 82 EmitUint8(0x6A); 83 EmitUint8(imm.value() & 0xFF); 84 } else { 85 EmitUint8(0x68); 86 EmitImmediate(imm); 87 } 88} 89 90 91void X86_64Assembler::popq(CpuRegister reg) { 92 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 93 EmitOptionalRex32(reg); 94 EmitUint8(0x58 + reg.LowBits()); 95} 96 97 98void X86_64Assembler::popq(const Address& address) { 99 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 100 EmitOptionalRex32(address); 101 EmitUint8(0x8F); 102 EmitOperand(0, address); 103} 104 105 106void X86_64Assembler::movq(CpuRegister dst, const Immediate& imm) { 107 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 108 if (imm.is_int32()) { 109 // 32 bit. Note: sign-extends. 110 EmitRex64(dst); 111 EmitUint8(0xC7); 112 EmitRegisterOperand(0, dst.LowBits()); 113 EmitInt32(static_cast<int32_t>(imm.value())); 114 } else { 115 EmitRex64(dst); 116 EmitUint8(0xB8 + dst.LowBits()); 117 EmitInt64(imm.value()); 118 } 119} 120 121 122void X86_64Assembler::movl(CpuRegister dst, const Immediate& imm) { 123 CHECK(imm.is_int32()); 124 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 125 EmitOptionalRex32(dst); 126 EmitUint8(0xB8 + dst.LowBits()); 127 EmitImmediate(imm); 128} 129 130 131void X86_64Assembler::movq(CpuRegister dst, CpuRegister src) { 132 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 133 // 0x89 is movq r/m64 <- r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 134 EmitRex64(src, dst); 135 EmitUint8(0x89); 136 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 137} 138 139 140void X86_64Assembler::movl(CpuRegister dst, CpuRegister src) { 141 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 142 EmitOptionalRex32(dst, src); 143 EmitUint8(0x8B); 144 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 145} 146 147 148void X86_64Assembler::movq(CpuRegister dst, const Address& src) { 149 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 150 EmitRex64(dst, src); 151 EmitUint8(0x8B); 152 EmitOperand(dst.LowBits(), src); 153} 154 155 156void X86_64Assembler::movl(CpuRegister dst, const Address& src) { 157 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 158 EmitOptionalRex32(dst, src); 159 EmitUint8(0x8B); 160 EmitOperand(dst.LowBits(), src); 161} 162 163 164void X86_64Assembler::movq(const Address& dst, CpuRegister src) { 165 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 166 EmitRex64(src, dst); 167 EmitUint8(0x89); 168 EmitOperand(src.LowBits(), dst); 169} 170 171 172void X86_64Assembler::movl(const Address& dst, CpuRegister src) { 173 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 174 EmitOptionalRex32(src, dst); 175 EmitUint8(0x89); 176 EmitOperand(src.LowBits(), dst); 177} 178 179void X86_64Assembler::movl(const Address& dst, const Immediate& imm) { 180 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 181 EmitOptionalRex32(dst); 182 EmitUint8(0xC7); 183 EmitOperand(0, dst); 184 EmitImmediate(imm); 185} 186 187void X86_64Assembler::movzxb(CpuRegister dst, CpuRegister src) { 188 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 189 EmitOptionalByteRegNormalizingRex32(dst, src); 190 EmitUint8(0x0F); 191 EmitUint8(0xB6); 192 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 193} 194 195 196void X86_64Assembler::movzxb(CpuRegister dst, const Address& src) { 197 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 198 EmitOptionalByteRegNormalizingRex32(dst, src); 199 EmitUint8(0x0F); 200 EmitUint8(0xB6); 201 EmitOperand(dst.LowBits(), src); 202} 203 204 205void X86_64Assembler::movsxb(CpuRegister dst, CpuRegister src) { 206 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 207 EmitOptionalByteRegNormalizingRex32(dst, src); 208 EmitUint8(0x0F); 209 EmitUint8(0xBE); 210 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 211} 212 213 214void X86_64Assembler::movsxb(CpuRegister dst, const Address& src) { 215 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 216 EmitOptionalByteRegNormalizingRex32(dst, src); 217 EmitUint8(0x0F); 218 EmitUint8(0xBE); 219 EmitOperand(dst.LowBits(), src); 220} 221 222 223void X86_64Assembler::movb(CpuRegister /*dst*/, const Address& /*src*/) { 224 LOG(FATAL) << "Use movzxb or movsxb instead."; 225} 226 227 228void X86_64Assembler::movb(const Address& dst, CpuRegister src) { 229 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 230 EmitOptionalByteRegNormalizingRex32(src, dst); 231 EmitUint8(0x88); 232 EmitOperand(src.LowBits(), dst); 233} 234 235 236void X86_64Assembler::movb(const Address& dst, const Immediate& imm) { 237 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 238 EmitOptionalRex32(dst); 239 EmitUint8(0xC6); 240 EmitOperand(Register::RAX, dst); 241 CHECK(imm.is_int8()); 242 EmitUint8(imm.value() & 0xFF); 243} 244 245 246void X86_64Assembler::movzxw(CpuRegister dst, CpuRegister src) { 247 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 248 EmitOptionalRex32(dst, src); 249 EmitUint8(0x0F); 250 EmitUint8(0xB7); 251 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 252} 253 254 255void X86_64Assembler::movzxw(CpuRegister dst, const Address& src) { 256 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 257 EmitOptionalRex32(dst, src); 258 EmitUint8(0x0F); 259 EmitUint8(0xB7); 260 EmitOperand(dst.LowBits(), src); 261} 262 263 264void X86_64Assembler::movsxw(CpuRegister dst, CpuRegister src) { 265 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 266 EmitOptionalRex32(dst, src); 267 EmitUint8(0x0F); 268 EmitUint8(0xBF); 269 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 270} 271 272 273void X86_64Assembler::movsxw(CpuRegister dst, const Address& src) { 274 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 275 EmitOptionalRex32(dst, src); 276 EmitUint8(0x0F); 277 EmitUint8(0xBF); 278 EmitOperand(dst.LowBits(), src); 279} 280 281 282void X86_64Assembler::movw(CpuRegister /*dst*/, const Address& /*src*/) { 283 LOG(FATAL) << "Use movzxw or movsxw instead."; 284} 285 286 287void X86_64Assembler::movw(const Address& dst, CpuRegister src) { 288 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 289 EmitOperandSizeOverride(); 290 EmitOptionalRex32(src, dst); 291 EmitUint8(0x89); 292 EmitOperand(src.LowBits(), dst); 293} 294 295 296void X86_64Assembler::movw(const Address& dst, const Immediate& imm) { 297 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 298 EmitOperandSizeOverride(); 299 EmitOptionalRex32(dst); 300 EmitUint8(0xC7); 301 EmitOperand(Register::RAX, dst); 302 CHECK(imm.is_uint16() || imm.is_int16()); 303 EmitUint8(imm.value() & 0xFF); 304 EmitUint8(imm.value() >> 8); 305} 306 307 308void X86_64Assembler::leaq(CpuRegister dst, const Address& src) { 309 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 310 EmitRex64(dst, src); 311 EmitUint8(0x8D); 312 EmitOperand(dst.LowBits(), src); 313} 314 315 316void X86_64Assembler::movaps(XmmRegister dst, XmmRegister src) { 317 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 318 EmitOptionalRex32(dst, src); 319 EmitUint8(0x0F); 320 EmitUint8(0x28); 321 EmitXmmRegisterOperand(dst.LowBits(), src); 322} 323 324 325void X86_64Assembler::movss(XmmRegister dst, const Address& src) { 326 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 327 EmitUint8(0xF3); 328 EmitOptionalRex32(dst, src); 329 EmitUint8(0x0F); 330 EmitUint8(0x10); 331 EmitOperand(dst.LowBits(), src); 332} 333 334 335void X86_64Assembler::movss(const Address& dst, XmmRegister src) { 336 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 337 EmitUint8(0xF3); 338 EmitOptionalRex32(src, dst); 339 EmitUint8(0x0F); 340 EmitUint8(0x11); 341 EmitOperand(src.LowBits(), dst); 342} 343 344 345void X86_64Assembler::movss(XmmRegister dst, XmmRegister src) { 346 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 347 EmitUint8(0xF3); 348 EmitOptionalRex32(src, dst); // Movss is MR encoding instead of the usual RM. 349 EmitUint8(0x0F); 350 EmitUint8(0x11); 351 EmitXmmRegisterOperand(src.LowBits(), dst); 352} 353 354 355void X86_64Assembler::movsxd(CpuRegister dst, CpuRegister src) { 356 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 357 EmitRex64(dst, src); 358 EmitUint8(0x63); 359 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 360} 361 362 363void X86_64Assembler::movsxd(CpuRegister dst, const Address& src) { 364 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 365 EmitRex64(dst); 366 EmitUint8(0x63); 367 EmitOperand(dst.LowBits(), src); 368} 369 370 371void X86_64Assembler::movd(XmmRegister dst, CpuRegister src) { 372 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 373 EmitUint8(0x66); 374 EmitRex64(dst, src); 375 EmitUint8(0x0F); 376 EmitUint8(0x6E); 377 EmitOperand(dst.LowBits(), Operand(src)); 378} 379 380 381void X86_64Assembler::movd(CpuRegister dst, XmmRegister src) { 382 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 383 EmitUint8(0x66); 384 EmitRex64(src, dst); 385 EmitUint8(0x0F); 386 EmitUint8(0x7E); 387 EmitOperand(src.LowBits(), Operand(dst)); 388} 389 390 391void X86_64Assembler::addss(XmmRegister dst, XmmRegister src) { 392 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 393 EmitUint8(0xF3); 394 EmitOptionalRex32(dst, src); 395 EmitUint8(0x0F); 396 EmitUint8(0x58); 397 EmitXmmRegisterOperand(dst.LowBits(), src); 398} 399 400 401void X86_64Assembler::addss(XmmRegister dst, const Address& src) { 402 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 403 EmitUint8(0xF3); 404 EmitOptionalRex32(dst, src); 405 EmitUint8(0x0F); 406 EmitUint8(0x58); 407 EmitOperand(dst.LowBits(), src); 408} 409 410 411void X86_64Assembler::subss(XmmRegister dst, XmmRegister src) { 412 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 413 EmitUint8(0xF3); 414 EmitOptionalRex32(dst, src); 415 EmitUint8(0x0F); 416 EmitUint8(0x5C); 417 EmitXmmRegisterOperand(dst.LowBits(), src); 418} 419 420 421void X86_64Assembler::subss(XmmRegister dst, const Address& src) { 422 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 423 EmitUint8(0xF3); 424 EmitOptionalRex32(dst, src); 425 EmitUint8(0x0F); 426 EmitUint8(0x5C); 427 EmitOperand(dst.LowBits(), src); 428} 429 430 431void X86_64Assembler::mulss(XmmRegister dst, XmmRegister src) { 432 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 433 EmitUint8(0xF3); 434 EmitOptionalRex32(dst, src); 435 EmitUint8(0x0F); 436 EmitUint8(0x59); 437 EmitXmmRegisterOperand(dst.LowBits(), src); 438} 439 440 441void X86_64Assembler::mulss(XmmRegister dst, const Address& src) { 442 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 443 EmitUint8(0xF3); 444 EmitOptionalRex32(dst, src); 445 EmitUint8(0x0F); 446 EmitUint8(0x59); 447 EmitOperand(dst.LowBits(), src); 448} 449 450 451void X86_64Assembler::divss(XmmRegister dst, XmmRegister src) { 452 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 453 EmitUint8(0xF3); 454 EmitOptionalRex32(dst, src); 455 EmitUint8(0x0F); 456 EmitUint8(0x5E); 457 EmitXmmRegisterOperand(dst.LowBits(), src); 458} 459 460 461void X86_64Assembler::divss(XmmRegister dst, const Address& src) { 462 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 463 EmitUint8(0xF3); 464 EmitOptionalRex32(dst, src); 465 EmitUint8(0x0F); 466 EmitUint8(0x5E); 467 EmitOperand(dst.LowBits(), src); 468} 469 470 471void X86_64Assembler::flds(const Address& src) { 472 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 473 EmitUint8(0xD9); 474 EmitOperand(0, src); 475} 476 477 478void X86_64Assembler::fstps(const Address& dst) { 479 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 480 EmitUint8(0xD9); 481 EmitOperand(3, dst); 482} 483 484 485void X86_64Assembler::movsd(XmmRegister dst, const Address& src) { 486 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 487 EmitUint8(0xF2); 488 EmitOptionalRex32(dst, src); 489 EmitUint8(0x0F); 490 EmitUint8(0x10); 491 EmitOperand(dst.LowBits(), src); 492} 493 494 495void X86_64Assembler::movsd(const Address& dst, XmmRegister src) { 496 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 497 EmitUint8(0xF2); 498 EmitOptionalRex32(src, dst); 499 EmitUint8(0x0F); 500 EmitUint8(0x11); 501 EmitOperand(src.LowBits(), dst); 502} 503 504 505void X86_64Assembler::movsd(XmmRegister dst, XmmRegister src) { 506 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 507 EmitUint8(0xF2); 508 EmitOptionalRex32(src, dst); // Movsd is MR encoding instead of the usual RM. 509 EmitUint8(0x0F); 510 EmitUint8(0x11); 511 EmitXmmRegisterOperand(src.LowBits(), dst); 512} 513 514 515void X86_64Assembler::addsd(XmmRegister dst, XmmRegister src) { 516 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 517 EmitUint8(0xF2); 518 EmitOptionalRex32(dst, src); 519 EmitUint8(0x0F); 520 EmitUint8(0x58); 521 EmitXmmRegisterOperand(dst.LowBits(), src); 522} 523 524 525void X86_64Assembler::addsd(XmmRegister dst, const Address& src) { 526 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 527 EmitUint8(0xF2); 528 EmitOptionalRex32(dst, src); 529 EmitUint8(0x0F); 530 EmitUint8(0x58); 531 EmitOperand(dst.LowBits(), src); 532} 533 534 535void X86_64Assembler::subsd(XmmRegister dst, XmmRegister src) { 536 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 537 EmitUint8(0xF2); 538 EmitOptionalRex32(dst, src); 539 EmitUint8(0x0F); 540 EmitUint8(0x5C); 541 EmitXmmRegisterOperand(dst.LowBits(), src); 542} 543 544 545void X86_64Assembler::subsd(XmmRegister dst, const Address& src) { 546 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 547 EmitUint8(0xF2); 548 EmitOptionalRex32(dst, src); 549 EmitUint8(0x0F); 550 EmitUint8(0x5C); 551 EmitOperand(dst.LowBits(), src); 552} 553 554 555void X86_64Assembler::mulsd(XmmRegister dst, XmmRegister src) { 556 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 557 EmitUint8(0xF2); 558 EmitOptionalRex32(dst, src); 559 EmitUint8(0x0F); 560 EmitUint8(0x59); 561 EmitXmmRegisterOperand(dst.LowBits(), src); 562} 563 564 565void X86_64Assembler::mulsd(XmmRegister dst, const Address& src) { 566 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 567 EmitUint8(0xF2); 568 EmitOptionalRex32(dst, src); 569 EmitUint8(0x0F); 570 EmitUint8(0x59); 571 EmitOperand(dst.LowBits(), src); 572} 573 574 575void X86_64Assembler::divsd(XmmRegister dst, XmmRegister src) { 576 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 577 EmitUint8(0xF2); 578 EmitOptionalRex32(dst, src); 579 EmitUint8(0x0F); 580 EmitUint8(0x5E); 581 EmitXmmRegisterOperand(dst.LowBits(), src); 582} 583 584 585void X86_64Assembler::divsd(XmmRegister dst, const Address& src) { 586 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 587 EmitUint8(0xF2); 588 EmitOptionalRex32(dst, src); 589 EmitUint8(0x0F); 590 EmitUint8(0x5E); 591 EmitOperand(dst.LowBits(), src); 592} 593 594 595void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) { 596 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 597 EmitUint8(0xF3); 598 EmitOptionalRex32(dst, src); 599 EmitUint8(0x0F); 600 EmitUint8(0x2A); 601 EmitOperand(dst.LowBits(), Operand(src)); 602} 603 604 605void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src) { 606 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 607 EmitUint8(0xF2); 608 EmitOptionalRex32(dst, src); 609 EmitUint8(0x0F); 610 EmitUint8(0x2A); 611 EmitOperand(dst.LowBits(), Operand(src)); 612} 613 614 615void X86_64Assembler::cvtss2si(CpuRegister dst, XmmRegister src) { 616 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 617 EmitUint8(0xF3); 618 EmitOptionalRex32(dst, src); 619 EmitUint8(0x0F); 620 EmitUint8(0x2D); 621 EmitXmmRegisterOperand(dst.LowBits(), src); 622} 623 624 625void X86_64Assembler::cvtss2sd(XmmRegister dst, XmmRegister src) { 626 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 627 EmitUint8(0xF3); 628 EmitOptionalRex32(dst, src); 629 EmitUint8(0x0F); 630 EmitUint8(0x5A); 631 EmitXmmRegisterOperand(dst.LowBits(), src); 632} 633 634 635void X86_64Assembler::cvtsd2si(CpuRegister dst, XmmRegister src) { 636 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 637 EmitUint8(0xF2); 638 EmitOptionalRex32(dst, src); 639 EmitUint8(0x0F); 640 EmitUint8(0x2D); 641 EmitXmmRegisterOperand(dst.LowBits(), src); 642} 643 644 645void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src) { 646 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 647 EmitUint8(0xF3); 648 EmitOptionalRex32(dst, src); 649 EmitUint8(0x0F); 650 EmitUint8(0x2C); 651 EmitXmmRegisterOperand(dst.LowBits(), src); 652} 653 654 655void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src) { 656 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 657 EmitUint8(0xF2); 658 EmitOptionalRex32(dst, src); 659 EmitUint8(0x0F); 660 EmitUint8(0x2C); 661 EmitXmmRegisterOperand(dst.LowBits(), src); 662} 663 664 665void X86_64Assembler::cvtsd2ss(XmmRegister dst, XmmRegister src) { 666 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 667 EmitUint8(0xF2); 668 EmitOptionalRex32(dst, src); 669 EmitUint8(0x0F); 670 EmitUint8(0x5A); 671 EmitXmmRegisterOperand(dst.LowBits(), src); 672} 673 674 675void X86_64Assembler::cvtdq2pd(XmmRegister dst, XmmRegister src) { 676 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 677 EmitUint8(0xF3); 678 EmitOptionalRex32(dst, src); 679 EmitUint8(0x0F); 680 EmitUint8(0xE6); 681 EmitXmmRegisterOperand(dst.LowBits(), src); 682} 683 684 685void X86_64Assembler::comiss(XmmRegister a, XmmRegister b) { 686 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 687 EmitOptionalRex32(a, b); 688 EmitUint8(0x0F); 689 EmitUint8(0x2F); 690 EmitXmmRegisterOperand(a.LowBits(), b); 691} 692 693 694void X86_64Assembler::comisd(XmmRegister a, XmmRegister b) { 695 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 696 EmitUint8(0x66); 697 EmitOptionalRex32(a, b); 698 EmitUint8(0x0F); 699 EmitUint8(0x2F); 700 EmitXmmRegisterOperand(a.LowBits(), b); 701} 702 703 704void X86_64Assembler::sqrtsd(XmmRegister dst, XmmRegister src) { 705 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 706 EmitUint8(0xF2); 707 EmitOptionalRex32(dst, src); 708 EmitUint8(0x0F); 709 EmitUint8(0x51); 710 EmitXmmRegisterOperand(dst.LowBits(), src); 711} 712 713 714void X86_64Assembler::sqrtss(XmmRegister dst, XmmRegister src) { 715 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 716 EmitUint8(0xF3); 717 EmitOptionalRex32(dst, src); 718 EmitUint8(0x0F); 719 EmitUint8(0x51); 720 EmitXmmRegisterOperand(dst.LowBits(), src); 721} 722 723 724void X86_64Assembler::xorpd(XmmRegister dst, const Address& src) { 725 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 726 EmitUint8(0x66); 727 EmitOptionalRex32(dst, src); 728 EmitUint8(0x0F); 729 EmitUint8(0x57); 730 EmitOperand(dst.LowBits(), src); 731} 732 733 734void X86_64Assembler::xorpd(XmmRegister dst, XmmRegister src) { 735 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 736 EmitUint8(0x66); 737 EmitOptionalRex32(dst, src); 738 EmitUint8(0x0F); 739 EmitUint8(0x57); 740 EmitXmmRegisterOperand(dst.LowBits(), src); 741} 742 743 744void X86_64Assembler::xorps(XmmRegister dst, const Address& src) { 745 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 746 EmitOptionalRex32(dst, src); 747 EmitUint8(0x0F); 748 EmitUint8(0x57); 749 EmitOperand(dst.LowBits(), src); 750} 751 752 753void X86_64Assembler::xorps(XmmRegister dst, XmmRegister src) { 754 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 755 EmitOptionalRex32(dst, src); 756 EmitUint8(0x0F); 757 EmitUint8(0x57); 758 EmitXmmRegisterOperand(dst.LowBits(), src); 759} 760 761 762void X86_64Assembler::andpd(XmmRegister dst, const Address& src) { 763 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 764 EmitUint8(0x66); 765 EmitOptionalRex32(dst, src); 766 EmitUint8(0x0F); 767 EmitUint8(0x54); 768 EmitOperand(dst.LowBits(), src); 769} 770 771 772void X86_64Assembler::fldl(const Address& src) { 773 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 774 EmitUint8(0xDD); 775 EmitOperand(0, src); 776} 777 778 779void X86_64Assembler::fstpl(const Address& dst) { 780 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 781 EmitUint8(0xDD); 782 EmitOperand(3, dst); 783} 784 785 786void X86_64Assembler::fnstcw(const Address& dst) { 787 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 788 EmitUint8(0xD9); 789 EmitOperand(7, dst); 790} 791 792 793void X86_64Assembler::fldcw(const Address& src) { 794 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 795 EmitUint8(0xD9); 796 EmitOperand(5, src); 797} 798 799 800void X86_64Assembler::fistpl(const Address& dst) { 801 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 802 EmitUint8(0xDF); 803 EmitOperand(7, dst); 804} 805 806 807void X86_64Assembler::fistps(const Address& dst) { 808 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 809 EmitUint8(0xDB); 810 EmitOperand(3, dst); 811} 812 813 814void X86_64Assembler::fildl(const Address& src) { 815 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 816 EmitUint8(0xDF); 817 EmitOperand(5, src); 818} 819 820 821void X86_64Assembler::fincstp() { 822 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 823 EmitUint8(0xD9); 824 EmitUint8(0xF7); 825} 826 827 828void X86_64Assembler::ffree(const Immediate& index) { 829 CHECK_LT(index.value(), 7); 830 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 831 EmitUint8(0xDD); 832 EmitUint8(0xC0 + index.value()); 833} 834 835 836void X86_64Assembler::fsin() { 837 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 838 EmitUint8(0xD9); 839 EmitUint8(0xFE); 840} 841 842 843void X86_64Assembler::fcos() { 844 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 845 EmitUint8(0xD9); 846 EmitUint8(0xFF); 847} 848 849 850void X86_64Assembler::fptan() { 851 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 852 EmitUint8(0xD9); 853 EmitUint8(0xF2); 854} 855 856 857void X86_64Assembler::xchgl(CpuRegister dst, CpuRegister src) { 858 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 859 // There is a short version for rax. 860 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't 861 // work. 862 const bool src_rax = src.AsRegister() == RAX; 863 const bool dst_rax = dst.AsRegister() == RAX; 864 if (src_rax || dst_rax) { 865 EmitOptionalRex32(src_rax ? dst : src); 866 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits())); 867 return; 868 } 869 870 // General case. 871 EmitOptionalRex32(src, dst); 872 EmitUint8(0x87); 873 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 874} 875 876 877void X86_64Assembler::xchgq(CpuRegister dst, CpuRegister src) { 878 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 879 // There is a short version for rax. 880 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't 881 // work. 882 const bool src_rax = src.AsRegister() == RAX; 883 const bool dst_rax = dst.AsRegister() == RAX; 884 if (src_rax || dst_rax) { 885 // If src == target, emit a nop instead. 886 if (src_rax && dst_rax) { 887 EmitUint8(0x90); 888 } else { 889 EmitRex64(src_rax ? dst : src); 890 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits())); 891 } 892 return; 893 } 894 895 // General case. 896 EmitRex64(src, dst); 897 EmitUint8(0x87); 898 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 899} 900 901 902void X86_64Assembler::xchgl(CpuRegister reg, const Address& address) { 903 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 904 EmitOptionalRex32(reg, address); 905 EmitUint8(0x87); 906 EmitOperand(reg.LowBits(), address); 907} 908 909 910void X86_64Assembler::cmpw(const Address& address, const Immediate& imm) { 911 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 912 EmitOptionalRex32(address); 913 EmitUint8(0x66); 914 EmitComplex(7, address, imm); 915} 916 917 918void X86_64Assembler::cmpl(CpuRegister reg, const Immediate& imm) { 919 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 920 EmitOptionalRex32(reg); 921 EmitComplex(7, Operand(reg), imm); 922} 923 924 925void X86_64Assembler::cmpl(CpuRegister reg0, CpuRegister reg1) { 926 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 927 EmitOptionalRex32(reg0, reg1); 928 EmitUint8(0x3B); 929 EmitOperand(reg0.LowBits(), Operand(reg1)); 930} 931 932 933void X86_64Assembler::cmpl(CpuRegister reg, const Address& address) { 934 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 935 EmitOptionalRex32(reg, address); 936 EmitUint8(0x3B); 937 EmitOperand(reg.LowBits(), address); 938} 939 940 941void X86_64Assembler::cmpl(const Address& address, CpuRegister reg) { 942 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 943 EmitOptionalRex32(reg, address); 944 EmitUint8(0x39); 945 EmitOperand(reg.LowBits(), address); 946} 947 948 949void X86_64Assembler::cmpl(const Address& address, const Immediate& imm) { 950 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 951 EmitOptionalRex32(address); 952 EmitComplex(7, address, imm); 953} 954 955 956void X86_64Assembler::cmpq(CpuRegister reg0, CpuRegister reg1) { 957 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 958 EmitRex64(reg0, reg1); 959 EmitUint8(0x3B); 960 EmitOperand(reg0.LowBits(), Operand(reg1)); 961} 962 963 964void X86_64Assembler::cmpq(CpuRegister reg, const Immediate& imm) { 965 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 966 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 967 EmitRex64(reg); 968 EmitComplex(7, Operand(reg), imm); 969} 970 971 972void X86_64Assembler::cmpq(CpuRegister reg, const Address& address) { 973 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 974 EmitRex64(reg); 975 EmitUint8(0x3B); 976 EmitOperand(reg.LowBits(), address); 977} 978 979 980void X86_64Assembler::cmpq(const Address& address, const Immediate& imm) { 981 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 982 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 983 EmitRex64(address); 984 EmitComplex(7, address, imm); 985} 986 987 988void X86_64Assembler::addl(CpuRegister dst, CpuRegister src) { 989 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 990 EmitOptionalRex32(dst, src); 991 EmitUint8(0x03); 992 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 993} 994 995 996void X86_64Assembler::addl(CpuRegister reg, const Address& address) { 997 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 998 EmitOptionalRex32(reg, address); 999 EmitUint8(0x03); 1000 EmitOperand(reg.LowBits(), address); 1001} 1002 1003 1004void X86_64Assembler::testl(CpuRegister reg1, CpuRegister reg2) { 1005 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1006 EmitOptionalRex32(reg1, reg2); 1007 EmitUint8(0x85); 1008 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 1009} 1010 1011 1012void X86_64Assembler::testl(CpuRegister reg, const Immediate& immediate) { 1013 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1014 // For registers that have a byte variant (RAX, RBX, RCX, and RDX) 1015 // we only test the byte CpuRegister to keep the encoding short. 1016 if (immediate.is_uint8() && reg.AsRegister() < 4) { 1017 // Use zero-extended 8-bit immediate. 1018 if (reg.AsRegister() == RAX) { 1019 EmitUint8(0xA8); 1020 } else { 1021 EmitUint8(0xF6); 1022 EmitUint8(0xC0 + reg.AsRegister()); 1023 } 1024 EmitUint8(immediate.value() & 0xFF); 1025 } else if (reg.AsRegister() == RAX) { 1026 // Use short form if the destination is RAX. 1027 EmitUint8(0xA9); 1028 EmitImmediate(immediate); 1029 } else { 1030 EmitOptionalRex32(reg); 1031 EmitUint8(0xF7); 1032 EmitOperand(0, Operand(reg)); 1033 EmitImmediate(immediate); 1034 } 1035} 1036 1037 1038void X86_64Assembler::testq(CpuRegister reg1, CpuRegister reg2) { 1039 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1040 EmitRex64(reg1, reg2); 1041 EmitUint8(0x85); 1042 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 1043} 1044 1045 1046void X86_64Assembler::testq(CpuRegister reg, const Address& address) { 1047 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1048 EmitRex64(reg); 1049 EmitUint8(0x85); 1050 EmitOperand(reg.LowBits(), address); 1051} 1052 1053 1054void X86_64Assembler::andl(CpuRegister dst, CpuRegister src) { 1055 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1056 EmitOptionalRex32(dst, src); 1057 EmitUint8(0x23); 1058 EmitOperand(dst.LowBits(), Operand(src)); 1059} 1060 1061 1062void X86_64Assembler::andl(CpuRegister reg, const Address& address) { 1063 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1064 EmitOptionalRex32(reg, address); 1065 EmitUint8(0x23); 1066 EmitOperand(reg.LowBits(), address); 1067} 1068 1069 1070void X86_64Assembler::andl(CpuRegister dst, const Immediate& imm) { 1071 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1072 EmitOptionalRex32(dst); 1073 EmitComplex(4, Operand(dst), imm); 1074} 1075 1076 1077void X86_64Assembler::andq(CpuRegister reg, const Immediate& imm) { 1078 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1079 CHECK(imm.is_int32()); // andq only supports 32b immediate. 1080 EmitRex64(reg); 1081 EmitComplex(4, Operand(reg), imm); 1082} 1083 1084 1085void X86_64Assembler::andq(CpuRegister dst, CpuRegister src) { 1086 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1087 EmitRex64(dst, src); 1088 EmitUint8(0x23); 1089 EmitOperand(dst.LowBits(), Operand(src)); 1090} 1091 1092 1093void X86_64Assembler::orl(CpuRegister dst, CpuRegister src) { 1094 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1095 EmitOptionalRex32(dst, src); 1096 EmitUint8(0x0B); 1097 EmitOperand(dst.LowBits(), Operand(src)); 1098} 1099 1100 1101void X86_64Assembler::orl(CpuRegister reg, const Address& address) { 1102 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1103 EmitOptionalRex32(reg, address); 1104 EmitUint8(0x0B); 1105 EmitOperand(reg.LowBits(), address); 1106} 1107 1108 1109void X86_64Assembler::orl(CpuRegister dst, const Immediate& imm) { 1110 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1111 EmitOptionalRex32(dst); 1112 EmitComplex(1, Operand(dst), imm); 1113} 1114 1115 1116void X86_64Assembler::orq(CpuRegister dst, CpuRegister src) { 1117 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1118 EmitRex64(dst, src); 1119 EmitUint8(0x0B); 1120 EmitOperand(dst.LowBits(), Operand(src)); 1121} 1122 1123 1124void X86_64Assembler::xorl(CpuRegister dst, CpuRegister src) { 1125 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1126 EmitOptionalRex32(dst, src); 1127 EmitUint8(0x33); 1128 EmitOperand(dst.LowBits(), Operand(src)); 1129} 1130 1131 1132void X86_64Assembler::xorl(CpuRegister reg, const Address& address) { 1133 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1134 EmitOptionalRex32(reg, address); 1135 EmitUint8(0x33); 1136 EmitOperand(reg.LowBits(), address); 1137} 1138 1139 1140void X86_64Assembler::xorl(CpuRegister dst, const Immediate& imm) { 1141 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1142 EmitOptionalRex32(dst); 1143 EmitComplex(6, Operand(dst), imm); 1144} 1145 1146 1147void X86_64Assembler::xorq(CpuRegister dst, CpuRegister src) { 1148 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1149 EmitRex64(dst, src); 1150 EmitUint8(0x33); 1151 EmitOperand(dst.LowBits(), Operand(src)); 1152} 1153 1154 1155void X86_64Assembler::xorq(CpuRegister dst, const Immediate& imm) { 1156 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1157 CHECK(imm.is_int32()); // xorq only supports 32b immediate. 1158 EmitRex64(dst); 1159 EmitComplex(6, Operand(dst), imm); 1160} 1161 1162#if 0 1163void X86_64Assembler::rex(bool force, bool w, Register* r, Register* x, Register* b) { 1164 // REX.WRXB 1165 // W - 64-bit operand 1166 // R - MODRM.reg 1167 // X - SIB.index 1168 // B - MODRM.rm/SIB.base 1169 uint8_t rex = force ? 0x40 : 0; 1170 if (w) { 1171 rex |= 0x48; // REX.W000 1172 } 1173 if (r != nullptr && *r >= Register::R8 && *r < Register::kNumberOfCpuRegisters) { 1174 rex |= 0x44; // REX.0R00 1175 *r = static_cast<Register>(*r - 8); 1176 } 1177 if (x != nullptr && *x >= Register::R8 && *x < Register::kNumberOfCpuRegisters) { 1178 rex |= 0x42; // REX.00X0 1179 *x = static_cast<Register>(*x - 8); 1180 } 1181 if (b != nullptr && *b >= Register::R8 && *b < Register::kNumberOfCpuRegisters) { 1182 rex |= 0x41; // REX.000B 1183 *b = static_cast<Register>(*b - 8); 1184 } 1185 if (rex != 0) { 1186 EmitUint8(rex); 1187 } 1188} 1189 1190void X86_64Assembler::rex_reg_mem(bool force, bool w, Register* dst, const Address& mem) { 1191 // REX.WRXB 1192 // W - 64-bit operand 1193 // R - MODRM.reg 1194 // X - SIB.index 1195 // B - MODRM.rm/SIB.base 1196 uint8_t rex = mem->rex(); 1197 if (force) { 1198 rex |= 0x40; // REX.0000 1199 } 1200 if (w) { 1201 rex |= 0x48; // REX.W000 1202 } 1203 if (dst != nullptr && *dst >= Register::R8 && *dst < Register::kNumberOfCpuRegisters) { 1204 rex |= 0x44; // REX.0R00 1205 *dst = static_cast<Register>(*dst - 8); 1206 } 1207 if (rex != 0) { 1208 EmitUint8(rex); 1209 } 1210} 1211 1212void rex_mem_reg(bool force, bool w, Address* mem, Register* src); 1213#endif 1214 1215void X86_64Assembler::addl(CpuRegister reg, const Immediate& imm) { 1216 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1217 EmitOptionalRex32(reg); 1218 EmitComplex(0, Operand(reg), imm); 1219} 1220 1221 1222void X86_64Assembler::addq(CpuRegister reg, const Immediate& imm) { 1223 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1224 CHECK(imm.is_int32()); // addq only supports 32b immediate. 1225 EmitRex64(reg); 1226 EmitComplex(0, Operand(reg), imm); 1227} 1228 1229 1230void X86_64Assembler::addq(CpuRegister dst, const Address& address) { 1231 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1232 EmitRex64(dst); 1233 EmitUint8(0x03); 1234 EmitOperand(dst.LowBits(), address); 1235} 1236 1237 1238void X86_64Assembler::addq(CpuRegister dst, CpuRegister src) { 1239 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1240 // 0x01 is addq r/m64 <- r/m64 + r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 1241 EmitRex64(src, dst); 1242 EmitUint8(0x01); 1243 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1244} 1245 1246 1247void X86_64Assembler::addl(const Address& address, CpuRegister reg) { 1248 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1249 EmitOptionalRex32(reg, address); 1250 EmitUint8(0x01); 1251 EmitOperand(reg.LowBits(), address); 1252} 1253 1254 1255void X86_64Assembler::addl(const Address& address, const Immediate& imm) { 1256 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1257 EmitOptionalRex32(address); 1258 EmitComplex(0, address, imm); 1259} 1260 1261 1262void X86_64Assembler::subl(CpuRegister dst, CpuRegister src) { 1263 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1264 EmitOptionalRex32(dst, src); 1265 EmitUint8(0x2B); 1266 EmitOperand(dst.LowBits(), Operand(src)); 1267} 1268 1269 1270void X86_64Assembler::subl(CpuRegister reg, const Immediate& imm) { 1271 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1272 EmitOptionalRex32(reg); 1273 EmitComplex(5, Operand(reg), imm); 1274} 1275 1276 1277void X86_64Assembler::subq(CpuRegister reg, const Immediate& imm) { 1278 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1279 CHECK(imm.is_int32()); // subq only supports 32b immediate. 1280 EmitRex64(reg); 1281 EmitComplex(5, Operand(reg), imm); 1282} 1283 1284 1285void X86_64Assembler::subq(CpuRegister dst, CpuRegister src) { 1286 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1287 EmitRex64(dst, src); 1288 EmitUint8(0x2B); 1289 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1290} 1291 1292 1293void X86_64Assembler::subq(CpuRegister reg, const Address& address) { 1294 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1295 EmitRex64(reg); 1296 EmitUint8(0x2B); 1297 EmitOperand(reg.LowBits() & 7, address); 1298} 1299 1300 1301void X86_64Assembler::subl(CpuRegister reg, const Address& address) { 1302 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1303 EmitOptionalRex32(reg, address); 1304 EmitUint8(0x2B); 1305 EmitOperand(reg.LowBits(), address); 1306} 1307 1308 1309void X86_64Assembler::cdq() { 1310 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1311 EmitUint8(0x99); 1312} 1313 1314 1315void X86_64Assembler::cqo() { 1316 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1317 EmitRex64(); 1318 EmitUint8(0x99); 1319} 1320 1321 1322void X86_64Assembler::idivl(CpuRegister reg) { 1323 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1324 EmitOptionalRex32(reg); 1325 EmitUint8(0xF7); 1326 EmitUint8(0xF8 | reg.LowBits()); 1327} 1328 1329 1330void X86_64Assembler::idivq(CpuRegister reg) { 1331 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1332 EmitRex64(reg); 1333 EmitUint8(0xF7); 1334 EmitUint8(0xF8 | reg.LowBits()); 1335} 1336 1337 1338void X86_64Assembler::imull(CpuRegister dst, CpuRegister src) { 1339 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1340 EmitOptionalRex32(dst, src); 1341 EmitUint8(0x0F); 1342 EmitUint8(0xAF); 1343 EmitOperand(dst.LowBits(), Operand(src)); 1344} 1345 1346void X86_64Assembler::imull(CpuRegister reg, const Immediate& imm) { 1347 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1348 CHECK(imm.is_int32()); // imull only supports 32b immediate. 1349 1350 EmitOptionalRex32(reg, reg); 1351 1352 // See whether imm can be represented as a sign-extended 8bit value. 1353 int32_t v32 = static_cast<int32_t>(imm.value()); 1354 if (IsInt32(8, v32)) { 1355 // Sign-extension works. 1356 EmitUint8(0x6B); 1357 EmitOperand(reg.LowBits(), Operand(reg)); 1358 EmitUint8(static_cast<uint8_t>(v32 & 0xFF)); 1359 } else { 1360 // Not representable, use full immediate. 1361 EmitUint8(0x69); 1362 EmitOperand(reg.LowBits(), Operand(reg)); 1363 EmitImmediate(imm); 1364 } 1365} 1366 1367 1368void X86_64Assembler::imull(CpuRegister reg, const Address& address) { 1369 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1370 EmitOptionalRex32(reg, address); 1371 EmitUint8(0x0F); 1372 EmitUint8(0xAF); 1373 EmitOperand(reg.LowBits(), address); 1374} 1375 1376 1377void X86_64Assembler::imulq(CpuRegister dst, CpuRegister src) { 1378 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1379 EmitRex64(dst, src); 1380 EmitUint8(0x0F); 1381 EmitUint8(0xAF); 1382 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1383} 1384 1385 1386void X86_64Assembler::imulq(CpuRegister reg, const Immediate& imm) { 1387 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1388 CHECK(imm.is_int32()); // imulq only supports 32b immediate. 1389 1390 EmitRex64(reg, reg); 1391 1392 // See whether imm can be represented as a sign-extended 8bit value. 1393 int64_t v64 = imm.value(); 1394 if (IsInt64(8, v64)) { 1395 // Sign-extension works. 1396 EmitUint8(0x6B); 1397 EmitOperand(reg.LowBits(), Operand(reg)); 1398 EmitUint8(static_cast<uint8_t>(v64 & 0xFF)); 1399 } else { 1400 // Not representable, use full immediate. 1401 EmitUint8(0x69); 1402 EmitOperand(reg.LowBits(), Operand(reg)); 1403 EmitImmediate(imm); 1404 } 1405} 1406 1407 1408void X86_64Assembler::imulq(CpuRegister reg, const Address& address) { 1409 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1410 EmitRex64(reg, address); 1411 EmitUint8(0x0F); 1412 EmitUint8(0xAF); 1413 EmitOperand(reg.LowBits(), address); 1414} 1415 1416 1417void X86_64Assembler::imull(CpuRegister reg) { 1418 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1419 EmitOptionalRex32(reg); 1420 EmitUint8(0xF7); 1421 EmitOperand(5, Operand(reg)); 1422} 1423 1424 1425void X86_64Assembler::imull(const Address& address) { 1426 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1427 EmitOptionalRex32(address); 1428 EmitUint8(0xF7); 1429 EmitOperand(5, address); 1430} 1431 1432 1433void X86_64Assembler::mull(CpuRegister reg) { 1434 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1435 EmitOptionalRex32(reg); 1436 EmitUint8(0xF7); 1437 EmitOperand(4, Operand(reg)); 1438} 1439 1440 1441void X86_64Assembler::mull(const Address& address) { 1442 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1443 EmitOptionalRex32(address); 1444 EmitUint8(0xF7); 1445 EmitOperand(4, address); 1446} 1447 1448 1449void X86_64Assembler::shll(CpuRegister reg, const Immediate& imm) { 1450 EmitGenericShift(false, 4, reg, imm); 1451} 1452 1453 1454void X86_64Assembler::shlq(CpuRegister reg, const Immediate& imm) { 1455 EmitGenericShift(true, 4, reg, imm); 1456} 1457 1458 1459void X86_64Assembler::shll(CpuRegister operand, CpuRegister shifter) { 1460 EmitGenericShift(false, 4, operand, shifter); 1461} 1462 1463 1464void X86_64Assembler::shlq(CpuRegister operand, CpuRegister shifter) { 1465 EmitGenericShift(true, 4, operand, shifter); 1466} 1467 1468 1469void X86_64Assembler::shrl(CpuRegister reg, const Immediate& imm) { 1470 EmitGenericShift(false, 5, reg, imm); 1471} 1472 1473 1474void X86_64Assembler::shrq(CpuRegister reg, const Immediate& imm) { 1475 EmitGenericShift(true, 5, reg, imm); 1476} 1477 1478 1479void X86_64Assembler::shrl(CpuRegister operand, CpuRegister shifter) { 1480 EmitGenericShift(false, 5, operand, shifter); 1481} 1482 1483 1484void X86_64Assembler::shrq(CpuRegister operand, CpuRegister shifter) { 1485 EmitGenericShift(true, 5, operand, shifter); 1486} 1487 1488 1489void X86_64Assembler::sarl(CpuRegister reg, const Immediate& imm) { 1490 EmitGenericShift(false, 7, reg, imm); 1491} 1492 1493 1494void X86_64Assembler::sarl(CpuRegister operand, CpuRegister shifter) { 1495 EmitGenericShift(false, 7, operand, shifter); 1496} 1497 1498 1499void X86_64Assembler::sarq(CpuRegister reg, const Immediate& imm) { 1500 EmitGenericShift(true, 7, reg, imm); 1501} 1502 1503 1504void X86_64Assembler::sarq(CpuRegister operand, CpuRegister shifter) { 1505 EmitGenericShift(true, 7, operand, shifter); 1506} 1507 1508 1509void X86_64Assembler::negl(CpuRegister reg) { 1510 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1511 EmitOptionalRex32(reg); 1512 EmitUint8(0xF7); 1513 EmitOperand(3, Operand(reg)); 1514} 1515 1516 1517void X86_64Assembler::negq(CpuRegister reg) { 1518 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1519 EmitRex64(reg); 1520 EmitUint8(0xF7); 1521 EmitOperand(3, Operand(reg)); 1522} 1523 1524 1525void X86_64Assembler::notl(CpuRegister reg) { 1526 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1527 EmitOptionalRex32(reg); 1528 EmitUint8(0xF7); 1529 EmitUint8(0xD0 | reg.LowBits()); 1530} 1531 1532 1533void X86_64Assembler::notq(CpuRegister reg) { 1534 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1535 EmitRex64(reg); 1536 EmitUint8(0xF7); 1537 EmitOperand(2, Operand(reg)); 1538} 1539 1540 1541void X86_64Assembler::enter(const Immediate& imm) { 1542 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1543 EmitUint8(0xC8); 1544 CHECK(imm.is_uint16()); 1545 EmitUint8(imm.value() & 0xFF); 1546 EmitUint8((imm.value() >> 8) & 0xFF); 1547 EmitUint8(0x00); 1548} 1549 1550 1551void X86_64Assembler::leave() { 1552 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1553 EmitUint8(0xC9); 1554} 1555 1556 1557void X86_64Assembler::ret() { 1558 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1559 EmitUint8(0xC3); 1560} 1561 1562 1563void X86_64Assembler::ret(const Immediate& imm) { 1564 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1565 EmitUint8(0xC2); 1566 CHECK(imm.is_uint16()); 1567 EmitUint8(imm.value() & 0xFF); 1568 EmitUint8((imm.value() >> 8) & 0xFF); 1569} 1570 1571 1572 1573void X86_64Assembler::nop() { 1574 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1575 EmitUint8(0x90); 1576} 1577 1578 1579void X86_64Assembler::int3() { 1580 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1581 EmitUint8(0xCC); 1582} 1583 1584 1585void X86_64Assembler::hlt() { 1586 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1587 EmitUint8(0xF4); 1588} 1589 1590 1591void X86_64Assembler::j(Condition condition, Label* label) { 1592 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1593 if (label->IsBound()) { 1594 static const int kShortSize = 2; 1595 static const int kLongSize = 6; 1596 int offset = label->Position() - buffer_.Size(); 1597 CHECK_LE(offset, 0); 1598 if (IsInt(8, offset - kShortSize)) { 1599 EmitUint8(0x70 + condition); 1600 EmitUint8((offset - kShortSize) & 0xFF); 1601 } else { 1602 EmitUint8(0x0F); 1603 EmitUint8(0x80 + condition); 1604 EmitInt32(offset - kLongSize); 1605 } 1606 } else { 1607 EmitUint8(0x0F); 1608 EmitUint8(0x80 + condition); 1609 EmitLabelLink(label); 1610 } 1611} 1612 1613 1614void X86_64Assembler::jmp(CpuRegister reg) { 1615 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1616 EmitOptionalRex32(reg); 1617 EmitUint8(0xFF); 1618 EmitRegisterOperand(4, reg.LowBits()); 1619} 1620 1621void X86_64Assembler::jmp(const Address& address) { 1622 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1623 EmitOptionalRex32(address); 1624 EmitUint8(0xFF); 1625 EmitOperand(4, address); 1626} 1627 1628void X86_64Assembler::jmp(Label* label) { 1629 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1630 if (label->IsBound()) { 1631 static const int kShortSize = 2; 1632 static const int kLongSize = 5; 1633 int offset = label->Position() - buffer_.Size(); 1634 CHECK_LE(offset, 0); 1635 if (IsInt(8, offset - kShortSize)) { 1636 EmitUint8(0xEB); 1637 EmitUint8((offset - kShortSize) & 0xFF); 1638 } else { 1639 EmitUint8(0xE9); 1640 EmitInt32(offset - kLongSize); 1641 } 1642 } else { 1643 EmitUint8(0xE9); 1644 EmitLabelLink(label); 1645 } 1646} 1647 1648 1649X86_64Assembler* X86_64Assembler::lock() { 1650 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1651 EmitUint8(0xF0); 1652 return this; 1653} 1654 1655 1656void X86_64Assembler::cmpxchgl(const Address& address, CpuRegister reg) { 1657 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1658 EmitUint8(0x0F); 1659 EmitUint8(0xB1); 1660 EmitOperand(reg.LowBits(), address); 1661} 1662 1663void X86_64Assembler::mfence() { 1664 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1665 EmitUint8(0x0F); 1666 EmitUint8(0xAE); 1667 EmitUint8(0xF0); 1668} 1669 1670 1671X86_64Assembler* X86_64Assembler::gs() { 1672 // TODO: gs is a prefix and not an instruction 1673 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1674 EmitUint8(0x65); 1675 return this; 1676} 1677 1678 1679void X86_64Assembler::AddImmediate(CpuRegister reg, const Immediate& imm) { 1680 int value = imm.value(); 1681 if (value != 0) { 1682 if (value > 0) { 1683 addl(reg, imm); 1684 } else { 1685 subl(reg, Immediate(value)); 1686 } 1687 } 1688} 1689 1690 1691void X86_64Assembler::setcc(Condition condition, CpuRegister dst) { 1692 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1693 // RSP, RBP, RDI, RSI need rex prefix (else the pattern encodes ah/bh/ch/dh). 1694 if (dst.NeedsRex() || dst.AsRegister() > 3) { 1695 EmitOptionalRex(true, false, false, false, dst.NeedsRex()); 1696 } 1697 EmitUint8(0x0F); 1698 EmitUint8(0x90 + condition); 1699 EmitUint8(0xC0 + dst.LowBits()); 1700} 1701 1702 1703void X86_64Assembler::LoadDoubleConstant(XmmRegister dst, double value) { 1704 // TODO: Need to have a code constants table. 1705 int64_t constant = bit_cast<int64_t, double>(value); 1706 pushq(Immediate(High32Bits(constant))); 1707 pushq(Immediate(Low32Bits(constant))); 1708 movsd(dst, Address(CpuRegister(RSP), 0)); 1709 addq(CpuRegister(RSP), Immediate(2 * sizeof(intptr_t))); 1710} 1711 1712 1713void X86_64Assembler::FloatNegate(XmmRegister f) { 1714 static const struct { 1715 uint32_t a; 1716 uint32_t b; 1717 uint32_t c; 1718 uint32_t d; 1719 } float_negate_constant __attribute__((aligned(16))) = 1720 { 0x80000000, 0x00000000, 0x80000000, 0x00000000 }; 1721 xorps(f, Address::Absolute(reinterpret_cast<uintptr_t>(&float_negate_constant))); 1722} 1723 1724 1725void X86_64Assembler::DoubleNegate(XmmRegister d) { 1726 static const struct { 1727 uint64_t a; 1728 uint64_t b; 1729 } double_negate_constant __attribute__((aligned(16))) = 1730 {0x8000000000000000LL, 0x8000000000000000LL}; 1731 xorpd(d, Address::Absolute(reinterpret_cast<uintptr_t>(&double_negate_constant))); 1732} 1733 1734 1735void X86_64Assembler::DoubleAbs(XmmRegister reg) { 1736 static const struct { 1737 uint64_t a; 1738 uint64_t b; 1739 } double_abs_constant __attribute__((aligned(16))) = 1740 {0x7FFFFFFFFFFFFFFFLL, 0x7FFFFFFFFFFFFFFFLL}; 1741 andpd(reg, Address::Absolute(reinterpret_cast<uintptr_t>(&double_abs_constant))); 1742} 1743 1744 1745void X86_64Assembler::Align(int alignment, int offset) { 1746 CHECK(IsPowerOfTwo(alignment)); 1747 // Emit nop instruction until the real position is aligned. 1748 while (((offset + buffer_.GetPosition()) & (alignment-1)) != 0) { 1749 nop(); 1750 } 1751} 1752 1753 1754void X86_64Assembler::Bind(Label* label) { 1755 int bound = buffer_.Size(); 1756 CHECK(!label->IsBound()); // Labels can only be bound once. 1757 while (label->IsLinked()) { 1758 int position = label->LinkPosition(); 1759 int next = buffer_.Load<int32_t>(position); 1760 buffer_.Store<int32_t>(position, bound - (position + 4)); 1761 label->position_ = next; 1762 } 1763 label->BindTo(bound); 1764} 1765 1766 1767void X86_64Assembler::EmitOperand(uint8_t reg_or_opcode, const Operand& operand) { 1768 CHECK_GE(reg_or_opcode, 0); 1769 CHECK_LT(reg_or_opcode, 8); 1770 const int length = operand.length_; 1771 CHECK_GT(length, 0); 1772 // Emit the ModRM byte updated with the given reg value. 1773 CHECK_EQ(operand.encoding_[0] & 0x38, 0); 1774 EmitUint8(operand.encoding_[0] + (reg_or_opcode << 3)); 1775 // Emit the rest of the encoded operand. 1776 for (int i = 1; i < length; i++) { 1777 EmitUint8(operand.encoding_[i]); 1778 } 1779} 1780 1781 1782void X86_64Assembler::EmitImmediate(const Immediate& imm) { 1783 if (imm.is_int32()) { 1784 EmitInt32(static_cast<int32_t>(imm.value())); 1785 } else { 1786 EmitInt64(imm.value()); 1787 } 1788} 1789 1790 1791void X86_64Assembler::EmitComplex(uint8_t reg_or_opcode, 1792 const Operand& operand, 1793 const Immediate& immediate) { 1794 CHECK_GE(reg_or_opcode, 0); 1795 CHECK_LT(reg_or_opcode, 8); 1796 if (immediate.is_int8()) { 1797 // Use sign-extended 8-bit immediate. 1798 EmitUint8(0x83); 1799 EmitOperand(reg_or_opcode, operand); 1800 EmitUint8(immediate.value() & 0xFF); 1801 } else if (operand.IsRegister(CpuRegister(RAX))) { 1802 // Use short form if the destination is eax. 1803 EmitUint8(0x05 + (reg_or_opcode << 3)); 1804 EmitImmediate(immediate); 1805 } else { 1806 EmitUint8(0x81); 1807 EmitOperand(reg_or_opcode, operand); 1808 EmitImmediate(immediate); 1809 } 1810} 1811 1812 1813void X86_64Assembler::EmitLabel(Label* label, int instruction_size) { 1814 if (label->IsBound()) { 1815 int offset = label->Position() - buffer_.Size(); 1816 CHECK_LE(offset, 0); 1817 EmitInt32(offset - instruction_size); 1818 } else { 1819 EmitLabelLink(label); 1820 } 1821} 1822 1823 1824void X86_64Assembler::EmitLabelLink(Label* label) { 1825 CHECK(!label->IsBound()); 1826 int position = buffer_.Size(); 1827 EmitInt32(label->position_); 1828 label->LinkTo(position); 1829} 1830 1831 1832void X86_64Assembler::EmitGenericShift(bool wide, 1833 int reg_or_opcode, 1834 CpuRegister reg, 1835 const Immediate& imm) { 1836 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1837 CHECK(imm.is_int8()); 1838 if (wide) { 1839 EmitRex64(reg); 1840 } else { 1841 EmitOptionalRex32(reg); 1842 } 1843 if (imm.value() == 1) { 1844 EmitUint8(0xD1); 1845 EmitOperand(reg_or_opcode, Operand(reg)); 1846 } else { 1847 EmitUint8(0xC1); 1848 EmitOperand(reg_or_opcode, Operand(reg)); 1849 EmitUint8(imm.value() & 0xFF); 1850 } 1851} 1852 1853 1854void X86_64Assembler::EmitGenericShift(bool wide, 1855 int reg_or_opcode, 1856 CpuRegister operand, 1857 CpuRegister shifter) { 1858 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1859 CHECK_EQ(shifter.AsRegister(), RCX); 1860 if (wide) { 1861 EmitRex64(operand); 1862 } else { 1863 EmitOptionalRex32(operand); 1864 } 1865 EmitUint8(0xD3); 1866 EmitOperand(reg_or_opcode, Operand(operand)); 1867} 1868 1869void X86_64Assembler::EmitOptionalRex(bool force, bool w, bool r, bool x, bool b) { 1870 // REX.WRXB 1871 // W - 64-bit operand 1872 // R - MODRM.reg 1873 // X - SIB.index 1874 // B - MODRM.rm/SIB.base 1875 uint8_t rex = force ? 0x40 : 0; 1876 if (w) { 1877 rex |= 0x48; // REX.W000 1878 } 1879 if (r) { 1880 rex |= 0x44; // REX.0R00 1881 } 1882 if (x) { 1883 rex |= 0x42; // REX.00X0 1884 } 1885 if (b) { 1886 rex |= 0x41; // REX.000B 1887 } 1888 if (rex != 0) { 1889 EmitUint8(rex); 1890 } 1891} 1892 1893void X86_64Assembler::EmitOptionalRex32(CpuRegister reg) { 1894 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 1895} 1896 1897void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, CpuRegister src) { 1898 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1899} 1900 1901void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, XmmRegister src) { 1902 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1903} 1904 1905void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, XmmRegister src) { 1906 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1907} 1908 1909void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, CpuRegister src) { 1910 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1911} 1912 1913void X86_64Assembler::EmitOptionalRex32(const Operand& operand) { 1914 uint8_t rex = operand.rex(); 1915 if (rex != 0) { 1916 EmitUint8(rex); 1917 } 1918} 1919 1920void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, const Operand& operand) { 1921 uint8_t rex = operand.rex(); 1922 if (dst.NeedsRex()) { 1923 rex |= 0x44; // REX.0R00 1924 } 1925 if (rex != 0) { 1926 EmitUint8(rex); 1927 } 1928} 1929 1930void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, const Operand& operand) { 1931 uint8_t rex = operand.rex(); 1932 if (dst.NeedsRex()) { 1933 rex |= 0x44; // REX.0R00 1934 } 1935 if (rex != 0) { 1936 EmitUint8(rex); 1937 } 1938} 1939 1940void X86_64Assembler::EmitRex64() { 1941 EmitOptionalRex(false, true, false, false, false); 1942} 1943 1944void X86_64Assembler::EmitRex64(CpuRegister reg) { 1945 EmitOptionalRex(false, true, false, false, reg.NeedsRex()); 1946} 1947 1948void X86_64Assembler::EmitRex64(const Operand& operand) { 1949 uint8_t rex = operand.rex(); 1950 rex |= 0x48; // REX.W000 1951 EmitUint8(rex); 1952} 1953 1954void X86_64Assembler::EmitRex64(CpuRegister dst, CpuRegister src) { 1955 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 1956} 1957 1958void X86_64Assembler::EmitRex64(XmmRegister dst, CpuRegister src) { 1959 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 1960} 1961 1962void X86_64Assembler::EmitRex64(CpuRegister dst, const Operand& operand) { 1963 uint8_t rex = 0x48 | operand.rex(); // REX.W000 1964 if (dst.NeedsRex()) { 1965 rex |= 0x44; // REX.0R00 1966 } 1967 if (rex != 0) { 1968 EmitUint8(rex); 1969 } 1970} 1971 1972void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, CpuRegister src) { 1973 EmitOptionalRex(true, false, dst.NeedsRex(), false, src.NeedsRex()); 1974} 1975 1976void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand) { 1977 uint8_t rex = 0x40 | operand.rex(); // REX.0000 1978 if (dst.NeedsRex()) { 1979 rex |= 0x44; // REX.0R00 1980 } 1981 if (rex != 0) { 1982 EmitUint8(rex); 1983 } 1984} 1985 1986void X86_64Assembler::InitializeFrameDescriptionEntry() { 1987 WriteFDEHeader(&cfi_info_, true /* is_64bit */); 1988} 1989 1990void X86_64Assembler::FinalizeFrameDescriptionEntry() { 1991 WriteFDEAddressRange(&cfi_info_, buffer_.Size(), true /* is_64bit */); 1992 PadCFI(&cfi_info_); 1993 WriteCFILength(&cfi_info_, true /* is_64bit */); 1994} 1995 1996constexpr size_t kFramePointerSize = 8; 1997 1998void X86_64Assembler::BuildFrame(size_t frame_size, ManagedRegister method_reg, 1999 const std::vector<ManagedRegister>& spill_regs, 2000 const ManagedRegisterEntrySpills& entry_spills) { 2001 cfi_cfa_offset_ = kFramePointerSize; // Only return address on stack 2002 cfi_pc_ = buffer_.Size(); // Nothing emitted yet 2003 DCHECK_EQ(cfi_pc_, 0U); 2004 2005 uint32_t reg_offset = 1; 2006 CHECK_ALIGNED(frame_size, kStackAlignment); 2007 int gpr_count = 0; 2008 for (int i = spill_regs.size() - 1; i >= 0; --i) { 2009 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2010 if (spill.IsCpuRegister()) { 2011 pushq(spill.AsCpuRegister()); 2012 gpr_count++; 2013 2014 // DW_CFA_advance_loc 2015 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 2016 cfi_pc_ = buffer_.Size(); 2017 // DW_CFA_def_cfa_offset 2018 cfi_cfa_offset_ += kFramePointerSize; 2019 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 2020 // DW_CFA_offset reg offset 2021 reg_offset++; 2022 DW_CFA_offset(&cfi_info_, spill.DWARFRegId(), reg_offset); 2023 } 2024 } 2025 // return address then method on stack 2026 int64_t rest_of_frame = static_cast<int64_t>(frame_size) 2027 - (gpr_count * kFramePointerSize) 2028 - kFramePointerSize /*return address*/; 2029 subq(CpuRegister(RSP), Immediate(rest_of_frame)); 2030 // DW_CFA_advance_loc 2031 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 2032 cfi_pc_ = buffer_.Size(); 2033 // DW_CFA_def_cfa_offset 2034 cfi_cfa_offset_ += rest_of_frame; 2035 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 2036 2037 // spill xmms 2038 int64_t offset = rest_of_frame; 2039 for (int i = spill_regs.size() - 1; i >= 0; --i) { 2040 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2041 if (spill.IsXmmRegister()) { 2042 offset -= sizeof(double); 2043 movsd(Address(CpuRegister(RSP), offset), spill.AsXmmRegister()); 2044 } 2045 } 2046 2047 DCHECK_EQ(4U, sizeof(StackReference<mirror::ArtMethod>)); 2048 2049 movl(Address(CpuRegister(RSP), 0), method_reg.AsX86_64().AsCpuRegister()); 2050 2051 for (size_t i = 0; i < entry_spills.size(); ++i) { 2052 ManagedRegisterSpill spill = entry_spills.at(i); 2053 if (spill.AsX86_64().IsCpuRegister()) { 2054 if (spill.getSize() == 8) { 2055 movq(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), 2056 spill.AsX86_64().AsCpuRegister()); 2057 } else { 2058 CHECK_EQ(spill.getSize(), 4); 2059 movl(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsCpuRegister()); 2060 } 2061 } else { 2062 if (spill.getSize() == 8) { 2063 movsd(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 2064 } else { 2065 CHECK_EQ(spill.getSize(), 4); 2066 movss(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 2067 } 2068 } 2069 } 2070} 2071 2072void X86_64Assembler::RemoveFrame(size_t frame_size, 2073 const std::vector<ManagedRegister>& spill_regs) { 2074 CHECK_ALIGNED(frame_size, kStackAlignment); 2075 int gpr_count = 0; 2076 // unspill xmms 2077 int64_t offset = static_cast<int64_t>(frame_size) - (spill_regs.size() * kFramePointerSize) - 2 * kFramePointerSize; 2078 for (size_t i = 0; i < spill_regs.size(); ++i) { 2079 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2080 if (spill.IsXmmRegister()) { 2081 offset += sizeof(double); 2082 movsd(spill.AsXmmRegister(), Address(CpuRegister(RSP), offset)); 2083 } else { 2084 gpr_count++; 2085 } 2086 } 2087 addq(CpuRegister(RSP), Immediate(static_cast<int64_t>(frame_size) - (gpr_count * kFramePointerSize) - kFramePointerSize)); 2088 for (size_t i = 0; i < spill_regs.size(); ++i) { 2089 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2090 if (spill.IsCpuRegister()) { 2091 popq(spill.AsCpuRegister()); 2092 } 2093 } 2094 ret(); 2095} 2096 2097void X86_64Assembler::IncreaseFrameSize(size_t adjust) { 2098 CHECK_ALIGNED(adjust, kStackAlignment); 2099 addq(CpuRegister(RSP), Immediate(-static_cast<int64_t>(adjust))); 2100 // DW_CFA_advance_loc 2101 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 2102 cfi_pc_ = buffer_.Size(); 2103 // DW_CFA_def_cfa_offset 2104 cfi_cfa_offset_ += adjust; 2105 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 2106} 2107 2108void X86_64Assembler::DecreaseFrameSize(size_t adjust) { 2109 CHECK_ALIGNED(adjust, kStackAlignment); 2110 addq(CpuRegister(RSP), Immediate(adjust)); 2111} 2112 2113void X86_64Assembler::Store(FrameOffset offs, ManagedRegister msrc, size_t size) { 2114 X86_64ManagedRegister src = msrc.AsX86_64(); 2115 if (src.IsNoRegister()) { 2116 CHECK_EQ(0u, size); 2117 } else if (src.IsCpuRegister()) { 2118 if (size == 4) { 2119 CHECK_EQ(4u, size); 2120 movl(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 2121 } else { 2122 CHECK_EQ(8u, size); 2123 movq(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 2124 } 2125 } else if (src.IsRegisterPair()) { 2126 CHECK_EQ(0u, size); 2127 movq(Address(CpuRegister(RSP), offs), src.AsRegisterPairLow()); 2128 movq(Address(CpuRegister(RSP), FrameOffset(offs.Int32Value()+4)), 2129 src.AsRegisterPairHigh()); 2130 } else if (src.IsX87Register()) { 2131 if (size == 4) { 2132 fstps(Address(CpuRegister(RSP), offs)); 2133 } else { 2134 fstpl(Address(CpuRegister(RSP), offs)); 2135 } 2136 } else { 2137 CHECK(src.IsXmmRegister()); 2138 if (size == 4) { 2139 movss(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 2140 } else { 2141 movsd(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 2142 } 2143 } 2144} 2145 2146void X86_64Assembler::StoreRef(FrameOffset dest, ManagedRegister msrc) { 2147 X86_64ManagedRegister src = msrc.AsX86_64(); 2148 CHECK(src.IsCpuRegister()); 2149 movl(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 2150} 2151 2152void X86_64Assembler::StoreRawPtr(FrameOffset dest, ManagedRegister msrc) { 2153 X86_64ManagedRegister src = msrc.AsX86_64(); 2154 CHECK(src.IsCpuRegister()); 2155 movq(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 2156} 2157 2158void X86_64Assembler::StoreImmediateToFrame(FrameOffset dest, uint32_t imm, 2159 ManagedRegister) { 2160 movl(Address(CpuRegister(RSP), dest), Immediate(imm)); // TODO(64) movq? 2161} 2162 2163void X86_64Assembler::StoreImmediateToThread64(ThreadOffset<8> dest, uint32_t imm, 2164 ManagedRegister) { 2165 gs()->movl(Address::Absolute(dest, true), Immediate(imm)); // TODO(64) movq? 2166} 2167 2168void X86_64Assembler::StoreStackOffsetToThread64(ThreadOffset<8> thr_offs, 2169 FrameOffset fr_offs, 2170 ManagedRegister mscratch) { 2171 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2172 CHECK(scratch.IsCpuRegister()); 2173 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), fr_offs)); 2174 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2175} 2176 2177void X86_64Assembler::StoreStackPointerToThread64(ThreadOffset<8> thr_offs) { 2178 gs()->movq(Address::Absolute(thr_offs, true), CpuRegister(RSP)); 2179} 2180 2181void X86_64Assembler::StoreSpanning(FrameOffset /*dst*/, ManagedRegister /*src*/, 2182 FrameOffset /*in_off*/, ManagedRegister /*scratch*/) { 2183 UNIMPLEMENTED(FATAL); // this case only currently exists for ARM 2184} 2185 2186void X86_64Assembler::Load(ManagedRegister mdest, FrameOffset src, size_t size) { 2187 X86_64ManagedRegister dest = mdest.AsX86_64(); 2188 if (dest.IsNoRegister()) { 2189 CHECK_EQ(0u, size); 2190 } else if (dest.IsCpuRegister()) { 2191 if (size == 4) { 2192 CHECK_EQ(4u, size); 2193 movl(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2194 } else { 2195 CHECK_EQ(8u, size); 2196 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2197 } 2198 } else if (dest.IsRegisterPair()) { 2199 CHECK_EQ(0u, size); 2200 movq(dest.AsRegisterPairLow(), Address(CpuRegister(RSP), src)); 2201 movq(dest.AsRegisterPairHigh(), Address(CpuRegister(RSP), FrameOffset(src.Int32Value()+4))); 2202 } else if (dest.IsX87Register()) { 2203 if (size == 4) { 2204 flds(Address(CpuRegister(RSP), src)); 2205 } else { 2206 fldl(Address(CpuRegister(RSP), src)); 2207 } 2208 } else { 2209 CHECK(dest.IsXmmRegister()); 2210 if (size == 4) { 2211 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2212 } else { 2213 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2214 } 2215 } 2216} 2217 2218void X86_64Assembler::LoadFromThread64(ManagedRegister mdest, ThreadOffset<8> src, size_t size) { 2219 X86_64ManagedRegister dest = mdest.AsX86_64(); 2220 if (dest.IsNoRegister()) { 2221 CHECK_EQ(0u, size); 2222 } else if (dest.IsCpuRegister()) { 2223 CHECK_EQ(4u, size); 2224 gs()->movl(dest.AsCpuRegister(), Address::Absolute(src, true)); 2225 } else if (dest.IsRegisterPair()) { 2226 CHECK_EQ(8u, size); 2227 gs()->movq(dest.AsRegisterPairLow(), Address::Absolute(src, true)); 2228 } else if (dest.IsX87Register()) { 2229 if (size == 4) { 2230 gs()->flds(Address::Absolute(src, true)); 2231 } else { 2232 gs()->fldl(Address::Absolute(src, true)); 2233 } 2234 } else { 2235 CHECK(dest.IsXmmRegister()); 2236 if (size == 4) { 2237 gs()->movss(dest.AsXmmRegister(), Address::Absolute(src, true)); 2238 } else { 2239 gs()->movsd(dest.AsXmmRegister(), Address::Absolute(src, true)); 2240 } 2241 } 2242} 2243 2244void X86_64Assembler::LoadRef(ManagedRegister mdest, FrameOffset src) { 2245 X86_64ManagedRegister dest = mdest.AsX86_64(); 2246 CHECK(dest.IsCpuRegister()); 2247 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2248} 2249 2250void X86_64Assembler::LoadRef(ManagedRegister mdest, ManagedRegister base, 2251 MemberOffset offs) { 2252 X86_64ManagedRegister dest = mdest.AsX86_64(); 2253 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2254 movl(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2255 if (kPoisonHeapReferences) { 2256 negl(dest.AsCpuRegister()); 2257 } 2258} 2259 2260void X86_64Assembler::LoadRawPtr(ManagedRegister mdest, ManagedRegister base, 2261 Offset offs) { 2262 X86_64ManagedRegister dest = mdest.AsX86_64(); 2263 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2264 movq(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2265} 2266 2267void X86_64Assembler::LoadRawPtrFromThread64(ManagedRegister mdest, ThreadOffset<8> offs) { 2268 X86_64ManagedRegister dest = mdest.AsX86_64(); 2269 CHECK(dest.IsCpuRegister()); 2270 gs()->movq(dest.AsCpuRegister(), Address::Absolute(offs, true)); 2271} 2272 2273void X86_64Assembler::SignExtend(ManagedRegister mreg, size_t size) { 2274 X86_64ManagedRegister reg = mreg.AsX86_64(); 2275 CHECK(size == 1 || size == 2) << size; 2276 CHECK(reg.IsCpuRegister()) << reg; 2277 if (size == 1) { 2278 movsxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2279 } else { 2280 movsxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2281 } 2282} 2283 2284void X86_64Assembler::ZeroExtend(ManagedRegister mreg, size_t size) { 2285 X86_64ManagedRegister reg = mreg.AsX86_64(); 2286 CHECK(size == 1 || size == 2) << size; 2287 CHECK(reg.IsCpuRegister()) << reg; 2288 if (size == 1) { 2289 movzxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2290 } else { 2291 movzxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2292 } 2293} 2294 2295void X86_64Assembler::Move(ManagedRegister mdest, ManagedRegister msrc, size_t size) { 2296 X86_64ManagedRegister dest = mdest.AsX86_64(); 2297 X86_64ManagedRegister src = msrc.AsX86_64(); 2298 if (!dest.Equals(src)) { 2299 if (dest.IsCpuRegister() && src.IsCpuRegister()) { 2300 movq(dest.AsCpuRegister(), src.AsCpuRegister()); 2301 } else if (src.IsX87Register() && dest.IsXmmRegister()) { 2302 // Pass via stack and pop X87 register 2303 subl(CpuRegister(RSP), Immediate(16)); 2304 if (size == 4) { 2305 CHECK_EQ(src.AsX87Register(), ST0); 2306 fstps(Address(CpuRegister(RSP), 0)); 2307 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2308 } else { 2309 CHECK_EQ(src.AsX87Register(), ST0); 2310 fstpl(Address(CpuRegister(RSP), 0)); 2311 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2312 } 2313 addq(CpuRegister(RSP), Immediate(16)); 2314 } else { 2315 // TODO: x87, SSE 2316 UNIMPLEMENTED(FATAL) << ": Move " << dest << ", " << src; 2317 } 2318 } 2319} 2320 2321void X86_64Assembler::CopyRef(FrameOffset dest, FrameOffset src, 2322 ManagedRegister mscratch) { 2323 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2324 CHECK(scratch.IsCpuRegister()); 2325 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2326 movl(Address(CpuRegister(RSP), dest), scratch.AsCpuRegister()); 2327} 2328 2329void X86_64Assembler::CopyRawPtrFromThread64(FrameOffset fr_offs, 2330 ThreadOffset<8> thr_offs, 2331 ManagedRegister mscratch) { 2332 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2333 CHECK(scratch.IsCpuRegister()); 2334 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(thr_offs, true)); 2335 Store(fr_offs, scratch, 8); 2336} 2337 2338void X86_64Assembler::CopyRawPtrToThread64(ThreadOffset<8> thr_offs, 2339 FrameOffset fr_offs, 2340 ManagedRegister mscratch) { 2341 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2342 CHECK(scratch.IsCpuRegister()); 2343 Load(scratch, fr_offs, 8); 2344 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2345} 2346 2347void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src, 2348 ManagedRegister mscratch, 2349 size_t size) { 2350 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2351 if (scratch.IsCpuRegister() && size == 8) { 2352 Load(scratch, src, 4); 2353 Store(dest, scratch, 4); 2354 Load(scratch, FrameOffset(src.Int32Value() + 4), 4); 2355 Store(FrameOffset(dest.Int32Value() + 4), scratch, 4); 2356 } else { 2357 Load(scratch, src, size); 2358 Store(dest, scratch, size); 2359 } 2360} 2361 2362void X86_64Assembler::Copy(FrameOffset /*dst*/, ManagedRegister /*src_base*/, Offset /*src_offset*/, 2363 ManagedRegister /*scratch*/, size_t /*size*/) { 2364 UNIMPLEMENTED(FATAL); 2365} 2366 2367void X86_64Assembler::Copy(ManagedRegister dest_base, Offset dest_offset, FrameOffset src, 2368 ManagedRegister scratch, size_t size) { 2369 CHECK(scratch.IsNoRegister()); 2370 CHECK_EQ(size, 4u); 2371 pushq(Address(CpuRegister(RSP), src)); 2372 popq(Address(dest_base.AsX86_64().AsCpuRegister(), dest_offset)); 2373} 2374 2375void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src_base, Offset src_offset, 2376 ManagedRegister mscratch, size_t size) { 2377 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2378 CHECK_EQ(size, 4u); 2379 movq(scratch, Address(CpuRegister(RSP), src_base)); 2380 movq(scratch, Address(scratch, src_offset)); 2381 movq(Address(CpuRegister(RSP), dest), scratch); 2382} 2383 2384void X86_64Assembler::Copy(ManagedRegister dest, Offset dest_offset, 2385 ManagedRegister src, Offset src_offset, 2386 ManagedRegister scratch, size_t size) { 2387 CHECK_EQ(size, 4u); 2388 CHECK(scratch.IsNoRegister()); 2389 pushq(Address(src.AsX86_64().AsCpuRegister(), src_offset)); 2390 popq(Address(dest.AsX86_64().AsCpuRegister(), dest_offset)); 2391} 2392 2393void X86_64Assembler::Copy(FrameOffset dest, Offset dest_offset, FrameOffset src, Offset src_offset, 2394 ManagedRegister mscratch, size_t size) { 2395 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2396 CHECK_EQ(size, 4u); 2397 CHECK_EQ(dest.Int32Value(), src.Int32Value()); 2398 movq(scratch, Address(CpuRegister(RSP), src)); 2399 pushq(Address(scratch, src_offset)); 2400 popq(Address(scratch, dest_offset)); 2401} 2402 2403void X86_64Assembler::MemoryBarrier(ManagedRegister) { 2404 mfence(); 2405} 2406 2407void X86_64Assembler::CreateHandleScopeEntry(ManagedRegister mout_reg, 2408 FrameOffset handle_scope_offset, 2409 ManagedRegister min_reg, bool null_allowed) { 2410 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2411 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2412 if (in_reg.IsNoRegister()) { // TODO(64): && null_allowed 2413 // Use out_reg as indicator of NULL 2414 in_reg = out_reg; 2415 // TODO: movzwl 2416 movl(in_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2417 } 2418 CHECK(in_reg.IsCpuRegister()); 2419 CHECK(out_reg.IsCpuRegister()); 2420 VerifyObject(in_reg, null_allowed); 2421 if (null_allowed) { 2422 Label null_arg; 2423 if (!out_reg.Equals(in_reg)) { 2424 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2425 } 2426 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2427 j(kZero, &null_arg); 2428 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2429 Bind(&null_arg); 2430 } else { 2431 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2432 } 2433} 2434 2435void X86_64Assembler::CreateHandleScopeEntry(FrameOffset out_off, 2436 FrameOffset handle_scope_offset, 2437 ManagedRegister mscratch, 2438 bool null_allowed) { 2439 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2440 CHECK(scratch.IsCpuRegister()); 2441 if (null_allowed) { 2442 Label null_arg; 2443 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2444 testl(scratch.AsCpuRegister(), scratch.AsCpuRegister()); 2445 j(kZero, &null_arg); 2446 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2447 Bind(&null_arg); 2448 } else { 2449 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2450 } 2451 Store(out_off, scratch, 8); 2452} 2453 2454// Given a handle scope entry, load the associated reference. 2455void X86_64Assembler::LoadReferenceFromHandleScope(ManagedRegister mout_reg, 2456 ManagedRegister min_reg) { 2457 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2458 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2459 CHECK(out_reg.IsCpuRegister()); 2460 CHECK(in_reg.IsCpuRegister()); 2461 Label null_arg; 2462 if (!out_reg.Equals(in_reg)) { 2463 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2464 } 2465 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2466 j(kZero, &null_arg); 2467 movq(out_reg.AsCpuRegister(), Address(in_reg.AsCpuRegister(), 0)); 2468 Bind(&null_arg); 2469} 2470 2471void X86_64Assembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) { 2472 // TODO: not validating references 2473} 2474 2475void X86_64Assembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) { 2476 // TODO: not validating references 2477} 2478 2479void X86_64Assembler::Call(ManagedRegister mbase, Offset offset, ManagedRegister) { 2480 X86_64ManagedRegister base = mbase.AsX86_64(); 2481 CHECK(base.IsCpuRegister()); 2482 call(Address(base.AsCpuRegister(), offset.Int32Value())); 2483 // TODO: place reference map on call 2484} 2485 2486void X86_64Assembler::Call(FrameOffset base, Offset offset, ManagedRegister mscratch) { 2487 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2488 movl(scratch, Address(CpuRegister(RSP), base)); 2489 call(Address(scratch, offset)); 2490} 2491 2492void X86_64Assembler::CallFromThread64(ThreadOffset<8> offset, ManagedRegister /*mscratch*/) { 2493 gs()->call(Address::Absolute(offset, true)); 2494} 2495 2496void X86_64Assembler::GetCurrentThread(ManagedRegister tr) { 2497 gs()->movq(tr.AsX86_64().AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2498} 2499 2500void X86_64Assembler::GetCurrentThread(FrameOffset offset, ManagedRegister mscratch) { 2501 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2502 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2503 movq(Address(CpuRegister(RSP), offset), scratch.AsCpuRegister()); 2504} 2505 2506// Slowpath entered when Thread::Current()->_exception is non-null 2507class X86_64ExceptionSlowPath FINAL : public SlowPath { 2508 public: 2509 explicit X86_64ExceptionSlowPath(size_t stack_adjust) : stack_adjust_(stack_adjust) {} 2510 virtual void Emit(Assembler *sp_asm) OVERRIDE; 2511 private: 2512 const size_t stack_adjust_; 2513}; 2514 2515void X86_64Assembler::ExceptionPoll(ManagedRegister /*scratch*/, size_t stack_adjust) { 2516 X86_64ExceptionSlowPath* slow = new X86_64ExceptionSlowPath(stack_adjust); 2517 buffer_.EnqueueSlowPath(slow); 2518 gs()->cmpl(Address::Absolute(Thread::ExceptionOffset<8>(), true), Immediate(0)); 2519 j(kNotEqual, slow->Entry()); 2520} 2521 2522void X86_64ExceptionSlowPath::Emit(Assembler *sasm) { 2523 X86_64Assembler* sp_asm = down_cast<X86_64Assembler*>(sasm); 2524#define __ sp_asm-> 2525 __ Bind(&entry_); 2526 // Note: the return value is dead 2527 if (stack_adjust_ != 0) { // Fix up the frame. 2528 __ DecreaseFrameSize(stack_adjust_); 2529 } 2530 // Pass exception as argument in RDI 2531 __ gs()->movq(CpuRegister(RDI), Address::Absolute(Thread::ExceptionOffset<8>(), true)); 2532 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(8, pDeliverException), true)); 2533 // this call should never return 2534 __ int3(); 2535#undef __ 2536} 2537 2538} // namespace x86_64 2539} // namespace art 2540