assembler_x86_64.cc revision 96f89a290eb67d7bf4b1636798fa28df14309cc7
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "assembler_x86_64.h" 18 19#include "base/casts.h" 20#include "entrypoints/quick/quick_entrypoints.h" 21#include "memory_region.h" 22#include "thread.h" 23 24namespace art { 25namespace x86_64 { 26 27std::ostream& operator<<(std::ostream& os, const CpuRegister& reg) { 28 return os << reg.AsRegister(); 29} 30 31std::ostream& operator<<(std::ostream& os, const XmmRegister& reg) { 32 return os << reg.AsFloatRegister(); 33} 34 35std::ostream& operator<<(std::ostream& os, const X87Register& reg) { 36 return os << "ST" << static_cast<int>(reg); 37} 38 39void X86_64Assembler::call(CpuRegister reg) { 40 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 41 EmitOptionalRex32(reg); 42 EmitUint8(0xFF); 43 EmitRegisterOperand(2, reg.LowBits()); 44} 45 46 47void X86_64Assembler::call(const Address& address) { 48 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 49 EmitOptionalRex32(address); 50 EmitUint8(0xFF); 51 EmitOperand(2, address); 52} 53 54 55void X86_64Assembler::call(Label* label) { 56 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 57 EmitUint8(0xE8); 58 static const int kSize = 5; 59 EmitLabel(label, kSize); 60} 61 62void X86_64Assembler::pushq(CpuRegister reg) { 63 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 64 EmitOptionalRex32(reg); 65 EmitUint8(0x50 + reg.LowBits()); 66} 67 68 69void X86_64Assembler::pushq(const Address& address) { 70 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 71 EmitOptionalRex32(address); 72 EmitUint8(0xFF); 73 EmitOperand(6, address); 74} 75 76 77void X86_64Assembler::pushq(const Immediate& imm) { 78 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 79 CHECK(imm.is_int32()); // pushq only supports 32b immediate. 80 if (imm.is_int8()) { 81 EmitUint8(0x6A); 82 EmitUint8(imm.value() & 0xFF); 83 } else { 84 EmitUint8(0x68); 85 EmitImmediate(imm); 86 } 87} 88 89 90void X86_64Assembler::popq(CpuRegister reg) { 91 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 92 EmitOptionalRex32(reg); 93 EmitUint8(0x58 + reg.LowBits()); 94} 95 96 97void X86_64Assembler::popq(const Address& address) { 98 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 99 EmitOptionalRex32(address); 100 EmitUint8(0x8F); 101 EmitOperand(0, address); 102} 103 104 105void X86_64Assembler::movq(CpuRegister dst, const Immediate& imm) { 106 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 107 if (imm.is_int32()) { 108 // 32 bit. Note: sign-extends. 109 EmitRex64(dst); 110 EmitUint8(0xC7); 111 EmitRegisterOperand(0, dst.LowBits()); 112 EmitInt32(static_cast<int32_t>(imm.value())); 113 } else { 114 EmitRex64(dst); 115 EmitUint8(0xB8 + dst.LowBits()); 116 EmitInt64(imm.value()); 117 } 118} 119 120 121void X86_64Assembler::movl(CpuRegister dst, const Immediate& imm) { 122 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 123 EmitOptionalRex32(dst); 124 EmitUint8(0xB8 + dst.LowBits()); 125 EmitImmediate(imm); 126} 127 128 129void X86_64Assembler::movq(CpuRegister dst, CpuRegister src) { 130 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 131 // 0x89 is movq r/m64 <- r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 132 EmitRex64(src, dst); 133 EmitUint8(0x89); 134 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 135} 136 137 138void X86_64Assembler::movl(CpuRegister dst, CpuRegister src) { 139 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 140 EmitOptionalRex32(dst, src); 141 EmitUint8(0x8B); 142 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 143} 144 145 146void X86_64Assembler::movq(CpuRegister dst, const Address& src) { 147 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 148 EmitRex64(dst, src); 149 EmitUint8(0x8B); 150 EmitOperand(dst.LowBits(), src); 151} 152 153 154void X86_64Assembler::movl(CpuRegister dst, const Address& src) { 155 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 156 EmitOptionalRex32(dst, src); 157 EmitUint8(0x8B); 158 EmitOperand(dst.LowBits(), src); 159} 160 161 162void X86_64Assembler::movq(const Address& dst, CpuRegister src) { 163 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 164 EmitRex64(src, dst); 165 EmitUint8(0x89); 166 EmitOperand(src.LowBits(), dst); 167} 168 169 170void X86_64Assembler::movl(const Address& dst, CpuRegister src) { 171 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 172 EmitOptionalRex32(src, dst); 173 EmitUint8(0x89); 174 EmitOperand(src.LowBits(), dst); 175} 176 177void X86_64Assembler::movl(const Address& dst, const Immediate& imm) { 178 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 179 EmitOptionalRex32(dst); 180 EmitUint8(0xC7); 181 EmitOperand(0, dst); 182 EmitImmediate(imm); 183} 184 185void X86_64Assembler::movzxb(CpuRegister dst, CpuRegister src) { 186 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 187 EmitOptionalByteRegNormalizingRex32(dst, src); 188 EmitUint8(0x0F); 189 EmitUint8(0xB6); 190 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 191} 192 193 194void X86_64Assembler::movzxb(CpuRegister dst, const Address& src) { 195 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 196 EmitOptionalByteRegNormalizingRex32(dst, src); 197 EmitUint8(0x0F); 198 EmitUint8(0xB6); 199 EmitOperand(dst.LowBits(), src); 200} 201 202 203void X86_64Assembler::movsxb(CpuRegister dst, CpuRegister src) { 204 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 205 EmitOptionalByteRegNormalizingRex32(dst, src); 206 EmitUint8(0x0F); 207 EmitUint8(0xBE); 208 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 209} 210 211 212void X86_64Assembler::movsxb(CpuRegister dst, const Address& src) { 213 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 214 EmitOptionalByteRegNormalizingRex32(dst, src); 215 EmitUint8(0x0F); 216 EmitUint8(0xBE); 217 EmitOperand(dst.LowBits(), src); 218} 219 220 221void X86_64Assembler::movb(CpuRegister /*dst*/, const Address& /*src*/) { 222 LOG(FATAL) << "Use movzxb or movsxb instead."; 223} 224 225 226void X86_64Assembler::movb(const Address& dst, CpuRegister src) { 227 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 228 EmitOptionalByteRegNormalizingRex32(src, dst); 229 EmitUint8(0x88); 230 EmitOperand(src.LowBits(), dst); 231} 232 233 234void X86_64Assembler::movb(const Address& dst, const Immediate& imm) { 235 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 236 EmitUint8(0xC6); 237 EmitOperand(Register::RAX, dst); 238 CHECK(imm.is_int8()); 239 EmitUint8(imm.value() & 0xFF); 240} 241 242 243void X86_64Assembler::movzxw(CpuRegister dst, CpuRegister src) { 244 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 245 EmitOptionalRex32(dst, src); 246 EmitUint8(0x0F); 247 EmitUint8(0xB7); 248 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 249} 250 251 252void X86_64Assembler::movzxw(CpuRegister dst, const Address& src) { 253 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 254 EmitOptionalRex32(dst, src); 255 EmitUint8(0x0F); 256 EmitUint8(0xB7); 257 EmitOperand(dst.LowBits(), src); 258} 259 260 261void X86_64Assembler::movsxw(CpuRegister dst, CpuRegister src) { 262 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 263 EmitOptionalRex32(dst, src); 264 EmitUint8(0x0F); 265 EmitUint8(0xBF); 266 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 267} 268 269 270void X86_64Assembler::movsxw(CpuRegister dst, const Address& src) { 271 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 272 EmitOptionalRex32(dst, src); 273 EmitUint8(0x0F); 274 EmitUint8(0xBF); 275 EmitOperand(dst.LowBits(), src); 276} 277 278 279void X86_64Assembler::movw(CpuRegister /*dst*/, const Address& /*src*/) { 280 LOG(FATAL) << "Use movzxw or movsxw instead."; 281} 282 283 284void X86_64Assembler::movw(const Address& dst, CpuRegister src) { 285 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 286 EmitOptionalRex32(src, dst); 287 EmitOperandSizeOverride(); 288 EmitUint8(0x89); 289 EmitOperand(src.LowBits(), dst); 290} 291 292 293void X86_64Assembler::leaq(CpuRegister dst, const Address& src) { 294 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 295 EmitRex64(dst, src); 296 EmitUint8(0x8D); 297 EmitOperand(dst.LowBits(), src); 298} 299 300 301void X86_64Assembler::movss(XmmRegister dst, const Address& src) { 302 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 303 EmitUint8(0xF3); 304 EmitOptionalRex32(dst, src); 305 EmitUint8(0x0F); 306 EmitUint8(0x10); 307 EmitOperand(dst.LowBits(), src); 308} 309 310 311void X86_64Assembler::movss(const Address& dst, XmmRegister src) { 312 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 313 EmitUint8(0xF3); 314 EmitOptionalRex32(src, dst); 315 EmitUint8(0x0F); 316 EmitUint8(0x11); 317 EmitOperand(src.LowBits(), dst); 318} 319 320 321void X86_64Assembler::movss(XmmRegister dst, XmmRegister src) { 322 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 323 EmitUint8(0xF3); 324 EmitOptionalRex32(dst, src); 325 EmitUint8(0x0F); 326 EmitUint8(0x11); 327 EmitXmmRegisterOperand(src.LowBits(), dst); 328} 329 330 331void X86_64Assembler::movd(XmmRegister dst, CpuRegister src) { 332 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 333 EmitUint8(0x66); 334 EmitOptionalRex32(dst, src); 335 EmitUint8(0x0F); 336 EmitUint8(0x6E); 337 EmitOperand(dst.LowBits(), Operand(src)); 338} 339 340 341void X86_64Assembler::movd(CpuRegister dst, XmmRegister src) { 342 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 343 EmitUint8(0x66); 344 EmitOptionalRex32(src, dst); 345 EmitUint8(0x0F); 346 EmitUint8(0x7E); 347 EmitOperand(src.LowBits(), Operand(dst)); 348} 349 350 351void X86_64Assembler::addss(XmmRegister dst, XmmRegister src) { 352 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 353 EmitUint8(0xF3); 354 EmitOptionalRex32(dst, src); 355 EmitUint8(0x0F); 356 EmitUint8(0x58); 357 EmitXmmRegisterOperand(dst.LowBits(), src); 358} 359 360 361void X86_64Assembler::addss(XmmRegister dst, const Address& src) { 362 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 363 EmitUint8(0xF3); 364 EmitOptionalRex32(dst, src); 365 EmitUint8(0x0F); 366 EmitUint8(0x58); 367 EmitOperand(dst.LowBits(), src); 368} 369 370 371void X86_64Assembler::subss(XmmRegister dst, XmmRegister src) { 372 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 373 EmitUint8(0xF3); 374 EmitOptionalRex32(dst, src); 375 EmitUint8(0x0F); 376 EmitUint8(0x5C); 377 EmitXmmRegisterOperand(dst.LowBits(), src); 378} 379 380 381void X86_64Assembler::subss(XmmRegister dst, const Address& src) { 382 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 383 EmitUint8(0xF3); 384 EmitOptionalRex32(dst, src); 385 EmitUint8(0x0F); 386 EmitUint8(0x5C); 387 EmitOperand(dst.LowBits(), src); 388} 389 390 391void X86_64Assembler::mulss(XmmRegister dst, XmmRegister src) { 392 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 393 EmitUint8(0xF3); 394 EmitOptionalRex32(dst, src); 395 EmitUint8(0x0F); 396 EmitUint8(0x59); 397 EmitXmmRegisterOperand(dst.LowBits(), src); 398} 399 400 401void X86_64Assembler::mulss(XmmRegister dst, const Address& src) { 402 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 403 EmitUint8(0xF3); 404 EmitOptionalRex32(dst, src); 405 EmitUint8(0x0F); 406 EmitUint8(0x59); 407 EmitOperand(dst.LowBits(), src); 408} 409 410 411void X86_64Assembler::divss(XmmRegister dst, XmmRegister src) { 412 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 413 EmitUint8(0xF3); 414 EmitOptionalRex32(dst, src); 415 EmitUint8(0x0F); 416 EmitUint8(0x5E); 417 EmitXmmRegisterOperand(dst.LowBits(), src); 418} 419 420 421void X86_64Assembler::divss(XmmRegister dst, const Address& src) { 422 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 423 EmitUint8(0xF3); 424 EmitOptionalRex32(dst, src); 425 EmitUint8(0x0F); 426 EmitUint8(0x5E); 427 EmitOperand(dst.LowBits(), src); 428} 429 430 431void X86_64Assembler::flds(const Address& src) { 432 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 433 EmitUint8(0xD9); 434 EmitOperand(0, src); 435} 436 437 438void X86_64Assembler::fstps(const Address& dst) { 439 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 440 EmitUint8(0xD9); 441 EmitOperand(3, dst); 442} 443 444 445void X86_64Assembler::movsd(XmmRegister dst, const Address& src) { 446 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 447 EmitUint8(0xF2); 448 EmitOptionalRex32(dst, src); 449 EmitUint8(0x0F); 450 EmitUint8(0x10); 451 EmitOperand(dst.LowBits(), src); 452} 453 454 455void X86_64Assembler::movsd(const Address& dst, XmmRegister src) { 456 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 457 EmitUint8(0xF2); 458 EmitOptionalRex32(src, dst); 459 EmitUint8(0x0F); 460 EmitUint8(0x11); 461 EmitOperand(src.LowBits(), dst); 462} 463 464 465void X86_64Assembler::movsd(XmmRegister dst, XmmRegister src) { 466 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 467 EmitUint8(0xF2); 468 EmitOptionalRex32(dst, src); 469 EmitUint8(0x0F); 470 EmitUint8(0x11); 471 EmitXmmRegisterOperand(src.LowBits(), dst); 472} 473 474 475void X86_64Assembler::addsd(XmmRegister dst, XmmRegister src) { 476 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 477 EmitUint8(0xF2); 478 EmitOptionalRex32(dst, src); 479 EmitUint8(0x0F); 480 EmitUint8(0x58); 481 EmitXmmRegisterOperand(dst.LowBits(), src); 482} 483 484 485void X86_64Assembler::addsd(XmmRegister dst, const Address& src) { 486 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 487 EmitUint8(0xF2); 488 EmitOptionalRex32(dst, src); 489 EmitUint8(0x0F); 490 EmitUint8(0x58); 491 EmitOperand(dst.LowBits(), src); 492} 493 494 495void X86_64Assembler::subsd(XmmRegister dst, XmmRegister src) { 496 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 497 EmitUint8(0xF2); 498 EmitOptionalRex32(dst, src); 499 EmitUint8(0x0F); 500 EmitUint8(0x5C); 501 EmitXmmRegisterOperand(dst.LowBits(), src); 502} 503 504 505void X86_64Assembler::subsd(XmmRegister dst, const Address& src) { 506 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 507 EmitUint8(0xF2); 508 EmitOptionalRex32(dst, src); 509 EmitUint8(0x0F); 510 EmitUint8(0x5C); 511 EmitOperand(dst.LowBits(), src); 512} 513 514 515void X86_64Assembler::mulsd(XmmRegister dst, XmmRegister src) { 516 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 517 EmitUint8(0xF2); 518 EmitOptionalRex32(dst, src); 519 EmitUint8(0x0F); 520 EmitUint8(0x59); 521 EmitXmmRegisterOperand(dst.LowBits(), src); 522} 523 524 525void X86_64Assembler::mulsd(XmmRegister dst, const Address& src) { 526 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 527 EmitUint8(0xF2); 528 EmitOptionalRex32(dst, src); 529 EmitUint8(0x0F); 530 EmitUint8(0x59); 531 EmitOperand(dst.LowBits(), src); 532} 533 534 535void X86_64Assembler::divsd(XmmRegister dst, XmmRegister src) { 536 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 537 EmitUint8(0xF2); 538 EmitOptionalRex32(dst, src); 539 EmitUint8(0x0F); 540 EmitUint8(0x5E); 541 EmitXmmRegisterOperand(dst.LowBits(), src); 542} 543 544 545void X86_64Assembler::divsd(XmmRegister dst, const Address& src) { 546 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 547 EmitUint8(0xF2); 548 EmitOptionalRex32(dst, src); 549 EmitUint8(0x0F); 550 EmitUint8(0x5E); 551 EmitOperand(dst.LowBits(), src); 552} 553 554 555void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) { 556 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 557 EmitUint8(0xF3); 558 EmitOptionalRex32(dst, src); 559 EmitUint8(0x0F); 560 EmitUint8(0x2A); 561 EmitOperand(dst.LowBits(), Operand(src)); 562} 563 564 565void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src) { 566 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 567 EmitUint8(0xF2); 568 EmitOptionalRex32(dst, src); 569 EmitUint8(0x0F); 570 EmitUint8(0x2A); 571 EmitOperand(dst.LowBits(), Operand(src)); 572} 573 574 575void X86_64Assembler::cvtss2si(CpuRegister dst, XmmRegister src) { 576 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 577 EmitUint8(0xF3); 578 EmitOptionalRex32(dst, src); 579 EmitUint8(0x0F); 580 EmitUint8(0x2D); 581 EmitXmmRegisterOperand(dst.LowBits(), src); 582} 583 584 585void X86_64Assembler::cvtss2sd(XmmRegister dst, XmmRegister src) { 586 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 587 EmitUint8(0xF3); 588 EmitOptionalRex32(dst, src); 589 EmitUint8(0x0F); 590 EmitUint8(0x5A); 591 EmitXmmRegisterOperand(dst.LowBits(), src); 592} 593 594 595void X86_64Assembler::cvtsd2si(CpuRegister dst, XmmRegister src) { 596 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 597 EmitUint8(0xF2); 598 EmitOptionalRex32(dst, src); 599 EmitUint8(0x0F); 600 EmitUint8(0x2D); 601 EmitXmmRegisterOperand(dst.LowBits(), src); 602} 603 604 605void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src) { 606 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 607 EmitUint8(0xF3); 608 EmitOptionalRex32(dst, src); 609 EmitUint8(0x0F); 610 EmitUint8(0x2C); 611 EmitXmmRegisterOperand(dst.LowBits(), src); 612} 613 614 615void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src) { 616 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 617 EmitUint8(0xF2); 618 EmitOptionalRex32(dst, src); 619 EmitUint8(0x0F); 620 EmitUint8(0x2C); 621 EmitXmmRegisterOperand(dst.LowBits(), src); 622} 623 624 625void X86_64Assembler::cvtsd2ss(XmmRegister dst, XmmRegister src) { 626 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 627 EmitUint8(0xF2); 628 EmitOptionalRex32(dst, src); 629 EmitUint8(0x0F); 630 EmitUint8(0x5A); 631 EmitXmmRegisterOperand(dst.LowBits(), src); 632} 633 634 635void X86_64Assembler::cvtdq2pd(XmmRegister dst, XmmRegister src) { 636 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 637 EmitUint8(0xF3); 638 EmitOptionalRex32(dst, src); 639 EmitUint8(0x0F); 640 EmitUint8(0xE6); 641 EmitXmmRegisterOperand(dst.LowBits(), src); 642} 643 644 645void X86_64Assembler::comiss(XmmRegister a, XmmRegister b) { 646 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 647 EmitOptionalRex32(a, b); 648 EmitUint8(0x0F); 649 EmitUint8(0x2F); 650 EmitXmmRegisterOperand(a.LowBits(), b); 651} 652 653 654void X86_64Assembler::comisd(XmmRegister a, XmmRegister b) { 655 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 656 EmitUint8(0x66); 657 EmitOptionalRex32(a, b); 658 EmitUint8(0x0F); 659 EmitUint8(0x2F); 660 EmitXmmRegisterOperand(a.LowBits(), b); 661} 662 663 664void X86_64Assembler::sqrtsd(XmmRegister dst, XmmRegister src) { 665 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 666 EmitUint8(0xF2); 667 EmitOptionalRex32(dst, src); 668 EmitUint8(0x0F); 669 EmitUint8(0x51); 670 EmitXmmRegisterOperand(dst.LowBits(), src); 671} 672 673 674void X86_64Assembler::sqrtss(XmmRegister dst, XmmRegister src) { 675 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 676 EmitUint8(0xF3); 677 EmitOptionalRex32(dst, src); 678 EmitUint8(0x0F); 679 EmitUint8(0x51); 680 EmitXmmRegisterOperand(dst.LowBits(), src); 681} 682 683 684void X86_64Assembler::xorpd(XmmRegister dst, const Address& src) { 685 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 686 EmitUint8(0x66); 687 EmitOptionalRex32(dst, src); 688 EmitUint8(0x0F); 689 EmitUint8(0x57); 690 EmitOperand(dst.LowBits(), src); 691} 692 693 694void X86_64Assembler::xorpd(XmmRegister dst, XmmRegister src) { 695 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 696 EmitUint8(0x66); 697 EmitOptionalRex32(dst, src); 698 EmitUint8(0x0F); 699 EmitUint8(0x57); 700 EmitXmmRegisterOperand(dst.LowBits(), src); 701} 702 703 704void X86_64Assembler::xorps(XmmRegister dst, const Address& src) { 705 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 706 EmitOptionalRex32(dst, src); 707 EmitUint8(0x0F); 708 EmitUint8(0x57); 709 EmitOperand(dst.LowBits(), src); 710} 711 712 713void X86_64Assembler::xorps(XmmRegister dst, XmmRegister src) { 714 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 715 EmitOptionalRex32(dst, src); 716 EmitUint8(0x0F); 717 EmitUint8(0x57); 718 EmitXmmRegisterOperand(dst.LowBits(), src); 719} 720 721 722void X86_64Assembler::andpd(XmmRegister dst, const Address& src) { 723 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 724 EmitUint8(0x66); 725 EmitOptionalRex32(dst, src); 726 EmitUint8(0x0F); 727 EmitUint8(0x54); 728 EmitOperand(dst.LowBits(), src); 729} 730 731 732void X86_64Assembler::fldl(const Address& src) { 733 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 734 EmitUint8(0xDD); 735 EmitOperand(0, src); 736} 737 738 739void X86_64Assembler::fstpl(const Address& dst) { 740 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 741 EmitUint8(0xDD); 742 EmitOperand(3, dst); 743} 744 745 746void X86_64Assembler::fnstcw(const Address& dst) { 747 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 748 EmitUint8(0xD9); 749 EmitOperand(7, dst); 750} 751 752 753void X86_64Assembler::fldcw(const Address& src) { 754 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 755 EmitUint8(0xD9); 756 EmitOperand(5, src); 757} 758 759 760void X86_64Assembler::fistpl(const Address& dst) { 761 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 762 EmitUint8(0xDF); 763 EmitOperand(7, dst); 764} 765 766 767void X86_64Assembler::fistps(const Address& dst) { 768 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 769 EmitUint8(0xDB); 770 EmitOperand(3, dst); 771} 772 773 774void X86_64Assembler::fildl(const Address& src) { 775 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 776 EmitUint8(0xDF); 777 EmitOperand(5, src); 778} 779 780 781void X86_64Assembler::fincstp() { 782 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 783 EmitUint8(0xD9); 784 EmitUint8(0xF7); 785} 786 787 788void X86_64Assembler::ffree(const Immediate& index) { 789 CHECK_LT(index.value(), 7); 790 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 791 EmitUint8(0xDD); 792 EmitUint8(0xC0 + index.value()); 793} 794 795 796void X86_64Assembler::fsin() { 797 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 798 EmitUint8(0xD9); 799 EmitUint8(0xFE); 800} 801 802 803void X86_64Assembler::fcos() { 804 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 805 EmitUint8(0xD9); 806 EmitUint8(0xFF); 807} 808 809 810void X86_64Assembler::fptan() { 811 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 812 EmitUint8(0xD9); 813 EmitUint8(0xF2); 814} 815 816 817void X86_64Assembler::xchgl(CpuRegister dst, CpuRegister src) { 818 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 819 EmitOptionalRex32(dst, src); 820 EmitUint8(0x87); 821 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 822} 823 824 825void X86_64Assembler::xchgq(CpuRegister dst, CpuRegister src) { 826 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 827 EmitRex64(dst, src); 828 EmitUint8(0x87); 829 EmitOperand(dst.LowBits(), Operand(src)); 830} 831 832 833void X86_64Assembler::xchgl(CpuRegister reg, const Address& address) { 834 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 835 EmitOptionalRex32(reg, address); 836 EmitUint8(0x87); 837 EmitOperand(reg.LowBits(), address); 838} 839 840 841void X86_64Assembler::cmpl(CpuRegister reg, const Immediate& imm) { 842 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 843 EmitOptionalRex32(reg); 844 EmitComplex(7, Operand(reg), imm); 845} 846 847 848void X86_64Assembler::cmpl(CpuRegister reg0, CpuRegister reg1) { 849 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 850 EmitOptionalRex32(reg0, reg1); 851 EmitUint8(0x3B); 852 EmitOperand(reg0.LowBits(), Operand(reg1)); 853} 854 855 856void X86_64Assembler::cmpl(CpuRegister reg, const Address& address) { 857 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 858 EmitOptionalRex32(reg, address); 859 EmitUint8(0x3B); 860 EmitOperand(reg.LowBits(), address); 861} 862 863 864void X86_64Assembler::cmpq(CpuRegister reg0, CpuRegister reg1) { 865 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 866 EmitRex64(reg0, reg1); 867 EmitUint8(0x3B); 868 EmitOperand(reg0.LowBits(), Operand(reg1)); 869} 870 871 872void X86_64Assembler::cmpq(CpuRegister reg, const Immediate& imm) { 873 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 874 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 875 EmitRex64(reg); 876 EmitComplex(7, Operand(reg), imm); 877} 878 879 880void X86_64Assembler::cmpq(CpuRegister reg, const Address& address) { 881 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 882 EmitRex64(reg); 883 EmitUint8(0x3B); 884 EmitOperand(reg.LowBits(), address); 885} 886 887 888void X86_64Assembler::addl(CpuRegister dst, CpuRegister src) { 889 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 890 EmitOptionalRex32(dst, src); 891 EmitUint8(0x03); 892 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 893} 894 895 896void X86_64Assembler::addl(CpuRegister reg, const Address& address) { 897 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 898 EmitOptionalRex32(reg, address); 899 EmitUint8(0x03); 900 EmitOperand(reg.LowBits(), address); 901} 902 903 904void X86_64Assembler::cmpl(const Address& address, CpuRegister reg) { 905 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 906 EmitOptionalRex32(reg, address); 907 EmitUint8(0x39); 908 EmitOperand(reg.LowBits(), address); 909} 910 911 912void X86_64Assembler::cmpl(const Address& address, const Immediate& imm) { 913 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 914 EmitOptionalRex32(address); 915 EmitComplex(7, address, imm); 916} 917 918 919void X86_64Assembler::testl(CpuRegister reg1, CpuRegister reg2) { 920 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 921 EmitOptionalRex32(reg1, reg2); 922 EmitUint8(0x85); 923 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 924} 925 926 927void X86_64Assembler::testl(CpuRegister reg, const Immediate& immediate) { 928 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 929 // For registers that have a byte variant (RAX, RBX, RCX, and RDX) 930 // we only test the byte CpuRegister to keep the encoding short. 931 if (immediate.is_uint8() && reg.AsRegister() < 4) { 932 // Use zero-extended 8-bit immediate. 933 if (reg.AsRegister() == RAX) { 934 EmitUint8(0xA8); 935 } else { 936 EmitUint8(0xF6); 937 EmitUint8(0xC0 + reg.AsRegister()); 938 } 939 EmitUint8(immediate.value() & 0xFF); 940 } else if (reg.AsRegister() == RAX) { 941 // Use short form if the destination is RAX. 942 EmitUint8(0xA9); 943 EmitImmediate(immediate); 944 } else { 945 EmitOptionalRex32(reg); 946 EmitUint8(0xF7); 947 EmitOperand(0, Operand(reg)); 948 EmitImmediate(immediate); 949 } 950} 951 952 953void X86_64Assembler::andl(CpuRegister dst, CpuRegister src) { 954 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 955 EmitOptionalRex32(dst, src); 956 EmitUint8(0x23); 957 EmitOperand(dst.LowBits(), Operand(src)); 958} 959 960 961void X86_64Assembler::andl(CpuRegister dst, const Immediate& imm) { 962 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 963 EmitOptionalRex32(dst); 964 EmitComplex(4, Operand(dst), imm); 965} 966 967 968void X86_64Assembler::andq(CpuRegister reg, const Immediate& imm) { 969 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 970 CHECK(imm.is_int32()); // andq only supports 32b immediate. 971 EmitRex64(reg); 972 EmitComplex(4, Operand(reg), imm); 973} 974 975 976void X86_64Assembler::orl(CpuRegister dst, CpuRegister src) { 977 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 978 EmitOptionalRex32(dst, src); 979 EmitUint8(0x0B); 980 EmitOperand(dst.LowBits(), Operand(src)); 981} 982 983 984void X86_64Assembler::orl(CpuRegister dst, const Immediate& imm) { 985 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 986 EmitOptionalRex32(dst); 987 EmitComplex(1, Operand(dst), imm); 988} 989 990 991void X86_64Assembler::xorl(CpuRegister dst, CpuRegister src) { 992 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 993 EmitOptionalRex32(dst, src); 994 EmitUint8(0x33); 995 EmitOperand(dst.LowBits(), Operand(src)); 996} 997 998 999void X86_64Assembler::xorq(CpuRegister dst, CpuRegister src) { 1000 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1001 EmitRex64(dst, src); 1002 EmitUint8(0x33); 1003 EmitOperand(dst.LowBits(), Operand(src)); 1004} 1005 1006 1007void X86_64Assembler::xorq(CpuRegister dst, const Immediate& imm) { 1008 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1009 CHECK(imm.is_int32()); // xorq only supports 32b immediate. 1010 EmitRex64(dst); 1011 EmitComplex(6, Operand(dst), imm); 1012} 1013 1014#if 0 1015void X86_64Assembler::rex(bool force, bool w, Register* r, Register* x, Register* b) { 1016 // REX.WRXB 1017 // W - 64-bit operand 1018 // R - MODRM.reg 1019 // X - SIB.index 1020 // B - MODRM.rm/SIB.base 1021 uint8_t rex = force ? 0x40 : 0; 1022 if (w) { 1023 rex |= 0x48; // REX.W000 1024 } 1025 if (r != nullptr && *r >= Register::R8 && *r < Register::kNumberOfCpuRegisters) { 1026 rex |= 0x44; // REX.0R00 1027 *r = static_cast<Register>(*r - 8); 1028 } 1029 if (x != nullptr && *x >= Register::R8 && *x < Register::kNumberOfCpuRegisters) { 1030 rex |= 0x42; // REX.00X0 1031 *x = static_cast<Register>(*x - 8); 1032 } 1033 if (b != nullptr && *b >= Register::R8 && *b < Register::kNumberOfCpuRegisters) { 1034 rex |= 0x41; // REX.000B 1035 *b = static_cast<Register>(*b - 8); 1036 } 1037 if (rex != 0) { 1038 EmitUint8(rex); 1039 } 1040} 1041 1042void X86_64Assembler::rex_reg_mem(bool force, bool w, Register* dst, const Address& mem) { 1043 // REX.WRXB 1044 // W - 64-bit operand 1045 // R - MODRM.reg 1046 // X - SIB.index 1047 // B - MODRM.rm/SIB.base 1048 uint8_t rex = mem->rex(); 1049 if (force) { 1050 rex |= 0x40; // REX.0000 1051 } 1052 if (w) { 1053 rex |= 0x48; // REX.W000 1054 } 1055 if (dst != nullptr && *dst >= Register::R8 && *dst < Register::kNumberOfCpuRegisters) { 1056 rex |= 0x44; // REX.0R00 1057 *dst = static_cast<Register>(*dst - 8); 1058 } 1059 if (rex != 0) { 1060 EmitUint8(rex); 1061 } 1062} 1063 1064void rex_mem_reg(bool force, bool w, Address* mem, Register* src); 1065#endif 1066 1067void X86_64Assembler::addl(CpuRegister reg, const Immediate& imm) { 1068 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1069 EmitOptionalRex32(reg); 1070 EmitComplex(0, Operand(reg), imm); 1071} 1072 1073 1074void X86_64Assembler::addq(CpuRegister reg, const Immediate& imm) { 1075 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1076 CHECK(imm.is_int32()); // addq only supports 32b immediate. 1077 EmitRex64(reg); 1078 EmitComplex(0, Operand(reg), imm); 1079} 1080 1081 1082void X86_64Assembler::addq(CpuRegister dst, const Address& address) { 1083 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1084 EmitRex64(dst); 1085 EmitUint8(0x03); 1086 EmitOperand(dst.LowBits(), address); 1087} 1088 1089 1090void X86_64Assembler::addq(CpuRegister dst, CpuRegister src) { 1091 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1092 // 0x01 is addq r/m64 <- r/m64 + r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 1093 EmitRex64(src, dst); 1094 EmitUint8(0x01); 1095 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1096} 1097 1098 1099void X86_64Assembler::addl(const Address& address, CpuRegister reg) { 1100 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1101 EmitOptionalRex32(reg, address); 1102 EmitUint8(0x01); 1103 EmitOperand(reg.LowBits(), address); 1104} 1105 1106 1107void X86_64Assembler::addl(const Address& address, const Immediate& imm) { 1108 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1109 EmitOptionalRex32(address); 1110 EmitComplex(0, address, imm); 1111} 1112 1113 1114void X86_64Assembler::subl(CpuRegister dst, CpuRegister src) { 1115 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1116 EmitOptionalRex32(dst, src); 1117 EmitUint8(0x2B); 1118 EmitOperand(dst.LowBits(), Operand(src)); 1119} 1120 1121 1122void X86_64Assembler::subl(CpuRegister reg, const Immediate& imm) { 1123 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1124 EmitOptionalRex32(reg); 1125 EmitComplex(5, Operand(reg), imm); 1126} 1127 1128 1129void X86_64Assembler::subq(CpuRegister reg, const Immediate& imm) { 1130 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1131 CHECK(imm.is_int32()); // subq only supports 32b immediate. 1132 EmitRex64(reg); 1133 EmitComplex(5, Operand(reg), imm); 1134} 1135 1136 1137void X86_64Assembler::subq(CpuRegister dst, CpuRegister src) { 1138 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1139 EmitRex64(dst, src); 1140 EmitUint8(0x2B); 1141 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1142} 1143 1144 1145void X86_64Assembler::subq(CpuRegister reg, const Address& address) { 1146 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1147 EmitRex64(reg); 1148 EmitUint8(0x2B); 1149 EmitOperand(reg.LowBits() & 7, address); 1150} 1151 1152 1153void X86_64Assembler::subl(CpuRegister reg, const Address& address) { 1154 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1155 EmitOptionalRex32(reg, address); 1156 EmitUint8(0x2B); 1157 EmitOperand(reg.LowBits(), address); 1158} 1159 1160 1161void X86_64Assembler::cdq() { 1162 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1163 EmitUint8(0x99); 1164} 1165 1166 1167void X86_64Assembler::idivl(CpuRegister reg) { 1168 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1169 EmitOptionalRex32(reg); 1170 EmitUint8(0xF7); 1171 EmitUint8(0xF8 | reg.LowBits()); 1172} 1173 1174 1175void X86_64Assembler::imull(CpuRegister dst, CpuRegister src) { 1176 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1177 EmitOptionalRex32(dst, src); 1178 EmitUint8(0x0F); 1179 EmitUint8(0xAF); 1180 EmitOperand(dst.LowBits(), Operand(src)); 1181} 1182 1183 1184void X86_64Assembler::imull(CpuRegister reg, const Immediate& imm) { 1185 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1186 EmitOptionalRex32(reg); 1187 EmitUint8(0x69); 1188 EmitOperand(reg.LowBits(), Operand(reg)); 1189 EmitImmediate(imm); 1190} 1191 1192 1193void X86_64Assembler::imull(CpuRegister reg, const Address& address) { 1194 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1195 EmitOptionalRex32(reg, address); 1196 EmitUint8(0x0F); 1197 EmitUint8(0xAF); 1198 EmitOperand(reg.LowBits(), address); 1199} 1200 1201 1202void X86_64Assembler::imull(CpuRegister reg) { 1203 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1204 EmitOptionalRex32(reg); 1205 EmitUint8(0xF7); 1206 EmitOperand(5, Operand(reg)); 1207} 1208 1209 1210void X86_64Assembler::imull(const Address& address) { 1211 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1212 EmitOptionalRex32(address); 1213 EmitUint8(0xF7); 1214 EmitOperand(5, address); 1215} 1216 1217 1218void X86_64Assembler::mull(CpuRegister reg) { 1219 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1220 EmitOptionalRex32(reg); 1221 EmitUint8(0xF7); 1222 EmitOperand(4, Operand(reg)); 1223} 1224 1225 1226void X86_64Assembler::mull(const Address& address) { 1227 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1228 EmitOptionalRex32(address); 1229 EmitUint8(0xF7); 1230 EmitOperand(4, address); 1231} 1232 1233 1234 1235void X86_64Assembler::shll(CpuRegister reg, const Immediate& imm) { 1236 EmitGenericShift(4, reg, imm); 1237} 1238 1239 1240void X86_64Assembler::shll(CpuRegister operand, CpuRegister shifter) { 1241 EmitGenericShift(4, operand, shifter); 1242} 1243 1244 1245void X86_64Assembler::shrl(CpuRegister reg, const Immediate& imm) { 1246 EmitGenericShift(5, reg, imm); 1247} 1248 1249 1250void X86_64Assembler::shrl(CpuRegister operand, CpuRegister shifter) { 1251 EmitGenericShift(5, operand, shifter); 1252} 1253 1254 1255void X86_64Assembler::sarl(CpuRegister reg, const Immediate& imm) { 1256 EmitGenericShift(7, reg, imm); 1257} 1258 1259 1260void X86_64Assembler::sarl(CpuRegister operand, CpuRegister shifter) { 1261 EmitGenericShift(7, operand, shifter); 1262} 1263 1264 1265void X86_64Assembler::negl(CpuRegister reg) { 1266 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1267 EmitOptionalRex32(reg); 1268 EmitUint8(0xF7); 1269 EmitOperand(3, Operand(reg)); 1270} 1271 1272 1273void X86_64Assembler::notl(CpuRegister reg) { 1274 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1275 EmitOptionalRex32(reg); 1276 EmitUint8(0xF7); 1277 EmitUint8(0xD0 | reg.LowBits()); 1278} 1279 1280 1281void X86_64Assembler::enter(const Immediate& imm) { 1282 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1283 EmitUint8(0xC8); 1284 CHECK(imm.is_uint16()); 1285 EmitUint8(imm.value() & 0xFF); 1286 EmitUint8((imm.value() >> 8) & 0xFF); 1287 EmitUint8(0x00); 1288} 1289 1290 1291void X86_64Assembler::leave() { 1292 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1293 EmitUint8(0xC9); 1294} 1295 1296 1297void X86_64Assembler::ret() { 1298 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1299 EmitUint8(0xC3); 1300} 1301 1302 1303void X86_64Assembler::ret(const Immediate& imm) { 1304 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1305 EmitUint8(0xC2); 1306 CHECK(imm.is_uint16()); 1307 EmitUint8(imm.value() & 0xFF); 1308 EmitUint8((imm.value() >> 8) & 0xFF); 1309} 1310 1311 1312 1313void X86_64Assembler::nop() { 1314 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1315 EmitUint8(0x90); 1316} 1317 1318 1319void X86_64Assembler::int3() { 1320 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1321 EmitUint8(0xCC); 1322} 1323 1324 1325void X86_64Assembler::hlt() { 1326 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1327 EmitUint8(0xF4); 1328} 1329 1330 1331void X86_64Assembler::j(Condition condition, Label* label) { 1332 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1333 if (label->IsBound()) { 1334 static const int kShortSize = 2; 1335 static const int kLongSize = 6; 1336 int offset = label->Position() - buffer_.Size(); 1337 CHECK_LE(offset, 0); 1338 if (IsInt(8, offset - kShortSize)) { 1339 EmitUint8(0x70 + condition); 1340 EmitUint8((offset - kShortSize) & 0xFF); 1341 } else { 1342 EmitUint8(0x0F); 1343 EmitUint8(0x80 + condition); 1344 EmitInt32(offset - kLongSize); 1345 } 1346 } else { 1347 EmitUint8(0x0F); 1348 EmitUint8(0x80 + condition); 1349 EmitLabelLink(label); 1350 } 1351} 1352 1353 1354void X86_64Assembler::jmp(CpuRegister reg) { 1355 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1356 EmitOptionalRex32(reg); 1357 EmitUint8(0xFF); 1358 EmitRegisterOperand(4, reg.LowBits()); 1359} 1360 1361void X86_64Assembler::jmp(const Address& address) { 1362 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1363 EmitOptionalRex32(address); 1364 EmitUint8(0xFF); 1365 EmitOperand(4, address); 1366} 1367 1368void X86_64Assembler::jmp(Label* label) { 1369 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1370 if (label->IsBound()) { 1371 static const int kShortSize = 2; 1372 static const int kLongSize = 5; 1373 int offset = label->Position() - buffer_.Size(); 1374 CHECK_LE(offset, 0); 1375 if (IsInt(8, offset - kShortSize)) { 1376 EmitUint8(0xEB); 1377 EmitUint8((offset - kShortSize) & 0xFF); 1378 } else { 1379 EmitUint8(0xE9); 1380 EmitInt32(offset - kLongSize); 1381 } 1382 } else { 1383 EmitUint8(0xE9); 1384 EmitLabelLink(label); 1385 } 1386} 1387 1388 1389X86_64Assembler* X86_64Assembler::lock() { 1390 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1391 EmitUint8(0xF0); 1392 return this; 1393} 1394 1395 1396void X86_64Assembler::cmpxchgl(const Address& address, CpuRegister reg) { 1397 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1398 EmitUint8(0x0F); 1399 EmitUint8(0xB1); 1400 EmitOperand(reg.LowBits(), address); 1401} 1402 1403void X86_64Assembler::mfence() { 1404 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1405 EmitUint8(0x0F); 1406 EmitUint8(0xAE); 1407 EmitUint8(0xF0); 1408} 1409 1410 1411X86_64Assembler* X86_64Assembler::gs() { 1412 // TODO: gs is a prefix and not an instruction 1413 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1414 EmitUint8(0x65); 1415 return this; 1416} 1417 1418 1419void X86_64Assembler::AddImmediate(CpuRegister reg, const Immediate& imm) { 1420 int value = imm.value(); 1421 if (value != 0) { 1422 if (value > 0) { 1423 addl(reg, imm); 1424 } else { 1425 subl(reg, Immediate(value)); 1426 } 1427 } 1428} 1429 1430 1431void X86_64Assembler::setcc(Condition condition, CpuRegister dst) { 1432 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1433 // RSP, RBP, RDI, RSI need rex prefix (else the pattern encodes ah/bh/ch/dh). 1434 if (dst.NeedsRex() || dst.AsRegister() > 3) { 1435 EmitOptionalRex(true, false, false, false, dst.NeedsRex()); 1436 } 1437 EmitUint8(0x0F); 1438 EmitUint8(0x90 + condition); 1439 EmitUint8(0xC0 + dst.LowBits()); 1440} 1441 1442 1443void X86_64Assembler::LoadDoubleConstant(XmmRegister dst, double value) { 1444 // TODO: Need to have a code constants table. 1445 int64_t constant = bit_cast<int64_t, double>(value); 1446 pushq(Immediate(High32Bits(constant))); 1447 pushq(Immediate(Low32Bits(constant))); 1448 movsd(dst, Address(CpuRegister(RSP), 0)); 1449 addq(CpuRegister(RSP), Immediate(2 * kWordSize)); 1450} 1451 1452 1453void X86_64Assembler::FloatNegate(XmmRegister f) { 1454 static const struct { 1455 uint32_t a; 1456 uint32_t b; 1457 uint32_t c; 1458 uint32_t d; 1459 } float_negate_constant __attribute__((aligned(16))) = 1460 { 0x80000000, 0x00000000, 0x80000000, 0x00000000 }; 1461 xorps(f, Address::Absolute(reinterpret_cast<uword>(&float_negate_constant))); 1462} 1463 1464 1465void X86_64Assembler::DoubleNegate(XmmRegister d) { 1466 static const struct { 1467 uint64_t a; 1468 uint64_t b; 1469 } double_negate_constant __attribute__((aligned(16))) = 1470 {0x8000000000000000LL, 0x8000000000000000LL}; 1471 xorpd(d, Address::Absolute(reinterpret_cast<uword>(&double_negate_constant))); 1472} 1473 1474 1475void X86_64Assembler::DoubleAbs(XmmRegister reg) { 1476 static const struct { 1477 uint64_t a; 1478 uint64_t b; 1479 } double_abs_constant __attribute__((aligned(16))) = 1480 {0x7FFFFFFFFFFFFFFFLL, 0x7FFFFFFFFFFFFFFFLL}; 1481 andpd(reg, Address::Absolute(reinterpret_cast<uword>(&double_abs_constant))); 1482} 1483 1484 1485void X86_64Assembler::Align(int alignment, int offset) { 1486 CHECK(IsPowerOfTwo(alignment)); 1487 // Emit nop instruction until the real position is aligned. 1488 while (((offset + buffer_.GetPosition()) & (alignment-1)) != 0) { 1489 nop(); 1490 } 1491} 1492 1493 1494void X86_64Assembler::Bind(Label* label) { 1495 int bound = buffer_.Size(); 1496 CHECK(!label->IsBound()); // Labels can only be bound once. 1497 while (label->IsLinked()) { 1498 int position = label->LinkPosition(); 1499 int next = buffer_.Load<int32_t>(position); 1500 buffer_.Store<int32_t>(position, bound - (position + 4)); 1501 label->position_ = next; 1502 } 1503 label->BindTo(bound); 1504} 1505 1506 1507void X86_64Assembler::EmitOperand(uint8_t reg_or_opcode, const Operand& operand) { 1508 CHECK_GE(reg_or_opcode, 0); 1509 CHECK_LT(reg_or_opcode, 8); 1510 const int length = operand.length_; 1511 CHECK_GT(length, 0); 1512 // Emit the ModRM byte updated with the given reg value. 1513 CHECK_EQ(operand.encoding_[0] & 0x38, 0); 1514 EmitUint8(operand.encoding_[0] + (reg_or_opcode << 3)); 1515 // Emit the rest of the encoded operand. 1516 for (int i = 1; i < length; i++) { 1517 EmitUint8(operand.encoding_[i]); 1518 } 1519} 1520 1521 1522void X86_64Assembler::EmitImmediate(const Immediate& imm) { 1523 if (imm.is_int32()) { 1524 EmitInt32(static_cast<int32_t>(imm.value())); 1525 } else { 1526 EmitInt64(imm.value()); 1527 } 1528} 1529 1530 1531void X86_64Assembler::EmitComplex(uint8_t reg_or_opcode, 1532 const Operand& operand, 1533 const Immediate& immediate) { 1534 CHECK_GE(reg_or_opcode, 0); 1535 CHECK_LT(reg_or_opcode, 8); 1536 if (immediate.is_int8()) { 1537 // Use sign-extended 8-bit immediate. 1538 EmitUint8(0x83); 1539 EmitOperand(reg_or_opcode, operand); 1540 EmitUint8(immediate.value() & 0xFF); 1541 } else if (operand.IsRegister(CpuRegister(RAX))) { 1542 // Use short form if the destination is eax. 1543 EmitUint8(0x05 + (reg_or_opcode << 3)); 1544 EmitImmediate(immediate); 1545 } else { 1546 EmitUint8(0x81); 1547 EmitOperand(reg_or_opcode, operand); 1548 EmitImmediate(immediate); 1549 } 1550} 1551 1552 1553void X86_64Assembler::EmitLabel(Label* label, int instruction_size) { 1554 if (label->IsBound()) { 1555 int offset = label->Position() - buffer_.Size(); 1556 CHECK_LE(offset, 0); 1557 EmitInt32(offset - instruction_size); 1558 } else { 1559 EmitLabelLink(label); 1560 } 1561} 1562 1563 1564void X86_64Assembler::EmitLabelLink(Label* label) { 1565 CHECK(!label->IsBound()); 1566 int position = buffer_.Size(); 1567 EmitInt32(label->position_); 1568 label->LinkTo(position); 1569} 1570 1571 1572void X86_64Assembler::EmitGenericShift(int reg_or_opcode, 1573 CpuRegister reg, 1574 const Immediate& imm) { 1575 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1576 CHECK(imm.is_int8()); 1577 if (imm.value() == 1) { 1578 EmitUint8(0xD1); 1579 EmitOperand(reg_or_opcode, Operand(reg)); 1580 } else { 1581 EmitUint8(0xC1); 1582 EmitOperand(reg_or_opcode, Operand(reg)); 1583 EmitUint8(imm.value() & 0xFF); 1584 } 1585} 1586 1587 1588void X86_64Assembler::EmitGenericShift(int reg_or_opcode, 1589 CpuRegister operand, 1590 CpuRegister shifter) { 1591 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1592 CHECK_EQ(shifter.AsRegister(), RCX); 1593 EmitUint8(0xD3); 1594 EmitOperand(reg_or_opcode, Operand(operand)); 1595} 1596 1597void X86_64Assembler::EmitOptionalRex(bool force, bool w, bool r, bool x, bool b) { 1598 // REX.WRXB 1599 // W - 64-bit operand 1600 // R - MODRM.reg 1601 // X - SIB.index 1602 // B - MODRM.rm/SIB.base 1603 uint8_t rex = force ? 0x40 : 0; 1604 if (w) { 1605 rex |= 0x48; // REX.W000 1606 } 1607 if (r) { 1608 rex |= 0x44; // REX.0R00 1609 } 1610 if (x) { 1611 rex |= 0x42; // REX.00X0 1612 } 1613 if (b) { 1614 rex |= 0x41; // REX.000B 1615 } 1616 if (rex != 0) { 1617 EmitUint8(rex); 1618 } 1619} 1620 1621void X86_64Assembler::EmitOptionalRex32(CpuRegister reg) { 1622 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 1623} 1624 1625void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, CpuRegister src) { 1626 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1627} 1628 1629void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, XmmRegister src) { 1630 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1631} 1632 1633void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, XmmRegister src) { 1634 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1635} 1636 1637void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, CpuRegister src) { 1638 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1639} 1640 1641void X86_64Assembler::EmitOptionalRex32(const Operand& operand) { 1642 uint8_t rex = operand.rex(); 1643 if (rex != 0) { 1644 EmitUint8(rex); 1645 } 1646} 1647 1648void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, const Operand& operand) { 1649 uint8_t rex = operand.rex(); 1650 if (dst.NeedsRex()) { 1651 rex |= 0x44; // REX.0R00 1652 } 1653 if (rex != 0) { 1654 EmitUint8(rex); 1655 } 1656} 1657 1658void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, const Operand& operand) { 1659 uint8_t rex = operand.rex(); 1660 if (dst.NeedsRex()) { 1661 rex |= 0x44; // REX.0R00 1662 } 1663 if (rex != 0) { 1664 EmitUint8(rex); 1665 } 1666} 1667 1668void X86_64Assembler::EmitRex64(CpuRegister reg) { 1669 EmitOptionalRex(false, true, false, false, reg.NeedsRex()); 1670} 1671 1672void X86_64Assembler::EmitRex64(CpuRegister dst, CpuRegister src) { 1673 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 1674} 1675 1676void X86_64Assembler::EmitRex64(CpuRegister dst, const Operand& operand) { 1677 uint8_t rex = 0x48 | operand.rex(); // REX.W000 1678 if (dst.NeedsRex()) { 1679 rex |= 0x44; // REX.0R00 1680 } 1681 if (rex != 0) { 1682 EmitUint8(rex); 1683 } 1684} 1685 1686void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, CpuRegister src) { 1687 EmitOptionalRex(true, false, dst.NeedsRex(), false, src.NeedsRex()); 1688} 1689 1690void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand) { 1691 uint8_t rex = 0x40 | operand.rex(); // REX.0000 1692 if (dst.NeedsRex()) { 1693 rex |= 0x44; // REX.0R00 1694 } 1695 if (rex != 0) { 1696 EmitUint8(rex); 1697 } 1698} 1699 1700constexpr size_t kFramePointerSize = 8; 1701 1702void X86_64Assembler::BuildFrame(size_t frame_size, ManagedRegister method_reg, 1703 const std::vector<ManagedRegister>& spill_regs, 1704 const ManagedRegisterEntrySpills& entry_spills) { 1705 CHECK_ALIGNED(frame_size, kStackAlignment); 1706 int gpr_count = 0; 1707 for (int i = spill_regs.size() - 1; i >= 0; --i) { 1708 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 1709 if (spill.IsCpuRegister()) { 1710 pushq(spill.AsCpuRegister()); 1711 gpr_count++; 1712 } 1713 } 1714 // return address then method on stack 1715 int64_t rest_of_frame = static_cast<int64_t>(frame_size) 1716 - (gpr_count * kFramePointerSize) 1717 - kFramePointerSize /*return address*/; 1718 subq(CpuRegister(RSP), Immediate(rest_of_frame)); 1719 // spill xmms 1720 int64_t offset = rest_of_frame; 1721 for (int i = spill_regs.size() - 1; i >= 0; --i) { 1722 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 1723 if (spill.IsXmmRegister()) { 1724 offset -= sizeof(double); 1725 movsd(Address(CpuRegister(RSP), offset), spill.AsXmmRegister()); 1726 } 1727 } 1728 1729 DCHECK_EQ(4U, sizeof(StackReference<mirror::ArtMethod>)); 1730 1731 movl(Address(CpuRegister(RSP), 0), method_reg.AsX86_64().AsCpuRegister()); 1732 1733 for (size_t i = 0; i < entry_spills.size(); ++i) { 1734 ManagedRegisterSpill spill = entry_spills.at(i); 1735 if (spill.AsX86_64().IsCpuRegister()) { 1736 if (spill.getSize() == 8) { 1737 movq(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), 1738 spill.AsX86_64().AsCpuRegister()); 1739 } else { 1740 CHECK_EQ(spill.getSize(), 4); 1741 movl(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsCpuRegister()); 1742 } 1743 } else { 1744 if (spill.getSize() == 8) { 1745 movsd(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 1746 } else { 1747 CHECK_EQ(spill.getSize(), 4); 1748 movss(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 1749 } 1750 } 1751 } 1752} 1753 1754void X86_64Assembler::RemoveFrame(size_t frame_size, 1755 const std::vector<ManagedRegister>& spill_regs) { 1756 CHECK_ALIGNED(frame_size, kStackAlignment); 1757 int gpr_count = 0; 1758 // unspill xmms 1759 int64_t offset = static_cast<int64_t>(frame_size) - (spill_regs.size() * kFramePointerSize) - 2 * kFramePointerSize; 1760 for (size_t i = 0; i < spill_regs.size(); ++i) { 1761 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 1762 if (spill.IsXmmRegister()) { 1763 offset += sizeof(double); 1764 movsd(spill.AsXmmRegister(), Address(CpuRegister(RSP), offset)); 1765 } else { 1766 gpr_count++; 1767 } 1768 } 1769 addq(CpuRegister(RSP), Immediate(static_cast<int64_t>(frame_size) - (gpr_count * kFramePointerSize) - kFramePointerSize)); 1770 for (size_t i = 0; i < spill_regs.size(); ++i) { 1771 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 1772 if (spill.IsCpuRegister()) { 1773 popq(spill.AsCpuRegister()); 1774 } 1775 } 1776 ret(); 1777} 1778 1779void X86_64Assembler::IncreaseFrameSize(size_t adjust) { 1780 CHECK_ALIGNED(adjust, kStackAlignment); 1781 addq(CpuRegister(RSP), Immediate(-static_cast<int64_t>(adjust))); 1782} 1783 1784void X86_64Assembler::DecreaseFrameSize(size_t adjust) { 1785 CHECK_ALIGNED(adjust, kStackAlignment); 1786 addq(CpuRegister(RSP), Immediate(adjust)); 1787} 1788 1789void X86_64Assembler::Store(FrameOffset offs, ManagedRegister msrc, size_t size) { 1790 X86_64ManagedRegister src = msrc.AsX86_64(); 1791 if (src.IsNoRegister()) { 1792 CHECK_EQ(0u, size); 1793 } else if (src.IsCpuRegister()) { 1794 if (size == 4) { 1795 CHECK_EQ(4u, size); 1796 movl(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 1797 } else { 1798 CHECK_EQ(8u, size); 1799 movq(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 1800 } 1801 } else if (src.IsRegisterPair()) { 1802 CHECK_EQ(0u, size); 1803 movq(Address(CpuRegister(RSP), offs), src.AsRegisterPairLow()); 1804 movq(Address(CpuRegister(RSP), FrameOffset(offs.Int32Value()+4)), 1805 src.AsRegisterPairHigh()); 1806 } else if (src.IsX87Register()) { 1807 if (size == 4) { 1808 fstps(Address(CpuRegister(RSP), offs)); 1809 } else { 1810 fstpl(Address(CpuRegister(RSP), offs)); 1811 } 1812 } else { 1813 CHECK(src.IsXmmRegister()); 1814 if (size == 4) { 1815 movss(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 1816 } else { 1817 movsd(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 1818 } 1819 } 1820} 1821 1822void X86_64Assembler::StoreRef(FrameOffset dest, ManagedRegister msrc) { 1823 X86_64ManagedRegister src = msrc.AsX86_64(); 1824 CHECK(src.IsCpuRegister()); 1825 movl(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 1826} 1827 1828void X86_64Assembler::StoreRawPtr(FrameOffset dest, ManagedRegister msrc) { 1829 X86_64ManagedRegister src = msrc.AsX86_64(); 1830 CHECK(src.IsCpuRegister()); 1831 movq(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 1832} 1833 1834void X86_64Assembler::StoreImmediateToFrame(FrameOffset dest, uint32_t imm, 1835 ManagedRegister) { 1836 movl(Address(CpuRegister(RSP), dest), Immediate(imm)); // TODO(64) movq? 1837} 1838 1839void X86_64Assembler::StoreImmediateToThread64(ThreadOffset<8> dest, uint32_t imm, 1840 ManagedRegister) { 1841 gs()->movl(Address::Absolute(dest, true), Immediate(imm)); // TODO(64) movq? 1842} 1843 1844void X86_64Assembler::StoreStackOffsetToThread64(ThreadOffset<8> thr_offs, 1845 FrameOffset fr_offs, 1846 ManagedRegister mscratch) { 1847 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 1848 CHECK(scratch.IsCpuRegister()); 1849 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), fr_offs)); 1850 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 1851} 1852 1853void X86_64Assembler::StoreStackPointerToThread64(ThreadOffset<8> thr_offs) { 1854 gs()->movq(Address::Absolute(thr_offs, true), CpuRegister(RSP)); 1855} 1856 1857void X86_64Assembler::StoreSpanning(FrameOffset /*dst*/, ManagedRegister /*src*/, 1858 FrameOffset /*in_off*/, ManagedRegister /*scratch*/) { 1859 UNIMPLEMENTED(FATAL); // this case only currently exists for ARM 1860} 1861 1862void X86_64Assembler::Load(ManagedRegister mdest, FrameOffset src, size_t size) { 1863 X86_64ManagedRegister dest = mdest.AsX86_64(); 1864 if (dest.IsNoRegister()) { 1865 CHECK_EQ(0u, size); 1866 } else if (dest.IsCpuRegister()) { 1867 if (size == 4) { 1868 CHECK_EQ(4u, size); 1869 movl(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 1870 } else { 1871 CHECK_EQ(8u, size); 1872 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 1873 } 1874 } else if (dest.IsRegisterPair()) { 1875 CHECK_EQ(0u, size); 1876 movq(dest.AsRegisterPairLow(), Address(CpuRegister(RSP), src)); 1877 movq(dest.AsRegisterPairHigh(), Address(CpuRegister(RSP), FrameOffset(src.Int32Value()+4))); 1878 } else if (dest.IsX87Register()) { 1879 if (size == 4) { 1880 flds(Address(CpuRegister(RSP), src)); 1881 } else { 1882 fldl(Address(CpuRegister(RSP), src)); 1883 } 1884 } else { 1885 CHECK(dest.IsXmmRegister()); 1886 if (size == 4) { 1887 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 1888 } else { 1889 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 1890 } 1891 } 1892} 1893 1894void X86_64Assembler::LoadFromThread64(ManagedRegister mdest, ThreadOffset<8> src, size_t size) { 1895 X86_64ManagedRegister dest = mdest.AsX86_64(); 1896 if (dest.IsNoRegister()) { 1897 CHECK_EQ(0u, size); 1898 } else if (dest.IsCpuRegister()) { 1899 CHECK_EQ(4u, size); 1900 gs()->movl(dest.AsCpuRegister(), Address::Absolute(src, true)); 1901 } else if (dest.IsRegisterPair()) { 1902 CHECK_EQ(8u, size); 1903 gs()->movq(dest.AsRegisterPairLow(), Address::Absolute(src, true)); 1904 } else if (dest.IsX87Register()) { 1905 if (size == 4) { 1906 gs()->flds(Address::Absolute(src, true)); 1907 } else { 1908 gs()->fldl(Address::Absolute(src, true)); 1909 } 1910 } else { 1911 CHECK(dest.IsXmmRegister()); 1912 if (size == 4) { 1913 gs()->movss(dest.AsXmmRegister(), Address::Absolute(src, true)); 1914 } else { 1915 gs()->movsd(dest.AsXmmRegister(), Address::Absolute(src, true)); 1916 } 1917 } 1918} 1919 1920void X86_64Assembler::LoadRef(ManagedRegister mdest, FrameOffset src) { 1921 X86_64ManagedRegister dest = mdest.AsX86_64(); 1922 CHECK(dest.IsCpuRegister()); 1923 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 1924} 1925 1926void X86_64Assembler::LoadRef(ManagedRegister mdest, ManagedRegister base, 1927 MemberOffset offs) { 1928 X86_64ManagedRegister dest = mdest.AsX86_64(); 1929 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 1930 movq(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 1931} 1932 1933void X86_64Assembler::LoadRawPtr(ManagedRegister mdest, ManagedRegister base, 1934 Offset offs) { 1935 X86_64ManagedRegister dest = mdest.AsX86_64(); 1936 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 1937 movq(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 1938} 1939 1940void X86_64Assembler::LoadRawPtrFromThread64(ManagedRegister mdest, ThreadOffset<8> offs) { 1941 X86_64ManagedRegister dest = mdest.AsX86_64(); 1942 CHECK(dest.IsCpuRegister()); 1943 gs()->movq(dest.AsCpuRegister(), Address::Absolute(offs, true)); 1944} 1945 1946void X86_64Assembler::SignExtend(ManagedRegister mreg, size_t size) { 1947 X86_64ManagedRegister reg = mreg.AsX86_64(); 1948 CHECK(size == 1 || size == 2) << size; 1949 CHECK(reg.IsCpuRegister()) << reg; 1950 if (size == 1) { 1951 movsxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 1952 } else { 1953 movsxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 1954 } 1955} 1956 1957void X86_64Assembler::ZeroExtend(ManagedRegister mreg, size_t size) { 1958 X86_64ManagedRegister reg = mreg.AsX86_64(); 1959 CHECK(size == 1 || size == 2) << size; 1960 CHECK(reg.IsCpuRegister()) << reg; 1961 if (size == 1) { 1962 movzxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 1963 } else { 1964 movzxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 1965 } 1966} 1967 1968void X86_64Assembler::Move(ManagedRegister mdest, ManagedRegister msrc, size_t size) { 1969 X86_64ManagedRegister dest = mdest.AsX86_64(); 1970 X86_64ManagedRegister src = msrc.AsX86_64(); 1971 if (!dest.Equals(src)) { 1972 if (dest.IsCpuRegister() && src.IsCpuRegister()) { 1973 movq(dest.AsCpuRegister(), src.AsCpuRegister()); 1974 } else if (src.IsX87Register() && dest.IsXmmRegister()) { 1975 // Pass via stack and pop X87 register 1976 subl(CpuRegister(RSP), Immediate(16)); 1977 if (size == 4) { 1978 CHECK_EQ(src.AsX87Register(), ST0); 1979 fstps(Address(CpuRegister(RSP), 0)); 1980 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 1981 } else { 1982 CHECK_EQ(src.AsX87Register(), ST0); 1983 fstpl(Address(CpuRegister(RSP), 0)); 1984 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 1985 } 1986 addq(CpuRegister(RSP), Immediate(16)); 1987 } else { 1988 // TODO: x87, SSE 1989 UNIMPLEMENTED(FATAL) << ": Move " << dest << ", " << src; 1990 } 1991 } 1992} 1993 1994void X86_64Assembler::CopyRef(FrameOffset dest, FrameOffset src, 1995 ManagedRegister mscratch) { 1996 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 1997 CHECK(scratch.IsCpuRegister()); 1998 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), src)); 1999 movl(Address(CpuRegister(RSP), dest), scratch.AsCpuRegister()); 2000} 2001 2002void X86_64Assembler::CopyRawPtrFromThread64(FrameOffset fr_offs, 2003 ThreadOffset<8> thr_offs, 2004 ManagedRegister mscratch) { 2005 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2006 CHECK(scratch.IsCpuRegister()); 2007 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(thr_offs, true)); 2008 Store(fr_offs, scratch, 8); 2009} 2010 2011void X86_64Assembler::CopyRawPtrToThread64(ThreadOffset<8> thr_offs, 2012 FrameOffset fr_offs, 2013 ManagedRegister mscratch) { 2014 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2015 CHECK(scratch.IsCpuRegister()); 2016 Load(scratch, fr_offs, 8); 2017 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2018} 2019 2020void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src, 2021 ManagedRegister mscratch, 2022 size_t size) { 2023 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2024 if (scratch.IsCpuRegister() && size == 8) { 2025 Load(scratch, src, 4); 2026 Store(dest, scratch, 4); 2027 Load(scratch, FrameOffset(src.Int32Value() + 4), 4); 2028 Store(FrameOffset(dest.Int32Value() + 4), scratch, 4); 2029 } else { 2030 Load(scratch, src, size); 2031 Store(dest, scratch, size); 2032 } 2033} 2034 2035void X86_64Assembler::Copy(FrameOffset /*dst*/, ManagedRegister /*src_base*/, Offset /*src_offset*/, 2036 ManagedRegister /*scratch*/, size_t /*size*/) { 2037 UNIMPLEMENTED(FATAL); 2038} 2039 2040void X86_64Assembler::Copy(ManagedRegister dest_base, Offset dest_offset, FrameOffset src, 2041 ManagedRegister scratch, size_t size) { 2042 CHECK(scratch.IsNoRegister()); 2043 CHECK_EQ(size, 4u); 2044 pushq(Address(CpuRegister(RSP), src)); 2045 popq(Address(dest_base.AsX86_64().AsCpuRegister(), dest_offset)); 2046} 2047 2048void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src_base, Offset src_offset, 2049 ManagedRegister mscratch, size_t size) { 2050 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2051 CHECK_EQ(size, 4u); 2052 movq(scratch, Address(CpuRegister(RSP), src_base)); 2053 movq(scratch, Address(scratch, src_offset)); 2054 movq(Address(CpuRegister(RSP), dest), scratch); 2055} 2056 2057void X86_64Assembler::Copy(ManagedRegister dest, Offset dest_offset, 2058 ManagedRegister src, Offset src_offset, 2059 ManagedRegister scratch, size_t size) { 2060 CHECK_EQ(size, 4u); 2061 CHECK(scratch.IsNoRegister()); 2062 pushq(Address(src.AsX86_64().AsCpuRegister(), src_offset)); 2063 popq(Address(dest.AsX86_64().AsCpuRegister(), dest_offset)); 2064} 2065 2066void X86_64Assembler::Copy(FrameOffset dest, Offset dest_offset, FrameOffset src, Offset src_offset, 2067 ManagedRegister mscratch, size_t size) { 2068 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2069 CHECK_EQ(size, 4u); 2070 CHECK_EQ(dest.Int32Value(), src.Int32Value()); 2071 movq(scratch, Address(CpuRegister(RSP), src)); 2072 pushq(Address(scratch, src_offset)); 2073 popq(Address(scratch, dest_offset)); 2074} 2075 2076void X86_64Assembler::MemoryBarrier(ManagedRegister) { 2077#if ANDROID_SMP != 0 2078 mfence(); 2079#endif 2080} 2081 2082void X86_64Assembler::CreateHandleScopeEntry(ManagedRegister mout_reg, 2083 FrameOffset handle_scope_offset, 2084 ManagedRegister min_reg, bool null_allowed) { 2085 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2086 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2087 if (in_reg.IsNoRegister()) { // TODO(64): && null_allowed 2088 // Use out_reg as indicator of NULL 2089 in_reg = out_reg; 2090 // TODO: movzwl 2091 movl(in_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2092 } 2093 CHECK(in_reg.IsCpuRegister()); 2094 CHECK(out_reg.IsCpuRegister()); 2095 VerifyObject(in_reg, null_allowed); 2096 if (null_allowed) { 2097 Label null_arg; 2098 if (!out_reg.Equals(in_reg)) { 2099 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2100 } 2101 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2102 j(kZero, &null_arg); 2103 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2104 Bind(&null_arg); 2105 } else { 2106 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2107 } 2108} 2109 2110void X86_64Assembler::CreateHandleScopeEntry(FrameOffset out_off, 2111 FrameOffset handle_scope_offset, 2112 ManagedRegister mscratch, 2113 bool null_allowed) { 2114 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2115 CHECK(scratch.IsCpuRegister()); 2116 if (null_allowed) { 2117 Label null_arg; 2118 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2119 testl(scratch.AsCpuRegister(), scratch.AsCpuRegister()); 2120 j(kZero, &null_arg); 2121 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2122 Bind(&null_arg); 2123 } else { 2124 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2125 } 2126 Store(out_off, scratch, 8); 2127} 2128 2129// Given a handle scope entry, load the associated reference. 2130void X86_64Assembler::LoadReferenceFromHandleScope(ManagedRegister mout_reg, 2131 ManagedRegister min_reg) { 2132 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2133 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2134 CHECK(out_reg.IsCpuRegister()); 2135 CHECK(in_reg.IsCpuRegister()); 2136 Label null_arg; 2137 if (!out_reg.Equals(in_reg)) { 2138 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2139 } 2140 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2141 j(kZero, &null_arg); 2142 movq(out_reg.AsCpuRegister(), Address(in_reg.AsCpuRegister(), 0)); 2143 Bind(&null_arg); 2144} 2145 2146void X86_64Assembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) { 2147 // TODO: not validating references 2148} 2149 2150void X86_64Assembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) { 2151 // TODO: not validating references 2152} 2153 2154void X86_64Assembler::Call(ManagedRegister mbase, Offset offset, ManagedRegister) { 2155 X86_64ManagedRegister base = mbase.AsX86_64(); 2156 CHECK(base.IsCpuRegister()); 2157 call(Address(base.AsCpuRegister(), offset.Int32Value())); 2158 // TODO: place reference map on call 2159} 2160 2161void X86_64Assembler::Call(FrameOffset base, Offset offset, ManagedRegister mscratch) { 2162 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2163 movl(scratch, Address(CpuRegister(RSP), base)); 2164 call(Address(scratch, offset)); 2165} 2166 2167void X86_64Assembler::CallFromThread64(ThreadOffset<8> offset, ManagedRegister /*mscratch*/) { 2168 gs()->call(Address::Absolute(offset, true)); 2169} 2170 2171void X86_64Assembler::GetCurrentThread(ManagedRegister tr) { 2172 gs()->movq(tr.AsX86_64().AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2173} 2174 2175void X86_64Assembler::GetCurrentThread(FrameOffset offset, ManagedRegister mscratch) { 2176 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2177 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2178 movq(Address(CpuRegister(RSP), offset), scratch.AsCpuRegister()); 2179} 2180 2181// Slowpath entered when Thread::Current()->_exception is non-null 2182class X86_64ExceptionSlowPath FINAL : public SlowPath { 2183 public: 2184 explicit X86_64ExceptionSlowPath(size_t stack_adjust) : stack_adjust_(stack_adjust) {} 2185 virtual void Emit(Assembler *sp_asm) OVERRIDE; 2186 private: 2187 const size_t stack_adjust_; 2188}; 2189 2190void X86_64Assembler::ExceptionPoll(ManagedRegister /*scratch*/, size_t stack_adjust) { 2191 X86_64ExceptionSlowPath* slow = new X86_64ExceptionSlowPath(stack_adjust); 2192 buffer_.EnqueueSlowPath(slow); 2193 gs()->cmpl(Address::Absolute(Thread::ExceptionOffset<8>(), true), Immediate(0)); 2194 j(kNotEqual, slow->Entry()); 2195} 2196 2197void X86_64ExceptionSlowPath::Emit(Assembler *sasm) { 2198 X86_64Assembler* sp_asm = down_cast<X86_64Assembler*>(sasm); 2199#define __ sp_asm-> 2200 __ Bind(&entry_); 2201 // Note: the return value is dead 2202 if (stack_adjust_ != 0) { // Fix up the frame. 2203 __ DecreaseFrameSize(stack_adjust_); 2204 } 2205 // Pass exception as argument in RDI 2206 __ gs()->movq(CpuRegister(RDI), Address::Absolute(Thread::ExceptionOffset<8>(), true)); 2207 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(8, pDeliverException), true)); 2208 // this call should never return 2209 __ int3(); 2210#undef __ 2211} 2212 2213} // namespace x86_64 2214} // namespace art 2215 2216