assembler_x86_64.cc revision 102cbed1e52b7c5f09458b44903fe97bb3e14d5f
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "assembler_x86_64.h" 18 19#include "base/casts.h" 20#include "entrypoints/quick/quick_entrypoints.h" 21#include "memory_region.h" 22#include "thread.h" 23#include "utils/dwarf_cfi.h" 24 25namespace art { 26namespace x86_64 { 27 28std::ostream& operator<<(std::ostream& os, const CpuRegister& reg) { 29 return os << reg.AsRegister(); 30} 31 32std::ostream& operator<<(std::ostream& os, const XmmRegister& reg) { 33 return os << reg.AsFloatRegister(); 34} 35 36std::ostream& operator<<(std::ostream& os, const X87Register& reg) { 37 return os << "ST" << static_cast<int>(reg); 38} 39 40void X86_64Assembler::call(CpuRegister reg) { 41 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 42 EmitOptionalRex32(reg); 43 EmitUint8(0xFF); 44 EmitRegisterOperand(2, reg.LowBits()); 45} 46 47 48void X86_64Assembler::call(const Address& address) { 49 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 50 EmitOptionalRex32(address); 51 EmitUint8(0xFF); 52 EmitOperand(2, address); 53} 54 55 56void X86_64Assembler::call(Label* label) { 57 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 58 EmitUint8(0xE8); 59 static const int kSize = 5; 60 EmitLabel(label, kSize); 61} 62 63void X86_64Assembler::pushq(CpuRegister reg) { 64 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 65 EmitOptionalRex32(reg); 66 EmitUint8(0x50 + reg.LowBits()); 67} 68 69 70void X86_64Assembler::pushq(const Address& address) { 71 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 72 EmitOptionalRex32(address); 73 EmitUint8(0xFF); 74 EmitOperand(6, address); 75} 76 77 78void X86_64Assembler::pushq(const Immediate& imm) { 79 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 80 CHECK(imm.is_int32()); // pushq only supports 32b immediate. 81 if (imm.is_int8()) { 82 EmitUint8(0x6A); 83 EmitUint8(imm.value() & 0xFF); 84 } else { 85 EmitUint8(0x68); 86 EmitImmediate(imm); 87 } 88} 89 90 91void X86_64Assembler::popq(CpuRegister reg) { 92 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 93 EmitOptionalRex32(reg); 94 EmitUint8(0x58 + reg.LowBits()); 95} 96 97 98void X86_64Assembler::popq(const Address& address) { 99 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 100 EmitOptionalRex32(address); 101 EmitUint8(0x8F); 102 EmitOperand(0, address); 103} 104 105 106void X86_64Assembler::movq(CpuRegister dst, const Immediate& imm) { 107 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 108 if (imm.is_int32()) { 109 // 32 bit. Note: sign-extends. 110 EmitRex64(dst); 111 EmitUint8(0xC7); 112 EmitRegisterOperand(0, dst.LowBits()); 113 EmitInt32(static_cast<int32_t>(imm.value())); 114 } else { 115 EmitRex64(dst); 116 EmitUint8(0xB8 + dst.LowBits()); 117 EmitInt64(imm.value()); 118 } 119} 120 121 122void X86_64Assembler::movl(CpuRegister dst, const Immediate& imm) { 123 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 124 EmitOptionalRex32(dst); 125 EmitUint8(0xB8 + dst.LowBits()); 126 EmitImmediate(imm); 127} 128 129 130void X86_64Assembler::movq(CpuRegister dst, CpuRegister src) { 131 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 132 // 0x89 is movq r/m64 <- r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 133 EmitRex64(src, dst); 134 EmitUint8(0x89); 135 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 136} 137 138 139void X86_64Assembler::movl(CpuRegister dst, CpuRegister src) { 140 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 141 EmitOptionalRex32(dst, src); 142 EmitUint8(0x8B); 143 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 144} 145 146 147void X86_64Assembler::movq(CpuRegister dst, const Address& src) { 148 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 149 EmitRex64(dst, src); 150 EmitUint8(0x8B); 151 EmitOperand(dst.LowBits(), src); 152} 153 154 155void X86_64Assembler::movl(CpuRegister dst, const Address& src) { 156 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 157 EmitOptionalRex32(dst, src); 158 EmitUint8(0x8B); 159 EmitOperand(dst.LowBits(), src); 160} 161 162 163void X86_64Assembler::movq(const Address& dst, CpuRegister src) { 164 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 165 EmitRex64(src, dst); 166 EmitUint8(0x89); 167 EmitOperand(src.LowBits(), dst); 168} 169 170 171void X86_64Assembler::movl(const Address& dst, CpuRegister src) { 172 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 173 EmitOptionalRex32(src, dst); 174 EmitUint8(0x89); 175 EmitOperand(src.LowBits(), dst); 176} 177 178void X86_64Assembler::movl(const Address& dst, const Immediate& imm) { 179 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 180 EmitOptionalRex32(dst); 181 EmitUint8(0xC7); 182 EmitOperand(0, dst); 183 EmitImmediate(imm); 184} 185 186void X86_64Assembler::movzxb(CpuRegister dst, CpuRegister src) { 187 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 188 EmitOptionalByteRegNormalizingRex32(dst, src); 189 EmitUint8(0x0F); 190 EmitUint8(0xB6); 191 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 192} 193 194 195void X86_64Assembler::movzxb(CpuRegister dst, const Address& src) { 196 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 197 EmitOptionalByteRegNormalizingRex32(dst, src); 198 EmitUint8(0x0F); 199 EmitUint8(0xB6); 200 EmitOperand(dst.LowBits(), src); 201} 202 203 204void X86_64Assembler::movsxb(CpuRegister dst, CpuRegister src) { 205 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 206 EmitOptionalByteRegNormalizingRex32(dst, src); 207 EmitUint8(0x0F); 208 EmitUint8(0xBE); 209 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 210} 211 212 213void X86_64Assembler::movsxb(CpuRegister dst, const Address& src) { 214 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 215 EmitOptionalByteRegNormalizingRex32(dst, src); 216 EmitUint8(0x0F); 217 EmitUint8(0xBE); 218 EmitOperand(dst.LowBits(), src); 219} 220 221 222void X86_64Assembler::movb(CpuRegister /*dst*/, const Address& /*src*/) { 223 LOG(FATAL) << "Use movzxb or movsxb instead."; 224} 225 226 227void X86_64Assembler::movb(const Address& dst, CpuRegister src) { 228 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 229 EmitOptionalByteRegNormalizingRex32(src, dst); 230 EmitUint8(0x88); 231 EmitOperand(src.LowBits(), dst); 232} 233 234 235void X86_64Assembler::movb(const Address& dst, const Immediate& imm) { 236 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 237 EmitOptionalRex32(dst); 238 EmitUint8(0xC6); 239 EmitOperand(Register::RAX, dst); 240 CHECK(imm.is_int8()); 241 EmitUint8(imm.value() & 0xFF); 242} 243 244 245void X86_64Assembler::movzxw(CpuRegister dst, CpuRegister src) { 246 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 247 EmitOptionalRex32(dst, src); 248 EmitUint8(0x0F); 249 EmitUint8(0xB7); 250 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 251} 252 253 254void X86_64Assembler::movzxw(CpuRegister dst, const Address& src) { 255 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 256 EmitOptionalRex32(dst, src); 257 EmitUint8(0x0F); 258 EmitUint8(0xB7); 259 EmitOperand(dst.LowBits(), src); 260} 261 262 263void X86_64Assembler::movsxw(CpuRegister dst, CpuRegister src) { 264 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 265 EmitOptionalRex32(dst, src); 266 EmitUint8(0x0F); 267 EmitUint8(0xBF); 268 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 269} 270 271 272void X86_64Assembler::movsxw(CpuRegister dst, const Address& src) { 273 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 274 EmitOptionalRex32(dst, src); 275 EmitUint8(0x0F); 276 EmitUint8(0xBF); 277 EmitOperand(dst.LowBits(), src); 278} 279 280 281void X86_64Assembler::movw(CpuRegister /*dst*/, const Address& /*src*/) { 282 LOG(FATAL) << "Use movzxw or movsxw instead."; 283} 284 285 286void X86_64Assembler::movw(const Address& dst, CpuRegister src) { 287 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 288 EmitOperandSizeOverride(); 289 EmitOptionalRex32(src, dst); 290 EmitUint8(0x89); 291 EmitOperand(src.LowBits(), dst); 292} 293 294 295void X86_64Assembler::movw(const Address& dst, const Immediate& imm) { 296 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 297 EmitOperandSizeOverride(); 298 EmitOptionalRex32(dst); 299 EmitUint8(0xC7); 300 EmitOperand(Register::RAX, dst); 301 CHECK(imm.is_uint16() || imm.is_int16()); 302 EmitUint8(imm.value() & 0xFF); 303 EmitUint8(imm.value() >> 8); 304} 305 306 307void X86_64Assembler::leaq(CpuRegister dst, const Address& src) { 308 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 309 EmitRex64(dst, src); 310 EmitUint8(0x8D); 311 EmitOperand(dst.LowBits(), src); 312} 313 314 315void X86_64Assembler::movaps(XmmRegister dst, XmmRegister src) { 316 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 317 EmitOptionalRex32(dst, src); 318 EmitUint8(0x0F); 319 EmitUint8(0x28); 320 EmitXmmRegisterOperand(dst.LowBits(), src); 321} 322 323 324void X86_64Assembler::movss(XmmRegister dst, const Address& src) { 325 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 326 EmitUint8(0xF3); 327 EmitOptionalRex32(dst, src); 328 EmitUint8(0x0F); 329 EmitUint8(0x10); 330 EmitOperand(dst.LowBits(), src); 331} 332 333 334void X86_64Assembler::movss(const Address& dst, XmmRegister src) { 335 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 336 EmitUint8(0xF3); 337 EmitOptionalRex32(src, dst); 338 EmitUint8(0x0F); 339 EmitUint8(0x11); 340 EmitOperand(src.LowBits(), dst); 341} 342 343 344void X86_64Assembler::movss(XmmRegister dst, XmmRegister src) { 345 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 346 EmitUint8(0xF3); 347 EmitOptionalRex32(dst, src); 348 EmitUint8(0x0F); 349 EmitUint8(0x11); 350 EmitXmmRegisterOperand(src.LowBits(), dst); 351} 352 353 354void X86_64Assembler::movd(XmmRegister dst, CpuRegister src) { 355 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 356 EmitUint8(0x66); 357 EmitRex64(dst, src); 358 EmitUint8(0x0F); 359 EmitUint8(0x6E); 360 EmitOperand(dst.LowBits(), Operand(src)); 361} 362 363 364void X86_64Assembler::movd(CpuRegister dst, XmmRegister src) { 365 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 366 EmitUint8(0x66); 367 EmitRex64(src, dst); 368 EmitUint8(0x0F); 369 EmitUint8(0x7E); 370 EmitOperand(src.LowBits(), Operand(dst)); 371} 372 373 374void X86_64Assembler::addss(XmmRegister dst, XmmRegister src) { 375 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 376 EmitUint8(0xF3); 377 EmitOptionalRex32(dst, src); 378 EmitUint8(0x0F); 379 EmitUint8(0x58); 380 EmitXmmRegisterOperand(dst.LowBits(), src); 381} 382 383 384void X86_64Assembler::addss(XmmRegister dst, const Address& src) { 385 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 386 EmitUint8(0xF3); 387 EmitOptionalRex32(dst, src); 388 EmitUint8(0x0F); 389 EmitUint8(0x58); 390 EmitOperand(dst.LowBits(), src); 391} 392 393 394void X86_64Assembler::subss(XmmRegister dst, XmmRegister src) { 395 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 396 EmitUint8(0xF3); 397 EmitOptionalRex32(dst, src); 398 EmitUint8(0x0F); 399 EmitUint8(0x5C); 400 EmitXmmRegisterOperand(dst.LowBits(), src); 401} 402 403 404void X86_64Assembler::subss(XmmRegister dst, const Address& src) { 405 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 406 EmitUint8(0xF3); 407 EmitOptionalRex32(dst, src); 408 EmitUint8(0x0F); 409 EmitUint8(0x5C); 410 EmitOperand(dst.LowBits(), src); 411} 412 413 414void X86_64Assembler::mulss(XmmRegister dst, XmmRegister src) { 415 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 416 EmitUint8(0xF3); 417 EmitOptionalRex32(dst, src); 418 EmitUint8(0x0F); 419 EmitUint8(0x59); 420 EmitXmmRegisterOperand(dst.LowBits(), src); 421} 422 423 424void X86_64Assembler::mulss(XmmRegister dst, const Address& src) { 425 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 426 EmitUint8(0xF3); 427 EmitOptionalRex32(dst, src); 428 EmitUint8(0x0F); 429 EmitUint8(0x59); 430 EmitOperand(dst.LowBits(), src); 431} 432 433 434void X86_64Assembler::divss(XmmRegister dst, XmmRegister src) { 435 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 436 EmitUint8(0xF3); 437 EmitOptionalRex32(dst, src); 438 EmitUint8(0x0F); 439 EmitUint8(0x5E); 440 EmitXmmRegisterOperand(dst.LowBits(), src); 441} 442 443 444void X86_64Assembler::divss(XmmRegister dst, const Address& src) { 445 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 446 EmitUint8(0xF3); 447 EmitOptionalRex32(dst, src); 448 EmitUint8(0x0F); 449 EmitUint8(0x5E); 450 EmitOperand(dst.LowBits(), src); 451} 452 453 454void X86_64Assembler::flds(const Address& src) { 455 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 456 EmitUint8(0xD9); 457 EmitOperand(0, src); 458} 459 460 461void X86_64Assembler::fstps(const Address& dst) { 462 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 463 EmitUint8(0xD9); 464 EmitOperand(3, dst); 465} 466 467 468void X86_64Assembler::movsd(XmmRegister dst, const Address& src) { 469 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 470 EmitUint8(0xF2); 471 EmitOptionalRex32(dst, src); 472 EmitUint8(0x0F); 473 EmitUint8(0x10); 474 EmitOperand(dst.LowBits(), src); 475} 476 477 478void X86_64Assembler::movsd(const Address& dst, XmmRegister src) { 479 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 480 EmitUint8(0xF2); 481 EmitOptionalRex32(src, dst); 482 EmitUint8(0x0F); 483 EmitUint8(0x11); 484 EmitOperand(src.LowBits(), dst); 485} 486 487 488void X86_64Assembler::movsd(XmmRegister dst, XmmRegister src) { 489 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 490 EmitUint8(0xF2); 491 EmitOptionalRex32(dst, src); 492 EmitUint8(0x0F); 493 EmitUint8(0x11); 494 EmitXmmRegisterOperand(src.LowBits(), dst); 495} 496 497 498void X86_64Assembler::addsd(XmmRegister dst, XmmRegister src) { 499 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 500 EmitUint8(0xF2); 501 EmitOptionalRex32(dst, src); 502 EmitUint8(0x0F); 503 EmitUint8(0x58); 504 EmitXmmRegisterOperand(dst.LowBits(), src); 505} 506 507 508void X86_64Assembler::addsd(XmmRegister dst, const Address& src) { 509 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 510 EmitUint8(0xF2); 511 EmitOptionalRex32(dst, src); 512 EmitUint8(0x0F); 513 EmitUint8(0x58); 514 EmitOperand(dst.LowBits(), src); 515} 516 517 518void X86_64Assembler::subsd(XmmRegister dst, XmmRegister src) { 519 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 520 EmitUint8(0xF2); 521 EmitOptionalRex32(dst, src); 522 EmitUint8(0x0F); 523 EmitUint8(0x5C); 524 EmitXmmRegisterOperand(dst.LowBits(), src); 525} 526 527 528void X86_64Assembler::subsd(XmmRegister dst, const Address& src) { 529 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 530 EmitUint8(0xF2); 531 EmitOptionalRex32(dst, src); 532 EmitUint8(0x0F); 533 EmitUint8(0x5C); 534 EmitOperand(dst.LowBits(), src); 535} 536 537 538void X86_64Assembler::mulsd(XmmRegister dst, XmmRegister src) { 539 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 540 EmitUint8(0xF2); 541 EmitOptionalRex32(dst, src); 542 EmitUint8(0x0F); 543 EmitUint8(0x59); 544 EmitXmmRegisterOperand(dst.LowBits(), src); 545} 546 547 548void X86_64Assembler::mulsd(XmmRegister dst, const Address& src) { 549 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 550 EmitUint8(0xF2); 551 EmitOptionalRex32(dst, src); 552 EmitUint8(0x0F); 553 EmitUint8(0x59); 554 EmitOperand(dst.LowBits(), src); 555} 556 557 558void X86_64Assembler::divsd(XmmRegister dst, XmmRegister src) { 559 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 560 EmitUint8(0xF2); 561 EmitOptionalRex32(dst, src); 562 EmitUint8(0x0F); 563 EmitUint8(0x5E); 564 EmitXmmRegisterOperand(dst.LowBits(), src); 565} 566 567 568void X86_64Assembler::divsd(XmmRegister dst, const Address& src) { 569 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 570 EmitUint8(0xF2); 571 EmitOptionalRex32(dst, src); 572 EmitUint8(0x0F); 573 EmitUint8(0x5E); 574 EmitOperand(dst.LowBits(), src); 575} 576 577 578void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) { 579 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 580 EmitUint8(0xF3); 581 EmitOptionalRex32(dst, src); 582 EmitUint8(0x0F); 583 EmitUint8(0x2A); 584 EmitOperand(dst.LowBits(), Operand(src)); 585} 586 587 588void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src) { 589 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 590 EmitUint8(0xF2); 591 EmitOptionalRex32(dst, src); 592 EmitUint8(0x0F); 593 EmitUint8(0x2A); 594 EmitOperand(dst.LowBits(), Operand(src)); 595} 596 597 598void X86_64Assembler::cvtss2si(CpuRegister dst, XmmRegister src) { 599 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 600 EmitUint8(0xF3); 601 EmitOptionalRex32(dst, src); 602 EmitUint8(0x0F); 603 EmitUint8(0x2D); 604 EmitXmmRegisterOperand(dst.LowBits(), src); 605} 606 607 608void X86_64Assembler::cvtss2sd(XmmRegister dst, XmmRegister src) { 609 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 610 EmitUint8(0xF3); 611 EmitOptionalRex32(dst, src); 612 EmitUint8(0x0F); 613 EmitUint8(0x5A); 614 EmitXmmRegisterOperand(dst.LowBits(), src); 615} 616 617 618void X86_64Assembler::cvtsd2si(CpuRegister dst, XmmRegister src) { 619 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 620 EmitUint8(0xF2); 621 EmitOptionalRex32(dst, src); 622 EmitUint8(0x0F); 623 EmitUint8(0x2D); 624 EmitXmmRegisterOperand(dst.LowBits(), src); 625} 626 627 628void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src) { 629 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 630 EmitUint8(0xF3); 631 EmitOptionalRex32(dst, src); 632 EmitUint8(0x0F); 633 EmitUint8(0x2C); 634 EmitXmmRegisterOperand(dst.LowBits(), src); 635} 636 637 638void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src) { 639 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 640 EmitUint8(0xF2); 641 EmitOptionalRex32(dst, src); 642 EmitUint8(0x0F); 643 EmitUint8(0x2C); 644 EmitXmmRegisterOperand(dst.LowBits(), src); 645} 646 647 648void X86_64Assembler::cvtsd2ss(XmmRegister dst, XmmRegister src) { 649 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 650 EmitUint8(0xF2); 651 EmitOptionalRex32(dst, src); 652 EmitUint8(0x0F); 653 EmitUint8(0x5A); 654 EmitXmmRegisterOperand(dst.LowBits(), src); 655} 656 657 658void X86_64Assembler::cvtdq2pd(XmmRegister dst, XmmRegister src) { 659 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 660 EmitUint8(0xF3); 661 EmitOptionalRex32(dst, src); 662 EmitUint8(0x0F); 663 EmitUint8(0xE6); 664 EmitXmmRegisterOperand(dst.LowBits(), src); 665} 666 667 668void X86_64Assembler::comiss(XmmRegister a, XmmRegister b) { 669 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 670 EmitOptionalRex32(a, b); 671 EmitUint8(0x0F); 672 EmitUint8(0x2F); 673 EmitXmmRegisterOperand(a.LowBits(), b); 674} 675 676 677void X86_64Assembler::comisd(XmmRegister a, XmmRegister b) { 678 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 679 EmitUint8(0x66); 680 EmitOptionalRex32(a, b); 681 EmitUint8(0x0F); 682 EmitUint8(0x2F); 683 EmitXmmRegisterOperand(a.LowBits(), b); 684} 685 686 687void X86_64Assembler::sqrtsd(XmmRegister dst, XmmRegister src) { 688 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 689 EmitUint8(0xF2); 690 EmitOptionalRex32(dst, src); 691 EmitUint8(0x0F); 692 EmitUint8(0x51); 693 EmitXmmRegisterOperand(dst.LowBits(), src); 694} 695 696 697void X86_64Assembler::sqrtss(XmmRegister dst, XmmRegister src) { 698 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 699 EmitUint8(0xF3); 700 EmitOptionalRex32(dst, src); 701 EmitUint8(0x0F); 702 EmitUint8(0x51); 703 EmitXmmRegisterOperand(dst.LowBits(), src); 704} 705 706 707void X86_64Assembler::xorpd(XmmRegister dst, const Address& src) { 708 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 709 EmitUint8(0x66); 710 EmitOptionalRex32(dst, src); 711 EmitUint8(0x0F); 712 EmitUint8(0x57); 713 EmitOperand(dst.LowBits(), src); 714} 715 716 717void X86_64Assembler::xorpd(XmmRegister dst, XmmRegister src) { 718 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 719 EmitUint8(0x66); 720 EmitOptionalRex32(dst, src); 721 EmitUint8(0x0F); 722 EmitUint8(0x57); 723 EmitXmmRegisterOperand(dst.LowBits(), src); 724} 725 726 727void X86_64Assembler::xorps(XmmRegister dst, const Address& src) { 728 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 729 EmitOptionalRex32(dst, src); 730 EmitUint8(0x0F); 731 EmitUint8(0x57); 732 EmitOperand(dst.LowBits(), src); 733} 734 735 736void X86_64Assembler::xorps(XmmRegister dst, XmmRegister src) { 737 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 738 EmitOptionalRex32(dst, src); 739 EmitUint8(0x0F); 740 EmitUint8(0x57); 741 EmitXmmRegisterOperand(dst.LowBits(), src); 742} 743 744 745void X86_64Assembler::andpd(XmmRegister dst, const Address& src) { 746 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 747 EmitUint8(0x66); 748 EmitOptionalRex32(dst, src); 749 EmitUint8(0x0F); 750 EmitUint8(0x54); 751 EmitOperand(dst.LowBits(), src); 752} 753 754 755void X86_64Assembler::fldl(const Address& src) { 756 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 757 EmitUint8(0xDD); 758 EmitOperand(0, src); 759} 760 761 762void X86_64Assembler::fstpl(const Address& dst) { 763 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 764 EmitUint8(0xDD); 765 EmitOperand(3, dst); 766} 767 768 769void X86_64Assembler::fnstcw(const Address& dst) { 770 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 771 EmitUint8(0xD9); 772 EmitOperand(7, dst); 773} 774 775 776void X86_64Assembler::fldcw(const Address& src) { 777 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 778 EmitUint8(0xD9); 779 EmitOperand(5, src); 780} 781 782 783void X86_64Assembler::fistpl(const Address& dst) { 784 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 785 EmitUint8(0xDF); 786 EmitOperand(7, dst); 787} 788 789 790void X86_64Assembler::fistps(const Address& dst) { 791 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 792 EmitUint8(0xDB); 793 EmitOperand(3, dst); 794} 795 796 797void X86_64Assembler::fildl(const Address& src) { 798 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 799 EmitUint8(0xDF); 800 EmitOperand(5, src); 801} 802 803 804void X86_64Assembler::fincstp() { 805 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 806 EmitUint8(0xD9); 807 EmitUint8(0xF7); 808} 809 810 811void X86_64Assembler::ffree(const Immediate& index) { 812 CHECK_LT(index.value(), 7); 813 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 814 EmitUint8(0xDD); 815 EmitUint8(0xC0 + index.value()); 816} 817 818 819void X86_64Assembler::fsin() { 820 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 821 EmitUint8(0xD9); 822 EmitUint8(0xFE); 823} 824 825 826void X86_64Assembler::fcos() { 827 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 828 EmitUint8(0xD9); 829 EmitUint8(0xFF); 830} 831 832 833void X86_64Assembler::fptan() { 834 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 835 EmitUint8(0xD9); 836 EmitUint8(0xF2); 837} 838 839 840void X86_64Assembler::xchgl(CpuRegister dst, CpuRegister src) { 841 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 842 EmitOptionalRex32(dst, src); 843 EmitUint8(0x87); 844 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 845} 846 847 848void X86_64Assembler::xchgq(CpuRegister dst, CpuRegister src) { 849 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 850 EmitRex64(dst, src); 851 EmitUint8(0x87); 852 EmitOperand(dst.LowBits(), Operand(src)); 853} 854 855 856void X86_64Assembler::xchgl(CpuRegister reg, const Address& address) { 857 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 858 EmitOptionalRex32(reg, address); 859 EmitUint8(0x87); 860 EmitOperand(reg.LowBits(), address); 861} 862 863 864void X86_64Assembler::cmpw(const Address& address, const Immediate& imm) { 865 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 866 EmitOptionalRex32(address); 867 EmitUint8(0x66); 868 EmitComplex(7, address, imm); 869} 870 871 872void X86_64Assembler::cmpl(CpuRegister reg, const Immediate& imm) { 873 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 874 EmitOptionalRex32(reg); 875 EmitComplex(7, Operand(reg), imm); 876} 877 878 879void X86_64Assembler::cmpl(CpuRegister reg0, CpuRegister reg1) { 880 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 881 EmitOptionalRex32(reg0, reg1); 882 EmitUint8(0x3B); 883 EmitOperand(reg0.LowBits(), Operand(reg1)); 884} 885 886 887void X86_64Assembler::cmpl(CpuRegister reg, const Address& address) { 888 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 889 EmitOptionalRex32(reg, address); 890 EmitUint8(0x3B); 891 EmitOperand(reg.LowBits(), address); 892} 893 894 895void X86_64Assembler::cmpq(CpuRegister reg0, CpuRegister reg1) { 896 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 897 EmitRex64(reg0, reg1); 898 EmitUint8(0x3B); 899 EmitOperand(reg0.LowBits(), Operand(reg1)); 900} 901 902 903void X86_64Assembler::cmpq(CpuRegister reg, const Immediate& imm) { 904 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 905 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 906 EmitRex64(reg); 907 EmitComplex(7, Operand(reg), imm); 908} 909 910 911void X86_64Assembler::cmpq(CpuRegister reg, const Address& address) { 912 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 913 EmitRex64(reg); 914 EmitUint8(0x3B); 915 EmitOperand(reg.LowBits(), address); 916} 917 918 919void X86_64Assembler::addl(CpuRegister dst, CpuRegister src) { 920 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 921 EmitOptionalRex32(dst, src); 922 EmitUint8(0x03); 923 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 924} 925 926 927void X86_64Assembler::addl(CpuRegister reg, const Address& address) { 928 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 929 EmitOptionalRex32(reg, address); 930 EmitUint8(0x03); 931 EmitOperand(reg.LowBits(), address); 932} 933 934 935void X86_64Assembler::cmpl(const Address& address, CpuRegister reg) { 936 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 937 EmitOptionalRex32(reg, address); 938 EmitUint8(0x39); 939 EmitOperand(reg.LowBits(), address); 940} 941 942 943void X86_64Assembler::cmpl(const Address& address, const Immediate& imm) { 944 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 945 EmitOptionalRex32(address); 946 EmitComplex(7, address, imm); 947} 948 949 950void X86_64Assembler::testl(CpuRegister reg1, CpuRegister reg2) { 951 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 952 EmitOptionalRex32(reg1, reg2); 953 EmitUint8(0x85); 954 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 955} 956 957 958void X86_64Assembler::testl(CpuRegister reg, const Immediate& immediate) { 959 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 960 // For registers that have a byte variant (RAX, RBX, RCX, and RDX) 961 // we only test the byte CpuRegister to keep the encoding short. 962 if (immediate.is_uint8() && reg.AsRegister() < 4) { 963 // Use zero-extended 8-bit immediate. 964 if (reg.AsRegister() == RAX) { 965 EmitUint8(0xA8); 966 } else { 967 EmitUint8(0xF6); 968 EmitUint8(0xC0 + reg.AsRegister()); 969 } 970 EmitUint8(immediate.value() & 0xFF); 971 } else if (reg.AsRegister() == RAX) { 972 // Use short form if the destination is RAX. 973 EmitUint8(0xA9); 974 EmitImmediate(immediate); 975 } else { 976 EmitOptionalRex32(reg); 977 EmitUint8(0xF7); 978 EmitOperand(0, Operand(reg)); 979 EmitImmediate(immediate); 980 } 981} 982 983 984void X86_64Assembler::testq(CpuRegister reg, const Address& address) { 985 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 986 EmitRex64(reg); 987 EmitUint8(0x85); 988 EmitOperand(reg.LowBits(), address); 989} 990 991 992void X86_64Assembler::andl(CpuRegister dst, CpuRegister src) { 993 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 994 EmitOptionalRex32(dst, src); 995 EmitUint8(0x23); 996 EmitOperand(dst.LowBits(), Operand(src)); 997} 998 999 1000void X86_64Assembler::andl(CpuRegister dst, const Immediate& imm) { 1001 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1002 EmitOptionalRex32(dst); 1003 EmitComplex(4, Operand(dst), imm); 1004} 1005 1006 1007void X86_64Assembler::andq(CpuRegister reg, const Immediate& imm) { 1008 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1009 CHECK(imm.is_int32()); // andq only supports 32b immediate. 1010 EmitRex64(reg); 1011 EmitComplex(4, Operand(reg), imm); 1012} 1013 1014 1015void X86_64Assembler::orl(CpuRegister dst, CpuRegister src) { 1016 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1017 EmitOptionalRex32(dst, src); 1018 EmitUint8(0x0B); 1019 EmitOperand(dst.LowBits(), Operand(src)); 1020} 1021 1022 1023void X86_64Assembler::orl(CpuRegister dst, const Immediate& imm) { 1024 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1025 EmitOptionalRex32(dst); 1026 EmitComplex(1, Operand(dst), imm); 1027} 1028 1029 1030void X86_64Assembler::xorl(CpuRegister dst, CpuRegister src) { 1031 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1032 EmitOptionalRex32(dst, src); 1033 EmitUint8(0x33); 1034 EmitOperand(dst.LowBits(), Operand(src)); 1035} 1036 1037 1038void X86_64Assembler::xorq(CpuRegister dst, CpuRegister src) { 1039 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1040 EmitRex64(dst, src); 1041 EmitUint8(0x33); 1042 EmitOperand(dst.LowBits(), Operand(src)); 1043} 1044 1045 1046void X86_64Assembler::xorq(CpuRegister dst, const Immediate& imm) { 1047 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1048 CHECK(imm.is_int32()); // xorq only supports 32b immediate. 1049 EmitRex64(dst); 1050 EmitComplex(6, Operand(dst), imm); 1051} 1052 1053#if 0 1054void X86_64Assembler::rex(bool force, bool w, Register* r, Register* x, Register* b) { 1055 // REX.WRXB 1056 // W - 64-bit operand 1057 // R - MODRM.reg 1058 // X - SIB.index 1059 // B - MODRM.rm/SIB.base 1060 uint8_t rex = force ? 0x40 : 0; 1061 if (w) { 1062 rex |= 0x48; // REX.W000 1063 } 1064 if (r != nullptr && *r >= Register::R8 && *r < Register::kNumberOfCpuRegisters) { 1065 rex |= 0x44; // REX.0R00 1066 *r = static_cast<Register>(*r - 8); 1067 } 1068 if (x != nullptr && *x >= Register::R8 && *x < Register::kNumberOfCpuRegisters) { 1069 rex |= 0x42; // REX.00X0 1070 *x = static_cast<Register>(*x - 8); 1071 } 1072 if (b != nullptr && *b >= Register::R8 && *b < Register::kNumberOfCpuRegisters) { 1073 rex |= 0x41; // REX.000B 1074 *b = static_cast<Register>(*b - 8); 1075 } 1076 if (rex != 0) { 1077 EmitUint8(rex); 1078 } 1079} 1080 1081void X86_64Assembler::rex_reg_mem(bool force, bool w, Register* dst, const Address& mem) { 1082 // REX.WRXB 1083 // W - 64-bit operand 1084 // R - MODRM.reg 1085 // X - SIB.index 1086 // B - MODRM.rm/SIB.base 1087 uint8_t rex = mem->rex(); 1088 if (force) { 1089 rex |= 0x40; // REX.0000 1090 } 1091 if (w) { 1092 rex |= 0x48; // REX.W000 1093 } 1094 if (dst != nullptr && *dst >= Register::R8 && *dst < Register::kNumberOfCpuRegisters) { 1095 rex |= 0x44; // REX.0R00 1096 *dst = static_cast<Register>(*dst - 8); 1097 } 1098 if (rex != 0) { 1099 EmitUint8(rex); 1100 } 1101} 1102 1103void rex_mem_reg(bool force, bool w, Address* mem, Register* src); 1104#endif 1105 1106void X86_64Assembler::addl(CpuRegister reg, const Immediate& imm) { 1107 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1108 EmitOptionalRex32(reg); 1109 EmitComplex(0, Operand(reg), imm); 1110} 1111 1112 1113void X86_64Assembler::addq(CpuRegister reg, const Immediate& imm) { 1114 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1115 CHECK(imm.is_int32()); // addq only supports 32b immediate. 1116 EmitRex64(reg); 1117 EmitComplex(0, Operand(reg), imm); 1118} 1119 1120 1121void X86_64Assembler::addq(CpuRegister dst, const Address& address) { 1122 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1123 EmitRex64(dst); 1124 EmitUint8(0x03); 1125 EmitOperand(dst.LowBits(), address); 1126} 1127 1128 1129void X86_64Assembler::addq(CpuRegister dst, CpuRegister src) { 1130 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1131 // 0x01 is addq r/m64 <- r/m64 + r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 1132 EmitRex64(src, dst); 1133 EmitUint8(0x01); 1134 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1135} 1136 1137 1138void X86_64Assembler::addl(const Address& address, CpuRegister reg) { 1139 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1140 EmitOptionalRex32(reg, address); 1141 EmitUint8(0x01); 1142 EmitOperand(reg.LowBits(), address); 1143} 1144 1145 1146void X86_64Assembler::addl(const Address& address, const Immediate& imm) { 1147 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1148 EmitOptionalRex32(address); 1149 EmitComplex(0, address, imm); 1150} 1151 1152 1153void X86_64Assembler::subl(CpuRegister dst, CpuRegister src) { 1154 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1155 EmitOptionalRex32(dst, src); 1156 EmitUint8(0x2B); 1157 EmitOperand(dst.LowBits(), Operand(src)); 1158} 1159 1160 1161void X86_64Assembler::subl(CpuRegister reg, const Immediate& imm) { 1162 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1163 EmitOptionalRex32(reg); 1164 EmitComplex(5, Operand(reg), imm); 1165} 1166 1167 1168void X86_64Assembler::subq(CpuRegister reg, const Immediate& imm) { 1169 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1170 CHECK(imm.is_int32()); // subq only supports 32b immediate. 1171 EmitRex64(reg); 1172 EmitComplex(5, Operand(reg), imm); 1173} 1174 1175 1176void X86_64Assembler::subq(CpuRegister dst, CpuRegister src) { 1177 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1178 EmitRex64(dst, src); 1179 EmitUint8(0x2B); 1180 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1181} 1182 1183 1184void X86_64Assembler::subq(CpuRegister reg, const Address& address) { 1185 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1186 EmitRex64(reg); 1187 EmitUint8(0x2B); 1188 EmitOperand(reg.LowBits() & 7, address); 1189} 1190 1191 1192void X86_64Assembler::subl(CpuRegister reg, const Address& address) { 1193 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1194 EmitOptionalRex32(reg, address); 1195 EmitUint8(0x2B); 1196 EmitOperand(reg.LowBits(), address); 1197} 1198 1199 1200void X86_64Assembler::cdq() { 1201 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1202 EmitUint8(0x99); 1203} 1204 1205 1206void X86_64Assembler::idivl(CpuRegister reg) { 1207 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1208 EmitOptionalRex32(reg); 1209 EmitUint8(0xF7); 1210 EmitUint8(0xF8 | reg.LowBits()); 1211} 1212 1213 1214void X86_64Assembler::imull(CpuRegister dst, CpuRegister src) { 1215 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1216 EmitOptionalRex32(dst, src); 1217 EmitUint8(0x0F); 1218 EmitUint8(0xAF); 1219 EmitOperand(dst.LowBits(), Operand(src)); 1220} 1221 1222 1223void X86_64Assembler::imull(CpuRegister reg, const Immediate& imm) { 1224 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1225 EmitOptionalRex32(reg); 1226 EmitUint8(0x69); 1227 EmitOperand(reg.LowBits(), Operand(reg)); 1228 EmitImmediate(imm); 1229} 1230 1231 1232void X86_64Assembler::imull(CpuRegister reg, const Address& address) { 1233 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1234 EmitOptionalRex32(reg, address); 1235 EmitUint8(0x0F); 1236 EmitUint8(0xAF); 1237 EmitOperand(reg.LowBits(), address); 1238} 1239 1240 1241void X86_64Assembler::imulq(CpuRegister dst, CpuRegister src) { 1242 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1243 EmitRex64(dst, src); 1244 EmitUint8(0x0F); 1245 EmitUint8(0xAF); 1246 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1247} 1248 1249 1250void X86_64Assembler::imulq(CpuRegister reg, const Immediate& imm) { 1251 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1252 CHECK(imm.is_int32()); // imulq only supports 32b immediate. 1253 EmitRex64(reg); 1254 EmitUint8(0x69); 1255 EmitOperand(reg.LowBits(), Operand(reg)); 1256 EmitImmediate(imm); 1257} 1258 1259 1260void X86_64Assembler::imulq(CpuRegister reg, const Address& address) { 1261 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1262 EmitRex64(reg, address); 1263 EmitUint8(0x0F); 1264 EmitUint8(0xAF); 1265 EmitOperand(reg.LowBits(), address); 1266} 1267 1268 1269void X86_64Assembler::imull(CpuRegister reg) { 1270 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1271 EmitOptionalRex32(reg); 1272 EmitUint8(0xF7); 1273 EmitOperand(5, Operand(reg)); 1274} 1275 1276 1277void X86_64Assembler::imull(const Address& address) { 1278 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1279 EmitOptionalRex32(address); 1280 EmitUint8(0xF7); 1281 EmitOperand(5, address); 1282} 1283 1284 1285void X86_64Assembler::mull(CpuRegister reg) { 1286 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1287 EmitOptionalRex32(reg); 1288 EmitUint8(0xF7); 1289 EmitOperand(4, Operand(reg)); 1290} 1291 1292 1293void X86_64Assembler::mull(const Address& address) { 1294 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1295 EmitOptionalRex32(address); 1296 EmitUint8(0xF7); 1297 EmitOperand(4, address); 1298} 1299 1300 1301void X86_64Assembler::shll(CpuRegister reg, const Immediate& imm) { 1302 EmitGenericShift(false, 4, reg, imm); 1303} 1304 1305 1306void X86_64Assembler::shll(CpuRegister operand, CpuRegister shifter) { 1307 EmitGenericShift(4, operand, shifter); 1308} 1309 1310 1311void X86_64Assembler::shrl(CpuRegister reg, const Immediate& imm) { 1312 EmitGenericShift(false, 5, reg, imm); 1313} 1314 1315 1316void X86_64Assembler::shrq(CpuRegister reg, const Immediate& imm) { 1317 EmitGenericShift(true, 5, reg, imm); 1318} 1319 1320 1321void X86_64Assembler::shrl(CpuRegister operand, CpuRegister shifter) { 1322 EmitGenericShift(5, operand, shifter); 1323} 1324 1325 1326void X86_64Assembler::sarl(CpuRegister reg, const Immediate& imm) { 1327 EmitGenericShift(false, 7, reg, imm); 1328} 1329 1330 1331void X86_64Assembler::sarl(CpuRegister operand, CpuRegister shifter) { 1332 EmitGenericShift(7, operand, shifter); 1333} 1334 1335 1336void X86_64Assembler::negl(CpuRegister reg) { 1337 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1338 EmitOptionalRex32(reg); 1339 EmitUint8(0xF7); 1340 EmitOperand(3, Operand(reg)); 1341} 1342 1343 1344void X86_64Assembler::notl(CpuRegister reg) { 1345 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1346 EmitOptionalRex32(reg); 1347 EmitUint8(0xF7); 1348 EmitUint8(0xD0 | reg.LowBits()); 1349} 1350 1351 1352void X86_64Assembler::enter(const Immediate& imm) { 1353 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1354 EmitUint8(0xC8); 1355 CHECK(imm.is_uint16()); 1356 EmitUint8(imm.value() & 0xFF); 1357 EmitUint8((imm.value() >> 8) & 0xFF); 1358 EmitUint8(0x00); 1359} 1360 1361 1362void X86_64Assembler::leave() { 1363 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1364 EmitUint8(0xC9); 1365} 1366 1367 1368void X86_64Assembler::ret() { 1369 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1370 EmitUint8(0xC3); 1371} 1372 1373 1374void X86_64Assembler::ret(const Immediate& imm) { 1375 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1376 EmitUint8(0xC2); 1377 CHECK(imm.is_uint16()); 1378 EmitUint8(imm.value() & 0xFF); 1379 EmitUint8((imm.value() >> 8) & 0xFF); 1380} 1381 1382 1383 1384void X86_64Assembler::nop() { 1385 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1386 EmitUint8(0x90); 1387} 1388 1389 1390void X86_64Assembler::int3() { 1391 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1392 EmitUint8(0xCC); 1393} 1394 1395 1396void X86_64Assembler::hlt() { 1397 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1398 EmitUint8(0xF4); 1399} 1400 1401 1402void X86_64Assembler::j(Condition condition, Label* label) { 1403 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1404 if (label->IsBound()) { 1405 static const int kShortSize = 2; 1406 static const int kLongSize = 6; 1407 int offset = label->Position() - buffer_.Size(); 1408 CHECK_LE(offset, 0); 1409 if (IsInt(8, offset - kShortSize)) { 1410 EmitUint8(0x70 + condition); 1411 EmitUint8((offset - kShortSize) & 0xFF); 1412 } else { 1413 EmitUint8(0x0F); 1414 EmitUint8(0x80 + condition); 1415 EmitInt32(offset - kLongSize); 1416 } 1417 } else { 1418 EmitUint8(0x0F); 1419 EmitUint8(0x80 + condition); 1420 EmitLabelLink(label); 1421 } 1422} 1423 1424 1425void X86_64Assembler::jmp(CpuRegister reg) { 1426 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1427 EmitOptionalRex32(reg); 1428 EmitUint8(0xFF); 1429 EmitRegisterOperand(4, reg.LowBits()); 1430} 1431 1432void X86_64Assembler::jmp(const Address& address) { 1433 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1434 EmitOptionalRex32(address); 1435 EmitUint8(0xFF); 1436 EmitOperand(4, address); 1437} 1438 1439void X86_64Assembler::jmp(Label* label) { 1440 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1441 if (label->IsBound()) { 1442 static const int kShortSize = 2; 1443 static const int kLongSize = 5; 1444 int offset = label->Position() - buffer_.Size(); 1445 CHECK_LE(offset, 0); 1446 if (IsInt(8, offset - kShortSize)) { 1447 EmitUint8(0xEB); 1448 EmitUint8((offset - kShortSize) & 0xFF); 1449 } else { 1450 EmitUint8(0xE9); 1451 EmitInt32(offset - kLongSize); 1452 } 1453 } else { 1454 EmitUint8(0xE9); 1455 EmitLabelLink(label); 1456 } 1457} 1458 1459 1460X86_64Assembler* X86_64Assembler::lock() { 1461 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1462 EmitUint8(0xF0); 1463 return this; 1464} 1465 1466 1467void X86_64Assembler::cmpxchgl(const Address& address, CpuRegister reg) { 1468 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1469 EmitUint8(0x0F); 1470 EmitUint8(0xB1); 1471 EmitOperand(reg.LowBits(), address); 1472} 1473 1474void X86_64Assembler::mfence() { 1475 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1476 EmitUint8(0x0F); 1477 EmitUint8(0xAE); 1478 EmitUint8(0xF0); 1479} 1480 1481 1482X86_64Assembler* X86_64Assembler::gs() { 1483 // TODO: gs is a prefix and not an instruction 1484 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1485 EmitUint8(0x65); 1486 return this; 1487} 1488 1489 1490void X86_64Assembler::AddImmediate(CpuRegister reg, const Immediate& imm) { 1491 int value = imm.value(); 1492 if (value != 0) { 1493 if (value > 0) { 1494 addl(reg, imm); 1495 } else { 1496 subl(reg, Immediate(value)); 1497 } 1498 } 1499} 1500 1501 1502void X86_64Assembler::setcc(Condition condition, CpuRegister dst) { 1503 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1504 // RSP, RBP, RDI, RSI need rex prefix (else the pattern encodes ah/bh/ch/dh). 1505 if (dst.NeedsRex() || dst.AsRegister() > 3) { 1506 EmitOptionalRex(true, false, false, false, dst.NeedsRex()); 1507 } 1508 EmitUint8(0x0F); 1509 EmitUint8(0x90 + condition); 1510 EmitUint8(0xC0 + dst.LowBits()); 1511} 1512 1513 1514void X86_64Assembler::LoadDoubleConstant(XmmRegister dst, double value) { 1515 // TODO: Need to have a code constants table. 1516 int64_t constant = bit_cast<int64_t, double>(value); 1517 pushq(Immediate(High32Bits(constant))); 1518 pushq(Immediate(Low32Bits(constant))); 1519 movsd(dst, Address(CpuRegister(RSP), 0)); 1520 addq(CpuRegister(RSP), Immediate(2 * sizeof(intptr_t))); 1521} 1522 1523 1524void X86_64Assembler::FloatNegate(XmmRegister f) { 1525 static const struct { 1526 uint32_t a; 1527 uint32_t b; 1528 uint32_t c; 1529 uint32_t d; 1530 } float_negate_constant __attribute__((aligned(16))) = 1531 { 0x80000000, 0x00000000, 0x80000000, 0x00000000 }; 1532 xorps(f, Address::Absolute(reinterpret_cast<uintptr_t>(&float_negate_constant))); 1533} 1534 1535 1536void X86_64Assembler::DoubleNegate(XmmRegister d) { 1537 static const struct { 1538 uint64_t a; 1539 uint64_t b; 1540 } double_negate_constant __attribute__((aligned(16))) = 1541 {0x8000000000000000LL, 0x8000000000000000LL}; 1542 xorpd(d, Address::Absolute(reinterpret_cast<uintptr_t>(&double_negate_constant))); 1543} 1544 1545 1546void X86_64Assembler::DoubleAbs(XmmRegister reg) { 1547 static const struct { 1548 uint64_t a; 1549 uint64_t b; 1550 } double_abs_constant __attribute__((aligned(16))) = 1551 {0x7FFFFFFFFFFFFFFFLL, 0x7FFFFFFFFFFFFFFFLL}; 1552 andpd(reg, Address::Absolute(reinterpret_cast<uintptr_t>(&double_abs_constant))); 1553} 1554 1555 1556void X86_64Assembler::Align(int alignment, int offset) { 1557 CHECK(IsPowerOfTwo(alignment)); 1558 // Emit nop instruction until the real position is aligned. 1559 while (((offset + buffer_.GetPosition()) & (alignment-1)) != 0) { 1560 nop(); 1561 } 1562} 1563 1564 1565void X86_64Assembler::Bind(Label* label) { 1566 int bound = buffer_.Size(); 1567 CHECK(!label->IsBound()); // Labels can only be bound once. 1568 while (label->IsLinked()) { 1569 int position = label->LinkPosition(); 1570 int next = buffer_.Load<int32_t>(position); 1571 buffer_.Store<int32_t>(position, bound - (position + 4)); 1572 label->position_ = next; 1573 } 1574 label->BindTo(bound); 1575} 1576 1577 1578void X86_64Assembler::EmitOperand(uint8_t reg_or_opcode, const Operand& operand) { 1579 CHECK_GE(reg_or_opcode, 0); 1580 CHECK_LT(reg_or_opcode, 8); 1581 const int length = operand.length_; 1582 CHECK_GT(length, 0); 1583 // Emit the ModRM byte updated with the given reg value. 1584 CHECK_EQ(operand.encoding_[0] & 0x38, 0); 1585 EmitUint8(operand.encoding_[0] + (reg_or_opcode << 3)); 1586 // Emit the rest of the encoded operand. 1587 for (int i = 1; i < length; i++) { 1588 EmitUint8(operand.encoding_[i]); 1589 } 1590} 1591 1592 1593void X86_64Assembler::EmitImmediate(const Immediate& imm) { 1594 if (imm.is_int32()) { 1595 EmitInt32(static_cast<int32_t>(imm.value())); 1596 } else { 1597 EmitInt64(imm.value()); 1598 } 1599} 1600 1601 1602void X86_64Assembler::EmitComplex(uint8_t reg_or_opcode, 1603 const Operand& operand, 1604 const Immediate& immediate) { 1605 CHECK_GE(reg_or_opcode, 0); 1606 CHECK_LT(reg_or_opcode, 8); 1607 if (immediate.is_int8()) { 1608 // Use sign-extended 8-bit immediate. 1609 EmitUint8(0x83); 1610 EmitOperand(reg_or_opcode, operand); 1611 EmitUint8(immediate.value() & 0xFF); 1612 } else if (operand.IsRegister(CpuRegister(RAX))) { 1613 // Use short form if the destination is eax. 1614 EmitUint8(0x05 + (reg_or_opcode << 3)); 1615 EmitImmediate(immediate); 1616 } else { 1617 EmitUint8(0x81); 1618 EmitOperand(reg_or_opcode, operand); 1619 EmitImmediate(immediate); 1620 } 1621} 1622 1623 1624void X86_64Assembler::EmitLabel(Label* label, int instruction_size) { 1625 if (label->IsBound()) { 1626 int offset = label->Position() - buffer_.Size(); 1627 CHECK_LE(offset, 0); 1628 EmitInt32(offset - instruction_size); 1629 } else { 1630 EmitLabelLink(label); 1631 } 1632} 1633 1634 1635void X86_64Assembler::EmitLabelLink(Label* label) { 1636 CHECK(!label->IsBound()); 1637 int position = buffer_.Size(); 1638 EmitInt32(label->position_); 1639 label->LinkTo(position); 1640} 1641 1642 1643void X86_64Assembler::EmitGenericShift(bool wide, 1644 int reg_or_opcode, 1645 CpuRegister reg, 1646 const Immediate& imm) { 1647 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1648 CHECK(imm.is_int8()); 1649 if (wide) { 1650 EmitRex64(reg); 1651 } 1652 if (imm.value() == 1) { 1653 EmitUint8(0xD1); 1654 EmitOperand(reg_or_opcode, Operand(reg)); 1655 } else { 1656 EmitUint8(0xC1); 1657 EmitOperand(reg_or_opcode, Operand(reg)); 1658 EmitUint8(imm.value() & 0xFF); 1659 } 1660} 1661 1662 1663void X86_64Assembler::EmitGenericShift(int reg_or_opcode, 1664 CpuRegister operand, 1665 CpuRegister shifter) { 1666 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1667 CHECK_EQ(shifter.AsRegister(), RCX); 1668 EmitUint8(0xD3); 1669 EmitOperand(reg_or_opcode, Operand(operand)); 1670} 1671 1672void X86_64Assembler::EmitOptionalRex(bool force, bool w, bool r, bool x, bool b) { 1673 // REX.WRXB 1674 // W - 64-bit operand 1675 // R - MODRM.reg 1676 // X - SIB.index 1677 // B - MODRM.rm/SIB.base 1678 uint8_t rex = force ? 0x40 : 0; 1679 if (w) { 1680 rex |= 0x48; // REX.W000 1681 } 1682 if (r) { 1683 rex |= 0x44; // REX.0R00 1684 } 1685 if (x) { 1686 rex |= 0x42; // REX.00X0 1687 } 1688 if (b) { 1689 rex |= 0x41; // REX.000B 1690 } 1691 if (rex != 0) { 1692 EmitUint8(rex); 1693 } 1694} 1695 1696void X86_64Assembler::EmitOptionalRex32(CpuRegister reg) { 1697 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 1698} 1699 1700void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, CpuRegister src) { 1701 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1702} 1703 1704void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, XmmRegister src) { 1705 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1706} 1707 1708void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, XmmRegister src) { 1709 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1710} 1711 1712void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, CpuRegister src) { 1713 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1714} 1715 1716void X86_64Assembler::EmitOptionalRex32(const Operand& operand) { 1717 uint8_t rex = operand.rex(); 1718 if (rex != 0) { 1719 EmitUint8(rex); 1720 } 1721} 1722 1723void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, const Operand& operand) { 1724 uint8_t rex = operand.rex(); 1725 if (dst.NeedsRex()) { 1726 rex |= 0x44; // REX.0R00 1727 } 1728 if (rex != 0) { 1729 EmitUint8(rex); 1730 } 1731} 1732 1733void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, const Operand& operand) { 1734 uint8_t rex = operand.rex(); 1735 if (dst.NeedsRex()) { 1736 rex |= 0x44; // REX.0R00 1737 } 1738 if (rex != 0) { 1739 EmitUint8(rex); 1740 } 1741} 1742 1743void X86_64Assembler::EmitRex64(CpuRegister reg) { 1744 EmitOptionalRex(false, true, false, false, reg.NeedsRex()); 1745} 1746 1747void X86_64Assembler::EmitRex64(CpuRegister dst, CpuRegister src) { 1748 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 1749} 1750 1751void X86_64Assembler::EmitRex64(XmmRegister dst, CpuRegister src) { 1752 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 1753} 1754 1755void X86_64Assembler::EmitRex64(CpuRegister dst, const Operand& operand) { 1756 uint8_t rex = 0x48 | operand.rex(); // REX.W000 1757 if (dst.NeedsRex()) { 1758 rex |= 0x44; // REX.0R00 1759 } 1760 if (rex != 0) { 1761 EmitUint8(rex); 1762 } 1763} 1764 1765void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, CpuRegister src) { 1766 EmitOptionalRex(true, false, dst.NeedsRex(), false, src.NeedsRex()); 1767} 1768 1769void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand) { 1770 uint8_t rex = 0x40 | operand.rex(); // REX.0000 1771 if (dst.NeedsRex()) { 1772 rex |= 0x44; // REX.0R00 1773 } 1774 if (rex != 0) { 1775 EmitUint8(rex); 1776 } 1777} 1778 1779void X86_64Assembler::InitializeFrameDescriptionEntry() { 1780 WriteFDEHeader(&cfi_info_, true /* is_64bit */); 1781} 1782 1783void X86_64Assembler::FinalizeFrameDescriptionEntry() { 1784 WriteFDEAddressRange(&cfi_info_, buffer_.Size(), true /* is_64bit */); 1785 PadCFI(&cfi_info_); 1786 WriteCFILength(&cfi_info_, true /* is_64bit */); 1787} 1788 1789constexpr size_t kFramePointerSize = 8; 1790 1791void X86_64Assembler::BuildFrame(size_t frame_size, ManagedRegister method_reg, 1792 const std::vector<ManagedRegister>& spill_regs, 1793 const ManagedRegisterEntrySpills& entry_spills) { 1794 cfi_cfa_offset_ = kFramePointerSize; // Only return address on stack 1795 cfi_pc_ = buffer_.Size(); // Nothing emitted yet 1796 DCHECK_EQ(cfi_pc_, 0U); 1797 1798 uint32_t reg_offset = 1; 1799 CHECK_ALIGNED(frame_size, kStackAlignment); 1800 int gpr_count = 0; 1801 for (int i = spill_regs.size() - 1; i >= 0; --i) { 1802 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 1803 if (spill.IsCpuRegister()) { 1804 pushq(spill.AsCpuRegister()); 1805 gpr_count++; 1806 1807 // DW_CFA_advance_loc 1808 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 1809 cfi_pc_ = buffer_.Size(); 1810 // DW_CFA_def_cfa_offset 1811 cfi_cfa_offset_ += kFramePointerSize; 1812 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 1813 // DW_CFA_offset reg offset 1814 reg_offset++; 1815 DW_CFA_offset(&cfi_info_, spill.DWARFRegId(), reg_offset); 1816 } 1817 } 1818 // return address then method on stack 1819 int64_t rest_of_frame = static_cast<int64_t>(frame_size) 1820 - (gpr_count * kFramePointerSize) 1821 - kFramePointerSize /*return address*/; 1822 subq(CpuRegister(RSP), Immediate(rest_of_frame)); 1823 // DW_CFA_advance_loc 1824 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 1825 cfi_pc_ = buffer_.Size(); 1826 // DW_CFA_def_cfa_offset 1827 cfi_cfa_offset_ += rest_of_frame; 1828 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 1829 1830 // spill xmms 1831 int64_t offset = rest_of_frame; 1832 for (int i = spill_regs.size() - 1; i >= 0; --i) { 1833 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 1834 if (spill.IsXmmRegister()) { 1835 offset -= sizeof(double); 1836 movsd(Address(CpuRegister(RSP), offset), spill.AsXmmRegister()); 1837 } 1838 } 1839 1840 DCHECK_EQ(4U, sizeof(StackReference<mirror::ArtMethod>)); 1841 1842 movl(Address(CpuRegister(RSP), 0), method_reg.AsX86_64().AsCpuRegister()); 1843 1844 for (size_t i = 0; i < entry_spills.size(); ++i) { 1845 ManagedRegisterSpill spill = entry_spills.at(i); 1846 if (spill.AsX86_64().IsCpuRegister()) { 1847 if (spill.getSize() == 8) { 1848 movq(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), 1849 spill.AsX86_64().AsCpuRegister()); 1850 } else { 1851 CHECK_EQ(spill.getSize(), 4); 1852 movl(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsCpuRegister()); 1853 } 1854 } else { 1855 if (spill.getSize() == 8) { 1856 movsd(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 1857 } else { 1858 CHECK_EQ(spill.getSize(), 4); 1859 movss(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 1860 } 1861 } 1862 } 1863} 1864 1865void X86_64Assembler::RemoveFrame(size_t frame_size, 1866 const std::vector<ManagedRegister>& spill_regs) { 1867 CHECK_ALIGNED(frame_size, kStackAlignment); 1868 int gpr_count = 0; 1869 // unspill xmms 1870 int64_t offset = static_cast<int64_t>(frame_size) - (spill_regs.size() * kFramePointerSize) - 2 * kFramePointerSize; 1871 for (size_t i = 0; i < spill_regs.size(); ++i) { 1872 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 1873 if (spill.IsXmmRegister()) { 1874 offset += sizeof(double); 1875 movsd(spill.AsXmmRegister(), Address(CpuRegister(RSP), offset)); 1876 } else { 1877 gpr_count++; 1878 } 1879 } 1880 addq(CpuRegister(RSP), Immediate(static_cast<int64_t>(frame_size) - (gpr_count * kFramePointerSize) - kFramePointerSize)); 1881 for (size_t i = 0; i < spill_regs.size(); ++i) { 1882 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 1883 if (spill.IsCpuRegister()) { 1884 popq(spill.AsCpuRegister()); 1885 } 1886 } 1887 ret(); 1888} 1889 1890void X86_64Assembler::IncreaseFrameSize(size_t adjust) { 1891 CHECK_ALIGNED(adjust, kStackAlignment); 1892 addq(CpuRegister(RSP), Immediate(-static_cast<int64_t>(adjust))); 1893 // DW_CFA_advance_loc 1894 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 1895 cfi_pc_ = buffer_.Size(); 1896 // DW_CFA_def_cfa_offset 1897 cfi_cfa_offset_ += adjust; 1898 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 1899} 1900 1901void X86_64Assembler::DecreaseFrameSize(size_t adjust) { 1902 CHECK_ALIGNED(adjust, kStackAlignment); 1903 addq(CpuRegister(RSP), Immediate(adjust)); 1904} 1905 1906void X86_64Assembler::Store(FrameOffset offs, ManagedRegister msrc, size_t size) { 1907 X86_64ManagedRegister src = msrc.AsX86_64(); 1908 if (src.IsNoRegister()) { 1909 CHECK_EQ(0u, size); 1910 } else if (src.IsCpuRegister()) { 1911 if (size == 4) { 1912 CHECK_EQ(4u, size); 1913 movl(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 1914 } else { 1915 CHECK_EQ(8u, size); 1916 movq(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 1917 } 1918 } else if (src.IsRegisterPair()) { 1919 CHECK_EQ(0u, size); 1920 movq(Address(CpuRegister(RSP), offs), src.AsRegisterPairLow()); 1921 movq(Address(CpuRegister(RSP), FrameOffset(offs.Int32Value()+4)), 1922 src.AsRegisterPairHigh()); 1923 } else if (src.IsX87Register()) { 1924 if (size == 4) { 1925 fstps(Address(CpuRegister(RSP), offs)); 1926 } else { 1927 fstpl(Address(CpuRegister(RSP), offs)); 1928 } 1929 } else { 1930 CHECK(src.IsXmmRegister()); 1931 if (size == 4) { 1932 movss(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 1933 } else { 1934 movsd(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 1935 } 1936 } 1937} 1938 1939void X86_64Assembler::StoreRef(FrameOffset dest, ManagedRegister msrc) { 1940 X86_64ManagedRegister src = msrc.AsX86_64(); 1941 CHECK(src.IsCpuRegister()); 1942 movl(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 1943} 1944 1945void X86_64Assembler::StoreRawPtr(FrameOffset dest, ManagedRegister msrc) { 1946 X86_64ManagedRegister src = msrc.AsX86_64(); 1947 CHECK(src.IsCpuRegister()); 1948 movq(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 1949} 1950 1951void X86_64Assembler::StoreImmediateToFrame(FrameOffset dest, uint32_t imm, 1952 ManagedRegister) { 1953 movl(Address(CpuRegister(RSP), dest), Immediate(imm)); // TODO(64) movq? 1954} 1955 1956void X86_64Assembler::StoreImmediateToThread64(ThreadOffset<8> dest, uint32_t imm, 1957 ManagedRegister) { 1958 gs()->movl(Address::Absolute(dest, true), Immediate(imm)); // TODO(64) movq? 1959} 1960 1961void X86_64Assembler::StoreStackOffsetToThread64(ThreadOffset<8> thr_offs, 1962 FrameOffset fr_offs, 1963 ManagedRegister mscratch) { 1964 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 1965 CHECK(scratch.IsCpuRegister()); 1966 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), fr_offs)); 1967 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 1968} 1969 1970void X86_64Assembler::StoreStackPointerToThread64(ThreadOffset<8> thr_offs) { 1971 gs()->movq(Address::Absolute(thr_offs, true), CpuRegister(RSP)); 1972} 1973 1974void X86_64Assembler::StoreSpanning(FrameOffset /*dst*/, ManagedRegister /*src*/, 1975 FrameOffset /*in_off*/, ManagedRegister /*scratch*/) { 1976 UNIMPLEMENTED(FATAL); // this case only currently exists for ARM 1977} 1978 1979void X86_64Assembler::Load(ManagedRegister mdest, FrameOffset src, size_t size) { 1980 X86_64ManagedRegister dest = mdest.AsX86_64(); 1981 if (dest.IsNoRegister()) { 1982 CHECK_EQ(0u, size); 1983 } else if (dest.IsCpuRegister()) { 1984 if (size == 4) { 1985 CHECK_EQ(4u, size); 1986 movl(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 1987 } else { 1988 CHECK_EQ(8u, size); 1989 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 1990 } 1991 } else if (dest.IsRegisterPair()) { 1992 CHECK_EQ(0u, size); 1993 movq(dest.AsRegisterPairLow(), Address(CpuRegister(RSP), src)); 1994 movq(dest.AsRegisterPairHigh(), Address(CpuRegister(RSP), FrameOffset(src.Int32Value()+4))); 1995 } else if (dest.IsX87Register()) { 1996 if (size == 4) { 1997 flds(Address(CpuRegister(RSP), src)); 1998 } else { 1999 fldl(Address(CpuRegister(RSP), src)); 2000 } 2001 } else { 2002 CHECK(dest.IsXmmRegister()); 2003 if (size == 4) { 2004 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2005 } else { 2006 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2007 } 2008 } 2009} 2010 2011void X86_64Assembler::LoadFromThread64(ManagedRegister mdest, ThreadOffset<8> src, size_t size) { 2012 X86_64ManagedRegister dest = mdest.AsX86_64(); 2013 if (dest.IsNoRegister()) { 2014 CHECK_EQ(0u, size); 2015 } else if (dest.IsCpuRegister()) { 2016 CHECK_EQ(4u, size); 2017 gs()->movl(dest.AsCpuRegister(), Address::Absolute(src, true)); 2018 } else if (dest.IsRegisterPair()) { 2019 CHECK_EQ(8u, size); 2020 gs()->movq(dest.AsRegisterPairLow(), Address::Absolute(src, true)); 2021 } else if (dest.IsX87Register()) { 2022 if (size == 4) { 2023 gs()->flds(Address::Absolute(src, true)); 2024 } else { 2025 gs()->fldl(Address::Absolute(src, true)); 2026 } 2027 } else { 2028 CHECK(dest.IsXmmRegister()); 2029 if (size == 4) { 2030 gs()->movss(dest.AsXmmRegister(), Address::Absolute(src, true)); 2031 } else { 2032 gs()->movsd(dest.AsXmmRegister(), Address::Absolute(src, true)); 2033 } 2034 } 2035} 2036 2037void X86_64Assembler::LoadRef(ManagedRegister mdest, FrameOffset src) { 2038 X86_64ManagedRegister dest = mdest.AsX86_64(); 2039 CHECK(dest.IsCpuRegister()); 2040 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2041} 2042 2043void X86_64Assembler::LoadRef(ManagedRegister mdest, ManagedRegister base, 2044 MemberOffset offs) { 2045 X86_64ManagedRegister dest = mdest.AsX86_64(); 2046 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2047 movl(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2048 if (kPoisonHeapReferences) { 2049 negl(dest.AsCpuRegister()); 2050 } 2051} 2052 2053void X86_64Assembler::LoadRawPtr(ManagedRegister mdest, ManagedRegister base, 2054 Offset offs) { 2055 X86_64ManagedRegister dest = mdest.AsX86_64(); 2056 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2057 movq(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2058} 2059 2060void X86_64Assembler::LoadRawPtrFromThread64(ManagedRegister mdest, ThreadOffset<8> offs) { 2061 X86_64ManagedRegister dest = mdest.AsX86_64(); 2062 CHECK(dest.IsCpuRegister()); 2063 gs()->movq(dest.AsCpuRegister(), Address::Absolute(offs, true)); 2064} 2065 2066void X86_64Assembler::SignExtend(ManagedRegister mreg, size_t size) { 2067 X86_64ManagedRegister reg = mreg.AsX86_64(); 2068 CHECK(size == 1 || size == 2) << size; 2069 CHECK(reg.IsCpuRegister()) << reg; 2070 if (size == 1) { 2071 movsxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2072 } else { 2073 movsxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2074 } 2075} 2076 2077void X86_64Assembler::ZeroExtend(ManagedRegister mreg, size_t size) { 2078 X86_64ManagedRegister reg = mreg.AsX86_64(); 2079 CHECK(size == 1 || size == 2) << size; 2080 CHECK(reg.IsCpuRegister()) << reg; 2081 if (size == 1) { 2082 movzxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2083 } else { 2084 movzxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2085 } 2086} 2087 2088void X86_64Assembler::Move(ManagedRegister mdest, ManagedRegister msrc, size_t size) { 2089 X86_64ManagedRegister dest = mdest.AsX86_64(); 2090 X86_64ManagedRegister src = msrc.AsX86_64(); 2091 if (!dest.Equals(src)) { 2092 if (dest.IsCpuRegister() && src.IsCpuRegister()) { 2093 movq(dest.AsCpuRegister(), src.AsCpuRegister()); 2094 } else if (src.IsX87Register() && dest.IsXmmRegister()) { 2095 // Pass via stack and pop X87 register 2096 subl(CpuRegister(RSP), Immediate(16)); 2097 if (size == 4) { 2098 CHECK_EQ(src.AsX87Register(), ST0); 2099 fstps(Address(CpuRegister(RSP), 0)); 2100 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2101 } else { 2102 CHECK_EQ(src.AsX87Register(), ST0); 2103 fstpl(Address(CpuRegister(RSP), 0)); 2104 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2105 } 2106 addq(CpuRegister(RSP), Immediate(16)); 2107 } else { 2108 // TODO: x87, SSE 2109 UNIMPLEMENTED(FATAL) << ": Move " << dest << ", " << src; 2110 } 2111 } 2112} 2113 2114void X86_64Assembler::CopyRef(FrameOffset dest, FrameOffset src, 2115 ManagedRegister mscratch) { 2116 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2117 CHECK(scratch.IsCpuRegister()); 2118 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2119 movl(Address(CpuRegister(RSP), dest), scratch.AsCpuRegister()); 2120} 2121 2122void X86_64Assembler::CopyRawPtrFromThread64(FrameOffset fr_offs, 2123 ThreadOffset<8> thr_offs, 2124 ManagedRegister mscratch) { 2125 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2126 CHECK(scratch.IsCpuRegister()); 2127 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(thr_offs, true)); 2128 Store(fr_offs, scratch, 8); 2129} 2130 2131void X86_64Assembler::CopyRawPtrToThread64(ThreadOffset<8> thr_offs, 2132 FrameOffset fr_offs, 2133 ManagedRegister mscratch) { 2134 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2135 CHECK(scratch.IsCpuRegister()); 2136 Load(scratch, fr_offs, 8); 2137 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2138} 2139 2140void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src, 2141 ManagedRegister mscratch, 2142 size_t size) { 2143 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2144 if (scratch.IsCpuRegister() && size == 8) { 2145 Load(scratch, src, 4); 2146 Store(dest, scratch, 4); 2147 Load(scratch, FrameOffset(src.Int32Value() + 4), 4); 2148 Store(FrameOffset(dest.Int32Value() + 4), scratch, 4); 2149 } else { 2150 Load(scratch, src, size); 2151 Store(dest, scratch, size); 2152 } 2153} 2154 2155void X86_64Assembler::Copy(FrameOffset /*dst*/, ManagedRegister /*src_base*/, Offset /*src_offset*/, 2156 ManagedRegister /*scratch*/, size_t /*size*/) { 2157 UNIMPLEMENTED(FATAL); 2158} 2159 2160void X86_64Assembler::Copy(ManagedRegister dest_base, Offset dest_offset, FrameOffset src, 2161 ManagedRegister scratch, size_t size) { 2162 CHECK(scratch.IsNoRegister()); 2163 CHECK_EQ(size, 4u); 2164 pushq(Address(CpuRegister(RSP), src)); 2165 popq(Address(dest_base.AsX86_64().AsCpuRegister(), dest_offset)); 2166} 2167 2168void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src_base, Offset src_offset, 2169 ManagedRegister mscratch, size_t size) { 2170 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2171 CHECK_EQ(size, 4u); 2172 movq(scratch, Address(CpuRegister(RSP), src_base)); 2173 movq(scratch, Address(scratch, src_offset)); 2174 movq(Address(CpuRegister(RSP), dest), scratch); 2175} 2176 2177void X86_64Assembler::Copy(ManagedRegister dest, Offset dest_offset, 2178 ManagedRegister src, Offset src_offset, 2179 ManagedRegister scratch, size_t size) { 2180 CHECK_EQ(size, 4u); 2181 CHECK(scratch.IsNoRegister()); 2182 pushq(Address(src.AsX86_64().AsCpuRegister(), src_offset)); 2183 popq(Address(dest.AsX86_64().AsCpuRegister(), dest_offset)); 2184} 2185 2186void X86_64Assembler::Copy(FrameOffset dest, Offset dest_offset, FrameOffset src, Offset src_offset, 2187 ManagedRegister mscratch, size_t size) { 2188 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2189 CHECK_EQ(size, 4u); 2190 CHECK_EQ(dest.Int32Value(), src.Int32Value()); 2191 movq(scratch, Address(CpuRegister(RSP), src)); 2192 pushq(Address(scratch, src_offset)); 2193 popq(Address(scratch, dest_offset)); 2194} 2195 2196void X86_64Assembler::MemoryBarrier(ManagedRegister) { 2197#if ANDROID_SMP != 0 2198 mfence(); 2199#endif 2200} 2201 2202void X86_64Assembler::CreateHandleScopeEntry(ManagedRegister mout_reg, 2203 FrameOffset handle_scope_offset, 2204 ManagedRegister min_reg, bool null_allowed) { 2205 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2206 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2207 if (in_reg.IsNoRegister()) { // TODO(64): && null_allowed 2208 // Use out_reg as indicator of NULL 2209 in_reg = out_reg; 2210 // TODO: movzwl 2211 movl(in_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2212 } 2213 CHECK(in_reg.IsCpuRegister()); 2214 CHECK(out_reg.IsCpuRegister()); 2215 VerifyObject(in_reg, null_allowed); 2216 if (null_allowed) { 2217 Label null_arg; 2218 if (!out_reg.Equals(in_reg)) { 2219 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2220 } 2221 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2222 j(kZero, &null_arg); 2223 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2224 Bind(&null_arg); 2225 } else { 2226 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2227 } 2228} 2229 2230void X86_64Assembler::CreateHandleScopeEntry(FrameOffset out_off, 2231 FrameOffset handle_scope_offset, 2232 ManagedRegister mscratch, 2233 bool null_allowed) { 2234 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2235 CHECK(scratch.IsCpuRegister()); 2236 if (null_allowed) { 2237 Label null_arg; 2238 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2239 testl(scratch.AsCpuRegister(), scratch.AsCpuRegister()); 2240 j(kZero, &null_arg); 2241 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2242 Bind(&null_arg); 2243 } else { 2244 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2245 } 2246 Store(out_off, scratch, 8); 2247} 2248 2249// Given a handle scope entry, load the associated reference. 2250void X86_64Assembler::LoadReferenceFromHandleScope(ManagedRegister mout_reg, 2251 ManagedRegister min_reg) { 2252 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2253 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2254 CHECK(out_reg.IsCpuRegister()); 2255 CHECK(in_reg.IsCpuRegister()); 2256 Label null_arg; 2257 if (!out_reg.Equals(in_reg)) { 2258 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2259 } 2260 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2261 j(kZero, &null_arg); 2262 movq(out_reg.AsCpuRegister(), Address(in_reg.AsCpuRegister(), 0)); 2263 Bind(&null_arg); 2264} 2265 2266void X86_64Assembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) { 2267 // TODO: not validating references 2268} 2269 2270void X86_64Assembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) { 2271 // TODO: not validating references 2272} 2273 2274void X86_64Assembler::Call(ManagedRegister mbase, Offset offset, ManagedRegister) { 2275 X86_64ManagedRegister base = mbase.AsX86_64(); 2276 CHECK(base.IsCpuRegister()); 2277 call(Address(base.AsCpuRegister(), offset.Int32Value())); 2278 // TODO: place reference map on call 2279} 2280 2281void X86_64Assembler::Call(FrameOffset base, Offset offset, ManagedRegister mscratch) { 2282 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2283 movl(scratch, Address(CpuRegister(RSP), base)); 2284 call(Address(scratch, offset)); 2285} 2286 2287void X86_64Assembler::CallFromThread64(ThreadOffset<8> offset, ManagedRegister /*mscratch*/) { 2288 gs()->call(Address::Absolute(offset, true)); 2289} 2290 2291void X86_64Assembler::GetCurrentThread(ManagedRegister tr) { 2292 gs()->movq(tr.AsX86_64().AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2293} 2294 2295void X86_64Assembler::GetCurrentThread(FrameOffset offset, ManagedRegister mscratch) { 2296 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2297 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2298 movq(Address(CpuRegister(RSP), offset), scratch.AsCpuRegister()); 2299} 2300 2301// Slowpath entered when Thread::Current()->_exception is non-null 2302class X86_64ExceptionSlowPath FINAL : public SlowPath { 2303 public: 2304 explicit X86_64ExceptionSlowPath(size_t stack_adjust) : stack_adjust_(stack_adjust) {} 2305 virtual void Emit(Assembler *sp_asm) OVERRIDE; 2306 private: 2307 const size_t stack_adjust_; 2308}; 2309 2310void X86_64Assembler::ExceptionPoll(ManagedRegister /*scratch*/, size_t stack_adjust) { 2311 X86_64ExceptionSlowPath* slow = new X86_64ExceptionSlowPath(stack_adjust); 2312 buffer_.EnqueueSlowPath(slow); 2313 gs()->cmpl(Address::Absolute(Thread::ExceptionOffset<8>(), true), Immediate(0)); 2314 j(kNotEqual, slow->Entry()); 2315} 2316 2317void X86_64ExceptionSlowPath::Emit(Assembler *sasm) { 2318 X86_64Assembler* sp_asm = down_cast<X86_64Assembler*>(sasm); 2319#define __ sp_asm-> 2320 __ Bind(&entry_); 2321 // Note: the return value is dead 2322 if (stack_adjust_ != 0) { // Fix up the frame. 2323 __ DecreaseFrameSize(stack_adjust_); 2324 } 2325 // Pass exception as argument in RDI 2326 __ gs()->movq(CpuRegister(RDI), Address::Absolute(Thread::ExceptionOffset<8>(), true)); 2327 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(8, pDeliverException), true)); 2328 // this call should never return 2329 __ int3(); 2330#undef __ 2331} 2332 2333} // namespace x86_64 2334} // namespace art 2335