assembler_x86_64.cc revision dff1f2812ecdaea89978c5351f0c70cdabbc0821
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "assembler_x86_64.h" 18 19#include "base/casts.h" 20#include "entrypoints/quick/quick_entrypoints.h" 21#include "memory_region.h" 22#include "thread.h" 23#include "utils/dwarf_cfi.h" 24 25namespace art { 26namespace x86_64 { 27 28std::ostream& operator<<(std::ostream& os, const CpuRegister& reg) { 29 return os << reg.AsRegister(); 30} 31 32std::ostream& operator<<(std::ostream& os, const XmmRegister& reg) { 33 return os << reg.AsFloatRegister(); 34} 35 36std::ostream& operator<<(std::ostream& os, const X87Register& reg) { 37 return os << "ST" << static_cast<int>(reg); 38} 39 40void X86_64Assembler::call(CpuRegister reg) { 41 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 42 EmitOptionalRex32(reg); 43 EmitUint8(0xFF); 44 EmitRegisterOperand(2, reg.LowBits()); 45} 46 47 48void X86_64Assembler::call(const Address& address) { 49 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 50 EmitOptionalRex32(address); 51 EmitUint8(0xFF); 52 EmitOperand(2, address); 53} 54 55 56void X86_64Assembler::call(Label* label) { 57 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 58 EmitUint8(0xE8); 59 static const int kSize = 5; 60 EmitLabel(label, kSize); 61} 62 63void X86_64Assembler::pushq(CpuRegister reg) { 64 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 65 EmitOptionalRex32(reg); 66 EmitUint8(0x50 + reg.LowBits()); 67} 68 69 70void X86_64Assembler::pushq(const Address& address) { 71 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 72 EmitOptionalRex32(address); 73 EmitUint8(0xFF); 74 EmitOperand(6, address); 75} 76 77 78void X86_64Assembler::pushq(const Immediate& imm) { 79 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 80 CHECK(imm.is_int32()); // pushq only supports 32b immediate. 81 if (imm.is_int8()) { 82 EmitUint8(0x6A); 83 EmitUint8(imm.value() & 0xFF); 84 } else { 85 EmitUint8(0x68); 86 EmitImmediate(imm); 87 } 88} 89 90 91void X86_64Assembler::popq(CpuRegister reg) { 92 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 93 EmitOptionalRex32(reg); 94 EmitUint8(0x58 + reg.LowBits()); 95} 96 97 98void X86_64Assembler::popq(const Address& address) { 99 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 100 EmitOptionalRex32(address); 101 EmitUint8(0x8F); 102 EmitOperand(0, address); 103} 104 105 106void X86_64Assembler::movq(CpuRegister dst, const Immediate& imm) { 107 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 108 if (imm.is_int32()) { 109 // 32 bit. Note: sign-extends. 110 EmitRex64(dst); 111 EmitUint8(0xC7); 112 EmitRegisterOperand(0, dst.LowBits()); 113 EmitInt32(static_cast<int32_t>(imm.value())); 114 } else { 115 EmitRex64(dst); 116 EmitUint8(0xB8 + dst.LowBits()); 117 EmitInt64(imm.value()); 118 } 119} 120 121 122void X86_64Assembler::movl(CpuRegister dst, const Immediate& imm) { 123 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 124 EmitOptionalRex32(dst); 125 EmitUint8(0xB8 + dst.LowBits()); 126 EmitImmediate(imm); 127} 128 129 130void X86_64Assembler::movq(CpuRegister dst, CpuRegister src) { 131 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 132 // 0x89 is movq r/m64 <- r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 133 EmitRex64(src, dst); 134 EmitUint8(0x89); 135 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 136} 137 138 139void X86_64Assembler::movl(CpuRegister dst, CpuRegister src) { 140 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 141 EmitOptionalRex32(dst, src); 142 EmitUint8(0x8B); 143 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 144} 145 146 147void X86_64Assembler::movq(CpuRegister dst, const Address& src) { 148 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 149 EmitRex64(dst, src); 150 EmitUint8(0x8B); 151 EmitOperand(dst.LowBits(), src); 152} 153 154 155void X86_64Assembler::movl(CpuRegister dst, const Address& src) { 156 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 157 EmitOptionalRex32(dst, src); 158 EmitUint8(0x8B); 159 EmitOperand(dst.LowBits(), src); 160} 161 162 163void X86_64Assembler::movq(const Address& dst, CpuRegister src) { 164 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 165 EmitRex64(src, dst); 166 EmitUint8(0x89); 167 EmitOperand(src.LowBits(), dst); 168} 169 170 171void X86_64Assembler::movl(const Address& dst, CpuRegister src) { 172 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 173 EmitOptionalRex32(src, dst); 174 EmitUint8(0x89); 175 EmitOperand(src.LowBits(), dst); 176} 177 178void X86_64Assembler::movl(const Address& dst, const Immediate& imm) { 179 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 180 EmitOptionalRex32(dst); 181 EmitUint8(0xC7); 182 EmitOperand(0, dst); 183 EmitImmediate(imm); 184} 185 186void X86_64Assembler::movzxb(CpuRegister dst, CpuRegister src) { 187 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 188 EmitOptionalByteRegNormalizingRex32(dst, src); 189 EmitUint8(0x0F); 190 EmitUint8(0xB6); 191 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 192} 193 194 195void X86_64Assembler::movzxb(CpuRegister dst, const Address& src) { 196 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 197 EmitOptionalByteRegNormalizingRex32(dst, src); 198 EmitUint8(0x0F); 199 EmitUint8(0xB6); 200 EmitOperand(dst.LowBits(), src); 201} 202 203 204void X86_64Assembler::movsxb(CpuRegister dst, CpuRegister src) { 205 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 206 EmitOptionalByteRegNormalizingRex32(dst, src); 207 EmitUint8(0x0F); 208 EmitUint8(0xBE); 209 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 210} 211 212 213void X86_64Assembler::movsxb(CpuRegister dst, const Address& src) { 214 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 215 EmitOptionalByteRegNormalizingRex32(dst, src); 216 EmitUint8(0x0F); 217 EmitUint8(0xBE); 218 EmitOperand(dst.LowBits(), src); 219} 220 221 222void X86_64Assembler::movb(CpuRegister /*dst*/, const Address& /*src*/) { 223 LOG(FATAL) << "Use movzxb or movsxb instead."; 224} 225 226 227void X86_64Assembler::movb(const Address& dst, CpuRegister src) { 228 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 229 EmitOptionalByteRegNormalizingRex32(src, dst); 230 EmitUint8(0x88); 231 EmitOperand(src.LowBits(), dst); 232} 233 234 235void X86_64Assembler::movb(const Address& dst, const Immediate& imm) { 236 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 237 EmitOptionalRex32(dst); 238 EmitUint8(0xC6); 239 EmitOperand(Register::RAX, dst); 240 CHECK(imm.is_int8()); 241 EmitUint8(imm.value() & 0xFF); 242} 243 244 245void X86_64Assembler::movzxw(CpuRegister dst, CpuRegister src) { 246 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 247 EmitOptionalRex32(dst, src); 248 EmitUint8(0x0F); 249 EmitUint8(0xB7); 250 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 251} 252 253 254void X86_64Assembler::movzxw(CpuRegister dst, const Address& src) { 255 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 256 EmitOptionalRex32(dst, src); 257 EmitUint8(0x0F); 258 EmitUint8(0xB7); 259 EmitOperand(dst.LowBits(), src); 260} 261 262 263void X86_64Assembler::movsxw(CpuRegister dst, CpuRegister src) { 264 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 265 EmitOptionalRex32(dst, src); 266 EmitUint8(0x0F); 267 EmitUint8(0xBF); 268 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 269} 270 271 272void X86_64Assembler::movsxw(CpuRegister dst, const Address& src) { 273 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 274 EmitOptionalRex32(dst, src); 275 EmitUint8(0x0F); 276 EmitUint8(0xBF); 277 EmitOperand(dst.LowBits(), src); 278} 279 280 281void X86_64Assembler::movw(CpuRegister /*dst*/, const Address& /*src*/) { 282 LOG(FATAL) << "Use movzxw or movsxw instead."; 283} 284 285 286void X86_64Assembler::movw(const Address& dst, CpuRegister src) { 287 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 288 EmitOperandSizeOverride(); 289 EmitOptionalRex32(src, dst); 290 EmitUint8(0x89); 291 EmitOperand(src.LowBits(), dst); 292} 293 294 295void X86_64Assembler::movw(const Address& dst, const Immediate& imm) { 296 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 297 EmitOperandSizeOverride(); 298 EmitOptionalRex32(dst); 299 EmitUint8(0xC7); 300 EmitOperand(Register::RAX, dst); 301 CHECK(imm.is_uint16() || imm.is_int16()); 302 EmitUint8(imm.value() & 0xFF); 303 EmitUint8(imm.value() >> 8); 304} 305 306 307void X86_64Assembler::leaq(CpuRegister dst, const Address& src) { 308 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 309 EmitRex64(dst, src); 310 EmitUint8(0x8D); 311 EmitOperand(dst.LowBits(), src); 312} 313 314 315void X86_64Assembler::movaps(XmmRegister dst, XmmRegister src) { 316 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 317 EmitOptionalRex32(dst, src); 318 EmitUint8(0x0F); 319 EmitUint8(0x28); 320 EmitXmmRegisterOperand(dst.LowBits(), src); 321} 322 323 324void X86_64Assembler::movss(XmmRegister dst, const Address& src) { 325 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 326 EmitUint8(0xF3); 327 EmitOptionalRex32(dst, src); 328 EmitUint8(0x0F); 329 EmitUint8(0x10); 330 EmitOperand(dst.LowBits(), src); 331} 332 333 334void X86_64Assembler::movss(const Address& dst, XmmRegister src) { 335 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 336 EmitUint8(0xF3); 337 EmitOptionalRex32(src, dst); 338 EmitUint8(0x0F); 339 EmitUint8(0x11); 340 EmitOperand(src.LowBits(), dst); 341} 342 343 344void X86_64Assembler::movss(XmmRegister dst, XmmRegister src) { 345 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 346 EmitUint8(0xF3); 347 EmitOptionalRex32(dst, src); 348 EmitUint8(0x0F); 349 EmitUint8(0x11); 350 EmitXmmRegisterOperand(src.LowBits(), dst); 351} 352 353 354void X86_64Assembler::movsxd(CpuRegister dst, CpuRegister src) { 355 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 356 EmitRex64(dst); 357 EmitUint8(0x63); 358 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 359} 360 361 362void X86_64Assembler::movsxd(CpuRegister dst, const Address& src) { 363 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 364 EmitRex64(dst); 365 EmitUint8(0x63); 366 EmitOperand(dst.LowBits(), src); 367} 368 369 370void X86_64Assembler::movd(XmmRegister dst, CpuRegister src) { 371 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 372 EmitUint8(0x66); 373 EmitRex64(dst, src); 374 EmitUint8(0x0F); 375 EmitUint8(0x6E); 376 EmitOperand(dst.LowBits(), Operand(src)); 377} 378 379 380void X86_64Assembler::movd(CpuRegister dst, XmmRegister src) { 381 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 382 EmitUint8(0x66); 383 EmitRex64(src, dst); 384 EmitUint8(0x0F); 385 EmitUint8(0x7E); 386 EmitOperand(src.LowBits(), Operand(dst)); 387} 388 389 390void X86_64Assembler::addss(XmmRegister dst, XmmRegister src) { 391 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 392 EmitUint8(0xF3); 393 EmitOptionalRex32(dst, src); 394 EmitUint8(0x0F); 395 EmitUint8(0x58); 396 EmitXmmRegisterOperand(dst.LowBits(), src); 397} 398 399 400void X86_64Assembler::addss(XmmRegister dst, const Address& src) { 401 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 402 EmitUint8(0xF3); 403 EmitOptionalRex32(dst, src); 404 EmitUint8(0x0F); 405 EmitUint8(0x58); 406 EmitOperand(dst.LowBits(), src); 407} 408 409 410void X86_64Assembler::subss(XmmRegister dst, XmmRegister src) { 411 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 412 EmitUint8(0xF3); 413 EmitOptionalRex32(dst, src); 414 EmitUint8(0x0F); 415 EmitUint8(0x5C); 416 EmitXmmRegisterOperand(dst.LowBits(), src); 417} 418 419 420void X86_64Assembler::subss(XmmRegister dst, const Address& src) { 421 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 422 EmitUint8(0xF3); 423 EmitOptionalRex32(dst, src); 424 EmitUint8(0x0F); 425 EmitUint8(0x5C); 426 EmitOperand(dst.LowBits(), src); 427} 428 429 430void X86_64Assembler::mulss(XmmRegister dst, XmmRegister src) { 431 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 432 EmitUint8(0xF3); 433 EmitOptionalRex32(dst, src); 434 EmitUint8(0x0F); 435 EmitUint8(0x59); 436 EmitXmmRegisterOperand(dst.LowBits(), src); 437} 438 439 440void X86_64Assembler::mulss(XmmRegister dst, const Address& src) { 441 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 442 EmitUint8(0xF3); 443 EmitOptionalRex32(dst, src); 444 EmitUint8(0x0F); 445 EmitUint8(0x59); 446 EmitOperand(dst.LowBits(), src); 447} 448 449 450void X86_64Assembler::divss(XmmRegister dst, XmmRegister src) { 451 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 452 EmitUint8(0xF3); 453 EmitOptionalRex32(dst, src); 454 EmitUint8(0x0F); 455 EmitUint8(0x5E); 456 EmitXmmRegisterOperand(dst.LowBits(), src); 457} 458 459 460void X86_64Assembler::divss(XmmRegister dst, const Address& src) { 461 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 462 EmitUint8(0xF3); 463 EmitOptionalRex32(dst, src); 464 EmitUint8(0x0F); 465 EmitUint8(0x5E); 466 EmitOperand(dst.LowBits(), src); 467} 468 469 470void X86_64Assembler::flds(const Address& src) { 471 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 472 EmitUint8(0xD9); 473 EmitOperand(0, src); 474} 475 476 477void X86_64Assembler::fstps(const Address& dst) { 478 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 479 EmitUint8(0xD9); 480 EmitOperand(3, dst); 481} 482 483 484void X86_64Assembler::movsd(XmmRegister dst, const Address& src) { 485 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 486 EmitUint8(0xF2); 487 EmitOptionalRex32(dst, src); 488 EmitUint8(0x0F); 489 EmitUint8(0x10); 490 EmitOperand(dst.LowBits(), src); 491} 492 493 494void X86_64Assembler::movsd(const Address& dst, XmmRegister src) { 495 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 496 EmitUint8(0xF2); 497 EmitOptionalRex32(src, dst); 498 EmitUint8(0x0F); 499 EmitUint8(0x11); 500 EmitOperand(src.LowBits(), dst); 501} 502 503 504void X86_64Assembler::movsd(XmmRegister dst, XmmRegister src) { 505 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 506 EmitUint8(0xF2); 507 EmitOptionalRex32(dst, src); 508 EmitUint8(0x0F); 509 EmitUint8(0x11); 510 EmitXmmRegisterOperand(src.LowBits(), dst); 511} 512 513 514void X86_64Assembler::addsd(XmmRegister dst, XmmRegister src) { 515 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 516 EmitUint8(0xF2); 517 EmitOptionalRex32(dst, src); 518 EmitUint8(0x0F); 519 EmitUint8(0x58); 520 EmitXmmRegisterOperand(dst.LowBits(), src); 521} 522 523 524void X86_64Assembler::addsd(XmmRegister dst, const Address& src) { 525 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 526 EmitUint8(0xF2); 527 EmitOptionalRex32(dst, src); 528 EmitUint8(0x0F); 529 EmitUint8(0x58); 530 EmitOperand(dst.LowBits(), src); 531} 532 533 534void X86_64Assembler::subsd(XmmRegister dst, XmmRegister src) { 535 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 536 EmitUint8(0xF2); 537 EmitOptionalRex32(dst, src); 538 EmitUint8(0x0F); 539 EmitUint8(0x5C); 540 EmitXmmRegisterOperand(dst.LowBits(), src); 541} 542 543 544void X86_64Assembler::subsd(XmmRegister dst, const Address& src) { 545 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 546 EmitUint8(0xF2); 547 EmitOptionalRex32(dst, src); 548 EmitUint8(0x0F); 549 EmitUint8(0x5C); 550 EmitOperand(dst.LowBits(), src); 551} 552 553 554void X86_64Assembler::mulsd(XmmRegister dst, XmmRegister src) { 555 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 556 EmitUint8(0xF2); 557 EmitOptionalRex32(dst, src); 558 EmitUint8(0x0F); 559 EmitUint8(0x59); 560 EmitXmmRegisterOperand(dst.LowBits(), src); 561} 562 563 564void X86_64Assembler::mulsd(XmmRegister dst, const Address& src) { 565 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 566 EmitUint8(0xF2); 567 EmitOptionalRex32(dst, src); 568 EmitUint8(0x0F); 569 EmitUint8(0x59); 570 EmitOperand(dst.LowBits(), src); 571} 572 573 574void X86_64Assembler::divsd(XmmRegister dst, XmmRegister src) { 575 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 576 EmitUint8(0xF2); 577 EmitOptionalRex32(dst, src); 578 EmitUint8(0x0F); 579 EmitUint8(0x5E); 580 EmitXmmRegisterOperand(dst.LowBits(), src); 581} 582 583 584void X86_64Assembler::divsd(XmmRegister dst, const Address& src) { 585 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 586 EmitUint8(0xF2); 587 EmitOptionalRex32(dst, src); 588 EmitUint8(0x0F); 589 EmitUint8(0x5E); 590 EmitOperand(dst.LowBits(), src); 591} 592 593 594void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) { 595 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 596 EmitUint8(0xF3); 597 EmitOptionalRex32(dst, src); 598 EmitUint8(0x0F); 599 EmitUint8(0x2A); 600 EmitOperand(dst.LowBits(), Operand(src)); 601} 602 603 604void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src) { 605 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 606 EmitUint8(0xF2); 607 EmitOptionalRex32(dst, src); 608 EmitUint8(0x0F); 609 EmitUint8(0x2A); 610 EmitOperand(dst.LowBits(), Operand(src)); 611} 612 613 614void X86_64Assembler::cvtss2si(CpuRegister dst, XmmRegister src) { 615 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 616 EmitUint8(0xF3); 617 EmitOptionalRex32(dst, src); 618 EmitUint8(0x0F); 619 EmitUint8(0x2D); 620 EmitXmmRegisterOperand(dst.LowBits(), src); 621} 622 623 624void X86_64Assembler::cvtss2sd(XmmRegister dst, XmmRegister src) { 625 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 626 EmitUint8(0xF3); 627 EmitOptionalRex32(dst, src); 628 EmitUint8(0x0F); 629 EmitUint8(0x5A); 630 EmitXmmRegisterOperand(dst.LowBits(), src); 631} 632 633 634void X86_64Assembler::cvtsd2si(CpuRegister dst, XmmRegister src) { 635 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 636 EmitUint8(0xF2); 637 EmitOptionalRex32(dst, src); 638 EmitUint8(0x0F); 639 EmitUint8(0x2D); 640 EmitXmmRegisterOperand(dst.LowBits(), src); 641} 642 643 644void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src) { 645 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 646 EmitUint8(0xF3); 647 EmitOptionalRex32(dst, src); 648 EmitUint8(0x0F); 649 EmitUint8(0x2C); 650 EmitXmmRegisterOperand(dst.LowBits(), src); 651} 652 653 654void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src) { 655 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 656 EmitUint8(0xF2); 657 EmitOptionalRex32(dst, src); 658 EmitUint8(0x0F); 659 EmitUint8(0x2C); 660 EmitXmmRegisterOperand(dst.LowBits(), src); 661} 662 663 664void X86_64Assembler::cvtsd2ss(XmmRegister dst, XmmRegister src) { 665 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 666 EmitUint8(0xF2); 667 EmitOptionalRex32(dst, src); 668 EmitUint8(0x0F); 669 EmitUint8(0x5A); 670 EmitXmmRegisterOperand(dst.LowBits(), src); 671} 672 673 674void X86_64Assembler::cvtdq2pd(XmmRegister dst, XmmRegister src) { 675 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 676 EmitUint8(0xF3); 677 EmitOptionalRex32(dst, src); 678 EmitUint8(0x0F); 679 EmitUint8(0xE6); 680 EmitXmmRegisterOperand(dst.LowBits(), src); 681} 682 683 684void X86_64Assembler::comiss(XmmRegister a, XmmRegister b) { 685 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 686 EmitOptionalRex32(a, b); 687 EmitUint8(0x0F); 688 EmitUint8(0x2F); 689 EmitXmmRegisterOperand(a.LowBits(), b); 690} 691 692 693void X86_64Assembler::comisd(XmmRegister a, XmmRegister b) { 694 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 695 EmitUint8(0x66); 696 EmitOptionalRex32(a, b); 697 EmitUint8(0x0F); 698 EmitUint8(0x2F); 699 EmitXmmRegisterOperand(a.LowBits(), b); 700} 701 702 703void X86_64Assembler::sqrtsd(XmmRegister dst, XmmRegister src) { 704 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 705 EmitUint8(0xF2); 706 EmitOptionalRex32(dst, src); 707 EmitUint8(0x0F); 708 EmitUint8(0x51); 709 EmitXmmRegisterOperand(dst.LowBits(), src); 710} 711 712 713void X86_64Assembler::sqrtss(XmmRegister dst, XmmRegister src) { 714 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 715 EmitUint8(0xF3); 716 EmitOptionalRex32(dst, src); 717 EmitUint8(0x0F); 718 EmitUint8(0x51); 719 EmitXmmRegisterOperand(dst.LowBits(), src); 720} 721 722 723void X86_64Assembler::xorpd(XmmRegister dst, const Address& src) { 724 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 725 EmitUint8(0x66); 726 EmitOptionalRex32(dst, src); 727 EmitUint8(0x0F); 728 EmitUint8(0x57); 729 EmitOperand(dst.LowBits(), src); 730} 731 732 733void X86_64Assembler::xorpd(XmmRegister dst, XmmRegister src) { 734 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 735 EmitUint8(0x66); 736 EmitOptionalRex32(dst, src); 737 EmitUint8(0x0F); 738 EmitUint8(0x57); 739 EmitXmmRegisterOperand(dst.LowBits(), src); 740} 741 742 743void X86_64Assembler::xorps(XmmRegister dst, const Address& src) { 744 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 745 EmitOptionalRex32(dst, src); 746 EmitUint8(0x0F); 747 EmitUint8(0x57); 748 EmitOperand(dst.LowBits(), src); 749} 750 751 752void X86_64Assembler::xorps(XmmRegister dst, XmmRegister src) { 753 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 754 EmitOptionalRex32(dst, src); 755 EmitUint8(0x0F); 756 EmitUint8(0x57); 757 EmitXmmRegisterOperand(dst.LowBits(), src); 758} 759 760 761void X86_64Assembler::andpd(XmmRegister dst, const Address& src) { 762 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 763 EmitUint8(0x66); 764 EmitOptionalRex32(dst, src); 765 EmitUint8(0x0F); 766 EmitUint8(0x54); 767 EmitOperand(dst.LowBits(), src); 768} 769 770 771void X86_64Assembler::fldl(const Address& src) { 772 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 773 EmitUint8(0xDD); 774 EmitOperand(0, src); 775} 776 777 778void X86_64Assembler::fstpl(const Address& dst) { 779 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 780 EmitUint8(0xDD); 781 EmitOperand(3, dst); 782} 783 784 785void X86_64Assembler::fnstcw(const Address& dst) { 786 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 787 EmitUint8(0xD9); 788 EmitOperand(7, dst); 789} 790 791 792void X86_64Assembler::fldcw(const Address& src) { 793 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 794 EmitUint8(0xD9); 795 EmitOperand(5, src); 796} 797 798 799void X86_64Assembler::fistpl(const Address& dst) { 800 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 801 EmitUint8(0xDF); 802 EmitOperand(7, dst); 803} 804 805 806void X86_64Assembler::fistps(const Address& dst) { 807 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 808 EmitUint8(0xDB); 809 EmitOperand(3, dst); 810} 811 812 813void X86_64Assembler::fildl(const Address& src) { 814 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 815 EmitUint8(0xDF); 816 EmitOperand(5, src); 817} 818 819 820void X86_64Assembler::fincstp() { 821 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 822 EmitUint8(0xD9); 823 EmitUint8(0xF7); 824} 825 826 827void X86_64Assembler::ffree(const Immediate& index) { 828 CHECK_LT(index.value(), 7); 829 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 830 EmitUint8(0xDD); 831 EmitUint8(0xC0 + index.value()); 832} 833 834 835void X86_64Assembler::fsin() { 836 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 837 EmitUint8(0xD9); 838 EmitUint8(0xFE); 839} 840 841 842void X86_64Assembler::fcos() { 843 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 844 EmitUint8(0xD9); 845 EmitUint8(0xFF); 846} 847 848 849void X86_64Assembler::fptan() { 850 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 851 EmitUint8(0xD9); 852 EmitUint8(0xF2); 853} 854 855 856void X86_64Assembler::xchgl(CpuRegister dst, CpuRegister src) { 857 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 858 EmitOptionalRex32(dst, src); 859 EmitUint8(0x87); 860 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 861} 862 863 864void X86_64Assembler::xchgq(CpuRegister dst, CpuRegister src) { 865 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 866 EmitRex64(dst, src); 867 EmitUint8(0x87); 868 EmitOperand(dst.LowBits(), Operand(src)); 869} 870 871 872void X86_64Assembler::xchgl(CpuRegister reg, const Address& address) { 873 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 874 EmitOptionalRex32(reg, address); 875 EmitUint8(0x87); 876 EmitOperand(reg.LowBits(), address); 877} 878 879 880void X86_64Assembler::cmpw(const Address& address, const Immediate& imm) { 881 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 882 EmitOptionalRex32(address); 883 EmitUint8(0x66); 884 EmitComplex(7, address, imm); 885} 886 887 888void X86_64Assembler::cmpl(CpuRegister reg, const Immediate& imm) { 889 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 890 EmitOptionalRex32(reg); 891 EmitComplex(7, Operand(reg), imm); 892} 893 894 895void X86_64Assembler::cmpl(CpuRegister reg0, CpuRegister reg1) { 896 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 897 EmitOptionalRex32(reg0, reg1); 898 EmitUint8(0x3B); 899 EmitOperand(reg0.LowBits(), Operand(reg1)); 900} 901 902 903void X86_64Assembler::cmpl(CpuRegister reg, const Address& address) { 904 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 905 EmitOptionalRex32(reg, address); 906 EmitUint8(0x3B); 907 EmitOperand(reg.LowBits(), address); 908} 909 910 911void X86_64Assembler::cmpq(CpuRegister reg0, CpuRegister reg1) { 912 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 913 EmitRex64(reg0, reg1); 914 EmitUint8(0x3B); 915 EmitOperand(reg0.LowBits(), Operand(reg1)); 916} 917 918 919void X86_64Assembler::cmpq(CpuRegister reg, const Immediate& imm) { 920 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 921 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 922 EmitRex64(reg); 923 EmitComplex(7, Operand(reg), imm); 924} 925 926 927void X86_64Assembler::cmpq(CpuRegister reg, const Address& address) { 928 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 929 EmitRex64(reg); 930 EmitUint8(0x3B); 931 EmitOperand(reg.LowBits(), address); 932} 933 934 935void X86_64Assembler::addl(CpuRegister dst, CpuRegister src) { 936 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 937 EmitOptionalRex32(dst, src); 938 EmitUint8(0x03); 939 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 940} 941 942 943void X86_64Assembler::addl(CpuRegister reg, const Address& address) { 944 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 945 EmitOptionalRex32(reg, address); 946 EmitUint8(0x03); 947 EmitOperand(reg.LowBits(), address); 948} 949 950 951void X86_64Assembler::cmpl(const Address& address, CpuRegister reg) { 952 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 953 EmitOptionalRex32(reg, address); 954 EmitUint8(0x39); 955 EmitOperand(reg.LowBits(), address); 956} 957 958 959void X86_64Assembler::cmpl(const Address& address, const Immediate& imm) { 960 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 961 EmitOptionalRex32(address); 962 EmitComplex(7, address, imm); 963} 964 965 966void X86_64Assembler::testl(CpuRegister reg1, CpuRegister reg2) { 967 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 968 EmitOptionalRex32(reg1, reg2); 969 EmitUint8(0x85); 970 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 971} 972 973 974void X86_64Assembler::testl(CpuRegister reg, const Immediate& immediate) { 975 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 976 // For registers that have a byte variant (RAX, RBX, RCX, and RDX) 977 // we only test the byte CpuRegister to keep the encoding short. 978 if (immediate.is_uint8() && reg.AsRegister() < 4) { 979 // Use zero-extended 8-bit immediate. 980 if (reg.AsRegister() == RAX) { 981 EmitUint8(0xA8); 982 } else { 983 EmitUint8(0xF6); 984 EmitUint8(0xC0 + reg.AsRegister()); 985 } 986 EmitUint8(immediate.value() & 0xFF); 987 } else if (reg.AsRegister() == RAX) { 988 // Use short form if the destination is RAX. 989 EmitUint8(0xA9); 990 EmitImmediate(immediate); 991 } else { 992 EmitOptionalRex32(reg); 993 EmitUint8(0xF7); 994 EmitOperand(0, Operand(reg)); 995 EmitImmediate(immediate); 996 } 997} 998 999 1000void X86_64Assembler::testq(CpuRegister reg, const Address& address) { 1001 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1002 EmitRex64(reg); 1003 EmitUint8(0x85); 1004 EmitOperand(reg.LowBits(), address); 1005} 1006 1007 1008void X86_64Assembler::andl(CpuRegister dst, CpuRegister src) { 1009 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1010 EmitOptionalRex32(dst, src); 1011 EmitUint8(0x23); 1012 EmitOperand(dst.LowBits(), Operand(src)); 1013} 1014 1015 1016void X86_64Assembler::andl(CpuRegister dst, const Immediate& imm) { 1017 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1018 EmitOptionalRex32(dst); 1019 EmitComplex(4, Operand(dst), imm); 1020} 1021 1022 1023void X86_64Assembler::andq(CpuRegister reg, const Immediate& imm) { 1024 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1025 CHECK(imm.is_int32()); // andq only supports 32b immediate. 1026 EmitRex64(reg); 1027 EmitComplex(4, Operand(reg), imm); 1028} 1029 1030 1031void X86_64Assembler::orl(CpuRegister dst, CpuRegister src) { 1032 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1033 EmitOptionalRex32(dst, src); 1034 EmitUint8(0x0B); 1035 EmitOperand(dst.LowBits(), Operand(src)); 1036} 1037 1038 1039void X86_64Assembler::orl(CpuRegister dst, const Immediate& imm) { 1040 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1041 EmitOptionalRex32(dst); 1042 EmitComplex(1, Operand(dst), imm); 1043} 1044 1045 1046void X86_64Assembler::xorl(CpuRegister dst, CpuRegister src) { 1047 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1048 EmitOptionalRex32(dst, src); 1049 EmitUint8(0x33); 1050 EmitOperand(dst.LowBits(), Operand(src)); 1051} 1052 1053 1054void X86_64Assembler::xorq(CpuRegister dst, CpuRegister src) { 1055 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1056 EmitRex64(dst, src); 1057 EmitUint8(0x33); 1058 EmitOperand(dst.LowBits(), Operand(src)); 1059} 1060 1061 1062void X86_64Assembler::xorq(CpuRegister dst, const Immediate& imm) { 1063 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1064 CHECK(imm.is_int32()); // xorq only supports 32b immediate. 1065 EmitRex64(dst); 1066 EmitComplex(6, Operand(dst), imm); 1067} 1068 1069#if 0 1070void X86_64Assembler::rex(bool force, bool w, Register* r, Register* x, Register* b) { 1071 // REX.WRXB 1072 // W - 64-bit operand 1073 // R - MODRM.reg 1074 // X - SIB.index 1075 // B - MODRM.rm/SIB.base 1076 uint8_t rex = force ? 0x40 : 0; 1077 if (w) { 1078 rex |= 0x48; // REX.W000 1079 } 1080 if (r != nullptr && *r >= Register::R8 && *r < Register::kNumberOfCpuRegisters) { 1081 rex |= 0x44; // REX.0R00 1082 *r = static_cast<Register>(*r - 8); 1083 } 1084 if (x != nullptr && *x >= Register::R8 && *x < Register::kNumberOfCpuRegisters) { 1085 rex |= 0x42; // REX.00X0 1086 *x = static_cast<Register>(*x - 8); 1087 } 1088 if (b != nullptr && *b >= Register::R8 && *b < Register::kNumberOfCpuRegisters) { 1089 rex |= 0x41; // REX.000B 1090 *b = static_cast<Register>(*b - 8); 1091 } 1092 if (rex != 0) { 1093 EmitUint8(rex); 1094 } 1095} 1096 1097void X86_64Assembler::rex_reg_mem(bool force, bool w, Register* dst, const Address& mem) { 1098 // REX.WRXB 1099 // W - 64-bit operand 1100 // R - MODRM.reg 1101 // X - SIB.index 1102 // B - MODRM.rm/SIB.base 1103 uint8_t rex = mem->rex(); 1104 if (force) { 1105 rex |= 0x40; // REX.0000 1106 } 1107 if (w) { 1108 rex |= 0x48; // REX.W000 1109 } 1110 if (dst != nullptr && *dst >= Register::R8 && *dst < Register::kNumberOfCpuRegisters) { 1111 rex |= 0x44; // REX.0R00 1112 *dst = static_cast<Register>(*dst - 8); 1113 } 1114 if (rex != 0) { 1115 EmitUint8(rex); 1116 } 1117} 1118 1119void rex_mem_reg(bool force, bool w, Address* mem, Register* src); 1120#endif 1121 1122void X86_64Assembler::addl(CpuRegister reg, const Immediate& imm) { 1123 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1124 EmitOptionalRex32(reg); 1125 EmitComplex(0, Operand(reg), imm); 1126} 1127 1128 1129void X86_64Assembler::addq(CpuRegister reg, const Immediate& imm) { 1130 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1131 CHECK(imm.is_int32()); // addq only supports 32b immediate. 1132 EmitRex64(reg); 1133 EmitComplex(0, Operand(reg), imm); 1134} 1135 1136 1137void X86_64Assembler::addq(CpuRegister dst, const Address& address) { 1138 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1139 EmitRex64(dst); 1140 EmitUint8(0x03); 1141 EmitOperand(dst.LowBits(), address); 1142} 1143 1144 1145void X86_64Assembler::addq(CpuRegister dst, CpuRegister src) { 1146 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1147 // 0x01 is addq r/m64 <- r/m64 + r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 1148 EmitRex64(src, dst); 1149 EmitUint8(0x01); 1150 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1151} 1152 1153 1154void X86_64Assembler::addl(const Address& address, CpuRegister reg) { 1155 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1156 EmitOptionalRex32(reg, address); 1157 EmitUint8(0x01); 1158 EmitOperand(reg.LowBits(), address); 1159} 1160 1161 1162void X86_64Assembler::addl(const Address& address, const Immediate& imm) { 1163 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1164 EmitOptionalRex32(address); 1165 EmitComplex(0, address, imm); 1166} 1167 1168 1169void X86_64Assembler::subl(CpuRegister dst, CpuRegister src) { 1170 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1171 EmitOptionalRex32(dst, src); 1172 EmitUint8(0x2B); 1173 EmitOperand(dst.LowBits(), Operand(src)); 1174} 1175 1176 1177void X86_64Assembler::subl(CpuRegister reg, const Immediate& imm) { 1178 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1179 EmitOptionalRex32(reg); 1180 EmitComplex(5, Operand(reg), imm); 1181} 1182 1183 1184void X86_64Assembler::subq(CpuRegister reg, const Immediate& imm) { 1185 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1186 CHECK(imm.is_int32()); // subq only supports 32b immediate. 1187 EmitRex64(reg); 1188 EmitComplex(5, Operand(reg), imm); 1189} 1190 1191 1192void X86_64Assembler::subq(CpuRegister dst, CpuRegister src) { 1193 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1194 EmitRex64(dst, src); 1195 EmitUint8(0x2B); 1196 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1197} 1198 1199 1200void X86_64Assembler::subq(CpuRegister reg, const Address& address) { 1201 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1202 EmitRex64(reg); 1203 EmitUint8(0x2B); 1204 EmitOperand(reg.LowBits() & 7, address); 1205} 1206 1207 1208void X86_64Assembler::subl(CpuRegister reg, const Address& address) { 1209 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1210 EmitOptionalRex32(reg, address); 1211 EmitUint8(0x2B); 1212 EmitOperand(reg.LowBits(), address); 1213} 1214 1215 1216void X86_64Assembler::cdq() { 1217 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1218 EmitUint8(0x99); 1219} 1220 1221 1222void X86_64Assembler::idivl(CpuRegister reg) { 1223 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1224 EmitOptionalRex32(reg); 1225 EmitUint8(0xF7); 1226 EmitUint8(0xF8 | reg.LowBits()); 1227} 1228 1229 1230void X86_64Assembler::imull(CpuRegister dst, CpuRegister src) { 1231 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1232 EmitOptionalRex32(dst, src); 1233 EmitUint8(0x0F); 1234 EmitUint8(0xAF); 1235 EmitOperand(dst.LowBits(), Operand(src)); 1236} 1237 1238 1239void X86_64Assembler::imull(CpuRegister reg, const Immediate& imm) { 1240 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1241 EmitOptionalRex32(reg, reg); 1242 EmitUint8(0x69); 1243 EmitOperand(reg.LowBits(), Operand(reg)); 1244 EmitImmediate(imm); 1245} 1246 1247 1248void X86_64Assembler::imull(CpuRegister reg, const Address& address) { 1249 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1250 EmitOptionalRex32(reg, address); 1251 EmitUint8(0x0F); 1252 EmitUint8(0xAF); 1253 EmitOperand(reg.LowBits(), address); 1254} 1255 1256 1257void X86_64Assembler::imulq(CpuRegister dst, CpuRegister src) { 1258 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1259 EmitRex64(dst, src); 1260 EmitUint8(0x0F); 1261 EmitUint8(0xAF); 1262 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1263} 1264 1265 1266void X86_64Assembler::imulq(CpuRegister reg, const Immediate& imm) { 1267 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1268 CHECK(imm.is_int32()); // imulq only supports 32b immediate. 1269 EmitRex64(reg); 1270 EmitUint8(0x69); 1271 EmitOperand(reg.LowBits(), Operand(reg)); 1272 EmitImmediate(imm); 1273} 1274 1275 1276void X86_64Assembler::imulq(CpuRegister reg, const Address& address) { 1277 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1278 EmitRex64(reg, address); 1279 EmitUint8(0x0F); 1280 EmitUint8(0xAF); 1281 EmitOperand(reg.LowBits(), address); 1282} 1283 1284 1285void X86_64Assembler::imull(CpuRegister reg) { 1286 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1287 EmitOptionalRex32(reg); 1288 EmitUint8(0xF7); 1289 EmitOperand(5, Operand(reg)); 1290} 1291 1292 1293void X86_64Assembler::imull(const Address& address) { 1294 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1295 EmitOptionalRex32(address); 1296 EmitUint8(0xF7); 1297 EmitOperand(5, address); 1298} 1299 1300 1301void X86_64Assembler::mull(CpuRegister reg) { 1302 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1303 EmitOptionalRex32(reg); 1304 EmitUint8(0xF7); 1305 EmitOperand(4, Operand(reg)); 1306} 1307 1308 1309void X86_64Assembler::mull(const Address& address) { 1310 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1311 EmitOptionalRex32(address); 1312 EmitUint8(0xF7); 1313 EmitOperand(4, address); 1314} 1315 1316 1317void X86_64Assembler::shll(CpuRegister reg, const Immediate& imm) { 1318 EmitGenericShift(false, 4, reg, imm); 1319} 1320 1321 1322void X86_64Assembler::shll(CpuRegister operand, CpuRegister shifter) { 1323 EmitGenericShift(4, operand, shifter); 1324} 1325 1326 1327void X86_64Assembler::shrl(CpuRegister reg, const Immediate& imm) { 1328 EmitGenericShift(false, 5, reg, imm); 1329} 1330 1331 1332void X86_64Assembler::shrq(CpuRegister reg, const Immediate& imm) { 1333 EmitGenericShift(true, 5, reg, imm); 1334} 1335 1336 1337void X86_64Assembler::shrl(CpuRegister operand, CpuRegister shifter) { 1338 EmitGenericShift(5, operand, shifter); 1339} 1340 1341 1342void X86_64Assembler::sarl(CpuRegister reg, const Immediate& imm) { 1343 EmitGenericShift(false, 7, reg, imm); 1344} 1345 1346 1347void X86_64Assembler::sarl(CpuRegister operand, CpuRegister shifter) { 1348 EmitGenericShift(7, operand, shifter); 1349} 1350 1351 1352void X86_64Assembler::negl(CpuRegister reg) { 1353 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1354 EmitOptionalRex32(reg); 1355 EmitUint8(0xF7); 1356 EmitOperand(3, Operand(reg)); 1357} 1358 1359 1360void X86_64Assembler::negq(CpuRegister reg) { 1361 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1362 EmitRex64(reg); 1363 EmitUint8(0xF7); 1364 EmitOperand(3, Operand(reg)); 1365} 1366 1367 1368void X86_64Assembler::notl(CpuRegister reg) { 1369 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1370 EmitOptionalRex32(reg); 1371 EmitUint8(0xF7); 1372 EmitUint8(0xD0 | reg.LowBits()); 1373} 1374 1375 1376void X86_64Assembler::notq(CpuRegister reg) { 1377 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1378 EmitRex64(reg); 1379 EmitUint8(0xF7); 1380 EmitOperand(2, Operand(reg)); 1381} 1382 1383 1384void X86_64Assembler::enter(const Immediate& imm) { 1385 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1386 EmitUint8(0xC8); 1387 CHECK(imm.is_uint16()); 1388 EmitUint8(imm.value() & 0xFF); 1389 EmitUint8((imm.value() >> 8) & 0xFF); 1390 EmitUint8(0x00); 1391} 1392 1393 1394void X86_64Assembler::leave() { 1395 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1396 EmitUint8(0xC9); 1397} 1398 1399 1400void X86_64Assembler::ret() { 1401 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1402 EmitUint8(0xC3); 1403} 1404 1405 1406void X86_64Assembler::ret(const Immediate& imm) { 1407 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1408 EmitUint8(0xC2); 1409 CHECK(imm.is_uint16()); 1410 EmitUint8(imm.value() & 0xFF); 1411 EmitUint8((imm.value() >> 8) & 0xFF); 1412} 1413 1414 1415 1416void X86_64Assembler::nop() { 1417 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1418 EmitUint8(0x90); 1419} 1420 1421 1422void X86_64Assembler::int3() { 1423 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1424 EmitUint8(0xCC); 1425} 1426 1427 1428void X86_64Assembler::hlt() { 1429 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1430 EmitUint8(0xF4); 1431} 1432 1433 1434void X86_64Assembler::j(Condition condition, Label* label) { 1435 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1436 if (label->IsBound()) { 1437 static const int kShortSize = 2; 1438 static const int kLongSize = 6; 1439 int offset = label->Position() - buffer_.Size(); 1440 CHECK_LE(offset, 0); 1441 if (IsInt(8, offset - kShortSize)) { 1442 EmitUint8(0x70 + condition); 1443 EmitUint8((offset - kShortSize) & 0xFF); 1444 } else { 1445 EmitUint8(0x0F); 1446 EmitUint8(0x80 + condition); 1447 EmitInt32(offset - kLongSize); 1448 } 1449 } else { 1450 EmitUint8(0x0F); 1451 EmitUint8(0x80 + condition); 1452 EmitLabelLink(label); 1453 } 1454} 1455 1456 1457void X86_64Assembler::jmp(CpuRegister reg) { 1458 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1459 EmitOptionalRex32(reg); 1460 EmitUint8(0xFF); 1461 EmitRegisterOperand(4, reg.LowBits()); 1462} 1463 1464void X86_64Assembler::jmp(const Address& address) { 1465 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1466 EmitOptionalRex32(address); 1467 EmitUint8(0xFF); 1468 EmitOperand(4, address); 1469} 1470 1471void X86_64Assembler::jmp(Label* label) { 1472 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1473 if (label->IsBound()) { 1474 static const int kShortSize = 2; 1475 static const int kLongSize = 5; 1476 int offset = label->Position() - buffer_.Size(); 1477 CHECK_LE(offset, 0); 1478 if (IsInt(8, offset - kShortSize)) { 1479 EmitUint8(0xEB); 1480 EmitUint8((offset - kShortSize) & 0xFF); 1481 } else { 1482 EmitUint8(0xE9); 1483 EmitInt32(offset - kLongSize); 1484 } 1485 } else { 1486 EmitUint8(0xE9); 1487 EmitLabelLink(label); 1488 } 1489} 1490 1491 1492X86_64Assembler* X86_64Assembler::lock() { 1493 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1494 EmitUint8(0xF0); 1495 return this; 1496} 1497 1498 1499void X86_64Assembler::cmpxchgl(const Address& address, CpuRegister reg) { 1500 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1501 EmitUint8(0x0F); 1502 EmitUint8(0xB1); 1503 EmitOperand(reg.LowBits(), address); 1504} 1505 1506void X86_64Assembler::mfence() { 1507 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1508 EmitUint8(0x0F); 1509 EmitUint8(0xAE); 1510 EmitUint8(0xF0); 1511} 1512 1513 1514X86_64Assembler* X86_64Assembler::gs() { 1515 // TODO: gs is a prefix and not an instruction 1516 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1517 EmitUint8(0x65); 1518 return this; 1519} 1520 1521 1522void X86_64Assembler::AddImmediate(CpuRegister reg, const Immediate& imm) { 1523 int value = imm.value(); 1524 if (value != 0) { 1525 if (value > 0) { 1526 addl(reg, imm); 1527 } else { 1528 subl(reg, Immediate(value)); 1529 } 1530 } 1531} 1532 1533 1534void X86_64Assembler::setcc(Condition condition, CpuRegister dst) { 1535 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1536 // RSP, RBP, RDI, RSI need rex prefix (else the pattern encodes ah/bh/ch/dh). 1537 if (dst.NeedsRex() || dst.AsRegister() > 3) { 1538 EmitOptionalRex(true, false, false, false, dst.NeedsRex()); 1539 } 1540 EmitUint8(0x0F); 1541 EmitUint8(0x90 + condition); 1542 EmitUint8(0xC0 + dst.LowBits()); 1543} 1544 1545 1546void X86_64Assembler::LoadDoubleConstant(XmmRegister dst, double value) { 1547 // TODO: Need to have a code constants table. 1548 int64_t constant = bit_cast<int64_t, double>(value); 1549 pushq(Immediate(High32Bits(constant))); 1550 pushq(Immediate(Low32Bits(constant))); 1551 movsd(dst, Address(CpuRegister(RSP), 0)); 1552 addq(CpuRegister(RSP), Immediate(2 * sizeof(intptr_t))); 1553} 1554 1555 1556void X86_64Assembler::FloatNegate(XmmRegister f) { 1557 static const struct { 1558 uint32_t a; 1559 uint32_t b; 1560 uint32_t c; 1561 uint32_t d; 1562 } float_negate_constant __attribute__((aligned(16))) = 1563 { 0x80000000, 0x00000000, 0x80000000, 0x00000000 }; 1564 xorps(f, Address::Absolute(reinterpret_cast<uintptr_t>(&float_negate_constant))); 1565} 1566 1567 1568void X86_64Assembler::DoubleNegate(XmmRegister d) { 1569 static const struct { 1570 uint64_t a; 1571 uint64_t b; 1572 } double_negate_constant __attribute__((aligned(16))) = 1573 {0x8000000000000000LL, 0x8000000000000000LL}; 1574 xorpd(d, Address::Absolute(reinterpret_cast<uintptr_t>(&double_negate_constant))); 1575} 1576 1577 1578void X86_64Assembler::DoubleAbs(XmmRegister reg) { 1579 static const struct { 1580 uint64_t a; 1581 uint64_t b; 1582 } double_abs_constant __attribute__((aligned(16))) = 1583 {0x7FFFFFFFFFFFFFFFLL, 0x7FFFFFFFFFFFFFFFLL}; 1584 andpd(reg, Address::Absolute(reinterpret_cast<uintptr_t>(&double_abs_constant))); 1585} 1586 1587 1588void X86_64Assembler::Align(int alignment, int offset) { 1589 CHECK(IsPowerOfTwo(alignment)); 1590 // Emit nop instruction until the real position is aligned. 1591 while (((offset + buffer_.GetPosition()) & (alignment-1)) != 0) { 1592 nop(); 1593 } 1594} 1595 1596 1597void X86_64Assembler::Bind(Label* label) { 1598 int bound = buffer_.Size(); 1599 CHECK(!label->IsBound()); // Labels can only be bound once. 1600 while (label->IsLinked()) { 1601 int position = label->LinkPosition(); 1602 int next = buffer_.Load<int32_t>(position); 1603 buffer_.Store<int32_t>(position, bound - (position + 4)); 1604 label->position_ = next; 1605 } 1606 label->BindTo(bound); 1607} 1608 1609 1610void X86_64Assembler::EmitOperand(uint8_t reg_or_opcode, const Operand& operand) { 1611 CHECK_GE(reg_or_opcode, 0); 1612 CHECK_LT(reg_or_opcode, 8); 1613 const int length = operand.length_; 1614 CHECK_GT(length, 0); 1615 // Emit the ModRM byte updated with the given reg value. 1616 CHECK_EQ(operand.encoding_[0] & 0x38, 0); 1617 EmitUint8(operand.encoding_[0] + (reg_or_opcode << 3)); 1618 // Emit the rest of the encoded operand. 1619 for (int i = 1; i < length; i++) { 1620 EmitUint8(operand.encoding_[i]); 1621 } 1622} 1623 1624 1625void X86_64Assembler::EmitImmediate(const Immediate& imm) { 1626 if (imm.is_int32()) { 1627 EmitInt32(static_cast<int32_t>(imm.value())); 1628 } else { 1629 EmitInt64(imm.value()); 1630 } 1631} 1632 1633 1634void X86_64Assembler::EmitComplex(uint8_t reg_or_opcode, 1635 const Operand& operand, 1636 const Immediate& immediate) { 1637 CHECK_GE(reg_or_opcode, 0); 1638 CHECK_LT(reg_or_opcode, 8); 1639 if (immediate.is_int8()) { 1640 // Use sign-extended 8-bit immediate. 1641 EmitUint8(0x83); 1642 EmitOperand(reg_or_opcode, operand); 1643 EmitUint8(immediate.value() & 0xFF); 1644 } else if (operand.IsRegister(CpuRegister(RAX))) { 1645 // Use short form if the destination is eax. 1646 EmitUint8(0x05 + (reg_or_opcode << 3)); 1647 EmitImmediate(immediate); 1648 } else { 1649 EmitUint8(0x81); 1650 EmitOperand(reg_or_opcode, operand); 1651 EmitImmediate(immediate); 1652 } 1653} 1654 1655 1656void X86_64Assembler::EmitLabel(Label* label, int instruction_size) { 1657 if (label->IsBound()) { 1658 int offset = label->Position() - buffer_.Size(); 1659 CHECK_LE(offset, 0); 1660 EmitInt32(offset - instruction_size); 1661 } else { 1662 EmitLabelLink(label); 1663 } 1664} 1665 1666 1667void X86_64Assembler::EmitLabelLink(Label* label) { 1668 CHECK(!label->IsBound()); 1669 int position = buffer_.Size(); 1670 EmitInt32(label->position_); 1671 label->LinkTo(position); 1672} 1673 1674 1675void X86_64Assembler::EmitGenericShift(bool wide, 1676 int reg_or_opcode, 1677 CpuRegister reg, 1678 const Immediate& imm) { 1679 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1680 CHECK(imm.is_int8()); 1681 if (wide) { 1682 EmitRex64(reg); 1683 } 1684 if (imm.value() == 1) { 1685 EmitUint8(0xD1); 1686 EmitOperand(reg_or_opcode, Operand(reg)); 1687 } else { 1688 EmitUint8(0xC1); 1689 EmitOperand(reg_or_opcode, Operand(reg)); 1690 EmitUint8(imm.value() & 0xFF); 1691 } 1692} 1693 1694 1695void X86_64Assembler::EmitGenericShift(int reg_or_opcode, 1696 CpuRegister operand, 1697 CpuRegister shifter) { 1698 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1699 CHECK_EQ(shifter.AsRegister(), RCX); 1700 EmitUint8(0xD3); 1701 EmitOperand(reg_or_opcode, Operand(operand)); 1702} 1703 1704void X86_64Assembler::EmitOptionalRex(bool force, bool w, bool r, bool x, bool b) { 1705 // REX.WRXB 1706 // W - 64-bit operand 1707 // R - MODRM.reg 1708 // X - SIB.index 1709 // B - MODRM.rm/SIB.base 1710 uint8_t rex = force ? 0x40 : 0; 1711 if (w) { 1712 rex |= 0x48; // REX.W000 1713 } 1714 if (r) { 1715 rex |= 0x44; // REX.0R00 1716 } 1717 if (x) { 1718 rex |= 0x42; // REX.00X0 1719 } 1720 if (b) { 1721 rex |= 0x41; // REX.000B 1722 } 1723 if (rex != 0) { 1724 EmitUint8(rex); 1725 } 1726} 1727 1728void X86_64Assembler::EmitOptionalRex32(CpuRegister reg) { 1729 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 1730} 1731 1732void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, CpuRegister src) { 1733 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1734} 1735 1736void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, XmmRegister src) { 1737 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1738} 1739 1740void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, XmmRegister src) { 1741 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1742} 1743 1744void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, CpuRegister src) { 1745 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1746} 1747 1748void X86_64Assembler::EmitOptionalRex32(const Operand& operand) { 1749 uint8_t rex = operand.rex(); 1750 if (rex != 0) { 1751 EmitUint8(rex); 1752 } 1753} 1754 1755void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, const Operand& operand) { 1756 uint8_t rex = operand.rex(); 1757 if (dst.NeedsRex()) { 1758 rex |= 0x44; // REX.0R00 1759 } 1760 if (rex != 0) { 1761 EmitUint8(rex); 1762 } 1763} 1764 1765void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, const Operand& operand) { 1766 uint8_t rex = operand.rex(); 1767 if (dst.NeedsRex()) { 1768 rex |= 0x44; // REX.0R00 1769 } 1770 if (rex != 0) { 1771 EmitUint8(rex); 1772 } 1773} 1774 1775void X86_64Assembler::EmitRex64(CpuRegister reg) { 1776 EmitOptionalRex(false, true, false, false, reg.NeedsRex()); 1777} 1778 1779void X86_64Assembler::EmitRex64(CpuRegister dst, CpuRegister src) { 1780 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 1781} 1782 1783void X86_64Assembler::EmitRex64(XmmRegister dst, CpuRegister src) { 1784 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 1785} 1786 1787void X86_64Assembler::EmitRex64(CpuRegister dst, const Operand& operand) { 1788 uint8_t rex = 0x48 | operand.rex(); // REX.W000 1789 if (dst.NeedsRex()) { 1790 rex |= 0x44; // REX.0R00 1791 } 1792 if (rex != 0) { 1793 EmitUint8(rex); 1794 } 1795} 1796 1797void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, CpuRegister src) { 1798 EmitOptionalRex(true, false, dst.NeedsRex(), false, src.NeedsRex()); 1799} 1800 1801void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand) { 1802 uint8_t rex = 0x40 | operand.rex(); // REX.0000 1803 if (dst.NeedsRex()) { 1804 rex |= 0x44; // REX.0R00 1805 } 1806 if (rex != 0) { 1807 EmitUint8(rex); 1808 } 1809} 1810 1811void X86_64Assembler::InitializeFrameDescriptionEntry() { 1812 WriteFDEHeader(&cfi_info_, true /* is_64bit */); 1813} 1814 1815void X86_64Assembler::FinalizeFrameDescriptionEntry() { 1816 WriteFDEAddressRange(&cfi_info_, buffer_.Size(), true /* is_64bit */); 1817 PadCFI(&cfi_info_); 1818 WriteCFILength(&cfi_info_, true /* is_64bit */); 1819} 1820 1821constexpr size_t kFramePointerSize = 8; 1822 1823void X86_64Assembler::BuildFrame(size_t frame_size, ManagedRegister method_reg, 1824 const std::vector<ManagedRegister>& spill_regs, 1825 const ManagedRegisterEntrySpills& entry_spills) { 1826 cfi_cfa_offset_ = kFramePointerSize; // Only return address on stack 1827 cfi_pc_ = buffer_.Size(); // Nothing emitted yet 1828 DCHECK_EQ(cfi_pc_, 0U); 1829 1830 uint32_t reg_offset = 1; 1831 CHECK_ALIGNED(frame_size, kStackAlignment); 1832 int gpr_count = 0; 1833 for (int i = spill_regs.size() - 1; i >= 0; --i) { 1834 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 1835 if (spill.IsCpuRegister()) { 1836 pushq(spill.AsCpuRegister()); 1837 gpr_count++; 1838 1839 // DW_CFA_advance_loc 1840 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 1841 cfi_pc_ = buffer_.Size(); 1842 // DW_CFA_def_cfa_offset 1843 cfi_cfa_offset_ += kFramePointerSize; 1844 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 1845 // DW_CFA_offset reg offset 1846 reg_offset++; 1847 DW_CFA_offset(&cfi_info_, spill.DWARFRegId(), reg_offset); 1848 } 1849 } 1850 // return address then method on stack 1851 int64_t rest_of_frame = static_cast<int64_t>(frame_size) 1852 - (gpr_count * kFramePointerSize) 1853 - kFramePointerSize /*return address*/; 1854 subq(CpuRegister(RSP), Immediate(rest_of_frame)); 1855 // DW_CFA_advance_loc 1856 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 1857 cfi_pc_ = buffer_.Size(); 1858 // DW_CFA_def_cfa_offset 1859 cfi_cfa_offset_ += rest_of_frame; 1860 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 1861 1862 // spill xmms 1863 int64_t offset = rest_of_frame; 1864 for (int i = spill_regs.size() - 1; i >= 0; --i) { 1865 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 1866 if (spill.IsXmmRegister()) { 1867 offset -= sizeof(double); 1868 movsd(Address(CpuRegister(RSP), offset), spill.AsXmmRegister()); 1869 } 1870 } 1871 1872 DCHECK_EQ(4U, sizeof(StackReference<mirror::ArtMethod>)); 1873 1874 movl(Address(CpuRegister(RSP), 0), method_reg.AsX86_64().AsCpuRegister()); 1875 1876 for (size_t i = 0; i < entry_spills.size(); ++i) { 1877 ManagedRegisterSpill spill = entry_spills.at(i); 1878 if (spill.AsX86_64().IsCpuRegister()) { 1879 if (spill.getSize() == 8) { 1880 movq(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), 1881 spill.AsX86_64().AsCpuRegister()); 1882 } else { 1883 CHECK_EQ(spill.getSize(), 4); 1884 movl(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsCpuRegister()); 1885 } 1886 } else { 1887 if (spill.getSize() == 8) { 1888 movsd(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 1889 } else { 1890 CHECK_EQ(spill.getSize(), 4); 1891 movss(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 1892 } 1893 } 1894 } 1895} 1896 1897void X86_64Assembler::RemoveFrame(size_t frame_size, 1898 const std::vector<ManagedRegister>& spill_regs) { 1899 CHECK_ALIGNED(frame_size, kStackAlignment); 1900 int gpr_count = 0; 1901 // unspill xmms 1902 int64_t offset = static_cast<int64_t>(frame_size) - (spill_regs.size() * kFramePointerSize) - 2 * kFramePointerSize; 1903 for (size_t i = 0; i < spill_regs.size(); ++i) { 1904 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 1905 if (spill.IsXmmRegister()) { 1906 offset += sizeof(double); 1907 movsd(spill.AsXmmRegister(), Address(CpuRegister(RSP), offset)); 1908 } else { 1909 gpr_count++; 1910 } 1911 } 1912 addq(CpuRegister(RSP), Immediate(static_cast<int64_t>(frame_size) - (gpr_count * kFramePointerSize) - kFramePointerSize)); 1913 for (size_t i = 0; i < spill_regs.size(); ++i) { 1914 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 1915 if (spill.IsCpuRegister()) { 1916 popq(spill.AsCpuRegister()); 1917 } 1918 } 1919 ret(); 1920} 1921 1922void X86_64Assembler::IncreaseFrameSize(size_t adjust) { 1923 CHECK_ALIGNED(adjust, kStackAlignment); 1924 addq(CpuRegister(RSP), Immediate(-static_cast<int64_t>(adjust))); 1925 // DW_CFA_advance_loc 1926 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 1927 cfi_pc_ = buffer_.Size(); 1928 // DW_CFA_def_cfa_offset 1929 cfi_cfa_offset_ += adjust; 1930 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 1931} 1932 1933void X86_64Assembler::DecreaseFrameSize(size_t adjust) { 1934 CHECK_ALIGNED(adjust, kStackAlignment); 1935 addq(CpuRegister(RSP), Immediate(adjust)); 1936} 1937 1938void X86_64Assembler::Store(FrameOffset offs, ManagedRegister msrc, size_t size) { 1939 X86_64ManagedRegister src = msrc.AsX86_64(); 1940 if (src.IsNoRegister()) { 1941 CHECK_EQ(0u, size); 1942 } else if (src.IsCpuRegister()) { 1943 if (size == 4) { 1944 CHECK_EQ(4u, size); 1945 movl(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 1946 } else { 1947 CHECK_EQ(8u, size); 1948 movq(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 1949 } 1950 } else if (src.IsRegisterPair()) { 1951 CHECK_EQ(0u, size); 1952 movq(Address(CpuRegister(RSP), offs), src.AsRegisterPairLow()); 1953 movq(Address(CpuRegister(RSP), FrameOffset(offs.Int32Value()+4)), 1954 src.AsRegisterPairHigh()); 1955 } else if (src.IsX87Register()) { 1956 if (size == 4) { 1957 fstps(Address(CpuRegister(RSP), offs)); 1958 } else { 1959 fstpl(Address(CpuRegister(RSP), offs)); 1960 } 1961 } else { 1962 CHECK(src.IsXmmRegister()); 1963 if (size == 4) { 1964 movss(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 1965 } else { 1966 movsd(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 1967 } 1968 } 1969} 1970 1971void X86_64Assembler::StoreRef(FrameOffset dest, ManagedRegister msrc) { 1972 X86_64ManagedRegister src = msrc.AsX86_64(); 1973 CHECK(src.IsCpuRegister()); 1974 movl(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 1975} 1976 1977void X86_64Assembler::StoreRawPtr(FrameOffset dest, ManagedRegister msrc) { 1978 X86_64ManagedRegister src = msrc.AsX86_64(); 1979 CHECK(src.IsCpuRegister()); 1980 movq(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 1981} 1982 1983void X86_64Assembler::StoreImmediateToFrame(FrameOffset dest, uint32_t imm, 1984 ManagedRegister) { 1985 movl(Address(CpuRegister(RSP), dest), Immediate(imm)); // TODO(64) movq? 1986} 1987 1988void X86_64Assembler::StoreImmediateToThread64(ThreadOffset<8> dest, uint32_t imm, 1989 ManagedRegister) { 1990 gs()->movl(Address::Absolute(dest, true), Immediate(imm)); // TODO(64) movq? 1991} 1992 1993void X86_64Assembler::StoreStackOffsetToThread64(ThreadOffset<8> thr_offs, 1994 FrameOffset fr_offs, 1995 ManagedRegister mscratch) { 1996 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 1997 CHECK(scratch.IsCpuRegister()); 1998 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), fr_offs)); 1999 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2000} 2001 2002void X86_64Assembler::StoreStackPointerToThread64(ThreadOffset<8> thr_offs) { 2003 gs()->movq(Address::Absolute(thr_offs, true), CpuRegister(RSP)); 2004} 2005 2006void X86_64Assembler::StoreSpanning(FrameOffset /*dst*/, ManagedRegister /*src*/, 2007 FrameOffset /*in_off*/, ManagedRegister /*scratch*/) { 2008 UNIMPLEMENTED(FATAL); // this case only currently exists for ARM 2009} 2010 2011void X86_64Assembler::Load(ManagedRegister mdest, FrameOffset src, size_t size) { 2012 X86_64ManagedRegister dest = mdest.AsX86_64(); 2013 if (dest.IsNoRegister()) { 2014 CHECK_EQ(0u, size); 2015 } else if (dest.IsCpuRegister()) { 2016 if (size == 4) { 2017 CHECK_EQ(4u, size); 2018 movl(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2019 } else { 2020 CHECK_EQ(8u, size); 2021 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2022 } 2023 } else if (dest.IsRegisterPair()) { 2024 CHECK_EQ(0u, size); 2025 movq(dest.AsRegisterPairLow(), Address(CpuRegister(RSP), src)); 2026 movq(dest.AsRegisterPairHigh(), Address(CpuRegister(RSP), FrameOffset(src.Int32Value()+4))); 2027 } else if (dest.IsX87Register()) { 2028 if (size == 4) { 2029 flds(Address(CpuRegister(RSP), src)); 2030 } else { 2031 fldl(Address(CpuRegister(RSP), src)); 2032 } 2033 } else { 2034 CHECK(dest.IsXmmRegister()); 2035 if (size == 4) { 2036 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2037 } else { 2038 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2039 } 2040 } 2041} 2042 2043void X86_64Assembler::LoadFromThread64(ManagedRegister mdest, ThreadOffset<8> src, size_t size) { 2044 X86_64ManagedRegister dest = mdest.AsX86_64(); 2045 if (dest.IsNoRegister()) { 2046 CHECK_EQ(0u, size); 2047 } else if (dest.IsCpuRegister()) { 2048 CHECK_EQ(4u, size); 2049 gs()->movl(dest.AsCpuRegister(), Address::Absolute(src, true)); 2050 } else if (dest.IsRegisterPair()) { 2051 CHECK_EQ(8u, size); 2052 gs()->movq(dest.AsRegisterPairLow(), Address::Absolute(src, true)); 2053 } else if (dest.IsX87Register()) { 2054 if (size == 4) { 2055 gs()->flds(Address::Absolute(src, true)); 2056 } else { 2057 gs()->fldl(Address::Absolute(src, true)); 2058 } 2059 } else { 2060 CHECK(dest.IsXmmRegister()); 2061 if (size == 4) { 2062 gs()->movss(dest.AsXmmRegister(), Address::Absolute(src, true)); 2063 } else { 2064 gs()->movsd(dest.AsXmmRegister(), Address::Absolute(src, true)); 2065 } 2066 } 2067} 2068 2069void X86_64Assembler::LoadRef(ManagedRegister mdest, FrameOffset src) { 2070 X86_64ManagedRegister dest = mdest.AsX86_64(); 2071 CHECK(dest.IsCpuRegister()); 2072 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2073} 2074 2075void X86_64Assembler::LoadRef(ManagedRegister mdest, ManagedRegister base, 2076 MemberOffset offs) { 2077 X86_64ManagedRegister dest = mdest.AsX86_64(); 2078 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2079 movl(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2080 if (kPoisonHeapReferences) { 2081 negl(dest.AsCpuRegister()); 2082 } 2083} 2084 2085void X86_64Assembler::LoadRawPtr(ManagedRegister mdest, ManagedRegister base, 2086 Offset offs) { 2087 X86_64ManagedRegister dest = mdest.AsX86_64(); 2088 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2089 movq(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2090} 2091 2092void X86_64Assembler::LoadRawPtrFromThread64(ManagedRegister mdest, ThreadOffset<8> offs) { 2093 X86_64ManagedRegister dest = mdest.AsX86_64(); 2094 CHECK(dest.IsCpuRegister()); 2095 gs()->movq(dest.AsCpuRegister(), Address::Absolute(offs, true)); 2096} 2097 2098void X86_64Assembler::SignExtend(ManagedRegister mreg, size_t size) { 2099 X86_64ManagedRegister reg = mreg.AsX86_64(); 2100 CHECK(size == 1 || size == 2) << size; 2101 CHECK(reg.IsCpuRegister()) << reg; 2102 if (size == 1) { 2103 movsxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2104 } else { 2105 movsxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2106 } 2107} 2108 2109void X86_64Assembler::ZeroExtend(ManagedRegister mreg, size_t size) { 2110 X86_64ManagedRegister reg = mreg.AsX86_64(); 2111 CHECK(size == 1 || size == 2) << size; 2112 CHECK(reg.IsCpuRegister()) << reg; 2113 if (size == 1) { 2114 movzxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2115 } else { 2116 movzxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2117 } 2118} 2119 2120void X86_64Assembler::Move(ManagedRegister mdest, ManagedRegister msrc, size_t size) { 2121 X86_64ManagedRegister dest = mdest.AsX86_64(); 2122 X86_64ManagedRegister src = msrc.AsX86_64(); 2123 if (!dest.Equals(src)) { 2124 if (dest.IsCpuRegister() && src.IsCpuRegister()) { 2125 movq(dest.AsCpuRegister(), src.AsCpuRegister()); 2126 } else if (src.IsX87Register() && dest.IsXmmRegister()) { 2127 // Pass via stack and pop X87 register 2128 subl(CpuRegister(RSP), Immediate(16)); 2129 if (size == 4) { 2130 CHECK_EQ(src.AsX87Register(), ST0); 2131 fstps(Address(CpuRegister(RSP), 0)); 2132 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2133 } else { 2134 CHECK_EQ(src.AsX87Register(), ST0); 2135 fstpl(Address(CpuRegister(RSP), 0)); 2136 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2137 } 2138 addq(CpuRegister(RSP), Immediate(16)); 2139 } else { 2140 // TODO: x87, SSE 2141 UNIMPLEMENTED(FATAL) << ": Move " << dest << ", " << src; 2142 } 2143 } 2144} 2145 2146void X86_64Assembler::CopyRef(FrameOffset dest, FrameOffset src, 2147 ManagedRegister mscratch) { 2148 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2149 CHECK(scratch.IsCpuRegister()); 2150 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2151 movl(Address(CpuRegister(RSP), dest), scratch.AsCpuRegister()); 2152} 2153 2154void X86_64Assembler::CopyRawPtrFromThread64(FrameOffset fr_offs, 2155 ThreadOffset<8> thr_offs, 2156 ManagedRegister mscratch) { 2157 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2158 CHECK(scratch.IsCpuRegister()); 2159 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(thr_offs, true)); 2160 Store(fr_offs, scratch, 8); 2161} 2162 2163void X86_64Assembler::CopyRawPtrToThread64(ThreadOffset<8> thr_offs, 2164 FrameOffset fr_offs, 2165 ManagedRegister mscratch) { 2166 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2167 CHECK(scratch.IsCpuRegister()); 2168 Load(scratch, fr_offs, 8); 2169 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2170} 2171 2172void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src, 2173 ManagedRegister mscratch, 2174 size_t size) { 2175 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2176 if (scratch.IsCpuRegister() && size == 8) { 2177 Load(scratch, src, 4); 2178 Store(dest, scratch, 4); 2179 Load(scratch, FrameOffset(src.Int32Value() + 4), 4); 2180 Store(FrameOffset(dest.Int32Value() + 4), scratch, 4); 2181 } else { 2182 Load(scratch, src, size); 2183 Store(dest, scratch, size); 2184 } 2185} 2186 2187void X86_64Assembler::Copy(FrameOffset /*dst*/, ManagedRegister /*src_base*/, Offset /*src_offset*/, 2188 ManagedRegister /*scratch*/, size_t /*size*/) { 2189 UNIMPLEMENTED(FATAL); 2190} 2191 2192void X86_64Assembler::Copy(ManagedRegister dest_base, Offset dest_offset, FrameOffset src, 2193 ManagedRegister scratch, size_t size) { 2194 CHECK(scratch.IsNoRegister()); 2195 CHECK_EQ(size, 4u); 2196 pushq(Address(CpuRegister(RSP), src)); 2197 popq(Address(dest_base.AsX86_64().AsCpuRegister(), dest_offset)); 2198} 2199 2200void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src_base, Offset src_offset, 2201 ManagedRegister mscratch, size_t size) { 2202 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2203 CHECK_EQ(size, 4u); 2204 movq(scratch, Address(CpuRegister(RSP), src_base)); 2205 movq(scratch, Address(scratch, src_offset)); 2206 movq(Address(CpuRegister(RSP), dest), scratch); 2207} 2208 2209void X86_64Assembler::Copy(ManagedRegister dest, Offset dest_offset, 2210 ManagedRegister src, Offset src_offset, 2211 ManagedRegister scratch, size_t size) { 2212 CHECK_EQ(size, 4u); 2213 CHECK(scratch.IsNoRegister()); 2214 pushq(Address(src.AsX86_64().AsCpuRegister(), src_offset)); 2215 popq(Address(dest.AsX86_64().AsCpuRegister(), dest_offset)); 2216} 2217 2218void X86_64Assembler::Copy(FrameOffset dest, Offset dest_offset, FrameOffset src, Offset src_offset, 2219 ManagedRegister mscratch, size_t size) { 2220 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2221 CHECK_EQ(size, 4u); 2222 CHECK_EQ(dest.Int32Value(), src.Int32Value()); 2223 movq(scratch, Address(CpuRegister(RSP), src)); 2224 pushq(Address(scratch, src_offset)); 2225 popq(Address(scratch, dest_offset)); 2226} 2227 2228void X86_64Assembler::MemoryBarrier(ManagedRegister) { 2229#if ANDROID_SMP != 0 2230 mfence(); 2231#endif 2232} 2233 2234void X86_64Assembler::CreateHandleScopeEntry(ManagedRegister mout_reg, 2235 FrameOffset handle_scope_offset, 2236 ManagedRegister min_reg, bool null_allowed) { 2237 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2238 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2239 if (in_reg.IsNoRegister()) { // TODO(64): && null_allowed 2240 // Use out_reg as indicator of NULL 2241 in_reg = out_reg; 2242 // TODO: movzwl 2243 movl(in_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2244 } 2245 CHECK(in_reg.IsCpuRegister()); 2246 CHECK(out_reg.IsCpuRegister()); 2247 VerifyObject(in_reg, null_allowed); 2248 if (null_allowed) { 2249 Label null_arg; 2250 if (!out_reg.Equals(in_reg)) { 2251 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2252 } 2253 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2254 j(kZero, &null_arg); 2255 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2256 Bind(&null_arg); 2257 } else { 2258 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2259 } 2260} 2261 2262void X86_64Assembler::CreateHandleScopeEntry(FrameOffset out_off, 2263 FrameOffset handle_scope_offset, 2264 ManagedRegister mscratch, 2265 bool null_allowed) { 2266 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2267 CHECK(scratch.IsCpuRegister()); 2268 if (null_allowed) { 2269 Label null_arg; 2270 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2271 testl(scratch.AsCpuRegister(), scratch.AsCpuRegister()); 2272 j(kZero, &null_arg); 2273 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2274 Bind(&null_arg); 2275 } else { 2276 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2277 } 2278 Store(out_off, scratch, 8); 2279} 2280 2281// Given a handle scope entry, load the associated reference. 2282void X86_64Assembler::LoadReferenceFromHandleScope(ManagedRegister mout_reg, 2283 ManagedRegister min_reg) { 2284 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2285 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2286 CHECK(out_reg.IsCpuRegister()); 2287 CHECK(in_reg.IsCpuRegister()); 2288 Label null_arg; 2289 if (!out_reg.Equals(in_reg)) { 2290 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2291 } 2292 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2293 j(kZero, &null_arg); 2294 movq(out_reg.AsCpuRegister(), Address(in_reg.AsCpuRegister(), 0)); 2295 Bind(&null_arg); 2296} 2297 2298void X86_64Assembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) { 2299 // TODO: not validating references 2300} 2301 2302void X86_64Assembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) { 2303 // TODO: not validating references 2304} 2305 2306void X86_64Assembler::Call(ManagedRegister mbase, Offset offset, ManagedRegister) { 2307 X86_64ManagedRegister base = mbase.AsX86_64(); 2308 CHECK(base.IsCpuRegister()); 2309 call(Address(base.AsCpuRegister(), offset.Int32Value())); 2310 // TODO: place reference map on call 2311} 2312 2313void X86_64Assembler::Call(FrameOffset base, Offset offset, ManagedRegister mscratch) { 2314 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2315 movl(scratch, Address(CpuRegister(RSP), base)); 2316 call(Address(scratch, offset)); 2317} 2318 2319void X86_64Assembler::CallFromThread64(ThreadOffset<8> offset, ManagedRegister /*mscratch*/) { 2320 gs()->call(Address::Absolute(offset, true)); 2321} 2322 2323void X86_64Assembler::GetCurrentThread(ManagedRegister tr) { 2324 gs()->movq(tr.AsX86_64().AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2325} 2326 2327void X86_64Assembler::GetCurrentThread(FrameOffset offset, ManagedRegister mscratch) { 2328 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2329 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2330 movq(Address(CpuRegister(RSP), offset), scratch.AsCpuRegister()); 2331} 2332 2333// Slowpath entered when Thread::Current()->_exception is non-null 2334class X86_64ExceptionSlowPath FINAL : public SlowPath { 2335 public: 2336 explicit X86_64ExceptionSlowPath(size_t stack_adjust) : stack_adjust_(stack_adjust) {} 2337 virtual void Emit(Assembler *sp_asm) OVERRIDE; 2338 private: 2339 const size_t stack_adjust_; 2340}; 2341 2342void X86_64Assembler::ExceptionPoll(ManagedRegister /*scratch*/, size_t stack_adjust) { 2343 X86_64ExceptionSlowPath* slow = new X86_64ExceptionSlowPath(stack_adjust); 2344 buffer_.EnqueueSlowPath(slow); 2345 gs()->cmpl(Address::Absolute(Thread::ExceptionOffset<8>(), true), Immediate(0)); 2346 j(kNotEqual, slow->Entry()); 2347} 2348 2349void X86_64ExceptionSlowPath::Emit(Assembler *sasm) { 2350 X86_64Assembler* sp_asm = down_cast<X86_64Assembler*>(sasm); 2351#define __ sp_asm-> 2352 __ Bind(&entry_); 2353 // Note: the return value is dead 2354 if (stack_adjust_ != 0) { // Fix up the frame. 2355 __ DecreaseFrameSize(stack_adjust_); 2356 } 2357 // Pass exception as argument in RDI 2358 __ gs()->movq(CpuRegister(RDI), Address::Absolute(Thread::ExceptionOffset<8>(), true)); 2359 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(8, pDeliverException), true)); 2360 // this call should never return 2361 __ int3(); 2362#undef __ 2363} 2364 2365} // namespace x86_64 2366} // namespace art 2367