assembler_x86_64.cc revision 547cdfd21ee21e4ab9ca8692d6ef47c62ee7ea52
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "assembler_x86_64.h" 18 19#include "base/casts.h" 20#include "entrypoints/quick/quick_entrypoints.h" 21#include "memory_region.h" 22#include "thread.h" 23#include "utils/dwarf_cfi.h" 24 25namespace art { 26namespace x86_64 { 27 28std::ostream& operator<<(std::ostream& os, const CpuRegister& reg) { 29 return os << reg.AsRegister(); 30} 31 32std::ostream& operator<<(std::ostream& os, const XmmRegister& reg) { 33 return os << reg.AsFloatRegister(); 34} 35 36std::ostream& operator<<(std::ostream& os, const X87Register& reg) { 37 return os << "ST" << static_cast<int>(reg); 38} 39 40void X86_64Assembler::call(CpuRegister reg) { 41 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 42 EmitOptionalRex32(reg); 43 EmitUint8(0xFF); 44 EmitRegisterOperand(2, reg.LowBits()); 45} 46 47 48void X86_64Assembler::call(const Address& address) { 49 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 50 EmitOptionalRex32(address); 51 EmitUint8(0xFF); 52 EmitOperand(2, address); 53} 54 55 56void X86_64Assembler::call(Label* label) { 57 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 58 EmitUint8(0xE8); 59 static const int kSize = 5; 60 EmitLabel(label, kSize); 61} 62 63void X86_64Assembler::pushq(CpuRegister reg) { 64 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 65 EmitOptionalRex32(reg); 66 EmitUint8(0x50 + reg.LowBits()); 67} 68 69 70void X86_64Assembler::pushq(const Address& address) { 71 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 72 EmitOptionalRex32(address); 73 EmitUint8(0xFF); 74 EmitOperand(6, address); 75} 76 77 78void X86_64Assembler::pushq(const Immediate& imm) { 79 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 80 CHECK(imm.is_int32()); // pushq only supports 32b immediate. 81 if (imm.is_int8()) { 82 EmitUint8(0x6A); 83 EmitUint8(imm.value() & 0xFF); 84 } else { 85 EmitUint8(0x68); 86 EmitImmediate(imm); 87 } 88} 89 90 91void X86_64Assembler::popq(CpuRegister reg) { 92 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 93 EmitOptionalRex32(reg); 94 EmitUint8(0x58 + reg.LowBits()); 95} 96 97 98void X86_64Assembler::popq(const Address& address) { 99 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 100 EmitOptionalRex32(address); 101 EmitUint8(0x8F); 102 EmitOperand(0, address); 103} 104 105 106void X86_64Assembler::movq(CpuRegister dst, const Immediate& imm) { 107 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 108 if (imm.is_int32()) { 109 // 32 bit. Note: sign-extends. 110 EmitRex64(dst); 111 EmitUint8(0xC7); 112 EmitRegisterOperand(0, dst.LowBits()); 113 EmitInt32(static_cast<int32_t>(imm.value())); 114 } else { 115 EmitRex64(dst); 116 EmitUint8(0xB8 + dst.LowBits()); 117 EmitInt64(imm.value()); 118 } 119} 120 121 122void X86_64Assembler::movl(CpuRegister dst, const Immediate& imm) { 123 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 124 EmitOptionalRex32(dst); 125 EmitUint8(0xB8 + dst.LowBits()); 126 EmitImmediate(imm); 127} 128 129 130void X86_64Assembler::movq(CpuRegister dst, CpuRegister src) { 131 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 132 // 0x89 is movq r/m64 <- r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 133 EmitRex64(src, dst); 134 EmitUint8(0x89); 135 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 136} 137 138 139void X86_64Assembler::movl(CpuRegister dst, CpuRegister src) { 140 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 141 EmitOptionalRex32(dst, src); 142 EmitUint8(0x8B); 143 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 144} 145 146 147void X86_64Assembler::movq(CpuRegister dst, const Address& src) { 148 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 149 EmitRex64(dst, src); 150 EmitUint8(0x8B); 151 EmitOperand(dst.LowBits(), src); 152} 153 154 155void X86_64Assembler::movl(CpuRegister dst, const Address& src) { 156 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 157 EmitOptionalRex32(dst, src); 158 EmitUint8(0x8B); 159 EmitOperand(dst.LowBits(), src); 160} 161 162 163void X86_64Assembler::movq(const Address& dst, CpuRegister src) { 164 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 165 EmitRex64(src, dst); 166 EmitUint8(0x89); 167 EmitOperand(src.LowBits(), dst); 168} 169 170 171void X86_64Assembler::movl(const Address& dst, CpuRegister src) { 172 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 173 EmitOptionalRex32(src, dst); 174 EmitUint8(0x89); 175 EmitOperand(src.LowBits(), dst); 176} 177 178void X86_64Assembler::movl(const Address& dst, const Immediate& imm) { 179 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 180 EmitOptionalRex32(dst); 181 EmitUint8(0xC7); 182 EmitOperand(0, dst); 183 EmitImmediate(imm); 184} 185 186void X86_64Assembler::movzxb(CpuRegister dst, CpuRegister src) { 187 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 188 EmitOptionalByteRegNormalizingRex32(dst, src); 189 EmitUint8(0x0F); 190 EmitUint8(0xB6); 191 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 192} 193 194 195void X86_64Assembler::movzxb(CpuRegister dst, const Address& src) { 196 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 197 EmitOptionalByteRegNormalizingRex32(dst, src); 198 EmitUint8(0x0F); 199 EmitUint8(0xB6); 200 EmitOperand(dst.LowBits(), src); 201} 202 203 204void X86_64Assembler::movsxb(CpuRegister dst, CpuRegister src) { 205 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 206 EmitOptionalByteRegNormalizingRex32(dst, src); 207 EmitUint8(0x0F); 208 EmitUint8(0xBE); 209 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 210} 211 212 213void X86_64Assembler::movsxb(CpuRegister dst, const Address& src) { 214 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 215 EmitOptionalByteRegNormalizingRex32(dst, src); 216 EmitUint8(0x0F); 217 EmitUint8(0xBE); 218 EmitOperand(dst.LowBits(), src); 219} 220 221 222void X86_64Assembler::movb(CpuRegister /*dst*/, const Address& /*src*/) { 223 LOG(FATAL) << "Use movzxb or movsxb instead."; 224} 225 226 227void X86_64Assembler::movb(const Address& dst, CpuRegister src) { 228 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 229 EmitOptionalByteRegNormalizingRex32(src, dst); 230 EmitUint8(0x88); 231 EmitOperand(src.LowBits(), dst); 232} 233 234 235void X86_64Assembler::movb(const Address& dst, const Immediate& imm) { 236 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 237 EmitUint8(0xC6); 238 EmitOperand(Register::RAX, dst); 239 CHECK(imm.is_int8()); 240 EmitUint8(imm.value() & 0xFF); 241} 242 243 244void X86_64Assembler::movzxw(CpuRegister dst, CpuRegister src) { 245 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 246 EmitOptionalRex32(dst, src); 247 EmitUint8(0x0F); 248 EmitUint8(0xB7); 249 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 250} 251 252 253void X86_64Assembler::movzxw(CpuRegister dst, const Address& src) { 254 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 255 EmitOptionalRex32(dst, src); 256 EmitUint8(0x0F); 257 EmitUint8(0xB7); 258 EmitOperand(dst.LowBits(), src); 259} 260 261 262void X86_64Assembler::movsxw(CpuRegister dst, CpuRegister src) { 263 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 264 EmitOptionalRex32(dst, src); 265 EmitUint8(0x0F); 266 EmitUint8(0xBF); 267 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 268} 269 270 271void X86_64Assembler::movsxw(CpuRegister dst, const Address& src) { 272 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 273 EmitOptionalRex32(dst, src); 274 EmitUint8(0x0F); 275 EmitUint8(0xBF); 276 EmitOperand(dst.LowBits(), src); 277} 278 279 280void X86_64Assembler::movw(CpuRegister /*dst*/, const Address& /*src*/) { 281 LOG(FATAL) << "Use movzxw or movsxw instead."; 282} 283 284 285void X86_64Assembler::movw(const Address& dst, CpuRegister src) { 286 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 287 EmitOperandSizeOverride(); 288 EmitOptionalRex32(src, dst); 289 EmitUint8(0x89); 290 EmitOperand(src.LowBits(), dst); 291} 292 293 294void X86_64Assembler::leaq(CpuRegister dst, const Address& src) { 295 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 296 EmitRex64(dst, src); 297 EmitUint8(0x8D); 298 EmitOperand(dst.LowBits(), src); 299} 300 301 302void X86_64Assembler::movss(XmmRegister dst, const Address& src) { 303 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 304 EmitUint8(0xF3); 305 EmitOptionalRex32(dst, src); 306 EmitUint8(0x0F); 307 EmitUint8(0x10); 308 EmitOperand(dst.LowBits(), src); 309} 310 311 312void X86_64Assembler::movss(const Address& dst, XmmRegister src) { 313 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 314 EmitUint8(0xF3); 315 EmitOptionalRex32(src, dst); 316 EmitUint8(0x0F); 317 EmitUint8(0x11); 318 EmitOperand(src.LowBits(), dst); 319} 320 321 322void X86_64Assembler::movss(XmmRegister dst, XmmRegister src) { 323 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 324 EmitUint8(0xF3); 325 EmitOptionalRex32(dst, src); 326 EmitUint8(0x0F); 327 EmitUint8(0x11); 328 EmitXmmRegisterOperand(src.LowBits(), dst); 329} 330 331 332void X86_64Assembler::movd(XmmRegister dst, CpuRegister src) { 333 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 334 EmitUint8(0x66); 335 EmitOptionalRex32(dst, src); 336 EmitUint8(0x0F); 337 EmitUint8(0x6E); 338 EmitOperand(dst.LowBits(), Operand(src)); 339} 340 341 342void X86_64Assembler::movd(CpuRegister dst, XmmRegister src) { 343 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 344 EmitUint8(0x66); 345 EmitOptionalRex32(src, dst); 346 EmitUint8(0x0F); 347 EmitUint8(0x7E); 348 EmitOperand(src.LowBits(), Operand(dst)); 349} 350 351 352void X86_64Assembler::addss(XmmRegister dst, XmmRegister src) { 353 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 354 EmitUint8(0xF3); 355 EmitOptionalRex32(dst, src); 356 EmitUint8(0x0F); 357 EmitUint8(0x58); 358 EmitXmmRegisterOperand(dst.LowBits(), src); 359} 360 361 362void X86_64Assembler::addss(XmmRegister dst, const Address& src) { 363 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 364 EmitUint8(0xF3); 365 EmitOptionalRex32(dst, src); 366 EmitUint8(0x0F); 367 EmitUint8(0x58); 368 EmitOperand(dst.LowBits(), src); 369} 370 371 372void X86_64Assembler::subss(XmmRegister dst, XmmRegister src) { 373 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 374 EmitUint8(0xF3); 375 EmitOptionalRex32(dst, src); 376 EmitUint8(0x0F); 377 EmitUint8(0x5C); 378 EmitXmmRegisterOperand(dst.LowBits(), src); 379} 380 381 382void X86_64Assembler::subss(XmmRegister dst, const Address& src) { 383 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 384 EmitUint8(0xF3); 385 EmitOptionalRex32(dst, src); 386 EmitUint8(0x0F); 387 EmitUint8(0x5C); 388 EmitOperand(dst.LowBits(), src); 389} 390 391 392void X86_64Assembler::mulss(XmmRegister dst, XmmRegister src) { 393 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 394 EmitUint8(0xF3); 395 EmitOptionalRex32(dst, src); 396 EmitUint8(0x0F); 397 EmitUint8(0x59); 398 EmitXmmRegisterOperand(dst.LowBits(), src); 399} 400 401 402void X86_64Assembler::mulss(XmmRegister dst, const Address& src) { 403 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 404 EmitUint8(0xF3); 405 EmitOptionalRex32(dst, src); 406 EmitUint8(0x0F); 407 EmitUint8(0x59); 408 EmitOperand(dst.LowBits(), src); 409} 410 411 412void X86_64Assembler::divss(XmmRegister dst, XmmRegister src) { 413 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 414 EmitUint8(0xF3); 415 EmitOptionalRex32(dst, src); 416 EmitUint8(0x0F); 417 EmitUint8(0x5E); 418 EmitXmmRegisterOperand(dst.LowBits(), src); 419} 420 421 422void X86_64Assembler::divss(XmmRegister dst, const Address& src) { 423 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 424 EmitUint8(0xF3); 425 EmitOptionalRex32(dst, src); 426 EmitUint8(0x0F); 427 EmitUint8(0x5E); 428 EmitOperand(dst.LowBits(), src); 429} 430 431 432void X86_64Assembler::flds(const Address& src) { 433 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 434 EmitUint8(0xD9); 435 EmitOperand(0, src); 436} 437 438 439void X86_64Assembler::fstps(const Address& dst) { 440 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 441 EmitUint8(0xD9); 442 EmitOperand(3, dst); 443} 444 445 446void X86_64Assembler::movsd(XmmRegister dst, const Address& src) { 447 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 448 EmitUint8(0xF2); 449 EmitOptionalRex32(dst, src); 450 EmitUint8(0x0F); 451 EmitUint8(0x10); 452 EmitOperand(dst.LowBits(), src); 453} 454 455 456void X86_64Assembler::movsd(const Address& dst, XmmRegister src) { 457 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 458 EmitUint8(0xF2); 459 EmitOptionalRex32(src, dst); 460 EmitUint8(0x0F); 461 EmitUint8(0x11); 462 EmitOperand(src.LowBits(), dst); 463} 464 465 466void X86_64Assembler::movsd(XmmRegister dst, XmmRegister src) { 467 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 468 EmitUint8(0xF2); 469 EmitOptionalRex32(dst, src); 470 EmitUint8(0x0F); 471 EmitUint8(0x11); 472 EmitXmmRegisterOperand(src.LowBits(), dst); 473} 474 475 476void X86_64Assembler::addsd(XmmRegister dst, XmmRegister src) { 477 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 478 EmitUint8(0xF2); 479 EmitOptionalRex32(dst, src); 480 EmitUint8(0x0F); 481 EmitUint8(0x58); 482 EmitXmmRegisterOperand(dst.LowBits(), src); 483} 484 485 486void X86_64Assembler::addsd(XmmRegister dst, const Address& src) { 487 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 488 EmitUint8(0xF2); 489 EmitOptionalRex32(dst, src); 490 EmitUint8(0x0F); 491 EmitUint8(0x58); 492 EmitOperand(dst.LowBits(), src); 493} 494 495 496void X86_64Assembler::subsd(XmmRegister dst, XmmRegister src) { 497 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 498 EmitUint8(0xF2); 499 EmitOptionalRex32(dst, src); 500 EmitUint8(0x0F); 501 EmitUint8(0x5C); 502 EmitXmmRegisterOperand(dst.LowBits(), src); 503} 504 505 506void X86_64Assembler::subsd(XmmRegister dst, const Address& src) { 507 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 508 EmitUint8(0xF2); 509 EmitOptionalRex32(dst, src); 510 EmitUint8(0x0F); 511 EmitUint8(0x5C); 512 EmitOperand(dst.LowBits(), src); 513} 514 515 516void X86_64Assembler::mulsd(XmmRegister dst, XmmRegister src) { 517 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 518 EmitUint8(0xF2); 519 EmitOptionalRex32(dst, src); 520 EmitUint8(0x0F); 521 EmitUint8(0x59); 522 EmitXmmRegisterOperand(dst.LowBits(), src); 523} 524 525 526void X86_64Assembler::mulsd(XmmRegister dst, const Address& src) { 527 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 528 EmitUint8(0xF2); 529 EmitOptionalRex32(dst, src); 530 EmitUint8(0x0F); 531 EmitUint8(0x59); 532 EmitOperand(dst.LowBits(), src); 533} 534 535 536void X86_64Assembler::divsd(XmmRegister dst, XmmRegister src) { 537 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 538 EmitUint8(0xF2); 539 EmitOptionalRex32(dst, src); 540 EmitUint8(0x0F); 541 EmitUint8(0x5E); 542 EmitXmmRegisterOperand(dst.LowBits(), src); 543} 544 545 546void X86_64Assembler::divsd(XmmRegister dst, const Address& src) { 547 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 548 EmitUint8(0xF2); 549 EmitOptionalRex32(dst, src); 550 EmitUint8(0x0F); 551 EmitUint8(0x5E); 552 EmitOperand(dst.LowBits(), src); 553} 554 555 556void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) { 557 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 558 EmitUint8(0xF3); 559 EmitOptionalRex32(dst, src); 560 EmitUint8(0x0F); 561 EmitUint8(0x2A); 562 EmitOperand(dst.LowBits(), Operand(src)); 563} 564 565 566void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src) { 567 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 568 EmitUint8(0xF2); 569 EmitOptionalRex32(dst, src); 570 EmitUint8(0x0F); 571 EmitUint8(0x2A); 572 EmitOperand(dst.LowBits(), Operand(src)); 573} 574 575 576void X86_64Assembler::cvtss2si(CpuRegister dst, XmmRegister src) { 577 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 578 EmitUint8(0xF3); 579 EmitOptionalRex32(dst, src); 580 EmitUint8(0x0F); 581 EmitUint8(0x2D); 582 EmitXmmRegisterOperand(dst.LowBits(), src); 583} 584 585 586void X86_64Assembler::cvtss2sd(XmmRegister dst, XmmRegister src) { 587 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 588 EmitUint8(0xF3); 589 EmitOptionalRex32(dst, src); 590 EmitUint8(0x0F); 591 EmitUint8(0x5A); 592 EmitXmmRegisterOperand(dst.LowBits(), src); 593} 594 595 596void X86_64Assembler::cvtsd2si(CpuRegister dst, XmmRegister src) { 597 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 598 EmitUint8(0xF2); 599 EmitOptionalRex32(dst, src); 600 EmitUint8(0x0F); 601 EmitUint8(0x2D); 602 EmitXmmRegisterOperand(dst.LowBits(), src); 603} 604 605 606void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src) { 607 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 608 EmitUint8(0xF3); 609 EmitOptionalRex32(dst, src); 610 EmitUint8(0x0F); 611 EmitUint8(0x2C); 612 EmitXmmRegisterOperand(dst.LowBits(), src); 613} 614 615 616void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src) { 617 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 618 EmitUint8(0xF2); 619 EmitOptionalRex32(dst, src); 620 EmitUint8(0x0F); 621 EmitUint8(0x2C); 622 EmitXmmRegisterOperand(dst.LowBits(), src); 623} 624 625 626void X86_64Assembler::cvtsd2ss(XmmRegister dst, XmmRegister src) { 627 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 628 EmitUint8(0xF2); 629 EmitOptionalRex32(dst, src); 630 EmitUint8(0x0F); 631 EmitUint8(0x5A); 632 EmitXmmRegisterOperand(dst.LowBits(), src); 633} 634 635 636void X86_64Assembler::cvtdq2pd(XmmRegister dst, XmmRegister src) { 637 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 638 EmitUint8(0xF3); 639 EmitOptionalRex32(dst, src); 640 EmitUint8(0x0F); 641 EmitUint8(0xE6); 642 EmitXmmRegisterOperand(dst.LowBits(), src); 643} 644 645 646void X86_64Assembler::comiss(XmmRegister a, XmmRegister b) { 647 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 648 EmitOptionalRex32(a, b); 649 EmitUint8(0x0F); 650 EmitUint8(0x2F); 651 EmitXmmRegisterOperand(a.LowBits(), b); 652} 653 654 655void X86_64Assembler::comisd(XmmRegister a, XmmRegister b) { 656 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 657 EmitUint8(0x66); 658 EmitOptionalRex32(a, b); 659 EmitUint8(0x0F); 660 EmitUint8(0x2F); 661 EmitXmmRegisterOperand(a.LowBits(), b); 662} 663 664 665void X86_64Assembler::sqrtsd(XmmRegister dst, XmmRegister src) { 666 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 667 EmitUint8(0xF2); 668 EmitOptionalRex32(dst, src); 669 EmitUint8(0x0F); 670 EmitUint8(0x51); 671 EmitXmmRegisterOperand(dst.LowBits(), src); 672} 673 674 675void X86_64Assembler::sqrtss(XmmRegister dst, XmmRegister src) { 676 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 677 EmitUint8(0xF3); 678 EmitOptionalRex32(dst, src); 679 EmitUint8(0x0F); 680 EmitUint8(0x51); 681 EmitXmmRegisterOperand(dst.LowBits(), src); 682} 683 684 685void X86_64Assembler::xorpd(XmmRegister dst, const Address& src) { 686 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 687 EmitUint8(0x66); 688 EmitOptionalRex32(dst, src); 689 EmitUint8(0x0F); 690 EmitUint8(0x57); 691 EmitOperand(dst.LowBits(), src); 692} 693 694 695void X86_64Assembler::xorpd(XmmRegister dst, XmmRegister src) { 696 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 697 EmitUint8(0x66); 698 EmitOptionalRex32(dst, src); 699 EmitUint8(0x0F); 700 EmitUint8(0x57); 701 EmitXmmRegisterOperand(dst.LowBits(), src); 702} 703 704 705void X86_64Assembler::xorps(XmmRegister dst, const Address& src) { 706 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 707 EmitOptionalRex32(dst, src); 708 EmitUint8(0x0F); 709 EmitUint8(0x57); 710 EmitOperand(dst.LowBits(), src); 711} 712 713 714void X86_64Assembler::xorps(XmmRegister dst, XmmRegister src) { 715 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 716 EmitOptionalRex32(dst, src); 717 EmitUint8(0x0F); 718 EmitUint8(0x57); 719 EmitXmmRegisterOperand(dst.LowBits(), src); 720} 721 722 723void X86_64Assembler::andpd(XmmRegister dst, const Address& src) { 724 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 725 EmitUint8(0x66); 726 EmitOptionalRex32(dst, src); 727 EmitUint8(0x0F); 728 EmitUint8(0x54); 729 EmitOperand(dst.LowBits(), src); 730} 731 732 733void X86_64Assembler::fldl(const Address& src) { 734 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 735 EmitUint8(0xDD); 736 EmitOperand(0, src); 737} 738 739 740void X86_64Assembler::fstpl(const Address& dst) { 741 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 742 EmitUint8(0xDD); 743 EmitOperand(3, dst); 744} 745 746 747void X86_64Assembler::fnstcw(const Address& dst) { 748 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 749 EmitUint8(0xD9); 750 EmitOperand(7, dst); 751} 752 753 754void X86_64Assembler::fldcw(const Address& src) { 755 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 756 EmitUint8(0xD9); 757 EmitOperand(5, src); 758} 759 760 761void X86_64Assembler::fistpl(const Address& dst) { 762 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 763 EmitUint8(0xDF); 764 EmitOperand(7, dst); 765} 766 767 768void X86_64Assembler::fistps(const Address& dst) { 769 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 770 EmitUint8(0xDB); 771 EmitOperand(3, dst); 772} 773 774 775void X86_64Assembler::fildl(const Address& src) { 776 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 777 EmitUint8(0xDF); 778 EmitOperand(5, src); 779} 780 781 782void X86_64Assembler::fincstp() { 783 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 784 EmitUint8(0xD9); 785 EmitUint8(0xF7); 786} 787 788 789void X86_64Assembler::ffree(const Immediate& index) { 790 CHECK_LT(index.value(), 7); 791 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 792 EmitUint8(0xDD); 793 EmitUint8(0xC0 + index.value()); 794} 795 796 797void X86_64Assembler::fsin() { 798 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 799 EmitUint8(0xD9); 800 EmitUint8(0xFE); 801} 802 803 804void X86_64Assembler::fcos() { 805 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 806 EmitUint8(0xD9); 807 EmitUint8(0xFF); 808} 809 810 811void X86_64Assembler::fptan() { 812 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 813 EmitUint8(0xD9); 814 EmitUint8(0xF2); 815} 816 817 818void X86_64Assembler::xchgl(CpuRegister dst, CpuRegister src) { 819 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 820 EmitOptionalRex32(dst, src); 821 EmitUint8(0x87); 822 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 823} 824 825 826void X86_64Assembler::xchgq(CpuRegister dst, CpuRegister src) { 827 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 828 EmitRex64(dst, src); 829 EmitUint8(0x87); 830 EmitOperand(dst.LowBits(), Operand(src)); 831} 832 833 834void X86_64Assembler::xchgl(CpuRegister reg, const Address& address) { 835 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 836 EmitOptionalRex32(reg, address); 837 EmitUint8(0x87); 838 EmitOperand(reg.LowBits(), address); 839} 840 841 842void X86_64Assembler::cmpl(CpuRegister reg, const Immediate& imm) { 843 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 844 EmitOptionalRex32(reg); 845 EmitComplex(7, Operand(reg), imm); 846} 847 848 849void X86_64Assembler::cmpl(CpuRegister reg0, CpuRegister reg1) { 850 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 851 EmitOptionalRex32(reg0, reg1); 852 EmitUint8(0x3B); 853 EmitOperand(reg0.LowBits(), Operand(reg1)); 854} 855 856 857void X86_64Assembler::cmpl(CpuRegister reg, const Address& address) { 858 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 859 EmitOptionalRex32(reg, address); 860 EmitUint8(0x3B); 861 EmitOperand(reg.LowBits(), address); 862} 863 864 865void X86_64Assembler::cmpq(CpuRegister reg0, CpuRegister reg1) { 866 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 867 EmitRex64(reg0, reg1); 868 EmitUint8(0x3B); 869 EmitOperand(reg0.LowBits(), Operand(reg1)); 870} 871 872 873void X86_64Assembler::cmpq(CpuRegister reg, const Immediate& imm) { 874 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 875 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 876 EmitRex64(reg); 877 EmitComplex(7, Operand(reg), imm); 878} 879 880 881void X86_64Assembler::cmpq(CpuRegister reg, const Address& address) { 882 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 883 EmitRex64(reg); 884 EmitUint8(0x3B); 885 EmitOperand(reg.LowBits(), address); 886} 887 888 889void X86_64Assembler::addl(CpuRegister dst, CpuRegister src) { 890 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 891 EmitOptionalRex32(dst, src); 892 EmitUint8(0x03); 893 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 894} 895 896 897void X86_64Assembler::addl(CpuRegister reg, const Address& address) { 898 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 899 EmitOptionalRex32(reg, address); 900 EmitUint8(0x03); 901 EmitOperand(reg.LowBits(), address); 902} 903 904 905void X86_64Assembler::cmpl(const Address& address, CpuRegister reg) { 906 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 907 EmitOptionalRex32(reg, address); 908 EmitUint8(0x39); 909 EmitOperand(reg.LowBits(), address); 910} 911 912 913void X86_64Assembler::cmpl(const Address& address, const Immediate& imm) { 914 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 915 EmitOptionalRex32(address); 916 EmitComplex(7, address, imm); 917} 918 919 920void X86_64Assembler::testl(CpuRegister reg1, CpuRegister reg2) { 921 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 922 EmitOptionalRex32(reg1, reg2); 923 EmitUint8(0x85); 924 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 925} 926 927 928void X86_64Assembler::testl(CpuRegister reg, const Immediate& immediate) { 929 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 930 // For registers that have a byte variant (RAX, RBX, RCX, and RDX) 931 // we only test the byte CpuRegister to keep the encoding short. 932 if (immediate.is_uint8() && reg.AsRegister() < 4) { 933 // Use zero-extended 8-bit immediate. 934 if (reg.AsRegister() == RAX) { 935 EmitUint8(0xA8); 936 } else { 937 EmitUint8(0xF6); 938 EmitUint8(0xC0 + reg.AsRegister()); 939 } 940 EmitUint8(immediate.value() & 0xFF); 941 } else if (reg.AsRegister() == RAX) { 942 // Use short form if the destination is RAX. 943 EmitUint8(0xA9); 944 EmitImmediate(immediate); 945 } else { 946 EmitOptionalRex32(reg); 947 EmitUint8(0xF7); 948 EmitOperand(0, Operand(reg)); 949 EmitImmediate(immediate); 950 } 951} 952 953 954void X86_64Assembler::testq(CpuRegister reg, const Address& address) { 955 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 956 EmitRex64(reg); 957 EmitUint8(0x85); 958 EmitOperand(reg.LowBits(), address); 959} 960 961 962void X86_64Assembler::andl(CpuRegister dst, CpuRegister src) { 963 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 964 EmitOptionalRex32(dst, src); 965 EmitUint8(0x23); 966 EmitOperand(dst.LowBits(), Operand(src)); 967} 968 969 970void X86_64Assembler::andl(CpuRegister dst, const Immediate& imm) { 971 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 972 EmitOptionalRex32(dst); 973 EmitComplex(4, Operand(dst), imm); 974} 975 976 977void X86_64Assembler::andq(CpuRegister reg, const Immediate& imm) { 978 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 979 CHECK(imm.is_int32()); // andq only supports 32b immediate. 980 EmitRex64(reg); 981 EmitComplex(4, Operand(reg), imm); 982} 983 984 985void X86_64Assembler::orl(CpuRegister dst, CpuRegister src) { 986 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 987 EmitOptionalRex32(dst, src); 988 EmitUint8(0x0B); 989 EmitOperand(dst.LowBits(), Operand(src)); 990} 991 992 993void X86_64Assembler::orl(CpuRegister dst, const Immediate& imm) { 994 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 995 EmitOptionalRex32(dst); 996 EmitComplex(1, Operand(dst), imm); 997} 998 999 1000void X86_64Assembler::xorl(CpuRegister dst, CpuRegister src) { 1001 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1002 EmitOptionalRex32(dst, src); 1003 EmitUint8(0x33); 1004 EmitOperand(dst.LowBits(), Operand(src)); 1005} 1006 1007 1008void X86_64Assembler::xorq(CpuRegister dst, CpuRegister src) { 1009 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1010 EmitRex64(dst, src); 1011 EmitUint8(0x33); 1012 EmitOperand(dst.LowBits(), Operand(src)); 1013} 1014 1015 1016void X86_64Assembler::xorq(CpuRegister dst, const Immediate& imm) { 1017 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1018 CHECK(imm.is_int32()); // xorq only supports 32b immediate. 1019 EmitRex64(dst); 1020 EmitComplex(6, Operand(dst), imm); 1021} 1022 1023#if 0 1024void X86_64Assembler::rex(bool force, bool w, Register* r, Register* x, Register* b) { 1025 // REX.WRXB 1026 // W - 64-bit operand 1027 // R - MODRM.reg 1028 // X - SIB.index 1029 // B - MODRM.rm/SIB.base 1030 uint8_t rex = force ? 0x40 : 0; 1031 if (w) { 1032 rex |= 0x48; // REX.W000 1033 } 1034 if (r != nullptr && *r >= Register::R8 && *r < Register::kNumberOfCpuRegisters) { 1035 rex |= 0x44; // REX.0R00 1036 *r = static_cast<Register>(*r - 8); 1037 } 1038 if (x != nullptr && *x >= Register::R8 && *x < Register::kNumberOfCpuRegisters) { 1039 rex |= 0x42; // REX.00X0 1040 *x = static_cast<Register>(*x - 8); 1041 } 1042 if (b != nullptr && *b >= Register::R8 && *b < Register::kNumberOfCpuRegisters) { 1043 rex |= 0x41; // REX.000B 1044 *b = static_cast<Register>(*b - 8); 1045 } 1046 if (rex != 0) { 1047 EmitUint8(rex); 1048 } 1049} 1050 1051void X86_64Assembler::rex_reg_mem(bool force, bool w, Register* dst, const Address& mem) { 1052 // REX.WRXB 1053 // W - 64-bit operand 1054 // R - MODRM.reg 1055 // X - SIB.index 1056 // B - MODRM.rm/SIB.base 1057 uint8_t rex = mem->rex(); 1058 if (force) { 1059 rex |= 0x40; // REX.0000 1060 } 1061 if (w) { 1062 rex |= 0x48; // REX.W000 1063 } 1064 if (dst != nullptr && *dst >= Register::R8 && *dst < Register::kNumberOfCpuRegisters) { 1065 rex |= 0x44; // REX.0R00 1066 *dst = static_cast<Register>(*dst - 8); 1067 } 1068 if (rex != 0) { 1069 EmitUint8(rex); 1070 } 1071} 1072 1073void rex_mem_reg(bool force, bool w, Address* mem, Register* src); 1074#endif 1075 1076void X86_64Assembler::addl(CpuRegister reg, const Immediate& imm) { 1077 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1078 EmitOptionalRex32(reg); 1079 EmitComplex(0, Operand(reg), imm); 1080} 1081 1082 1083void X86_64Assembler::addq(CpuRegister reg, const Immediate& imm) { 1084 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1085 CHECK(imm.is_int32()); // addq only supports 32b immediate. 1086 EmitRex64(reg); 1087 EmitComplex(0, Operand(reg), imm); 1088} 1089 1090 1091void X86_64Assembler::addq(CpuRegister dst, const Address& address) { 1092 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1093 EmitRex64(dst); 1094 EmitUint8(0x03); 1095 EmitOperand(dst.LowBits(), address); 1096} 1097 1098 1099void X86_64Assembler::addq(CpuRegister dst, CpuRegister src) { 1100 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1101 // 0x01 is addq r/m64 <- r/m64 + r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 1102 EmitRex64(src, dst); 1103 EmitUint8(0x01); 1104 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1105} 1106 1107 1108void X86_64Assembler::addl(const Address& address, CpuRegister reg) { 1109 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1110 EmitOptionalRex32(reg, address); 1111 EmitUint8(0x01); 1112 EmitOperand(reg.LowBits(), address); 1113} 1114 1115 1116void X86_64Assembler::addl(const Address& address, const Immediate& imm) { 1117 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1118 EmitOptionalRex32(address); 1119 EmitComplex(0, address, imm); 1120} 1121 1122 1123void X86_64Assembler::subl(CpuRegister dst, CpuRegister src) { 1124 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1125 EmitOptionalRex32(dst, src); 1126 EmitUint8(0x2B); 1127 EmitOperand(dst.LowBits(), Operand(src)); 1128} 1129 1130 1131void X86_64Assembler::subl(CpuRegister reg, const Immediate& imm) { 1132 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1133 EmitOptionalRex32(reg); 1134 EmitComplex(5, Operand(reg), imm); 1135} 1136 1137 1138void X86_64Assembler::subq(CpuRegister reg, const Immediate& imm) { 1139 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1140 CHECK(imm.is_int32()); // subq only supports 32b immediate. 1141 EmitRex64(reg); 1142 EmitComplex(5, Operand(reg), imm); 1143} 1144 1145 1146void X86_64Assembler::subq(CpuRegister dst, CpuRegister src) { 1147 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1148 EmitRex64(dst, src); 1149 EmitUint8(0x2B); 1150 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1151} 1152 1153 1154void X86_64Assembler::subq(CpuRegister reg, const Address& address) { 1155 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1156 EmitRex64(reg); 1157 EmitUint8(0x2B); 1158 EmitOperand(reg.LowBits() & 7, address); 1159} 1160 1161 1162void X86_64Assembler::subl(CpuRegister reg, const Address& address) { 1163 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1164 EmitOptionalRex32(reg, address); 1165 EmitUint8(0x2B); 1166 EmitOperand(reg.LowBits(), address); 1167} 1168 1169 1170void X86_64Assembler::cdq() { 1171 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1172 EmitUint8(0x99); 1173} 1174 1175 1176void X86_64Assembler::idivl(CpuRegister reg) { 1177 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1178 EmitOptionalRex32(reg); 1179 EmitUint8(0xF7); 1180 EmitUint8(0xF8 | reg.LowBits()); 1181} 1182 1183 1184void X86_64Assembler::imull(CpuRegister dst, CpuRegister src) { 1185 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1186 EmitOptionalRex32(dst, src); 1187 EmitUint8(0x0F); 1188 EmitUint8(0xAF); 1189 EmitOperand(dst.LowBits(), Operand(src)); 1190} 1191 1192 1193void X86_64Assembler::imull(CpuRegister reg, const Immediate& imm) { 1194 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1195 EmitOptionalRex32(reg); 1196 EmitUint8(0x69); 1197 EmitOperand(reg.LowBits(), Operand(reg)); 1198 EmitImmediate(imm); 1199} 1200 1201 1202void X86_64Assembler::imull(CpuRegister reg, const Address& address) { 1203 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1204 EmitOptionalRex32(reg, address); 1205 EmitUint8(0x0F); 1206 EmitUint8(0xAF); 1207 EmitOperand(reg.LowBits(), address); 1208} 1209 1210 1211void X86_64Assembler::imull(CpuRegister reg) { 1212 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1213 EmitOptionalRex32(reg); 1214 EmitUint8(0xF7); 1215 EmitOperand(5, Operand(reg)); 1216} 1217 1218 1219void X86_64Assembler::imull(const Address& address) { 1220 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1221 EmitOptionalRex32(address); 1222 EmitUint8(0xF7); 1223 EmitOperand(5, address); 1224} 1225 1226 1227void X86_64Assembler::mull(CpuRegister reg) { 1228 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1229 EmitOptionalRex32(reg); 1230 EmitUint8(0xF7); 1231 EmitOperand(4, Operand(reg)); 1232} 1233 1234 1235void X86_64Assembler::mull(const Address& address) { 1236 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1237 EmitOptionalRex32(address); 1238 EmitUint8(0xF7); 1239 EmitOperand(4, address); 1240} 1241 1242 1243 1244void X86_64Assembler::shll(CpuRegister reg, const Immediate& imm) { 1245 EmitGenericShift(false, 4, reg, imm); 1246} 1247 1248 1249void X86_64Assembler::shll(CpuRegister operand, CpuRegister shifter) { 1250 EmitGenericShift(4, operand, shifter); 1251} 1252 1253 1254void X86_64Assembler::shrl(CpuRegister reg, const Immediate& imm) { 1255 EmitGenericShift(false, 5, reg, imm); 1256} 1257 1258 1259void X86_64Assembler::shrq(CpuRegister reg, const Immediate& imm) { 1260 EmitGenericShift(true, 5, reg, imm); 1261} 1262 1263 1264void X86_64Assembler::shrl(CpuRegister operand, CpuRegister shifter) { 1265 EmitGenericShift(5, operand, shifter); 1266} 1267 1268 1269void X86_64Assembler::sarl(CpuRegister reg, const Immediate& imm) { 1270 EmitGenericShift(false, 7, reg, imm); 1271} 1272 1273 1274void X86_64Assembler::sarl(CpuRegister operand, CpuRegister shifter) { 1275 EmitGenericShift(7, operand, shifter); 1276} 1277 1278 1279void X86_64Assembler::negl(CpuRegister reg) { 1280 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1281 EmitOptionalRex32(reg); 1282 EmitUint8(0xF7); 1283 EmitOperand(3, Operand(reg)); 1284} 1285 1286 1287void X86_64Assembler::notl(CpuRegister reg) { 1288 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1289 EmitOptionalRex32(reg); 1290 EmitUint8(0xF7); 1291 EmitUint8(0xD0 | reg.LowBits()); 1292} 1293 1294 1295void X86_64Assembler::enter(const Immediate& imm) { 1296 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1297 EmitUint8(0xC8); 1298 CHECK(imm.is_uint16()); 1299 EmitUint8(imm.value() & 0xFF); 1300 EmitUint8((imm.value() >> 8) & 0xFF); 1301 EmitUint8(0x00); 1302} 1303 1304 1305void X86_64Assembler::leave() { 1306 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1307 EmitUint8(0xC9); 1308} 1309 1310 1311void X86_64Assembler::ret() { 1312 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1313 EmitUint8(0xC3); 1314} 1315 1316 1317void X86_64Assembler::ret(const Immediate& imm) { 1318 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1319 EmitUint8(0xC2); 1320 CHECK(imm.is_uint16()); 1321 EmitUint8(imm.value() & 0xFF); 1322 EmitUint8((imm.value() >> 8) & 0xFF); 1323} 1324 1325 1326 1327void X86_64Assembler::nop() { 1328 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1329 EmitUint8(0x90); 1330} 1331 1332 1333void X86_64Assembler::int3() { 1334 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1335 EmitUint8(0xCC); 1336} 1337 1338 1339void X86_64Assembler::hlt() { 1340 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1341 EmitUint8(0xF4); 1342} 1343 1344 1345void X86_64Assembler::j(Condition condition, Label* label) { 1346 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1347 if (label->IsBound()) { 1348 static const int kShortSize = 2; 1349 static const int kLongSize = 6; 1350 int offset = label->Position() - buffer_.Size(); 1351 CHECK_LE(offset, 0); 1352 if (IsInt(8, offset - kShortSize)) { 1353 EmitUint8(0x70 + condition); 1354 EmitUint8((offset - kShortSize) & 0xFF); 1355 } else { 1356 EmitUint8(0x0F); 1357 EmitUint8(0x80 + condition); 1358 EmitInt32(offset - kLongSize); 1359 } 1360 } else { 1361 EmitUint8(0x0F); 1362 EmitUint8(0x80 + condition); 1363 EmitLabelLink(label); 1364 } 1365} 1366 1367 1368void X86_64Assembler::jmp(CpuRegister reg) { 1369 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1370 EmitOptionalRex32(reg); 1371 EmitUint8(0xFF); 1372 EmitRegisterOperand(4, reg.LowBits()); 1373} 1374 1375void X86_64Assembler::jmp(const Address& address) { 1376 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1377 EmitOptionalRex32(address); 1378 EmitUint8(0xFF); 1379 EmitOperand(4, address); 1380} 1381 1382void X86_64Assembler::jmp(Label* label) { 1383 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1384 if (label->IsBound()) { 1385 static const int kShortSize = 2; 1386 static const int kLongSize = 5; 1387 int offset = label->Position() - buffer_.Size(); 1388 CHECK_LE(offset, 0); 1389 if (IsInt(8, offset - kShortSize)) { 1390 EmitUint8(0xEB); 1391 EmitUint8((offset - kShortSize) & 0xFF); 1392 } else { 1393 EmitUint8(0xE9); 1394 EmitInt32(offset - kLongSize); 1395 } 1396 } else { 1397 EmitUint8(0xE9); 1398 EmitLabelLink(label); 1399 } 1400} 1401 1402 1403X86_64Assembler* X86_64Assembler::lock() { 1404 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1405 EmitUint8(0xF0); 1406 return this; 1407} 1408 1409 1410void X86_64Assembler::cmpxchgl(const Address& address, CpuRegister reg) { 1411 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1412 EmitUint8(0x0F); 1413 EmitUint8(0xB1); 1414 EmitOperand(reg.LowBits(), address); 1415} 1416 1417void X86_64Assembler::mfence() { 1418 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1419 EmitUint8(0x0F); 1420 EmitUint8(0xAE); 1421 EmitUint8(0xF0); 1422} 1423 1424 1425X86_64Assembler* X86_64Assembler::gs() { 1426 // TODO: gs is a prefix and not an instruction 1427 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1428 EmitUint8(0x65); 1429 return this; 1430} 1431 1432 1433void X86_64Assembler::AddImmediate(CpuRegister reg, const Immediate& imm) { 1434 int value = imm.value(); 1435 if (value != 0) { 1436 if (value > 0) { 1437 addl(reg, imm); 1438 } else { 1439 subl(reg, Immediate(value)); 1440 } 1441 } 1442} 1443 1444 1445void X86_64Assembler::setcc(Condition condition, CpuRegister dst) { 1446 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1447 // RSP, RBP, RDI, RSI need rex prefix (else the pattern encodes ah/bh/ch/dh). 1448 if (dst.NeedsRex() || dst.AsRegister() > 3) { 1449 EmitOptionalRex(true, false, false, false, dst.NeedsRex()); 1450 } 1451 EmitUint8(0x0F); 1452 EmitUint8(0x90 + condition); 1453 EmitUint8(0xC0 + dst.LowBits()); 1454} 1455 1456 1457void X86_64Assembler::LoadDoubleConstant(XmmRegister dst, double value) { 1458 // TODO: Need to have a code constants table. 1459 int64_t constant = bit_cast<int64_t, double>(value); 1460 pushq(Immediate(High32Bits(constant))); 1461 pushq(Immediate(Low32Bits(constant))); 1462 movsd(dst, Address(CpuRegister(RSP), 0)); 1463 addq(CpuRegister(RSP), Immediate(2 * kWordSize)); 1464} 1465 1466 1467void X86_64Assembler::FloatNegate(XmmRegister f) { 1468 static const struct { 1469 uint32_t a; 1470 uint32_t b; 1471 uint32_t c; 1472 uint32_t d; 1473 } float_negate_constant __attribute__((aligned(16))) = 1474 { 0x80000000, 0x00000000, 0x80000000, 0x00000000 }; 1475 xorps(f, Address::Absolute(reinterpret_cast<uword>(&float_negate_constant))); 1476} 1477 1478 1479void X86_64Assembler::DoubleNegate(XmmRegister d) { 1480 static const struct { 1481 uint64_t a; 1482 uint64_t b; 1483 } double_negate_constant __attribute__((aligned(16))) = 1484 {0x8000000000000000LL, 0x8000000000000000LL}; 1485 xorpd(d, Address::Absolute(reinterpret_cast<uword>(&double_negate_constant))); 1486} 1487 1488 1489void X86_64Assembler::DoubleAbs(XmmRegister reg) { 1490 static const struct { 1491 uint64_t a; 1492 uint64_t b; 1493 } double_abs_constant __attribute__((aligned(16))) = 1494 {0x7FFFFFFFFFFFFFFFLL, 0x7FFFFFFFFFFFFFFFLL}; 1495 andpd(reg, Address::Absolute(reinterpret_cast<uword>(&double_abs_constant))); 1496} 1497 1498 1499void X86_64Assembler::Align(int alignment, int offset) { 1500 CHECK(IsPowerOfTwo(alignment)); 1501 // Emit nop instruction until the real position is aligned. 1502 while (((offset + buffer_.GetPosition()) & (alignment-1)) != 0) { 1503 nop(); 1504 } 1505} 1506 1507 1508void X86_64Assembler::Bind(Label* label) { 1509 int bound = buffer_.Size(); 1510 CHECK(!label->IsBound()); // Labels can only be bound once. 1511 while (label->IsLinked()) { 1512 int position = label->LinkPosition(); 1513 int next = buffer_.Load<int32_t>(position); 1514 buffer_.Store<int32_t>(position, bound - (position + 4)); 1515 label->position_ = next; 1516 } 1517 label->BindTo(bound); 1518} 1519 1520 1521void X86_64Assembler::EmitOperand(uint8_t reg_or_opcode, const Operand& operand) { 1522 CHECK_GE(reg_or_opcode, 0); 1523 CHECK_LT(reg_or_opcode, 8); 1524 const int length = operand.length_; 1525 CHECK_GT(length, 0); 1526 // Emit the ModRM byte updated with the given reg value. 1527 CHECK_EQ(operand.encoding_[0] & 0x38, 0); 1528 EmitUint8(operand.encoding_[0] + (reg_or_opcode << 3)); 1529 // Emit the rest of the encoded operand. 1530 for (int i = 1; i < length; i++) { 1531 EmitUint8(operand.encoding_[i]); 1532 } 1533} 1534 1535 1536void X86_64Assembler::EmitImmediate(const Immediate& imm) { 1537 if (imm.is_int32()) { 1538 EmitInt32(static_cast<int32_t>(imm.value())); 1539 } else { 1540 EmitInt64(imm.value()); 1541 } 1542} 1543 1544 1545void X86_64Assembler::EmitComplex(uint8_t reg_or_opcode, 1546 const Operand& operand, 1547 const Immediate& immediate) { 1548 CHECK_GE(reg_or_opcode, 0); 1549 CHECK_LT(reg_or_opcode, 8); 1550 if (immediate.is_int8()) { 1551 // Use sign-extended 8-bit immediate. 1552 EmitUint8(0x83); 1553 EmitOperand(reg_or_opcode, operand); 1554 EmitUint8(immediate.value() & 0xFF); 1555 } else if (operand.IsRegister(CpuRegister(RAX))) { 1556 // Use short form if the destination is eax. 1557 EmitUint8(0x05 + (reg_or_opcode << 3)); 1558 EmitImmediate(immediate); 1559 } else { 1560 EmitUint8(0x81); 1561 EmitOperand(reg_or_opcode, operand); 1562 EmitImmediate(immediate); 1563 } 1564} 1565 1566 1567void X86_64Assembler::EmitLabel(Label* label, int instruction_size) { 1568 if (label->IsBound()) { 1569 int offset = label->Position() - buffer_.Size(); 1570 CHECK_LE(offset, 0); 1571 EmitInt32(offset - instruction_size); 1572 } else { 1573 EmitLabelLink(label); 1574 } 1575} 1576 1577 1578void X86_64Assembler::EmitLabelLink(Label* label) { 1579 CHECK(!label->IsBound()); 1580 int position = buffer_.Size(); 1581 EmitInt32(label->position_); 1582 label->LinkTo(position); 1583} 1584 1585 1586void X86_64Assembler::EmitGenericShift(bool wide, 1587 int reg_or_opcode, 1588 CpuRegister reg, 1589 const Immediate& imm) { 1590 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1591 CHECK(imm.is_int8()); 1592 if (wide) { 1593 EmitRex64(reg); 1594 } 1595 if (imm.value() == 1) { 1596 EmitUint8(0xD1); 1597 EmitOperand(reg_or_opcode, Operand(reg)); 1598 } else { 1599 EmitUint8(0xC1); 1600 EmitOperand(reg_or_opcode, Operand(reg)); 1601 EmitUint8(imm.value() & 0xFF); 1602 } 1603} 1604 1605 1606void X86_64Assembler::EmitGenericShift(int reg_or_opcode, 1607 CpuRegister operand, 1608 CpuRegister shifter) { 1609 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1610 CHECK_EQ(shifter.AsRegister(), RCX); 1611 EmitUint8(0xD3); 1612 EmitOperand(reg_or_opcode, Operand(operand)); 1613} 1614 1615void X86_64Assembler::EmitOptionalRex(bool force, bool w, bool r, bool x, bool b) { 1616 // REX.WRXB 1617 // W - 64-bit operand 1618 // R - MODRM.reg 1619 // X - SIB.index 1620 // B - MODRM.rm/SIB.base 1621 uint8_t rex = force ? 0x40 : 0; 1622 if (w) { 1623 rex |= 0x48; // REX.W000 1624 } 1625 if (r) { 1626 rex |= 0x44; // REX.0R00 1627 } 1628 if (x) { 1629 rex |= 0x42; // REX.00X0 1630 } 1631 if (b) { 1632 rex |= 0x41; // REX.000B 1633 } 1634 if (rex != 0) { 1635 EmitUint8(rex); 1636 } 1637} 1638 1639void X86_64Assembler::EmitOptionalRex32(CpuRegister reg) { 1640 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 1641} 1642 1643void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, CpuRegister src) { 1644 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1645} 1646 1647void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, XmmRegister src) { 1648 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1649} 1650 1651void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, XmmRegister src) { 1652 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1653} 1654 1655void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, CpuRegister src) { 1656 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 1657} 1658 1659void X86_64Assembler::EmitOptionalRex32(const Operand& operand) { 1660 uint8_t rex = operand.rex(); 1661 if (rex != 0) { 1662 EmitUint8(rex); 1663 } 1664} 1665 1666void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, const Operand& operand) { 1667 uint8_t rex = operand.rex(); 1668 if (dst.NeedsRex()) { 1669 rex |= 0x44; // REX.0R00 1670 } 1671 if (rex != 0) { 1672 EmitUint8(rex); 1673 } 1674} 1675 1676void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, const Operand& operand) { 1677 uint8_t rex = operand.rex(); 1678 if (dst.NeedsRex()) { 1679 rex |= 0x44; // REX.0R00 1680 } 1681 if (rex != 0) { 1682 EmitUint8(rex); 1683 } 1684} 1685 1686void X86_64Assembler::EmitRex64(CpuRegister reg) { 1687 EmitOptionalRex(false, true, false, false, reg.NeedsRex()); 1688} 1689 1690void X86_64Assembler::EmitRex64(CpuRegister dst, CpuRegister src) { 1691 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 1692} 1693 1694void X86_64Assembler::EmitRex64(CpuRegister dst, const Operand& operand) { 1695 uint8_t rex = 0x48 | operand.rex(); // REX.W000 1696 if (dst.NeedsRex()) { 1697 rex |= 0x44; // REX.0R00 1698 } 1699 if (rex != 0) { 1700 EmitUint8(rex); 1701 } 1702} 1703 1704void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, CpuRegister src) { 1705 EmitOptionalRex(true, false, dst.NeedsRex(), false, src.NeedsRex()); 1706} 1707 1708void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand) { 1709 uint8_t rex = 0x40 | operand.rex(); // REX.0000 1710 if (dst.NeedsRex()) { 1711 rex |= 0x44; // REX.0R00 1712 } 1713 if (rex != 0) { 1714 EmitUint8(rex); 1715 } 1716} 1717 1718void X86_64Assembler::InitializeFrameDescriptionEntry() { 1719 WriteFDEHeader(&cfi_info_); 1720} 1721 1722void X86_64Assembler::FinalizeFrameDescriptionEntry() { 1723 WriteFDEAddressRange(&cfi_info_, buffer_.Size()); 1724 PadCFI(&cfi_info_); 1725 WriteCFILength(&cfi_info_); 1726} 1727 1728constexpr size_t kFramePointerSize = 8; 1729 1730void X86_64Assembler::BuildFrame(size_t frame_size, ManagedRegister method_reg, 1731 const std::vector<ManagedRegister>& spill_regs, 1732 const ManagedRegisterEntrySpills& entry_spills) { 1733 cfi_cfa_offset_ = kFramePointerSize; // Only return address on stack 1734 cfi_pc_ = buffer_.Size(); // Nothing emitted yet 1735 DCHECK_EQ(cfi_pc_, 0U); 1736 1737 uint32_t reg_offset = 1; 1738 CHECK_ALIGNED(frame_size, kStackAlignment); 1739 int gpr_count = 0; 1740 for (int i = spill_regs.size() - 1; i >= 0; --i) { 1741 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 1742 if (spill.IsCpuRegister()) { 1743 pushq(spill.AsCpuRegister()); 1744 gpr_count++; 1745 1746 // DW_CFA_advance_loc 1747 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 1748 cfi_pc_ = buffer_.Size(); 1749 // DW_CFA_def_cfa_offset 1750 cfi_cfa_offset_ += kFramePointerSize; 1751 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 1752 // DW_CFA_offset reg offset 1753 reg_offset++; 1754 DW_CFA_offset(&cfi_info_, spill.DWARFRegId(), reg_offset); 1755 } 1756 } 1757 // return address then method on stack 1758 int64_t rest_of_frame = static_cast<int64_t>(frame_size) 1759 - (gpr_count * kFramePointerSize) 1760 - kFramePointerSize /*return address*/; 1761 subq(CpuRegister(RSP), Immediate(rest_of_frame)); 1762 // DW_CFA_advance_loc 1763 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 1764 cfi_pc_ = buffer_.Size(); 1765 // DW_CFA_def_cfa_offset 1766 cfi_cfa_offset_ += rest_of_frame; 1767 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 1768 1769 // spill xmms 1770 int64_t offset = rest_of_frame; 1771 for (int i = spill_regs.size() - 1; i >= 0; --i) { 1772 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 1773 if (spill.IsXmmRegister()) { 1774 offset -= sizeof(double); 1775 movsd(Address(CpuRegister(RSP), offset), spill.AsXmmRegister()); 1776 } 1777 } 1778 1779 DCHECK_EQ(4U, sizeof(StackReference<mirror::ArtMethod>)); 1780 1781 movl(Address(CpuRegister(RSP), 0), method_reg.AsX86_64().AsCpuRegister()); 1782 1783 for (size_t i = 0; i < entry_spills.size(); ++i) { 1784 ManagedRegisterSpill spill = entry_spills.at(i); 1785 if (spill.AsX86_64().IsCpuRegister()) { 1786 if (spill.getSize() == 8) { 1787 movq(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), 1788 spill.AsX86_64().AsCpuRegister()); 1789 } else { 1790 CHECK_EQ(spill.getSize(), 4); 1791 movl(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsCpuRegister()); 1792 } 1793 } else { 1794 if (spill.getSize() == 8) { 1795 movsd(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 1796 } else { 1797 CHECK_EQ(spill.getSize(), 4); 1798 movss(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 1799 } 1800 } 1801 } 1802} 1803 1804void X86_64Assembler::RemoveFrame(size_t frame_size, 1805 const std::vector<ManagedRegister>& spill_regs) { 1806 CHECK_ALIGNED(frame_size, kStackAlignment); 1807 int gpr_count = 0; 1808 // unspill xmms 1809 int64_t offset = static_cast<int64_t>(frame_size) - (spill_regs.size() * kFramePointerSize) - 2 * kFramePointerSize; 1810 for (size_t i = 0; i < spill_regs.size(); ++i) { 1811 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 1812 if (spill.IsXmmRegister()) { 1813 offset += sizeof(double); 1814 movsd(spill.AsXmmRegister(), Address(CpuRegister(RSP), offset)); 1815 } else { 1816 gpr_count++; 1817 } 1818 } 1819 addq(CpuRegister(RSP), Immediate(static_cast<int64_t>(frame_size) - (gpr_count * kFramePointerSize) - kFramePointerSize)); 1820 for (size_t i = 0; i < spill_regs.size(); ++i) { 1821 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 1822 if (spill.IsCpuRegister()) { 1823 popq(spill.AsCpuRegister()); 1824 } 1825 } 1826 ret(); 1827} 1828 1829void X86_64Assembler::IncreaseFrameSize(size_t adjust) { 1830 CHECK_ALIGNED(adjust, kStackAlignment); 1831 addq(CpuRegister(RSP), Immediate(-static_cast<int64_t>(adjust))); 1832 // DW_CFA_advance_loc 1833 DW_CFA_advance_loc(&cfi_info_, buffer_.Size() - cfi_pc_); 1834 cfi_pc_ = buffer_.Size(); 1835 // DW_CFA_def_cfa_offset 1836 cfi_cfa_offset_ += adjust; 1837 DW_CFA_def_cfa_offset(&cfi_info_, cfi_cfa_offset_); 1838} 1839 1840void X86_64Assembler::DecreaseFrameSize(size_t adjust) { 1841 CHECK_ALIGNED(adjust, kStackAlignment); 1842 addq(CpuRegister(RSP), Immediate(adjust)); 1843} 1844 1845void X86_64Assembler::Store(FrameOffset offs, ManagedRegister msrc, size_t size) { 1846 X86_64ManagedRegister src = msrc.AsX86_64(); 1847 if (src.IsNoRegister()) { 1848 CHECK_EQ(0u, size); 1849 } else if (src.IsCpuRegister()) { 1850 if (size == 4) { 1851 CHECK_EQ(4u, size); 1852 movl(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 1853 } else { 1854 CHECK_EQ(8u, size); 1855 movq(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 1856 } 1857 } else if (src.IsRegisterPair()) { 1858 CHECK_EQ(0u, size); 1859 movq(Address(CpuRegister(RSP), offs), src.AsRegisterPairLow()); 1860 movq(Address(CpuRegister(RSP), FrameOffset(offs.Int32Value()+4)), 1861 src.AsRegisterPairHigh()); 1862 } else if (src.IsX87Register()) { 1863 if (size == 4) { 1864 fstps(Address(CpuRegister(RSP), offs)); 1865 } else { 1866 fstpl(Address(CpuRegister(RSP), offs)); 1867 } 1868 } else { 1869 CHECK(src.IsXmmRegister()); 1870 if (size == 4) { 1871 movss(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 1872 } else { 1873 movsd(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 1874 } 1875 } 1876} 1877 1878void X86_64Assembler::StoreRef(FrameOffset dest, ManagedRegister msrc) { 1879 X86_64ManagedRegister src = msrc.AsX86_64(); 1880 CHECK(src.IsCpuRegister()); 1881 movl(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 1882} 1883 1884void X86_64Assembler::StoreRawPtr(FrameOffset dest, ManagedRegister msrc) { 1885 X86_64ManagedRegister src = msrc.AsX86_64(); 1886 CHECK(src.IsCpuRegister()); 1887 movq(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 1888} 1889 1890void X86_64Assembler::StoreImmediateToFrame(FrameOffset dest, uint32_t imm, 1891 ManagedRegister) { 1892 movl(Address(CpuRegister(RSP), dest), Immediate(imm)); // TODO(64) movq? 1893} 1894 1895void X86_64Assembler::StoreImmediateToThread64(ThreadOffset<8> dest, uint32_t imm, 1896 ManagedRegister) { 1897 gs()->movl(Address::Absolute(dest, true), Immediate(imm)); // TODO(64) movq? 1898} 1899 1900void X86_64Assembler::StoreStackOffsetToThread64(ThreadOffset<8> thr_offs, 1901 FrameOffset fr_offs, 1902 ManagedRegister mscratch) { 1903 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 1904 CHECK(scratch.IsCpuRegister()); 1905 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), fr_offs)); 1906 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 1907} 1908 1909void X86_64Assembler::StoreStackPointerToThread64(ThreadOffset<8> thr_offs) { 1910 gs()->movq(Address::Absolute(thr_offs, true), CpuRegister(RSP)); 1911} 1912 1913void X86_64Assembler::StoreSpanning(FrameOffset /*dst*/, ManagedRegister /*src*/, 1914 FrameOffset /*in_off*/, ManagedRegister /*scratch*/) { 1915 UNIMPLEMENTED(FATAL); // this case only currently exists for ARM 1916} 1917 1918void X86_64Assembler::Load(ManagedRegister mdest, FrameOffset src, size_t size) { 1919 X86_64ManagedRegister dest = mdest.AsX86_64(); 1920 if (dest.IsNoRegister()) { 1921 CHECK_EQ(0u, size); 1922 } else if (dest.IsCpuRegister()) { 1923 if (size == 4) { 1924 CHECK_EQ(4u, size); 1925 movl(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 1926 } else { 1927 CHECK_EQ(8u, size); 1928 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 1929 } 1930 } else if (dest.IsRegisterPair()) { 1931 CHECK_EQ(0u, size); 1932 movq(dest.AsRegisterPairLow(), Address(CpuRegister(RSP), src)); 1933 movq(dest.AsRegisterPairHigh(), Address(CpuRegister(RSP), FrameOffset(src.Int32Value()+4))); 1934 } else if (dest.IsX87Register()) { 1935 if (size == 4) { 1936 flds(Address(CpuRegister(RSP), src)); 1937 } else { 1938 fldl(Address(CpuRegister(RSP), src)); 1939 } 1940 } else { 1941 CHECK(dest.IsXmmRegister()); 1942 if (size == 4) { 1943 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 1944 } else { 1945 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 1946 } 1947 } 1948} 1949 1950void X86_64Assembler::LoadFromThread64(ManagedRegister mdest, ThreadOffset<8> src, size_t size) { 1951 X86_64ManagedRegister dest = mdest.AsX86_64(); 1952 if (dest.IsNoRegister()) { 1953 CHECK_EQ(0u, size); 1954 } else if (dest.IsCpuRegister()) { 1955 CHECK_EQ(4u, size); 1956 gs()->movl(dest.AsCpuRegister(), Address::Absolute(src, true)); 1957 } else if (dest.IsRegisterPair()) { 1958 CHECK_EQ(8u, size); 1959 gs()->movq(dest.AsRegisterPairLow(), Address::Absolute(src, true)); 1960 } else if (dest.IsX87Register()) { 1961 if (size == 4) { 1962 gs()->flds(Address::Absolute(src, true)); 1963 } else { 1964 gs()->fldl(Address::Absolute(src, true)); 1965 } 1966 } else { 1967 CHECK(dest.IsXmmRegister()); 1968 if (size == 4) { 1969 gs()->movss(dest.AsXmmRegister(), Address::Absolute(src, true)); 1970 } else { 1971 gs()->movsd(dest.AsXmmRegister(), Address::Absolute(src, true)); 1972 } 1973 } 1974} 1975 1976void X86_64Assembler::LoadRef(ManagedRegister mdest, FrameOffset src) { 1977 X86_64ManagedRegister dest = mdest.AsX86_64(); 1978 CHECK(dest.IsCpuRegister()); 1979 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 1980} 1981 1982void X86_64Assembler::LoadRef(ManagedRegister mdest, ManagedRegister base, 1983 MemberOffset offs) { 1984 X86_64ManagedRegister dest = mdest.AsX86_64(); 1985 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 1986 movq(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 1987} 1988 1989void X86_64Assembler::LoadRawPtr(ManagedRegister mdest, ManagedRegister base, 1990 Offset offs) { 1991 X86_64ManagedRegister dest = mdest.AsX86_64(); 1992 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 1993 movq(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 1994} 1995 1996void X86_64Assembler::LoadRawPtrFromThread64(ManagedRegister mdest, ThreadOffset<8> offs) { 1997 X86_64ManagedRegister dest = mdest.AsX86_64(); 1998 CHECK(dest.IsCpuRegister()); 1999 gs()->movq(dest.AsCpuRegister(), Address::Absolute(offs, true)); 2000} 2001 2002void X86_64Assembler::SignExtend(ManagedRegister mreg, size_t size) { 2003 X86_64ManagedRegister reg = mreg.AsX86_64(); 2004 CHECK(size == 1 || size == 2) << size; 2005 CHECK(reg.IsCpuRegister()) << reg; 2006 if (size == 1) { 2007 movsxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2008 } else { 2009 movsxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2010 } 2011} 2012 2013void X86_64Assembler::ZeroExtend(ManagedRegister mreg, size_t size) { 2014 X86_64ManagedRegister reg = mreg.AsX86_64(); 2015 CHECK(size == 1 || size == 2) << size; 2016 CHECK(reg.IsCpuRegister()) << reg; 2017 if (size == 1) { 2018 movzxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2019 } else { 2020 movzxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2021 } 2022} 2023 2024void X86_64Assembler::Move(ManagedRegister mdest, ManagedRegister msrc, size_t size) { 2025 X86_64ManagedRegister dest = mdest.AsX86_64(); 2026 X86_64ManagedRegister src = msrc.AsX86_64(); 2027 if (!dest.Equals(src)) { 2028 if (dest.IsCpuRegister() && src.IsCpuRegister()) { 2029 movq(dest.AsCpuRegister(), src.AsCpuRegister()); 2030 } else if (src.IsX87Register() && dest.IsXmmRegister()) { 2031 // Pass via stack and pop X87 register 2032 subl(CpuRegister(RSP), Immediate(16)); 2033 if (size == 4) { 2034 CHECK_EQ(src.AsX87Register(), ST0); 2035 fstps(Address(CpuRegister(RSP), 0)); 2036 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2037 } else { 2038 CHECK_EQ(src.AsX87Register(), ST0); 2039 fstpl(Address(CpuRegister(RSP), 0)); 2040 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2041 } 2042 addq(CpuRegister(RSP), Immediate(16)); 2043 } else { 2044 // TODO: x87, SSE 2045 UNIMPLEMENTED(FATAL) << ": Move " << dest << ", " << src; 2046 } 2047 } 2048} 2049 2050void X86_64Assembler::CopyRef(FrameOffset dest, FrameOffset src, 2051 ManagedRegister mscratch) { 2052 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2053 CHECK(scratch.IsCpuRegister()); 2054 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2055 movl(Address(CpuRegister(RSP), dest), scratch.AsCpuRegister()); 2056} 2057 2058void X86_64Assembler::CopyRawPtrFromThread64(FrameOffset fr_offs, 2059 ThreadOffset<8> thr_offs, 2060 ManagedRegister mscratch) { 2061 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2062 CHECK(scratch.IsCpuRegister()); 2063 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(thr_offs, true)); 2064 Store(fr_offs, scratch, 8); 2065} 2066 2067void X86_64Assembler::CopyRawPtrToThread64(ThreadOffset<8> thr_offs, 2068 FrameOffset fr_offs, 2069 ManagedRegister mscratch) { 2070 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2071 CHECK(scratch.IsCpuRegister()); 2072 Load(scratch, fr_offs, 8); 2073 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2074} 2075 2076void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src, 2077 ManagedRegister mscratch, 2078 size_t size) { 2079 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2080 if (scratch.IsCpuRegister() && size == 8) { 2081 Load(scratch, src, 4); 2082 Store(dest, scratch, 4); 2083 Load(scratch, FrameOffset(src.Int32Value() + 4), 4); 2084 Store(FrameOffset(dest.Int32Value() + 4), scratch, 4); 2085 } else { 2086 Load(scratch, src, size); 2087 Store(dest, scratch, size); 2088 } 2089} 2090 2091void X86_64Assembler::Copy(FrameOffset /*dst*/, ManagedRegister /*src_base*/, Offset /*src_offset*/, 2092 ManagedRegister /*scratch*/, size_t /*size*/) { 2093 UNIMPLEMENTED(FATAL); 2094} 2095 2096void X86_64Assembler::Copy(ManagedRegister dest_base, Offset dest_offset, FrameOffset src, 2097 ManagedRegister scratch, size_t size) { 2098 CHECK(scratch.IsNoRegister()); 2099 CHECK_EQ(size, 4u); 2100 pushq(Address(CpuRegister(RSP), src)); 2101 popq(Address(dest_base.AsX86_64().AsCpuRegister(), dest_offset)); 2102} 2103 2104void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src_base, Offset src_offset, 2105 ManagedRegister mscratch, size_t size) { 2106 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2107 CHECK_EQ(size, 4u); 2108 movq(scratch, Address(CpuRegister(RSP), src_base)); 2109 movq(scratch, Address(scratch, src_offset)); 2110 movq(Address(CpuRegister(RSP), dest), scratch); 2111} 2112 2113void X86_64Assembler::Copy(ManagedRegister dest, Offset dest_offset, 2114 ManagedRegister src, Offset src_offset, 2115 ManagedRegister scratch, size_t size) { 2116 CHECK_EQ(size, 4u); 2117 CHECK(scratch.IsNoRegister()); 2118 pushq(Address(src.AsX86_64().AsCpuRegister(), src_offset)); 2119 popq(Address(dest.AsX86_64().AsCpuRegister(), dest_offset)); 2120} 2121 2122void X86_64Assembler::Copy(FrameOffset dest, Offset dest_offset, FrameOffset src, Offset src_offset, 2123 ManagedRegister mscratch, size_t size) { 2124 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2125 CHECK_EQ(size, 4u); 2126 CHECK_EQ(dest.Int32Value(), src.Int32Value()); 2127 movq(scratch, Address(CpuRegister(RSP), src)); 2128 pushq(Address(scratch, src_offset)); 2129 popq(Address(scratch, dest_offset)); 2130} 2131 2132void X86_64Assembler::MemoryBarrier(ManagedRegister) { 2133#if ANDROID_SMP != 0 2134 mfence(); 2135#endif 2136} 2137 2138void X86_64Assembler::CreateHandleScopeEntry(ManagedRegister mout_reg, 2139 FrameOffset handle_scope_offset, 2140 ManagedRegister min_reg, bool null_allowed) { 2141 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2142 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2143 if (in_reg.IsNoRegister()) { // TODO(64): && null_allowed 2144 // Use out_reg as indicator of NULL 2145 in_reg = out_reg; 2146 // TODO: movzwl 2147 movl(in_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2148 } 2149 CHECK(in_reg.IsCpuRegister()); 2150 CHECK(out_reg.IsCpuRegister()); 2151 VerifyObject(in_reg, null_allowed); 2152 if (null_allowed) { 2153 Label null_arg; 2154 if (!out_reg.Equals(in_reg)) { 2155 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2156 } 2157 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2158 j(kZero, &null_arg); 2159 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2160 Bind(&null_arg); 2161 } else { 2162 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2163 } 2164} 2165 2166void X86_64Assembler::CreateHandleScopeEntry(FrameOffset out_off, 2167 FrameOffset handle_scope_offset, 2168 ManagedRegister mscratch, 2169 bool null_allowed) { 2170 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2171 CHECK(scratch.IsCpuRegister()); 2172 if (null_allowed) { 2173 Label null_arg; 2174 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2175 testl(scratch.AsCpuRegister(), scratch.AsCpuRegister()); 2176 j(kZero, &null_arg); 2177 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2178 Bind(&null_arg); 2179 } else { 2180 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2181 } 2182 Store(out_off, scratch, 8); 2183} 2184 2185// Given a handle scope entry, load the associated reference. 2186void X86_64Assembler::LoadReferenceFromHandleScope(ManagedRegister mout_reg, 2187 ManagedRegister min_reg) { 2188 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2189 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2190 CHECK(out_reg.IsCpuRegister()); 2191 CHECK(in_reg.IsCpuRegister()); 2192 Label null_arg; 2193 if (!out_reg.Equals(in_reg)) { 2194 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2195 } 2196 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2197 j(kZero, &null_arg); 2198 movq(out_reg.AsCpuRegister(), Address(in_reg.AsCpuRegister(), 0)); 2199 Bind(&null_arg); 2200} 2201 2202void X86_64Assembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) { 2203 // TODO: not validating references 2204} 2205 2206void X86_64Assembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) { 2207 // TODO: not validating references 2208} 2209 2210void X86_64Assembler::Call(ManagedRegister mbase, Offset offset, ManagedRegister) { 2211 X86_64ManagedRegister base = mbase.AsX86_64(); 2212 CHECK(base.IsCpuRegister()); 2213 call(Address(base.AsCpuRegister(), offset.Int32Value())); 2214 // TODO: place reference map on call 2215} 2216 2217void X86_64Assembler::Call(FrameOffset base, Offset offset, ManagedRegister mscratch) { 2218 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2219 movl(scratch, Address(CpuRegister(RSP), base)); 2220 call(Address(scratch, offset)); 2221} 2222 2223void X86_64Assembler::CallFromThread64(ThreadOffset<8> offset, ManagedRegister /*mscratch*/) { 2224 gs()->call(Address::Absolute(offset, true)); 2225} 2226 2227void X86_64Assembler::GetCurrentThread(ManagedRegister tr) { 2228 gs()->movq(tr.AsX86_64().AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2229} 2230 2231void X86_64Assembler::GetCurrentThread(FrameOffset offset, ManagedRegister mscratch) { 2232 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2233 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2234 movq(Address(CpuRegister(RSP), offset), scratch.AsCpuRegister()); 2235} 2236 2237// Slowpath entered when Thread::Current()->_exception is non-null 2238class X86_64ExceptionSlowPath FINAL : public SlowPath { 2239 public: 2240 explicit X86_64ExceptionSlowPath(size_t stack_adjust) : stack_adjust_(stack_adjust) {} 2241 virtual void Emit(Assembler *sp_asm) OVERRIDE; 2242 private: 2243 const size_t stack_adjust_; 2244}; 2245 2246void X86_64Assembler::ExceptionPoll(ManagedRegister /*scratch*/, size_t stack_adjust) { 2247 X86_64ExceptionSlowPath* slow = new X86_64ExceptionSlowPath(stack_adjust); 2248 buffer_.EnqueueSlowPath(slow); 2249 gs()->cmpl(Address::Absolute(Thread::ExceptionOffset<8>(), true), Immediate(0)); 2250 j(kNotEqual, slow->Entry()); 2251} 2252 2253void X86_64ExceptionSlowPath::Emit(Assembler *sasm) { 2254 X86_64Assembler* sp_asm = down_cast<X86_64Assembler*>(sasm); 2255#define __ sp_asm-> 2256 __ Bind(&entry_); 2257 // Note: the return value is dead 2258 if (stack_adjust_ != 0) { // Fix up the frame. 2259 __ DecreaseFrameSize(stack_adjust_); 2260 } 2261 // Pass exception as argument in RDI 2262 __ gs()->movq(CpuRegister(RDI), Address::Absolute(Thread::ExceptionOffset<8>(), true)); 2263 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(8, pDeliverException), true)); 2264 // this call should never return 2265 __ int3(); 2266#undef __ 2267} 2268 2269} // namespace x86_64 2270} // namespace art 2271 2272