1// Copyright 2012 the V8 project authors. All rights reserved.
2// Redistribution and use in source and binary forms, with or without
3// modification, are permitted provided that the following conditions are
4// met:
5//
6//     * Redistributions of source code must retain the above copyright
7//       notice, this list of conditions and the following disclaimer.
8//     * Redistributions in binary form must reproduce the above
9//       copyright notice, this list of conditions and the following
10//       disclaimer in the documentation and/or other materials provided
11//       with the distribution.
12//     * Neither the name of Google Inc. nor the names of its
13//       contributors may be used to endorse or promote products derived
14//       from this software without specific prior written permission.
15//
16// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28#include "v8.h"
29
30#if V8_TARGET_ARCH_X64
31
32#include "bootstrapper.h"
33#include "codegen.h"
34#include "cpu-profiler.h"
35#include "assembler-x64.h"
36#include "macro-assembler-x64.h"
37#include "serialize.h"
38#include "debug.h"
39#include "heap.h"
40
41namespace v8 {
42namespace internal {
43
44MacroAssembler::MacroAssembler(Isolate* arg_isolate, void* buffer, int size)
45    : Assembler(arg_isolate, buffer, size),
46      generating_stub_(false),
47      allow_stub_calls_(true),
48      has_frame_(false),
49      root_array_available_(true) {
50  if (isolate() != NULL) {
51    code_object_ = Handle<Object>(isolate()->heap()->undefined_value(),
52                                  isolate());
53  }
54}
55
56
57static const int kInvalidRootRegisterDelta = -1;
58
59
60intptr_t MacroAssembler::RootRegisterDelta(ExternalReference other) {
61  if (predictable_code_size() &&
62      (other.address() < reinterpret_cast<Address>(isolate()) ||
63       other.address() >= reinterpret_cast<Address>(isolate() + 1))) {
64    return kInvalidRootRegisterDelta;
65  }
66  Address roots_register_value = kRootRegisterBias +
67      reinterpret_cast<Address>(isolate()->heap()->roots_array_start());
68  intptr_t delta = other.address() - roots_register_value;
69  return delta;
70}
71
72
73Operand MacroAssembler::ExternalOperand(ExternalReference target,
74                                        Register scratch) {
75  if (root_array_available_ && !Serializer::enabled()) {
76    intptr_t delta = RootRegisterDelta(target);
77    if (delta != kInvalidRootRegisterDelta && is_int32(delta)) {
78      Serializer::TooLateToEnableNow();
79      return Operand(kRootRegister, static_cast<int32_t>(delta));
80    }
81  }
82  movq(scratch, target);
83  return Operand(scratch, 0);
84}
85
86
87void MacroAssembler::Load(Register destination, ExternalReference source) {
88  if (root_array_available_ && !Serializer::enabled()) {
89    intptr_t delta = RootRegisterDelta(source);
90    if (delta != kInvalidRootRegisterDelta && is_int32(delta)) {
91      Serializer::TooLateToEnableNow();
92      movq(destination, Operand(kRootRegister, static_cast<int32_t>(delta)));
93      return;
94    }
95  }
96  // Safe code.
97  if (destination.is(rax)) {
98    load_rax(source);
99  } else {
100    movq(kScratchRegister, source);
101    movq(destination, Operand(kScratchRegister, 0));
102  }
103}
104
105
106void MacroAssembler::Store(ExternalReference destination, Register source) {
107  if (root_array_available_ && !Serializer::enabled()) {
108    intptr_t delta = RootRegisterDelta(destination);
109    if (delta != kInvalidRootRegisterDelta && is_int32(delta)) {
110      Serializer::TooLateToEnableNow();
111      movq(Operand(kRootRegister, static_cast<int32_t>(delta)), source);
112      return;
113    }
114  }
115  // Safe code.
116  if (source.is(rax)) {
117    store_rax(destination);
118  } else {
119    movq(kScratchRegister, destination);
120    movq(Operand(kScratchRegister, 0), source);
121  }
122}
123
124
125void MacroAssembler::LoadAddress(Register destination,
126                                 ExternalReference source) {
127  if (root_array_available_ && !Serializer::enabled()) {
128    intptr_t delta = RootRegisterDelta(source);
129    if (delta != kInvalidRootRegisterDelta && is_int32(delta)) {
130      Serializer::TooLateToEnableNow();
131      lea(destination, Operand(kRootRegister, static_cast<int32_t>(delta)));
132      return;
133    }
134  }
135  // Safe code.
136  movq(destination, source);
137}
138
139
140int MacroAssembler::LoadAddressSize(ExternalReference source) {
141  if (root_array_available_ && !Serializer::enabled()) {
142    // This calculation depends on the internals of LoadAddress.
143    // It's correctness is ensured by the asserts in the Call
144    // instruction below.
145    intptr_t delta = RootRegisterDelta(source);
146    if (delta != kInvalidRootRegisterDelta && is_int32(delta)) {
147      Serializer::TooLateToEnableNow();
148      // Operand is lea(scratch, Operand(kRootRegister, delta));
149      // Opcodes : REX.W 8D ModRM Disp8/Disp32  - 4 or 7.
150      int size = 4;
151      if (!is_int8(static_cast<int32_t>(delta))) {
152        size += 3;  // Need full four-byte displacement in lea.
153      }
154      return size;
155    }
156  }
157  // Size of movq(destination, src);
158  return Assembler::kMoveAddressIntoScratchRegisterInstructionLength;
159}
160
161
162void MacroAssembler::PushAddress(ExternalReference source) {
163  int64_t address = reinterpret_cast<int64_t>(source.address());
164  if (is_int32(address) && !Serializer::enabled()) {
165    if (emit_debug_code()) {
166      movq(kScratchRegister, BitCast<int64_t>(kZapValue), RelocInfo::NONE64);
167    }
168    push(Immediate(static_cast<int32_t>(address)));
169    return;
170  }
171  LoadAddress(kScratchRegister, source);
172  push(kScratchRegister);
173}
174
175
176void MacroAssembler::LoadRoot(Register destination, Heap::RootListIndex index) {
177  ASSERT(root_array_available_);
178  movq(destination, Operand(kRootRegister,
179                            (index << kPointerSizeLog2) - kRootRegisterBias));
180}
181
182
183void MacroAssembler::LoadRootIndexed(Register destination,
184                                     Register variable_offset,
185                                     int fixed_offset) {
186  ASSERT(root_array_available_);
187  movq(destination,
188       Operand(kRootRegister,
189               variable_offset, times_pointer_size,
190               (fixed_offset << kPointerSizeLog2) - kRootRegisterBias));
191}
192
193
194void MacroAssembler::StoreRoot(Register source, Heap::RootListIndex index) {
195  ASSERT(root_array_available_);
196  movq(Operand(kRootRegister, (index << kPointerSizeLog2) - kRootRegisterBias),
197       source);
198}
199
200
201void MacroAssembler::PushRoot(Heap::RootListIndex index) {
202  ASSERT(root_array_available_);
203  push(Operand(kRootRegister, (index << kPointerSizeLog2) - kRootRegisterBias));
204}
205
206
207void MacroAssembler::CompareRoot(Register with, Heap::RootListIndex index) {
208  ASSERT(root_array_available_);
209  cmpq(with, Operand(kRootRegister,
210                     (index << kPointerSizeLog2) - kRootRegisterBias));
211}
212
213
214void MacroAssembler::CompareRoot(const Operand& with,
215                                 Heap::RootListIndex index) {
216  ASSERT(root_array_available_);
217  ASSERT(!with.AddressUsesRegister(kScratchRegister));
218  LoadRoot(kScratchRegister, index);
219  cmpq(with, kScratchRegister);
220}
221
222
223void MacroAssembler::RememberedSetHelper(Register object,  // For debug tests.
224                                         Register addr,
225                                         Register scratch,
226                                         SaveFPRegsMode save_fp,
227                                         RememberedSetFinalAction and_then) {
228  if (emit_debug_code()) {
229    Label ok;
230    JumpIfNotInNewSpace(object, scratch, &ok, Label::kNear);
231    int3();
232    bind(&ok);
233  }
234  // Load store buffer top.
235  LoadRoot(scratch, Heap::kStoreBufferTopRootIndex);
236  // Store pointer to buffer.
237  movq(Operand(scratch, 0), addr);
238  // Increment buffer top.
239  addq(scratch, Immediate(kPointerSize));
240  // Write back new top of buffer.
241  StoreRoot(scratch, Heap::kStoreBufferTopRootIndex);
242  // Call stub on end of buffer.
243  Label done;
244  // Check for end of buffer.
245  testq(scratch, Immediate(StoreBuffer::kStoreBufferOverflowBit));
246  if (and_then == kReturnAtEnd) {
247    Label buffer_overflowed;
248    j(not_equal, &buffer_overflowed, Label::kNear);
249    ret(0);
250    bind(&buffer_overflowed);
251  } else {
252    ASSERT(and_then == kFallThroughAtEnd);
253    j(equal, &done, Label::kNear);
254  }
255  StoreBufferOverflowStub store_buffer_overflow =
256      StoreBufferOverflowStub(save_fp);
257  CallStub(&store_buffer_overflow);
258  if (and_then == kReturnAtEnd) {
259    ret(0);
260  } else {
261    ASSERT(and_then == kFallThroughAtEnd);
262    bind(&done);
263  }
264}
265
266
267void MacroAssembler::InNewSpace(Register object,
268                                Register scratch,
269                                Condition cc,
270                                Label* branch,
271                                Label::Distance distance) {
272  if (Serializer::enabled()) {
273    // Can't do arithmetic on external references if it might get serialized.
274    // The mask isn't really an address.  We load it as an external reference in
275    // case the size of the new space is different between the snapshot maker
276    // and the running system.
277    if (scratch.is(object)) {
278      movq(kScratchRegister, ExternalReference::new_space_mask(isolate()));
279      and_(scratch, kScratchRegister);
280    } else {
281      movq(scratch, ExternalReference::new_space_mask(isolate()));
282      and_(scratch, object);
283    }
284    movq(kScratchRegister, ExternalReference::new_space_start(isolate()));
285    cmpq(scratch, kScratchRegister);
286    j(cc, branch, distance);
287  } else {
288    ASSERT(is_int32(static_cast<int64_t>(HEAP->NewSpaceMask())));
289    intptr_t new_space_start =
290        reinterpret_cast<intptr_t>(HEAP->NewSpaceStart());
291    movq(kScratchRegister, -new_space_start, RelocInfo::NONE64);
292    if (scratch.is(object)) {
293      addq(scratch, kScratchRegister);
294    } else {
295      lea(scratch, Operand(object, kScratchRegister, times_1, 0));
296    }
297    and_(scratch, Immediate(static_cast<int32_t>(HEAP->NewSpaceMask())));
298    j(cc, branch, distance);
299  }
300}
301
302
303void MacroAssembler::RecordWriteField(
304    Register object,
305    int offset,
306    Register value,
307    Register dst,
308    SaveFPRegsMode save_fp,
309    RememberedSetAction remembered_set_action,
310    SmiCheck smi_check) {
311  // The compiled code assumes that record write doesn't change the
312  // context register, so we check that none of the clobbered
313  // registers are rsi.
314  ASSERT(!value.is(rsi) && !dst.is(rsi));
315
316  // First, check if a write barrier is even needed. The tests below
317  // catch stores of Smis.
318  Label done;
319
320  // Skip barrier if writing a smi.
321  if (smi_check == INLINE_SMI_CHECK) {
322    JumpIfSmi(value, &done);
323  }
324
325  // Although the object register is tagged, the offset is relative to the start
326  // of the object, so so offset must be a multiple of kPointerSize.
327  ASSERT(IsAligned(offset, kPointerSize));
328
329  lea(dst, FieldOperand(object, offset));
330  if (emit_debug_code()) {
331    Label ok;
332    testb(dst, Immediate((1 << kPointerSizeLog2) - 1));
333    j(zero, &ok, Label::kNear);
334    int3();
335    bind(&ok);
336  }
337
338  RecordWrite(
339      object, dst, value, save_fp, remembered_set_action, OMIT_SMI_CHECK);
340
341  bind(&done);
342
343  // Clobber clobbered input registers when running with the debug-code flag
344  // turned on to provoke errors.
345  if (emit_debug_code()) {
346    movq(value, BitCast<int64_t>(kZapValue), RelocInfo::NONE64);
347    movq(dst, BitCast<int64_t>(kZapValue), RelocInfo::NONE64);
348  }
349}
350
351
352void MacroAssembler::RecordWriteArray(Register object,
353                                      Register value,
354                                      Register index,
355                                      SaveFPRegsMode save_fp,
356                                      RememberedSetAction remembered_set_action,
357                                      SmiCheck smi_check) {
358  // First, check if a write barrier is even needed. The tests below
359  // catch stores of Smis.
360  Label done;
361
362  // Skip barrier if writing a smi.
363  if (smi_check == INLINE_SMI_CHECK) {
364    JumpIfSmi(value, &done);
365  }
366
367  // Array access: calculate the destination address. Index is not a smi.
368  Register dst = index;
369  lea(dst, Operand(object, index, times_pointer_size,
370                   FixedArray::kHeaderSize - kHeapObjectTag));
371
372  RecordWrite(
373      object, dst, value, save_fp, remembered_set_action, OMIT_SMI_CHECK);
374
375  bind(&done);
376
377  // Clobber clobbered input registers when running with the debug-code flag
378  // turned on to provoke errors.
379  if (emit_debug_code()) {
380    movq(value, BitCast<int64_t>(kZapValue), RelocInfo::NONE64);
381    movq(index, BitCast<int64_t>(kZapValue), RelocInfo::NONE64);
382  }
383}
384
385
386void MacroAssembler::RecordWrite(Register object,
387                                 Register address,
388                                 Register value,
389                                 SaveFPRegsMode fp_mode,
390                                 RememberedSetAction remembered_set_action,
391                                 SmiCheck smi_check) {
392  // The compiled code assumes that record write doesn't change the
393  // context register, so we check that none of the clobbered
394  // registers are rsi.
395  ASSERT(!value.is(rsi) && !address.is(rsi));
396
397  ASSERT(!object.is(value));
398  ASSERT(!object.is(address));
399  ASSERT(!value.is(address));
400  AssertNotSmi(object);
401
402  if (remembered_set_action == OMIT_REMEMBERED_SET &&
403      !FLAG_incremental_marking) {
404    return;
405  }
406
407  if (emit_debug_code()) {
408    Label ok;
409    cmpq(value, Operand(address, 0));
410    j(equal, &ok, Label::kNear);
411    int3();
412    bind(&ok);
413  }
414
415  // First, check if a write barrier is even needed. The tests below
416  // catch stores of smis and stores into the young generation.
417  Label done;
418
419  if (smi_check == INLINE_SMI_CHECK) {
420    // Skip barrier if writing a smi.
421    JumpIfSmi(value, &done);
422  }
423
424  CheckPageFlag(value,
425                value,  // Used as scratch.
426                MemoryChunk::kPointersToHereAreInterestingMask,
427                zero,
428                &done,
429                Label::kNear);
430
431  CheckPageFlag(object,
432                value,  // Used as scratch.
433                MemoryChunk::kPointersFromHereAreInterestingMask,
434                zero,
435                &done,
436                Label::kNear);
437
438  RecordWriteStub stub(object, value, address, remembered_set_action, fp_mode);
439  CallStub(&stub);
440
441  bind(&done);
442
443  // Clobber clobbered registers when running with the debug-code flag
444  // turned on to provoke errors.
445  if (emit_debug_code()) {
446    movq(address, BitCast<int64_t>(kZapValue), RelocInfo::NONE64);
447    movq(value, BitCast<int64_t>(kZapValue), RelocInfo::NONE64);
448  }
449}
450
451
452void MacroAssembler::Assert(Condition cc, BailoutReason reason) {
453  if (emit_debug_code()) Check(cc, reason);
454}
455
456
457void MacroAssembler::AssertFastElements(Register elements) {
458  if (emit_debug_code()) {
459    Label ok;
460    CompareRoot(FieldOperand(elements, HeapObject::kMapOffset),
461                Heap::kFixedArrayMapRootIndex);
462    j(equal, &ok, Label::kNear);
463    CompareRoot(FieldOperand(elements, HeapObject::kMapOffset),
464                Heap::kFixedDoubleArrayMapRootIndex);
465    j(equal, &ok, Label::kNear);
466    CompareRoot(FieldOperand(elements, HeapObject::kMapOffset),
467                Heap::kFixedCOWArrayMapRootIndex);
468    j(equal, &ok, Label::kNear);
469    Abort(kJSObjectWithFastElementsMapHasSlowElements);
470    bind(&ok);
471  }
472}
473
474
475void MacroAssembler::Check(Condition cc, BailoutReason reason) {
476  Label L;
477  j(cc, &L, Label::kNear);
478  Abort(reason);
479  // Control will not return here.
480  bind(&L);
481}
482
483
484void MacroAssembler::CheckStackAlignment() {
485  int frame_alignment = OS::ActivationFrameAlignment();
486  int frame_alignment_mask = frame_alignment - 1;
487  if (frame_alignment > kPointerSize) {
488    ASSERT(IsPowerOf2(frame_alignment));
489    Label alignment_as_expected;
490    testq(rsp, Immediate(frame_alignment_mask));
491    j(zero, &alignment_as_expected, Label::kNear);
492    // Abort if stack is not aligned.
493    int3();
494    bind(&alignment_as_expected);
495  }
496}
497
498
499void MacroAssembler::NegativeZeroTest(Register result,
500                                      Register op,
501                                      Label* then_label) {
502  Label ok;
503  testl(result, result);
504  j(not_zero, &ok, Label::kNear);
505  testl(op, op);
506  j(sign, then_label);
507  bind(&ok);
508}
509
510
511void MacroAssembler::Abort(BailoutReason reason) {
512  // We want to pass the msg string like a smi to avoid GC
513  // problems, however msg is not guaranteed to be aligned
514  // properly. Instead, we pass an aligned pointer that is
515  // a proper v8 smi, but also pass the alignment difference
516  // from the real pointer as a smi.
517  const char* msg = GetBailoutReason(reason);
518  intptr_t p1 = reinterpret_cast<intptr_t>(msg);
519  intptr_t p0 = (p1 & ~kSmiTagMask) + kSmiTag;
520  // Note: p0 might not be a valid Smi _value_, but it has a valid Smi tag.
521  ASSERT(reinterpret_cast<Object*>(p0)->IsSmi());
522#ifdef DEBUG
523  if (msg != NULL) {
524    RecordComment("Abort message: ");
525    RecordComment(msg);
526  }
527#endif
528  push(rax);
529  movq(kScratchRegister, p0, RelocInfo::NONE64);
530  push(kScratchRegister);
531  movq(kScratchRegister,
532       reinterpret_cast<intptr_t>(Smi::FromInt(static_cast<int>(p1 - p0))),
533       RelocInfo::NONE64);
534  push(kScratchRegister);
535
536  if (!has_frame_) {
537    // We don't actually want to generate a pile of code for this, so just
538    // claim there is a stack frame, without generating one.
539    FrameScope scope(this, StackFrame::NONE);
540    CallRuntime(Runtime::kAbort, 2);
541  } else {
542    CallRuntime(Runtime::kAbort, 2);
543  }
544  // Control will not return here.
545  int3();
546}
547
548
549void MacroAssembler::CallStub(CodeStub* stub, TypeFeedbackId ast_id) {
550  ASSERT(AllowThisStubCall(stub));  // Calls are not allowed in some stubs
551  Call(stub->GetCode(isolate()), RelocInfo::CODE_TARGET, ast_id);
552}
553
554
555void MacroAssembler::TailCallStub(CodeStub* stub) {
556  ASSERT(allow_stub_calls_ ||
557         stub->CompilingCallsToThisStubIsGCSafe(isolate()));
558  Jump(stub->GetCode(isolate()), RelocInfo::CODE_TARGET);
559}
560
561
562void MacroAssembler::StubReturn(int argc) {
563  ASSERT(argc >= 1 && generating_stub());
564  ret((argc - 1) * kPointerSize);
565}
566
567
568bool MacroAssembler::AllowThisStubCall(CodeStub* stub) {
569  if (!has_frame_ && stub->SometimesSetsUpAFrame()) return false;
570  return allow_stub_calls_ || stub->CompilingCallsToThisStubIsGCSafe(isolate());
571}
572
573
574void MacroAssembler::IllegalOperation(int num_arguments) {
575  if (num_arguments > 0) {
576    addq(rsp, Immediate(num_arguments * kPointerSize));
577  }
578  LoadRoot(rax, Heap::kUndefinedValueRootIndex);
579}
580
581
582void MacroAssembler::IndexFromHash(Register hash, Register index) {
583  // The assert checks that the constants for the maximum number of digits
584  // for an array index cached in the hash field and the number of bits
585  // reserved for it does not conflict.
586  ASSERT(TenToThe(String::kMaxCachedArrayIndexLength) <
587         (1 << String::kArrayIndexValueBits));
588  // We want the smi-tagged index in key. Even if we subsequently go to
589  // the slow case, converting the key to a smi is always valid.
590  // key: string key
591  // hash: key's hash field, including its array index value.
592  and_(hash, Immediate(String::kArrayIndexValueMask));
593  shr(hash, Immediate(String::kHashShift));
594  // Here we actually clobber the key which will be used if calling into
595  // runtime later. However as the new key is the numeric value of a string key
596  // there is no difference in using either key.
597  Integer32ToSmi(index, hash);
598}
599
600
601void MacroAssembler::CallRuntime(Runtime::FunctionId id, int num_arguments) {
602  CallRuntime(Runtime::FunctionForId(id), num_arguments);
603}
604
605
606void MacroAssembler::CallRuntimeSaveDoubles(Runtime::FunctionId id) {
607  const Runtime::Function* function = Runtime::FunctionForId(id);
608  Set(rax, function->nargs);
609  LoadAddress(rbx, ExternalReference(function, isolate()));
610  CEntryStub ces(1, kSaveFPRegs);
611  CallStub(&ces);
612}
613
614
615void MacroAssembler::CallRuntime(const Runtime::Function* f,
616                                 int num_arguments) {
617  // If the expected number of arguments of the runtime function is
618  // constant, we check that the actual number of arguments match the
619  // expectation.
620  if (f->nargs >= 0 && f->nargs != num_arguments) {
621    IllegalOperation(num_arguments);
622    return;
623  }
624
625  // TODO(1236192): Most runtime routines don't need the number of
626  // arguments passed in because it is constant. At some point we
627  // should remove this need and make the runtime routine entry code
628  // smarter.
629  Set(rax, num_arguments);
630  LoadAddress(rbx, ExternalReference(f, isolate()));
631  CEntryStub ces(f->result_size);
632  CallStub(&ces);
633}
634
635
636void MacroAssembler::CallExternalReference(const ExternalReference& ext,
637                                           int num_arguments) {
638  Set(rax, num_arguments);
639  LoadAddress(rbx, ext);
640
641  CEntryStub stub(1);
642  CallStub(&stub);
643}
644
645
646void MacroAssembler::TailCallExternalReference(const ExternalReference& ext,
647                                               int num_arguments,
648                                               int result_size) {
649  // ----------- S t a t e -------------
650  //  -- rsp[0]                 : return address
651  //  -- rsp[8]                 : argument num_arguments - 1
652  //  ...
653  //  -- rsp[8 * num_arguments] : argument 0 (receiver)
654  // -----------------------------------
655
656  // TODO(1236192): Most runtime routines don't need the number of
657  // arguments passed in because it is constant. At some point we
658  // should remove this need and make the runtime routine entry code
659  // smarter.
660  Set(rax, num_arguments);
661  JumpToExternalReference(ext, result_size);
662}
663
664
665void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid,
666                                     int num_arguments,
667                                     int result_size) {
668  TailCallExternalReference(ExternalReference(fid, isolate()),
669                            num_arguments,
670                            result_size);
671}
672
673
674static int Offset(ExternalReference ref0, ExternalReference ref1) {
675  int64_t offset = (ref0.address() - ref1.address());
676  // Check that fits into int.
677  ASSERT(static_cast<int>(offset) == offset);
678  return static_cast<int>(offset);
679}
680
681
682void MacroAssembler::PrepareCallApiFunction(int arg_stack_space,
683                                            bool returns_handle) {
684#if defined(_WIN64) && !defined(__MINGW64__)
685  if (!returns_handle) {
686    EnterApiExitFrame(arg_stack_space);
687    return;
688  }
689  // We need to prepare a slot for result handle on stack and put
690  // a pointer to it into 1st arg register.
691  EnterApiExitFrame(arg_stack_space + 1);
692
693  // rcx must be used to pass the pointer to the return value slot.
694  lea(rcx, StackSpaceOperand(arg_stack_space));
695#else
696  EnterApiExitFrame(arg_stack_space);
697#endif
698}
699
700
701void MacroAssembler::CallApiFunctionAndReturn(Address function_address,
702                                              Address thunk_address,
703                                              Register thunk_last_arg,
704                                              int stack_space,
705                                              bool returns_handle,
706                                              int return_value_offset) {
707  Label prologue;
708  Label promote_scheduled_exception;
709  Label delete_allocated_handles;
710  Label leave_exit_frame;
711  Label write_back;
712
713  Factory* factory = isolate()->factory();
714  ExternalReference next_address =
715      ExternalReference::handle_scope_next_address(isolate());
716  const int kNextOffset = 0;
717  const int kLimitOffset = Offset(
718      ExternalReference::handle_scope_limit_address(isolate()),
719      next_address);
720  const int kLevelOffset = Offset(
721      ExternalReference::handle_scope_level_address(isolate()),
722      next_address);
723  ExternalReference scheduled_exception_address =
724      ExternalReference::scheduled_exception_address(isolate());
725
726  // Allocate HandleScope in callee-save registers.
727  Register prev_next_address_reg = r14;
728  Register prev_limit_reg = rbx;
729  Register base_reg = r15;
730  movq(base_reg, next_address);
731  movq(prev_next_address_reg, Operand(base_reg, kNextOffset));
732  movq(prev_limit_reg, Operand(base_reg, kLimitOffset));
733  addl(Operand(base_reg, kLevelOffset), Immediate(1));
734
735  if (FLAG_log_timer_events) {
736    FrameScope frame(this, StackFrame::MANUAL);
737    PushSafepointRegisters();
738    PrepareCallCFunction(1);
739    LoadAddress(arg_reg_1, ExternalReference::isolate_address(isolate()));
740    CallCFunction(ExternalReference::log_enter_external_function(isolate()), 1);
741    PopSafepointRegisters();
742  }
743
744
745  Label profiler_disabled;
746  Label end_profiler_check;
747  bool* is_profiling_flag =
748      isolate()->cpu_profiler()->is_profiling_address();
749  STATIC_ASSERT(sizeof(*is_profiling_flag) == 1);
750  movq(rax, is_profiling_flag, RelocInfo::EXTERNAL_REFERENCE);
751  cmpb(Operand(rax, 0), Immediate(0));
752  j(zero, &profiler_disabled);
753
754  // Third parameter is the address of the actual getter function.
755  movq(thunk_last_arg, function_address, RelocInfo::EXTERNAL_REFERENCE);
756  movq(rax, thunk_address, RelocInfo::EXTERNAL_REFERENCE);
757  jmp(&end_profiler_check);
758
759  bind(&profiler_disabled);
760  // Call the api function!
761  movq(rax, reinterpret_cast<int64_t>(function_address),
762       RelocInfo::EXTERNAL_REFERENCE);
763
764  bind(&end_profiler_check);
765
766  // Call the api function!
767  call(rax);
768
769  if (FLAG_log_timer_events) {
770    FrameScope frame(this, StackFrame::MANUAL);
771    PushSafepointRegisters();
772    PrepareCallCFunction(1);
773    LoadAddress(arg_reg_1, ExternalReference::isolate_address(isolate()));
774    CallCFunction(ExternalReference::log_leave_external_function(isolate()), 1);
775    PopSafepointRegisters();
776  }
777
778  // Can skip the result check for new-style callbacks
779  // TODO(dcarney): may need to pass this information down
780  // as some function_addresses might not have been registered
781  if (returns_handle) {
782    Label empty_result;
783#if defined(_WIN64) && !defined(__MINGW64__)
784    // rax keeps a pointer to v8::Handle, unpack it.
785    movq(rax, Operand(rax, 0));
786#endif
787    // Check if the result handle holds 0.
788    testq(rax, rax);
789    j(zero, &empty_result);
790    // It was non-zero.  Dereference to get the result value.
791    movq(rax, Operand(rax, 0));
792    jmp(&prologue);
793    bind(&empty_result);
794  }
795  // Load the value from ReturnValue
796  movq(rax, Operand(rbp, return_value_offset * kPointerSize));
797  bind(&prologue);
798
799  // No more valid handles (the result handle was the last one). Restore
800  // previous handle scope.
801  subl(Operand(base_reg, kLevelOffset), Immediate(1));
802  movq(Operand(base_reg, kNextOffset), prev_next_address_reg);
803  cmpq(prev_limit_reg, Operand(base_reg, kLimitOffset));
804  j(not_equal, &delete_allocated_handles);
805  bind(&leave_exit_frame);
806
807  // Check if the function scheduled an exception.
808  movq(rsi, scheduled_exception_address);
809  Cmp(Operand(rsi, 0), factory->the_hole_value());
810  j(not_equal, &promote_scheduled_exception);
811
812#if ENABLE_EXTRA_CHECKS
813  // Check if the function returned a valid JavaScript value.
814  Label ok;
815  Register return_value = rax;
816  Register map = rcx;
817
818  JumpIfSmi(return_value, &ok, Label::kNear);
819  movq(map, FieldOperand(return_value, HeapObject::kMapOffset));
820
821  CmpInstanceType(map, FIRST_NONSTRING_TYPE);
822  j(below, &ok, Label::kNear);
823
824  CmpInstanceType(map, FIRST_SPEC_OBJECT_TYPE);
825  j(above_equal, &ok, Label::kNear);
826
827  CompareRoot(map, Heap::kHeapNumberMapRootIndex);
828  j(equal, &ok, Label::kNear);
829
830  CompareRoot(return_value, Heap::kUndefinedValueRootIndex);
831  j(equal, &ok, Label::kNear);
832
833  CompareRoot(return_value, Heap::kTrueValueRootIndex);
834  j(equal, &ok, Label::kNear);
835
836  CompareRoot(return_value, Heap::kFalseValueRootIndex);
837  j(equal, &ok, Label::kNear);
838
839  CompareRoot(return_value, Heap::kNullValueRootIndex);
840  j(equal, &ok, Label::kNear);
841
842  Abort(kAPICallReturnedInvalidObject);
843
844  bind(&ok);
845#endif
846
847  LeaveApiExitFrame();
848  ret(stack_space * kPointerSize);
849
850  bind(&promote_scheduled_exception);
851  TailCallRuntime(Runtime::kPromoteScheduledException, 0, 1);
852
853  // HandleScope limit has changed. Delete allocated extensions.
854  bind(&delete_allocated_handles);
855  movq(Operand(base_reg, kLimitOffset), prev_limit_reg);
856  movq(prev_limit_reg, rax);
857  LoadAddress(arg_reg_1, ExternalReference::isolate_address(isolate()));
858  LoadAddress(rax,
859              ExternalReference::delete_handle_scope_extensions(isolate()));
860  call(rax);
861  movq(rax, prev_limit_reg);
862  jmp(&leave_exit_frame);
863}
864
865
866void MacroAssembler::JumpToExternalReference(const ExternalReference& ext,
867                                             int result_size) {
868  // Set the entry point and jump to the C entry runtime stub.
869  LoadAddress(rbx, ext);
870  CEntryStub ces(result_size);
871  jmp(ces.GetCode(isolate()), RelocInfo::CODE_TARGET);
872}
873
874
875void MacroAssembler::InvokeBuiltin(Builtins::JavaScript id,
876                                   InvokeFlag flag,
877                                   const CallWrapper& call_wrapper) {
878  // You can't call a builtin without a valid frame.
879  ASSERT(flag == JUMP_FUNCTION || has_frame());
880
881  // Rely on the assertion to check that the number of provided
882  // arguments match the expected number of arguments. Fake a
883  // parameter count to avoid emitting code to do the check.
884  ParameterCount expected(0);
885  GetBuiltinEntry(rdx, id);
886  InvokeCode(rdx, expected, expected, flag, call_wrapper, CALL_AS_METHOD);
887}
888
889
890void MacroAssembler::GetBuiltinFunction(Register target,
891                                        Builtins::JavaScript id) {
892  // Load the builtins object into target register.
893  movq(target, Operand(rsi, Context::SlotOffset(Context::GLOBAL_OBJECT_INDEX)));
894  movq(target, FieldOperand(target, GlobalObject::kBuiltinsOffset));
895  movq(target, FieldOperand(target,
896                            JSBuiltinsObject::OffsetOfFunctionWithId(id)));
897}
898
899
900void MacroAssembler::GetBuiltinEntry(Register target, Builtins::JavaScript id) {
901  ASSERT(!target.is(rdi));
902  // Load the JavaScript builtin function from the builtins object.
903  GetBuiltinFunction(rdi, id);
904  movq(target, FieldOperand(rdi, JSFunction::kCodeEntryOffset));
905}
906
907
908#define REG(Name) { kRegister_ ## Name ## _Code }
909
910static const Register saved_regs[] = {
911  REG(rax), REG(rcx), REG(rdx), REG(rbx), REG(rbp), REG(rsi), REG(rdi), REG(r8),
912  REG(r9), REG(r10), REG(r11)
913};
914
915#undef REG
916
917static const int kNumberOfSavedRegs = sizeof(saved_regs) / sizeof(Register);
918
919
920void MacroAssembler::PushCallerSaved(SaveFPRegsMode fp_mode,
921                                     Register exclusion1,
922                                     Register exclusion2,
923                                     Register exclusion3) {
924  // We don't allow a GC during a store buffer overflow so there is no need to
925  // store the registers in any particular way, but we do have to store and
926  // restore them.
927  for (int i = 0; i < kNumberOfSavedRegs; i++) {
928    Register reg = saved_regs[i];
929    if (!reg.is(exclusion1) && !reg.is(exclusion2) && !reg.is(exclusion3)) {
930      push(reg);
931    }
932  }
933  // R12 to r15 are callee save on all platforms.
934  if (fp_mode == kSaveFPRegs) {
935    subq(rsp, Immediate(kDoubleSize * XMMRegister::kMaxNumRegisters));
936    for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) {
937      XMMRegister reg = XMMRegister::from_code(i);
938      movsd(Operand(rsp, i * kDoubleSize), reg);
939    }
940  }
941}
942
943
944void MacroAssembler::PopCallerSaved(SaveFPRegsMode fp_mode,
945                                    Register exclusion1,
946                                    Register exclusion2,
947                                    Register exclusion3) {
948  if (fp_mode == kSaveFPRegs) {
949    for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) {
950      XMMRegister reg = XMMRegister::from_code(i);
951      movsd(reg, Operand(rsp, i * kDoubleSize));
952    }
953    addq(rsp, Immediate(kDoubleSize * XMMRegister::kMaxNumRegisters));
954  }
955  for (int i = kNumberOfSavedRegs - 1; i >= 0; i--) {
956    Register reg = saved_regs[i];
957    if (!reg.is(exclusion1) && !reg.is(exclusion2) && !reg.is(exclusion3)) {
958      pop(reg);
959    }
960  }
961}
962
963
964void MacroAssembler::Set(Register dst, int64_t x) {
965  if (x == 0) {
966    xorl(dst, dst);
967  } else if (is_uint32(x)) {
968    movl(dst, Immediate(static_cast<uint32_t>(x)));
969  } else if (is_int32(x)) {
970    movq(dst, Immediate(static_cast<int32_t>(x)));
971  } else {
972    movq(dst, x, RelocInfo::NONE64);
973  }
974}
975
976
977void MacroAssembler::Set(const Operand& dst, int64_t x) {
978  if (is_int32(x)) {
979    movq(dst, Immediate(static_cast<int32_t>(x)));
980  } else {
981    Set(kScratchRegister, x);
982    movq(dst, kScratchRegister);
983  }
984}
985
986
987bool MacroAssembler::IsUnsafeInt(const int x) {
988  static const int kMaxBits = 17;
989  return !is_intn(x, kMaxBits);
990}
991
992
993void MacroAssembler::SafeMove(Register dst, Smi* src) {
994  ASSERT(!dst.is(kScratchRegister));
995  ASSERT(kSmiValueSize == 32);  // JIT cookie can be converted to Smi.
996  if (IsUnsafeInt(src->value()) && jit_cookie() != 0) {
997    Move(dst, Smi::FromInt(src->value() ^ jit_cookie()));
998    Move(kScratchRegister, Smi::FromInt(jit_cookie()));
999    xor_(dst, kScratchRegister);
1000  } else {
1001    Move(dst, src);
1002  }
1003}
1004
1005
1006void MacroAssembler::SafePush(Smi* src) {
1007  ASSERT(kSmiValueSize == 32);  // JIT cookie can be converted to Smi.
1008  if (IsUnsafeInt(src->value()) && jit_cookie() != 0) {
1009    Push(Smi::FromInt(src->value() ^ jit_cookie()));
1010    Move(kScratchRegister, Smi::FromInt(jit_cookie()));
1011    xor_(Operand(rsp, 0), kScratchRegister);
1012  } else {
1013    Push(src);
1014  }
1015}
1016
1017
1018// ----------------------------------------------------------------------------
1019// Smi tagging, untagging and tag detection.
1020
1021Register MacroAssembler::GetSmiConstant(Smi* source) {
1022  int value = source->value();
1023  if (value == 0) {
1024    xorl(kScratchRegister, kScratchRegister);
1025    return kScratchRegister;
1026  }
1027  if (value == 1) {
1028    return kSmiConstantRegister;
1029  }
1030  LoadSmiConstant(kScratchRegister, source);
1031  return kScratchRegister;
1032}
1033
1034
1035void MacroAssembler::LoadSmiConstant(Register dst, Smi* source) {
1036  if (emit_debug_code()) {
1037    movq(dst,
1038         reinterpret_cast<uint64_t>(Smi::FromInt(kSmiConstantRegisterValue)),
1039         RelocInfo::NONE64);
1040    cmpq(dst, kSmiConstantRegister);
1041    if (allow_stub_calls()) {
1042      Assert(equal, kUninitializedKSmiConstantRegister);
1043    } else {
1044      Label ok;
1045      j(equal, &ok, Label::kNear);
1046      int3();
1047      bind(&ok);
1048    }
1049  }
1050  int value = source->value();
1051  if (value == 0) {
1052    xorl(dst, dst);
1053    return;
1054  }
1055  bool negative = value < 0;
1056  unsigned int uvalue = negative ? -value : value;
1057
1058  switch (uvalue) {
1059    case 9:
1060      lea(dst, Operand(kSmiConstantRegister, kSmiConstantRegister, times_8, 0));
1061      break;
1062    case 8:
1063      xorl(dst, dst);
1064      lea(dst, Operand(dst, kSmiConstantRegister, times_8, 0));
1065      break;
1066    case 4:
1067      xorl(dst, dst);
1068      lea(dst, Operand(dst, kSmiConstantRegister, times_4, 0));
1069      break;
1070    case 5:
1071      lea(dst, Operand(kSmiConstantRegister, kSmiConstantRegister, times_4, 0));
1072      break;
1073    case 3:
1074      lea(dst, Operand(kSmiConstantRegister, kSmiConstantRegister, times_2, 0));
1075      break;
1076    case 2:
1077      lea(dst, Operand(kSmiConstantRegister, kSmiConstantRegister, times_1, 0));
1078      break;
1079    case 1:
1080      movq(dst, kSmiConstantRegister);
1081      break;
1082    case 0:
1083      UNREACHABLE();
1084      return;
1085    default:
1086      movq(dst, reinterpret_cast<uint64_t>(source), RelocInfo::NONE64);
1087      return;
1088  }
1089  if (negative) {
1090    neg(dst);
1091  }
1092}
1093
1094
1095void MacroAssembler::Integer32ToSmi(Register dst, Register src) {
1096  STATIC_ASSERT(kSmiTag == 0);
1097  if (!dst.is(src)) {
1098    movl(dst, src);
1099  }
1100  shl(dst, Immediate(kSmiShift));
1101}
1102
1103
1104void MacroAssembler::Integer32ToSmiField(const Operand& dst, Register src) {
1105  if (emit_debug_code()) {
1106    testb(dst, Immediate(0x01));
1107    Label ok;
1108    j(zero, &ok, Label::kNear);
1109    if (allow_stub_calls()) {
1110      Abort(kInteger32ToSmiFieldWritingToNonSmiLocation);
1111    } else {
1112      int3();
1113    }
1114    bind(&ok);
1115  }
1116  ASSERT(kSmiShift % kBitsPerByte == 0);
1117  movl(Operand(dst, kSmiShift / kBitsPerByte), src);
1118}
1119
1120
1121void MacroAssembler::Integer64PlusConstantToSmi(Register dst,
1122                                                Register src,
1123                                                int constant) {
1124  if (dst.is(src)) {
1125    addl(dst, Immediate(constant));
1126  } else {
1127    leal(dst, Operand(src, constant));
1128  }
1129  shl(dst, Immediate(kSmiShift));
1130}
1131
1132
1133void MacroAssembler::SmiToInteger32(Register dst, Register src) {
1134  STATIC_ASSERT(kSmiTag == 0);
1135  if (!dst.is(src)) {
1136    movq(dst, src);
1137  }
1138  shr(dst, Immediate(kSmiShift));
1139}
1140
1141
1142void MacroAssembler::SmiToInteger32(Register dst, const Operand& src) {
1143  movl(dst, Operand(src, kSmiShift / kBitsPerByte));
1144}
1145
1146
1147void MacroAssembler::SmiToInteger64(Register dst, Register src) {
1148  STATIC_ASSERT(kSmiTag == 0);
1149  if (!dst.is(src)) {
1150    movq(dst, src);
1151  }
1152  sar(dst, Immediate(kSmiShift));
1153}
1154
1155
1156void MacroAssembler::SmiToInteger64(Register dst, const Operand& src) {
1157  movsxlq(dst, Operand(src, kSmiShift / kBitsPerByte));
1158}
1159
1160
1161void MacroAssembler::SmiTest(Register src) {
1162  AssertSmi(src);
1163  testq(src, src);
1164}
1165
1166
1167void MacroAssembler::SmiCompare(Register smi1, Register smi2) {
1168  AssertSmi(smi1);
1169  AssertSmi(smi2);
1170  cmpq(smi1, smi2);
1171}
1172
1173
1174void MacroAssembler::SmiCompare(Register dst, Smi* src) {
1175  AssertSmi(dst);
1176  Cmp(dst, src);
1177}
1178
1179
1180void MacroAssembler::Cmp(Register dst, Smi* src) {
1181  ASSERT(!dst.is(kScratchRegister));
1182  if (src->value() == 0) {
1183    testq(dst, dst);
1184  } else {
1185    Register constant_reg = GetSmiConstant(src);
1186    cmpq(dst, constant_reg);
1187  }
1188}
1189
1190
1191void MacroAssembler::SmiCompare(Register dst, const Operand& src) {
1192  AssertSmi(dst);
1193  AssertSmi(src);
1194  cmpq(dst, src);
1195}
1196
1197
1198void MacroAssembler::SmiCompare(const Operand& dst, Register src) {
1199  AssertSmi(dst);
1200  AssertSmi(src);
1201  cmpq(dst, src);
1202}
1203
1204
1205void MacroAssembler::SmiCompare(const Operand& dst, Smi* src) {
1206  AssertSmi(dst);
1207  cmpl(Operand(dst, kSmiShift / kBitsPerByte), Immediate(src->value()));
1208}
1209
1210
1211void MacroAssembler::Cmp(const Operand& dst, Smi* src) {
1212  // The Operand cannot use the smi register.
1213  Register smi_reg = GetSmiConstant(src);
1214  ASSERT(!dst.AddressUsesRegister(smi_reg));
1215  cmpq(dst, smi_reg);
1216}
1217
1218
1219void MacroAssembler::SmiCompareInteger32(const Operand& dst, Register src) {
1220  cmpl(Operand(dst, kSmiShift / kBitsPerByte), src);
1221}
1222
1223
1224void MacroAssembler::PositiveSmiTimesPowerOfTwoToInteger64(Register dst,
1225                                                           Register src,
1226                                                           int power) {
1227  ASSERT(power >= 0);
1228  ASSERT(power < 64);
1229  if (power == 0) {
1230    SmiToInteger64(dst, src);
1231    return;
1232  }
1233  if (!dst.is(src)) {
1234    movq(dst, src);
1235  }
1236  if (power < kSmiShift) {
1237    sar(dst, Immediate(kSmiShift - power));
1238  } else if (power > kSmiShift) {
1239    shl(dst, Immediate(power - kSmiShift));
1240  }
1241}
1242
1243
1244void MacroAssembler::PositiveSmiDivPowerOfTwoToInteger32(Register dst,
1245                                                         Register src,
1246                                                         int power) {
1247  ASSERT((0 <= power) && (power < 32));
1248  if (dst.is(src)) {
1249    shr(dst, Immediate(power + kSmiShift));
1250  } else {
1251    UNIMPLEMENTED();  // Not used.
1252  }
1253}
1254
1255
1256void MacroAssembler::SmiOrIfSmis(Register dst, Register src1, Register src2,
1257                                 Label* on_not_smis,
1258                                 Label::Distance near_jump) {
1259  if (dst.is(src1) || dst.is(src2)) {
1260    ASSERT(!src1.is(kScratchRegister));
1261    ASSERT(!src2.is(kScratchRegister));
1262    movq(kScratchRegister, src1);
1263    or_(kScratchRegister, src2);
1264    JumpIfNotSmi(kScratchRegister, on_not_smis, near_jump);
1265    movq(dst, kScratchRegister);
1266  } else {
1267    movq(dst, src1);
1268    or_(dst, src2);
1269    JumpIfNotSmi(dst, on_not_smis, near_jump);
1270  }
1271}
1272
1273
1274Condition MacroAssembler::CheckSmi(Register src) {
1275  STATIC_ASSERT(kSmiTag == 0);
1276  testb(src, Immediate(kSmiTagMask));
1277  return zero;
1278}
1279
1280
1281Condition MacroAssembler::CheckSmi(const Operand& src) {
1282  STATIC_ASSERT(kSmiTag == 0);
1283  testb(src, Immediate(kSmiTagMask));
1284  return zero;
1285}
1286
1287
1288Condition MacroAssembler::CheckNonNegativeSmi(Register src) {
1289  STATIC_ASSERT(kSmiTag == 0);
1290  // Test that both bits of the mask 0x8000000000000001 are zero.
1291  movq(kScratchRegister, src);
1292  rol(kScratchRegister, Immediate(1));
1293  testb(kScratchRegister, Immediate(3));
1294  return zero;
1295}
1296
1297
1298Condition MacroAssembler::CheckBothSmi(Register first, Register second) {
1299  if (first.is(second)) {
1300    return CheckSmi(first);
1301  }
1302  STATIC_ASSERT(kSmiTag == 0 && kHeapObjectTag == 1 && kHeapObjectTagMask == 3);
1303  leal(kScratchRegister, Operand(first, second, times_1, 0));
1304  testb(kScratchRegister, Immediate(0x03));
1305  return zero;
1306}
1307
1308
1309Condition MacroAssembler::CheckBothNonNegativeSmi(Register first,
1310                                                  Register second) {
1311  if (first.is(second)) {
1312    return CheckNonNegativeSmi(first);
1313  }
1314  movq(kScratchRegister, first);
1315  or_(kScratchRegister, second);
1316  rol(kScratchRegister, Immediate(1));
1317  testl(kScratchRegister, Immediate(3));
1318  return zero;
1319}
1320
1321
1322Condition MacroAssembler::CheckEitherSmi(Register first,
1323                                         Register second,
1324                                         Register scratch) {
1325  if (first.is(second)) {
1326    return CheckSmi(first);
1327  }
1328  if (scratch.is(second)) {
1329    andl(scratch, first);
1330  } else {
1331    if (!scratch.is(first)) {
1332      movl(scratch, first);
1333    }
1334    andl(scratch, second);
1335  }
1336  testb(scratch, Immediate(kSmiTagMask));
1337  return zero;
1338}
1339
1340
1341Condition MacroAssembler::CheckIsMinSmi(Register src) {
1342  ASSERT(!src.is(kScratchRegister));
1343  // If we overflow by subtracting one, it's the minimal smi value.
1344  cmpq(src, kSmiConstantRegister);
1345  return overflow;
1346}
1347
1348
1349Condition MacroAssembler::CheckInteger32ValidSmiValue(Register src) {
1350  // A 32-bit integer value can always be converted to a smi.
1351  return always;
1352}
1353
1354
1355Condition MacroAssembler::CheckUInteger32ValidSmiValue(Register src) {
1356  // An unsigned 32-bit integer value is valid as long as the high bit
1357  // is not set.
1358  testl(src, src);
1359  return positive;
1360}
1361
1362
1363void MacroAssembler::CheckSmiToIndicator(Register dst, Register src) {
1364  if (dst.is(src)) {
1365    andl(dst, Immediate(kSmiTagMask));
1366  } else {
1367    movl(dst, Immediate(kSmiTagMask));
1368    andl(dst, src);
1369  }
1370}
1371
1372
1373void MacroAssembler::CheckSmiToIndicator(Register dst, const Operand& src) {
1374  if (!(src.AddressUsesRegister(dst))) {
1375    movl(dst, Immediate(kSmiTagMask));
1376    andl(dst, src);
1377  } else {
1378    movl(dst, src);
1379    andl(dst, Immediate(kSmiTagMask));
1380  }
1381}
1382
1383
1384void MacroAssembler::JumpIfNotValidSmiValue(Register src,
1385                                            Label* on_invalid,
1386                                            Label::Distance near_jump) {
1387  Condition is_valid = CheckInteger32ValidSmiValue(src);
1388  j(NegateCondition(is_valid), on_invalid, near_jump);
1389}
1390
1391
1392void MacroAssembler::JumpIfUIntNotValidSmiValue(Register src,
1393                                                Label* on_invalid,
1394                                                Label::Distance near_jump) {
1395  Condition is_valid = CheckUInteger32ValidSmiValue(src);
1396  j(NegateCondition(is_valid), on_invalid, near_jump);
1397}
1398
1399
1400void MacroAssembler::JumpIfSmi(Register src,
1401                               Label* on_smi,
1402                               Label::Distance near_jump) {
1403  Condition smi = CheckSmi(src);
1404  j(smi, on_smi, near_jump);
1405}
1406
1407
1408void MacroAssembler::JumpIfNotSmi(Register src,
1409                                  Label* on_not_smi,
1410                                  Label::Distance near_jump) {
1411  Condition smi = CheckSmi(src);
1412  j(NegateCondition(smi), on_not_smi, near_jump);
1413}
1414
1415
1416void MacroAssembler::JumpUnlessNonNegativeSmi(
1417    Register src, Label* on_not_smi_or_negative,
1418    Label::Distance near_jump) {
1419  Condition non_negative_smi = CheckNonNegativeSmi(src);
1420  j(NegateCondition(non_negative_smi), on_not_smi_or_negative, near_jump);
1421}
1422
1423
1424void MacroAssembler::JumpIfSmiEqualsConstant(Register src,
1425                                             Smi* constant,
1426                                             Label* on_equals,
1427                                             Label::Distance near_jump) {
1428  SmiCompare(src, constant);
1429  j(equal, on_equals, near_jump);
1430}
1431
1432
1433void MacroAssembler::JumpIfNotBothSmi(Register src1,
1434                                      Register src2,
1435                                      Label* on_not_both_smi,
1436                                      Label::Distance near_jump) {
1437  Condition both_smi = CheckBothSmi(src1, src2);
1438  j(NegateCondition(both_smi), on_not_both_smi, near_jump);
1439}
1440
1441
1442void MacroAssembler::JumpUnlessBothNonNegativeSmi(Register src1,
1443                                                  Register src2,
1444                                                  Label* on_not_both_smi,
1445                                                  Label::Distance near_jump) {
1446  Condition both_smi = CheckBothNonNegativeSmi(src1, src2);
1447  j(NegateCondition(both_smi), on_not_both_smi, near_jump);
1448}
1449
1450
1451void MacroAssembler::SmiTryAddConstant(Register dst,
1452                                       Register src,
1453                                       Smi* constant,
1454                                       Label* on_not_smi_result,
1455                                       Label::Distance near_jump) {
1456  // Does not assume that src is a smi.
1457  ASSERT_EQ(static_cast<int>(1), static_cast<int>(kSmiTagMask));
1458  STATIC_ASSERT(kSmiTag == 0);
1459  ASSERT(!dst.is(kScratchRegister));
1460  ASSERT(!src.is(kScratchRegister));
1461
1462  JumpIfNotSmi(src, on_not_smi_result, near_jump);
1463  Register tmp = (dst.is(src) ? kScratchRegister : dst);
1464  LoadSmiConstant(tmp, constant);
1465  addq(tmp, src);
1466  j(overflow, on_not_smi_result, near_jump);
1467  if (dst.is(src)) {
1468    movq(dst, tmp);
1469  }
1470}
1471
1472
1473void MacroAssembler::SmiAddConstant(Register dst, Register src, Smi* constant) {
1474  if (constant->value() == 0) {
1475    if (!dst.is(src)) {
1476      movq(dst, src);
1477    }
1478    return;
1479  } else if (dst.is(src)) {
1480    ASSERT(!dst.is(kScratchRegister));
1481    switch (constant->value()) {
1482      case 1:
1483        addq(dst, kSmiConstantRegister);
1484        return;
1485      case 2:
1486        lea(dst, Operand(src, kSmiConstantRegister, times_2, 0));
1487        return;
1488      case 4:
1489        lea(dst, Operand(src, kSmiConstantRegister, times_4, 0));
1490        return;
1491      case 8:
1492        lea(dst, Operand(src, kSmiConstantRegister, times_8, 0));
1493        return;
1494      default:
1495        Register constant_reg = GetSmiConstant(constant);
1496        addq(dst, constant_reg);
1497        return;
1498    }
1499  } else {
1500    switch (constant->value()) {
1501      case 1:
1502        lea(dst, Operand(src, kSmiConstantRegister, times_1, 0));
1503        return;
1504      case 2:
1505        lea(dst, Operand(src, kSmiConstantRegister, times_2, 0));
1506        return;
1507      case 4:
1508        lea(dst, Operand(src, kSmiConstantRegister, times_4, 0));
1509        return;
1510      case 8:
1511        lea(dst, Operand(src, kSmiConstantRegister, times_8, 0));
1512        return;
1513      default:
1514        LoadSmiConstant(dst, constant);
1515        addq(dst, src);
1516        return;
1517    }
1518  }
1519}
1520
1521
1522void MacroAssembler::SmiAddConstant(const Operand& dst, Smi* constant) {
1523  if (constant->value() != 0) {
1524    addl(Operand(dst, kSmiShift / kBitsPerByte), Immediate(constant->value()));
1525  }
1526}
1527
1528
1529void MacroAssembler::SmiAddConstant(Register dst,
1530                                    Register src,
1531                                    Smi* constant,
1532                                    Label* on_not_smi_result,
1533                                    Label::Distance near_jump) {
1534  if (constant->value() == 0) {
1535    if (!dst.is(src)) {
1536      movq(dst, src);
1537    }
1538  } else if (dst.is(src)) {
1539    ASSERT(!dst.is(kScratchRegister));
1540
1541    LoadSmiConstant(kScratchRegister, constant);
1542    addq(kScratchRegister, src);
1543    j(overflow, on_not_smi_result, near_jump);
1544    movq(dst, kScratchRegister);
1545  } else {
1546    LoadSmiConstant(dst, constant);
1547    addq(dst, src);
1548    j(overflow, on_not_smi_result, near_jump);
1549  }
1550}
1551
1552
1553void MacroAssembler::SmiSubConstant(Register dst, Register src, Smi* constant) {
1554  if (constant->value() == 0) {
1555    if (!dst.is(src)) {
1556      movq(dst, src);
1557    }
1558  } else if (dst.is(src)) {
1559    ASSERT(!dst.is(kScratchRegister));
1560    Register constant_reg = GetSmiConstant(constant);
1561    subq(dst, constant_reg);
1562  } else {
1563    if (constant->value() == Smi::kMinValue) {
1564      LoadSmiConstant(dst, constant);
1565      // Adding and subtracting the min-value gives the same result, it only
1566      // differs on the overflow bit, which we don't check here.
1567      addq(dst, src);
1568    } else {
1569      // Subtract by adding the negation.
1570      LoadSmiConstant(dst, Smi::FromInt(-constant->value()));
1571      addq(dst, src);
1572    }
1573  }
1574}
1575
1576
1577void MacroAssembler::SmiSubConstant(Register dst,
1578                                    Register src,
1579                                    Smi* constant,
1580                                    Label* on_not_smi_result,
1581                                    Label::Distance near_jump) {
1582  if (constant->value() == 0) {
1583    if (!dst.is(src)) {
1584      movq(dst, src);
1585    }
1586  } else if (dst.is(src)) {
1587    ASSERT(!dst.is(kScratchRegister));
1588    if (constant->value() == Smi::kMinValue) {
1589      // Subtracting min-value from any non-negative value will overflow.
1590      // We test the non-negativeness before doing the subtraction.
1591      testq(src, src);
1592      j(not_sign, on_not_smi_result, near_jump);
1593      LoadSmiConstant(kScratchRegister, constant);
1594      subq(dst, kScratchRegister);
1595    } else {
1596      // Subtract by adding the negation.
1597      LoadSmiConstant(kScratchRegister, Smi::FromInt(-constant->value()));
1598      addq(kScratchRegister, dst);
1599      j(overflow, on_not_smi_result, near_jump);
1600      movq(dst, kScratchRegister);
1601    }
1602  } else {
1603    if (constant->value() == Smi::kMinValue) {
1604      // Subtracting min-value from any non-negative value will overflow.
1605      // We test the non-negativeness before doing the subtraction.
1606      testq(src, src);
1607      j(not_sign, on_not_smi_result, near_jump);
1608      LoadSmiConstant(dst, constant);
1609      // Adding and subtracting the min-value gives the same result, it only
1610      // differs on the overflow bit, which we don't check here.
1611      addq(dst, src);
1612    } else {
1613      // Subtract by adding the negation.
1614      LoadSmiConstant(dst, Smi::FromInt(-(constant->value())));
1615      addq(dst, src);
1616      j(overflow, on_not_smi_result, near_jump);
1617    }
1618  }
1619}
1620
1621
1622void MacroAssembler::SmiNeg(Register dst,
1623                            Register src,
1624                            Label* on_smi_result,
1625                            Label::Distance near_jump) {
1626  if (dst.is(src)) {
1627    ASSERT(!dst.is(kScratchRegister));
1628    movq(kScratchRegister, src);
1629    neg(dst);  // Low 32 bits are retained as zero by negation.
1630    // Test if result is zero or Smi::kMinValue.
1631    cmpq(dst, kScratchRegister);
1632    j(not_equal, on_smi_result, near_jump);
1633    movq(src, kScratchRegister);
1634  } else {
1635    movq(dst, src);
1636    neg(dst);
1637    cmpq(dst, src);
1638    // If the result is zero or Smi::kMinValue, negation failed to create a smi.
1639    j(not_equal, on_smi_result, near_jump);
1640  }
1641}
1642
1643
1644void MacroAssembler::SmiAdd(Register dst,
1645                            Register src1,
1646                            Register src2,
1647                            Label* on_not_smi_result,
1648                            Label::Distance near_jump) {
1649  ASSERT_NOT_NULL(on_not_smi_result);
1650  ASSERT(!dst.is(src2));
1651  if (dst.is(src1)) {
1652    movq(kScratchRegister, src1);
1653    addq(kScratchRegister, src2);
1654    j(overflow, on_not_smi_result, near_jump);
1655    movq(dst, kScratchRegister);
1656  } else {
1657    movq(dst, src1);
1658    addq(dst, src2);
1659    j(overflow, on_not_smi_result, near_jump);
1660  }
1661}
1662
1663
1664void MacroAssembler::SmiAdd(Register dst,
1665                            Register src1,
1666                            const Operand& src2,
1667                            Label* on_not_smi_result,
1668                            Label::Distance near_jump) {
1669  ASSERT_NOT_NULL(on_not_smi_result);
1670  if (dst.is(src1)) {
1671    movq(kScratchRegister, src1);
1672    addq(kScratchRegister, src2);
1673    j(overflow, on_not_smi_result, near_jump);
1674    movq(dst, kScratchRegister);
1675  } else {
1676    ASSERT(!src2.AddressUsesRegister(dst));
1677    movq(dst, src1);
1678    addq(dst, src2);
1679    j(overflow, on_not_smi_result, near_jump);
1680  }
1681}
1682
1683
1684void MacroAssembler::SmiAdd(Register dst,
1685                            Register src1,
1686                            Register src2) {
1687  // No overflow checking. Use only when it's known that
1688  // overflowing is impossible.
1689  if (!dst.is(src1)) {
1690    if (emit_debug_code()) {
1691      movq(kScratchRegister, src1);
1692      addq(kScratchRegister, src2);
1693      Check(no_overflow, kSmiAdditionOverflow);
1694    }
1695    lea(dst, Operand(src1, src2, times_1, 0));
1696  } else {
1697    addq(dst, src2);
1698    Assert(no_overflow, kSmiAdditionOverflow);
1699  }
1700}
1701
1702
1703void MacroAssembler::SmiSub(Register dst,
1704                            Register src1,
1705                            Register src2,
1706                            Label* on_not_smi_result,
1707                            Label::Distance near_jump) {
1708  ASSERT_NOT_NULL(on_not_smi_result);
1709  ASSERT(!dst.is(src2));
1710  if (dst.is(src1)) {
1711    cmpq(dst, src2);
1712    j(overflow, on_not_smi_result, near_jump);
1713    subq(dst, src2);
1714  } else {
1715    movq(dst, src1);
1716    subq(dst, src2);
1717    j(overflow, on_not_smi_result, near_jump);
1718  }
1719}
1720
1721
1722void MacroAssembler::SmiSub(Register dst, Register src1, Register src2) {
1723  // No overflow checking. Use only when it's known that
1724  // overflowing is impossible (e.g., subtracting two positive smis).
1725  ASSERT(!dst.is(src2));
1726  if (!dst.is(src1)) {
1727    movq(dst, src1);
1728  }
1729  subq(dst, src2);
1730  Assert(no_overflow, kSmiSubtractionOverflow);
1731}
1732
1733
1734void MacroAssembler::SmiSub(Register dst,
1735                            Register src1,
1736                            const Operand& src2,
1737                            Label* on_not_smi_result,
1738                            Label::Distance near_jump) {
1739  ASSERT_NOT_NULL(on_not_smi_result);
1740  if (dst.is(src1)) {
1741    movq(kScratchRegister, src2);
1742    cmpq(src1, kScratchRegister);
1743    j(overflow, on_not_smi_result, near_jump);
1744    subq(src1, kScratchRegister);
1745  } else {
1746    movq(dst, src1);
1747    subq(dst, src2);
1748    j(overflow, on_not_smi_result, near_jump);
1749  }
1750}
1751
1752
1753void MacroAssembler::SmiSub(Register dst,
1754                            Register src1,
1755                            const Operand& src2) {
1756  // No overflow checking. Use only when it's known that
1757  // overflowing is impossible (e.g., subtracting two positive smis).
1758  if (!dst.is(src1)) {
1759    movq(dst, src1);
1760  }
1761  subq(dst, src2);
1762  Assert(no_overflow, kSmiSubtractionOverflow);
1763}
1764
1765
1766void MacroAssembler::SmiMul(Register dst,
1767                            Register src1,
1768                            Register src2,
1769                            Label* on_not_smi_result,
1770                            Label::Distance near_jump) {
1771  ASSERT(!dst.is(src2));
1772  ASSERT(!dst.is(kScratchRegister));
1773  ASSERT(!src1.is(kScratchRegister));
1774  ASSERT(!src2.is(kScratchRegister));
1775
1776  if (dst.is(src1)) {
1777    Label failure, zero_correct_result;
1778    movq(kScratchRegister, src1);  // Create backup for later testing.
1779    SmiToInteger64(dst, src1);
1780    imul(dst, src2);
1781    j(overflow, &failure, Label::kNear);
1782
1783    // Check for negative zero result.  If product is zero, and one
1784    // argument is negative, go to slow case.
1785    Label correct_result;
1786    testq(dst, dst);
1787    j(not_zero, &correct_result, Label::kNear);
1788
1789    movq(dst, kScratchRegister);
1790    xor_(dst, src2);
1791    // Result was positive zero.
1792    j(positive, &zero_correct_result, Label::kNear);
1793
1794    bind(&failure);  // Reused failure exit, restores src1.
1795    movq(src1, kScratchRegister);
1796    jmp(on_not_smi_result, near_jump);
1797
1798    bind(&zero_correct_result);
1799    Set(dst, 0);
1800
1801    bind(&correct_result);
1802  } else {
1803    SmiToInteger64(dst, src1);
1804    imul(dst, src2);
1805    j(overflow, on_not_smi_result, near_jump);
1806    // Check for negative zero result.  If product is zero, and one
1807    // argument is negative, go to slow case.
1808    Label correct_result;
1809    testq(dst, dst);
1810    j(not_zero, &correct_result, Label::kNear);
1811    // One of src1 and src2 is zero, the check whether the other is
1812    // negative.
1813    movq(kScratchRegister, src1);
1814    xor_(kScratchRegister, src2);
1815    j(negative, on_not_smi_result, near_jump);
1816    bind(&correct_result);
1817  }
1818}
1819
1820
1821void MacroAssembler::SmiDiv(Register dst,
1822                            Register src1,
1823                            Register src2,
1824                            Label* on_not_smi_result,
1825                            Label::Distance near_jump) {
1826  ASSERT(!src1.is(kScratchRegister));
1827  ASSERT(!src2.is(kScratchRegister));
1828  ASSERT(!dst.is(kScratchRegister));
1829  ASSERT(!src2.is(rax));
1830  ASSERT(!src2.is(rdx));
1831  ASSERT(!src1.is(rdx));
1832
1833  // Check for 0 divisor (result is +/-Infinity).
1834  testq(src2, src2);
1835  j(zero, on_not_smi_result, near_jump);
1836
1837  if (src1.is(rax)) {
1838    movq(kScratchRegister, src1);
1839  }
1840  SmiToInteger32(rax, src1);
1841  // We need to rule out dividing Smi::kMinValue by -1, since that would
1842  // overflow in idiv and raise an exception.
1843  // We combine this with negative zero test (negative zero only happens
1844  // when dividing zero by a negative number).
1845
1846  // We overshoot a little and go to slow case if we divide min-value
1847  // by any negative value, not just -1.
1848  Label safe_div;
1849  testl(rax, Immediate(0x7fffffff));
1850  j(not_zero, &safe_div, Label::kNear);
1851  testq(src2, src2);
1852  if (src1.is(rax)) {
1853    j(positive, &safe_div, Label::kNear);
1854    movq(src1, kScratchRegister);
1855    jmp(on_not_smi_result, near_jump);
1856  } else {
1857    j(negative, on_not_smi_result, near_jump);
1858  }
1859  bind(&safe_div);
1860
1861  SmiToInteger32(src2, src2);
1862  // Sign extend src1 into edx:eax.
1863  cdq();
1864  idivl(src2);
1865  Integer32ToSmi(src2, src2);
1866  // Check that the remainder is zero.
1867  testl(rdx, rdx);
1868  if (src1.is(rax)) {
1869    Label smi_result;
1870    j(zero, &smi_result, Label::kNear);
1871    movq(src1, kScratchRegister);
1872    jmp(on_not_smi_result, near_jump);
1873    bind(&smi_result);
1874  } else {
1875    j(not_zero, on_not_smi_result, near_jump);
1876  }
1877  if (!dst.is(src1) && src1.is(rax)) {
1878    movq(src1, kScratchRegister);
1879  }
1880  Integer32ToSmi(dst, rax);
1881}
1882
1883
1884void MacroAssembler::SmiMod(Register dst,
1885                            Register src1,
1886                            Register src2,
1887                            Label* on_not_smi_result,
1888                            Label::Distance near_jump) {
1889  ASSERT(!dst.is(kScratchRegister));
1890  ASSERT(!src1.is(kScratchRegister));
1891  ASSERT(!src2.is(kScratchRegister));
1892  ASSERT(!src2.is(rax));
1893  ASSERT(!src2.is(rdx));
1894  ASSERT(!src1.is(rdx));
1895  ASSERT(!src1.is(src2));
1896
1897  testq(src2, src2);
1898  j(zero, on_not_smi_result, near_jump);
1899
1900  if (src1.is(rax)) {
1901    movq(kScratchRegister, src1);
1902  }
1903  SmiToInteger32(rax, src1);
1904  SmiToInteger32(src2, src2);
1905
1906  // Test for the edge case of dividing Smi::kMinValue by -1 (will overflow).
1907  Label safe_div;
1908  cmpl(rax, Immediate(Smi::kMinValue));
1909  j(not_equal, &safe_div, Label::kNear);
1910  cmpl(src2, Immediate(-1));
1911  j(not_equal, &safe_div, Label::kNear);
1912  // Retag inputs and go slow case.
1913  Integer32ToSmi(src2, src2);
1914  if (src1.is(rax)) {
1915    movq(src1, kScratchRegister);
1916  }
1917  jmp(on_not_smi_result, near_jump);
1918  bind(&safe_div);
1919
1920  // Sign extend eax into edx:eax.
1921  cdq();
1922  idivl(src2);
1923  // Restore smi tags on inputs.
1924  Integer32ToSmi(src2, src2);
1925  if (src1.is(rax)) {
1926    movq(src1, kScratchRegister);
1927  }
1928  // Check for a negative zero result.  If the result is zero, and the
1929  // dividend is negative, go slow to return a floating point negative zero.
1930  Label smi_result;
1931  testl(rdx, rdx);
1932  j(not_zero, &smi_result, Label::kNear);
1933  testq(src1, src1);
1934  j(negative, on_not_smi_result, near_jump);
1935  bind(&smi_result);
1936  Integer32ToSmi(dst, rdx);
1937}
1938
1939
1940void MacroAssembler::SmiNot(Register dst, Register src) {
1941  ASSERT(!dst.is(kScratchRegister));
1942  ASSERT(!src.is(kScratchRegister));
1943  // Set tag and padding bits before negating, so that they are zero afterwards.
1944  movl(kScratchRegister, Immediate(~0));
1945  if (dst.is(src)) {
1946    xor_(dst, kScratchRegister);
1947  } else {
1948    lea(dst, Operand(src, kScratchRegister, times_1, 0));
1949  }
1950  not_(dst);
1951}
1952
1953
1954void MacroAssembler::SmiAnd(Register dst, Register src1, Register src2) {
1955  ASSERT(!dst.is(src2));
1956  if (!dst.is(src1)) {
1957    movq(dst, src1);
1958  }
1959  and_(dst, src2);
1960}
1961
1962
1963void MacroAssembler::SmiAndConstant(Register dst, Register src, Smi* constant) {
1964  if (constant->value() == 0) {
1965    Set(dst, 0);
1966  } else if (dst.is(src)) {
1967    ASSERT(!dst.is(kScratchRegister));
1968    Register constant_reg = GetSmiConstant(constant);
1969    and_(dst, constant_reg);
1970  } else {
1971    LoadSmiConstant(dst, constant);
1972    and_(dst, src);
1973  }
1974}
1975
1976
1977void MacroAssembler::SmiOr(Register dst, Register src1, Register src2) {
1978  if (!dst.is(src1)) {
1979    ASSERT(!src1.is(src2));
1980    movq(dst, src1);
1981  }
1982  or_(dst, src2);
1983}
1984
1985
1986void MacroAssembler::SmiOrConstant(Register dst, Register src, Smi* constant) {
1987  if (dst.is(src)) {
1988    ASSERT(!dst.is(kScratchRegister));
1989    Register constant_reg = GetSmiConstant(constant);
1990    or_(dst, constant_reg);
1991  } else {
1992    LoadSmiConstant(dst, constant);
1993    or_(dst, src);
1994  }
1995}
1996
1997
1998void MacroAssembler::SmiXor(Register dst, Register src1, Register src2) {
1999  if (!dst.is(src1)) {
2000    ASSERT(!src1.is(src2));
2001    movq(dst, src1);
2002  }
2003  xor_(dst, src2);
2004}
2005
2006
2007void MacroAssembler::SmiXorConstant(Register dst, Register src, Smi* constant) {
2008  if (dst.is(src)) {
2009    ASSERT(!dst.is(kScratchRegister));
2010    Register constant_reg = GetSmiConstant(constant);
2011    xor_(dst, constant_reg);
2012  } else {
2013    LoadSmiConstant(dst, constant);
2014    xor_(dst, src);
2015  }
2016}
2017
2018
2019void MacroAssembler::SmiShiftArithmeticRightConstant(Register dst,
2020                                                     Register src,
2021                                                     int shift_value) {
2022  ASSERT(is_uint5(shift_value));
2023  if (shift_value > 0) {
2024    if (dst.is(src)) {
2025      sar(dst, Immediate(shift_value + kSmiShift));
2026      shl(dst, Immediate(kSmiShift));
2027    } else {
2028      UNIMPLEMENTED();  // Not used.
2029    }
2030  }
2031}
2032
2033
2034void MacroAssembler::SmiShiftLeftConstant(Register dst,
2035                                          Register src,
2036                                          int shift_value) {
2037  if (!dst.is(src)) {
2038    movq(dst, src);
2039  }
2040  if (shift_value > 0) {
2041    shl(dst, Immediate(shift_value));
2042  }
2043}
2044
2045
2046void MacroAssembler::SmiShiftLogicalRightConstant(
2047    Register dst, Register src, int shift_value,
2048    Label* on_not_smi_result, Label::Distance near_jump) {
2049  // Logic right shift interprets its result as an *unsigned* number.
2050  if (dst.is(src)) {
2051    UNIMPLEMENTED();  // Not used.
2052  } else {
2053    movq(dst, src);
2054    if (shift_value == 0) {
2055      testq(dst, dst);
2056      j(negative, on_not_smi_result, near_jump);
2057    }
2058    shr(dst, Immediate(shift_value + kSmiShift));
2059    shl(dst, Immediate(kSmiShift));
2060  }
2061}
2062
2063
2064void MacroAssembler::SmiShiftLeft(Register dst,
2065                                  Register src1,
2066                                  Register src2) {
2067  ASSERT(!dst.is(rcx));
2068  // Untag shift amount.
2069  if (!dst.is(src1)) {
2070    movq(dst, src1);
2071  }
2072  SmiToInteger32(rcx, src2);
2073  // Shift amount specified by lower 5 bits, not six as the shl opcode.
2074  and_(rcx, Immediate(0x1f));
2075  shl_cl(dst);
2076}
2077
2078
2079void MacroAssembler::SmiShiftLogicalRight(Register dst,
2080                                          Register src1,
2081                                          Register src2,
2082                                          Label* on_not_smi_result,
2083                                          Label::Distance near_jump) {
2084  ASSERT(!dst.is(kScratchRegister));
2085  ASSERT(!src1.is(kScratchRegister));
2086  ASSERT(!src2.is(kScratchRegister));
2087  ASSERT(!dst.is(rcx));
2088  // dst and src1 can be the same, because the one case that bails out
2089  // is a shift by 0, which leaves dst, and therefore src1, unchanged.
2090  if (src1.is(rcx) || src2.is(rcx)) {
2091    movq(kScratchRegister, rcx);
2092  }
2093  if (!dst.is(src1)) {
2094    movq(dst, src1);
2095  }
2096  SmiToInteger32(rcx, src2);
2097  orl(rcx, Immediate(kSmiShift));
2098  shr_cl(dst);  // Shift is rcx modulo 0x1f + 32.
2099  shl(dst, Immediate(kSmiShift));
2100  testq(dst, dst);
2101  if (src1.is(rcx) || src2.is(rcx)) {
2102    Label positive_result;
2103    j(positive, &positive_result, Label::kNear);
2104    if (src1.is(rcx)) {
2105      movq(src1, kScratchRegister);
2106    } else {
2107      movq(src2, kScratchRegister);
2108    }
2109    jmp(on_not_smi_result, near_jump);
2110    bind(&positive_result);
2111  } else {
2112    // src2 was zero and src1 negative.
2113    j(negative, on_not_smi_result, near_jump);
2114  }
2115}
2116
2117
2118void MacroAssembler::SmiShiftArithmeticRight(Register dst,
2119                                             Register src1,
2120                                             Register src2) {
2121  ASSERT(!dst.is(kScratchRegister));
2122  ASSERT(!src1.is(kScratchRegister));
2123  ASSERT(!src2.is(kScratchRegister));
2124  ASSERT(!dst.is(rcx));
2125  if (src1.is(rcx)) {
2126    movq(kScratchRegister, src1);
2127  } else if (src2.is(rcx)) {
2128    movq(kScratchRegister, src2);
2129  }
2130  if (!dst.is(src1)) {
2131    movq(dst, src1);
2132  }
2133  SmiToInteger32(rcx, src2);
2134  orl(rcx, Immediate(kSmiShift));
2135  sar_cl(dst);  // Shift 32 + original rcx & 0x1f.
2136  shl(dst, Immediate(kSmiShift));
2137  if (src1.is(rcx)) {
2138    movq(src1, kScratchRegister);
2139  } else if (src2.is(rcx)) {
2140    movq(src2, kScratchRegister);
2141  }
2142}
2143
2144
2145void MacroAssembler::SelectNonSmi(Register dst,
2146                                  Register src1,
2147                                  Register src2,
2148                                  Label* on_not_smis,
2149                                  Label::Distance near_jump) {
2150  ASSERT(!dst.is(kScratchRegister));
2151  ASSERT(!src1.is(kScratchRegister));
2152  ASSERT(!src2.is(kScratchRegister));
2153  ASSERT(!dst.is(src1));
2154  ASSERT(!dst.is(src2));
2155  // Both operands must not be smis.
2156#ifdef DEBUG
2157  if (allow_stub_calls()) {  // Check contains a stub call.
2158    Condition not_both_smis = NegateCondition(CheckBothSmi(src1, src2));
2159    Check(not_both_smis, kBothRegistersWereSmisInSelectNonSmi);
2160  }
2161#endif
2162  STATIC_ASSERT(kSmiTag == 0);
2163  ASSERT_EQ(0, Smi::FromInt(0));
2164  movl(kScratchRegister, Immediate(kSmiTagMask));
2165  and_(kScratchRegister, src1);
2166  testl(kScratchRegister, src2);
2167  // If non-zero then both are smis.
2168  j(not_zero, on_not_smis, near_jump);
2169
2170  // Exactly one operand is a smi.
2171  ASSERT_EQ(1, static_cast<int>(kSmiTagMask));
2172  // kScratchRegister still holds src1 & kSmiTag, which is either zero or one.
2173  subq(kScratchRegister, Immediate(1));
2174  // If src1 is a smi, then scratch register all 1s, else it is all 0s.
2175  movq(dst, src1);
2176  xor_(dst, src2);
2177  and_(dst, kScratchRegister);
2178  // If src1 is a smi, dst holds src1 ^ src2, else it is zero.
2179  xor_(dst, src1);
2180  // If src1 is a smi, dst is src2, else it is src1, i.e., the non-smi.
2181}
2182
2183
2184SmiIndex MacroAssembler::SmiToIndex(Register dst,
2185                                    Register src,
2186                                    int shift) {
2187  ASSERT(is_uint6(shift));
2188  // There is a possible optimization if shift is in the range 60-63, but that
2189  // will (and must) never happen.
2190  if (!dst.is(src)) {
2191    movq(dst, src);
2192  }
2193  if (shift < kSmiShift) {
2194    sar(dst, Immediate(kSmiShift - shift));
2195  } else {
2196    shl(dst, Immediate(shift - kSmiShift));
2197  }
2198  return SmiIndex(dst, times_1);
2199}
2200
2201SmiIndex MacroAssembler::SmiToNegativeIndex(Register dst,
2202                                            Register src,
2203                                            int shift) {
2204  // Register src holds a positive smi.
2205  ASSERT(is_uint6(shift));
2206  if (!dst.is(src)) {
2207    movq(dst, src);
2208  }
2209  neg(dst);
2210  if (shift < kSmiShift) {
2211    sar(dst, Immediate(kSmiShift - shift));
2212  } else {
2213    shl(dst, Immediate(shift - kSmiShift));
2214  }
2215  return SmiIndex(dst, times_1);
2216}
2217
2218
2219void MacroAssembler::AddSmiField(Register dst, const Operand& src) {
2220  ASSERT_EQ(0, kSmiShift % kBitsPerByte);
2221  addl(dst, Operand(src, kSmiShift / kBitsPerByte));
2222}
2223
2224
2225void MacroAssembler::JumpIfNotString(Register object,
2226                                     Register object_map,
2227                                     Label* not_string,
2228                                     Label::Distance near_jump) {
2229  Condition is_smi = CheckSmi(object);
2230  j(is_smi, not_string, near_jump);
2231  CmpObjectType(object, FIRST_NONSTRING_TYPE, object_map);
2232  j(above_equal, not_string, near_jump);
2233}
2234
2235
2236void MacroAssembler::JumpIfNotBothSequentialAsciiStrings(
2237    Register first_object,
2238    Register second_object,
2239    Register scratch1,
2240    Register scratch2,
2241    Label* on_fail,
2242    Label::Distance near_jump) {
2243  // Check that both objects are not smis.
2244  Condition either_smi = CheckEitherSmi(first_object, second_object);
2245  j(either_smi, on_fail, near_jump);
2246
2247  // Load instance type for both strings.
2248  movq(scratch1, FieldOperand(first_object, HeapObject::kMapOffset));
2249  movq(scratch2, FieldOperand(second_object, HeapObject::kMapOffset));
2250  movzxbl(scratch1, FieldOperand(scratch1, Map::kInstanceTypeOffset));
2251  movzxbl(scratch2, FieldOperand(scratch2, Map::kInstanceTypeOffset));
2252
2253  // Check that both are flat ASCII strings.
2254  ASSERT(kNotStringTag != 0);
2255  const int kFlatAsciiStringMask =
2256      kIsNotStringMask | kStringRepresentationMask | kStringEncodingMask;
2257  const int kFlatAsciiStringTag =
2258      kStringTag | kOneByteStringTag | kSeqStringTag;
2259
2260  andl(scratch1, Immediate(kFlatAsciiStringMask));
2261  andl(scratch2, Immediate(kFlatAsciiStringMask));
2262  // Interleave the bits to check both scratch1 and scratch2 in one test.
2263  ASSERT_EQ(0, kFlatAsciiStringMask & (kFlatAsciiStringMask << 3));
2264  lea(scratch1, Operand(scratch1, scratch2, times_8, 0));
2265  cmpl(scratch1,
2266       Immediate(kFlatAsciiStringTag + (kFlatAsciiStringTag << 3)));
2267  j(not_equal, on_fail, near_jump);
2268}
2269
2270
2271void MacroAssembler::JumpIfInstanceTypeIsNotSequentialAscii(
2272    Register instance_type,
2273    Register scratch,
2274    Label* failure,
2275    Label::Distance near_jump) {
2276  if (!scratch.is(instance_type)) {
2277    movl(scratch, instance_type);
2278  }
2279
2280  const int kFlatAsciiStringMask =
2281      kIsNotStringMask | kStringRepresentationMask | kStringEncodingMask;
2282
2283  andl(scratch, Immediate(kFlatAsciiStringMask));
2284  cmpl(scratch, Immediate(kStringTag | kSeqStringTag | kOneByteStringTag));
2285  j(not_equal, failure, near_jump);
2286}
2287
2288
2289void MacroAssembler::JumpIfBothInstanceTypesAreNotSequentialAscii(
2290    Register first_object_instance_type,
2291    Register second_object_instance_type,
2292    Register scratch1,
2293    Register scratch2,
2294    Label* on_fail,
2295    Label::Distance near_jump) {
2296  // Load instance type for both strings.
2297  movq(scratch1, first_object_instance_type);
2298  movq(scratch2, second_object_instance_type);
2299
2300  // Check that both are flat ASCII strings.
2301  ASSERT(kNotStringTag != 0);
2302  const int kFlatAsciiStringMask =
2303      kIsNotStringMask | kStringRepresentationMask | kStringEncodingMask;
2304  const int kFlatAsciiStringTag =
2305      kStringTag | kOneByteStringTag | kSeqStringTag;
2306
2307  andl(scratch1, Immediate(kFlatAsciiStringMask));
2308  andl(scratch2, Immediate(kFlatAsciiStringMask));
2309  // Interleave the bits to check both scratch1 and scratch2 in one test.
2310  ASSERT_EQ(0, kFlatAsciiStringMask & (kFlatAsciiStringMask << 3));
2311  lea(scratch1, Operand(scratch1, scratch2, times_8, 0));
2312  cmpl(scratch1,
2313       Immediate(kFlatAsciiStringTag + (kFlatAsciiStringTag << 3)));
2314  j(not_equal, on_fail, near_jump);
2315}
2316
2317
2318template<class T>
2319static void JumpIfNotUniqueNameHelper(MacroAssembler* masm,
2320                                      T operand_or_register,
2321                                      Label* not_unique_name,
2322                                      Label::Distance distance) {
2323  STATIC_ASSERT(kInternalizedTag == 0 && kStringTag == 0);
2324  Label succeed;
2325  masm->testb(operand_or_register,
2326              Immediate(kIsNotStringMask | kIsNotInternalizedMask));
2327  masm->j(zero, &succeed, Label::kNear);
2328  masm->cmpb(operand_or_register, Immediate(static_cast<uint8_t>(SYMBOL_TYPE)));
2329  masm->j(not_equal, not_unique_name, distance);
2330
2331  masm->bind(&succeed);
2332}
2333
2334
2335void MacroAssembler::JumpIfNotUniqueName(Operand operand,
2336                                         Label* not_unique_name,
2337                                         Label::Distance distance) {
2338  JumpIfNotUniqueNameHelper<Operand>(this, operand, not_unique_name, distance);
2339}
2340
2341
2342void MacroAssembler::JumpIfNotUniqueName(Register reg,
2343                                         Label* not_unique_name,
2344                                         Label::Distance distance) {
2345  JumpIfNotUniqueNameHelper<Register>(this, reg, not_unique_name, distance);
2346}
2347
2348
2349void MacroAssembler::Move(Register dst, Register src) {
2350  if (!dst.is(src)) {
2351    movq(dst, src);
2352  }
2353}
2354
2355
2356void MacroAssembler::Move(Register dst, Handle<Object> source) {
2357  AllowDeferredHandleDereference smi_check;
2358  if (source->IsSmi()) {
2359    Move(dst, Smi::cast(*source));
2360  } else {
2361    ASSERT(source->IsHeapObject());
2362    movq(dst, source, RelocInfo::EMBEDDED_OBJECT);
2363  }
2364}
2365
2366
2367void MacroAssembler::Move(const Operand& dst, Handle<Object> source) {
2368  AllowDeferredHandleDereference smi_check;
2369  if (source->IsSmi()) {
2370    Move(dst, Smi::cast(*source));
2371  } else {
2372    ASSERT(source->IsHeapObject());
2373    movq(kScratchRegister, source, RelocInfo::EMBEDDED_OBJECT);
2374    movq(dst, kScratchRegister);
2375  }
2376}
2377
2378
2379void MacroAssembler::Cmp(Register dst, Handle<Object> source) {
2380  AllowDeferredHandleDereference smi_check;
2381  if (source->IsSmi()) {
2382    Cmp(dst, Smi::cast(*source));
2383  } else {
2384    ASSERT(source->IsHeapObject());
2385    movq(kScratchRegister, source, RelocInfo::EMBEDDED_OBJECT);
2386    cmpq(dst, kScratchRegister);
2387  }
2388}
2389
2390
2391void MacroAssembler::Cmp(const Operand& dst, Handle<Object> source) {
2392  AllowDeferredHandleDereference smi_check;
2393  if (source->IsSmi()) {
2394    Cmp(dst, Smi::cast(*source));
2395  } else {
2396    ASSERT(source->IsHeapObject());
2397    movq(kScratchRegister, source, RelocInfo::EMBEDDED_OBJECT);
2398    cmpq(dst, kScratchRegister);
2399  }
2400}
2401
2402
2403void MacroAssembler::Push(Handle<Object> source) {
2404  AllowDeferredHandleDereference smi_check;
2405  if (source->IsSmi()) {
2406    Push(Smi::cast(*source));
2407  } else {
2408    ASSERT(source->IsHeapObject());
2409    movq(kScratchRegister, source, RelocInfo::EMBEDDED_OBJECT);
2410    push(kScratchRegister);
2411  }
2412}
2413
2414
2415void MacroAssembler::LoadHeapObject(Register result,
2416                                    Handle<HeapObject> object) {
2417  AllowDeferredHandleDereference using_raw_address;
2418  if (isolate()->heap()->InNewSpace(*object)) {
2419    Handle<Cell> cell = isolate()->factory()->NewCell(object);
2420    movq(result, cell, RelocInfo::CELL);
2421    movq(result, Operand(result, 0));
2422  } else {
2423    Move(result, object);
2424  }
2425}
2426
2427
2428void MacroAssembler::CmpHeapObject(Register reg, Handle<HeapObject> object) {
2429  AllowDeferredHandleDereference using_raw_address;
2430  if (isolate()->heap()->InNewSpace(*object)) {
2431    Handle<Cell> cell = isolate()->factory()->NewCell(object);
2432    movq(kScratchRegister, cell, RelocInfo::CELL);
2433    cmpq(reg, Operand(kScratchRegister, 0));
2434  } else {
2435    Cmp(reg, object);
2436  }
2437}
2438
2439
2440void MacroAssembler::PushHeapObject(Handle<HeapObject> object) {
2441  AllowDeferredHandleDereference using_raw_address;
2442  if (isolate()->heap()->InNewSpace(*object)) {
2443    Handle<Cell> cell = isolate()->factory()->NewCell(object);
2444    movq(kScratchRegister, cell, RelocInfo::CELL);
2445    movq(kScratchRegister, Operand(kScratchRegister, 0));
2446    push(kScratchRegister);
2447  } else {
2448    Push(object);
2449  }
2450}
2451
2452
2453void MacroAssembler::LoadGlobalCell(Register dst, Handle<Cell> cell) {
2454  if (dst.is(rax)) {
2455    AllowDeferredHandleDereference embedding_raw_address;
2456    load_rax(cell.location(), RelocInfo::CELL);
2457  } else {
2458    movq(dst, cell, RelocInfo::CELL);
2459    movq(dst, Operand(dst, 0));
2460  }
2461}
2462
2463
2464void MacroAssembler::Push(Smi* source) {
2465  intptr_t smi = reinterpret_cast<intptr_t>(source);
2466  if (is_int32(smi)) {
2467    push(Immediate(static_cast<int32_t>(smi)));
2468  } else {
2469    Register constant = GetSmiConstant(source);
2470    push(constant);
2471  }
2472}
2473
2474
2475void MacroAssembler::Drop(int stack_elements) {
2476  if (stack_elements > 0) {
2477    addq(rsp, Immediate(stack_elements * kPointerSize));
2478  }
2479}
2480
2481
2482void MacroAssembler::Test(const Operand& src, Smi* source) {
2483  testl(Operand(src, kIntSize), Immediate(source->value()));
2484}
2485
2486
2487void MacroAssembler::TestBit(const Operand& src, int bits) {
2488  int byte_offset = bits / kBitsPerByte;
2489  int bit_in_byte = bits & (kBitsPerByte - 1);
2490  testb(Operand(src, byte_offset), Immediate(1 << bit_in_byte));
2491}
2492
2493
2494void MacroAssembler::Jump(ExternalReference ext) {
2495  LoadAddress(kScratchRegister, ext);
2496  jmp(kScratchRegister);
2497}
2498
2499
2500void MacroAssembler::Jump(Address destination, RelocInfo::Mode rmode) {
2501  movq(kScratchRegister, destination, rmode);
2502  jmp(kScratchRegister);
2503}
2504
2505
2506void MacroAssembler::Jump(Handle<Code> code_object, RelocInfo::Mode rmode) {
2507  // TODO(X64): Inline this
2508  jmp(code_object, rmode);
2509}
2510
2511
2512int MacroAssembler::CallSize(ExternalReference ext) {
2513  // Opcode for call kScratchRegister is: Rex.B FF D4 (three bytes).
2514  return LoadAddressSize(ext) +
2515         Assembler::kCallScratchRegisterInstructionLength;
2516}
2517
2518
2519void MacroAssembler::Call(ExternalReference ext) {
2520#ifdef DEBUG
2521  int end_position = pc_offset() + CallSize(ext);
2522#endif
2523  LoadAddress(kScratchRegister, ext);
2524  call(kScratchRegister);
2525#ifdef DEBUG
2526  CHECK_EQ(end_position, pc_offset());
2527#endif
2528}
2529
2530
2531void MacroAssembler::Call(Address destination, RelocInfo::Mode rmode) {
2532#ifdef DEBUG
2533  int end_position = pc_offset() + CallSize(destination, rmode);
2534#endif
2535  movq(kScratchRegister, destination, rmode);
2536  call(kScratchRegister);
2537#ifdef DEBUG
2538  CHECK_EQ(pc_offset(), end_position);
2539#endif
2540}
2541
2542
2543void MacroAssembler::Call(Handle<Code> code_object,
2544                          RelocInfo::Mode rmode,
2545                          TypeFeedbackId ast_id) {
2546#ifdef DEBUG
2547  int end_position = pc_offset() + CallSize(code_object);
2548#endif
2549  ASSERT(RelocInfo::IsCodeTarget(rmode));
2550  call(code_object, rmode, ast_id);
2551#ifdef DEBUG
2552  CHECK_EQ(end_position, pc_offset());
2553#endif
2554}
2555
2556
2557void MacroAssembler::Pushad() {
2558  push(rax);
2559  push(rcx);
2560  push(rdx);
2561  push(rbx);
2562  // Not pushing rsp or rbp.
2563  push(rsi);
2564  push(rdi);
2565  push(r8);
2566  push(r9);
2567  // r10 is kScratchRegister.
2568  push(r11);
2569  // r12 is kSmiConstantRegister.
2570  // r13 is kRootRegister.
2571  push(r14);
2572  push(r15);
2573  STATIC_ASSERT(11 == kNumSafepointSavedRegisters);
2574  // Use lea for symmetry with Popad.
2575  int sp_delta =
2576      (kNumSafepointRegisters - kNumSafepointSavedRegisters) * kPointerSize;
2577  lea(rsp, Operand(rsp, -sp_delta));
2578}
2579
2580
2581void MacroAssembler::Popad() {
2582  // Popad must not change the flags, so use lea instead of addq.
2583  int sp_delta =
2584      (kNumSafepointRegisters - kNumSafepointSavedRegisters) * kPointerSize;
2585  lea(rsp, Operand(rsp, sp_delta));
2586  pop(r15);
2587  pop(r14);
2588  pop(r11);
2589  pop(r9);
2590  pop(r8);
2591  pop(rdi);
2592  pop(rsi);
2593  pop(rbx);
2594  pop(rdx);
2595  pop(rcx);
2596  pop(rax);
2597}
2598
2599
2600void MacroAssembler::Dropad() {
2601  addq(rsp, Immediate(kNumSafepointRegisters * kPointerSize));
2602}
2603
2604
2605// Order general registers are pushed by Pushad:
2606// rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r11, r14, r15.
2607const int
2608MacroAssembler::kSafepointPushRegisterIndices[Register::kNumRegisters] = {
2609    0,
2610    1,
2611    2,
2612    3,
2613    -1,
2614    -1,
2615    4,
2616    5,
2617    6,
2618    7,
2619    -1,
2620    8,
2621    -1,
2622    -1,
2623    9,
2624    10
2625};
2626
2627
2628void MacroAssembler::StoreToSafepointRegisterSlot(Register dst,
2629                                                  const Immediate& imm) {
2630  movq(SafepointRegisterSlot(dst), imm);
2631}
2632
2633
2634void MacroAssembler::StoreToSafepointRegisterSlot(Register dst, Register src) {
2635  movq(SafepointRegisterSlot(dst), src);
2636}
2637
2638
2639void MacroAssembler::LoadFromSafepointRegisterSlot(Register dst, Register src) {
2640  movq(dst, SafepointRegisterSlot(src));
2641}
2642
2643
2644Operand MacroAssembler::SafepointRegisterSlot(Register reg) {
2645  return Operand(rsp, SafepointRegisterStackIndex(reg.code()) * kPointerSize);
2646}
2647
2648
2649void MacroAssembler::PushTryHandler(StackHandler::Kind kind,
2650                                    int handler_index) {
2651  // Adjust this code if not the case.
2652  STATIC_ASSERT(StackHandlerConstants::kSize == 5 * kPointerSize);
2653  STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
2654  STATIC_ASSERT(StackHandlerConstants::kCodeOffset == 1 * kPointerSize);
2655  STATIC_ASSERT(StackHandlerConstants::kStateOffset == 2 * kPointerSize);
2656  STATIC_ASSERT(StackHandlerConstants::kContextOffset == 3 * kPointerSize);
2657  STATIC_ASSERT(StackHandlerConstants::kFPOffset == 4 * kPointerSize);
2658
2659  // We will build up the handler from the bottom by pushing on the stack.
2660  // First push the frame pointer and context.
2661  if (kind == StackHandler::JS_ENTRY) {
2662    // The frame pointer does not point to a JS frame so we save NULL for
2663    // rbp. We expect the code throwing an exception to check rbp before
2664    // dereferencing it to restore the context.
2665    push(Immediate(0));  // NULL frame pointer.
2666    Push(Smi::FromInt(0));  // No context.
2667  } else {
2668    push(rbp);
2669    push(rsi);
2670  }
2671
2672  // Push the state and the code object.
2673  unsigned state =
2674      StackHandler::IndexField::encode(handler_index) |
2675      StackHandler::KindField::encode(kind);
2676  push(Immediate(state));
2677  Push(CodeObject());
2678
2679  // Link the current handler as the next handler.
2680  ExternalReference handler_address(Isolate::kHandlerAddress, isolate());
2681  push(ExternalOperand(handler_address));
2682  // Set this new handler as the current one.
2683  movq(ExternalOperand(handler_address), rsp);
2684}
2685
2686
2687void MacroAssembler::PopTryHandler() {
2688  STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
2689  ExternalReference handler_address(Isolate::kHandlerAddress, isolate());
2690  pop(ExternalOperand(handler_address));
2691  addq(rsp, Immediate(StackHandlerConstants::kSize - kPointerSize));
2692}
2693
2694
2695void MacroAssembler::JumpToHandlerEntry() {
2696  // Compute the handler entry address and jump to it.  The handler table is
2697  // a fixed array of (smi-tagged) code offsets.
2698  // rax = exception, rdi = code object, rdx = state.
2699  movq(rbx, FieldOperand(rdi, Code::kHandlerTableOffset));
2700  shr(rdx, Immediate(StackHandler::kKindWidth));
2701  movq(rdx,
2702       FieldOperand(rbx, rdx, times_pointer_size, FixedArray::kHeaderSize));
2703  SmiToInteger64(rdx, rdx);
2704  lea(rdi, FieldOperand(rdi, rdx, times_1, Code::kHeaderSize));
2705  jmp(rdi);
2706}
2707
2708
2709void MacroAssembler::Throw(Register value) {
2710  // Adjust this code if not the case.
2711  STATIC_ASSERT(StackHandlerConstants::kSize == 5 * kPointerSize);
2712  STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
2713  STATIC_ASSERT(StackHandlerConstants::kCodeOffset == 1 * kPointerSize);
2714  STATIC_ASSERT(StackHandlerConstants::kStateOffset == 2 * kPointerSize);
2715  STATIC_ASSERT(StackHandlerConstants::kContextOffset == 3 * kPointerSize);
2716  STATIC_ASSERT(StackHandlerConstants::kFPOffset == 4 * kPointerSize);
2717
2718  // The exception is expected in rax.
2719  if (!value.is(rax)) {
2720    movq(rax, value);
2721  }
2722  // Drop the stack pointer to the top of the top handler.
2723  ExternalReference handler_address(Isolate::kHandlerAddress, isolate());
2724  movq(rsp, ExternalOperand(handler_address));
2725  // Restore the next handler.
2726  pop(ExternalOperand(handler_address));
2727
2728  // Remove the code object and state, compute the handler address in rdi.
2729  pop(rdi);  // Code object.
2730  pop(rdx);  // Offset and state.
2731
2732  // Restore the context and frame pointer.
2733  pop(rsi);  // Context.
2734  pop(rbp);  // Frame pointer.
2735
2736  // If the handler is a JS frame, restore the context to the frame.
2737  // (kind == ENTRY) == (rbp == 0) == (rsi == 0), so we could test either
2738  // rbp or rsi.
2739  Label skip;
2740  testq(rsi, rsi);
2741  j(zero, &skip, Label::kNear);
2742  movq(Operand(rbp, StandardFrameConstants::kContextOffset), rsi);
2743  bind(&skip);
2744
2745  JumpToHandlerEntry();
2746}
2747
2748
2749void MacroAssembler::ThrowUncatchable(Register value) {
2750  // Adjust this code if not the case.
2751  STATIC_ASSERT(StackHandlerConstants::kSize == 5 * kPointerSize);
2752  STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
2753  STATIC_ASSERT(StackHandlerConstants::kCodeOffset == 1 * kPointerSize);
2754  STATIC_ASSERT(StackHandlerConstants::kStateOffset == 2 * kPointerSize);
2755  STATIC_ASSERT(StackHandlerConstants::kContextOffset == 3 * kPointerSize);
2756  STATIC_ASSERT(StackHandlerConstants::kFPOffset == 4 * kPointerSize);
2757
2758  // The exception is expected in rax.
2759  if (!value.is(rax)) {
2760    movq(rax, value);
2761  }
2762  // Drop the stack pointer to the top of the top stack handler.
2763  ExternalReference handler_address(Isolate::kHandlerAddress, isolate());
2764  Load(rsp, handler_address);
2765
2766  // Unwind the handlers until the top ENTRY handler is found.
2767  Label fetch_next, check_kind;
2768  jmp(&check_kind, Label::kNear);
2769  bind(&fetch_next);
2770  movq(rsp, Operand(rsp, StackHandlerConstants::kNextOffset));
2771
2772  bind(&check_kind);
2773  STATIC_ASSERT(StackHandler::JS_ENTRY == 0);
2774  testl(Operand(rsp, StackHandlerConstants::kStateOffset),
2775        Immediate(StackHandler::KindField::kMask));
2776  j(not_zero, &fetch_next);
2777
2778  // Set the top handler address to next handler past the top ENTRY handler.
2779  pop(ExternalOperand(handler_address));
2780
2781  // Remove the code object and state, compute the handler address in rdi.
2782  pop(rdi);  // Code object.
2783  pop(rdx);  // Offset and state.
2784
2785  // Clear the context pointer and frame pointer (0 was saved in the handler).
2786  pop(rsi);
2787  pop(rbp);
2788
2789  JumpToHandlerEntry();
2790}
2791
2792
2793void MacroAssembler::Ret() {
2794  ret(0);
2795}
2796
2797
2798void MacroAssembler::Ret(int bytes_dropped, Register scratch) {
2799  if (is_uint16(bytes_dropped)) {
2800    ret(bytes_dropped);
2801  } else {
2802    PopReturnAddressTo(scratch);
2803    addq(rsp, Immediate(bytes_dropped));
2804    PushReturnAddressFrom(scratch);
2805    ret(0);
2806  }
2807}
2808
2809
2810void MacroAssembler::FCmp() {
2811  fucomip();
2812  fstp(0);
2813}
2814
2815
2816void MacroAssembler::CmpObjectType(Register heap_object,
2817                                   InstanceType type,
2818                                   Register map) {
2819  movq(map, FieldOperand(heap_object, HeapObject::kMapOffset));
2820  CmpInstanceType(map, type);
2821}
2822
2823
2824void MacroAssembler::CmpInstanceType(Register map, InstanceType type) {
2825  cmpb(FieldOperand(map, Map::kInstanceTypeOffset),
2826       Immediate(static_cast<int8_t>(type)));
2827}
2828
2829
2830void MacroAssembler::CheckFastElements(Register map,
2831                                       Label* fail,
2832                                       Label::Distance distance) {
2833  STATIC_ASSERT(FAST_SMI_ELEMENTS == 0);
2834  STATIC_ASSERT(FAST_HOLEY_SMI_ELEMENTS == 1);
2835  STATIC_ASSERT(FAST_ELEMENTS == 2);
2836  STATIC_ASSERT(FAST_HOLEY_ELEMENTS == 3);
2837  cmpb(FieldOperand(map, Map::kBitField2Offset),
2838       Immediate(Map::kMaximumBitField2FastHoleyElementValue));
2839  j(above, fail, distance);
2840}
2841
2842
2843void MacroAssembler::CheckFastObjectElements(Register map,
2844                                             Label* fail,
2845                                             Label::Distance distance) {
2846  STATIC_ASSERT(FAST_SMI_ELEMENTS == 0);
2847  STATIC_ASSERT(FAST_HOLEY_SMI_ELEMENTS == 1);
2848  STATIC_ASSERT(FAST_ELEMENTS == 2);
2849  STATIC_ASSERT(FAST_HOLEY_ELEMENTS == 3);
2850  cmpb(FieldOperand(map, Map::kBitField2Offset),
2851       Immediate(Map::kMaximumBitField2FastHoleySmiElementValue));
2852  j(below_equal, fail, distance);
2853  cmpb(FieldOperand(map, Map::kBitField2Offset),
2854       Immediate(Map::kMaximumBitField2FastHoleyElementValue));
2855  j(above, fail, distance);
2856}
2857
2858
2859void MacroAssembler::CheckFastSmiElements(Register map,
2860                                          Label* fail,
2861                                          Label::Distance distance) {
2862  STATIC_ASSERT(FAST_SMI_ELEMENTS == 0);
2863  STATIC_ASSERT(FAST_HOLEY_SMI_ELEMENTS == 1);
2864  cmpb(FieldOperand(map, Map::kBitField2Offset),
2865       Immediate(Map::kMaximumBitField2FastHoleySmiElementValue));
2866  j(above, fail, distance);
2867}
2868
2869
2870void MacroAssembler::StoreNumberToDoubleElements(
2871    Register maybe_number,
2872    Register elements,
2873    Register index,
2874    XMMRegister xmm_scratch,
2875    Label* fail,
2876    int elements_offset) {
2877  Label smi_value, is_nan, maybe_nan, not_nan, have_double_value, done;
2878
2879  JumpIfSmi(maybe_number, &smi_value, Label::kNear);
2880
2881  CheckMap(maybe_number,
2882           isolate()->factory()->heap_number_map(),
2883           fail,
2884           DONT_DO_SMI_CHECK);
2885
2886  // Double value, canonicalize NaN.
2887  uint32_t offset = HeapNumber::kValueOffset + sizeof(kHoleNanLower32);
2888  cmpl(FieldOperand(maybe_number, offset),
2889       Immediate(kNaNOrInfinityLowerBoundUpper32));
2890  j(greater_equal, &maybe_nan, Label::kNear);
2891
2892  bind(&not_nan);
2893  movsd(xmm_scratch, FieldOperand(maybe_number, HeapNumber::kValueOffset));
2894  bind(&have_double_value);
2895  movsd(FieldOperand(elements, index, times_8,
2896                     FixedDoubleArray::kHeaderSize - elements_offset),
2897        xmm_scratch);
2898  jmp(&done);
2899
2900  bind(&maybe_nan);
2901  // Could be NaN or Infinity. If fraction is not zero, it's NaN, otherwise
2902  // it's an Infinity, and the non-NaN code path applies.
2903  j(greater, &is_nan, Label::kNear);
2904  cmpl(FieldOperand(maybe_number, HeapNumber::kValueOffset), Immediate(0));
2905  j(zero, &not_nan);
2906  bind(&is_nan);
2907  // Convert all NaNs to the same canonical NaN value when they are stored in
2908  // the double array.
2909  Set(kScratchRegister, BitCast<uint64_t>(
2910      FixedDoubleArray::canonical_not_the_hole_nan_as_double()));
2911  movq(xmm_scratch, kScratchRegister);
2912  jmp(&have_double_value, Label::kNear);
2913
2914  bind(&smi_value);
2915  // Value is a smi. convert to a double and store.
2916  // Preserve original value.
2917  SmiToInteger32(kScratchRegister, maybe_number);
2918  cvtlsi2sd(xmm_scratch, kScratchRegister);
2919  movsd(FieldOperand(elements, index, times_8,
2920                     FixedDoubleArray::kHeaderSize - elements_offset),
2921        xmm_scratch);
2922  bind(&done);
2923}
2924
2925
2926void MacroAssembler::CompareMap(Register obj,
2927                                Handle<Map> map,
2928                                Label* early_success) {
2929  Cmp(FieldOperand(obj, HeapObject::kMapOffset), map);
2930}
2931
2932
2933void MacroAssembler::CheckMap(Register obj,
2934                              Handle<Map> map,
2935                              Label* fail,
2936                              SmiCheckType smi_check_type) {
2937  if (smi_check_type == DO_SMI_CHECK) {
2938    JumpIfSmi(obj, fail);
2939  }
2940
2941  Label success;
2942  CompareMap(obj, map, &success);
2943  j(not_equal, fail);
2944  bind(&success);
2945}
2946
2947
2948void MacroAssembler::ClampUint8(Register reg) {
2949  Label done;
2950  testl(reg, Immediate(0xFFFFFF00));
2951  j(zero, &done, Label::kNear);
2952  setcc(negative, reg);  // 1 if negative, 0 if positive.
2953  decb(reg);  // 0 if negative, 255 if positive.
2954  bind(&done);
2955}
2956
2957
2958void MacroAssembler::ClampDoubleToUint8(XMMRegister input_reg,
2959                                        XMMRegister temp_xmm_reg,
2960                                        Register result_reg) {
2961  Label done;
2962  Label conv_failure;
2963  xorps(temp_xmm_reg, temp_xmm_reg);
2964  cvtsd2si(result_reg, input_reg);
2965  testl(result_reg, Immediate(0xFFFFFF00));
2966  j(zero, &done, Label::kNear);
2967  cmpl(result_reg, Immediate(0x80000000));
2968  j(equal, &conv_failure, Label::kNear);
2969  movl(result_reg, Immediate(0));
2970  setcc(above, result_reg);
2971  subl(result_reg, Immediate(1));
2972  andl(result_reg, Immediate(255));
2973  jmp(&done, Label::kNear);
2974  bind(&conv_failure);
2975  Set(result_reg, 0);
2976  ucomisd(input_reg, temp_xmm_reg);
2977  j(below, &done, Label::kNear);
2978  Set(result_reg, 255);
2979  bind(&done);
2980}
2981
2982
2983void MacroAssembler::LoadUint32(XMMRegister dst,
2984                                Register src,
2985                                XMMRegister scratch) {
2986  if (FLAG_debug_code) {
2987    cmpq(src, Immediate(0xffffffff));
2988    Assert(below_equal, kInputGPRIsExpectedToHaveUpper32Cleared);
2989  }
2990  cvtqsi2sd(dst, src);
2991}
2992
2993
2994void MacroAssembler::LoadInstanceDescriptors(Register map,
2995                                             Register descriptors) {
2996  movq(descriptors, FieldOperand(map, Map::kDescriptorsOffset));
2997}
2998
2999
3000void MacroAssembler::NumberOfOwnDescriptors(Register dst, Register map) {
3001  movq(dst, FieldOperand(map, Map::kBitField3Offset));
3002  DecodeField<Map::NumberOfOwnDescriptorsBits>(dst);
3003}
3004
3005
3006void MacroAssembler::EnumLength(Register dst, Register map) {
3007  STATIC_ASSERT(Map::EnumLengthBits::kShift == 0);
3008  movq(dst, FieldOperand(map, Map::kBitField3Offset));
3009  Move(kScratchRegister, Smi::FromInt(Map::EnumLengthBits::kMask));
3010  and_(dst, kScratchRegister);
3011}
3012
3013
3014void MacroAssembler::DispatchMap(Register obj,
3015                                 Register unused,
3016                                 Handle<Map> map,
3017                                 Handle<Code> success,
3018                                 SmiCheckType smi_check_type) {
3019  Label fail;
3020  if (smi_check_type == DO_SMI_CHECK) {
3021    JumpIfSmi(obj, &fail);
3022  }
3023  Cmp(FieldOperand(obj, HeapObject::kMapOffset), map);
3024  j(equal, success, RelocInfo::CODE_TARGET);
3025
3026  bind(&fail);
3027}
3028
3029
3030void MacroAssembler::AssertNumber(Register object) {
3031  if (emit_debug_code()) {
3032    Label ok;
3033    Condition is_smi = CheckSmi(object);
3034    j(is_smi, &ok, Label::kNear);
3035    Cmp(FieldOperand(object, HeapObject::kMapOffset),
3036        isolate()->factory()->heap_number_map());
3037    Check(equal, kOperandIsNotANumber);
3038    bind(&ok);
3039  }
3040}
3041
3042
3043void MacroAssembler::AssertNotSmi(Register object) {
3044  if (emit_debug_code()) {
3045    Condition is_smi = CheckSmi(object);
3046    Check(NegateCondition(is_smi), kOperandIsASmi);
3047  }
3048}
3049
3050
3051void MacroAssembler::AssertSmi(Register object) {
3052  if (emit_debug_code()) {
3053    Condition is_smi = CheckSmi(object);
3054    Check(is_smi, kOperandIsNotASmi);
3055  }
3056}
3057
3058
3059void MacroAssembler::AssertSmi(const Operand& object) {
3060  if (emit_debug_code()) {
3061    Condition is_smi = CheckSmi(object);
3062    Check(is_smi, kOperandIsNotASmi);
3063  }
3064}
3065
3066
3067void MacroAssembler::AssertZeroExtended(Register int32_register) {
3068  if (emit_debug_code()) {
3069    ASSERT(!int32_register.is(kScratchRegister));
3070    movq(kScratchRegister, 0x100000000l, RelocInfo::NONE64);
3071    cmpq(kScratchRegister, int32_register);
3072    Check(above_equal, k32BitValueInRegisterIsNotZeroExtended);
3073  }
3074}
3075
3076
3077void MacroAssembler::AssertString(Register object) {
3078  if (emit_debug_code()) {
3079    testb(object, Immediate(kSmiTagMask));
3080    Check(not_equal, kOperandIsASmiAndNotAString);
3081    push(object);
3082    movq(object, FieldOperand(object, HeapObject::kMapOffset));
3083    CmpInstanceType(object, FIRST_NONSTRING_TYPE);
3084    pop(object);
3085    Check(below, kOperandIsNotAString);
3086  }
3087}
3088
3089
3090void MacroAssembler::AssertName(Register object) {
3091  if (emit_debug_code()) {
3092    testb(object, Immediate(kSmiTagMask));
3093    Check(not_equal, kOperandIsASmiAndNotAName);
3094    push(object);
3095    movq(object, FieldOperand(object, HeapObject::kMapOffset));
3096    CmpInstanceType(object, LAST_NAME_TYPE);
3097    pop(object);
3098    Check(below_equal, kOperandIsNotAName);
3099  }
3100}
3101
3102
3103void MacroAssembler::AssertRootValue(Register src,
3104                                     Heap::RootListIndex root_value_index,
3105                                     BailoutReason reason) {
3106  if (emit_debug_code()) {
3107    ASSERT(!src.is(kScratchRegister));
3108    LoadRoot(kScratchRegister, root_value_index);
3109    cmpq(src, kScratchRegister);
3110    Check(equal, reason);
3111  }
3112}
3113
3114
3115
3116Condition MacroAssembler::IsObjectStringType(Register heap_object,
3117                                             Register map,
3118                                             Register instance_type) {
3119  movq(map, FieldOperand(heap_object, HeapObject::kMapOffset));
3120  movzxbl(instance_type, FieldOperand(map, Map::kInstanceTypeOffset));
3121  STATIC_ASSERT(kNotStringTag != 0);
3122  testb(instance_type, Immediate(kIsNotStringMask));
3123  return zero;
3124}
3125
3126
3127Condition MacroAssembler::IsObjectNameType(Register heap_object,
3128                                           Register map,
3129                                           Register instance_type) {
3130  movq(map, FieldOperand(heap_object, HeapObject::kMapOffset));
3131  movzxbl(instance_type, FieldOperand(map, Map::kInstanceTypeOffset));
3132  cmpb(instance_type, Immediate(static_cast<uint8_t>(LAST_NAME_TYPE)));
3133  return below_equal;
3134}
3135
3136
3137void MacroAssembler::TryGetFunctionPrototype(Register function,
3138                                             Register result,
3139                                             Label* miss,
3140                                             bool miss_on_bound_function) {
3141  // Check that the receiver isn't a smi.
3142  testl(function, Immediate(kSmiTagMask));
3143  j(zero, miss);
3144
3145  // Check that the function really is a function.
3146  CmpObjectType(function, JS_FUNCTION_TYPE, result);
3147  j(not_equal, miss);
3148
3149  if (miss_on_bound_function) {
3150    movq(kScratchRegister,
3151         FieldOperand(function, JSFunction::kSharedFunctionInfoOffset));
3152    // It's not smi-tagged (stored in the top half of a smi-tagged 8-byte
3153    // field).
3154    TestBit(FieldOperand(kScratchRegister,
3155                         SharedFunctionInfo::kCompilerHintsOffset),
3156            SharedFunctionInfo::kBoundFunction);
3157    j(not_zero, miss);
3158  }
3159
3160  // Make sure that the function has an instance prototype.
3161  Label non_instance;
3162  testb(FieldOperand(result, Map::kBitFieldOffset),
3163        Immediate(1 << Map::kHasNonInstancePrototype));
3164  j(not_zero, &non_instance, Label::kNear);
3165
3166  // Get the prototype or initial map from the function.
3167  movq(result,
3168       FieldOperand(function, JSFunction::kPrototypeOrInitialMapOffset));
3169
3170  // If the prototype or initial map is the hole, don't return it and
3171  // simply miss the cache instead. This will allow us to allocate a
3172  // prototype object on-demand in the runtime system.
3173  CompareRoot(result, Heap::kTheHoleValueRootIndex);
3174  j(equal, miss);
3175
3176  // If the function does not have an initial map, we're done.
3177  Label done;
3178  CmpObjectType(result, MAP_TYPE, kScratchRegister);
3179  j(not_equal, &done, Label::kNear);
3180
3181  // Get the prototype from the initial map.
3182  movq(result, FieldOperand(result, Map::kPrototypeOffset));
3183  jmp(&done, Label::kNear);
3184
3185  // Non-instance prototype: Fetch prototype from constructor field
3186  // in initial map.
3187  bind(&non_instance);
3188  movq(result, FieldOperand(result, Map::kConstructorOffset));
3189
3190  // All done.
3191  bind(&done);
3192}
3193
3194
3195void MacroAssembler::SetCounter(StatsCounter* counter, int value) {
3196  if (FLAG_native_code_counters && counter->Enabled()) {
3197    Operand counter_operand = ExternalOperand(ExternalReference(counter));
3198    movl(counter_operand, Immediate(value));
3199  }
3200}
3201
3202
3203void MacroAssembler::IncrementCounter(StatsCounter* counter, int value) {
3204  ASSERT(value > 0);
3205  if (FLAG_native_code_counters && counter->Enabled()) {
3206    Operand counter_operand = ExternalOperand(ExternalReference(counter));
3207    if (value == 1) {
3208      incl(counter_operand);
3209    } else {
3210      addl(counter_operand, Immediate(value));
3211    }
3212  }
3213}
3214
3215
3216void MacroAssembler::DecrementCounter(StatsCounter* counter, int value) {
3217  ASSERT(value > 0);
3218  if (FLAG_native_code_counters && counter->Enabled()) {
3219    Operand counter_operand = ExternalOperand(ExternalReference(counter));
3220    if (value == 1) {
3221      decl(counter_operand);
3222    } else {
3223      subl(counter_operand, Immediate(value));
3224    }
3225  }
3226}
3227
3228
3229#ifdef ENABLE_DEBUGGER_SUPPORT
3230void MacroAssembler::DebugBreak() {
3231  Set(rax, 0);  // No arguments.
3232  LoadAddress(rbx, ExternalReference(Runtime::kDebugBreak, isolate()));
3233  CEntryStub ces(1);
3234  ASSERT(AllowThisStubCall(&ces));
3235  Call(ces.GetCode(isolate()), RelocInfo::DEBUG_BREAK);
3236}
3237#endif  // ENABLE_DEBUGGER_SUPPORT
3238
3239
3240void MacroAssembler::SetCallKind(Register dst, CallKind call_kind) {
3241  // This macro takes the dst register to make the code more readable
3242  // at the call sites. However, the dst register has to be rcx to
3243  // follow the calling convention which requires the call type to be
3244  // in rcx.
3245  ASSERT(dst.is(rcx));
3246  if (call_kind == CALL_AS_FUNCTION) {
3247    LoadSmiConstant(dst, Smi::FromInt(1));
3248  } else {
3249    LoadSmiConstant(dst, Smi::FromInt(0));
3250  }
3251}
3252
3253
3254void MacroAssembler::InvokeCode(Register code,
3255                                const ParameterCount& expected,
3256                                const ParameterCount& actual,
3257                                InvokeFlag flag,
3258                                const CallWrapper& call_wrapper,
3259                                CallKind call_kind) {
3260  // You can't call a function without a valid frame.
3261  ASSERT(flag == JUMP_FUNCTION || has_frame());
3262
3263  Label done;
3264  bool definitely_mismatches = false;
3265  InvokePrologue(expected,
3266                 actual,
3267                 Handle<Code>::null(),
3268                 code,
3269                 &done,
3270                 &definitely_mismatches,
3271                 flag,
3272                 Label::kNear,
3273                 call_wrapper,
3274                 call_kind);
3275  if (!definitely_mismatches) {
3276    if (flag == CALL_FUNCTION) {
3277      call_wrapper.BeforeCall(CallSize(code));
3278      SetCallKind(rcx, call_kind);
3279      call(code);
3280      call_wrapper.AfterCall();
3281    } else {
3282      ASSERT(flag == JUMP_FUNCTION);
3283      SetCallKind(rcx, call_kind);
3284      jmp(code);
3285    }
3286    bind(&done);
3287  }
3288}
3289
3290
3291void MacroAssembler::InvokeCode(Handle<Code> code,
3292                                const ParameterCount& expected,
3293                                const ParameterCount& actual,
3294                                RelocInfo::Mode rmode,
3295                                InvokeFlag flag,
3296                                const CallWrapper& call_wrapper,
3297                                CallKind call_kind) {
3298  // You can't call a function without a valid frame.
3299  ASSERT(flag == JUMP_FUNCTION || has_frame());
3300
3301  Label done;
3302  bool definitely_mismatches = false;
3303  Register dummy = rax;
3304  InvokePrologue(expected,
3305                 actual,
3306                 code,
3307                 dummy,
3308                 &done,
3309                 &definitely_mismatches,
3310                 flag,
3311                 Label::kNear,
3312                 call_wrapper,
3313                 call_kind);
3314  if (!definitely_mismatches) {
3315    if (flag == CALL_FUNCTION) {
3316      call_wrapper.BeforeCall(CallSize(code));
3317      SetCallKind(rcx, call_kind);
3318      Call(code, rmode);
3319      call_wrapper.AfterCall();
3320    } else {
3321      ASSERT(flag == JUMP_FUNCTION);
3322      SetCallKind(rcx, call_kind);
3323      Jump(code, rmode);
3324    }
3325    bind(&done);
3326  }
3327}
3328
3329
3330void MacroAssembler::InvokeFunction(Register function,
3331                                    const ParameterCount& actual,
3332                                    InvokeFlag flag,
3333                                    const CallWrapper& call_wrapper,
3334                                    CallKind call_kind) {
3335  // You can't call a function without a valid frame.
3336  ASSERT(flag == JUMP_FUNCTION || has_frame());
3337
3338  ASSERT(function.is(rdi));
3339  movq(rdx, FieldOperand(function, JSFunction::kSharedFunctionInfoOffset));
3340  movq(rsi, FieldOperand(function, JSFunction::kContextOffset));
3341  movsxlq(rbx,
3342          FieldOperand(rdx, SharedFunctionInfo::kFormalParameterCountOffset));
3343  // Advances rdx to the end of the Code object header, to the start of
3344  // the executable code.
3345  movq(rdx, FieldOperand(rdi, JSFunction::kCodeEntryOffset));
3346
3347  ParameterCount expected(rbx);
3348  InvokeCode(rdx, expected, actual, flag, call_wrapper, call_kind);
3349}
3350
3351
3352void MacroAssembler::InvokeFunction(Handle<JSFunction> function,
3353                                    const ParameterCount& expected,
3354                                    const ParameterCount& actual,
3355                                    InvokeFlag flag,
3356                                    const CallWrapper& call_wrapper,
3357                                    CallKind call_kind) {
3358  // You can't call a function without a valid frame.
3359  ASSERT(flag == JUMP_FUNCTION || has_frame());
3360
3361  // Get the function and setup the context.
3362  LoadHeapObject(rdi, function);
3363  movq(rsi, FieldOperand(rdi, JSFunction::kContextOffset));
3364
3365  // We call indirectly through the code field in the function to
3366  // allow recompilation to take effect without changing any of the
3367  // call sites.
3368  movq(rdx, FieldOperand(rdi, JSFunction::kCodeEntryOffset));
3369  InvokeCode(rdx, expected, actual, flag, call_wrapper, call_kind);
3370}
3371
3372
3373void MacroAssembler::InvokePrologue(const ParameterCount& expected,
3374                                    const ParameterCount& actual,
3375                                    Handle<Code> code_constant,
3376                                    Register code_register,
3377                                    Label* done,
3378                                    bool* definitely_mismatches,
3379                                    InvokeFlag flag,
3380                                    Label::Distance near_jump,
3381                                    const CallWrapper& call_wrapper,
3382                                    CallKind call_kind) {
3383  bool definitely_matches = false;
3384  *definitely_mismatches = false;
3385  Label invoke;
3386  if (expected.is_immediate()) {
3387    ASSERT(actual.is_immediate());
3388    if (expected.immediate() == actual.immediate()) {
3389      definitely_matches = true;
3390    } else {
3391      Set(rax, actual.immediate());
3392      if (expected.immediate() ==
3393              SharedFunctionInfo::kDontAdaptArgumentsSentinel) {
3394        // Don't worry about adapting arguments for built-ins that
3395        // don't want that done. Skip adaption code by making it look
3396        // like we have a match between expected and actual number of
3397        // arguments.
3398        definitely_matches = true;
3399      } else {
3400        *definitely_mismatches = true;
3401        Set(rbx, expected.immediate());
3402      }
3403    }
3404  } else {
3405    if (actual.is_immediate()) {
3406      // Expected is in register, actual is immediate. This is the
3407      // case when we invoke function values without going through the
3408      // IC mechanism.
3409      cmpq(expected.reg(), Immediate(actual.immediate()));
3410      j(equal, &invoke, Label::kNear);
3411      ASSERT(expected.reg().is(rbx));
3412      Set(rax, actual.immediate());
3413    } else if (!expected.reg().is(actual.reg())) {
3414      // Both expected and actual are in (different) registers. This
3415      // is the case when we invoke functions using call and apply.
3416      cmpq(expected.reg(), actual.reg());
3417      j(equal, &invoke, Label::kNear);
3418      ASSERT(actual.reg().is(rax));
3419      ASSERT(expected.reg().is(rbx));
3420    }
3421  }
3422
3423  if (!definitely_matches) {
3424    Handle<Code> adaptor = isolate()->builtins()->ArgumentsAdaptorTrampoline();
3425    if (!code_constant.is_null()) {
3426      movq(rdx, code_constant, RelocInfo::EMBEDDED_OBJECT);
3427      addq(rdx, Immediate(Code::kHeaderSize - kHeapObjectTag));
3428    } else if (!code_register.is(rdx)) {
3429      movq(rdx, code_register);
3430    }
3431
3432    if (flag == CALL_FUNCTION) {
3433      call_wrapper.BeforeCall(CallSize(adaptor));
3434      SetCallKind(rcx, call_kind);
3435      Call(adaptor, RelocInfo::CODE_TARGET);
3436      call_wrapper.AfterCall();
3437      if (!*definitely_mismatches) {
3438        jmp(done, near_jump);
3439      }
3440    } else {
3441      SetCallKind(rcx, call_kind);
3442      Jump(adaptor, RelocInfo::CODE_TARGET);
3443    }
3444    bind(&invoke);
3445  }
3446}
3447
3448
3449void MacroAssembler::EnterFrame(StackFrame::Type type) {
3450  push(rbp);
3451  movq(rbp, rsp);
3452  push(rsi);  // Context.
3453  Push(Smi::FromInt(type));
3454  movq(kScratchRegister, CodeObject(), RelocInfo::EMBEDDED_OBJECT);
3455  push(kScratchRegister);
3456  if (emit_debug_code()) {
3457    movq(kScratchRegister,
3458         isolate()->factory()->undefined_value(),
3459         RelocInfo::EMBEDDED_OBJECT);
3460    cmpq(Operand(rsp, 0), kScratchRegister);
3461    Check(not_equal, kCodeObjectNotProperlyPatched);
3462  }
3463}
3464
3465
3466void MacroAssembler::LeaveFrame(StackFrame::Type type) {
3467  if (emit_debug_code()) {
3468    Move(kScratchRegister, Smi::FromInt(type));
3469    cmpq(Operand(rbp, StandardFrameConstants::kMarkerOffset), kScratchRegister);
3470    Check(equal, kStackFrameTypesMustMatch);
3471  }
3472  movq(rsp, rbp);
3473  pop(rbp);
3474}
3475
3476
3477void MacroAssembler::EnterExitFramePrologue(bool save_rax) {
3478  // Set up the frame structure on the stack.
3479  // All constants are relative to the frame pointer of the exit frame.
3480  ASSERT(ExitFrameConstants::kCallerSPDisplacement == +2 * kPointerSize);
3481  ASSERT(ExitFrameConstants::kCallerPCOffset == +1 * kPointerSize);
3482  ASSERT(ExitFrameConstants::kCallerFPOffset ==  0 * kPointerSize);
3483  push(rbp);
3484  movq(rbp, rsp);
3485
3486  // Reserve room for entry stack pointer and push the code object.
3487  ASSERT(ExitFrameConstants::kSPOffset == -1 * kPointerSize);
3488  push(Immediate(0));  // Saved entry sp, patched before call.
3489  movq(kScratchRegister, CodeObject(), RelocInfo::EMBEDDED_OBJECT);
3490  push(kScratchRegister);  // Accessed from EditFrame::code_slot.
3491
3492  // Save the frame pointer and the context in top.
3493  if (save_rax) {
3494    movq(r14, rax);  // Backup rax in callee-save register.
3495  }
3496
3497  Store(ExternalReference(Isolate::kCEntryFPAddress, isolate()), rbp);
3498  Store(ExternalReference(Isolate::kContextAddress, isolate()), rsi);
3499}
3500
3501
3502void MacroAssembler::EnterExitFrameEpilogue(int arg_stack_space,
3503                                            bool save_doubles) {
3504#ifdef _WIN64
3505  const int kShadowSpace = 4;
3506  arg_stack_space += kShadowSpace;
3507#endif
3508  // Optionally save all XMM registers.
3509  if (save_doubles) {
3510    int space = XMMRegister::kMaxNumRegisters * kDoubleSize +
3511        arg_stack_space * kPointerSize;
3512    subq(rsp, Immediate(space));
3513    int offset = -2 * kPointerSize;
3514    for (int i = 0; i < XMMRegister::NumAllocatableRegisters(); i++) {
3515      XMMRegister reg = XMMRegister::FromAllocationIndex(i);
3516      movsd(Operand(rbp, offset - ((i + 1) * kDoubleSize)), reg);
3517    }
3518  } else if (arg_stack_space > 0) {
3519    subq(rsp, Immediate(arg_stack_space * kPointerSize));
3520  }
3521
3522  // Get the required frame alignment for the OS.
3523  const int kFrameAlignment = OS::ActivationFrameAlignment();
3524  if (kFrameAlignment > 0) {
3525    ASSERT(IsPowerOf2(kFrameAlignment));
3526    ASSERT(is_int8(kFrameAlignment));
3527    and_(rsp, Immediate(-kFrameAlignment));
3528  }
3529
3530  // Patch the saved entry sp.
3531  movq(Operand(rbp, ExitFrameConstants::kSPOffset), rsp);
3532}
3533
3534
3535void MacroAssembler::EnterExitFrame(int arg_stack_space, bool save_doubles) {
3536  EnterExitFramePrologue(true);
3537
3538  // Set up argv in callee-saved register r15. It is reused in LeaveExitFrame,
3539  // so it must be retained across the C-call.
3540  int offset = StandardFrameConstants::kCallerSPOffset - kPointerSize;
3541  lea(r15, Operand(rbp, r14, times_pointer_size, offset));
3542
3543  EnterExitFrameEpilogue(arg_stack_space, save_doubles);
3544}
3545
3546
3547void MacroAssembler::EnterApiExitFrame(int arg_stack_space) {
3548  EnterExitFramePrologue(false);
3549  EnterExitFrameEpilogue(arg_stack_space, false);
3550}
3551
3552
3553void MacroAssembler::LeaveExitFrame(bool save_doubles) {
3554  // Registers:
3555  // r15 : argv
3556  if (save_doubles) {
3557    int offset = -2 * kPointerSize;
3558    for (int i = 0; i < XMMRegister::NumAllocatableRegisters(); i++) {
3559      XMMRegister reg = XMMRegister::FromAllocationIndex(i);
3560      movsd(reg, Operand(rbp, offset - ((i + 1) * kDoubleSize)));
3561    }
3562  }
3563  // Get the return address from the stack and restore the frame pointer.
3564  movq(rcx, Operand(rbp, 1 * kPointerSize));
3565  movq(rbp, Operand(rbp, 0 * kPointerSize));
3566
3567  // Drop everything up to and including the arguments and the receiver
3568  // from the caller stack.
3569  lea(rsp, Operand(r15, 1 * kPointerSize));
3570
3571  PushReturnAddressFrom(rcx);
3572
3573  LeaveExitFrameEpilogue();
3574}
3575
3576
3577void MacroAssembler::LeaveApiExitFrame() {
3578  movq(rsp, rbp);
3579  pop(rbp);
3580
3581  LeaveExitFrameEpilogue();
3582}
3583
3584
3585void MacroAssembler::LeaveExitFrameEpilogue() {
3586  // Restore current context from top and clear it in debug mode.
3587  ExternalReference context_address(Isolate::kContextAddress, isolate());
3588  Operand context_operand = ExternalOperand(context_address);
3589  movq(rsi, context_operand);
3590#ifdef DEBUG
3591  movq(context_operand, Immediate(0));
3592#endif
3593
3594  // Clear the top frame.
3595  ExternalReference c_entry_fp_address(Isolate::kCEntryFPAddress,
3596                                       isolate());
3597  Operand c_entry_fp_operand = ExternalOperand(c_entry_fp_address);
3598  movq(c_entry_fp_operand, Immediate(0));
3599}
3600
3601
3602void MacroAssembler::CheckAccessGlobalProxy(Register holder_reg,
3603                                            Register scratch,
3604                                            Label* miss) {
3605  Label same_contexts;
3606
3607  ASSERT(!holder_reg.is(scratch));
3608  ASSERT(!scratch.is(kScratchRegister));
3609  // Load current lexical context from the stack frame.
3610  movq(scratch, Operand(rbp, StandardFrameConstants::kContextOffset));
3611
3612  // When generating debug code, make sure the lexical context is set.
3613  if (emit_debug_code()) {
3614    cmpq(scratch, Immediate(0));
3615    Check(not_equal, kWeShouldNotHaveAnEmptyLexicalContext);
3616  }
3617  // Load the native context of the current context.
3618  int offset =
3619      Context::kHeaderSize + Context::GLOBAL_OBJECT_INDEX * kPointerSize;
3620  movq(scratch, FieldOperand(scratch, offset));
3621  movq(scratch, FieldOperand(scratch, GlobalObject::kNativeContextOffset));
3622
3623  // Check the context is a native context.
3624  if (emit_debug_code()) {
3625    Cmp(FieldOperand(scratch, HeapObject::kMapOffset),
3626        isolate()->factory()->native_context_map());
3627    Check(equal, kJSGlobalObjectNativeContextShouldBeANativeContext);
3628  }
3629
3630  // Check if both contexts are the same.
3631  cmpq(scratch, FieldOperand(holder_reg, JSGlobalProxy::kNativeContextOffset));
3632  j(equal, &same_contexts);
3633
3634  // Compare security tokens.
3635  // Check that the security token in the calling global object is
3636  // compatible with the security token in the receiving global
3637  // object.
3638
3639  // Check the context is a native context.
3640  if (emit_debug_code()) {
3641    // Preserve original value of holder_reg.
3642    push(holder_reg);
3643    movq(holder_reg,
3644         FieldOperand(holder_reg, JSGlobalProxy::kNativeContextOffset));
3645    CompareRoot(holder_reg, Heap::kNullValueRootIndex);
3646    Check(not_equal, kJSGlobalProxyContextShouldNotBeNull);
3647
3648    // Read the first word and compare to native_context_map(),
3649    movq(holder_reg, FieldOperand(holder_reg, HeapObject::kMapOffset));
3650    CompareRoot(holder_reg, Heap::kNativeContextMapRootIndex);
3651    Check(equal, kJSGlobalObjectNativeContextShouldBeANativeContext);
3652    pop(holder_reg);
3653  }
3654
3655  movq(kScratchRegister,
3656       FieldOperand(holder_reg, JSGlobalProxy::kNativeContextOffset));
3657  int token_offset =
3658      Context::kHeaderSize + Context::SECURITY_TOKEN_INDEX * kPointerSize;
3659  movq(scratch, FieldOperand(scratch, token_offset));
3660  cmpq(scratch, FieldOperand(kScratchRegister, token_offset));
3661  j(not_equal, miss);
3662
3663  bind(&same_contexts);
3664}
3665
3666
3667void MacroAssembler::GetNumberHash(Register r0, Register scratch) {
3668  // First of all we assign the hash seed to scratch.
3669  LoadRoot(scratch, Heap::kHashSeedRootIndex);
3670  SmiToInteger32(scratch, scratch);
3671
3672  // Xor original key with a seed.
3673  xorl(r0, scratch);
3674
3675  // Compute the hash code from the untagged key.  This must be kept in sync
3676  // with ComputeIntegerHash in utils.h.
3677  //
3678  // hash = ~hash + (hash << 15);
3679  movl(scratch, r0);
3680  notl(r0);
3681  shll(scratch, Immediate(15));
3682  addl(r0, scratch);
3683  // hash = hash ^ (hash >> 12);
3684  movl(scratch, r0);
3685  shrl(scratch, Immediate(12));
3686  xorl(r0, scratch);
3687  // hash = hash + (hash << 2);
3688  leal(r0, Operand(r0, r0, times_4, 0));
3689  // hash = hash ^ (hash >> 4);
3690  movl(scratch, r0);
3691  shrl(scratch, Immediate(4));
3692  xorl(r0, scratch);
3693  // hash = hash * 2057;
3694  imull(r0, r0, Immediate(2057));
3695  // hash = hash ^ (hash >> 16);
3696  movl(scratch, r0);
3697  shrl(scratch, Immediate(16));
3698  xorl(r0, scratch);
3699}
3700
3701
3702
3703void MacroAssembler::LoadFromNumberDictionary(Label* miss,
3704                                              Register elements,
3705                                              Register key,
3706                                              Register r0,
3707                                              Register r1,
3708                                              Register r2,
3709                                              Register result) {
3710  // Register use:
3711  //
3712  // elements - holds the slow-case elements of the receiver on entry.
3713  //            Unchanged unless 'result' is the same register.
3714  //
3715  // key      - holds the smi key on entry.
3716  //            Unchanged unless 'result' is the same register.
3717  //
3718  // Scratch registers:
3719  //
3720  // r0 - holds the untagged key on entry and holds the hash once computed.
3721  //
3722  // r1 - used to hold the capacity mask of the dictionary
3723  //
3724  // r2 - used for the index into the dictionary.
3725  //
3726  // result - holds the result on exit if the load succeeded.
3727  //          Allowed to be the same as 'key' or 'result'.
3728  //          Unchanged on bailout so 'key' or 'result' can be used
3729  //          in further computation.
3730
3731  Label done;
3732
3733  GetNumberHash(r0, r1);
3734
3735  // Compute capacity mask.
3736  SmiToInteger32(r1, FieldOperand(elements,
3737                                  SeededNumberDictionary::kCapacityOffset));
3738  decl(r1);
3739
3740  // Generate an unrolled loop that performs a few probes before giving up.
3741  const int kProbes = 4;
3742  for (int i = 0; i < kProbes; i++) {
3743    // Use r2 for index calculations and keep the hash intact in r0.
3744    movq(r2, r0);
3745    // Compute the masked index: (hash + i + i * i) & mask.
3746    if (i > 0) {
3747      addl(r2, Immediate(SeededNumberDictionary::GetProbeOffset(i)));
3748    }
3749    and_(r2, r1);
3750
3751    // Scale the index by multiplying by the entry size.
3752    ASSERT(SeededNumberDictionary::kEntrySize == 3);
3753    lea(r2, Operand(r2, r2, times_2, 0));  // r2 = r2 * 3
3754
3755    // Check if the key matches.
3756    cmpq(key, FieldOperand(elements,
3757                           r2,
3758                           times_pointer_size,
3759                           SeededNumberDictionary::kElementsStartOffset));
3760    if (i != (kProbes - 1)) {
3761      j(equal, &done);
3762    } else {
3763      j(not_equal, miss);
3764    }
3765  }
3766
3767  bind(&done);
3768  // Check that the value is a normal propety.
3769  const int kDetailsOffset =
3770      SeededNumberDictionary::kElementsStartOffset + 2 * kPointerSize;
3771  ASSERT_EQ(NORMAL, 0);
3772  Test(FieldOperand(elements, r2, times_pointer_size, kDetailsOffset),
3773       Smi::FromInt(PropertyDetails::TypeField::kMask));
3774  j(not_zero, miss);
3775
3776  // Get the value at the masked, scaled index.
3777  const int kValueOffset =
3778      SeededNumberDictionary::kElementsStartOffset + kPointerSize;
3779  movq(result, FieldOperand(elements, r2, times_pointer_size, kValueOffset));
3780}
3781
3782
3783void MacroAssembler::LoadAllocationTopHelper(Register result,
3784                                             Register scratch,
3785                                             AllocationFlags flags) {
3786  ExternalReference allocation_top =
3787      AllocationUtils::GetAllocationTopReference(isolate(), flags);
3788
3789  // Just return if allocation top is already known.
3790  if ((flags & RESULT_CONTAINS_TOP) != 0) {
3791    // No use of scratch if allocation top is provided.
3792    ASSERT(!scratch.is_valid());
3793#ifdef DEBUG
3794    // Assert that result actually contains top on entry.
3795    Operand top_operand = ExternalOperand(allocation_top);
3796    cmpq(result, top_operand);
3797    Check(equal, kUnexpectedAllocationTop);
3798#endif
3799    return;
3800  }
3801
3802  // Move address of new object to result. Use scratch register if available,
3803  // and keep address in scratch until call to UpdateAllocationTopHelper.
3804  if (scratch.is_valid()) {
3805    LoadAddress(scratch, allocation_top);
3806    movq(result, Operand(scratch, 0));
3807  } else {
3808    Load(result, allocation_top);
3809  }
3810}
3811
3812
3813void MacroAssembler::UpdateAllocationTopHelper(Register result_end,
3814                                               Register scratch,
3815                                               AllocationFlags flags) {
3816  if (emit_debug_code()) {
3817    testq(result_end, Immediate(kObjectAlignmentMask));
3818    Check(zero, kUnalignedAllocationInNewSpace);
3819  }
3820
3821  ExternalReference allocation_top =
3822      AllocationUtils::GetAllocationTopReference(isolate(), flags);
3823
3824  // Update new top.
3825  if (scratch.is_valid()) {
3826    // Scratch already contains address of allocation top.
3827    movq(Operand(scratch, 0), result_end);
3828  } else {
3829    Store(allocation_top, result_end);
3830  }
3831}
3832
3833
3834void MacroAssembler::Allocate(int object_size,
3835                              Register result,
3836                              Register result_end,
3837                              Register scratch,
3838                              Label* gc_required,
3839                              AllocationFlags flags) {
3840  ASSERT((flags & (RESULT_CONTAINS_TOP | SIZE_IN_WORDS)) == 0);
3841  ASSERT(object_size <= Page::kMaxNonCodeHeapObjectSize);
3842  if (!FLAG_inline_new) {
3843    if (emit_debug_code()) {
3844      // Trash the registers to simulate an allocation failure.
3845      movl(result, Immediate(0x7091));
3846      if (result_end.is_valid()) {
3847        movl(result_end, Immediate(0x7191));
3848      }
3849      if (scratch.is_valid()) {
3850        movl(scratch, Immediate(0x7291));
3851      }
3852    }
3853    jmp(gc_required);
3854    return;
3855  }
3856  ASSERT(!result.is(result_end));
3857
3858  // Load address of new object into result.
3859  LoadAllocationTopHelper(result, scratch, flags);
3860
3861  // Align the next allocation. Storing the filler map without checking top is
3862  // safe in new-space because the limit of the heap is aligned there.
3863  if (((flags & DOUBLE_ALIGNMENT) != 0) && FLAG_debug_code) {
3864    testq(result, Immediate(kDoubleAlignmentMask));
3865    Check(zero, kAllocationIsNotDoubleAligned);
3866  }
3867
3868  // Calculate new top and bail out if new space is exhausted.
3869  ExternalReference allocation_limit =
3870      AllocationUtils::GetAllocationLimitReference(isolate(), flags);
3871
3872  Register top_reg = result_end.is_valid() ? result_end : result;
3873
3874  if (!top_reg.is(result)) {
3875    movq(top_reg, result);
3876  }
3877  addq(top_reg, Immediate(object_size));
3878  j(carry, gc_required);
3879  Operand limit_operand = ExternalOperand(allocation_limit);
3880  cmpq(top_reg, limit_operand);
3881  j(above, gc_required);
3882
3883  // Update allocation top.
3884  UpdateAllocationTopHelper(top_reg, scratch, flags);
3885
3886  bool tag_result = (flags & TAG_OBJECT) != 0;
3887  if (top_reg.is(result)) {
3888    if (tag_result) {
3889      subq(result, Immediate(object_size - kHeapObjectTag));
3890    } else {
3891      subq(result, Immediate(object_size));
3892    }
3893  } else if (tag_result) {
3894    // Tag the result if requested.
3895    ASSERT(kHeapObjectTag == 1);
3896    incq(result);
3897  }
3898}
3899
3900
3901void MacroAssembler::Allocate(int header_size,
3902                              ScaleFactor element_size,
3903                              Register element_count,
3904                              Register result,
3905                              Register result_end,
3906                              Register scratch,
3907                              Label* gc_required,
3908                              AllocationFlags flags) {
3909  ASSERT((flags & SIZE_IN_WORDS) == 0);
3910  lea(result_end, Operand(element_count, element_size, header_size));
3911  Allocate(result_end, result, result_end, scratch, gc_required, flags);
3912}
3913
3914
3915void MacroAssembler::Allocate(Register object_size,
3916                              Register result,
3917                              Register result_end,
3918                              Register scratch,
3919                              Label* gc_required,
3920                              AllocationFlags flags) {
3921  ASSERT((flags & SIZE_IN_WORDS) == 0);
3922  if (!FLAG_inline_new) {
3923    if (emit_debug_code()) {
3924      // Trash the registers to simulate an allocation failure.
3925      movl(result, Immediate(0x7091));
3926      movl(result_end, Immediate(0x7191));
3927      if (scratch.is_valid()) {
3928        movl(scratch, Immediate(0x7291));
3929      }
3930      // object_size is left unchanged by this function.
3931    }
3932    jmp(gc_required);
3933    return;
3934  }
3935  ASSERT(!result.is(result_end));
3936
3937  // Load address of new object into result.
3938  LoadAllocationTopHelper(result, scratch, flags);
3939
3940  // Align the next allocation. Storing the filler map without checking top is
3941  // safe in new-space because the limit of the heap is aligned there.
3942  if (((flags & DOUBLE_ALIGNMENT) != 0) && FLAG_debug_code) {
3943    testq(result, Immediate(kDoubleAlignmentMask));
3944    Check(zero, kAllocationIsNotDoubleAligned);
3945  }
3946
3947  // Calculate new top and bail out if new space is exhausted.
3948  ExternalReference allocation_limit =
3949      AllocationUtils::GetAllocationLimitReference(isolate(), flags);
3950  if (!object_size.is(result_end)) {
3951    movq(result_end, object_size);
3952  }
3953  addq(result_end, result);
3954  j(carry, gc_required);
3955  Operand limit_operand = ExternalOperand(allocation_limit);
3956  cmpq(result_end, limit_operand);
3957  j(above, gc_required);
3958
3959  // Update allocation top.
3960  UpdateAllocationTopHelper(result_end, scratch, flags);
3961
3962  // Tag the result if requested.
3963  if ((flags & TAG_OBJECT) != 0) {
3964    addq(result, Immediate(kHeapObjectTag));
3965  }
3966}
3967
3968
3969void MacroAssembler::UndoAllocationInNewSpace(Register object) {
3970  ExternalReference new_space_allocation_top =
3971      ExternalReference::new_space_allocation_top_address(isolate());
3972
3973  // Make sure the object has no tag before resetting top.
3974  and_(object, Immediate(~kHeapObjectTagMask));
3975  Operand top_operand = ExternalOperand(new_space_allocation_top);
3976#ifdef DEBUG
3977  cmpq(object, top_operand);
3978  Check(below, kUndoAllocationOfNonAllocatedMemory);
3979#endif
3980  movq(top_operand, object);
3981}
3982
3983
3984void MacroAssembler::AllocateHeapNumber(Register result,
3985                                        Register scratch,
3986                                        Label* gc_required) {
3987  // Allocate heap number in new space.
3988  Allocate(HeapNumber::kSize, result, scratch, no_reg, gc_required, TAG_OBJECT);
3989
3990  // Set the map.
3991  LoadRoot(kScratchRegister, Heap::kHeapNumberMapRootIndex);
3992  movq(FieldOperand(result, HeapObject::kMapOffset), kScratchRegister);
3993}
3994
3995
3996void MacroAssembler::AllocateTwoByteString(Register result,
3997                                           Register length,
3998                                           Register scratch1,
3999                                           Register scratch2,
4000                                           Register scratch3,
4001                                           Label* gc_required) {
4002  // Calculate the number of bytes needed for the characters in the string while
4003  // observing object alignment.
4004  const int kHeaderAlignment = SeqTwoByteString::kHeaderSize &
4005                               kObjectAlignmentMask;
4006  ASSERT(kShortSize == 2);
4007  // scratch1 = length * 2 + kObjectAlignmentMask.
4008  lea(scratch1, Operand(length, length, times_1, kObjectAlignmentMask +
4009                kHeaderAlignment));
4010  and_(scratch1, Immediate(~kObjectAlignmentMask));
4011  if (kHeaderAlignment > 0) {
4012    subq(scratch1, Immediate(kHeaderAlignment));
4013  }
4014
4015  // Allocate two byte string in new space.
4016  Allocate(SeqTwoByteString::kHeaderSize,
4017           times_1,
4018           scratch1,
4019           result,
4020           scratch2,
4021           scratch3,
4022           gc_required,
4023           TAG_OBJECT);
4024
4025  // Set the map, length and hash field.
4026  LoadRoot(kScratchRegister, Heap::kStringMapRootIndex);
4027  movq(FieldOperand(result, HeapObject::kMapOffset), kScratchRegister);
4028  Integer32ToSmi(scratch1, length);
4029  movq(FieldOperand(result, String::kLengthOffset), scratch1);
4030  movq(FieldOperand(result, String::kHashFieldOffset),
4031       Immediate(String::kEmptyHashField));
4032}
4033
4034
4035void MacroAssembler::AllocateAsciiString(Register result,
4036                                         Register length,
4037                                         Register scratch1,
4038                                         Register scratch2,
4039                                         Register scratch3,
4040                                         Label* gc_required) {
4041  // Calculate the number of bytes needed for the characters in the string while
4042  // observing object alignment.
4043  const int kHeaderAlignment = SeqOneByteString::kHeaderSize &
4044                               kObjectAlignmentMask;
4045  movl(scratch1, length);
4046  ASSERT(kCharSize == 1);
4047  addq(scratch1, Immediate(kObjectAlignmentMask + kHeaderAlignment));
4048  and_(scratch1, Immediate(~kObjectAlignmentMask));
4049  if (kHeaderAlignment > 0) {
4050    subq(scratch1, Immediate(kHeaderAlignment));
4051  }
4052
4053  // Allocate ASCII string in new space.
4054  Allocate(SeqOneByteString::kHeaderSize,
4055           times_1,
4056           scratch1,
4057           result,
4058           scratch2,
4059           scratch3,
4060           gc_required,
4061           TAG_OBJECT);
4062
4063  // Set the map, length and hash field.
4064  LoadRoot(kScratchRegister, Heap::kAsciiStringMapRootIndex);
4065  movq(FieldOperand(result, HeapObject::kMapOffset), kScratchRegister);
4066  Integer32ToSmi(scratch1, length);
4067  movq(FieldOperand(result, String::kLengthOffset), scratch1);
4068  movq(FieldOperand(result, String::kHashFieldOffset),
4069       Immediate(String::kEmptyHashField));
4070}
4071
4072
4073void MacroAssembler::AllocateTwoByteConsString(Register result,
4074                                        Register scratch1,
4075                                        Register scratch2,
4076                                        Label* gc_required) {
4077  // Allocate heap number in new space.
4078  Allocate(ConsString::kSize, result, scratch1, scratch2, gc_required,
4079           TAG_OBJECT);
4080
4081  // Set the map. The other fields are left uninitialized.
4082  LoadRoot(kScratchRegister, Heap::kConsStringMapRootIndex);
4083  movq(FieldOperand(result, HeapObject::kMapOffset), kScratchRegister);
4084}
4085
4086
4087void MacroAssembler::AllocateAsciiConsString(Register result,
4088                                             Register scratch1,
4089                                             Register scratch2,
4090                                             Label* gc_required) {
4091  Label allocate_new_space, install_map;
4092  AllocationFlags flags = TAG_OBJECT;
4093
4094  ExternalReference high_promotion_mode = ExternalReference::
4095      new_space_high_promotion_mode_active_address(isolate());
4096
4097  Load(scratch1, high_promotion_mode);
4098  testb(scratch1, Immediate(1));
4099  j(zero, &allocate_new_space);
4100  Allocate(ConsString::kSize,
4101           result,
4102           scratch1,
4103           scratch2,
4104           gc_required,
4105           static_cast<AllocationFlags>(flags | PRETENURE_OLD_POINTER_SPACE));
4106
4107  jmp(&install_map);
4108
4109  bind(&allocate_new_space);
4110  Allocate(ConsString::kSize,
4111           result,
4112           scratch1,
4113           scratch2,
4114           gc_required,
4115           flags);
4116
4117  bind(&install_map);
4118
4119  // Set the map. The other fields are left uninitialized.
4120  LoadRoot(kScratchRegister, Heap::kConsAsciiStringMapRootIndex);
4121  movq(FieldOperand(result, HeapObject::kMapOffset), kScratchRegister);
4122}
4123
4124
4125void MacroAssembler::AllocateTwoByteSlicedString(Register result,
4126                                          Register scratch1,
4127                                          Register scratch2,
4128                                          Label* gc_required) {
4129  // Allocate heap number in new space.
4130  Allocate(SlicedString::kSize, result, scratch1, scratch2, gc_required,
4131           TAG_OBJECT);
4132
4133  // Set the map. The other fields are left uninitialized.
4134  LoadRoot(kScratchRegister, Heap::kSlicedStringMapRootIndex);
4135  movq(FieldOperand(result, HeapObject::kMapOffset), kScratchRegister);
4136}
4137
4138
4139void MacroAssembler::AllocateAsciiSlicedString(Register result,
4140                                               Register scratch1,
4141                                               Register scratch2,
4142                                               Label* gc_required) {
4143  // Allocate heap number in new space.
4144  Allocate(SlicedString::kSize, result, scratch1, scratch2, gc_required,
4145           TAG_OBJECT);
4146
4147  // Set the map. The other fields are left uninitialized.
4148  LoadRoot(kScratchRegister, Heap::kSlicedAsciiStringMapRootIndex);
4149  movq(FieldOperand(result, HeapObject::kMapOffset), kScratchRegister);
4150}
4151
4152
4153// Copy memory, byte-by-byte, from source to destination.  Not optimized for
4154// long or aligned copies.  The contents of scratch and length are destroyed.
4155// Destination is incremented by length, source, length and scratch are
4156// clobbered.
4157// A simpler loop is faster on small copies, but slower on large ones.
4158// The cld() instruction must have been emitted, to set the direction flag(),
4159// before calling this function.
4160void MacroAssembler::CopyBytes(Register destination,
4161                               Register source,
4162                               Register length,
4163                               int min_length,
4164                               Register scratch) {
4165  ASSERT(min_length >= 0);
4166  if (emit_debug_code()) {
4167    cmpl(length, Immediate(min_length));
4168    Assert(greater_equal, kInvalidMinLength);
4169  }
4170  Label loop, done, short_string, short_loop;
4171
4172  const int kLongStringLimit = 20;
4173  if (min_length <= kLongStringLimit) {
4174    cmpl(length, Immediate(kLongStringLimit));
4175    j(less_equal, &short_string);
4176  }
4177
4178  ASSERT(source.is(rsi));
4179  ASSERT(destination.is(rdi));
4180  ASSERT(length.is(rcx));
4181
4182  // Because source is 8-byte aligned in our uses of this function,
4183  // we keep source aligned for the rep movs operation by copying the odd bytes
4184  // at the end of the ranges.
4185  movq(scratch, length);
4186  shrl(length, Immediate(kPointerSizeLog2));
4187  repmovsq();
4188  // Move remaining bytes of length.
4189  andl(scratch, Immediate(kPointerSize - 1));
4190  movq(length, Operand(source, scratch, times_1, -kPointerSize));
4191  movq(Operand(destination, scratch, times_1, -kPointerSize), length);
4192  addq(destination, scratch);
4193
4194  if (min_length <= kLongStringLimit) {
4195    jmp(&done);
4196
4197    bind(&short_string);
4198    if (min_length == 0) {
4199      testl(length, length);
4200      j(zero, &done);
4201    }
4202    lea(scratch, Operand(destination, length, times_1, 0));
4203
4204    bind(&short_loop);
4205    movb(length, Operand(source, 0));
4206    movb(Operand(destination, 0), length);
4207    incq(source);
4208    incq(destination);
4209    cmpq(destination, scratch);
4210    j(not_equal, &short_loop);
4211
4212    bind(&done);
4213  }
4214}
4215
4216
4217void MacroAssembler::InitializeFieldsWithFiller(Register start_offset,
4218                                                Register end_offset,
4219                                                Register filler) {
4220  Label loop, entry;
4221  jmp(&entry);
4222  bind(&loop);
4223  movq(Operand(start_offset, 0), filler);
4224  addq(start_offset, Immediate(kPointerSize));
4225  bind(&entry);
4226  cmpq(start_offset, end_offset);
4227  j(less, &loop);
4228}
4229
4230
4231void MacroAssembler::LoadContext(Register dst, int context_chain_length) {
4232  if (context_chain_length > 0) {
4233    // Move up the chain of contexts to the context containing the slot.
4234    movq(dst, Operand(rsi, Context::SlotOffset(Context::PREVIOUS_INDEX)));
4235    for (int i = 1; i < context_chain_length; i++) {
4236      movq(dst, Operand(dst, Context::SlotOffset(Context::PREVIOUS_INDEX)));
4237    }
4238  } else {
4239    // Slot is in the current function context.  Move it into the
4240    // destination register in case we store into it (the write barrier
4241    // cannot be allowed to destroy the context in rsi).
4242    movq(dst, rsi);
4243  }
4244
4245  // We should not have found a with context by walking the context
4246  // chain (i.e., the static scope chain and runtime context chain do
4247  // not agree).  A variable occurring in such a scope should have
4248  // slot type LOOKUP and not CONTEXT.
4249  if (emit_debug_code()) {
4250    CompareRoot(FieldOperand(dst, HeapObject::kMapOffset),
4251                Heap::kWithContextMapRootIndex);
4252    Check(not_equal, kVariableResolvedToWithContext);
4253  }
4254}
4255
4256
4257void MacroAssembler::LoadTransitionedArrayMapConditional(
4258    ElementsKind expected_kind,
4259    ElementsKind transitioned_kind,
4260    Register map_in_out,
4261    Register scratch,
4262    Label* no_map_match) {
4263  // Load the global or builtins object from the current context.
4264  movq(scratch,
4265       Operand(rsi, Context::SlotOffset(Context::GLOBAL_OBJECT_INDEX)));
4266  movq(scratch, FieldOperand(scratch, GlobalObject::kNativeContextOffset));
4267
4268  // Check that the function's map is the same as the expected cached map.
4269  movq(scratch, Operand(scratch,
4270                        Context::SlotOffset(Context::JS_ARRAY_MAPS_INDEX)));
4271
4272  int offset = expected_kind * kPointerSize +
4273      FixedArrayBase::kHeaderSize;
4274  cmpq(map_in_out, FieldOperand(scratch, offset));
4275  j(not_equal, no_map_match);
4276
4277  // Use the transitioned cached map.
4278  offset = transitioned_kind * kPointerSize +
4279      FixedArrayBase::kHeaderSize;
4280  movq(map_in_out, FieldOperand(scratch, offset));
4281}
4282
4283
4284void MacroAssembler::LoadInitialArrayMap(
4285    Register function_in, Register scratch,
4286    Register map_out, bool can_have_holes) {
4287  ASSERT(!function_in.is(map_out));
4288  Label done;
4289  movq(map_out, FieldOperand(function_in,
4290                             JSFunction::kPrototypeOrInitialMapOffset));
4291  if (!FLAG_smi_only_arrays) {
4292    ElementsKind kind = can_have_holes ? FAST_HOLEY_ELEMENTS : FAST_ELEMENTS;
4293    LoadTransitionedArrayMapConditional(FAST_SMI_ELEMENTS,
4294                                        kind,
4295                                        map_out,
4296                                        scratch,
4297                                        &done);
4298  } else if (can_have_holes) {
4299    LoadTransitionedArrayMapConditional(FAST_SMI_ELEMENTS,
4300                                        FAST_HOLEY_SMI_ELEMENTS,
4301                                        map_out,
4302                                        scratch,
4303                                        &done);
4304  }
4305  bind(&done);
4306}
4307
4308#ifdef _WIN64
4309static const int kRegisterPassedArguments = 4;
4310#else
4311static const int kRegisterPassedArguments = 6;
4312#endif
4313
4314void MacroAssembler::LoadGlobalFunction(int index, Register function) {
4315  // Load the global or builtins object from the current context.
4316  movq(function,
4317       Operand(rsi, Context::SlotOffset(Context::GLOBAL_OBJECT_INDEX)));
4318  // Load the native context from the global or builtins object.
4319  movq(function, FieldOperand(function, GlobalObject::kNativeContextOffset));
4320  // Load the function from the native context.
4321  movq(function, Operand(function, Context::SlotOffset(index)));
4322}
4323
4324
4325void MacroAssembler::LoadArrayFunction(Register function) {
4326  movq(function,
4327       Operand(rsi, Context::SlotOffset(Context::GLOBAL_OBJECT_INDEX)));
4328  movq(function, FieldOperand(function, GlobalObject::kGlobalContextOffset));
4329  movq(function,
4330       Operand(function, Context::SlotOffset(Context::ARRAY_FUNCTION_INDEX)));
4331}
4332
4333
4334void MacroAssembler::LoadGlobalFunctionInitialMap(Register function,
4335                                                  Register map) {
4336  // Load the initial map.  The global functions all have initial maps.
4337  movq(map, FieldOperand(function, JSFunction::kPrototypeOrInitialMapOffset));
4338  if (emit_debug_code()) {
4339    Label ok, fail;
4340    CheckMap(map, isolate()->factory()->meta_map(), &fail, DO_SMI_CHECK);
4341    jmp(&ok);
4342    bind(&fail);
4343    Abort(kGlobalFunctionsMustHaveInitialMap);
4344    bind(&ok);
4345  }
4346}
4347
4348
4349int MacroAssembler::ArgumentStackSlotsForCFunctionCall(int num_arguments) {
4350  // On Windows 64 stack slots are reserved by the caller for all arguments
4351  // including the ones passed in registers, and space is always allocated for
4352  // the four register arguments even if the function takes fewer than four
4353  // arguments.
4354  // On AMD64 ABI (Linux/Mac) the first six arguments are passed in registers
4355  // and the caller does not reserve stack slots for them.
4356  ASSERT(num_arguments >= 0);
4357#ifdef _WIN64
4358  const int kMinimumStackSlots = kRegisterPassedArguments;
4359  if (num_arguments < kMinimumStackSlots) return kMinimumStackSlots;
4360  return num_arguments;
4361#else
4362  if (num_arguments < kRegisterPassedArguments) return 0;
4363  return num_arguments - kRegisterPassedArguments;
4364#endif
4365}
4366
4367
4368void MacroAssembler::PrepareCallCFunction(int num_arguments) {
4369  int frame_alignment = OS::ActivationFrameAlignment();
4370  ASSERT(frame_alignment != 0);
4371  ASSERT(num_arguments >= 0);
4372
4373  // Make stack end at alignment and allocate space for arguments and old rsp.
4374  movq(kScratchRegister, rsp);
4375  ASSERT(IsPowerOf2(frame_alignment));
4376  int argument_slots_on_stack =
4377      ArgumentStackSlotsForCFunctionCall(num_arguments);
4378  subq(rsp, Immediate((argument_slots_on_stack + 1) * kPointerSize));
4379  and_(rsp, Immediate(-frame_alignment));
4380  movq(Operand(rsp, argument_slots_on_stack * kPointerSize), kScratchRegister);
4381}
4382
4383
4384void MacroAssembler::CallCFunction(ExternalReference function,
4385                                   int num_arguments) {
4386  LoadAddress(rax, function);
4387  CallCFunction(rax, num_arguments);
4388}
4389
4390
4391void MacroAssembler::CallCFunction(Register function, int num_arguments) {
4392  ASSERT(has_frame());
4393  // Check stack alignment.
4394  if (emit_debug_code()) {
4395    CheckStackAlignment();
4396  }
4397
4398  call(function);
4399  ASSERT(OS::ActivationFrameAlignment() != 0);
4400  ASSERT(num_arguments >= 0);
4401  int argument_slots_on_stack =
4402      ArgumentStackSlotsForCFunctionCall(num_arguments);
4403  movq(rsp, Operand(rsp, argument_slots_on_stack * kPointerSize));
4404}
4405
4406
4407bool AreAliased(Register r1, Register r2, Register r3, Register r4) {
4408  if (r1.is(r2)) return true;
4409  if (r1.is(r3)) return true;
4410  if (r1.is(r4)) return true;
4411  if (r2.is(r3)) return true;
4412  if (r2.is(r4)) return true;
4413  if (r3.is(r4)) return true;
4414  return false;
4415}
4416
4417
4418CodePatcher::CodePatcher(byte* address, int size)
4419    : address_(address),
4420      size_(size),
4421      masm_(NULL, address, size + Assembler::kGap) {
4422  // Create a new macro assembler pointing to the address of the code to patch.
4423  // The size is adjusted with kGap on order for the assembler to generate size
4424  // bytes of instructions without failing with buffer size constraints.
4425  ASSERT(masm_.reloc_info_writer.pos() == address_ + size_ + Assembler::kGap);
4426}
4427
4428
4429CodePatcher::~CodePatcher() {
4430  // Indicate that code has changed.
4431  CPU::FlushICache(address_, size_);
4432
4433  // Check that the code was patched as expected.
4434  ASSERT(masm_.pc_ == address_ + size_);
4435  ASSERT(masm_.reloc_info_writer.pos() == address_ + size_ + Assembler::kGap);
4436}
4437
4438
4439void MacroAssembler::CheckPageFlag(
4440    Register object,
4441    Register scratch,
4442    int mask,
4443    Condition cc,
4444    Label* condition_met,
4445    Label::Distance condition_met_distance) {
4446  ASSERT(cc == zero || cc == not_zero);
4447  if (scratch.is(object)) {
4448    and_(scratch, Immediate(~Page::kPageAlignmentMask));
4449  } else {
4450    movq(scratch, Immediate(~Page::kPageAlignmentMask));
4451    and_(scratch, object);
4452  }
4453  if (mask < (1 << kBitsPerByte)) {
4454    testb(Operand(scratch, MemoryChunk::kFlagsOffset),
4455          Immediate(static_cast<uint8_t>(mask)));
4456  } else {
4457    testl(Operand(scratch, MemoryChunk::kFlagsOffset), Immediate(mask));
4458  }
4459  j(cc, condition_met, condition_met_distance);
4460}
4461
4462
4463void MacroAssembler::CheckMapDeprecated(Handle<Map> map,
4464                                        Register scratch,
4465                                        Label* if_deprecated) {
4466  if (map->CanBeDeprecated()) {
4467    Move(scratch, map);
4468    movq(scratch, FieldOperand(scratch, Map::kBitField3Offset));
4469    SmiToInteger32(scratch, scratch);
4470    and_(scratch, Immediate(Map::Deprecated::kMask));
4471    j(not_zero, if_deprecated);
4472  }
4473}
4474
4475
4476void MacroAssembler::JumpIfBlack(Register object,
4477                                 Register bitmap_scratch,
4478                                 Register mask_scratch,
4479                                 Label* on_black,
4480                                 Label::Distance on_black_distance) {
4481  ASSERT(!AreAliased(object, bitmap_scratch, mask_scratch, rcx));
4482  GetMarkBits(object, bitmap_scratch, mask_scratch);
4483
4484  ASSERT(strcmp(Marking::kBlackBitPattern, "10") == 0);
4485  // The mask_scratch register contains a 1 at the position of the first bit
4486  // and a 0 at all other positions, including the position of the second bit.
4487  movq(rcx, mask_scratch);
4488  // Make rcx into a mask that covers both marking bits using the operation
4489  // rcx = mask | (mask << 1).
4490  lea(rcx, Operand(mask_scratch, mask_scratch, times_2, 0));
4491  // Note that we are using a 4-byte aligned 8-byte load.
4492  and_(rcx, Operand(bitmap_scratch, MemoryChunk::kHeaderSize));
4493  cmpq(mask_scratch, rcx);
4494  j(equal, on_black, on_black_distance);
4495}
4496
4497
4498// Detect some, but not all, common pointer-free objects.  This is used by the
4499// incremental write barrier which doesn't care about oddballs (they are always
4500// marked black immediately so this code is not hit).
4501void MacroAssembler::JumpIfDataObject(
4502    Register value,
4503    Register scratch,
4504    Label* not_data_object,
4505    Label::Distance not_data_object_distance) {
4506  Label is_data_object;
4507  movq(scratch, FieldOperand(value, HeapObject::kMapOffset));
4508  CompareRoot(scratch, Heap::kHeapNumberMapRootIndex);
4509  j(equal, &is_data_object, Label::kNear);
4510  ASSERT(kIsIndirectStringTag == 1 && kIsIndirectStringMask == 1);
4511  ASSERT(kNotStringTag == 0x80 && kIsNotStringMask == 0x80);
4512  // If it's a string and it's not a cons string then it's an object containing
4513  // no GC pointers.
4514  testb(FieldOperand(scratch, Map::kInstanceTypeOffset),
4515        Immediate(kIsIndirectStringMask | kIsNotStringMask));
4516  j(not_zero, not_data_object, not_data_object_distance);
4517  bind(&is_data_object);
4518}
4519
4520
4521void MacroAssembler::GetMarkBits(Register addr_reg,
4522                                 Register bitmap_reg,
4523                                 Register mask_reg) {
4524  ASSERT(!AreAliased(addr_reg, bitmap_reg, mask_reg, rcx));
4525  movq(bitmap_reg, addr_reg);
4526  // Sign extended 32 bit immediate.
4527  and_(bitmap_reg, Immediate(~Page::kPageAlignmentMask));
4528  movq(rcx, addr_reg);
4529  int shift =
4530      Bitmap::kBitsPerCellLog2 + kPointerSizeLog2 - Bitmap::kBytesPerCellLog2;
4531  shrl(rcx, Immediate(shift));
4532  and_(rcx,
4533       Immediate((Page::kPageAlignmentMask >> shift) &
4534                 ~(Bitmap::kBytesPerCell - 1)));
4535
4536  addq(bitmap_reg, rcx);
4537  movq(rcx, addr_reg);
4538  shrl(rcx, Immediate(kPointerSizeLog2));
4539  and_(rcx, Immediate((1 << Bitmap::kBitsPerCellLog2) - 1));
4540  movl(mask_reg, Immediate(1));
4541  shl_cl(mask_reg);
4542}
4543
4544
4545void MacroAssembler::EnsureNotWhite(
4546    Register value,
4547    Register bitmap_scratch,
4548    Register mask_scratch,
4549    Label* value_is_white_and_not_data,
4550    Label::Distance distance) {
4551  ASSERT(!AreAliased(value, bitmap_scratch, mask_scratch, rcx));
4552  GetMarkBits(value, bitmap_scratch, mask_scratch);
4553
4554  // If the value is black or grey we don't need to do anything.
4555  ASSERT(strcmp(Marking::kWhiteBitPattern, "00") == 0);
4556  ASSERT(strcmp(Marking::kBlackBitPattern, "10") == 0);
4557  ASSERT(strcmp(Marking::kGreyBitPattern, "11") == 0);
4558  ASSERT(strcmp(Marking::kImpossibleBitPattern, "01") == 0);
4559
4560  Label done;
4561
4562  // Since both black and grey have a 1 in the first position and white does
4563  // not have a 1 there we only need to check one bit.
4564  testq(Operand(bitmap_scratch, MemoryChunk::kHeaderSize), mask_scratch);
4565  j(not_zero, &done, Label::kNear);
4566
4567  if (emit_debug_code()) {
4568    // Check for impossible bit pattern.
4569    Label ok;
4570    push(mask_scratch);
4571    // shl.  May overflow making the check conservative.
4572    addq(mask_scratch, mask_scratch);
4573    testq(Operand(bitmap_scratch, MemoryChunk::kHeaderSize), mask_scratch);
4574    j(zero, &ok, Label::kNear);
4575    int3();
4576    bind(&ok);
4577    pop(mask_scratch);
4578  }
4579
4580  // Value is white.  We check whether it is data that doesn't need scanning.
4581  // Currently only checks for HeapNumber and non-cons strings.
4582  Register map = rcx;  // Holds map while checking type.
4583  Register length = rcx;  // Holds length of object after checking type.
4584  Label not_heap_number;
4585  Label is_data_object;
4586
4587  // Check for heap-number
4588  movq(map, FieldOperand(value, HeapObject::kMapOffset));
4589  CompareRoot(map, Heap::kHeapNumberMapRootIndex);
4590  j(not_equal, &not_heap_number, Label::kNear);
4591  movq(length, Immediate(HeapNumber::kSize));
4592  jmp(&is_data_object, Label::kNear);
4593
4594  bind(&not_heap_number);
4595  // Check for strings.
4596  ASSERT(kIsIndirectStringTag == 1 && kIsIndirectStringMask == 1);
4597  ASSERT(kNotStringTag == 0x80 && kIsNotStringMask == 0x80);
4598  // If it's a string and it's not a cons string then it's an object containing
4599  // no GC pointers.
4600  Register instance_type = rcx;
4601  movzxbl(instance_type, FieldOperand(map, Map::kInstanceTypeOffset));
4602  testb(instance_type, Immediate(kIsIndirectStringMask | kIsNotStringMask));
4603  j(not_zero, value_is_white_and_not_data);
4604  // It's a non-indirect (non-cons and non-slice) string.
4605  // If it's external, the length is just ExternalString::kSize.
4606  // Otherwise it's String::kHeaderSize + string->length() * (1 or 2).
4607  Label not_external;
4608  // External strings are the only ones with the kExternalStringTag bit
4609  // set.
4610  ASSERT_EQ(0, kSeqStringTag & kExternalStringTag);
4611  ASSERT_EQ(0, kConsStringTag & kExternalStringTag);
4612  testb(instance_type, Immediate(kExternalStringTag));
4613  j(zero, &not_external, Label::kNear);
4614  movq(length, Immediate(ExternalString::kSize));
4615  jmp(&is_data_object, Label::kNear);
4616
4617  bind(&not_external);
4618  // Sequential string, either ASCII or UC16.
4619  ASSERT(kOneByteStringTag == 0x04);
4620  and_(length, Immediate(kStringEncodingMask));
4621  xor_(length, Immediate(kStringEncodingMask));
4622  addq(length, Immediate(0x04));
4623  // Value now either 4 (if ASCII) or 8 (if UC16), i.e. char-size shifted by 2.
4624  imul(length, FieldOperand(value, String::kLengthOffset));
4625  shr(length, Immediate(2 + kSmiTagSize + kSmiShiftSize));
4626  addq(length, Immediate(SeqString::kHeaderSize + kObjectAlignmentMask));
4627  and_(length, Immediate(~kObjectAlignmentMask));
4628
4629  bind(&is_data_object);
4630  // Value is a data object, and it is white.  Mark it black.  Since we know
4631  // that the object is white we can make it black by flipping one bit.
4632  or_(Operand(bitmap_scratch, MemoryChunk::kHeaderSize), mask_scratch);
4633
4634  and_(bitmap_scratch, Immediate(~Page::kPageAlignmentMask));
4635  addl(Operand(bitmap_scratch, MemoryChunk::kLiveBytesOffset), length);
4636
4637  bind(&done);
4638}
4639
4640
4641void MacroAssembler::CheckEnumCache(Register null_value, Label* call_runtime) {
4642  Label next, start;
4643  Register empty_fixed_array_value = r8;
4644  LoadRoot(empty_fixed_array_value, Heap::kEmptyFixedArrayRootIndex);
4645  movq(rcx, rax);
4646
4647  // Check if the enum length field is properly initialized, indicating that
4648  // there is an enum cache.
4649  movq(rbx, FieldOperand(rcx, HeapObject::kMapOffset));
4650
4651  EnumLength(rdx, rbx);
4652  Cmp(rdx, Smi::FromInt(Map::kInvalidEnumCache));
4653  j(equal, call_runtime);
4654
4655  jmp(&start);
4656
4657  bind(&next);
4658
4659  movq(rbx, FieldOperand(rcx, HeapObject::kMapOffset));
4660
4661  // For all objects but the receiver, check that the cache is empty.
4662  EnumLength(rdx, rbx);
4663  Cmp(rdx, Smi::FromInt(0));
4664  j(not_equal, call_runtime);
4665
4666  bind(&start);
4667
4668  // Check that there are no elements. Register rcx contains the current JS
4669  // object we've reached through the prototype chain.
4670  cmpq(empty_fixed_array_value,
4671       FieldOperand(rcx, JSObject::kElementsOffset));
4672  j(not_equal, call_runtime);
4673
4674  movq(rcx, FieldOperand(rbx, Map::kPrototypeOffset));
4675  cmpq(rcx, null_value);
4676  j(not_equal, &next);
4677}
4678
4679void MacroAssembler::TestJSArrayForAllocationMemento(
4680    Register receiver_reg,
4681    Register scratch_reg) {
4682  Label no_memento_available;
4683  ExternalReference new_space_start =
4684      ExternalReference::new_space_start(isolate());
4685  ExternalReference new_space_allocation_top =
4686      ExternalReference::new_space_allocation_top_address(isolate());
4687
4688  lea(scratch_reg, Operand(receiver_reg,
4689      JSArray::kSize + AllocationMemento::kSize - kHeapObjectTag));
4690  movq(kScratchRegister, new_space_start);
4691  cmpq(scratch_reg, kScratchRegister);
4692  j(less, &no_memento_available);
4693  cmpq(scratch_reg, ExternalOperand(new_space_allocation_top));
4694  j(greater, &no_memento_available);
4695  CompareRoot(MemOperand(scratch_reg, -AllocationMemento::kSize),
4696              Heap::kAllocationMementoMapRootIndex);
4697  bind(&no_memento_available);
4698}
4699
4700
4701} }  // namespace v8::internal
4702
4703#endif  // V8_TARGET_ARCH_X64
4704