1// Copyright 2013 the V8 project authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#if V8_TARGET_ARCH_ARM64 6 7#include "src/codegen.h" 8#include "src/ic/ic.h" 9#include "src/ic/ic-compiler.h" 10#include "src/ic/stub-cache.h" 11 12namespace v8 { 13namespace internal { 14 15 16#define __ ACCESS_MASM(masm) 17 18 19// "type" holds an instance type on entry and is not clobbered. 20// Generated code branch on "global_object" if type is any kind of global 21// JS object. 22static void GenerateGlobalInstanceTypeCheck(MacroAssembler* masm, Register type, 23 Label* global_object) { 24 __ Cmp(type, JS_GLOBAL_OBJECT_TYPE); 25 __ Ccmp(type, JS_GLOBAL_PROXY_TYPE, ZFlag, ne); 26 __ B(eq, global_object); 27} 28 29 30// Helper function used from LoadIC GenerateNormal. 31// 32// elements: Property dictionary. It is not clobbered if a jump to the miss 33// label is done. 34// name: Property name. It is not clobbered if a jump to the miss label is 35// done 36// result: Register for the result. It is only updated if a jump to the miss 37// label is not done. 38// The scratch registers need to be different from elements, name and result. 39// The generated code assumes that the receiver has slow properties, 40// is not a global object and does not have interceptors. 41static void GenerateDictionaryLoad(MacroAssembler* masm, Label* miss, 42 Register elements, Register name, 43 Register result, Register scratch1, 44 Register scratch2) { 45 DCHECK(!AreAliased(elements, name, scratch1, scratch2)); 46 DCHECK(!AreAliased(result, scratch1, scratch2)); 47 48 Label done; 49 50 // Probe the dictionary. 51 NameDictionaryLookupStub::GeneratePositiveLookup(masm, miss, &done, elements, 52 name, scratch1, scratch2); 53 54 // If probing finds an entry check that the value is a normal property. 55 __ Bind(&done); 56 57 static const int kElementsStartOffset = 58 NameDictionary::kHeaderSize + 59 NameDictionary::kElementsStartIndex * kPointerSize; 60 static const int kDetailsOffset = kElementsStartOffset + 2 * kPointerSize; 61 __ Ldr(scratch1, FieldMemOperand(scratch2, kDetailsOffset)); 62 __ Tst(scratch1, Smi::FromInt(PropertyDetails::TypeField::kMask)); 63 __ B(ne, miss); 64 65 // Get the value at the masked, scaled index and return. 66 __ Ldr(result, 67 FieldMemOperand(scratch2, kElementsStartOffset + 1 * kPointerSize)); 68} 69 70 71// Helper function used from StoreIC::GenerateNormal. 72// 73// elements: Property dictionary. It is not clobbered if a jump to the miss 74// label is done. 75// name: Property name. It is not clobbered if a jump to the miss label is 76// done 77// value: The value to store (never clobbered). 78// 79// The generated code assumes that the receiver has slow properties, 80// is not a global object and does not have interceptors. 81static void GenerateDictionaryStore(MacroAssembler* masm, Label* miss, 82 Register elements, Register name, 83 Register value, Register scratch1, 84 Register scratch2) { 85 DCHECK(!AreAliased(elements, name, value, scratch1, scratch2)); 86 87 Label done; 88 89 // Probe the dictionary. 90 NameDictionaryLookupStub::GeneratePositiveLookup(masm, miss, &done, elements, 91 name, scratch1, scratch2); 92 93 // If probing finds an entry in the dictionary check that the value 94 // is a normal property that is not read only. 95 __ Bind(&done); 96 97 static const int kElementsStartOffset = 98 NameDictionary::kHeaderSize + 99 NameDictionary::kElementsStartIndex * kPointerSize; 100 static const int kDetailsOffset = kElementsStartOffset + 2 * kPointerSize; 101 static const int kTypeAndReadOnlyMask = 102 PropertyDetails::TypeField::kMask | 103 PropertyDetails::AttributesField::encode(READ_ONLY); 104 __ Ldrsw(scratch1, UntagSmiFieldMemOperand(scratch2, kDetailsOffset)); 105 __ Tst(scratch1, kTypeAndReadOnlyMask); 106 __ B(ne, miss); 107 108 // Store the value at the masked, scaled index and return. 109 static const int kValueOffset = kElementsStartOffset + kPointerSize; 110 __ Add(scratch2, scratch2, kValueOffset - kHeapObjectTag); 111 __ Str(value, MemOperand(scratch2)); 112 113 // Update the write barrier. Make sure not to clobber the value. 114 __ Mov(scratch1, value); 115 __ RecordWrite(elements, scratch2, scratch1, kLRHasNotBeenSaved, 116 kDontSaveFPRegs); 117} 118 119 120// Checks the receiver for special cases (value type, slow case bits). 121// Falls through for regular JS object and return the map of the 122// receiver in 'map_scratch' if the receiver is not a SMI. 123static void GenerateKeyedLoadReceiverCheck(MacroAssembler* masm, 124 Register receiver, 125 Register map_scratch, 126 Register scratch, 127 int interceptor_bit, Label* slow) { 128 DCHECK(!AreAliased(map_scratch, scratch)); 129 130 // Check that the object isn't a smi. 131 __ JumpIfSmi(receiver, slow); 132 // Get the map of the receiver. 133 __ Ldr(map_scratch, FieldMemOperand(receiver, HeapObject::kMapOffset)); 134 // Check bit field. 135 __ Ldrb(scratch, FieldMemOperand(map_scratch, Map::kBitFieldOffset)); 136 __ Tbnz(scratch, Map::kIsAccessCheckNeeded, slow); 137 __ Tbnz(scratch, interceptor_bit, slow); 138 139 // Check that the object is some kind of JS object EXCEPT JS Value type. 140 // In the case that the object is a value-wrapper object, we enter the 141 // runtime system to make sure that indexing into string objects work 142 // as intended. 143 STATIC_ASSERT(JS_OBJECT_TYPE > JS_VALUE_TYPE); 144 __ Ldrb(scratch, FieldMemOperand(map_scratch, Map::kInstanceTypeOffset)); 145 __ Cmp(scratch, JS_OBJECT_TYPE); 146 __ B(lt, slow); 147} 148 149 150// Loads an indexed element from a fast case array. 151// 152// receiver - holds the receiver on entry. 153// Unchanged unless 'result' is the same register. 154// 155// key - holds the smi key on entry. 156// Unchanged unless 'result' is the same register. 157// 158// elements - holds the elements of the receiver and its prototypes. Clobbered. 159// 160// result - holds the result on exit if the load succeeded. 161// Allowed to be the the same as 'receiver' or 'key'. 162// Unchanged on bailout so 'receiver' and 'key' can be safely 163// used by further computation. 164static void GenerateFastArrayLoad(MacroAssembler* masm, Register receiver, 165 Register key, Register elements, 166 Register scratch1, Register scratch2, 167 Register result, Label* slow) { 168 DCHECK(!AreAliased(receiver, key, elements, scratch1, scratch2)); 169 170 Label check_prototypes, check_next_prototype; 171 Label done, in_bounds, absent; 172 173 // Check for fast array. 174 __ Ldr(elements, FieldMemOperand(receiver, JSObject::kElementsOffset)); 175 __ AssertFastElements(elements); 176 177 // Check that the key (index) is within bounds. 178 __ Ldr(scratch1, FieldMemOperand(elements, FixedArray::kLengthOffset)); 179 __ Cmp(key, scratch1); 180 __ B(lo, &in_bounds); 181 182 // Out of bounds. Check the prototype chain to see if we can just return 183 // 'undefined'. 184 __ Cmp(key, Operand(Smi::FromInt(0))); 185 __ B(lt, slow); // Negative keys can't take the fast OOB path. 186 __ Bind(&check_prototypes); 187 __ Ldr(scratch2, FieldMemOperand(receiver, HeapObject::kMapOffset)); 188 __ Bind(&check_next_prototype); 189 __ Ldr(scratch2, FieldMemOperand(scratch2, Map::kPrototypeOffset)); 190 // scratch2: current prototype 191 __ JumpIfRoot(scratch2, Heap::kNullValueRootIndex, &absent); 192 __ Ldr(elements, FieldMemOperand(scratch2, JSObject::kElementsOffset)); 193 __ Ldr(scratch2, FieldMemOperand(scratch2, HeapObject::kMapOffset)); 194 // elements: elements of current prototype 195 // scratch2: map of current prototype 196 __ CompareInstanceType(scratch2, scratch1, JS_OBJECT_TYPE); 197 __ B(lo, slow); 198 __ Ldrb(scratch1, FieldMemOperand(scratch2, Map::kBitFieldOffset)); 199 __ Tbnz(scratch1, Map::kIsAccessCheckNeeded, slow); 200 __ Tbnz(scratch1, Map::kHasIndexedInterceptor, slow); 201 __ JumpIfNotRoot(elements, Heap::kEmptyFixedArrayRootIndex, slow); 202 __ B(&check_next_prototype); 203 204 __ Bind(&absent); 205 __ LoadRoot(result, Heap::kUndefinedValueRootIndex); 206 __ B(&done); 207 208 __ Bind(&in_bounds); 209 // Fast case: Do the load. 210 __ Add(scratch1, elements, FixedArray::kHeaderSize - kHeapObjectTag); 211 __ SmiUntag(scratch2, key); 212 __ Ldr(scratch2, MemOperand(scratch1, scratch2, LSL, kPointerSizeLog2)); 213 214 // In case the loaded value is the_hole we have to check the prototype chain. 215 __ JumpIfRoot(scratch2, Heap::kTheHoleValueRootIndex, &check_prototypes); 216 217 // Move the value to the result register. 218 // 'result' can alias with 'receiver' or 'key' but these two must be 219 // preserved if we jump to 'slow'. 220 __ Mov(result, scratch2); 221 __ Bind(&done); 222} 223 224 225// Checks whether a key is an array index string or a unique name. 226// Falls through if a key is a unique name. 227// The map of the key is returned in 'map_scratch'. 228// If the jump to 'index_string' is done the hash of the key is left 229// in 'hash_scratch'. 230static void GenerateKeyNameCheck(MacroAssembler* masm, Register key, 231 Register map_scratch, Register hash_scratch, 232 Label* index_string, Label* not_unique) { 233 DCHECK(!AreAliased(key, map_scratch, hash_scratch)); 234 235 // Is the key a name? 236 Label unique; 237 __ JumpIfObjectType(key, map_scratch, hash_scratch, LAST_UNIQUE_NAME_TYPE, 238 not_unique, hi); 239 STATIC_ASSERT(LAST_UNIQUE_NAME_TYPE == FIRST_NONSTRING_TYPE); 240 __ B(eq, &unique); 241 242 // Is the string an array index with cached numeric value? 243 __ Ldr(hash_scratch.W(), FieldMemOperand(key, Name::kHashFieldOffset)); 244 __ TestAndBranchIfAllClear(hash_scratch, Name::kContainsCachedArrayIndexMask, 245 index_string); 246 247 // Is the string internalized? We know it's a string, so a single bit test is 248 // enough. 249 __ Ldrb(hash_scratch, FieldMemOperand(map_scratch, Map::kInstanceTypeOffset)); 250 STATIC_ASSERT(kInternalizedTag == 0); 251 __ TestAndBranchIfAnySet(hash_scratch, kIsNotInternalizedMask, not_unique); 252 253 __ Bind(&unique); 254 // Fall through if the key is a unique name. 255} 256 257void LoadIC::GenerateNormal(MacroAssembler* masm) { 258 Register dictionary = x0; 259 DCHECK(!dictionary.is(LoadDescriptor::ReceiverRegister())); 260 DCHECK(!dictionary.is(LoadDescriptor::NameRegister())); 261 Label slow; 262 263 __ Ldr(dictionary, FieldMemOperand(LoadDescriptor::ReceiverRegister(), 264 JSObject::kPropertiesOffset)); 265 GenerateDictionaryLoad(masm, &slow, dictionary, 266 LoadDescriptor::NameRegister(), x0, x3, x4); 267 __ Ret(); 268 269 // Dictionary load failed, go slow (but don't miss). 270 __ Bind(&slow); 271 GenerateRuntimeGetProperty(masm); 272} 273 274 275void LoadIC::GenerateMiss(MacroAssembler* masm) { 276 // The return address is in lr. 277 Isolate* isolate = masm->isolate(); 278 ASM_LOCATION("LoadIC::GenerateMiss"); 279 280 DCHECK(!AreAliased(x4, x5, LoadWithVectorDescriptor::SlotRegister(), 281 LoadWithVectorDescriptor::VectorRegister())); 282 __ IncrementCounter(isolate->counters()->ic_load_miss(), 1, x4, x5); 283 284 // Perform tail call to the entry. 285 __ Push(LoadWithVectorDescriptor::ReceiverRegister(), 286 LoadWithVectorDescriptor::NameRegister(), 287 LoadWithVectorDescriptor::SlotRegister(), 288 LoadWithVectorDescriptor::VectorRegister()); 289 __ TailCallRuntime(Runtime::kLoadIC_Miss); 290} 291 292void LoadIC::GenerateRuntimeGetProperty(MacroAssembler* masm) { 293 // The return address is in lr. 294 __ Push(LoadDescriptor::ReceiverRegister(), LoadDescriptor::NameRegister()); 295 296 // Do tail-call to runtime routine. 297 __ TailCallRuntime(Runtime::kGetProperty); 298} 299 300 301void KeyedLoadIC::GenerateMiss(MacroAssembler* masm) { 302 // The return address is in lr. 303 Isolate* isolate = masm->isolate(); 304 305 DCHECK(!AreAliased(x10, x11, LoadWithVectorDescriptor::SlotRegister(), 306 LoadWithVectorDescriptor::VectorRegister())); 307 __ IncrementCounter(isolate->counters()->ic_keyed_load_miss(), 1, x10, x11); 308 309 __ Push(LoadWithVectorDescriptor::ReceiverRegister(), 310 LoadWithVectorDescriptor::NameRegister(), 311 LoadWithVectorDescriptor::SlotRegister(), 312 LoadWithVectorDescriptor::VectorRegister()); 313 314 // Perform tail call to the entry. 315 __ TailCallRuntime(Runtime::kKeyedLoadIC_Miss); 316} 317 318void KeyedLoadIC::GenerateRuntimeGetProperty(MacroAssembler* masm) { 319 // The return address is in lr. 320 __ Push(LoadDescriptor::ReceiverRegister(), LoadDescriptor::NameRegister()); 321 322 // Do tail-call to runtime routine. 323 __ TailCallRuntime(Runtime::kKeyedGetProperty); 324} 325 326static void GenerateKeyedLoadWithSmiKey(MacroAssembler* masm, Register key, 327 Register receiver, Register scratch1, 328 Register scratch2, Register scratch3, 329 Register scratch4, Register scratch5, 330 Label* slow) { 331 DCHECK(!AreAliased(key, receiver, scratch1, scratch2, scratch3, scratch4, 332 scratch5)); 333 334 Isolate* isolate = masm->isolate(); 335 Label check_number_dictionary; 336 // If we can load the value, it should be returned in x0. 337 Register result = x0; 338 339 GenerateKeyedLoadReceiverCheck(masm, receiver, scratch1, scratch2, 340 Map::kHasIndexedInterceptor, slow); 341 342 // Check the receiver's map to see if it has fast elements. 343 __ CheckFastElements(scratch1, scratch2, &check_number_dictionary); 344 345 GenerateFastArrayLoad(masm, receiver, key, scratch3, scratch2, scratch1, 346 result, slow); 347 __ IncrementCounter(isolate->counters()->ic_keyed_load_generic_smi(), 1, 348 scratch1, scratch2); 349 __ Ret(); 350 351 __ Bind(&check_number_dictionary); 352 __ Ldr(scratch3, FieldMemOperand(receiver, JSObject::kElementsOffset)); 353 __ Ldr(scratch2, FieldMemOperand(scratch3, JSObject::kMapOffset)); 354 355 // Check whether we have a number dictionary. 356 __ JumpIfNotRoot(scratch2, Heap::kHashTableMapRootIndex, slow); 357 358 __ LoadFromNumberDictionary(slow, scratch3, key, result, scratch1, scratch2, 359 scratch4, scratch5); 360 __ Ret(); 361} 362 363static void GenerateKeyedLoadWithNameKey(MacroAssembler* masm, Register key, 364 Register receiver, Register scratch1, 365 Register scratch2, Register scratch3, 366 Register scratch4, Register scratch5, 367 Label* slow) { 368 DCHECK(!AreAliased(key, receiver, scratch1, scratch2, scratch3, scratch4, 369 scratch5)); 370 371 Isolate* isolate = masm->isolate(); 372 Label probe_dictionary, property_array_property; 373 // If we can load the value, it should be returned in x0. 374 Register result = x0; 375 376 GenerateKeyedLoadReceiverCheck(masm, receiver, scratch1, scratch2, 377 Map::kHasNamedInterceptor, slow); 378 379 // If the receiver is a fast-case object, check the stub cache. Otherwise 380 // probe the dictionary. 381 __ Ldr(scratch2, FieldMemOperand(receiver, JSObject::kPropertiesOffset)); 382 __ Ldr(scratch3, FieldMemOperand(scratch2, HeapObject::kMapOffset)); 383 __ JumpIfRoot(scratch3, Heap::kHashTableMapRootIndex, &probe_dictionary); 384 385 // The handlers in the stub cache expect a vector and slot. Since we won't 386 // change the IC from any downstream misses, a dummy vector can be used. 387 Register vector = LoadWithVectorDescriptor::VectorRegister(); 388 Register slot = LoadWithVectorDescriptor::SlotRegister(); 389 DCHECK(!AreAliased(vector, slot, scratch1, scratch2, scratch3, scratch4)); 390 Handle<TypeFeedbackVector> dummy_vector = 391 TypeFeedbackVector::DummyVector(masm->isolate()); 392 int slot_index = dummy_vector->GetIndex( 393 FeedbackVectorSlot(TypeFeedbackVector::kDummyKeyedLoadICSlot)); 394 __ LoadRoot(vector, Heap::kDummyVectorRootIndex); 395 __ Mov(slot, Operand(Smi::FromInt(slot_index))); 396 397 Code::Flags flags = 398 Code::RemoveHolderFromFlags(Code::ComputeHandlerFlags(Code::LOAD_IC)); 399 masm->isolate()->stub_cache()->GenerateProbe(masm, Code::KEYED_LOAD_IC, flags, 400 receiver, key, scratch1, 401 scratch2, scratch3, scratch4); 402 // Cache miss. 403 KeyedLoadIC::GenerateMiss(masm); 404 405 // Do a quick inline probe of the receiver's dictionary, if it exists. 406 __ Bind(&probe_dictionary); 407 __ Ldr(scratch1, FieldMemOperand(receiver, HeapObject::kMapOffset)); 408 __ Ldrb(scratch1, FieldMemOperand(scratch1, Map::kInstanceTypeOffset)); 409 GenerateGlobalInstanceTypeCheck(masm, scratch1, slow); 410 // Load the property. 411 GenerateDictionaryLoad(masm, slow, scratch2, key, result, scratch1, scratch3); 412 __ IncrementCounter(isolate->counters()->ic_keyed_load_generic_symbol(), 1, 413 scratch1, scratch2); 414 __ Ret(); 415} 416 417void KeyedLoadIC::GenerateMegamorphic(MacroAssembler* masm) { 418 // The return address is in lr. 419 Label slow, check_name, index_smi, index_name; 420 421 Register key = LoadDescriptor::NameRegister(); 422 Register receiver = LoadDescriptor::ReceiverRegister(); 423 DCHECK(key.is(x2)); 424 DCHECK(receiver.is(x1)); 425 426 __ JumpIfNotSmi(key, &check_name); 427 __ Bind(&index_smi); 428 // Now the key is known to be a smi. This place is also jumped to from below 429 // where a numeric string is converted to a smi. 430 GenerateKeyedLoadWithSmiKey(masm, key, receiver, x7, x3, x4, x5, x6, &slow); 431 432 // Slow case. 433 __ Bind(&slow); 434 __ IncrementCounter(masm->isolate()->counters()->ic_keyed_load_generic_slow(), 435 1, x4, x3); 436 GenerateRuntimeGetProperty(masm); 437 438 __ Bind(&check_name); 439 GenerateKeyNameCheck(masm, key, x0, x3, &index_name, &slow); 440 441 GenerateKeyedLoadWithNameKey(masm, key, receiver, x4, x5, x6, x7, x3, &slow); 442 443 __ Bind(&index_name); 444 __ IndexFromHash(x3, key); 445 // Now jump to the place where smi keys are handled. 446 __ B(&index_smi); 447} 448 449 450static void StoreIC_PushArgs(MacroAssembler* masm) { 451 __ Push(StoreDescriptor::ReceiverRegister(), StoreDescriptor::NameRegister(), 452 StoreDescriptor::ValueRegister(), 453 VectorStoreICDescriptor::SlotRegister(), 454 VectorStoreICDescriptor::VectorRegister()); 455} 456 457 458void KeyedStoreIC::GenerateMiss(MacroAssembler* masm) { 459 ASM_LOCATION("KeyedStoreIC::GenerateMiss"); 460 StoreIC_PushArgs(masm); 461 __ TailCallRuntime(Runtime::kKeyedStoreIC_Miss); 462} 463 464 465static void KeyedStoreGenerateMegamorphicHelper( 466 MacroAssembler* masm, Label* fast_object, Label* fast_double, Label* slow, 467 KeyedStoreCheckMap check_map, KeyedStoreIncrementLength increment_length, 468 Register value, Register key, Register receiver, Register receiver_map, 469 Register elements_map, Register elements) { 470 DCHECK(!AreAliased(value, key, receiver, receiver_map, elements_map, elements, 471 x10, x11)); 472 473 Label transition_smi_elements; 474 Label transition_double_elements; 475 Label fast_double_without_map_check; 476 Label non_double_value; 477 Label finish_store; 478 479 __ Bind(fast_object); 480 if (check_map == kCheckMap) { 481 __ Ldr(elements_map, FieldMemOperand(elements, HeapObject::kMapOffset)); 482 __ Cmp(elements_map, 483 Operand(masm->isolate()->factory()->fixed_array_map())); 484 __ B(ne, fast_double); 485 } 486 487 // HOLECHECK: guards "A[i] = V" 488 // We have to go to the runtime if the current value is the hole because there 489 // may be a callback on the element. 490 Label holecheck_passed; 491 __ Add(x10, elements, FixedArray::kHeaderSize - kHeapObjectTag); 492 __ Add(x10, x10, Operand::UntagSmiAndScale(key, kPointerSizeLog2)); 493 __ Ldr(x11, MemOperand(x10)); 494 __ JumpIfNotRoot(x11, Heap::kTheHoleValueRootIndex, &holecheck_passed); 495 __ JumpIfDictionaryInPrototypeChain(receiver, elements_map, x10, slow); 496 __ bind(&holecheck_passed); 497 498 // Smi stores don't require further checks. 499 __ JumpIfSmi(value, &finish_store); 500 501 // Escape to elements kind transition case. 502 __ CheckFastObjectElements(receiver_map, x10, &transition_smi_elements); 503 504 __ Bind(&finish_store); 505 if (increment_length == kIncrementLength) { 506 // Add 1 to receiver->length. 507 __ Add(x10, key, Smi::FromInt(1)); 508 __ Str(x10, FieldMemOperand(receiver, JSArray::kLengthOffset)); 509 } 510 511 Register address = x11; 512 __ Add(address, elements, FixedArray::kHeaderSize - kHeapObjectTag); 513 __ Add(address, address, Operand::UntagSmiAndScale(key, kPointerSizeLog2)); 514 __ Str(value, MemOperand(address)); 515 516 Label dont_record_write; 517 __ JumpIfSmi(value, &dont_record_write); 518 519 // Update write barrier for the elements array address. 520 __ Mov(x10, value); // Preserve the value which is returned. 521 __ RecordWrite(elements, address, x10, kLRHasNotBeenSaved, kDontSaveFPRegs, 522 EMIT_REMEMBERED_SET, OMIT_SMI_CHECK); 523 524 __ Bind(&dont_record_write); 525 __ Ret(); 526 527 528 __ Bind(fast_double); 529 if (check_map == kCheckMap) { 530 // Check for fast double array case. If this fails, call through to the 531 // runtime. 532 __ JumpIfNotRoot(elements_map, Heap::kFixedDoubleArrayMapRootIndex, slow); 533 } 534 535 // HOLECHECK: guards "A[i] double hole?" 536 // We have to see if the double version of the hole is present. If so go to 537 // the runtime. 538 __ Add(x10, elements, FixedDoubleArray::kHeaderSize - kHeapObjectTag); 539 __ Add(x10, x10, Operand::UntagSmiAndScale(key, kPointerSizeLog2)); 540 __ Ldr(x11, MemOperand(x10)); 541 __ CompareAndBranch(x11, kHoleNanInt64, ne, &fast_double_without_map_check); 542 __ JumpIfDictionaryInPrototypeChain(receiver, elements_map, x10, slow); 543 544 __ Bind(&fast_double_without_map_check); 545 __ StoreNumberToDoubleElements(value, key, elements, x10, d0, 546 &transition_double_elements); 547 if (increment_length == kIncrementLength) { 548 // Add 1 to receiver->length. 549 __ Add(x10, key, Smi::FromInt(1)); 550 __ Str(x10, FieldMemOperand(receiver, JSArray::kLengthOffset)); 551 } 552 __ Ret(); 553 554 555 __ Bind(&transition_smi_elements); 556 // Transition the array appropriately depending on the value type. 557 __ Ldr(x10, FieldMemOperand(value, HeapObject::kMapOffset)); 558 __ JumpIfNotRoot(x10, Heap::kHeapNumberMapRootIndex, &non_double_value); 559 560 // Value is a double. Transition FAST_SMI_ELEMENTS -> 561 // FAST_DOUBLE_ELEMENTS and complete the store. 562 __ LoadTransitionedArrayMapConditional( 563 FAST_SMI_ELEMENTS, FAST_DOUBLE_ELEMENTS, receiver_map, x10, x11, slow); 564 AllocationSiteMode mode = 565 AllocationSite::GetMode(FAST_SMI_ELEMENTS, FAST_DOUBLE_ELEMENTS); 566 ElementsTransitionGenerator::GenerateSmiToDouble(masm, receiver, key, value, 567 receiver_map, mode, slow); 568 __ Ldr(elements, FieldMemOperand(receiver, JSObject::kElementsOffset)); 569 __ B(&fast_double_without_map_check); 570 571 __ Bind(&non_double_value); 572 // Value is not a double, FAST_SMI_ELEMENTS -> FAST_ELEMENTS. 573 __ LoadTransitionedArrayMapConditional(FAST_SMI_ELEMENTS, FAST_ELEMENTS, 574 receiver_map, x10, x11, slow); 575 576 mode = AllocationSite::GetMode(FAST_SMI_ELEMENTS, FAST_ELEMENTS); 577 ElementsTransitionGenerator::GenerateMapChangeElementsTransition( 578 masm, receiver, key, value, receiver_map, mode, slow); 579 580 __ Ldr(elements, FieldMemOperand(receiver, JSObject::kElementsOffset)); 581 __ B(&finish_store); 582 583 __ Bind(&transition_double_elements); 584 // Elements are FAST_DOUBLE_ELEMENTS, but value is an Object that's not a 585 // HeapNumber. Make sure that the receiver is a Array with FAST_ELEMENTS and 586 // transition array from FAST_DOUBLE_ELEMENTS to FAST_ELEMENTS 587 __ LoadTransitionedArrayMapConditional(FAST_DOUBLE_ELEMENTS, FAST_ELEMENTS, 588 receiver_map, x10, x11, slow); 589 mode = AllocationSite::GetMode(FAST_DOUBLE_ELEMENTS, FAST_ELEMENTS); 590 ElementsTransitionGenerator::GenerateDoubleToObject( 591 masm, receiver, key, value, receiver_map, mode, slow); 592 __ Ldr(elements, FieldMemOperand(receiver, JSObject::kElementsOffset)); 593 __ B(&finish_store); 594} 595 596 597void KeyedStoreIC::GenerateMegamorphic(MacroAssembler* masm, 598 LanguageMode language_mode) { 599 ASM_LOCATION("KeyedStoreIC::GenerateMegamorphic"); 600 Label slow; 601 Label array; 602 Label fast_object; 603 Label extra; 604 Label fast_object_grow; 605 Label fast_double_grow; 606 Label fast_double; 607 Label maybe_name_key; 608 Label miss; 609 610 Register value = StoreDescriptor::ValueRegister(); 611 Register key = StoreDescriptor::NameRegister(); 612 Register receiver = StoreDescriptor::ReceiverRegister(); 613 DCHECK(receiver.is(x1)); 614 DCHECK(key.is(x2)); 615 DCHECK(value.is(x0)); 616 617 Register receiver_map = x3; 618 Register elements = x4; 619 Register elements_map = x5; 620 621 __ JumpIfNotSmi(key, &maybe_name_key); 622 __ JumpIfSmi(receiver, &slow); 623 __ Ldr(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset)); 624 625 // Check that the receiver does not require access checks. 626 // The generic stub does not perform map checks. 627 __ Ldrb(x10, FieldMemOperand(receiver_map, Map::kBitFieldOffset)); 628 __ TestAndBranchIfAnySet(x10, (1 << Map::kIsAccessCheckNeeded), &slow); 629 630 // Check if the object is a JS array or not. 631 Register instance_type = x10; 632 __ CompareInstanceType(receiver_map, instance_type, JS_ARRAY_TYPE); 633 __ B(eq, &array); 634 // Check that the object is some kind of JS object EXCEPT JS Value type. In 635 // the case that the object is a value-wrapper object, we enter the runtime 636 // system to make sure that indexing into string objects works as intended. 637 STATIC_ASSERT(JS_VALUE_TYPE < JS_OBJECT_TYPE); 638 __ Cmp(instance_type, JS_OBJECT_TYPE); 639 __ B(lo, &slow); 640 641 // Object case: Check key against length in the elements array. 642 __ Ldr(elements, FieldMemOperand(receiver, JSObject::kElementsOffset)); 643 // Check array bounds. Both the key and the length of FixedArray are smis. 644 __ Ldrsw(x10, UntagSmiFieldMemOperand(elements, FixedArray::kLengthOffset)); 645 __ Cmp(x10, Operand::UntagSmi(key)); 646 __ B(hi, &fast_object); 647 648 649 __ Bind(&slow); 650 // Slow case, handle jump to runtime. 651 // Live values: 652 // x0: value 653 // x1: key 654 // x2: receiver 655 PropertyICCompiler::GenerateRuntimeSetProperty(masm, language_mode); 656 // Never returns to here. 657 658 __ bind(&maybe_name_key); 659 __ Ldr(x10, FieldMemOperand(key, HeapObject::kMapOffset)); 660 __ Ldrb(x10, FieldMemOperand(x10, Map::kInstanceTypeOffset)); 661 __ JumpIfNotUniqueNameInstanceType(x10, &slow); 662 663 // The handlers in the stub cache expect a vector and slot. Since we won't 664 // change the IC from any downstream misses, a dummy vector can be used. 665 Register vector = VectorStoreICDescriptor::VectorRegister(); 666 Register slot = VectorStoreICDescriptor::SlotRegister(); 667 DCHECK(!AreAliased(vector, slot, x5, x6, x7, x8)); 668 Handle<TypeFeedbackVector> dummy_vector = 669 TypeFeedbackVector::DummyVector(masm->isolate()); 670 int slot_index = dummy_vector->GetIndex( 671 FeedbackVectorSlot(TypeFeedbackVector::kDummyKeyedStoreICSlot)); 672 __ LoadRoot(vector, Heap::kDummyVectorRootIndex); 673 __ Mov(slot, Operand(Smi::FromInt(slot_index))); 674 675 Code::Flags flags = 676 Code::RemoveHolderFromFlags(Code::ComputeHandlerFlags(Code::STORE_IC)); 677 masm->isolate()->stub_cache()->GenerateProbe( 678 masm, Code::KEYED_STORE_IC, flags, receiver, key, x5, x6, x7, x8); 679 // Cache miss. 680 __ B(&miss); 681 682 __ Bind(&extra); 683 // Extra capacity case: Check if there is extra capacity to 684 // perform the store and update the length. Used for adding one 685 // element to the array by writing to array[array.length]. 686 687 // Check for room in the elements backing store. 688 // Both the key and the length of FixedArray are smis. 689 __ Ldrsw(x10, UntagSmiFieldMemOperand(elements, FixedArray::kLengthOffset)); 690 __ Cmp(x10, Operand::UntagSmi(key)); 691 __ B(ls, &slow); 692 693 __ Ldr(elements_map, FieldMemOperand(elements, HeapObject::kMapOffset)); 694 __ Cmp(elements_map, Operand(masm->isolate()->factory()->fixed_array_map())); 695 __ B(eq, &fast_object_grow); 696 __ Cmp(elements_map, 697 Operand(masm->isolate()->factory()->fixed_double_array_map())); 698 __ B(eq, &fast_double_grow); 699 __ B(&slow); 700 701 702 __ Bind(&array); 703 // Array case: Get the length and the elements array from the JS 704 // array. Check that the array is in fast mode (and writable); if it 705 // is the length is always a smi. 706 707 __ Ldr(elements, FieldMemOperand(receiver, JSObject::kElementsOffset)); 708 709 // Check the key against the length in the array. 710 __ Ldrsw(x10, UntagSmiFieldMemOperand(receiver, JSArray::kLengthOffset)); 711 __ Cmp(x10, Operand::UntagSmi(key)); 712 __ B(eq, &extra); // We can handle the case where we are appending 1 element. 713 __ B(lo, &slow); 714 715 KeyedStoreGenerateMegamorphicHelper( 716 masm, &fast_object, &fast_double, &slow, kCheckMap, kDontIncrementLength, 717 value, key, receiver, receiver_map, elements_map, elements); 718 KeyedStoreGenerateMegamorphicHelper(masm, &fast_object_grow, 719 &fast_double_grow, &slow, kDontCheckMap, 720 kIncrementLength, value, key, receiver, 721 receiver_map, elements_map, elements); 722 723 __ bind(&miss); 724 GenerateMiss(masm); 725} 726 727void StoreIC::GenerateMiss(MacroAssembler* masm) { 728 StoreIC_PushArgs(masm); 729 730 // Tail call to the entry. 731 __ TailCallRuntime(Runtime::kStoreIC_Miss); 732} 733 734 735void StoreIC::GenerateNormal(MacroAssembler* masm) { 736 Label miss; 737 Register value = StoreDescriptor::ValueRegister(); 738 Register receiver = StoreDescriptor::ReceiverRegister(); 739 Register name = StoreDescriptor::NameRegister(); 740 Register dictionary = x5; 741 DCHECK(!AreAliased(value, receiver, name, 742 VectorStoreICDescriptor::SlotRegister(), 743 VectorStoreICDescriptor::VectorRegister(), x5, x6, x7)); 744 745 __ Ldr(dictionary, FieldMemOperand(receiver, JSObject::kPropertiesOffset)); 746 747 GenerateDictionaryStore(masm, &miss, dictionary, name, value, x6, x7); 748 Counters* counters = masm->isolate()->counters(); 749 __ IncrementCounter(counters->ic_store_normal_hit(), 1, x6, x7); 750 __ Ret(); 751 752 // Cache miss: Jump to runtime. 753 __ Bind(&miss); 754 __ IncrementCounter(counters->ic_store_normal_miss(), 1, x6, x7); 755 GenerateMiss(masm); 756} 757 758 759Condition CompareIC::ComputeCondition(Token::Value op) { 760 switch (op) { 761 case Token::EQ_STRICT: 762 case Token::EQ: 763 return eq; 764 case Token::LT: 765 return lt; 766 case Token::GT: 767 return gt; 768 case Token::LTE: 769 return le; 770 case Token::GTE: 771 return ge; 772 default: 773 UNREACHABLE(); 774 return al; 775 } 776} 777 778 779bool CompareIC::HasInlinedSmiCode(Address address) { 780 // The address of the instruction following the call. 781 Address info_address = Assembler::return_address_from_call_start(address); 782 783 InstructionSequence* patch_info = InstructionSequence::At(info_address); 784 return patch_info->IsInlineData(); 785} 786 787 788// Activate a SMI fast-path by patching the instructions generated by 789// JumpPatchSite::EmitJumpIf(Not)Smi(), using the information encoded by 790// JumpPatchSite::EmitPatchInfo(). 791void PatchInlinedSmiCode(Isolate* isolate, Address address, 792 InlinedSmiCheck check) { 793 // The patch information is encoded in the instruction stream using 794 // instructions which have no side effects, so we can safely execute them. 795 // The patch information is encoded directly after the call to the helper 796 // function which is requesting this patch operation. 797 Address info_address = Assembler::return_address_from_call_start(address); 798 InlineSmiCheckInfo info(info_address); 799 800 // Check and decode the patch information instruction. 801 if (!info.HasSmiCheck()) { 802 return; 803 } 804 805 if (FLAG_trace_ic) { 806 PrintF("[ Patching ic at %p, marker=%p, SMI check=%p\n", 807 static_cast<void*>(address), static_cast<void*>(info_address), 808 static_cast<void*>(info.SmiCheck())); 809 } 810 811 // Patch and activate code generated by JumpPatchSite::EmitJumpIfNotSmi() 812 // and JumpPatchSite::EmitJumpIfSmi(). 813 // Changing 814 // tb(n)z xzr, #0, <target> 815 // to 816 // tb(!n)z test_reg, #0, <target> 817 Instruction* to_patch = info.SmiCheck(); 818 PatchingAssembler patcher(isolate, to_patch, 1); 819 DCHECK(to_patch->IsTestBranch()); 820 DCHECK(to_patch->ImmTestBranchBit5() == 0); 821 DCHECK(to_patch->ImmTestBranchBit40() == 0); 822 823 STATIC_ASSERT(kSmiTag == 0); 824 STATIC_ASSERT(kSmiTagMask == 1); 825 826 int branch_imm = to_patch->ImmTestBranch(); 827 Register smi_reg; 828 if (check == ENABLE_INLINED_SMI_CHECK) { 829 DCHECK(to_patch->Rt() == xzr.code()); 830 smi_reg = info.SmiRegister(); 831 } else { 832 DCHECK(check == DISABLE_INLINED_SMI_CHECK); 833 DCHECK(to_patch->Rt() != xzr.code()); 834 smi_reg = xzr; 835 } 836 837 if (to_patch->Mask(TestBranchMask) == TBZ) { 838 // This is JumpIfNotSmi(smi_reg, branch_imm). 839 patcher.tbnz(smi_reg, 0, branch_imm); 840 } else { 841 DCHECK(to_patch->Mask(TestBranchMask) == TBNZ); 842 // This is JumpIfSmi(smi_reg, branch_imm). 843 patcher.tbz(smi_reg, 0, branch_imm); 844 } 845} 846} // namespace internal 847} // namespace v8 848 849#endif // V8_TARGET_ARCH_ARM64 850