1b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier/* 2b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier * Copyright (C) 2008 The Android Open Source Project 3b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier * 4b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier * Licensed under the Apache License, Version 2.0 (the "License"); 5b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier * you may not use this file except in compliance with the License. 6b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier * You may obtain a copy of the License at 7b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier * 8b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier * http://www.apache.org/licenses/LICENSE-2.0 9b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier * 10b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier * Unless required by applicable law or agreed to in writing, software 11b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier * distributed under the License is distributed on an "AS IS" BASIS, 12b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier * See the License for the specific language governing permissions and 14b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier * limitations under the License. 15b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier */ 16b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier 172dd0e2cea360bc9206eb88ecc40d259e796c239dIan Rogers#include "space_bitmap-inl.h" 18b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier 1946ee31b67d7ee1bd085fbc240502053caa3cf8faAndreas Gampe#include "android-base/stringprintf.h" 2046ee31b67d7ee1bd085fbc240502053caa3cf8faAndreas Gampe 21c785344b87221f5e4e6473e5b762e4e61fe65dcfMathieu Chartier#include "art_field-inl.h" 229e734c7ab4599d7747a05db0dc73c7b668cb6683David Sehr#include "dex/dex_file-inl.h" 2322d5e735f403c57525fe868304c7123f0ce66399Ian Rogers#include "mem_map.h" 24e401d146407d61eeb99f8d6176b2ac13c4df1e33Mathieu Chartier#include "mirror/class-inl.h" 258cf9cb386cd9286d67e879f1ee501ec00d72a4e1Andreas Gampe#include "mirror/object-inl.h" 2622d5e735f403c57525fe868304c7123f0ce66399Ian Rogers#include "mirror/object_array.h" 2722d5e735f403c57525fe868304c7123f0ce66399Ian Rogers 28b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartiernamespace art { 291d54e73444e017d3a65234e0f193846f3e27472bIan Rogersnamespace gc { 301d54e73444e017d3a65234e0f193846f3e27472bIan Rogersnamespace accounting { 31b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier 3246ee31b67d7ee1bd085fbc240502053caa3cf8faAndreas Gampeusing android::base::StringPrintf; 3346ee31b67d7ee1bd085fbc240502053caa3cf8faAndreas Gampe 34a8e8f9c0a8e259a807d7b99a148d14104c24209dMathieu Chartiertemplate<size_t kAlignment> 3573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartiersize_t SpaceBitmap<kAlignment>::ComputeBitmapSize(uint64_t capacity) { 368f7ea9ab1703ef52c0c5ca3490e2913ac67f2a16Roland Levillain // Number of space (heap) bytes covered by one bitmap word. 378f7ea9ab1703ef52c0c5ca3490e2913ac67f2a16Roland Levillain // (Word size in bytes = `sizeof(intptr_t)`, which is expected to be 388f7ea9ab1703ef52c0c5ca3490e2913ac67f2a16Roland Levillain // 4 on a 32-bit architecture and 8 on a 64-bit one.) 3913735955f39b3b304c37d2b2840663c131262c18Ian Rogers const uint64_t kBytesCoveredPerWord = kAlignment * kBitsPerIntPtrT; 408f7ea9ab1703ef52c0c5ca3490e2913ac67f2a16Roland Levillain // Calculate the number of words required to cover a space (heap) 418f7ea9ab1703ef52c0c5ca3490e2913ac67f2a16Roland Levillain // having a size of `capacity` bytes. 4213735955f39b3b304c37d2b2840663c131262c18Ian Rogers return (RoundUp(capacity, kBytesCoveredPerWord) / kBytesCoveredPerWord) * sizeof(intptr_t); 4373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier} 4473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier 4573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartiertemplate<size_t kAlignment> 46d39645e22b8db1767cf64dc1200a9e4b2f939ed2Mathieu Chartiersize_t SpaceBitmap<kAlignment>::ComputeHeapSize(uint64_t bitmap_bytes) { 47d39645e22b8db1767cf64dc1200a9e4b2f939ed2Mathieu Chartier return bitmap_bytes * kBitsPerByte * kAlignment; 48d39645e22b8db1767cf64dc1200a9e4b2f939ed2Mathieu Chartier} 49d39645e22b8db1767cf64dc1200a9e4b2f939ed2Mathieu Chartier 50d39645e22b8db1767cf64dc1200a9e4b2f939ed2Mathieu Chartiertemplate<size_t kAlignment> 51a8e8f9c0a8e259a807d7b99a148d14104c24209dMathieu ChartierSpaceBitmap<kAlignment>* SpaceBitmap<kAlignment>::CreateFromMemMap( 5213735955f39b3b304c37d2b2840663c131262c18Ian Rogers const std::string& name, MemMap* mem_map, uint8_t* heap_begin, size_t heap_capacity) { 5331e8925781c2302f1d1a9b39e216ba415bfe0d7eMathieu Chartier CHECK(mem_map != nullptr); 5413735955f39b3b304c37d2b2840663c131262c18Ian Rogers uintptr_t* bitmap_begin = reinterpret_cast<uintptr_t*>(mem_map->Begin()); 5573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier const size_t bitmap_size = ComputeBitmapSize(heap_capacity); 562a1513bdb0a3ecac628cd5620420d66c28413f1eMathieu Chartier return new SpaceBitmap(name, mem_map, bitmap_begin, bitmap_size, heap_begin, heap_capacity); 5731e8925781c2302f1d1a9b39e216ba415bfe0d7eMathieu Chartier} 5831e8925781c2302f1d1a9b39e216ba415bfe0d7eMathieu Chartier 59a8e8f9c0a8e259a807d7b99a148d14104c24209dMathieu Chartiertemplate<size_t kAlignment> 602a1513bdb0a3ecac628cd5620420d66c28413f1eMathieu ChartierSpaceBitmap<kAlignment>::SpaceBitmap(const std::string& name, 612a1513bdb0a3ecac628cd5620420d66c28413f1eMathieu Chartier MemMap* mem_map, 622a1513bdb0a3ecac628cd5620420d66c28413f1eMathieu Chartier uintptr_t* bitmap_begin, 632a1513bdb0a3ecac628cd5620420d66c28413f1eMathieu Chartier size_t bitmap_size, 642a1513bdb0a3ecac628cd5620420d66c28413f1eMathieu Chartier const void* heap_begin, 652a1513bdb0a3ecac628cd5620420d66c28413f1eMathieu Chartier size_t heap_capacity) 66c381c36aacf977f7e314e6a91e47b31b04639f62Mathieu Chartier : mem_map_(mem_map), 67c381c36aacf977f7e314e6a91e47b31b04639f62Mathieu Chartier bitmap_begin_(reinterpret_cast<Atomic<uintptr_t>*>(bitmap_begin)), 68c381c36aacf977f7e314e6a91e47b31b04639f62Mathieu Chartier bitmap_size_(bitmap_size), 69bbd695c71e0bf518f582e84524e1cdeb3de3896cMathieu Chartier heap_begin_(reinterpret_cast<uintptr_t>(heap_begin)), 702a1513bdb0a3ecac628cd5620420d66c28413f1eMathieu Chartier heap_limit_(reinterpret_cast<uintptr_t>(heap_begin) + heap_capacity), 71bbd695c71e0bf518f582e84524e1cdeb3de3896cMathieu Chartier name_(name) { 72bbd695c71e0bf518f582e84524e1cdeb3de3896cMathieu Chartier CHECK(bitmap_begin_ != nullptr); 73bbd695c71e0bf518f582e84524e1cdeb3de3896cMathieu Chartier CHECK_NE(bitmap_size, 0U); 74bbd695c71e0bf518f582e84524e1cdeb3de3896cMathieu Chartier} 75bbd695c71e0bf518f582e84524e1cdeb3de3896cMathieu Chartier 76bbd695c71e0bf518f582e84524e1cdeb3de3896cMathieu Chartiertemplate<size_t kAlignment> 7722d5e735f403c57525fe868304c7123f0ce66399Ian RogersSpaceBitmap<kAlignment>::~SpaceBitmap() {} 7822d5e735f403c57525fe868304c7123f0ce66399Ian Rogers 7922d5e735f403c57525fe868304c7123f0ce66399Ian Rogerstemplate<size_t kAlignment> 80a8e8f9c0a8e259a807d7b99a148d14104c24209dMathieu ChartierSpaceBitmap<kAlignment>* SpaceBitmap<kAlignment>::Create( 8113735955f39b3b304c37d2b2840663c131262c18Ian Rogers const std::string& name, uint8_t* heap_begin, size_t heap_capacity) { 828f7ea9ab1703ef52c0c5ca3490e2913ac67f2a16Roland Levillain // Round up since `heap_capacity` is not necessarily a multiple of `kAlignment * kBitsPerIntPtrT` 838f7ea9ab1703ef52c0c5ca3490e2913ac67f2a16Roland Levillain // (we represent one word as an `intptr_t`). 8473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier const size_t bitmap_size = ComputeBitmapSize(heap_capacity); 858d31bbd3d6536de12bc20e3d29cfe03fe848f9daIan Rogers std::string error_msg; 86700a402244a1a423da4f3ba8032459f4b65fa18fIan Rogers std::unique_ptr<MemMap> mem_map(MemMap::MapAnonymous(name.c_str(), nullptr, bitmap_size, 875c42c29b89286e5efa4a4613132b09051ce5945bVladimir Marko PROT_READ | PROT_WRITE, false, false, 885c42c29b89286e5efa4a4613132b09051ce5945bVladimir Marko &error_msg)); 898d31bbd3d6536de12bc20e3d29cfe03fe848f9daIan Rogers if (UNLIKELY(mem_map.get() == nullptr)) { 908d31bbd3d6536de12bc20e3d29cfe03fe848f9daIan Rogers LOG(ERROR) << "Failed to allocate bitmap " << name << ": " << error_msg; 91bbd695c71e0bf518f582e84524e1cdeb3de3896cMathieu Chartier return nullptr; 92b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier } 9331e8925781c2302f1d1a9b39e216ba415bfe0d7eMathieu Chartier return CreateFromMemMap(name, mem_map.release(), heap_begin, heap_capacity); 94b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier} 95b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier 96a8e8f9c0a8e259a807d7b99a148d14104c24209dMathieu Chartiertemplate<size_t kAlignment> 97a8e8f9c0a8e259a807d7b99a148d14104c24209dMathieu Chartiervoid SpaceBitmap<kAlignment>::SetHeapLimit(uintptr_t new_end) { 9814d90579f013b374638b599361970557ed4b3f09Roland Levillain DCHECK_ALIGNED(new_end, kBitsPerIntPtrT * kAlignment); 9913735955f39b3b304c37d2b2840663c131262c18Ian Rogers size_t new_size = OffsetToIndex(new_end - heap_begin_) * sizeof(intptr_t); 100cc236d74772dda5a4161d9bc5f497fd3d956eb87Mathieu Chartier if (new_size < bitmap_size_) { 101cc236d74772dda5a4161d9bc5f497fd3d956eb87Mathieu Chartier bitmap_size_ = new_size; 102cc236d74772dda5a4161d9bc5f497fd3d956eb87Mathieu Chartier } 1032a1513bdb0a3ecac628cd5620420d66c28413f1eMathieu Chartier heap_limit_ = new_end; 104cc236d74772dda5a4161d9bc5f497fd3d956eb87Mathieu Chartier // Not sure if doing this trim is necessary, since nothing past the end of the heap capacity 105cc236d74772dda5a4161d9bc5f497fd3d956eb87Mathieu Chartier // should be marked. 106cc236d74772dda5a4161d9bc5f497fd3d956eb87Mathieu Chartier} 107cc236d74772dda5a4161d9bc5f497fd3d956eb87Mathieu Chartier 108a8e8f9c0a8e259a807d7b99a148d14104c24209dMathieu Chartiertemplate<size_t kAlignment> 109576ca0cd692c0b6ae70e776de91015b8ff000a08Ian Rogersstd::string SpaceBitmap<kAlignment>::Dump() const { 110576ca0cd692c0b6ae70e776de91015b8ff000a08Ian Rogers return StringPrintf("%s: %p-%p", name_.c_str(), reinterpret_cast<void*>(HeapBegin()), 111576ca0cd692c0b6ae70e776de91015b8ff000a08Ian Rogers reinterpret_cast<void*>(HeapLimit())); 112576ca0cd692c0b6ae70e776de91015b8ff000a08Ian Rogers} 113576ca0cd692c0b6ae70e776de91015b8ff000a08Ian Rogers 114576ca0cd692c0b6ae70e776de91015b8ff000a08Ian Rogerstemplate<size_t kAlignment> 115a8e8f9c0a8e259a807d7b99a148d14104c24209dMathieu Chartiervoid SpaceBitmap<kAlignment>::Clear() { 116c5f17732d8144491c642776b6b48c85dfadf4b52Ian Rogers if (bitmap_begin_ != nullptr) { 117c5f17732d8144491c642776b6b48c85dfadf4b52Ian Rogers mem_map_->MadviseDontNeedAndZero(); 118b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier } 119b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier} 120b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier 121a8e8f9c0a8e259a807d7b99a148d14104c24209dMathieu Chartiertemplate<size_t kAlignment> 1227ec38dcb84d61f6172bbb5a303bd5ab7139f7409Mathieu Chartiervoid SpaceBitmap<kAlignment>::ClearRange(const mirror::Object* begin, const mirror::Object* end) { 1237ec38dcb84d61f6172bbb5a303bd5ab7139f7409Mathieu Chartier uintptr_t begin_offset = reinterpret_cast<uintptr_t>(begin) - heap_begin_; 1247ec38dcb84d61f6172bbb5a303bd5ab7139f7409Mathieu Chartier uintptr_t end_offset = reinterpret_cast<uintptr_t>(end) - heap_begin_; 1258f7ea9ab1703ef52c0c5ca3490e2913ac67f2a16Roland Levillain // Align begin and end to bitmap word boundaries. 1267ec38dcb84d61f6172bbb5a303bd5ab7139f7409Mathieu Chartier while (begin_offset < end_offset && OffsetBitIndex(begin_offset) != 0) { 1277ec38dcb84d61f6172bbb5a303bd5ab7139f7409Mathieu Chartier Clear(reinterpret_cast<mirror::Object*>(heap_begin_ + begin_offset)); 1287ec38dcb84d61f6172bbb5a303bd5ab7139f7409Mathieu Chartier begin_offset += kAlignment; 1297ec38dcb84d61f6172bbb5a303bd5ab7139f7409Mathieu Chartier } 1307ec38dcb84d61f6172bbb5a303bd5ab7139f7409Mathieu Chartier while (begin_offset < end_offset && OffsetBitIndex(end_offset) != 0) { 1317ec38dcb84d61f6172bbb5a303bd5ab7139f7409Mathieu Chartier end_offset -= kAlignment; 1327ec38dcb84d61f6172bbb5a303bd5ab7139f7409Mathieu Chartier Clear(reinterpret_cast<mirror::Object*>(heap_begin_ + end_offset)); 1337ec38dcb84d61f6172bbb5a303bd5ab7139f7409Mathieu Chartier } 1348f7ea9ab1703ef52c0c5ca3490e2913ac67f2a16Roland Levillain // Bitmap word boundaries. 1357ec38dcb84d61f6172bbb5a303bd5ab7139f7409Mathieu Chartier const uintptr_t start_index = OffsetToIndex(begin_offset); 1367ec38dcb84d61f6172bbb5a303bd5ab7139f7409Mathieu Chartier const uintptr_t end_index = OffsetToIndex(end_offset); 1376e6078ab06e16bb4032b363316360f3b76685e30Mathieu Chartier ZeroAndReleasePages(reinterpret_cast<uint8_t*>(&bitmap_begin_[start_index]), 1386e6078ab06e16bb4032b363316360f3b76685e30Mathieu Chartier (end_index - start_index) * sizeof(*bitmap_begin_)); 1397ec38dcb84d61f6172bbb5a303bd5ab7139f7409Mathieu Chartier} 1407ec38dcb84d61f6172bbb5a303bd5ab7139f7409Mathieu Chartier 1417ec38dcb84d61f6172bbb5a303bd5ab7139f7409Mathieu Chartiertemplate<size_t kAlignment> 142bbd695c71e0bf518f582e84524e1cdeb3de3896cMathieu Chartiervoid SpaceBitmap<kAlignment>::CopyFrom(SpaceBitmap* source_bitmap) { 143357e9be24c17a6bc2ae9fb53f25c73503116101dMathieu Chartier DCHECK_EQ(Size(), source_bitmap->Size()); 144c381c36aacf977f7e314e6a91e47b31b04639f62Mathieu Chartier const size_t count = source_bitmap->Size() / sizeof(intptr_t); 145c381c36aacf977f7e314e6a91e47b31b04639f62Mathieu Chartier Atomic<uintptr_t>* const src = source_bitmap->Begin(); 146c381c36aacf977f7e314e6a91e47b31b04639f62Mathieu Chartier Atomic<uintptr_t>* const dest = Begin(); 147c381c36aacf977f7e314e6a91e47b31b04639f62Mathieu Chartier for (size_t i = 0; i < count; ++i) { 148c381c36aacf977f7e314e6a91e47b31b04639f62Mathieu Chartier dest[i].StoreRelaxed(src[i].LoadRelaxed()); 149c381c36aacf977f7e314e6a91e47b31b04639f62Mathieu Chartier } 150357e9be24c17a6bc2ae9fb53f25c73503116101dMathieu Chartier} 151357e9be24c17a6bc2ae9fb53f25c73503116101dMathieu Chartier 152a8e8f9c0a8e259a807d7b99a148d14104c24209dMathieu Chartiertemplate<size_t kAlignment> 153a8e8f9c0a8e259a807d7b99a148d14104c24209dMathieu Chartiervoid SpaceBitmap<kAlignment>::SweepWalk(const SpaceBitmap<kAlignment>& live_bitmap, 154bbd695c71e0bf518f582e84524e1cdeb3de3896cMathieu Chartier const SpaceBitmap<kAlignment>& mark_bitmap, 155bbd695c71e0bf518f582e84524e1cdeb3de3896cMathieu Chartier uintptr_t sweep_begin, uintptr_t sweep_end, 156bbd695c71e0bf518f582e84524e1cdeb3de3896cMathieu Chartier SpaceBitmap::SweepCallback* callback, void* arg) { 157bbd695c71e0bf518f582e84524e1cdeb3de3896cMathieu Chartier CHECK(live_bitmap.bitmap_begin_ != nullptr); 158bbd695c71e0bf518f582e84524e1cdeb3de3896cMathieu Chartier CHECK(mark_bitmap.bitmap_begin_ != nullptr); 159b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier CHECK_EQ(live_bitmap.heap_begin_, mark_bitmap.heap_begin_); 160b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier CHECK_EQ(live_bitmap.bitmap_size_, mark_bitmap.bitmap_size_); 1612cebb24bfc3247d3e9be138a3350106737455918Mathieu Chartier CHECK(callback != nullptr); 162b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier CHECK_LE(sweep_begin, sweep_end); 163b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier CHECK_GE(sweep_begin, live_bitmap.heap_begin_); 164357e9be24c17a6bc2ae9fb53f25c73503116101dMathieu Chartier 165357e9be24c17a6bc2ae9fb53f25c73503116101dMathieu Chartier if (sweep_end <= sweep_begin) { 166b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier return; 167b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier } 168357e9be24c17a6bc2ae9fb53f25c73503116101dMathieu Chartier 1692dd0e2cea360bc9206eb88ecc40d259e796c239dIan Rogers // TODO: rewrite the callbacks to accept a std::vector<mirror::Object*> rather than a mirror::Object**? 17013735955f39b3b304c37d2b2840663c131262c18Ian Rogers constexpr size_t buffer_size = sizeof(intptr_t) * kBitsPerIntPtrT; 171262a0a36ee5f553880365fecf0be1a188526ca69Andreas Gampe#ifdef __LP64__ 172262a0a36ee5f553880365fecf0be1a188526ca69Andreas Gampe // Heap-allocate for smaller stack frame. 173262a0a36ee5f553880365fecf0be1a188526ca69Andreas Gampe std::unique_ptr<mirror::Object*[]> pointer_buf_ptr(new mirror::Object*[buffer_size]); 174262a0a36ee5f553880365fecf0be1a188526ca69Andreas Gampe mirror::Object** pointer_buf = pointer_buf_ptr.get(); 175262a0a36ee5f553880365fecf0be1a188526ca69Andreas Gampe#else 176262a0a36ee5f553880365fecf0be1a188526ca69Andreas Gampe // Stack-allocate buffer as it's small enough. 1772dd0e2cea360bc9206eb88ecc40d259e796c239dIan Rogers mirror::Object* pointer_buf[buffer_size]; 178262a0a36ee5f553880365fecf0be1a188526ca69Andreas Gampe#endif 1792dd0e2cea360bc9206eb88ecc40d259e796c239dIan Rogers mirror::Object** pb = &pointer_buf[0]; 180262a0a36ee5f553880365fecf0be1a188526ca69Andreas Gampe 181b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier size_t start = OffsetToIndex(sweep_begin - live_bitmap.heap_begin_); 182357e9be24c17a6bc2ae9fb53f25c73503116101dMathieu Chartier size_t end = OffsetToIndex(sweep_end - live_bitmap.heap_begin_ - 1); 18313735955f39b3b304c37d2b2840663c131262c18Ian Rogers CHECK_LT(end, live_bitmap.Size() / sizeof(intptr_t)); 184c381c36aacf977f7e314e6a91e47b31b04639f62Mathieu Chartier Atomic<uintptr_t>* live = live_bitmap.bitmap_begin_; 185c381c36aacf977f7e314e6a91e47b31b04639f62Mathieu Chartier Atomic<uintptr_t>* mark = mark_bitmap.bitmap_begin_; 186b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier for (size_t i = start; i <= end; i++) { 187c381c36aacf977f7e314e6a91e47b31b04639f62Mathieu Chartier uintptr_t garbage = live[i].LoadRelaxed() & ~mark[i].LoadRelaxed(); 188b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier if (UNLIKELY(garbage != 0)) { 189b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier uintptr_t ptr_base = IndexToOffset(i) + live_bitmap.heap_begin_; 190357e9be24c17a6bc2ae9fb53f25c73503116101dMathieu Chartier do { 191cb8aea4bd5e77857c713edeb62bffcb1f7f06a39Andreas Gampe const size_t shift = CTZ(garbage); 19213735955f39b3b304c37d2b2840663c131262c18Ian Rogers garbage ^= (static_cast<uintptr_t>(1)) << shift; 1932dd0e2cea360bc9206eb88ecc40d259e796c239dIan Rogers *pb++ = reinterpret_cast<mirror::Object*>(ptr_base + shift * kAlignment); 194357e9be24c17a6bc2ae9fb53f25c73503116101dMathieu Chartier } while (garbage != 0); 195b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier // Make sure that there are always enough slots available for an 196b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier // entire word of one bits. 19713735955f39b3b304c37d2b2840663c131262c18Ian Rogers if (pb >= &pointer_buf[buffer_size - kBitsPerIntPtrT]) { 198b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier (*callback)(pb - &pointer_buf[0], &pointer_buf[0], arg); 199b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier pb = &pointer_buf[0]; 200b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier } 201b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier } 202b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier } 203b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier if (pb > &pointer_buf[0]) { 204b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier (*callback)(pb - &pointer_buf[0], &pointer_buf[0], arg); 205b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier } 206b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier} 207b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier 208a8e8f9c0a8e259a807d7b99a148d14104c24209dMathieu Chartiertemplate class SpaceBitmap<kObjectAlignment>; 209a8e8f9c0a8e259a807d7b99a148d14104c24209dMathieu Chartiertemplate class SpaceBitmap<kPageSize>; 210a8e8f9c0a8e259a807d7b99a148d14104c24209dMathieu Chartier 2111d54e73444e017d3a65234e0f193846f3e27472bIan Rogers} // namespace accounting 2121d54e73444e017d3a65234e0f193846f3e27472bIan Rogers} // namespace gc 213b062fdd4cb097fbae69b4bcb479c34d83ecab8caMathieu Chartier} // namespace art 214