1/* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "HeifDecoderImpl" 19 20#include "HeifDecoderImpl.h" 21 22#include <stdio.h> 23 24#include <binder/IMemory.h> 25#include <drm/drm_framework_common.h> 26#include <media/IDataSource.h> 27#include <media/mediametadataretriever.h> 28#include <media/stagefright/foundation/ADebug.h> 29#include <media/stagefright/MediaSource.h> 30#include <private/media/VideoFrame.h> 31#include <utils/Log.h> 32#include <utils/RefBase.h> 33 34HeifDecoder* createHeifDecoder() { 35 return new android::HeifDecoderImpl(); 36} 37 38namespace android { 39 40/* 41 * HeifDataSource 42 * 43 * Proxies data requests over IDataSource interface from MediaMetadataRetriever 44 * to the HeifStream interface we received from the heif decoder client. 45 */ 46class HeifDataSource : public BnDataSource { 47public: 48 /* 49 * Constructs HeifDataSource; will take ownership of |stream|. 50 */ 51 HeifDataSource(HeifStream* stream) 52 : mStream(stream), mEOS(false), 53 mCachedOffset(0), mCachedSize(0), mCacheBufferSize(0) {} 54 55 ~HeifDataSource() override {} 56 57 /* 58 * Initializes internal resources. 59 */ 60 bool init(); 61 62 sp<IMemory> getIMemory() override { return mMemory; } 63 ssize_t readAt(off64_t offset, size_t size) override; 64 status_t getSize(off64_t* size) override ; 65 void close() {} 66 uint32_t getFlags() override { return 0; } 67 String8 toString() override { return String8("HeifDataSource"); } 68 sp<DecryptHandle> DrmInitialization(const char*) override { 69 return nullptr; 70 } 71 72private: 73 enum { 74 /* 75 * Buffer size for passing the read data to mediaserver. Set to 64K 76 * (which is what MediaDataSource Java API's jni implementation uses). 77 */ 78 kBufferSize = 64 * 1024, 79 /* 80 * Initial and max cache buffer size. 81 */ 82 kInitialCacheBufferSize = 4 * 1024 * 1024, 83 kMaxCacheBufferSize = 64 * 1024 * 1024, 84 }; 85 sp<IMemory> mMemory; 86 std::unique_ptr<HeifStream> mStream; 87 bool mEOS; 88 std::unique_ptr<uint8_t> mCache; 89 off64_t mCachedOffset; 90 size_t mCachedSize; 91 size_t mCacheBufferSize; 92}; 93 94bool HeifDataSource::init() { 95 sp<MemoryDealer> memoryDealer = 96 new MemoryDealer(kBufferSize, "HeifDataSource"); 97 mMemory = memoryDealer->allocate(kBufferSize); 98 if (mMemory == nullptr) { 99 ALOGE("Failed to allocate shared memory!"); 100 return false; 101 } 102 mCache.reset(new uint8_t[kInitialCacheBufferSize]); 103 if (mCache.get() == nullptr) { 104 ALOGE("mFailed to allocate cache!"); 105 return false; 106 } 107 mCacheBufferSize = kInitialCacheBufferSize; 108 return true; 109} 110 111ssize_t HeifDataSource::readAt(off64_t offset, size_t size) { 112 ALOGV("readAt: offset=%lld, size=%zu", (long long)offset, size); 113 114 if (offset < mCachedOffset) { 115 // try seek, then rewind/skip, fail if none worked 116 if (mStream->seek(offset)) { 117 ALOGV("readAt: seek to offset=%lld", (long long)offset); 118 mCachedOffset = offset; 119 mCachedSize = 0; 120 mEOS = false; 121 } else if (mStream->rewind()) { 122 ALOGV("readAt: rewind to offset=0"); 123 mCachedOffset = 0; 124 mCachedSize = 0; 125 mEOS = false; 126 } else { 127 ALOGE("readAt: couldn't seek or rewind!"); 128 mEOS = true; 129 } 130 } 131 132 if (mEOS && (offset < mCachedOffset || 133 offset >= (off64_t)(mCachedOffset + mCachedSize))) { 134 ALOGV("readAt: EOS"); 135 return ERROR_END_OF_STREAM; 136 } 137 138 // at this point, offset must be >= mCachedOffset, other cases should 139 // have been caught above. 140 CHECK(offset >= mCachedOffset); 141 142 off64_t resultOffset; 143 if (__builtin_add_overflow(offset, size, &resultOffset)) { 144 return ERROR_IO; 145 } 146 147 if (size == 0) { 148 return 0; 149 } 150 151 // Can only read max of kBufferSize 152 if (size > kBufferSize) { 153 size = kBufferSize; 154 } 155 156 // copy from cache if the request falls entirely in cache 157 if (offset + size <= mCachedOffset + mCachedSize) { 158 memcpy(mMemory->pointer(), mCache.get() + offset - mCachedOffset, size); 159 return size; 160 } 161 162 // need to fetch more, check if we need to expand the cache buffer. 163 if ((off64_t)(offset + size) > mCachedOffset + kMaxCacheBufferSize) { 164 // it's reaching max cache buffer size, need to roll window, and possibly 165 // expand the cache buffer. 166 size_t newCacheBufferSize = mCacheBufferSize; 167 std::unique_ptr<uint8_t> newCache; 168 uint8_t* dst = mCache.get(); 169 if (newCacheBufferSize < kMaxCacheBufferSize) { 170 newCacheBufferSize = kMaxCacheBufferSize; 171 newCache.reset(new uint8_t[newCacheBufferSize]); 172 dst = newCache.get(); 173 } 174 175 // when rolling the cache window, try to keep about half the old bytes 176 // in case that the client goes back. 177 off64_t newCachedOffset = offset - (off64_t)(newCacheBufferSize / 2); 178 if (newCachedOffset < mCachedOffset) { 179 newCachedOffset = mCachedOffset; 180 } 181 182 int64_t newCachedSize = (int64_t)(mCachedOffset + mCachedSize) - newCachedOffset; 183 if (newCachedSize > 0) { 184 // in this case, the new cache region partially overlop the old cache, 185 // move the portion of the cache we want to save to the beginning of 186 // the cache buffer. 187 memcpy(dst, mCache.get() + newCachedOffset - mCachedOffset, newCachedSize); 188 } else if (newCachedSize < 0){ 189 // in this case, the new cache region is entirely out of the old cache, 190 // in order to guarantee sequential read, we need to skip a number of 191 // bytes before reading. 192 size_t bytesToSkip = -newCachedSize; 193 size_t bytesSkipped = mStream->read(nullptr, bytesToSkip); 194 if (bytesSkipped != bytesToSkip) { 195 // bytesSkipped is invalid, there is not enough bytes to reach 196 // the requested offset. 197 ALOGE("readAt: skip failed, EOS"); 198 199 mEOS = true; 200 mCachedOffset = newCachedOffset; 201 mCachedSize = 0; 202 return ERROR_END_OF_STREAM; 203 } 204 // set cache size to 0, since we're not keeping any old cache 205 newCachedSize = 0; 206 } 207 208 if (newCache.get() != nullptr) { 209 mCache.reset(newCache.release()); 210 mCacheBufferSize = newCacheBufferSize; 211 } 212 mCachedOffset = newCachedOffset; 213 mCachedSize = newCachedSize; 214 215 ALOGV("readAt: rolling cache window to (%lld, %zu), cache buffer size %zu", 216 (long long)mCachedOffset, mCachedSize, mCacheBufferSize); 217 } else { 218 // expand cache buffer, but no need to roll the window 219 size_t newCacheBufferSize = mCacheBufferSize; 220 while (offset + size > mCachedOffset + newCacheBufferSize) { 221 newCacheBufferSize *= 2; 222 } 223 CHECK(newCacheBufferSize <= kMaxCacheBufferSize); 224 if (mCacheBufferSize < newCacheBufferSize) { 225 uint8_t* newCache = new uint8_t[newCacheBufferSize]; 226 memcpy(newCache, mCache.get(), mCachedSize); 227 mCache.reset(newCache); 228 mCacheBufferSize = newCacheBufferSize; 229 230 ALOGV("readAt: current cache window (%lld, %zu), new cache buffer size %zu", 231 (long long) mCachedOffset, mCachedSize, mCacheBufferSize); 232 } 233 } 234 size_t bytesToRead = offset + size - mCachedOffset - mCachedSize; 235 size_t bytesRead = mStream->read(mCache.get() + mCachedSize, bytesToRead); 236 if (bytesRead > bytesToRead || bytesRead == 0) { 237 // bytesRead is invalid 238 mEOS = true; 239 bytesRead = 0; 240 } else if (bytesRead < bytesToRead) { 241 // read some bytes but not all, set EOS 242 mEOS = true; 243 } 244 mCachedSize += bytesRead; 245 ALOGV("readAt: current cache window (%lld, %zu)", 246 (long long) mCachedOffset, mCachedSize); 247 248 // here bytesAvailable could be negative if offset jumped past EOS. 249 int64_t bytesAvailable = mCachedOffset + mCachedSize - offset; 250 if (bytesAvailable <= 0) { 251 return ERROR_END_OF_STREAM; 252 } 253 if (bytesAvailable < (int64_t)size) { 254 size = bytesAvailable; 255 } 256 memcpy(mMemory->pointer(), mCache.get() + offset - mCachedOffset, size); 257 return size; 258} 259 260status_t HeifDataSource::getSize(off64_t* size) { 261 if (!mStream->hasLength()) { 262 *size = -1; 263 ALOGE("getSize: not supported!"); 264 return ERROR_UNSUPPORTED; 265 } 266 *size = mStream->getLength(); 267 ALOGV("getSize: size=%lld", (long long)*size); 268 return OK; 269} 270 271///////////////////////////////////////////////////////////////////////// 272 273HeifDecoderImpl::HeifDecoderImpl() : 274 // output color format should always be set via setOutputColor(), in case 275 // it's not, default to HAL_PIXEL_FORMAT_RGB_565. 276 mOutputColor(HAL_PIXEL_FORMAT_RGB_565), 277 mCurScanline(0), 278 mFrameDecoded(false) { 279} 280 281HeifDecoderImpl::~HeifDecoderImpl() { 282} 283 284bool HeifDecoderImpl::init(HeifStream* stream, HeifFrameInfo* frameInfo) { 285 mFrameDecoded = false; 286 sp<HeifDataSource> dataSource = new HeifDataSource(stream); 287 if (!dataSource->init()) { 288 return false; 289 } 290 mDataSource = dataSource; 291 292 mRetriever = new MediaMetadataRetriever(); 293 status_t err = mRetriever->setDataSource(mDataSource, "video/mp4"); 294 if (err != OK) { 295 ALOGE("failed to set data source!"); 296 297 mRetriever.clear(); 298 mDataSource.clear(); 299 return false; 300 } 301 ALOGV("successfully set data source."); 302 303 const char* hasVideo = mRetriever->extractMetadata(METADATA_KEY_HAS_VIDEO); 304 if (!hasVideo || strcasecmp(hasVideo, "yes")) { 305 ALOGE("no video: %s", hasVideo ? hasVideo : "null"); 306 return false; 307 } 308 309 mFrameMemory = mRetriever->getFrameAtTime(0, 310 IMediaSource::ReadOptions::SEEK_PREVIOUS_SYNC, 311 mOutputColor, true /*metaOnly*/); 312 if (mFrameMemory == nullptr || mFrameMemory->pointer() == nullptr) { 313 ALOGE("getFrameAtTime: videoFrame is a nullptr"); 314 return false; 315 } 316 317 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->pointer()); 318 319 ALOGV("Meta dimension %dx%d, display %dx%d, angle %d, iccSize %d", 320 videoFrame->mWidth, 321 videoFrame->mHeight, 322 videoFrame->mDisplayWidth, 323 videoFrame->mDisplayHeight, 324 videoFrame->mRotationAngle, 325 videoFrame->mIccSize); 326 327 if (frameInfo != nullptr) { 328 frameInfo->set( 329 videoFrame->mWidth, 330 videoFrame->mHeight, 331 videoFrame->mRotationAngle, 332 videoFrame->mBytesPerPixel, 333 videoFrame->mIccSize, 334 videoFrame->getFlattenedIccData()); 335 } 336 return true; 337} 338 339bool HeifDecoderImpl::getEncodedColor(HeifEncodedColor* /*outColor*/) const { 340 ALOGW("getEncodedColor: not implemented!"); 341 return false; 342} 343 344bool HeifDecoderImpl::setOutputColor(HeifColorFormat heifColor) { 345 switch(heifColor) { 346 case kHeifColorFormat_RGB565: 347 { 348 mOutputColor = HAL_PIXEL_FORMAT_RGB_565; 349 return true; 350 } 351 case kHeifColorFormat_RGBA_8888: 352 { 353 mOutputColor = HAL_PIXEL_FORMAT_RGBA_8888; 354 return true; 355 } 356 case kHeifColorFormat_BGRA_8888: 357 { 358 mOutputColor = HAL_PIXEL_FORMAT_BGRA_8888; 359 return true; 360 } 361 default: 362 break; 363 } 364 ALOGE("Unsupported output color format %d", heifColor); 365 return false; 366} 367 368bool HeifDecoderImpl::decode(HeifFrameInfo* frameInfo) { 369 // reset scanline pointer 370 mCurScanline = 0; 371 372 if (mFrameDecoded) { 373 return true; 374 } 375 376 mFrameMemory = mRetriever->getFrameAtTime(0, 377 IMediaSource::ReadOptions::SEEK_PREVIOUS_SYNC, mOutputColor); 378 if (mFrameMemory == nullptr || mFrameMemory->pointer() == nullptr) { 379 ALOGE("getFrameAtTime: videoFrame is a nullptr"); 380 return false; 381 } 382 383 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->pointer()); 384 if (videoFrame->mSize == 0 || 385 mFrameMemory->size() < videoFrame->getFlattenedSize()) { 386 ALOGE("getFrameAtTime: videoFrame size is invalid"); 387 return false; 388 } 389 390 ALOGV("Decoded dimension %dx%d, display %dx%d, angle %d, rowbytes %d, size %d", 391 videoFrame->mWidth, 392 videoFrame->mHeight, 393 videoFrame->mDisplayWidth, 394 videoFrame->mDisplayHeight, 395 videoFrame->mRotationAngle, 396 videoFrame->mRowBytes, 397 videoFrame->mSize); 398 399 if (frameInfo != nullptr) { 400 frameInfo->set( 401 videoFrame->mWidth, 402 videoFrame->mHeight, 403 videoFrame->mRotationAngle, 404 videoFrame->mBytesPerPixel, 405 videoFrame->mIccSize, 406 videoFrame->getFlattenedIccData()); 407 } 408 mFrameDecoded = true; 409 410 // Aggressive clear to avoid holding on to resources 411 mRetriever.clear(); 412 mDataSource.clear(); 413 return true; 414} 415 416bool HeifDecoderImpl::getScanline(uint8_t* dst) { 417 if (mFrameMemory == nullptr || mFrameMemory->pointer() == nullptr) { 418 return false; 419 } 420 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->pointer()); 421 if (mCurScanline >= videoFrame->mHeight) { 422 ALOGE("no more scanline available"); 423 return false; 424 } 425 uint8_t* src = videoFrame->getFlattenedData() + videoFrame->mRowBytes * mCurScanline++; 426 memcpy(dst, src, videoFrame->mBytesPerPixel * videoFrame->mWidth); 427 return true; 428} 429 430size_t HeifDecoderImpl::skipScanlines(size_t count) { 431 if (mFrameMemory == nullptr || mFrameMemory->pointer() == nullptr) { 432 return 0; 433 } 434 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->pointer()); 435 436 uint32_t oldScanline = mCurScanline; 437 mCurScanline += count; 438 if (mCurScanline > videoFrame->mHeight) { 439 mCurScanline = videoFrame->mHeight; 440 } 441 return (mCurScanline > oldScanline) ? (mCurScanline - oldScanline) : 0; 442} 443 444} // namespace android 445