1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "WebmFrameThread" 19 20#include "WebmConstants.h" 21#include "WebmFrameThread.h" 22 23#include <media/stagefright/MetaData.h> 24#include <media/stagefright/foundation/ADebug.h> 25 26#include <utils/Log.h> 27#include <inttypes.h> 28 29using namespace webm; 30 31namespace android { 32 33void *WebmFrameThread::wrap(void *arg) { 34 WebmFrameThread *worker = reinterpret_cast<WebmFrameThread*>(arg); 35 worker->run(); 36 return NULL; 37} 38 39status_t WebmFrameThread::start() { 40 status_t err = OK; 41 pthread_attr_t attr; 42 pthread_attr_init(&attr); 43 pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); 44 if ((err = pthread_create(&mThread, &attr, WebmFrameThread::wrap, this))) { 45 mThread = 0; 46 } 47 pthread_attr_destroy(&attr); 48 return err; 49} 50 51status_t WebmFrameThread::stop() { 52 void *status = nullptr; 53 if (mThread) { 54 pthread_join(mThread, &status); 55 mThread = 0; 56 } 57 return (status_t)(intptr_t)status; 58} 59 60//================================================================================================= 61 62WebmFrameSourceThread::WebmFrameSourceThread( 63 int type, 64 LinkedBlockingQueue<const sp<WebmFrame> >& sink) 65 : mType(type), mSink(sink) { 66} 67 68//================================================================================================= 69 70WebmFrameSinkThread::WebmFrameSinkThread( 71 const int& fd, 72 const uint64_t& off, 73 sp<WebmFrameSourceThread> videoThread, 74 sp<WebmFrameSourceThread> audioThread, 75 List<sp<WebmElement> >& cues) 76 : mFd(fd), 77 mSegmentDataStart(off), 78 mVideoFrames(videoThread->mSink), 79 mAudioFrames(audioThread->mSink), 80 mCues(cues), 81 mDone(true) { 82} 83 84WebmFrameSinkThread::WebmFrameSinkThread( 85 const int& fd, 86 const uint64_t& off, 87 LinkedBlockingQueue<const sp<WebmFrame> >& videoSource, 88 LinkedBlockingQueue<const sp<WebmFrame> >& audioSource, 89 List<sp<WebmElement> >& cues) 90 : mFd(fd), 91 mSegmentDataStart(off), 92 mVideoFrames(videoSource), 93 mAudioFrames(audioSource), 94 mCues(cues), 95 mDone(true) { 96} 97 98// Initializes a webm cluster with its starting timecode. 99// 100// frames: 101// sequence of input audio/video frames received from the source. 102// 103// clusterTimecodeL: 104// the starting timecode of the cluster; this is the timecode of the first 105// frame since frames are ordered by timestamp. 106// 107// children: 108// list to hold child elements in a webm cluster (start timecode and 109// simple blocks). 110// 111// static 112void WebmFrameSinkThread::initCluster( 113 List<const sp<WebmFrame> >& frames, 114 uint64_t& clusterTimecodeL, 115 List<sp<WebmElement> >& children) { 116 CHECK(!frames.empty() && children.empty()); 117 118 const sp<WebmFrame> f = *(frames.begin()); 119 clusterTimecodeL = f->mAbsTimecode; 120 WebmUnsigned *clusterTimecode = new WebmUnsigned(kMkvTimecode, clusterTimecodeL); 121 children.clear(); 122 children.push_back(clusterTimecode); 123} 124 125void WebmFrameSinkThread::writeCluster(List<sp<WebmElement> >& children) { 126 // children must contain at least one simpleblock and its timecode 127 CHECK_GE(children.size(), 2); 128 129 uint64_t size; 130 sp<WebmElement> cluster = new WebmMaster(kMkvCluster, children); 131 cluster->write(mFd, size); 132 children.clear(); 133} 134 135// Write out (possibly multiple) webm cluster(s) from frames split on video key frames. 136// 137// last: 138// current flush is triggered by EOS instead of a second outstanding video key frame. 139void WebmFrameSinkThread::flushFrames(List<const sp<WebmFrame> >& frames, bool last) { 140 if (frames.empty()) { 141 return; 142 } 143 144 uint64_t clusterTimecodeL; 145 List<sp<WebmElement> > children; 146 initCluster(frames, clusterTimecodeL, children); 147 148 uint64_t cueTime = clusterTimecodeL; 149 off_t fpos = ::lseek(mFd, 0, SEEK_CUR); 150 size_t n = frames.size(); 151 if (!last) { 152 // If we are not flushing the last sequence of outstanding frames, flushFrames 153 // must have been called right after we have pushed a second outstanding video key 154 // frame (the last frame), which belongs to the next cluster; also hold back on 155 // flushing the second to last frame before we check its type. A audio frame 156 // should precede the aforementioned video key frame in the next sequence, a video 157 // frame should be the last frame in the current (to-be-flushed) sequence. 158 CHECK_GE(n, 2); 159 n -= 2; 160 } 161 162 for (size_t i = 0; i < n; i++) { 163 const sp<WebmFrame> f = *(frames.begin()); 164 if (f->mType == kVideoType && f->mKey) { 165 cueTime = f->mAbsTimecode; 166 } 167 168 if (f->mAbsTimecode - clusterTimecodeL > INT16_MAX) { 169 writeCluster(children); 170 initCluster(frames, clusterTimecodeL, children); 171 } 172 173 frames.erase(frames.begin()); 174 children.push_back(f->SimpleBlock(clusterTimecodeL)); 175 } 176 177 // equivalent to last==false 178 if (!frames.empty()) { 179 // decide whether to write out the second to last frame. 180 const sp<WebmFrame> secondLastFrame = *(frames.begin()); 181 if (secondLastFrame->mType == kVideoType) { 182 frames.erase(frames.begin()); 183 children.push_back(secondLastFrame->SimpleBlock(clusterTimecodeL)); 184 } 185 } 186 187 writeCluster(children); 188 sp<WebmElement> cuePoint = WebmElement::CuePointEntry(cueTime, 1, fpos - mSegmentDataStart); 189 mCues.push_back(cuePoint); 190} 191 192status_t WebmFrameSinkThread::start() { 193 mDone = false; 194 return WebmFrameThread::start(); 195} 196 197status_t WebmFrameSinkThread::stop() { 198 mDone = true; 199 mVideoFrames.push(WebmFrame::EOS); 200 mAudioFrames.push(WebmFrame::EOS); 201 return WebmFrameThread::stop(); 202} 203 204void WebmFrameSinkThread::run() { 205 int numVideoKeyFrames = 0; 206 List<const sp<WebmFrame> > outstandingFrames; 207 while (!mDone) { 208 ALOGV("wait v frame"); 209 const sp<WebmFrame> videoFrame = mVideoFrames.peek(); 210 ALOGV("v frame: %p", videoFrame.get()); 211 212 ALOGV("wait a frame"); 213 const sp<WebmFrame> audioFrame = mAudioFrames.peek(); 214 ALOGV("a frame: %p", audioFrame.get()); 215 216 if (videoFrame->mEos && audioFrame->mEos) { 217 break; 218 } 219 220 if (*audioFrame < *videoFrame) { 221 ALOGV("take a frame"); 222 mAudioFrames.take(); 223 outstandingFrames.push_back(audioFrame); 224 } else { 225 ALOGV("take v frame"); 226 mVideoFrames.take(); 227 outstandingFrames.push_back(videoFrame); 228 if (videoFrame->mKey) 229 numVideoKeyFrames++; 230 } 231 232 if (numVideoKeyFrames == 2) { 233 flushFrames(outstandingFrames, /* last = */ false); 234 numVideoKeyFrames--; 235 } 236 } 237 ALOGV("flushing last cluster (size %zu)", outstandingFrames.size()); 238 flushFrames(outstandingFrames, /* last = */ true); 239 mDone = true; 240} 241 242//================================================================================================= 243 244static const int64_t kInitialDelayTimeUs = 700000LL; 245 246void WebmFrameMediaSourceThread::clearFlags() { 247 mDone = false; 248 mPaused = false; 249 mResumed = false; 250 mStarted = false; 251 mReachedEOS = false; 252} 253 254WebmFrameMediaSourceThread::WebmFrameMediaSourceThread( 255 const sp<IMediaSource>& source, 256 int type, 257 LinkedBlockingQueue<const sp<WebmFrame> >& sink, 258 uint64_t timeCodeScale, 259 int64_t startTimeRealUs, 260 int32_t startTimeOffsetMs, 261 int numTracks, 262 bool realTimeRecording) 263 : WebmFrameSourceThread(type, sink), 264 mSource(source), 265 mTimeCodeScale(timeCodeScale), 266 mTrackDurationUs(0) { 267 clearFlags(); 268 mStartTimeUs = startTimeRealUs; 269 if (realTimeRecording && numTracks > 1) { 270 /* 271 * Copied from MPEG4Writer 272 * 273 * This extra delay of accepting incoming audio/video signals 274 * helps to align a/v start time at the beginning of a recording 275 * session, and it also helps eliminate the "recording" sound for 276 * camcorder applications. 277 * 278 * If client does not set the start time offset, we fall back to 279 * use the default initial delay value. 280 */ 281 int64_t startTimeOffsetUs = startTimeOffsetMs * 1000LL; 282 if (startTimeOffsetUs < 0) { // Start time offset was not set 283 startTimeOffsetUs = kInitialDelayTimeUs; 284 } 285 mStartTimeUs += startTimeOffsetUs; 286 ALOGI("Start time offset: %" PRId64 " us", startTimeOffsetUs); 287 } 288} 289 290status_t WebmFrameMediaSourceThread::start() { 291 sp<MetaData> meta = new MetaData; 292 meta->setInt64(kKeyTime, mStartTimeUs); 293 status_t err = mSource->start(meta.get()); 294 if (err != OK) { 295 mDone = true; 296 mReachedEOS = true; 297 return err; 298 } else { 299 mStarted = true; 300 return WebmFrameThread::start(); 301 } 302} 303 304status_t WebmFrameMediaSourceThread::resume() { 305 if (!mDone && mPaused) { 306 mPaused = false; 307 mResumed = true; 308 } 309 return OK; 310} 311 312status_t WebmFrameMediaSourceThread::pause() { 313 if (mStarted) { 314 mPaused = true; 315 } 316 return OK; 317} 318 319status_t WebmFrameMediaSourceThread::stop() { 320 if (mStarted) { 321 mStarted = false; 322 mDone = true; 323 mSource->stop(); 324 return WebmFrameThread::stop(); 325 } 326 return OK; 327} 328 329void WebmFrameMediaSourceThread::run() { 330 int32_t count = 0; 331 int64_t timestampUs = 0xdeadbeef; 332 int64_t lastTimestampUs = 0; // Previous sample time stamp 333 int64_t lastDurationUs = 0; // Previous sample duration 334 int64_t previousPausedDurationUs = 0; 335 336 const uint64_t kUninitialized = 0xffffffffffffffffL; 337 mStartTimeUs = kUninitialized; 338 339 status_t err = OK; 340 MediaBuffer *buffer; 341 while (!mDone && (err = mSource->read(&buffer, NULL)) == OK) { 342 if (buffer->range_length() == 0) { 343 buffer->release(); 344 buffer = NULL; 345 continue; 346 } 347 348 sp<MetaData> md = buffer->meta_data(); 349 CHECK(md->findInt64(kKeyTime, ×tampUs)); 350 if (mStartTimeUs == kUninitialized) { 351 mStartTimeUs = timestampUs; 352 } 353 timestampUs -= mStartTimeUs; 354 355 if (mPaused && !mResumed) { 356 lastDurationUs = timestampUs - lastTimestampUs; 357 lastTimestampUs = timestampUs; 358 buffer->release(); 359 buffer = NULL; 360 continue; 361 } 362 ++count; 363 364 // adjust time-stamps after pause/resume 365 if (mResumed) { 366 int64_t durExcludingEarlierPausesUs = timestampUs - previousPausedDurationUs; 367 CHECK_GE(durExcludingEarlierPausesUs, 0ll); 368 int64_t pausedDurationUs = durExcludingEarlierPausesUs - mTrackDurationUs; 369 CHECK_GE(pausedDurationUs, lastDurationUs); 370 previousPausedDurationUs += pausedDurationUs - lastDurationUs; 371 mResumed = false; 372 } 373 timestampUs -= previousPausedDurationUs; 374 CHECK_GE(timestampUs, 0ll); 375 376 int32_t isSync = false; 377 md->findInt32(kKeyIsSyncFrame, &isSync); 378 const sp<WebmFrame> f = new WebmFrame( 379 mType, 380 isSync, 381 timestampUs * 1000 / mTimeCodeScale, 382 buffer); 383 mSink.push(f); 384 385 ALOGV( 386 "%s %s frame at %" PRId64 " size %zu\n", 387 mType == kVideoType ? "video" : "audio", 388 isSync ? "I" : "P", 389 timestampUs * 1000 / mTimeCodeScale, 390 buffer->range_length()); 391 392 buffer->release(); 393 buffer = NULL; 394 395 if (timestampUs > mTrackDurationUs) { 396 mTrackDurationUs = timestampUs; 397 } 398 lastDurationUs = timestampUs - lastTimestampUs; 399 lastTimestampUs = timestampUs; 400 } 401 402 mTrackDurationUs += lastDurationUs; 403 mSink.push(WebmFrame::EOS); 404} 405} 406