1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "WebmFrameThread" 19 20#include "WebmConstants.h" 21#include "WebmFrameThread.h" 22 23#include <media/stagefright/MetaData.h> 24#include <media/stagefright/foundation/ADebug.h> 25 26#include <utils/Log.h> 27#include <inttypes.h> 28 29using namespace webm; 30 31namespace android { 32 33void *WebmFrameThread::wrap(void *arg) { 34 WebmFrameThread *worker = reinterpret_cast<WebmFrameThread*>(arg); 35 worker->run(); 36 return NULL; 37} 38 39status_t WebmFrameThread::start() { 40 pthread_attr_t attr; 41 pthread_attr_init(&attr); 42 pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); 43 pthread_create(&mThread, &attr, WebmFrameThread::wrap, this); 44 pthread_attr_destroy(&attr); 45 return OK; 46} 47 48status_t WebmFrameThread::stop() { 49 void *status; 50 pthread_join(mThread, &status); 51 return (status_t)(intptr_t)status; 52} 53 54//================================================================================================= 55 56WebmFrameSourceThread::WebmFrameSourceThread( 57 int type, 58 LinkedBlockingQueue<const sp<WebmFrame> >& sink) 59 : mType(type), mSink(sink) { 60} 61 62//================================================================================================= 63 64WebmFrameSinkThread::WebmFrameSinkThread( 65 const int& fd, 66 const uint64_t& off, 67 sp<WebmFrameSourceThread> videoThread, 68 sp<WebmFrameSourceThread> audioThread, 69 List<sp<WebmElement> >& cues) 70 : mFd(fd), 71 mSegmentDataStart(off), 72 mVideoFrames(videoThread->mSink), 73 mAudioFrames(audioThread->mSink), 74 mCues(cues), 75 mDone(true) { 76} 77 78WebmFrameSinkThread::WebmFrameSinkThread( 79 const int& fd, 80 const uint64_t& off, 81 LinkedBlockingQueue<const sp<WebmFrame> >& videoSource, 82 LinkedBlockingQueue<const sp<WebmFrame> >& audioSource, 83 List<sp<WebmElement> >& cues) 84 : mFd(fd), 85 mSegmentDataStart(off), 86 mVideoFrames(videoSource), 87 mAudioFrames(audioSource), 88 mCues(cues), 89 mDone(true) { 90} 91 92// Initializes a webm cluster with its starting timecode. 93// 94// frames: 95// sequence of input audio/video frames received from the source. 96// 97// clusterTimecodeL: 98// the starting timecode of the cluster; this is the timecode of the first 99// frame since frames are ordered by timestamp. 100// 101// children: 102// list to hold child elements in a webm cluster (start timecode and 103// simple blocks). 104// 105// static 106void WebmFrameSinkThread::initCluster( 107 List<const sp<WebmFrame> >& frames, 108 uint64_t& clusterTimecodeL, 109 List<sp<WebmElement> >& children) { 110 CHECK(!frames.empty() && children.empty()); 111 112 const sp<WebmFrame> f = *(frames.begin()); 113 clusterTimecodeL = f->mAbsTimecode; 114 WebmUnsigned *clusterTimecode = new WebmUnsigned(kMkvTimecode, clusterTimecodeL); 115 children.clear(); 116 children.push_back(clusterTimecode); 117} 118 119void WebmFrameSinkThread::writeCluster(List<sp<WebmElement> >& children) { 120 // children must contain at least one simpleblock and its timecode 121 CHECK_GE(children.size(), 2); 122 123 uint64_t size; 124 sp<WebmElement> cluster = new WebmMaster(kMkvCluster, children); 125 cluster->write(mFd, size); 126 children.clear(); 127} 128 129// Write out (possibly multiple) webm cluster(s) from frames split on video key frames. 130// 131// last: 132// current flush is triggered by EOS instead of a second outstanding video key frame. 133void WebmFrameSinkThread::flushFrames(List<const sp<WebmFrame> >& frames, bool last) { 134 if (frames.empty()) { 135 return; 136 } 137 138 uint64_t clusterTimecodeL; 139 List<sp<WebmElement> > children; 140 initCluster(frames, clusterTimecodeL, children); 141 142 uint64_t cueTime = clusterTimecodeL; 143 off_t fpos = ::lseek(mFd, 0, SEEK_CUR); 144 size_t n = frames.size(); 145 if (!last) { 146 // If we are not flushing the last sequence of outstanding frames, flushFrames 147 // must have been called right after we have pushed a second outstanding video key 148 // frame (the last frame), which belongs to the next cluster; also hold back on 149 // flushing the second to last frame before we check its type. A audio frame 150 // should precede the aforementioned video key frame in the next sequence, a video 151 // frame should be the last frame in the current (to-be-flushed) sequence. 152 CHECK_GE(n, 2); 153 n -= 2; 154 } 155 156 for (size_t i = 0; i < n; i++) { 157 const sp<WebmFrame> f = *(frames.begin()); 158 if (f->mType == kVideoType && f->mKey) { 159 cueTime = f->mAbsTimecode; 160 } 161 162 if (f->mAbsTimecode - clusterTimecodeL > INT16_MAX) { 163 writeCluster(children); 164 initCluster(frames, clusterTimecodeL, children); 165 } 166 167 frames.erase(frames.begin()); 168 children.push_back(f->SimpleBlock(clusterTimecodeL)); 169 } 170 171 // equivalent to last==false 172 if (!frames.empty()) { 173 // decide whether to write out the second to last frame. 174 const sp<WebmFrame> secondLastFrame = *(frames.begin()); 175 if (secondLastFrame->mType == kVideoType) { 176 frames.erase(frames.begin()); 177 children.push_back(secondLastFrame->SimpleBlock(clusterTimecodeL)); 178 } 179 } 180 181 writeCluster(children); 182 sp<WebmElement> cuePoint = WebmElement::CuePointEntry(cueTime, 1, fpos - mSegmentDataStart); 183 mCues.push_back(cuePoint); 184} 185 186status_t WebmFrameSinkThread::start() { 187 mDone = false; 188 return WebmFrameThread::start(); 189} 190 191status_t WebmFrameSinkThread::stop() { 192 mDone = true; 193 mVideoFrames.push(WebmFrame::EOS); 194 mAudioFrames.push(WebmFrame::EOS); 195 return WebmFrameThread::stop(); 196} 197 198void WebmFrameSinkThread::run() { 199 int numVideoKeyFrames = 0; 200 List<const sp<WebmFrame> > outstandingFrames; 201 while (!mDone) { 202 ALOGV("wait v frame"); 203 const sp<WebmFrame> videoFrame = mVideoFrames.peek(); 204 ALOGV("v frame: %p", videoFrame.get()); 205 206 ALOGV("wait a frame"); 207 const sp<WebmFrame> audioFrame = mAudioFrames.peek(); 208 ALOGV("a frame: %p", audioFrame.get()); 209 210 if (videoFrame->mEos && audioFrame->mEos) { 211 break; 212 } 213 214 if (*audioFrame < *videoFrame) { 215 ALOGV("take a frame"); 216 mAudioFrames.take(); 217 outstandingFrames.push_back(audioFrame); 218 } else { 219 ALOGV("take v frame"); 220 mVideoFrames.take(); 221 outstandingFrames.push_back(videoFrame); 222 if (videoFrame->mKey) 223 numVideoKeyFrames++; 224 } 225 226 if (numVideoKeyFrames == 2) { 227 flushFrames(outstandingFrames, /* last = */ false); 228 numVideoKeyFrames--; 229 } 230 } 231 ALOGV("flushing last cluster (size %zu)", outstandingFrames.size()); 232 flushFrames(outstandingFrames, /* last = */ true); 233 mDone = true; 234} 235 236//================================================================================================= 237 238static const int64_t kInitialDelayTimeUs = 700000LL; 239 240void WebmFrameMediaSourceThread::clearFlags() { 241 mDone = false; 242 mPaused = false; 243 mResumed = false; 244 mStarted = false; 245 mReachedEOS = false; 246} 247 248WebmFrameMediaSourceThread::WebmFrameMediaSourceThread( 249 const sp<MediaSource>& source, 250 int type, 251 LinkedBlockingQueue<const sp<WebmFrame> >& sink, 252 uint64_t timeCodeScale, 253 int64_t startTimeRealUs, 254 int32_t startTimeOffsetMs, 255 int numTracks, 256 bool realTimeRecording) 257 : WebmFrameSourceThread(type, sink), 258 mSource(source), 259 mTimeCodeScale(timeCodeScale), 260 mTrackDurationUs(0) { 261 clearFlags(); 262 mStartTimeUs = startTimeRealUs; 263 if (realTimeRecording && numTracks > 1) { 264 /* 265 * Copied from MPEG4Writer 266 * 267 * This extra delay of accepting incoming audio/video signals 268 * helps to align a/v start time at the beginning of a recording 269 * session, and it also helps eliminate the "recording" sound for 270 * camcorder applications. 271 * 272 * If client does not set the start time offset, we fall back to 273 * use the default initial delay value. 274 */ 275 int64_t startTimeOffsetUs = startTimeOffsetMs * 1000LL; 276 if (startTimeOffsetUs < 0) { // Start time offset was not set 277 startTimeOffsetUs = kInitialDelayTimeUs; 278 } 279 mStartTimeUs += startTimeOffsetUs; 280 ALOGI("Start time offset: %" PRId64 " us", startTimeOffsetUs); 281 } 282} 283 284status_t WebmFrameMediaSourceThread::start() { 285 sp<MetaData> meta = new MetaData; 286 meta->setInt64(kKeyTime, mStartTimeUs); 287 status_t err = mSource->start(meta.get()); 288 if (err != OK) { 289 mDone = true; 290 mReachedEOS = true; 291 return err; 292 } else { 293 mStarted = true; 294 return WebmFrameThread::start(); 295 } 296} 297 298status_t WebmFrameMediaSourceThread::resume() { 299 if (!mDone && mPaused) { 300 mPaused = false; 301 mResumed = true; 302 } 303 return OK; 304} 305 306status_t WebmFrameMediaSourceThread::pause() { 307 if (mStarted) { 308 mPaused = true; 309 } 310 return OK; 311} 312 313status_t WebmFrameMediaSourceThread::stop() { 314 if (mStarted) { 315 mStarted = false; 316 mDone = true; 317 mSource->stop(); 318 return WebmFrameThread::stop(); 319 } 320 return OK; 321} 322 323void WebmFrameMediaSourceThread::run() { 324 int32_t count = 0; 325 int64_t timestampUs = 0xdeadbeef; 326 int64_t lastTimestampUs = 0; // Previous sample time stamp 327 int64_t lastDurationUs = 0; // Previous sample duration 328 int64_t previousPausedDurationUs = 0; 329 330 const uint64_t kUninitialized = 0xffffffffffffffffL; 331 mStartTimeUs = kUninitialized; 332 333 status_t err = OK; 334 MediaBuffer *buffer; 335 while (!mDone && (err = mSource->read(&buffer, NULL)) == OK) { 336 if (buffer->range_length() == 0) { 337 buffer->release(); 338 buffer = NULL; 339 continue; 340 } 341 342 sp<MetaData> md = buffer->meta_data(); 343 CHECK(md->findInt64(kKeyTime, ×tampUs)); 344 if (mStartTimeUs == kUninitialized) { 345 mStartTimeUs = timestampUs; 346 } 347 timestampUs -= mStartTimeUs; 348 349 if (mPaused && !mResumed) { 350 lastDurationUs = timestampUs - lastTimestampUs; 351 lastTimestampUs = timestampUs; 352 buffer->release(); 353 buffer = NULL; 354 continue; 355 } 356 ++count; 357 358 // adjust time-stamps after pause/resume 359 if (mResumed) { 360 int64_t durExcludingEarlierPausesUs = timestampUs - previousPausedDurationUs; 361 CHECK_GE(durExcludingEarlierPausesUs, 0ll); 362 int64_t pausedDurationUs = durExcludingEarlierPausesUs - mTrackDurationUs; 363 CHECK_GE(pausedDurationUs, lastDurationUs); 364 previousPausedDurationUs += pausedDurationUs - lastDurationUs; 365 mResumed = false; 366 } 367 timestampUs -= previousPausedDurationUs; 368 CHECK_GE(timestampUs, 0ll); 369 370 int32_t isSync = false; 371 md->findInt32(kKeyIsSyncFrame, &isSync); 372 const sp<WebmFrame> f = new WebmFrame( 373 mType, 374 isSync, 375 timestampUs * 1000 / mTimeCodeScale, 376 buffer); 377 mSink.push(f); 378 379 ALOGV( 380 "%s %s frame at %" PRId64 " size %zu\n", 381 mType == kVideoType ? "video" : "audio", 382 isSync ? "I" : "P", 383 timestampUs * 1000 / mTimeCodeScale, 384 buffer->range_length()); 385 386 buffer->release(); 387 buffer = NULL; 388 389 if (timestampUs > mTrackDurationUs) { 390 mTrackDurationUs = timestampUs; 391 } 392 lastDurationUs = timestampUs - lastTimestampUs; 393 lastTimestampUs = timestampUs; 394 } 395 396 mTrackDurationUs += lastDurationUs; 397 mSink.push(WebmFrame::EOS); 398} 399} 400