mp4_stream_parser.cc revision 23730a6e56a168d1879203e4b3819bb36e3d8f1f
1// Copyright 2014 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "media/formats/mp4/mp4_stream_parser.h" 6 7#include "base/callback.h" 8#include "base/callback_helpers.h" 9#include "base/logging.h" 10#include "base/time/time.h" 11#include "media/base/audio_decoder_config.h" 12#include "media/base/stream_parser_buffer.h" 13#include "media/base/text_track_config.h" 14#include "media/base/video_decoder_config.h" 15#include "media/base/video_util.h" 16#include "media/formats/mp4/box_definitions.h" 17#include "media/formats/mp4/box_reader.h" 18#include "media/formats/mp4/es_descriptor.h" 19#include "media/formats/mp4/rcheck.h" 20#include "media/formats/mpeg/adts_constants.h" 21 22namespace media { 23namespace mp4 { 24 25// TODO(xhwang): Figure out the init data type appropriately once it's spec'ed. 26static const char kMp4InitDataType[] = "video/mp4"; 27 28MP4StreamParser::MP4StreamParser(const std::set<int>& audio_object_types, 29 bool has_sbr) 30 : state_(kWaitingForInit), 31 moof_head_(0), 32 mdat_tail_(0), 33 has_audio_(false), 34 has_video_(false), 35 audio_track_id_(0), 36 video_track_id_(0), 37 audio_object_types_(audio_object_types), 38 has_sbr_(has_sbr), 39 is_audio_track_encrypted_(false), 40 is_video_track_encrypted_(false) { 41} 42 43MP4StreamParser::~MP4StreamParser() {} 44 45void MP4StreamParser::Init(const InitCB& init_cb, 46 const NewConfigCB& config_cb, 47 const NewBuffersCB& new_buffers_cb, 48 bool /* ignore_text_tracks */ , 49 const NeedKeyCB& need_key_cb, 50 const NewMediaSegmentCB& new_segment_cb, 51 const base::Closure& end_of_segment_cb, 52 const LogCB& log_cb) { 53 DCHECK_EQ(state_, kWaitingForInit); 54 DCHECK(init_cb_.is_null()); 55 DCHECK(!init_cb.is_null()); 56 DCHECK(!config_cb.is_null()); 57 DCHECK(!new_buffers_cb.is_null()); 58 DCHECK(!need_key_cb.is_null()); 59 DCHECK(!end_of_segment_cb.is_null()); 60 61 ChangeState(kParsingBoxes); 62 init_cb_ = init_cb; 63 config_cb_ = config_cb; 64 new_buffers_cb_ = new_buffers_cb; 65 need_key_cb_ = need_key_cb; 66 new_segment_cb_ = new_segment_cb; 67 end_of_segment_cb_ = end_of_segment_cb; 68 log_cb_ = log_cb; 69} 70 71void MP4StreamParser::Reset() { 72 queue_.Reset(); 73 runs_.reset(); 74 moof_head_ = 0; 75 mdat_tail_ = 0; 76} 77 78void MP4StreamParser::Flush() { 79 DCHECK_NE(state_, kWaitingForInit); 80 Reset(); 81 ChangeState(kParsingBoxes); 82} 83 84bool MP4StreamParser::Parse(const uint8* buf, int size) { 85 DCHECK_NE(state_, kWaitingForInit); 86 87 if (state_ == kError) 88 return false; 89 90 queue_.Push(buf, size); 91 92 BufferQueue audio_buffers; 93 BufferQueue video_buffers; 94 95 bool result, err = false; 96 97 do { 98 if (state_ == kParsingBoxes) { 99 result = ParseBox(&err); 100 } else { 101 DCHECK_EQ(kEmittingSamples, state_); 102 result = EnqueueSample(&audio_buffers, &video_buffers, &err); 103 if (result) { 104 int64 max_clear = runs_->GetMaxClearOffset() + moof_head_; 105 err = !ReadAndDiscardMDATsUntil(max_clear); 106 } 107 } 108 } while (result && !err); 109 110 if (!err) 111 err = !SendAndFlushSamples(&audio_buffers, &video_buffers); 112 113 if (err) { 114 DLOG(ERROR) << "Error while parsing MP4"; 115 moov_.reset(); 116 Reset(); 117 ChangeState(kError); 118 return false; 119 } 120 121 return true; 122} 123 124bool MP4StreamParser::ParseBox(bool* err) { 125 const uint8* buf; 126 int size; 127 queue_.Peek(&buf, &size); 128 if (!size) return false; 129 130 scoped_ptr<BoxReader> reader( 131 BoxReader::ReadTopLevelBox(buf, size, log_cb_, err)); 132 if (reader.get() == NULL) return false; 133 134 if (reader->type() == FOURCC_MOOV) { 135 *err = !ParseMoov(reader.get()); 136 } else if (reader->type() == FOURCC_MOOF) { 137 moof_head_ = queue_.head(); 138 *err = !ParseMoof(reader.get()); 139 140 // Set up first mdat offset for ReadMDATsUntil(). 141 mdat_tail_ = queue_.head() + reader->size(); 142 143 // Return early to avoid evicting 'moof' data from queue. Auxiliary info may 144 // be located anywhere in the file, including inside the 'moof' itself. 145 // (Since 'default-base-is-moof' is mandated, no data references can come 146 // before the head of the 'moof', so keeping this box around is sufficient.) 147 return !(*err); 148 } else { 149 MEDIA_LOG(log_cb_) << "Skipping unrecognized top-level box: " 150 << FourCCToString(reader->type()); 151 } 152 153 queue_.Pop(reader->size()); 154 return !(*err); 155} 156 157 158bool MP4StreamParser::ParseMoov(BoxReader* reader) { 159 moov_.reset(new Movie); 160 RCHECK(moov_->Parse(reader)); 161 runs_.reset(); 162 163 has_audio_ = false; 164 has_video_ = false; 165 166 AudioDecoderConfig audio_config; 167 VideoDecoderConfig video_config; 168 169 for (std::vector<Track>::const_iterator track = moov_->tracks.begin(); 170 track != moov_->tracks.end(); ++track) { 171 // TODO(strobe): Only the first audio and video track present in a file are 172 // used. (Track selection is better accomplished via Source IDs, though, so 173 // adding support for track selection within a stream is low-priority.) 174 const SampleDescription& samp_descr = 175 track->media.information.sample_table.description; 176 177 // TODO(strobe): When codec reconfigurations are supported, detect and send 178 // a codec reconfiguration for fragments using a sample description index 179 // different from the previous one 180 size_t desc_idx = 0; 181 for (size_t t = 0; t < moov_->extends.tracks.size(); t++) { 182 const TrackExtends& trex = moov_->extends.tracks[t]; 183 if (trex.track_id == track->header.track_id) { 184 desc_idx = trex.default_sample_description_index; 185 break; 186 } 187 } 188 RCHECK(desc_idx > 0); 189 desc_idx -= 1; // BMFF descriptor index is one-based 190 191 if (track->media.handler.type == kAudio && !audio_config.IsValidConfig()) { 192 RCHECK(!samp_descr.audio_entries.empty()); 193 194 // It is not uncommon to find otherwise-valid files with incorrect sample 195 // description indices, so we fail gracefully in that case. 196 if (desc_idx >= samp_descr.audio_entries.size()) 197 desc_idx = 0; 198 const AudioSampleEntry& entry = samp_descr.audio_entries[desc_idx]; 199 const AAC& aac = entry.esds.aac; 200 201 if (!(entry.format == FOURCC_MP4A || 202 (entry.format == FOURCC_ENCA && 203 entry.sinf.format.format == FOURCC_MP4A))) { 204 MEDIA_LOG(log_cb_) << "Unsupported audio format 0x" 205 << std::hex << entry.format << " in stsd box."; 206 return false; 207 } 208 209 uint8 audio_type = entry.esds.object_type; 210 DVLOG(1) << "audio_type " << std::hex << audio_type; 211 if (audio_object_types_.find(audio_type) == audio_object_types_.end()) { 212 MEDIA_LOG(log_cb_) << "audio object type 0x" << std::hex << audio_type 213 << " does not match what is specified in the" 214 << " mimetype."; 215 return false; 216 } 217 218 AudioCodec codec = kUnknownAudioCodec; 219 ChannelLayout channel_layout = CHANNEL_LAYOUT_NONE; 220 int sample_per_second = 0; 221 std::vector<uint8> extra_data; 222 // Check if it is MPEG4 AAC defined in ISO 14496 Part 3 or 223 // supported MPEG2 AAC varients. 224 if (ESDescriptor::IsAAC(audio_type)) { 225 codec = kCodecAAC; 226 channel_layout = aac.GetChannelLayout(has_sbr_); 227 sample_per_second = aac.GetOutputSamplesPerSecond(has_sbr_); 228#if defined(OS_ANDROID) 229 extra_data = aac.codec_specific_data(); 230#endif 231 } else { 232 MEDIA_LOG(log_cb_) << "Unsupported audio object type 0x" << std::hex 233 << audio_type << " in esds."; 234 return false; 235 } 236 237 SampleFormat sample_format; 238 if (entry.samplesize == 8) { 239 sample_format = kSampleFormatU8; 240 } else if (entry.samplesize == 16) { 241 sample_format = kSampleFormatS16; 242 } else if (entry.samplesize == 32) { 243 sample_format = kSampleFormatS32; 244 } else { 245 LOG(ERROR) << "Unsupported sample size."; 246 return false; 247 } 248 249 is_audio_track_encrypted_ = entry.sinf.info.track_encryption.is_encrypted; 250 DVLOG(1) << "is_audio_track_encrypted_: " << is_audio_track_encrypted_; 251 audio_config.Initialize( 252 codec, sample_format, channel_layout, sample_per_second, 253 extra_data.size() ? &extra_data[0] : NULL, extra_data.size(), 254 is_audio_track_encrypted_, false, base::TimeDelta(), 255 base::TimeDelta()); 256 has_audio_ = true; 257 audio_track_id_ = track->header.track_id; 258 } 259 if (track->media.handler.type == kVideo && !video_config.IsValidConfig()) { 260 RCHECK(!samp_descr.video_entries.empty()); 261 if (desc_idx >= samp_descr.video_entries.size()) 262 desc_idx = 0; 263 const VideoSampleEntry& entry = samp_descr.video_entries[desc_idx]; 264 265 if (!entry.IsFormatValid()) { 266 MEDIA_LOG(log_cb_) << "Unsupported video format 0x" 267 << std::hex << entry.format << " in stsd box."; 268 return false; 269 } 270 271 // TODO(strobe): Recover correct crop box 272 gfx::Size coded_size(entry.width, entry.height); 273 gfx::Rect visible_rect(coded_size); 274 gfx::Size natural_size = GetNaturalSize(visible_rect.size(), 275 entry.pixel_aspect.h_spacing, 276 entry.pixel_aspect.v_spacing); 277 is_video_track_encrypted_ = entry.sinf.info.track_encryption.is_encrypted; 278 DVLOG(1) << "is_video_track_encrypted_: " << is_video_track_encrypted_; 279 video_config.Initialize(kCodecH264, H264PROFILE_MAIN, VideoFrame::YV12, 280 coded_size, visible_rect, natural_size, 281 // No decoder-specific buffer needed for AVC; 282 // SPS/PPS are embedded in the video stream 283 NULL, 0, is_video_track_encrypted_, false); 284 has_video_ = true; 285 video_track_id_ = track->header.track_id; 286 } 287 } 288 289 RCHECK(config_cb_.Run(audio_config, video_config, TextTrackConfigMap())); 290 291 base::TimeDelta duration; 292 if (moov_->extends.header.fragment_duration > 0) { 293 duration = TimeDeltaFromRational(moov_->extends.header.fragment_duration, 294 moov_->header.timescale); 295 } else if (moov_->header.duration > 0 && 296 moov_->header.duration != kuint64max) { 297 duration = TimeDeltaFromRational(moov_->header.duration, 298 moov_->header.timescale); 299 } else { 300 duration = kInfiniteDuration(); 301 } 302 303 if (!init_cb_.is_null()) 304 base::ResetAndReturn(&init_cb_).Run(true, duration, false); 305 306 EmitNeedKeyIfNecessary(moov_->pssh); 307 return true; 308} 309 310bool MP4StreamParser::ParseMoof(BoxReader* reader) { 311 RCHECK(moov_.get()); // Must already have initialization segment 312 MovieFragment moof; 313 RCHECK(moof.Parse(reader)); 314 if (!runs_) 315 runs_.reset(new TrackRunIterator(moov_.get(), log_cb_)); 316 RCHECK(runs_->Init(moof)); 317 EmitNeedKeyIfNecessary(moof.pssh); 318 new_segment_cb_.Run(); 319 ChangeState(kEmittingSamples); 320 return true; 321} 322 323void MP4StreamParser::EmitNeedKeyIfNecessary( 324 const std::vector<ProtectionSystemSpecificHeader>& headers) { 325 // TODO(strobe): ensure that the value of init_data (all PSSH headers 326 // concatenated in arbitrary order) matches the EME spec. 327 // See https://www.w3.org/Bugs/Public/show_bug.cgi?id=17673. 328 if (headers.empty()) 329 return; 330 331 size_t total_size = 0; 332 for (size_t i = 0; i < headers.size(); i++) 333 total_size += headers[i].raw_box.size(); 334 335 std::vector<uint8> init_data(total_size); 336 size_t pos = 0; 337 for (size_t i = 0; i < headers.size(); i++) { 338 memcpy(&init_data[pos], &headers[i].raw_box[0], 339 headers[i].raw_box.size()); 340 pos += headers[i].raw_box.size(); 341 } 342 need_key_cb_.Run(kMp4InitDataType, init_data); 343} 344 345bool MP4StreamParser::PrepareAVCBuffer( 346 const AVCDecoderConfigurationRecord& avc_config, 347 std::vector<uint8>* frame_buf, 348 std::vector<SubsampleEntry>* subsamples) const { 349 // Convert the AVC NALU length fields to Annex B headers, as expected by 350 // decoding libraries. Since this may enlarge the size of the buffer, we also 351 // update the clear byte count for each subsample if encryption is used to 352 // account for the difference in size between the length prefix and Annex B 353 // start code. 354 RCHECK(AVC::ConvertFrameToAnnexB(avc_config.length_size, frame_buf)); 355 if (!subsamples->empty()) { 356 const int nalu_size_diff = 4 - avc_config.length_size; 357 size_t expected_size = runs_->sample_size() + 358 subsamples->size() * nalu_size_diff; 359 RCHECK(frame_buf->size() == expected_size); 360 for (size_t i = 0; i < subsamples->size(); i++) 361 (*subsamples)[i].clear_bytes += nalu_size_diff; 362 } 363 364 if (runs_->is_keyframe()) { 365 // If this is a keyframe, we (re-)inject SPS and PPS headers at the start of 366 // a frame. If subsample info is present, we also update the clear byte 367 // count for that first subsample. 368 std::vector<uint8> param_sets; 369 RCHECK(AVC::ConvertConfigToAnnexB(avc_config, ¶m_sets)); 370 frame_buf->insert(frame_buf->begin(), 371 param_sets.begin(), param_sets.end()); 372 if (!subsamples->empty()) 373 (*subsamples)[0].clear_bytes += param_sets.size(); 374 } 375 return true; 376} 377 378bool MP4StreamParser::PrepareAACBuffer( 379 const AAC& aac_config, std::vector<uint8>* frame_buf, 380 std::vector<SubsampleEntry>* subsamples) const { 381 // Append an ADTS header to every audio sample. 382 RCHECK(aac_config.ConvertEsdsToADTS(frame_buf)); 383 384 // As above, adjust subsample information to account for the headers. AAC is 385 // not required to use subsample encryption, so we may need to add an entry. 386 if (subsamples->empty()) { 387 SubsampleEntry entry; 388 entry.clear_bytes = kADTSHeaderMinSize; 389 entry.cypher_bytes = frame_buf->size() - kADTSHeaderMinSize; 390 subsamples->push_back(entry); 391 } else { 392 (*subsamples)[0].clear_bytes += kADTSHeaderMinSize; 393 } 394 return true; 395} 396 397bool MP4StreamParser::EnqueueSample(BufferQueue* audio_buffers, 398 BufferQueue* video_buffers, 399 bool* err) { 400 if (!runs_->IsRunValid()) { 401 // Flush any buffers we've gotten in this chunk so that buffers don't 402 // cross NewSegment() calls 403 *err = !SendAndFlushSamples(audio_buffers, video_buffers); 404 if (*err) 405 return false; 406 407 // Remain in kEnqueueingSamples state, discarding data, until the end of 408 // the current 'mdat' box has been appended to the queue. 409 if (!queue_.Trim(mdat_tail_)) 410 return false; 411 412 ChangeState(kParsingBoxes); 413 end_of_segment_cb_.Run(); 414 return true; 415 } 416 417 if (!runs_->IsSampleValid()) { 418 runs_->AdvanceRun(); 419 return true; 420 } 421 422 DCHECK(!(*err)); 423 424 const uint8* buf; 425 int buf_size; 426 queue_.Peek(&buf, &buf_size); 427 if (!buf_size) return false; 428 429 bool audio = has_audio_ && audio_track_id_ == runs_->track_id(); 430 bool video = has_video_ && video_track_id_ == runs_->track_id(); 431 432 // Skip this entire track if it's not one we're interested in 433 if (!audio && !video) 434 runs_->AdvanceRun(); 435 436 // Attempt to cache the auxiliary information first. Aux info is usually 437 // placed in a contiguous block before the sample data, rather than being 438 // interleaved. If we didn't cache it, this would require that we retain the 439 // start of the segment buffer while reading samples. Aux info is typically 440 // quite small compared to sample data, so this pattern is useful on 441 // memory-constrained devices where the source buffer consumes a substantial 442 // portion of the total system memory. 443 if (runs_->AuxInfoNeedsToBeCached()) { 444 queue_.PeekAt(runs_->aux_info_offset() + moof_head_, &buf, &buf_size); 445 if (buf_size < runs_->aux_info_size()) return false; 446 *err = !runs_->CacheAuxInfo(buf, buf_size); 447 return !*err; 448 } 449 450 queue_.PeekAt(runs_->sample_offset() + moof_head_, &buf, &buf_size); 451 if (buf_size < runs_->sample_size()) return false; 452 453 scoped_ptr<DecryptConfig> decrypt_config; 454 std::vector<SubsampleEntry> subsamples; 455 if (runs_->is_encrypted()) { 456 decrypt_config = runs_->GetDecryptConfig(); 457 if (!decrypt_config) { 458 *err = true; 459 return false; 460 } 461 subsamples = decrypt_config->subsamples(); 462 } 463 464 std::vector<uint8> frame_buf(buf, buf + runs_->sample_size()); 465 if (video) { 466 if (!PrepareAVCBuffer(runs_->video_description().avcc, 467 &frame_buf, &subsamples)) { 468 MEDIA_LOG(log_cb_) << "Failed to prepare AVC sample for decode"; 469 *err = true; 470 return false; 471 } 472 } 473 474 if (audio) { 475 if (ESDescriptor::IsAAC(runs_->audio_description().esds.object_type) && 476 !PrepareAACBuffer(runs_->audio_description().esds.aac, 477 &frame_buf, &subsamples)) { 478 MEDIA_LOG(log_cb_) << "Failed to prepare AAC sample for decode"; 479 *err = true; 480 return false; 481 } 482 } 483 484 if (decrypt_config) { 485 if (!subsamples.empty()) { 486 // Create a new config with the updated subsamples. 487 decrypt_config.reset(new DecryptConfig( 488 decrypt_config->key_id(), 489 decrypt_config->iv(), 490 subsamples)); 491 } 492 // else, use the existing config. 493 } else if ((audio && is_audio_track_encrypted_) || 494 (video && is_video_track_encrypted_)) { 495 // The media pipeline requires a DecryptConfig with an empty |iv|. 496 // TODO(ddorwin): Refactor so we do not need a fake key ID ("1"); 497 decrypt_config.reset( 498 new DecryptConfig("1", "", std::vector<SubsampleEntry>())); 499 } 500 501 StreamParserBuffer::Type buffer_type = audio ? DemuxerStream::AUDIO : 502 DemuxerStream::VIDEO; 503 504 // TODO(wolenetz/acolwell): Validate and use a common cross-parser TrackId 505 // type and allow multiple tracks for same media type, if applicable. See 506 // https://crbug.com/341581. 507 scoped_refptr<StreamParserBuffer> stream_buf = 508 StreamParserBuffer::CopyFrom(&frame_buf[0], frame_buf.size(), 509 runs_->is_keyframe(), buffer_type, 0); 510 511 if (decrypt_config) 512 stream_buf->set_decrypt_config(decrypt_config.Pass()); 513 514 stream_buf->set_duration(runs_->duration()); 515 stream_buf->set_timestamp(runs_->cts()); 516 stream_buf->SetDecodeTimestamp(runs_->dts()); 517 518 DVLOG(3) << "Pushing frame: aud=" << audio 519 << ", key=" << runs_->is_keyframe() 520 << ", dur=" << runs_->duration().InMilliseconds() 521 << ", dts=" << runs_->dts().InMilliseconds() 522 << ", cts=" << runs_->cts().InMilliseconds() 523 << ", size=" << runs_->sample_size(); 524 525 if (audio) { 526 audio_buffers->push_back(stream_buf); 527 } else { 528 video_buffers->push_back(stream_buf); 529 } 530 531 runs_->AdvanceSample(); 532 return true; 533} 534 535bool MP4StreamParser::SendAndFlushSamples(BufferQueue* audio_buffers, 536 BufferQueue* video_buffers) { 537 if (audio_buffers->empty() && video_buffers->empty()) 538 return true; 539 540 TextBufferQueueMap empty_text_map; 541 bool success = new_buffers_cb_.Run(*audio_buffers, 542 *video_buffers, 543 empty_text_map); 544 audio_buffers->clear(); 545 video_buffers->clear(); 546 return success; 547} 548 549bool MP4StreamParser::ReadAndDiscardMDATsUntil(const int64 offset) { 550 bool err = false; 551 while (mdat_tail_ < offset) { 552 const uint8* buf; 553 int size; 554 queue_.PeekAt(mdat_tail_, &buf, &size); 555 556 FourCC type; 557 int box_sz; 558 if (!BoxReader::StartTopLevelBox(buf, size, log_cb_, 559 &type, &box_sz, &err)) 560 break; 561 562 if (type != FOURCC_MDAT) { 563 MEDIA_LOG(log_cb_) << "Unexpected box type while parsing MDATs: " 564 << FourCCToString(type); 565 } 566 mdat_tail_ += box_sz; 567 } 568 queue_.Trim(std::min(mdat_tail_, offset)); 569 return !err; 570} 571 572void MP4StreamParser::ChangeState(State new_state) { 573 DVLOG(2) << "Changing state: " << new_state; 574 state_ = new_state; 575} 576 577} // namespace mp4 578} // namespace media 579