mp4_stream_parser.cc revision 5c02ac1a9c1b504631c0a3d2b6e737b5d738bae1
1// Copyright 2014 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "media/formats/mp4/mp4_stream_parser.h" 6 7#include "base/callback.h" 8#include "base/callback_helpers.h" 9#include "base/logging.h" 10#include "base/time/time.h" 11#include "media/base/audio_decoder_config.h" 12#include "media/base/stream_parser_buffer.h" 13#include "media/base/text_track_config.h" 14#include "media/base/video_decoder_config.h" 15#include "media/base/video_util.h" 16#include "media/formats/mp4/box_definitions.h" 17#include "media/formats/mp4/box_reader.h" 18#include "media/formats/mp4/es_descriptor.h" 19#include "media/formats/mp4/rcheck.h" 20#include "media/formats/mpeg/adts_constants.h" 21 22namespace media { 23namespace mp4 { 24 25// TODO(xhwang): Figure out the init data type appropriately once it's spec'ed. 26static const char kMp4InitDataType[] = "video/mp4"; 27 28MP4StreamParser::MP4StreamParser(const std::set<int>& audio_object_types, 29 bool has_sbr) 30 : state_(kWaitingForInit), 31 moof_head_(0), 32 mdat_tail_(0), 33 has_audio_(false), 34 has_video_(false), 35 audio_track_id_(0), 36 video_track_id_(0), 37 audio_object_types_(audio_object_types), 38 has_sbr_(has_sbr), 39 is_audio_track_encrypted_(false), 40 is_video_track_encrypted_(false) { 41} 42 43MP4StreamParser::~MP4StreamParser() {} 44 45void MP4StreamParser::Init(const InitCB& init_cb, 46 const NewConfigCB& config_cb, 47 const NewBuffersCB& new_buffers_cb, 48 bool /* ignore_text_tracks */ , 49 const NeedKeyCB& need_key_cb, 50 const NewMediaSegmentCB& new_segment_cb, 51 const base::Closure& end_of_segment_cb, 52 const LogCB& log_cb) { 53 DCHECK_EQ(state_, kWaitingForInit); 54 DCHECK(init_cb_.is_null()); 55 DCHECK(!init_cb.is_null()); 56 DCHECK(!config_cb.is_null()); 57 DCHECK(!new_buffers_cb.is_null()); 58 DCHECK(!need_key_cb.is_null()); 59 DCHECK(!end_of_segment_cb.is_null()); 60 61 ChangeState(kParsingBoxes); 62 init_cb_ = init_cb; 63 config_cb_ = config_cb; 64 new_buffers_cb_ = new_buffers_cb; 65 need_key_cb_ = need_key_cb; 66 new_segment_cb_ = new_segment_cb; 67 end_of_segment_cb_ = end_of_segment_cb; 68 log_cb_ = log_cb; 69} 70 71void MP4StreamParser::Reset() { 72 queue_.Reset(); 73 runs_.reset(); 74 moof_head_ = 0; 75 mdat_tail_ = 0; 76} 77 78void MP4StreamParser::Flush() { 79 DCHECK_NE(state_, kWaitingForInit); 80 Reset(); 81 ChangeState(kParsingBoxes); 82} 83 84bool MP4StreamParser::Parse(const uint8* buf, int size) { 85 DCHECK_NE(state_, kWaitingForInit); 86 87 if (state_ == kError) 88 return false; 89 90 queue_.Push(buf, size); 91 92 BufferQueue audio_buffers; 93 BufferQueue video_buffers; 94 95 bool result, err = false; 96 97 do { 98 if (state_ == kParsingBoxes) { 99 result = ParseBox(&err); 100 } else { 101 DCHECK_EQ(kEmittingSamples, state_); 102 result = EnqueueSample(&audio_buffers, &video_buffers, &err); 103 if (result) { 104 int64 max_clear = runs_->GetMaxClearOffset() + moof_head_; 105 err = !ReadAndDiscardMDATsUntil(max_clear); 106 } 107 } 108 } while (result && !err); 109 110 if (!err) 111 err = !SendAndFlushSamples(&audio_buffers, &video_buffers); 112 113 if (err) { 114 DLOG(ERROR) << "Error while parsing MP4"; 115 moov_.reset(); 116 Reset(); 117 ChangeState(kError); 118 return false; 119 } 120 121 return true; 122} 123 124bool MP4StreamParser::ParseBox(bool* err) { 125 const uint8* buf; 126 int size; 127 queue_.Peek(&buf, &size); 128 if (!size) return false; 129 130 scoped_ptr<BoxReader> reader( 131 BoxReader::ReadTopLevelBox(buf, size, log_cb_, err)); 132 if (reader.get() == NULL) return false; 133 134 if (reader->type() == FOURCC_MOOV) { 135 *err = !ParseMoov(reader.get()); 136 } else if (reader->type() == FOURCC_MOOF) { 137 moof_head_ = queue_.head(); 138 *err = !ParseMoof(reader.get()); 139 140 // Set up first mdat offset for ReadMDATsUntil(). 141 mdat_tail_ = queue_.head() + reader->size(); 142 143 // Return early to avoid evicting 'moof' data from queue. Auxiliary info may 144 // be located anywhere in the file, including inside the 'moof' itself. 145 // (Since 'default-base-is-moof' is mandated, no data references can come 146 // before the head of the 'moof', so keeping this box around is sufficient.) 147 return !(*err); 148 } else { 149 MEDIA_LOG(log_cb_) << "Skipping unrecognized top-level box: " 150 << FourCCToString(reader->type()); 151 } 152 153 queue_.Pop(reader->size()); 154 return !(*err); 155} 156 157 158bool MP4StreamParser::ParseMoov(BoxReader* reader) { 159 moov_.reset(new Movie); 160 RCHECK(moov_->Parse(reader)); 161 runs_.reset(); 162 163 has_audio_ = false; 164 has_video_ = false; 165 166 AudioDecoderConfig audio_config; 167 VideoDecoderConfig video_config; 168 169 for (std::vector<Track>::const_iterator track = moov_->tracks.begin(); 170 track != moov_->tracks.end(); ++track) { 171 // TODO(strobe): Only the first audio and video track present in a file are 172 // used. (Track selection is better accomplished via Source IDs, though, so 173 // adding support for track selection within a stream is low-priority.) 174 const SampleDescription& samp_descr = 175 track->media.information.sample_table.description; 176 177 // TODO(strobe): When codec reconfigurations are supported, detect and send 178 // a codec reconfiguration for fragments using a sample description index 179 // different from the previous one 180 size_t desc_idx = 0; 181 for (size_t t = 0; t < moov_->extends.tracks.size(); t++) { 182 const TrackExtends& trex = moov_->extends.tracks[t]; 183 if (trex.track_id == track->header.track_id) { 184 desc_idx = trex.default_sample_description_index; 185 break; 186 } 187 } 188 RCHECK(desc_idx > 0); 189 desc_idx -= 1; // BMFF descriptor index is one-based 190 191 if (track->media.handler.type == kAudio && !audio_config.IsValidConfig()) { 192 RCHECK(!samp_descr.audio_entries.empty()); 193 194 // It is not uncommon to find otherwise-valid files with incorrect sample 195 // description indices, so we fail gracefully in that case. 196 if (desc_idx >= samp_descr.audio_entries.size()) 197 desc_idx = 0; 198 const AudioSampleEntry& entry = samp_descr.audio_entries[desc_idx]; 199 const AAC& aac = entry.esds.aac; 200 201 if (!(entry.format == FOURCC_MP4A || 202 (entry.format == FOURCC_ENCA && 203 entry.sinf.format.format == FOURCC_MP4A))) { 204 MEDIA_LOG(log_cb_) << "Unsupported audio format 0x" 205 << std::hex << entry.format << " in stsd box."; 206 return false; 207 } 208 209 uint8 audio_type = entry.esds.object_type; 210 DVLOG(1) << "audio_type " << std::hex << audio_type; 211 if (audio_object_types_.find(audio_type) == audio_object_types_.end()) { 212 MEDIA_LOG(log_cb_) << "audio object type 0x" << std::hex << audio_type 213 << " does not match what is specified in the" 214 << " mimetype."; 215 return false; 216 } 217 218 AudioCodec codec = kUnknownAudioCodec; 219 ChannelLayout channel_layout = CHANNEL_LAYOUT_NONE; 220 int sample_per_second = 0; 221 std::vector<uint8> extra_data; 222 // Check if it is MPEG4 AAC defined in ISO 14496 Part 3 or 223 // supported MPEG2 AAC varients. 224 if (ESDescriptor::IsAAC(audio_type)) { 225 codec = kCodecAAC; 226 channel_layout = aac.GetChannelLayout(has_sbr_); 227 sample_per_second = aac.GetOutputSamplesPerSecond(has_sbr_); 228#if defined(OS_ANDROID) 229 extra_data = aac.codec_specific_data(); 230#endif 231 } else { 232 MEDIA_LOG(log_cb_) << "Unsupported audio object type 0x" << std::hex 233 << audio_type << " in esds."; 234 return false; 235 } 236 237 SampleFormat sample_format; 238 if (entry.samplesize == 8) { 239 sample_format = kSampleFormatU8; 240 } else if (entry.samplesize == 16) { 241 sample_format = kSampleFormatS16; 242 } else if (entry.samplesize == 32) { 243 sample_format = kSampleFormatS32; 244 } else { 245 LOG(ERROR) << "Unsupported sample size."; 246 return false; 247 } 248 249 is_audio_track_encrypted_ = entry.sinf.info.track_encryption.is_encrypted; 250 DVLOG(1) << "is_audio_track_encrypted_: " << is_audio_track_encrypted_; 251 audio_config.Initialize( 252 codec, sample_format, channel_layout, sample_per_second, 253 extra_data.size() ? &extra_data[0] : NULL, extra_data.size(), 254 is_audio_track_encrypted_, false, base::TimeDelta(), 255 0); 256 has_audio_ = true; 257 audio_track_id_ = track->header.track_id; 258 } 259 if (track->media.handler.type == kVideo && !video_config.IsValidConfig()) { 260 RCHECK(!samp_descr.video_entries.empty()); 261 if (desc_idx >= samp_descr.video_entries.size()) 262 desc_idx = 0; 263 const VideoSampleEntry& entry = samp_descr.video_entries[desc_idx]; 264 265 if (!entry.IsFormatValid()) { 266 MEDIA_LOG(log_cb_) << "Unsupported video format 0x" 267 << std::hex << entry.format << " in stsd box."; 268 return false; 269 } 270 271 // TODO(strobe): Recover correct crop box 272 gfx::Size coded_size(entry.width, entry.height); 273 gfx::Rect visible_rect(coded_size); 274 gfx::Size natural_size = GetNaturalSize(visible_rect.size(), 275 entry.pixel_aspect.h_spacing, 276 entry.pixel_aspect.v_spacing); 277 is_video_track_encrypted_ = entry.sinf.info.track_encryption.is_encrypted; 278 DVLOG(1) << "is_video_track_encrypted_: " << is_video_track_encrypted_; 279 video_config.Initialize(kCodecH264, H264PROFILE_MAIN, VideoFrame::YV12, 280 coded_size, visible_rect, natural_size, 281 // No decoder-specific buffer needed for AVC; 282 // SPS/PPS are embedded in the video stream 283 NULL, 0, is_video_track_encrypted_, false); 284 has_video_ = true; 285 video_track_id_ = track->header.track_id; 286 } 287 } 288 289 RCHECK(config_cb_.Run(audio_config, video_config, TextTrackConfigMap())); 290 291 StreamParser::InitParameters params(kInfiniteDuration()); 292 if (moov_->extends.header.fragment_duration > 0) { 293 params.duration = TimeDeltaFromRational( 294 moov_->extends.header.fragment_duration, moov_->header.timescale); 295 } else if (moov_->header.duration > 0 && 296 moov_->header.duration != kuint64max) { 297 params.duration = 298 TimeDeltaFromRational(moov_->header.duration, moov_->header.timescale); 299 } 300 301 if (!init_cb_.is_null()) 302 base::ResetAndReturn(&init_cb_).Run(true, params); 303 304 EmitNeedKeyIfNecessary(moov_->pssh); 305 return true; 306} 307 308bool MP4StreamParser::ParseMoof(BoxReader* reader) { 309 RCHECK(moov_.get()); // Must already have initialization segment 310 MovieFragment moof; 311 RCHECK(moof.Parse(reader)); 312 if (!runs_) 313 runs_.reset(new TrackRunIterator(moov_.get(), log_cb_)); 314 RCHECK(runs_->Init(moof)); 315 EmitNeedKeyIfNecessary(moof.pssh); 316 new_segment_cb_.Run(); 317 ChangeState(kEmittingSamples); 318 return true; 319} 320 321void MP4StreamParser::EmitNeedKeyIfNecessary( 322 const std::vector<ProtectionSystemSpecificHeader>& headers) { 323 // TODO(strobe): ensure that the value of init_data (all PSSH headers 324 // concatenated in arbitrary order) matches the EME spec. 325 // See https://www.w3.org/Bugs/Public/show_bug.cgi?id=17673. 326 if (headers.empty()) 327 return; 328 329 size_t total_size = 0; 330 for (size_t i = 0; i < headers.size(); i++) 331 total_size += headers[i].raw_box.size(); 332 333 std::vector<uint8> init_data(total_size); 334 size_t pos = 0; 335 for (size_t i = 0; i < headers.size(); i++) { 336 memcpy(&init_data[pos], &headers[i].raw_box[0], 337 headers[i].raw_box.size()); 338 pos += headers[i].raw_box.size(); 339 } 340 need_key_cb_.Run(kMp4InitDataType, init_data); 341} 342 343bool MP4StreamParser::PrepareAVCBuffer( 344 const AVCDecoderConfigurationRecord& avc_config, 345 std::vector<uint8>* frame_buf, 346 std::vector<SubsampleEntry>* subsamples) const { 347 // Convert the AVC NALU length fields to Annex B headers, as expected by 348 // decoding libraries. Since this may enlarge the size of the buffer, we also 349 // update the clear byte count for each subsample if encryption is used to 350 // account for the difference in size between the length prefix and Annex B 351 // start code. 352 RCHECK(AVC::ConvertFrameToAnnexB(avc_config.length_size, frame_buf)); 353 if (!subsamples->empty()) { 354 const int nalu_size_diff = 4 - avc_config.length_size; 355 size_t expected_size = runs_->sample_size() + 356 subsamples->size() * nalu_size_diff; 357 RCHECK(frame_buf->size() == expected_size); 358 for (size_t i = 0; i < subsamples->size(); i++) 359 (*subsamples)[i].clear_bytes += nalu_size_diff; 360 } 361 362 if (runs_->is_keyframe()) { 363 // If this is a keyframe, we (re-)inject SPS and PPS headers at the start of 364 // a frame. If subsample info is present, we also update the clear byte 365 // count for that first subsample. 366 RCHECK(AVC::InsertParamSetsAnnexB(avc_config, frame_buf, subsamples)); 367 } 368 369 DCHECK(AVC::IsValidAnnexB(*frame_buf)); 370 return true; 371} 372 373bool MP4StreamParser::PrepareAACBuffer( 374 const AAC& aac_config, std::vector<uint8>* frame_buf, 375 std::vector<SubsampleEntry>* subsamples) const { 376 // Append an ADTS header to every audio sample. 377 RCHECK(aac_config.ConvertEsdsToADTS(frame_buf)); 378 379 // As above, adjust subsample information to account for the headers. AAC is 380 // not required to use subsample encryption, so we may need to add an entry. 381 if (subsamples->empty()) { 382 SubsampleEntry entry; 383 entry.clear_bytes = kADTSHeaderMinSize; 384 entry.cypher_bytes = frame_buf->size() - kADTSHeaderMinSize; 385 subsamples->push_back(entry); 386 } else { 387 (*subsamples)[0].clear_bytes += kADTSHeaderMinSize; 388 } 389 return true; 390} 391 392bool MP4StreamParser::EnqueueSample(BufferQueue* audio_buffers, 393 BufferQueue* video_buffers, 394 bool* err) { 395 if (!runs_->IsRunValid()) { 396 // Flush any buffers we've gotten in this chunk so that buffers don't 397 // cross NewSegment() calls 398 *err = !SendAndFlushSamples(audio_buffers, video_buffers); 399 if (*err) 400 return false; 401 402 // Remain in kEnqueueingSamples state, discarding data, until the end of 403 // the current 'mdat' box has been appended to the queue. 404 if (!queue_.Trim(mdat_tail_)) 405 return false; 406 407 ChangeState(kParsingBoxes); 408 end_of_segment_cb_.Run(); 409 return true; 410 } 411 412 if (!runs_->IsSampleValid()) { 413 runs_->AdvanceRun(); 414 return true; 415 } 416 417 DCHECK(!(*err)); 418 419 const uint8* buf; 420 int buf_size; 421 queue_.Peek(&buf, &buf_size); 422 if (!buf_size) return false; 423 424 bool audio = has_audio_ && audio_track_id_ == runs_->track_id(); 425 bool video = has_video_ && video_track_id_ == runs_->track_id(); 426 427 // Skip this entire track if it's not one we're interested in 428 if (!audio && !video) 429 runs_->AdvanceRun(); 430 431 // AuxInfo is required for encrypted samples. 432 // See ISO Common Encryption spec: ISO/IEC FDIS 23001-7:2011(E); 433 // Section 7: Common Encryption Sample Auxiliary Information. 434 if (runs_->is_encrypted() && !runs_->aux_info_size()) 435 return false; 436 437 // Attempt to cache the auxiliary information first. Aux info is usually 438 // placed in a contiguous block before the sample data, rather than being 439 // interleaved. If we didn't cache it, this would require that we retain the 440 // start of the segment buffer while reading samples. Aux info is typically 441 // quite small compared to sample data, so this pattern is useful on 442 // memory-constrained devices where the source buffer consumes a substantial 443 // portion of the total system memory. 444 if (runs_->AuxInfoNeedsToBeCached()) { 445 queue_.PeekAt(runs_->aux_info_offset() + moof_head_, &buf, &buf_size); 446 if (buf_size < runs_->aux_info_size()) return false; 447 *err = !runs_->CacheAuxInfo(buf, buf_size); 448 return !*err; 449 } 450 451 queue_.PeekAt(runs_->sample_offset() + moof_head_, &buf, &buf_size); 452 if (buf_size < runs_->sample_size()) return false; 453 454 scoped_ptr<DecryptConfig> decrypt_config; 455 std::vector<SubsampleEntry> subsamples; 456 if (runs_->is_encrypted()) { 457 decrypt_config = runs_->GetDecryptConfig(); 458 if (!decrypt_config) { 459 *err = true; 460 return false; 461 } 462 subsamples = decrypt_config->subsamples(); 463 } 464 465 std::vector<uint8> frame_buf(buf, buf + runs_->sample_size()); 466 if (video) { 467 if (!PrepareAVCBuffer(runs_->video_description().avcc, 468 &frame_buf, &subsamples)) { 469 MEDIA_LOG(log_cb_) << "Failed to prepare AVC sample for decode"; 470 *err = true; 471 return false; 472 } 473 } 474 475 if (audio) { 476 if (ESDescriptor::IsAAC(runs_->audio_description().esds.object_type) && 477 !PrepareAACBuffer(runs_->audio_description().esds.aac, 478 &frame_buf, &subsamples)) { 479 MEDIA_LOG(log_cb_) << "Failed to prepare AAC sample for decode"; 480 *err = true; 481 return false; 482 } 483 } 484 485 if (decrypt_config) { 486 if (!subsamples.empty()) { 487 // Create a new config with the updated subsamples. 488 decrypt_config.reset(new DecryptConfig( 489 decrypt_config->key_id(), 490 decrypt_config->iv(), 491 subsamples)); 492 } 493 // else, use the existing config. 494 } else if ((audio && is_audio_track_encrypted_) || 495 (video && is_video_track_encrypted_)) { 496 // The media pipeline requires a DecryptConfig with an empty |iv|. 497 // TODO(ddorwin): Refactor so we do not need a fake key ID ("1"); 498 decrypt_config.reset( 499 new DecryptConfig("1", "", std::vector<SubsampleEntry>())); 500 } 501 502 StreamParserBuffer::Type buffer_type = audio ? DemuxerStream::AUDIO : 503 DemuxerStream::VIDEO; 504 505 // TODO(wolenetz/acolwell): Validate and use a common cross-parser TrackId 506 // type and allow multiple tracks for same media type, if applicable. See 507 // https://crbug.com/341581. 508 scoped_refptr<StreamParserBuffer> stream_buf = 509 StreamParserBuffer::CopyFrom(&frame_buf[0], frame_buf.size(), 510 runs_->is_keyframe(), buffer_type, 0); 511 512 if (decrypt_config) 513 stream_buf->set_decrypt_config(decrypt_config.Pass()); 514 515 stream_buf->set_duration(runs_->duration()); 516 stream_buf->set_timestamp(runs_->cts()); 517 stream_buf->SetDecodeTimestamp(runs_->dts()); 518 519 DVLOG(3) << "Pushing frame: aud=" << audio 520 << ", key=" << runs_->is_keyframe() 521 << ", dur=" << runs_->duration().InMilliseconds() 522 << ", dts=" << runs_->dts().InMilliseconds() 523 << ", cts=" << runs_->cts().InMilliseconds() 524 << ", size=" << runs_->sample_size(); 525 526 if (audio) { 527 audio_buffers->push_back(stream_buf); 528 } else { 529 video_buffers->push_back(stream_buf); 530 } 531 532 runs_->AdvanceSample(); 533 return true; 534} 535 536bool MP4StreamParser::SendAndFlushSamples(BufferQueue* audio_buffers, 537 BufferQueue* video_buffers) { 538 if (audio_buffers->empty() && video_buffers->empty()) 539 return true; 540 541 TextBufferQueueMap empty_text_map; 542 bool success = new_buffers_cb_.Run(*audio_buffers, 543 *video_buffers, 544 empty_text_map); 545 audio_buffers->clear(); 546 video_buffers->clear(); 547 return success; 548} 549 550bool MP4StreamParser::ReadAndDiscardMDATsUntil(const int64 offset) { 551 bool err = false; 552 while (mdat_tail_ < offset) { 553 const uint8* buf; 554 int size; 555 queue_.PeekAt(mdat_tail_, &buf, &size); 556 557 FourCC type; 558 int box_sz; 559 if (!BoxReader::StartTopLevelBox(buf, size, log_cb_, 560 &type, &box_sz, &err)) 561 break; 562 563 if (type != FOURCC_MDAT) { 564 MEDIA_LOG(log_cb_) << "Unexpected box type while parsing MDATs: " 565 << FourCCToString(type); 566 } 567 mdat_tail_ += box_sz; 568 } 569 queue_.Trim(std::min(mdat_tail_, offset)); 570 return !err; 571} 572 573void MP4StreamParser::ChangeState(State new_state) { 574 DVLOG(2) << "Changing state: " << new_state; 575 state_ = new_state; 576} 577 578} // namespace mp4 579} // namespace media 580