1// Copyright 2014 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "media/formats/mp4/mp4_stream_parser.h"
6
7#include "base/callback.h"
8#include "base/callback_helpers.h"
9#include "base/logging.h"
10#include "base/time/time.h"
11#include "media/base/audio_decoder_config.h"
12#include "media/base/stream_parser_buffer.h"
13#include "media/base/text_track_config.h"
14#include "media/base/video_decoder_config.h"
15#include "media/base/video_util.h"
16#include "media/formats/mp4/box_definitions.h"
17#include "media/formats/mp4/box_reader.h"
18#include "media/formats/mp4/es_descriptor.h"
19#include "media/formats/mp4/rcheck.h"
20#include "media/formats/mpeg/adts_constants.h"
21
22namespace media {
23namespace mp4 {
24
25// TODO(xhwang): Figure out the init data type appropriately once it's spec'ed.
26static const char kMp4InitDataType[] = "video/mp4";
27
28MP4StreamParser::MP4StreamParser(const std::set<int>& audio_object_types,
29                                 bool has_sbr)
30    : state_(kWaitingForInit),
31      moof_head_(0),
32      mdat_tail_(0),
33      highest_end_offset_(0),
34      has_audio_(false),
35      has_video_(false),
36      audio_track_id_(0),
37      video_track_id_(0),
38      audio_object_types_(audio_object_types),
39      has_sbr_(has_sbr),
40      is_audio_track_encrypted_(false),
41      is_video_track_encrypted_(false) {
42}
43
44MP4StreamParser::~MP4StreamParser() {}
45
46void MP4StreamParser::Init(const InitCB& init_cb,
47                           const NewConfigCB& config_cb,
48                           const NewBuffersCB& new_buffers_cb,
49                           bool /* ignore_text_tracks */ ,
50                           const NeedKeyCB& need_key_cb,
51                           const NewMediaSegmentCB& new_segment_cb,
52                           const base::Closure& end_of_segment_cb,
53                           const LogCB& log_cb) {
54  DCHECK_EQ(state_, kWaitingForInit);
55  DCHECK(init_cb_.is_null());
56  DCHECK(!init_cb.is_null());
57  DCHECK(!config_cb.is_null());
58  DCHECK(!new_buffers_cb.is_null());
59  DCHECK(!need_key_cb.is_null());
60  DCHECK(!end_of_segment_cb.is_null());
61
62  ChangeState(kParsingBoxes);
63  init_cb_ = init_cb;
64  config_cb_ = config_cb;
65  new_buffers_cb_ = new_buffers_cb;
66  need_key_cb_ = need_key_cb;
67  new_segment_cb_ = new_segment_cb;
68  end_of_segment_cb_ = end_of_segment_cb;
69  log_cb_ = log_cb;
70}
71
72void MP4StreamParser::Reset() {
73  queue_.Reset();
74  runs_.reset();
75  moof_head_ = 0;
76  mdat_tail_ = 0;
77}
78
79void MP4StreamParser::Flush() {
80  DCHECK_NE(state_, kWaitingForInit);
81  Reset();
82  ChangeState(kParsingBoxes);
83}
84
85bool MP4StreamParser::Parse(const uint8* buf, int size) {
86  DCHECK_NE(state_, kWaitingForInit);
87
88  if (state_ == kError)
89    return false;
90
91  queue_.Push(buf, size);
92
93  BufferQueue audio_buffers;
94  BufferQueue video_buffers;
95
96  bool result = false;
97  bool err = false;
98
99  do {
100    switch (state_) {
101      case kWaitingForInit:
102      case kError:
103        NOTREACHED();
104        return false;
105
106      case kParsingBoxes:
107        result = ParseBox(&err);
108        break;
109
110      case kWaitingForSampleData:
111        result = HaveEnoughDataToEnqueueSamples();
112        if (result)
113          ChangeState(kEmittingSamples);
114        break;
115
116      case kEmittingSamples:
117        result = EnqueueSample(&audio_buffers, &video_buffers, &err);
118        if (result) {
119          int64 max_clear = runs_->GetMaxClearOffset() + moof_head_;
120          err = !ReadAndDiscardMDATsUntil(max_clear);
121        }
122        break;
123    }
124  } while (result && !err);
125
126  if (!err)
127    err = !SendAndFlushSamples(&audio_buffers, &video_buffers);
128
129  if (err) {
130    DLOG(ERROR) << "Error while parsing MP4";
131    moov_.reset();
132    Reset();
133    ChangeState(kError);
134    return false;
135  }
136
137  return true;
138}
139
140bool MP4StreamParser::ParseBox(bool* err) {
141  const uint8* buf;
142  int size;
143  queue_.Peek(&buf, &size);
144  if (!size) return false;
145
146  scoped_ptr<BoxReader> reader(
147      BoxReader::ReadTopLevelBox(buf, size, log_cb_, err));
148  if (reader.get() == NULL) return false;
149
150  if (reader->type() == FOURCC_MOOV) {
151    *err = !ParseMoov(reader.get());
152  } else if (reader->type() == FOURCC_MOOF) {
153    moof_head_ = queue_.head();
154    *err = !ParseMoof(reader.get());
155
156    // Set up first mdat offset for ReadMDATsUntil().
157    mdat_tail_ = queue_.head() + reader->size();
158
159    // Return early to avoid evicting 'moof' data from queue. Auxiliary info may
160    // be located anywhere in the file, including inside the 'moof' itself.
161    // (Since 'default-base-is-moof' is mandated, no data references can come
162    // before the head of the 'moof', so keeping this box around is sufficient.)
163    return !(*err);
164  } else {
165    MEDIA_LOG(log_cb_) << "Skipping unrecognized top-level box: "
166                       << FourCCToString(reader->type());
167  }
168
169  queue_.Pop(reader->size());
170  return !(*err);
171}
172
173
174bool MP4StreamParser::ParseMoov(BoxReader* reader) {
175  moov_.reset(new Movie);
176  RCHECK(moov_->Parse(reader));
177  runs_.reset();
178
179  has_audio_ = false;
180  has_video_ = false;
181
182  AudioDecoderConfig audio_config;
183  VideoDecoderConfig video_config;
184
185  for (std::vector<Track>::const_iterator track = moov_->tracks.begin();
186       track != moov_->tracks.end(); ++track) {
187    // TODO(strobe): Only the first audio and video track present in a file are
188    // used. (Track selection is better accomplished via Source IDs, though, so
189    // adding support for track selection within a stream is low-priority.)
190    const SampleDescription& samp_descr =
191        track->media.information.sample_table.description;
192
193    // TODO(strobe): When codec reconfigurations are supported, detect and send
194    // a codec reconfiguration for fragments using a sample description index
195    // different from the previous one
196    size_t desc_idx = 0;
197    for (size_t t = 0; t < moov_->extends.tracks.size(); t++) {
198      const TrackExtends& trex = moov_->extends.tracks[t];
199      if (trex.track_id == track->header.track_id) {
200        desc_idx = trex.default_sample_description_index;
201        break;
202      }
203    }
204    RCHECK(desc_idx > 0);
205    desc_idx -= 1;  // BMFF descriptor index is one-based
206
207    if (track->media.handler.type == kAudio && !audio_config.IsValidConfig()) {
208      RCHECK(!samp_descr.audio_entries.empty());
209
210      // It is not uncommon to find otherwise-valid files with incorrect sample
211      // description indices, so we fail gracefully in that case.
212      if (desc_idx >= samp_descr.audio_entries.size())
213        desc_idx = 0;
214      const AudioSampleEntry& entry = samp_descr.audio_entries[desc_idx];
215      const AAC& aac = entry.esds.aac;
216
217      if (!(entry.format == FOURCC_MP4A ||
218            (entry.format == FOURCC_ENCA &&
219             entry.sinf.format.format == FOURCC_MP4A))) {
220        MEDIA_LOG(log_cb_) << "Unsupported audio format 0x"
221                           << std::hex << entry.format << " in stsd box.";
222        return false;
223      }
224
225      uint8 audio_type = entry.esds.object_type;
226      DVLOG(1) << "audio_type " << std::hex << static_cast<int>(audio_type);
227      if (audio_object_types_.find(audio_type) == audio_object_types_.end()) {
228        MEDIA_LOG(log_cb_) << "audio object type 0x" << std::hex << audio_type
229                           << " does not match what is specified in the"
230                           << " mimetype.";
231        return false;
232      }
233
234      AudioCodec codec = kUnknownAudioCodec;
235      ChannelLayout channel_layout = CHANNEL_LAYOUT_NONE;
236      int sample_per_second = 0;
237      std::vector<uint8> extra_data;
238      // Check if it is MPEG4 AAC defined in ISO 14496 Part 3 or
239      // supported MPEG2 AAC varients.
240      if (ESDescriptor::IsAAC(audio_type)) {
241        codec = kCodecAAC;
242        channel_layout = aac.GetChannelLayout(has_sbr_);
243        sample_per_second = aac.GetOutputSamplesPerSecond(has_sbr_);
244#if defined(OS_ANDROID)
245        extra_data = aac.codec_specific_data();
246#endif
247      } else {
248        MEDIA_LOG(log_cb_) << "Unsupported audio object type 0x" << std::hex
249                           << audio_type << " in esds.";
250        return false;
251      }
252
253      SampleFormat sample_format;
254      if (entry.samplesize == 8) {
255        sample_format = kSampleFormatU8;
256      } else if (entry.samplesize == 16) {
257        sample_format = kSampleFormatS16;
258      } else if (entry.samplesize == 32) {
259        sample_format = kSampleFormatS32;
260      } else {
261        LOG(ERROR) << "Unsupported sample size.";
262        return false;
263      }
264
265      is_audio_track_encrypted_ = entry.sinf.info.track_encryption.is_encrypted;
266      DVLOG(1) << "is_audio_track_encrypted_: " << is_audio_track_encrypted_;
267      audio_config.Initialize(
268          codec, sample_format, channel_layout, sample_per_second,
269          extra_data.size() ? &extra_data[0] : NULL, extra_data.size(),
270          is_audio_track_encrypted_, false, base::TimeDelta(),
271          0);
272      has_audio_ = true;
273      audio_track_id_ = track->header.track_id;
274    }
275    if (track->media.handler.type == kVideo && !video_config.IsValidConfig()) {
276      RCHECK(!samp_descr.video_entries.empty());
277      if (desc_idx >= samp_descr.video_entries.size())
278        desc_idx = 0;
279      const VideoSampleEntry& entry = samp_descr.video_entries[desc_idx];
280
281      if (!entry.IsFormatValid()) {
282        MEDIA_LOG(log_cb_) << "Unsupported video format 0x"
283                           << std::hex << entry.format << " in stsd box.";
284        return false;
285      }
286
287      // TODO(strobe): Recover correct crop box
288      gfx::Size coded_size(entry.width, entry.height);
289      gfx::Rect visible_rect(coded_size);
290      gfx::Size natural_size = GetNaturalSize(visible_rect.size(),
291                                              entry.pixel_aspect.h_spacing,
292                                              entry.pixel_aspect.v_spacing);
293      is_video_track_encrypted_ = entry.sinf.info.track_encryption.is_encrypted;
294      DVLOG(1) << "is_video_track_encrypted_: " << is_video_track_encrypted_;
295      video_config.Initialize(kCodecH264, H264PROFILE_MAIN,  VideoFrame::YV12,
296                              coded_size, visible_rect, natural_size,
297                              // No decoder-specific buffer needed for AVC;
298                              // SPS/PPS are embedded in the video stream
299                              NULL, 0, is_video_track_encrypted_, false);
300      has_video_ = true;
301      video_track_id_ = track->header.track_id;
302    }
303  }
304
305  RCHECK(config_cb_.Run(audio_config, video_config, TextTrackConfigMap()));
306
307  StreamParser::InitParameters params(kInfiniteDuration());
308  if (moov_->extends.header.fragment_duration > 0) {
309    params.duration = TimeDeltaFromRational(
310        moov_->extends.header.fragment_duration, moov_->header.timescale);
311  } else if (moov_->header.duration > 0 &&
312             moov_->header.duration != kuint64max) {
313    params.duration =
314        TimeDeltaFromRational(moov_->header.duration, moov_->header.timescale);
315  }
316
317  if (!init_cb_.is_null())
318    base::ResetAndReturn(&init_cb_).Run(true, params);
319
320  EmitNeedKeyIfNecessary(moov_->pssh);
321  return true;
322}
323
324bool MP4StreamParser::ParseMoof(BoxReader* reader) {
325  RCHECK(moov_.get());  // Must already have initialization segment
326  MovieFragment moof;
327  RCHECK(moof.Parse(reader));
328  if (!runs_)
329    runs_.reset(new TrackRunIterator(moov_.get(), log_cb_));
330  RCHECK(runs_->Init(moof));
331  RCHECK(ComputeHighestEndOffset(moof));
332  EmitNeedKeyIfNecessary(moof.pssh);
333  new_segment_cb_.Run();
334  ChangeState(kWaitingForSampleData);
335  return true;
336}
337
338void MP4StreamParser::EmitNeedKeyIfNecessary(
339    const std::vector<ProtectionSystemSpecificHeader>& headers) {
340  // TODO(strobe): ensure that the value of init_data (all PSSH headers
341  // concatenated in arbitrary order) matches the EME spec.
342  // See https://www.w3.org/Bugs/Public/show_bug.cgi?id=17673.
343  if (headers.empty())
344    return;
345
346  size_t total_size = 0;
347  for (size_t i = 0; i < headers.size(); i++)
348    total_size += headers[i].raw_box.size();
349
350  std::vector<uint8> init_data(total_size);
351  size_t pos = 0;
352  for (size_t i = 0; i < headers.size(); i++) {
353    memcpy(&init_data[pos], &headers[i].raw_box[0],
354           headers[i].raw_box.size());
355    pos += headers[i].raw_box.size();
356  }
357  need_key_cb_.Run(kMp4InitDataType, init_data);
358}
359
360bool MP4StreamParser::PrepareAVCBuffer(
361    const AVCDecoderConfigurationRecord& avc_config,
362    std::vector<uint8>* frame_buf,
363    std::vector<SubsampleEntry>* subsamples) const {
364  // Convert the AVC NALU length fields to Annex B headers, as expected by
365  // decoding libraries. Since this may enlarge the size of the buffer, we also
366  // update the clear byte count for each subsample if encryption is used to
367  // account for the difference in size between the length prefix and Annex B
368  // start code.
369  RCHECK(AVC::ConvertFrameToAnnexB(avc_config.length_size, frame_buf));
370  if (!subsamples->empty()) {
371    const int nalu_size_diff = 4 - avc_config.length_size;
372    size_t expected_size = runs_->sample_size() +
373        subsamples->size() * nalu_size_diff;
374    RCHECK(frame_buf->size() == expected_size);
375    for (size_t i = 0; i < subsamples->size(); i++)
376      (*subsamples)[i].clear_bytes += nalu_size_diff;
377  }
378
379  if (runs_->is_keyframe()) {
380    // If this is a keyframe, we (re-)inject SPS and PPS headers at the start of
381    // a frame. If subsample info is present, we also update the clear byte
382    // count for that first subsample.
383    RCHECK(AVC::InsertParamSetsAnnexB(avc_config, frame_buf, subsamples));
384  }
385
386  DCHECK(AVC::IsValidAnnexB(*frame_buf, *subsamples));
387  return true;
388}
389
390bool MP4StreamParser::PrepareAACBuffer(
391    const AAC& aac_config, std::vector<uint8>* frame_buf,
392    std::vector<SubsampleEntry>* subsamples) const {
393  // Append an ADTS header to every audio sample.
394  RCHECK(aac_config.ConvertEsdsToADTS(frame_buf));
395
396  // As above, adjust subsample information to account for the headers. AAC is
397  // not required to use subsample encryption, so we may need to add an entry.
398  if (subsamples->empty()) {
399    SubsampleEntry entry;
400    entry.clear_bytes = kADTSHeaderMinSize;
401    entry.cypher_bytes = frame_buf->size() - kADTSHeaderMinSize;
402    subsamples->push_back(entry);
403  } else {
404    (*subsamples)[0].clear_bytes += kADTSHeaderMinSize;
405  }
406  return true;
407}
408
409bool MP4StreamParser::EnqueueSample(BufferQueue* audio_buffers,
410                                    BufferQueue* video_buffers,
411                                    bool* err) {
412  DCHECK_EQ(state_, kEmittingSamples);
413
414  if (!runs_->IsRunValid()) {
415    // Flush any buffers we've gotten in this chunk so that buffers don't
416    // cross NewSegment() calls
417    *err = !SendAndFlushSamples(audio_buffers, video_buffers);
418    if (*err)
419      return false;
420
421    // Remain in kEmittingSamples state, discarding data, until the end of
422    // the current 'mdat' box has been appended to the queue.
423    if (!queue_.Trim(mdat_tail_))
424      return false;
425
426    ChangeState(kParsingBoxes);
427    end_of_segment_cb_.Run();
428    return true;
429  }
430
431  if (!runs_->IsSampleValid()) {
432    runs_->AdvanceRun();
433    return true;
434  }
435
436  DCHECK(!(*err));
437
438  const uint8* buf;
439  int buf_size;
440  queue_.Peek(&buf, &buf_size);
441  if (!buf_size) return false;
442
443  bool audio = has_audio_ && audio_track_id_ == runs_->track_id();
444  bool video = has_video_ && video_track_id_ == runs_->track_id();
445
446  // Skip this entire track if it's not one we're interested in
447  if (!audio && !video) {
448    runs_->AdvanceRun();
449    return true;
450  }
451
452  // Attempt to cache the auxiliary information first. Aux info is usually
453  // placed in a contiguous block before the sample data, rather than being
454  // interleaved. If we didn't cache it, this would require that we retain the
455  // start of the segment buffer while reading samples. Aux info is typically
456  // quite small compared to sample data, so this pattern is useful on
457  // memory-constrained devices where the source buffer consumes a substantial
458  // portion of the total system memory.
459  if (runs_->AuxInfoNeedsToBeCached()) {
460    queue_.PeekAt(runs_->aux_info_offset() + moof_head_, &buf, &buf_size);
461    if (buf_size < runs_->aux_info_size()) return false;
462    *err = !runs_->CacheAuxInfo(buf, buf_size);
463    return !*err;
464  }
465
466  queue_.PeekAt(runs_->sample_offset() + moof_head_, &buf, &buf_size);
467  if (buf_size < runs_->sample_size()) return false;
468
469  scoped_ptr<DecryptConfig> decrypt_config;
470  std::vector<SubsampleEntry> subsamples;
471  if (runs_->is_encrypted()) {
472    decrypt_config = runs_->GetDecryptConfig();
473    if (!decrypt_config) {
474      *err = true;
475      return false;
476    }
477    subsamples = decrypt_config->subsamples();
478  }
479
480  std::vector<uint8> frame_buf(buf, buf + runs_->sample_size());
481  if (video) {
482    if (!PrepareAVCBuffer(runs_->video_description().avcc,
483                          &frame_buf, &subsamples)) {
484      MEDIA_LOG(log_cb_) << "Failed to prepare AVC sample for decode";
485      *err = true;
486      return false;
487    }
488  }
489
490  if (audio) {
491    if (ESDescriptor::IsAAC(runs_->audio_description().esds.object_type) &&
492        !PrepareAACBuffer(runs_->audio_description().esds.aac,
493                          &frame_buf, &subsamples)) {
494      MEDIA_LOG(log_cb_) << "Failed to prepare AAC sample for decode";
495      *err = true;
496      return false;
497    }
498  }
499
500  if (decrypt_config) {
501    if (!subsamples.empty()) {
502    // Create a new config with the updated subsamples.
503    decrypt_config.reset(new DecryptConfig(
504        decrypt_config->key_id(),
505        decrypt_config->iv(),
506        subsamples));
507    }
508    // else, use the existing config.
509  } else if ((audio && is_audio_track_encrypted_) ||
510             (video && is_video_track_encrypted_)) {
511    // The media pipeline requires a DecryptConfig with an empty |iv|.
512    // TODO(ddorwin): Refactor so we do not need a fake key ID ("1");
513    decrypt_config.reset(
514        new DecryptConfig("1", "", std::vector<SubsampleEntry>()));
515  }
516
517  StreamParserBuffer::Type buffer_type = audio ? DemuxerStream::AUDIO :
518      DemuxerStream::VIDEO;
519
520  // TODO(wolenetz/acolwell): Validate and use a common cross-parser TrackId
521  // type and allow multiple tracks for same media type, if applicable. See
522  // https://crbug.com/341581.
523  //
524  // NOTE: MPEG's "random access point" concept is equivalent to the
525  // downstream code's "is keyframe" concept.
526  scoped_refptr<StreamParserBuffer> stream_buf =
527      StreamParserBuffer::CopyFrom(&frame_buf[0], frame_buf.size(),
528                                   runs_->is_random_access_point(),
529                                   buffer_type, 0);
530
531  if (decrypt_config)
532    stream_buf->set_decrypt_config(decrypt_config.Pass());
533
534  stream_buf->set_duration(runs_->duration());
535  stream_buf->set_timestamp(runs_->cts());
536  stream_buf->SetDecodeTimestamp(runs_->dts());
537
538  DVLOG(3) << "Pushing frame: aud=" << audio
539           << ", key=" << runs_->is_keyframe()
540           << ", rap=" << runs_->is_random_access_point()
541           << ", dur=" << runs_->duration().InMilliseconds()
542           << ", dts=" << runs_->dts().InMilliseconds()
543           << ", cts=" << runs_->cts().InMilliseconds()
544           << ", size=" << runs_->sample_size();
545
546  if (audio) {
547    audio_buffers->push_back(stream_buf);
548  } else {
549    video_buffers->push_back(stream_buf);
550  }
551
552  runs_->AdvanceSample();
553  return true;
554}
555
556bool MP4StreamParser::SendAndFlushSamples(BufferQueue* audio_buffers,
557                                          BufferQueue* video_buffers) {
558  if (audio_buffers->empty() && video_buffers->empty())
559    return true;
560
561  TextBufferQueueMap empty_text_map;
562  bool success = new_buffers_cb_.Run(*audio_buffers,
563                                     *video_buffers,
564                                     empty_text_map);
565  audio_buffers->clear();
566  video_buffers->clear();
567  return success;
568}
569
570bool MP4StreamParser::ReadAndDiscardMDATsUntil(int64 max_clear_offset) {
571  bool err = false;
572  int64 upper_bound = std::min(max_clear_offset, queue_.tail());
573  while (mdat_tail_ < upper_bound) {
574    const uint8* buf = NULL;
575    int size = 0;
576    queue_.PeekAt(mdat_tail_, &buf, &size);
577
578    FourCC type;
579    int box_sz;
580    if (!BoxReader::StartTopLevelBox(buf, size, log_cb_,
581                                     &type, &box_sz, &err))
582      break;
583
584    if (type != FOURCC_MDAT) {
585      MEDIA_LOG(log_cb_) << "Unexpected box type while parsing MDATs: "
586                         << FourCCToString(type);
587    }
588    mdat_tail_ += box_sz;
589  }
590  queue_.Trim(std::min(mdat_tail_, upper_bound));
591  return !err;
592}
593
594void MP4StreamParser::ChangeState(State new_state) {
595  DVLOG(2) << "Changing state: " << new_state;
596  state_ = new_state;
597}
598
599bool MP4StreamParser::HaveEnoughDataToEnqueueSamples() {
600  DCHECK_EQ(state_, kWaitingForSampleData);
601  // For muxed content, make sure we have data up to |highest_end_offset_|
602  // so we can ensure proper enqueuing behavior. Otherwise assume we have enough
603  // data and allow per sample offset checks to meter sample enqueuing.
604  // TODO(acolwell): Fix trun box handling so we don't have to special case
605  // muxed content.
606  return !(has_audio_ && has_video_ &&
607           queue_.tail() < highest_end_offset_ + moof_head_);
608}
609
610bool MP4StreamParser::ComputeHighestEndOffset(const MovieFragment& moof) {
611  highest_end_offset_ = 0;
612
613  TrackRunIterator runs(moov_.get(), log_cb_);
614  RCHECK(runs.Init(moof));
615
616  while (runs.IsRunValid()) {
617    int64 aux_info_end_offset = runs.aux_info_offset() + runs.aux_info_size();
618    if (aux_info_end_offset > highest_end_offset_)
619      highest_end_offset_ = aux_info_end_offset;
620
621    while (runs.IsSampleValid()) {
622      int64 sample_end_offset = runs.sample_offset() + runs.sample_size();
623      if (sample_end_offset > highest_end_offset_)
624        highest_end_offset_ = sample_end_offset;
625
626      runs.AdvanceSample();
627    }
628    runs.AdvanceRun();
629  }
630
631  return true;
632}
633
634}  // namespace mp4
635}  // namespace media
636