1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5// Test application that simulates a cast sender - Data can be either generated
6// or read from a file.
7
8#include <queue>
9
10#include "base/at_exit.h"
11#include "base/base_paths.h"
12#include "base/command_line.h"
13#include "base/file_util.h"
14#include "base/files/file_path.h"
15#include "base/files/memory_mapped_file.h"
16#include "base/files/scoped_file.h"
17#include "base/json/json_writer.h"
18#include "base/logging.h"
19#include "base/memory/scoped_ptr.h"
20#include "base/path_service.h"
21#include "base/strings/string_number_conversions.h"
22#include "base/threading/thread.h"
23#include "base/time/default_tick_clock.h"
24#include "base/values.h"
25#include "media/audio/audio_parameters.h"
26#include "media/base/audio_buffer.h"
27#include "media/base/audio_bus.h"
28#include "media/base/audio_fifo.h"
29#include "media/base/audio_timestamp_helper.h"
30#include "media/base/media.h"
31#include "media/base/multi_channel_resampler.h"
32#include "media/base/video_frame.h"
33#include "media/base/video_util.h"
34#include "media/cast/cast_config.h"
35#include "media/cast/cast_environment.h"
36#include "media/cast/cast_sender.h"
37#include "media/cast/logging/encoding_event_subscriber.h"
38#include "media/cast/logging/log_serializer.h"
39#include "media/cast/logging/logging_defines.h"
40#include "media/cast/logging/proto/raw_events.pb.h"
41#include "media/cast/logging/receiver_time_offset_estimator_impl.h"
42#include "media/cast/logging/stats_event_subscriber.h"
43#include "media/cast/test/utility/audio_utility.h"
44#include "media/cast/test/utility/default_config.h"
45#include "media/cast/test/utility/input_builder.h"
46#include "media/cast/test/utility/video_utility.h"
47#include "media/cast/transport/cast_transport_defines.h"
48#include "media/cast/transport/cast_transport_sender.h"
49#include "media/cast/transport/transport/udp_transport.h"
50#include "media/ffmpeg/ffmpeg_common.h"
51#include "media/ffmpeg/ffmpeg_deleters.h"
52#include "media/filters/audio_renderer_algorithm.h"
53#include "media/filters/ffmpeg_demuxer.h"
54#include "media/filters/ffmpeg_glue.h"
55#include "media/filters/in_memory_url_protocol.h"
56#include "ui/gfx/size.h"
57
58namespace {
59static const int kAudioChannels = 2;
60static const int kAudioSamplingFrequency = 48000;
61static const int kSoundFrequency = 1234;  // Frequency of sinusoid wave.
62static const float kSoundVolume = 0.5f;
63static const int kAudioFrameMs = 10;  // Each audio frame is exactly 10ms.
64static const int kAudioPacketsPerSecond = 1000 / kAudioFrameMs;
65
66// The max allowed size of serialized log.
67const int kMaxSerializedLogBytes = 10 * 1000 * 1000;
68
69// Flags for this program:
70//
71// --address=xx.xx.xx.xx
72//   IP address of receiver.
73//
74// --port=xxxx
75//   Port number of receiver.
76//
77// --source-file=xxx.webm
78//   WebM file as source of video frames.
79//
80// --fps=xx
81//   Override framerate of the video stream.
82
83const char kSwitchAddress[] = "address";
84const char kSwitchPort[] = "port";
85const char kSwitchSourceFile[] = "source-file";
86const char kSwitchFps[] = "fps";
87
88}  // namespace
89
90namespace media {
91namespace cast {
92
93AudioSenderConfig GetAudioSenderConfig() {
94  AudioSenderConfig audio_config;
95
96  audio_config.rtcp_c_name = "audio_sender@a.b.c.d";
97
98  audio_config.use_external_encoder = false;
99  audio_config.frequency = kAudioSamplingFrequency;
100  audio_config.channels = kAudioChannels;
101  audio_config.bitrate = 64000;
102  audio_config.codec = transport::kOpus;
103  audio_config.rtp_config.ssrc = 1;
104  audio_config.incoming_feedback_ssrc = 2;
105  audio_config.rtp_config.payload_type = 127;
106  audio_config.rtp_config.max_delay_ms = 300;
107  return audio_config;
108}
109
110VideoSenderConfig GetVideoSenderConfig() {
111  VideoSenderConfig video_config;
112
113  video_config.rtcp_c_name = "video_sender@a.b.c.d";
114  video_config.use_external_encoder = false;
115
116  // Resolution.
117  video_config.width = 1280;
118  video_config.height = 720;
119  video_config.max_frame_rate = 30;
120
121  // Bitrates.
122  video_config.max_bitrate = 2500000;
123  video_config.min_bitrate = 100000;
124  video_config.start_bitrate = video_config.min_bitrate;
125
126  // Codec.
127  video_config.codec = transport::kVp8;
128  video_config.max_number_of_video_buffers_used = 1;
129  video_config.number_of_encode_threads = 2;
130
131  // Quality options.
132  video_config.min_qp = 4;
133  video_config.max_qp = 40;
134
135  // SSRCs and payload type. Don't change them.
136  video_config.rtp_config.ssrc = 11;
137  video_config.incoming_feedback_ssrc = 12;
138  video_config.rtp_config.payload_type = 96;
139  video_config.rtp_config.max_delay_ms = 300;
140  return video_config;
141}
142
143void AVFreeFrame(AVFrame* frame) { av_frame_free(&frame); }
144
145class SendProcess {
146 public:
147  SendProcess(scoped_refptr<base::SingleThreadTaskRunner> thread_proxy,
148              base::TickClock* clock,
149              const VideoSenderConfig& video_config)
150      : test_app_thread_proxy_(thread_proxy),
151        video_config_(video_config),
152        synthetic_count_(0),
153        clock_(clock),
154        audio_frame_count_(0),
155        video_frame_count_(0),
156        weak_factory_(this),
157        av_format_context_(NULL),
158        audio_stream_index_(-1),
159        playback_rate_(1.0),
160        video_stream_index_(-1),
161        video_frame_rate_numerator_(video_config.max_frame_rate),
162        video_frame_rate_denominator_(1),
163        video_first_pts_(0),
164        video_first_pts_set_(false) {
165    audio_bus_factory_.reset(new TestAudioBusFactory(kAudioChannels,
166                                                     kAudioSamplingFrequency,
167                                                     kSoundFrequency,
168                                                     kSoundVolume));
169    const CommandLine* cmd = CommandLine::ForCurrentProcess();
170    int override_fps = 0;
171    if (base::StringToInt(cmd->GetSwitchValueASCII(kSwitchFps),
172                          &override_fps)) {
173      video_config_.max_frame_rate = override_fps;
174      video_frame_rate_numerator_ = override_fps;
175    }
176
177    // Load source file and prepare FFmpeg demuxer.
178    base::FilePath source_path = cmd->GetSwitchValuePath(kSwitchSourceFile);
179    if (source_path.empty())
180      return;
181
182    LOG(INFO) << "Source: " << source_path.value();
183    if (!file_data_.Initialize(source_path)) {
184      LOG(ERROR) << "Cannot load file.";
185      return;
186    }
187    protocol_.reset(
188        new InMemoryUrlProtocol(file_data_.data(), file_data_.length(), false));
189    glue_.reset(new FFmpegGlue(protocol_.get()));
190
191    if (!glue_->OpenContext()) {
192      LOG(ERROR) << "Cannot open file.";
193      return;
194    }
195
196    // AVFormatContext is owned by the glue.
197    av_format_context_ = glue_->format_context();
198    if (avformat_find_stream_info(av_format_context_, NULL) < 0) {
199      LOG(ERROR) << "Cannot find stream information.";
200      return;
201    }
202
203    // Prepare FFmpeg decoders.
204    for (unsigned int i = 0; i < av_format_context_->nb_streams; ++i) {
205      AVStream* av_stream = av_format_context_->streams[i];
206      AVCodecContext* av_codec_context = av_stream->codec;
207      AVCodec* av_codec = avcodec_find_decoder(av_codec_context->codec_id);
208
209      if (!av_codec) {
210        LOG(ERROR) << "Cannot find decoder for the codec: "
211                   << av_codec_context->codec_id;
212        continue;
213      }
214
215      // Number of threads for decoding.
216      av_codec_context->thread_count = 2;
217      av_codec_context->error_concealment = FF_EC_GUESS_MVS | FF_EC_DEBLOCK;
218      av_codec_context->request_sample_fmt = AV_SAMPLE_FMT_S16;
219
220      if (avcodec_open2(av_codec_context, av_codec, NULL) < 0) {
221        LOG(ERROR) << "Cannot open AVCodecContext for the codec: "
222                   << av_codec_context->codec_id;
223        return;
224      }
225
226      if (av_codec->type == AVMEDIA_TYPE_AUDIO) {
227        if (av_codec_context->sample_fmt == AV_SAMPLE_FMT_S16P) {
228          LOG(ERROR) << "Audio format not supported.";
229          continue;
230        }
231        ChannelLayout layout = ChannelLayoutToChromeChannelLayout(
232            av_codec_context->channel_layout,
233            av_codec_context->channels);
234        if (layout == CHANNEL_LAYOUT_UNSUPPORTED) {
235          LOG(ERROR) << "Unsupported audio channels layout.";
236          continue;
237        }
238        if (audio_stream_index_ != -1) {
239          LOG(WARNING) << "Found multiple audio streams.";
240        }
241        audio_stream_index_ = static_cast<int>(i);
242        audio_params_.Reset(
243            AudioParameters::AUDIO_PCM_LINEAR,
244            layout,
245            av_codec_context->channels,
246            av_codec_context->channels,
247            av_codec_context->sample_rate,
248            8 * av_get_bytes_per_sample(av_codec_context->sample_fmt),
249            av_codec_context->sample_rate / kAudioPacketsPerSecond);
250        LOG(INFO) << "Source file has audio.";
251      } else if (av_codec->type == AVMEDIA_TYPE_VIDEO) {
252        VideoFrame::Format format =
253            PixelFormatToVideoFormat(av_codec_context->pix_fmt);
254        if (format != VideoFrame::YV12) {
255          LOG(ERROR) << "Cannot handle non YV12 video format: " << format;
256          continue;
257        }
258        if (video_stream_index_ != -1) {
259          LOG(WARNING) << "Found multiple video streams.";
260        }
261        video_stream_index_ = static_cast<int>(i);
262        if (!override_fps) {
263          video_frame_rate_numerator_ = av_stream->r_frame_rate.num;
264          video_frame_rate_denominator_ = av_stream->r_frame_rate.den;
265          // Max frame rate is rounded up.
266          video_config_.max_frame_rate =
267              video_frame_rate_denominator_ +
268              video_frame_rate_numerator_ - 1;
269          video_config_.max_frame_rate /= video_frame_rate_denominator_;
270        } else {
271          // If video is played at a manual speed audio needs to match.
272          playback_rate_ = 1.0 * override_fps *
273               av_stream->r_frame_rate.den /  av_stream->r_frame_rate.num;
274        }
275        LOG(INFO) << "Source file has video.";
276      } else {
277        LOG(ERROR) << "Unknown stream type; ignore.";
278      }
279    }
280
281    Rewind();
282  }
283
284  ~SendProcess() {
285  }
286
287  void Start(scoped_refptr<AudioFrameInput> audio_frame_input,
288             scoped_refptr<VideoFrameInput> video_frame_input) {
289    audio_frame_input_ = audio_frame_input;
290    video_frame_input_ = video_frame_input;
291
292    LOG(INFO) << "Max Frame rate: " << video_config_.max_frame_rate;
293    LOG(INFO) << "Real Frame rate: "
294              << video_frame_rate_numerator_ << "/"
295              << video_frame_rate_denominator_ << " fps.";
296    LOG(INFO) << "Audio playback rate: " << playback_rate_;
297
298    if (!is_transcoding_audio() && !is_transcoding_video()) {
299      // Send fake patterns.
300      test_app_thread_proxy_->PostTask(
301          FROM_HERE,
302          base::Bind(
303              &SendProcess::SendNextFakeFrame,
304              base::Unretained(this)));
305      return;
306    }
307
308    // Send transcoding streams.
309    audio_algo_.Initialize(playback_rate_, audio_params_);
310    audio_algo_.FlushBuffers();
311    audio_fifo_input_bus_ =
312        AudioBus::Create(
313            audio_params_.channels(), audio_params_.frames_per_buffer());
314    // Audio FIFO can carry all data fron AudioRendererAlgorithm.
315    audio_fifo_.reset(
316        new AudioFifo(audio_params_.channels(),
317                      audio_algo_.QueueCapacity()));
318    audio_resampler_.reset(new media::MultiChannelResampler(
319        audio_params_.channels(),
320        static_cast<double>(audio_params_.sample_rate()) /
321        kAudioSamplingFrequency,
322        audio_params_.frames_per_buffer(),
323        base::Bind(&SendProcess::ProvideData, base::Unretained(this))));
324    test_app_thread_proxy_->PostTask(
325        FROM_HERE,
326        base::Bind(
327            &SendProcess::SendNextFrame,
328            base::Unretained(this)));
329  }
330
331  void SendNextFakeFrame() {
332    gfx::Size size(video_config_.width, video_config_.height);
333    scoped_refptr<VideoFrame> video_frame =
334        VideoFrame::CreateBlackFrame(size);
335    PopulateVideoFrame(video_frame, synthetic_count_);
336    ++synthetic_count_;
337
338    base::TimeTicks now = clock_->NowTicks();
339    if (start_time_.is_null())
340      start_time_ = now;
341
342    base::TimeDelta video_time = VideoFrameTime(video_frame_count_);
343    video_frame->set_timestamp(video_time);
344    video_frame_input_->InsertRawVideoFrame(video_frame,
345                                            start_time_ + video_time);
346
347    // Send just enough audio data to match next video frame's time.
348    base::TimeDelta audio_time = AudioFrameTime(audio_frame_count_);
349    while (audio_time < video_time) {
350      if (is_transcoding_audio()) {
351        Decode(true);
352        CHECK(!audio_bus_queue_.empty()) << "No audio decoded.";
353        scoped_ptr<AudioBus> bus(audio_bus_queue_.front());
354        audio_bus_queue_.pop();
355        audio_frame_input_->InsertAudio(
356            bus.Pass(), start_time_ + audio_time);
357      } else {
358        audio_frame_input_->InsertAudio(
359            audio_bus_factory_->NextAudioBus(
360                base::TimeDelta::FromMilliseconds(kAudioFrameMs)),
361            start_time_ + audio_time);
362      }
363      audio_time = AudioFrameTime(++audio_frame_count_);
364    }
365
366    // This is the time since the stream started.
367    const base::TimeDelta elapsed_time = now - start_time_;
368
369    // Handle the case when frame generation cannot keep up.
370    // Move the time ahead to match the next frame.
371    while (video_time < elapsed_time) {
372      LOG(WARNING) << "Skipping one frame.";
373      video_time = VideoFrameTime(++video_frame_count_);
374    }
375
376    test_app_thread_proxy_->PostDelayedTask(
377        FROM_HERE,
378        base::Bind(&SendProcess::SendNextFakeFrame,
379                   weak_factory_.GetWeakPtr()),
380        video_time - elapsed_time);
381  }
382
383  // Return true if a frame was sent.
384  bool SendNextTranscodedVideo(base::TimeDelta elapsed_time) {
385    if (!is_transcoding_video())
386      return false;
387
388    Decode(false);
389    if (video_frame_queue_.empty())
390      return false;
391
392    scoped_refptr<VideoFrame> decoded_frame =
393        video_frame_queue_.front();
394    if (elapsed_time < decoded_frame->timestamp())
395      return false;
396
397    gfx::Size size(video_config_.width, video_config_.height);
398    scoped_refptr<VideoFrame> video_frame =
399        VideoFrame::CreateBlackFrame(size);
400    video_frame_queue_.pop();
401    media::CopyPlane(VideoFrame::kYPlane,
402                     decoded_frame->data(VideoFrame::kYPlane),
403                     decoded_frame->stride(VideoFrame::kYPlane),
404                     decoded_frame->rows(VideoFrame::kYPlane),
405                     video_frame);
406    media::CopyPlane(VideoFrame::kUPlane,
407                     decoded_frame->data(VideoFrame::kUPlane),
408                     decoded_frame->stride(VideoFrame::kUPlane),
409                     decoded_frame->rows(VideoFrame::kUPlane),
410                     video_frame);
411    media::CopyPlane(VideoFrame::kVPlane,
412                     decoded_frame->data(VideoFrame::kVPlane),
413                     decoded_frame->stride(VideoFrame::kVPlane),
414                     decoded_frame->rows(VideoFrame::kVPlane),
415                     video_frame);
416
417    base::TimeDelta video_time;
418    // Use the timestamp from the file if we're transcoding.
419    video_time = ScaleTimestamp(decoded_frame->timestamp());
420    video_frame_input_->InsertRawVideoFrame(
421        video_frame, start_time_ + video_time);
422
423    // Make sure queue is not empty.
424    Decode(false);
425    return true;
426  }
427
428  // Return true if a frame was sent.
429  bool SendNextTranscodedAudio(base::TimeDelta elapsed_time) {
430    if (!is_transcoding_audio())
431      return false;
432
433    Decode(true);
434    if (audio_bus_queue_.empty())
435      return false;
436
437    base::TimeDelta audio_time = audio_sent_ts_->GetTimestamp();
438    if (elapsed_time < audio_time)
439      return false;
440    scoped_ptr<AudioBus> bus(audio_bus_queue_.front());
441    audio_bus_queue_.pop();
442    audio_sent_ts_->AddFrames(bus->frames());
443    audio_frame_input_->InsertAudio(
444        bus.Pass(), start_time_ + audio_time);
445
446    // Make sure queue is not empty.
447    Decode(true);
448    return true;
449  }
450
451  void SendNextFrame() {
452    if (start_time_.is_null())
453      start_time_ = clock_->NowTicks();
454    if (start_time_.is_null())
455      start_time_ = clock_->NowTicks();
456
457    // Send as much as possible. Audio is sent according to
458    // system time.
459    while (SendNextTranscodedAudio(clock_->NowTicks() - start_time_));
460
461    // Video is sync'ed to audio.
462    while (SendNextTranscodedVideo(audio_sent_ts_->GetTimestamp()));
463
464    if (audio_bus_queue_.empty() && video_frame_queue_.empty()) {
465      // Both queues are empty can only mean that we have reached
466      // the end of the stream.
467      LOG(INFO) << "Rewind.";
468      Rewind();
469      start_time_ = base::TimeTicks();
470      audio_sent_ts_.reset();
471      video_first_pts_set_ = false;
472    }
473
474    // Send next send.
475    test_app_thread_proxy_->PostDelayedTask(
476        FROM_HERE,
477        base::Bind(
478            &SendProcess::SendNextFrame,
479            base::Unretained(this)),
480        base::TimeDelta::FromMilliseconds(kAudioFrameMs));
481  }
482
483  const VideoSenderConfig& get_video_config() const { return video_config_; }
484
485 private:
486  bool is_transcoding_audio() { return audio_stream_index_ >= 0; }
487  bool is_transcoding_video() { return video_stream_index_ >= 0; }
488
489  // Helper methods to compute timestamps for the frame number specified.
490  base::TimeDelta VideoFrameTime(int frame_number) {
491    return frame_number * base::TimeDelta::FromSeconds(1) *
492        video_frame_rate_denominator_ / video_frame_rate_numerator_;
493  }
494
495  base::TimeDelta ScaleTimestamp(base::TimeDelta timestamp) {
496    return base::TimeDelta::FromMicroseconds(
497        timestamp.InMicroseconds() / playback_rate_);
498  }
499
500  base::TimeDelta AudioFrameTime(int frame_number) {
501    return frame_number * base::TimeDelta::FromMilliseconds(kAudioFrameMs);
502  }
503
504  // Go to the beginning of the stream.
505  void Rewind() {
506    CHECK(av_seek_frame(av_format_context_, -1, 0, AVSEEK_FLAG_BACKWARD) >= 0)
507        << "Failed to rewind to the beginning.";
508  }
509
510  // Call FFmpeg to fetch one packet.
511  ScopedAVPacket DemuxOnePacket(bool* audio) {
512    ScopedAVPacket packet(new AVPacket());
513    if (av_read_frame(av_format_context_, packet.get()) < 0) {
514      LOG(ERROR) << "Failed to read one AVPacket.";
515      packet.reset();
516      return packet.Pass();
517    }
518
519    int stream_index = static_cast<int>(packet->stream_index);
520    if (stream_index == audio_stream_index_) {
521      *audio = true;
522    } else if (stream_index == video_stream_index_) {
523      *audio = false;
524    } else {
525      // Ignore unknown packet.
526      LOG(INFO) << "Unknown packet.";
527      packet.reset();
528    }
529    return packet.Pass();
530  }
531
532  void DecodeAudio(ScopedAVPacket packet) {
533    // Audio.
534    AVFrame* avframe = av_frame_alloc();
535
536    // Make a shallow copy of packet so we can slide packet.data as frames are
537    // decoded from the packet; otherwise av_free_packet() will corrupt memory.
538    AVPacket packet_temp = *packet.get();
539
540    do {
541      int frame_decoded = 0;
542      int result = avcodec_decode_audio4(
543          av_audio_context(), avframe, &frame_decoded, &packet_temp);
544      CHECK(result >= 0) << "Failed to decode audio.";
545      packet_temp.size -= result;
546      packet_temp.data += result;
547      if (!frame_decoded)
548        continue;
549
550      int frames_read = avframe->nb_samples;
551      if (frames_read < 0)
552        break;
553
554      if (!audio_sent_ts_) {
555        // Initialize the base time to the first packet in the file.
556        // This is set to the frequency we send to the receiver.
557        // Not the frequency of the source file. This is because we
558        // increment the frame count by samples we sent.
559        audio_sent_ts_.reset(
560            new AudioTimestampHelper(kAudioSamplingFrequency));
561        // For some files this is an invalid value.
562        base::TimeDelta base_ts;
563        audio_sent_ts_->SetBaseTimestamp(base_ts);
564      }
565
566      scoped_refptr<AudioBuffer> buffer =
567          AudioBuffer::CopyFrom(
568              AVSampleFormatToSampleFormat(
569                  av_audio_context()->sample_fmt),
570              ChannelLayoutToChromeChannelLayout(
571                  av_audio_context()->channel_layout,
572                  av_audio_context()->channels),
573              av_audio_context()->channels,
574              av_audio_context()->sample_rate,
575              frames_read,
576              &avframe->data[0],
577              // Note: Not all files have correct values for pkt_pts.
578              base::TimeDelta::FromMilliseconds(avframe->pkt_pts));
579      audio_algo_.EnqueueBuffer(buffer);
580      av_frame_unref(avframe);
581    } while (packet_temp.size > 0);
582    av_frame_free(&avframe);
583
584    const int frames_needed_to_scale =
585        playback_rate_ * av_audio_context()->sample_rate /
586        kAudioPacketsPerSecond;
587    while (frames_needed_to_scale <= audio_algo_.frames_buffered()) {
588      if (!audio_algo_.FillBuffer(audio_fifo_input_bus_.get(),
589                                  audio_fifo_input_bus_->frames())) {
590        // Nothing can be scaled. Decode some more.
591        return;
592      }
593
594      // Prevent overflow of audio data in the FIFO.
595      if (audio_fifo_input_bus_->frames() + audio_fifo_->frames()
596          <= audio_fifo_->max_frames()) {
597        audio_fifo_->Push(audio_fifo_input_bus_.get());
598      } else {
599        LOG(WARNING) << "Audio FIFO full; dropping samples.";
600      }
601
602      // Make sure there's enough data to resample audio.
603      if (audio_fifo_->frames() <
604          2 * audio_params_.sample_rate() / kAudioPacketsPerSecond) {
605        continue;
606      }
607
608      scoped_ptr<media::AudioBus> resampled_bus(
609          media::AudioBus::Create(
610              audio_params_.channels(),
611              kAudioSamplingFrequency / kAudioPacketsPerSecond));
612      audio_resampler_->Resample(resampled_bus->frames(),
613                                 resampled_bus.get());
614      audio_bus_queue_.push(resampled_bus.release());
615    }
616  }
617
618  void DecodeVideo(ScopedAVPacket packet) {
619    // Video.
620    int got_picture;
621    AVFrame* avframe = av_frame_alloc();
622    // Tell the decoder to reorder for us.
623    avframe->reordered_opaque =
624        av_video_context()->reordered_opaque = packet->pts;
625    CHECK(avcodec_decode_video2(
626        av_video_context(), avframe, &got_picture, packet.get()) >= 0)
627        << "Video decode error.";
628    if (!got_picture) {
629      av_frame_free(&avframe);
630      return;
631    }
632    gfx::Size size(av_video_context()->width, av_video_context()->height);
633    if (!video_first_pts_set_ ||
634        avframe->reordered_opaque < video_first_pts_) {
635      video_first_pts_set_ = true;
636      video_first_pts_ = avframe->reordered_opaque;
637    }
638    int64 pts = avframe->reordered_opaque - video_first_pts_;
639    video_frame_queue_.push(
640        VideoFrame::WrapExternalYuvData(
641            media::VideoFrame::YV12,
642            size,
643            gfx::Rect(size),
644            size,
645            avframe->linesize[0],
646            avframe->linesize[1],
647            avframe->linesize[2],
648            avframe->data[0],
649            avframe->data[1],
650            avframe->data[2],
651            base::TimeDelta::FromMilliseconds(pts),
652            base::Bind(&AVFreeFrame, avframe)));
653  }
654
655  void Decode(bool decode_audio) {
656    // Read the stream until one video frame can be decoded.
657    while (true) {
658      if (decode_audio && !audio_bus_queue_.empty())
659        return;
660      if (!decode_audio && !video_frame_queue_.empty())
661        return;
662
663      bool audio_packet = false;
664      ScopedAVPacket packet = DemuxOnePacket(&audio_packet);
665      if (!packet) {
666        LOG(INFO) << "End of stream.";
667        return;
668      }
669
670      if (audio_packet)
671        DecodeAudio(packet.Pass());
672      else
673        DecodeVideo(packet.Pass());
674    }
675  }
676
677  void ProvideData(int frame_delay, media::AudioBus* output_bus) {
678    if (audio_fifo_->frames() >= output_bus->frames()) {
679      audio_fifo_->Consume(output_bus, 0, output_bus->frames());
680    } else {
681      LOG(WARNING) << "Not enough audio data for resampling.";
682      output_bus->Zero();
683    }
684  }
685
686  AVStream* av_audio_stream() {
687    return av_format_context_->streams[audio_stream_index_];
688  }
689  AVStream* av_video_stream() {
690    return av_format_context_->streams[video_stream_index_];
691  }
692  AVCodecContext* av_audio_context() { return av_audio_stream()->codec; }
693  AVCodecContext* av_video_context() { return av_video_stream()->codec; }
694
695  scoped_refptr<base::SingleThreadTaskRunner> test_app_thread_proxy_;
696  VideoSenderConfig video_config_;
697  scoped_refptr<AudioFrameInput> audio_frame_input_;
698  scoped_refptr<VideoFrameInput> video_frame_input_;
699  uint8 synthetic_count_;
700  base::TickClock* const clock_;  // Not owned by this class.
701
702  // Time when the stream starts.
703  base::TimeTicks start_time_;
704
705  // The following three members are used only for fake frames.
706  int audio_frame_count_;  // Each audio frame is exactly 10ms.
707  int video_frame_count_;
708  scoped_ptr<TestAudioBusFactory> audio_bus_factory_;
709
710  // NOTE: Weak pointers must be invalidated before all other member variables.
711  base::WeakPtrFactory<SendProcess> weak_factory_;
712
713  base::MemoryMappedFile file_data_;
714  scoped_ptr<InMemoryUrlProtocol> protocol_;
715  scoped_ptr<FFmpegGlue> glue_;
716  AVFormatContext* av_format_context_;
717
718  int audio_stream_index_;
719  AudioParameters audio_params_;
720  double playback_rate_;
721
722  int video_stream_index_;
723  int video_frame_rate_numerator_;
724  int video_frame_rate_denominator_;
725
726  // These are used for audio resampling.
727  scoped_ptr<media::MultiChannelResampler> audio_resampler_;
728  scoped_ptr<media::AudioFifo> audio_fifo_;
729  scoped_ptr<media::AudioBus> audio_fifo_input_bus_;
730  media::AudioRendererAlgorithm audio_algo_;
731
732  // Track the timestamp of audio sent to the receiver.
733  scoped_ptr<media::AudioTimestampHelper> audio_sent_ts_;
734
735  std::queue<scoped_refptr<VideoFrame> > video_frame_queue_;
736  int64 video_first_pts_;
737  bool video_first_pts_set_;
738
739  std::queue<AudioBus*> audio_bus_queue_;
740
741  DISALLOW_COPY_AND_ASSIGN(SendProcess);
742};
743
744}  // namespace cast
745}  // namespace media
746
747namespace {
748void UpdateCastTransportStatus(
749    media::cast::transport::CastTransportStatus status) {
750  VLOG(1) << "Transport status: " << status;
751}
752
753void LogRawEvents(
754    const scoped_refptr<media::cast::CastEnvironment>& cast_environment,
755    const std::vector<media::cast::PacketEvent>& packet_events) {
756  VLOG(1) << "Got packet events from transport, size: " << packet_events.size();
757  for (std::vector<media::cast::PacketEvent>::const_iterator it =
758           packet_events.begin();
759       it != packet_events.end();
760       ++it) {
761    cast_environment->Logging()->InsertPacketEvent(it->timestamp,
762                                                   it->type,
763                                                   it->media_type,
764                                                   it->rtp_timestamp,
765                                                   it->frame_id,
766                                                   it->packet_id,
767                                                   it->max_packet_id,
768                                                   it->size);
769  }
770}
771
772void InitializationResult(media::cast::CastInitializationStatus result) {
773  bool end_result = result == media::cast::STATUS_AUDIO_INITIALIZED ||
774                    result == media::cast::STATUS_VIDEO_INITIALIZED;
775  CHECK(end_result) << "Cast sender uninitialized";
776}
777
778net::IPEndPoint CreateUDPAddress(std::string ip_str, int port) {
779  net::IPAddressNumber ip_number;
780  CHECK(net::ParseIPLiteralToNumber(ip_str, &ip_number));
781  return net::IPEndPoint(ip_number, port);
782}
783
784void DumpLoggingData(const media::cast::proto::LogMetadata& log_metadata,
785                     const media::cast::FrameEventList& frame_events,
786                     const media::cast::PacketEventList& packet_events,
787                     base::ScopedFILE log_file) {
788  VLOG(0) << "Frame map size: " << frame_events.size();
789  VLOG(0) << "Packet map size: " << packet_events.size();
790
791  scoped_ptr<char[]> event_log(new char[kMaxSerializedLogBytes]);
792  int event_log_bytes;
793  if (!media::cast::SerializeEvents(log_metadata,
794                                    frame_events,
795                                    packet_events,
796                                    true,
797                                    kMaxSerializedLogBytes,
798                                    event_log.get(),
799                                    &event_log_bytes)) {
800    VLOG(0) << "Failed to serialize events.";
801    return;
802  }
803
804  VLOG(0) << "Events serialized length: " << event_log_bytes;
805
806  int ret = fwrite(event_log.get(), 1, event_log_bytes, log_file.get());
807  if (ret != event_log_bytes)
808    VLOG(0) << "Failed to write logs to file.";
809}
810
811void WriteLogsToFileAndDestroySubscribers(
812    const scoped_refptr<media::cast::CastEnvironment>& cast_environment,
813    scoped_ptr<media::cast::EncodingEventSubscriber> video_event_subscriber,
814    scoped_ptr<media::cast::EncodingEventSubscriber> audio_event_subscriber,
815    base::ScopedFILE video_log_file,
816    base::ScopedFILE audio_log_file) {
817  cast_environment->Logging()->RemoveRawEventSubscriber(
818      video_event_subscriber.get());
819  cast_environment->Logging()->RemoveRawEventSubscriber(
820      audio_event_subscriber.get());
821
822  VLOG(0) << "Dumping logging data for video stream.";
823  media::cast::proto::LogMetadata log_metadata;
824  media::cast::FrameEventList frame_events;
825  media::cast::PacketEventList packet_events;
826  video_event_subscriber->GetEventsAndReset(
827      &log_metadata, &frame_events, &packet_events);
828
829  DumpLoggingData(log_metadata,
830                  frame_events,
831                  packet_events,
832                  video_log_file.Pass());
833
834  VLOG(0) << "Dumping logging data for audio stream.";
835  audio_event_subscriber->GetEventsAndReset(
836      &log_metadata, &frame_events, &packet_events);
837
838  DumpLoggingData(log_metadata,
839                  frame_events,
840                  packet_events,
841                  audio_log_file.Pass());
842}
843
844void WriteStatsAndDestroySubscribers(
845    const scoped_refptr<media::cast::CastEnvironment>& cast_environment,
846    scoped_ptr<media::cast::StatsEventSubscriber> video_event_subscriber,
847    scoped_ptr<media::cast::StatsEventSubscriber> audio_event_subscriber,
848    scoped_ptr<media::cast::ReceiverTimeOffsetEstimatorImpl> estimator) {
849  cast_environment->Logging()->RemoveRawEventSubscriber(
850      video_event_subscriber.get());
851  cast_environment->Logging()->RemoveRawEventSubscriber(
852      audio_event_subscriber.get());
853  cast_environment->Logging()->RemoveRawEventSubscriber(estimator.get());
854
855  scoped_ptr<base::DictionaryValue> stats = video_event_subscriber->GetStats();
856  std::string json;
857  base::JSONWriter::WriteWithOptions(
858      stats.get(), base::JSONWriter::OPTIONS_PRETTY_PRINT, &json);
859  VLOG(0) << "Video stats: " << json;
860
861  stats = audio_event_subscriber->GetStats();
862  json.clear();
863  base::JSONWriter::WriteWithOptions(
864      stats.get(), base::JSONWriter::OPTIONS_PRETTY_PRINT, &json);
865  VLOG(0) << "Audio stats: " << json;
866}
867
868}  // namespace
869
870int main(int argc, char** argv) {
871  base::AtExitManager at_exit;
872  CommandLine::Init(argc, argv);
873  InitLogging(logging::LoggingSettings());
874
875  // Load the media module for FFmpeg decoding.
876  base::FilePath path;
877  PathService::Get(base::DIR_MODULE, &path);
878  if (!media::InitializeMediaLibrary(path)) {
879    LOG(ERROR) << "Could not initialize media library.";
880    return 1;
881  }
882
883  base::Thread test_thread("Cast sender test app thread");
884  base::Thread audio_thread("Cast audio encoder thread");
885  base::Thread video_thread("Cast video encoder thread");
886  test_thread.Start();
887  audio_thread.Start();
888  video_thread.Start();
889
890  base::MessageLoopForIO io_message_loop;
891
892  // Default parameters.
893  CommandLine* cmd = CommandLine::ForCurrentProcess();
894  std::string remote_ip_address = cmd->GetSwitchValueASCII(kSwitchAddress);
895  if (remote_ip_address.empty())
896    remote_ip_address = "127.0.0.1";
897  int remote_port = 0;
898  if (!base::StringToInt(cmd->GetSwitchValueASCII(kSwitchPort),
899                         &remote_port)) {
900    remote_port = 2344;
901  }
902  LOG(INFO) << "Sending to " << remote_ip_address << ":" << remote_port
903            << ".";
904
905  media::cast::AudioSenderConfig audio_config =
906      media::cast::GetAudioSenderConfig();
907  media::cast::VideoSenderConfig video_config =
908      media::cast::GetVideoSenderConfig();
909
910  // Running transport on the main thread.
911  // Setting up transport config.
912  net::IPEndPoint remote_endpoint =
913      CreateUDPAddress(remote_ip_address, remote_port);
914
915  // Enable raw event and stats logging.
916  // Running transport on the main thread.
917  scoped_refptr<media::cast::CastEnvironment> cast_environment(
918      new media::cast::CastEnvironment(
919          make_scoped_ptr<base::TickClock>(new base::DefaultTickClock()),
920          io_message_loop.message_loop_proxy(),
921          audio_thread.message_loop_proxy(),
922          video_thread.message_loop_proxy()));
923
924  // SendProcess initialization.
925  scoped_ptr<media::cast::SendProcess> send_process(
926      new media::cast::SendProcess(test_thread.message_loop_proxy(),
927                                   cast_environment->Clock(),
928                                   video_config));
929
930  // CastTransportSender initialization.
931  scoped_ptr<media::cast::transport::CastTransportSender> transport_sender =
932      media::cast::transport::CastTransportSender::Create(
933          NULL,  // net log.
934          cast_environment->Clock(),
935          remote_endpoint,
936          base::Bind(&UpdateCastTransportStatus),
937          base::Bind(&LogRawEvents, cast_environment),
938          base::TimeDelta::FromSeconds(1),
939          io_message_loop.message_loop_proxy());
940
941  // CastSender initialization.
942  scoped_ptr<media::cast::CastSender> cast_sender =
943      media::cast::CastSender::Create(cast_environment, transport_sender.get());
944  cast_sender->InitializeVideo(
945      send_process->get_video_config(),
946      base::Bind(&InitializationResult),
947      media::cast::CreateDefaultVideoEncodeAcceleratorCallback(),
948      media::cast::CreateDefaultVideoEncodeMemoryCallback());
949  cast_sender->InitializeAudio(audio_config, base::Bind(&InitializationResult));
950  transport_sender->SetPacketReceiver(cast_sender->packet_receiver());
951
952  // Set up event subscribers.
953  scoped_ptr<media::cast::EncodingEventSubscriber> video_event_subscriber;
954  scoped_ptr<media::cast::EncodingEventSubscriber> audio_event_subscriber;
955  std::string video_log_file_name("/tmp/video_events.log.gz");
956  std::string audio_log_file_name("/tmp/audio_events.log.gz");
957  LOG(INFO) << "Logging audio events to: " << audio_log_file_name;
958  LOG(INFO) << "Logging video events to: " << video_log_file_name;
959  video_event_subscriber.reset(new media::cast::EncodingEventSubscriber(
960      media::cast::VIDEO_EVENT, 10000));
961  audio_event_subscriber.reset(new media::cast::EncodingEventSubscriber(
962      media::cast::AUDIO_EVENT, 10000));
963  cast_environment->Logging()->AddRawEventSubscriber(
964      video_event_subscriber.get());
965  cast_environment->Logging()->AddRawEventSubscriber(
966      audio_event_subscriber.get());
967
968  // Subscribers for stats.
969  scoped_ptr<media::cast::ReceiverTimeOffsetEstimatorImpl> offset_estimator(
970      new media::cast::ReceiverTimeOffsetEstimatorImpl);
971  cast_environment->Logging()->AddRawEventSubscriber(offset_estimator.get());
972  scoped_ptr<media::cast::StatsEventSubscriber> video_stats_subscriber(
973      new media::cast::StatsEventSubscriber(media::cast::VIDEO_EVENT,
974                                            cast_environment->Clock(),
975                                            offset_estimator.get()));
976  scoped_ptr<media::cast::StatsEventSubscriber> audio_stats_subscriber(
977      new media::cast::StatsEventSubscriber(media::cast::AUDIO_EVENT,
978                                            cast_environment->Clock(),
979                                            offset_estimator.get()));
980  cast_environment->Logging()->AddRawEventSubscriber(
981      video_stats_subscriber.get());
982  cast_environment->Logging()->AddRawEventSubscriber(
983      audio_stats_subscriber.get());
984
985  base::ScopedFILE video_log_file(fopen(video_log_file_name.c_str(), "w"));
986  if (!video_log_file) {
987    VLOG(1) << "Failed to open video log file for writing.";
988    exit(-1);
989  }
990
991  base::ScopedFILE audio_log_file(fopen(audio_log_file_name.c_str(), "w"));
992  if (!audio_log_file) {
993    VLOG(1) << "Failed to open audio log file for writing.";
994    exit(-1);
995  }
996
997  const int logging_duration_seconds = 10;
998  io_message_loop.message_loop_proxy()->PostDelayedTask(
999      FROM_HERE,
1000      base::Bind(&WriteLogsToFileAndDestroySubscribers,
1001                 cast_environment,
1002                 base::Passed(&video_event_subscriber),
1003                 base::Passed(&audio_event_subscriber),
1004                 base::Passed(&video_log_file),
1005                 base::Passed(&audio_log_file)),
1006      base::TimeDelta::FromSeconds(logging_duration_seconds));
1007
1008  io_message_loop.message_loop_proxy()->PostDelayedTask(
1009      FROM_HERE,
1010      base::Bind(&WriteStatsAndDestroySubscribers,
1011                 cast_environment,
1012                 base::Passed(&video_stats_subscriber),
1013                 base::Passed(&audio_stats_subscriber),
1014                 base::Passed(&offset_estimator)),
1015      base::TimeDelta::FromSeconds(logging_duration_seconds));
1016
1017  send_process->Start(cast_sender->audio_frame_input(),
1018                      cast_sender->video_frame_input());
1019
1020  io_message_loop.Run();
1021  return 0;
1022}
1023