browser/media/chrome_webrtc_video_quality_browsertest.cc

// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/environment.h"
#include "base/file_util.h"
#include "base/path_service.h"
#include "base/process_util.h"
#include "base/strings/string_split.h"
#include "base/strings/stringprintf.h"
#include "base/test/test_timeouts.h"
#include "base/time/time.h"
#include "chrome/browser/chrome_notification_types.h"
#include "chrome/browser/infobars/infobar.h"
#include "chrome/browser/infobars/infobar_service.h"
#include "chrome/browser/media/media_stream_infobar_delegate.h"
#include "chrome/browser/media/webrtc_browsertest_common.h"
#include "chrome/browser/profiles/profile.h"
#include "chrome/browser/ui/browser.h"
#include "chrome/browser/ui/browser_tabstrip.h"
#include "chrome/browser/ui/tabs/tab_strip_model.h"
#include "chrome/common/chrome_switches.h"
#include "chrome/test/base/in_process_browser_test.h"
#include "chrome/test/base/ui_test_utils.h"
#include "chrome/test/perf/perf_test.h"
#include "chrome/test/ui/ui_test.h"
#include "content/public/browser/notification_service.h"
#include "content/public/test/browser_test_utils.h"
#include "net/test/python_utils.h"
#include "net/test/spawned_test_server/spawned_test_server.h"

static const base::FilePath::CharType kPeerConnectionServer[] =
#if defined(OS_WIN)
    FILE_PATH_LITERAL("peerconnection_server.exe");
#else
    FILE_PATH_LITERAL("peerconnection_server");
#endif

static const base::FilePath::CharType kFrameAnalyzerExecutable[] =
#if defined(OS_WIN)
    FILE_PATH_LITERAL("frame_analyzer.exe");
#else
    FILE_PATH_LITERAL("frame_analyzer");
#endif

static const base::FilePath::CharType kArgbToI420ConverterExecutable[] =
#if defined(OS_WIN)
    FILE_PATH_LITERAL("rgba_to_i420_converter.exe");
#else
    FILE_PATH_LITERAL("rgba_to_i420_converter");
#endif

static const char kHomeEnvName[] =
#if defined(OS_WIN)
    "HOMEPATH";
#else
    "HOME";
#endif

// The working dir should be in the user's home folder.
static const base::FilePath::CharType kWorkingDirName[] =
    FILE_PATH_LITERAL("webrtc_video_quality");
static const base::FilePath::CharType kReferenceYuvFileName[] =
    FILE_PATH_LITERAL("reference_video.yuv");
static const base::FilePath::CharType kCapturedYuvFileName[] =
    FILE_PATH_LITERAL("captured_video.yuv");
static const base::FilePath::CharType kStatsFileName[] =
    FILE_PATH_LITERAL("stats.txt");
static const char kMainWebrtcTestHtmlPage[] =
    "files/webrtc/webrtc_jsep01_test.html";
static const char kCapturingWebrtcHtmlPage[] =
    "files/webrtc/webrtc_video_quality_test.html";
static const int kVgaWidth = 640;
static const int kVgaHeight = 480;

// If you change the port number, don't forget to modify video_extraction.js
// too!
static const char kPyWebSocketPortNumber[] = "12221";

// Test the video quality of the WebRTC output.
//
// Prerequisites: This test case must run on a machine with a virtual webcam
// that plays video from the reference file located in <the running users home
// folder>/kWorkingDirName/kReferenceYuvFileName.
//
// You must also compile the chromium_builder_webrtc target before you run this
// test to get all the tools built.
//
// The external compare_videos.py script also depends on two external
// executables which must be located in the PATH when running this test.
// * zxing (see the CPP version at https://code.google.com/p/zxing)
// * ffmpeg 0.11.1 or compatible version (see http://www.ffmpeg.org)
//
// The test case will launch a custom binary (peerconnection_server) which will
// allow two WebRTC clients to find each other.
//
// The test also runs several other custom binaries - rgba_to_i420 converter and
// frame_analyzer. Both tools can be found under third_party/webrtc/tools. The
// test also runs a stand alone Python implementation of a WebSocket server
// (pywebsocket) and a barcode_decoder script.
class WebrtcVideoQualityBrowserTest : public InProcessBrowserTest {
 public:
  WebrtcVideoQualityBrowserTest()
      : peerconnection_server_(0),
        pywebsocket_server_(0),
        environment_(base::Environment::Create()) {}

  virtual void SetUp() OVERRIDE {
    RunPeerConnectionServer();
    InProcessBrowserTest::SetUp();

    // Ensure we have the stuff we need.
    EXPECT_TRUE(base::PathExists(GetWorkingDir()))
        << "Cannot find the working directory for the reference video and "
           "the temporary files:" << GetWorkingDir().value();
    base::FilePath reference_file =
        GetWorkingDir().Append(kReferenceYuvFileName);
    EXPECT_TRUE(base::PathExists(reference_file))
        << "Cannot find the reference file to be used for video quality "
        << "comparison: " << reference_file.value();
  }

  virtual void TearDown() OVERRIDE {
    ShutdownPeerConnectionServer();
    InProcessBrowserTest::TearDown();
    ShutdownPyWebSocketServer();
  }

  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
    // TODO(phoglund): check that user actually has the requisite devices and
    // print a nice message if not; otherwise the test just times out which can
    // be confusing.
    // This test expects real device handling and requires a real webcam / audio
    // device; it will not work with fake devices.
    EXPECT_FALSE(
        command_line->HasSwitch(switches::kUseFakeDeviceForMediaStream))
        << "You cannot run this test with fake devices.";
  }

  void StartPyWebSocketServer() {
    base::FilePath path_pywebsocket_dir =
        GetSourceDir().Append(FILE_PATH_LITERAL("third_party/pywebsocket/src"));
    base::FilePath pywebsocket_server = path_pywebsocket_dir.Append(
        FILE_PATH_LITERAL("mod_pywebsocket/standalone.py"));
    base::FilePath path_to_data_handler =
        GetSourceDir().Append(FILE_PATH_LITERAL("chrome/test/functional"));

    EXPECT_TRUE(base::PathExists(pywebsocket_server))
        << "Fatal: missing pywebsocket server.";
    EXPECT_TRUE(base::PathExists(path_to_data_handler))
        << "Fatal: missing data handler for pywebsocket server.";

    AppendToPythonPath(path_pywebsocket_dir);
    CommandLine pywebsocket_command = MakePythonCommand(pywebsocket_server);

    // Construct the command line manually, the server doesn't support -arg=val.
    pywebsocket_command.AppendArg("-p");
    pywebsocket_command.AppendArg(kPyWebSocketPortNumber);
    pywebsocket_command.AppendArg("-d");
    pywebsocket_command.AppendArgPath(path_to_data_handler);

    LOG(INFO) << "Running " << pywebsocket_command.GetCommandLineString();
    EXPECT_TRUE(base::LaunchProcess(
        pywebsocket_command, base::LaunchOptions(), &pywebsocket_server_))
        << "Failed to launch pywebsocket server.";
  }

  void ShutdownPyWebSocketServer() {
    EXPECT_TRUE(base::KillProcess(pywebsocket_server_, 0, false))
        << "Failed to shut down pywebsocket server!";
  }

  // Convenience method which executes the provided javascript in the context
  // of the provided web contents and returns what it evaluated to.
  std::string ExecuteJavascript(const std::string& javascript,
                                content::WebContents* tab_contents) {
    std::string result;
    EXPECT_TRUE(content::ExecuteScriptAndExtractString(
        tab_contents, javascript, &result));
    return result;
  }

  void GetUserMedia(content::WebContents* tab_contents) {
    content::WindowedNotificationObserver infobar_added(
        chrome::NOTIFICATION_TAB_CONTENTS_INFOBAR_ADDED,
        content::NotificationService::AllSources());

    // Request user media: this will launch the media stream info bar.
    EXPECT_EQ("ok-requested",
              ExecuteJavascript("getUserMedia('{video: true, audio: true}');",
                                tab_contents));

    // Wait for the bar to pop up, then accept.
    infobar_added.Wait();
    content::Details<InfoBarAddedDetails> details(infobar_added.details());
    MediaStreamInfoBarDelegate* media_infobar =
        details.ptr()->AsMediaStreamInfoBarDelegate();
    media_infobar->Accept();

    // Wait for WebRTC to call the success callback.
    EXPECT_TRUE(PollingWaitUntil(
        "obtainGetUserMediaResult();", "ok-got-stream", tab_contents));
  }

  // Ensures we didn't get any errors asynchronously (e.g. while no javascript
  // call from this test was outstanding).
  // TODO(phoglund): this becomes obsolete when we switch to communicating with
  // the DOM message queue.
  void AssertNoAsynchronousErrors(content::WebContents* tab_contents) {
    EXPECT_EQ("ok-no-errors",
              ExecuteJavascript("getAnyTestFailures()", tab_contents));
  }

  // The peer connection server lets our two tabs find each other and talk to
  // each other (e.g. it is the application-specific "signaling solution").
  void ConnectToPeerConnectionServer(const std::string peer_name,
                                     content::WebContents* tab_contents) {
    std::string javascript = base::StringPrintf(
        "connect('http://localhost:8888', '%s');", peer_name.c_str());
    EXPECT_EQ("ok-connected", ExecuteJavascript(javascript, tab_contents));
  }

  void EstablishCall(content::WebContents* from_tab,
                     content::WebContents* to_tab) {
    EXPECT_EQ("ok-peerconnection-created",
              ExecuteJavascript("preparePeerConnection()", from_tab));
    EXPECT_EQ("ok-added", ExecuteJavascript("addLocalStream()", from_tab));
    EXPECT_EQ("ok-negotiating", ExecuteJavascript("negotiateCall()", from_tab));

    // Ensure the call gets up on both sides.
    EXPECT_TRUE(PollingWaitUntil(
        "getPeerConnectionReadyState()", "active", from_tab));
    EXPECT_TRUE(PollingWaitUntil(
        "getPeerConnectionReadyState()", "active", to_tab));
  }

  void HangUp(content::WebContents* from_tab) {
    EXPECT_EQ("ok-call-hung-up", ExecuteJavascript("hangUp()", from_tab));
  }

  void WaitUntilHangupVerified(content::WebContents* tab_contents) {
    EXPECT_TRUE(PollingWaitUntil(
        "getPeerConnectionReadyState()", "no-peer-connection", tab_contents));
  }

  // Runs the RGBA to I420 converter on the video in |capture_video_filename|,
  // which should contain frames of size |width| x |height|.
  //
  // The rgba_to_i420_converter is part of the webrtc_test_tools target which
  // should be build prior to running this test. The resulting binary should
  // live next to Chrome.
  void RunARGBtoI420Converter(int width,
                              int height,
                              const base::FilePath& captured_video_filename) {
    base::FilePath path_to_converter = base::MakeAbsoluteFilePath(
        GetBrowserDir().Append(kArgbToI420ConverterExecutable));
    EXPECT_TRUE(base::PathExists(path_to_converter))
        << "Missing ARGB->I420 converter: should be in "
        << path_to_converter.value();

    CommandLine converter_command(path_to_converter);
    converter_command.AppendSwitchPath("--frames_dir", GetWorkingDir());
    converter_command.AppendSwitchPath("--output_file",
                                       captured_video_filename);
    converter_command.AppendSwitchASCII("--width",
                                        base::StringPrintf("%d", width));
    converter_command.AppendSwitchASCII("--height",
                                        base::StringPrintf("%d", height));

    // We produce an output file that will later be used as an input to the
    // barcode decoder and frame analyzer tools.
    LOG(INFO) << "Running " << converter_command.GetCommandLineString();
    std::string result;
    EXPECT_TRUE(base::GetAppOutput(converter_command, &result));
    LOG(INFO) << "Output was:\n\n" << result;
  }

  // Compares the |captured_video_filename| with the |reference_video_filename|.
  //
  // The barcode decoder decodes the captured video containing barcodes overlaid
  // into every frame of the video (produced by rgba_to_i420_converter). It
  // produces a set of PNG images and a |stats_file| that maps each captured
  // frame to a frame in the reference video. The frames should be of size
  // |width| x |height|. The output of compare_videos.py is returned.
  std::string CompareVideos(int width,
                            int height,
                            const base::FilePath& captured_video_filename,
                            const base::FilePath& reference_video_filename,
                            const base::FilePath& stats_file) {
    base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath(
        GetBrowserDir().Append(kFrameAnalyzerExecutable));
    base::FilePath path_to_compare_script = GetSourceDir().Append(
        FILE_PATH_LITERAL("third_party/webrtc/tools/compare_videos.py"));

    EXPECT_TRUE(base::PathExists(path_to_analyzer))
        << "Missing frame analyzer: should be in " << path_to_analyzer.value();
    EXPECT_TRUE(base::PathExists(path_to_compare_script))
        << "Missing video compare script: should be in "
        << path_to_compare_script.value();

    CommandLine compare_command = MakePythonCommand(path_to_compare_script);
    compare_command.AppendSwitchPath("--ref_video", reference_video_filename);
    compare_command.AppendSwitchPath("--test_video", captured_video_filename);
    compare_command.AppendSwitchPath("--frame_analyzer", path_to_analyzer);
    compare_command.AppendSwitchASCII("--yuv_frame_width",
                                      base::StringPrintf("%d", width));
    compare_command.AppendSwitchASCII("--yuv_frame_height",
                                      base::StringPrintf("%d", height));
    compare_command.AppendSwitchPath("--stats_file", stats_file);

    LOG(INFO) << "Running " << compare_command.GetCommandLineString();
    std::string result;
    EXPECT_TRUE(base::GetAppOutput(compare_command, &result));
    LOG(INFO) << "Output was:\n\n" << result;
    return result;
  }

  // Processes the |frame_analyzer_output| for the different frame counts.
  //
  // The frame analyzer outputs additional information about the number of
  // unique frames captured, The max number of repeated frames in a sequence and
  // the max number of skipped frames. These values are then written to the Perf
  // Graph. (Note: Some of the repeated or skipped frames will probably be due
  // to the imperfection of JavaScript timers).
  void PrintFramesCountPerfResults(std::string frame_analyzer_output) {
    size_t unique_frames_pos =
        frame_analyzer_output.rfind("Unique_frames_count");
    EXPECT_NE(unique_frames_pos, std::string::npos)
        << "Missing Unique_frames_count in frame analyzer output:\n"
        << frame_analyzer_output;

    std::string unique_frame_counts =
        frame_analyzer_output.substr(unique_frames_pos);
    // TODO(phoglund): Fix ESTATS result to not have this silly newline.
    std::replace(
        unique_frame_counts.begin(), unique_frame_counts.end(), '\n', ' ');

    std::vector<std::pair<std::string, std::string> > key_values;
    base::SplitStringIntoKeyValuePairs(
        unique_frame_counts, ':', ' ', &key_values);
    std::vector<std::pair<std::string, std::string> >::const_iterator iter;
    for (iter = key_values.begin(); iter != key_values.end(); ++iter) {
      const std::pair<std::string, std::string>& key_value = *iter;
      perf_test::PrintResult(
          key_value.first, "", "VGA", key_value.second, "", false);
    }
  }

  // Processes the |frame_analyzer_output| to extract the PSNR and SSIM values.
  //
  // The frame analyzer produces PSNR and SSIM results for every unique frame
  // that has been captured. This method forms a list of all the psnr and ssim
  // values and passes it to PrintResultList() for printing on the Perf Graph.
  void PrintPsnrAndSsimPerfResults(std::string frame_analyzer_output) {
    size_t stats_start = frame_analyzer_output.find("BSTATS");
    EXPECT_NE(stats_start, std::string::npos)
        << "Missing BSTATS in frame analyzer output:\n"
        << frame_analyzer_output;
    size_t stats_end = frame_analyzer_output.find("ESTATS");
    EXPECT_NE(stats_end, std::string::npos)
        << "Missing ESTATS in frame analyzer output:\n"
        << frame_analyzer_output;

    stats_start += std::string("BSTATS").size();
    std::string psnr_ssim_stats =
        frame_analyzer_output.substr(stats_start, stats_end - stats_start);

    // PSNR and SSIM values aren't really key-value pairs but it is convenient
    // to parse them as such.
    // TODO(phoglund): make the format more convenient so we need less
    // processing here.
    std::vector<std::pair<std::string, std::string> > psnr_ssim_entries;
    base::SplitStringIntoKeyValuePairs(
        psnr_ssim_stats, ' ', ';', &psnr_ssim_entries);

    std::string psnr_value_list;
    std::string ssim_value_list;
    std::vector<std::pair<std::string, std::string> >::const_iterator iter;
    for (iter = psnr_ssim_entries.begin(); iter != psnr_ssim_entries.end();
         ++iter) {
      const std::pair<std::string, std::string>& psnr_and_ssim = *iter;
      psnr_value_list.append(psnr_and_ssim.first).append(",");
      ssim_value_list.append(psnr_and_ssim.second).append(",");
    }
    // Nuke last comma.
    psnr_value_list.erase(psnr_value_list.size() - 1);
    ssim_value_list.erase(ssim_value_list.size() - 1);

    perf_test::PrintResultList("PSNR", "", "VGA", psnr_value_list, "dB", false);
    perf_test::PrintResultList("SSIM", "", "VGA", ssim_value_list, "", false);
  }

  base::FilePath GetWorkingDir() {
    std::string home_dir;
    environment_->GetVar(kHomeEnvName, &home_dir);
    base::FilePath::StringType native_home_dir(home_dir.begin(),
                                               home_dir.end());
    return base::FilePath(native_home_dir).Append(kWorkingDirName);
  }

 private:
  void RunPeerConnectionServer() {
    // TODO(phoglund): de-dupe later: next line differs from original.
    base::FilePath peerconnection_server =
        GetBrowserDir().Append(kPeerConnectionServer);

    EXPECT_TRUE(base::PathExists(peerconnection_server))
        << "Missing peerconnection_server. You must build "
           "it so it ends up next to the browser test binary.";
    EXPECT_TRUE(base::LaunchProcess(CommandLine(peerconnection_server),
                                    base::LaunchOptions(),
                                    &peerconnection_server_))
        << "Failed to launch peerconnection_server.";
  }

  void ShutdownPeerConnectionServer() {
    EXPECT_TRUE(base::KillProcess(peerconnection_server_, 0, false))
        << "Failed to shut down peerconnection_server!";
  }

  base::FilePath GetSourceDir() {
    base::FilePath source_dir;
    PathService::Get(base::DIR_SOURCE_ROOT, &source_dir);
    return source_dir;
  }

  base::FilePath GetBrowserDir() {
    base::FilePath browser_dir;
    EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &browser_dir));
    return browser_dir;
  }

  CommandLine MakePythonCommand(base::FilePath python_script) {
    CommandLine python_command(CommandLine::NO_PROGRAM);
    EXPECT_TRUE(GetPythonCommand(&python_command));
    CommandLine complete_command(python_script);
    complete_command.PrependWrapper(python_command.GetCommandLineString());
    return complete_command;
  }

  base::ProcessHandle peerconnection_server_;
  base::ProcessHandle pywebsocket_server_;
  scoped_ptr<base::Environment> environment_;
};

// Broken on Win: failing to start pywebsocket_server. http://crbug.com/255499.
#define MAYBE_MANUAL_TestVGAVideoQuality DISABLED_MANUAL_TestVGAVideoQuality

IN_PROC_BROWSER_TEST_F(WebrtcVideoQualityBrowserTest,
                       MAYBE_MANUAL_TestVGAVideoQuality) {
  // TODO(phoglund): de-dupe from chrome_webrtc_browsertest.cc.
  StartPyWebSocketServer();

  EXPECT_TRUE(test_server()->Start());

  ui_test_utils::NavigateToURL(browser(),
                               test_server()->GetURL(kMainWebrtcTestHtmlPage));
  content::WebContents* left_tab =
      browser()->tab_strip_model()->GetActiveWebContents();
  GetUserMedia(left_tab);

  chrome::AddBlankTabAt(browser(), -1, true);
  content::WebContents* right_tab =
      browser()->tab_strip_model()->GetActiveWebContents();
  // TODO(phoglund): (de-dupe later) different from original flow.
  ui_test_utils::NavigateToURL(browser(),
                               test_server()->GetURL(kCapturingWebrtcHtmlPage));
  GetUserMedia(right_tab);

  ConnectToPeerConnectionServer("peer 1", left_tab);
  ConnectToPeerConnectionServer("peer 2", right_tab);

  EstablishCall(left_tab, right_tab);

  AssertNoAsynchronousErrors(left_tab);
  AssertNoAsynchronousErrors(right_tab);

  // Poll slower here to avoid flooding the log with messages: capturing and
  // sending frames take quite a bit of time.
  int polling_interval_msec = 1000;

  // TODO(phoglund): (de-dupe later) different from original flow.
  EXPECT_TRUE(PollingWaitUntil(
      "doneFrameCapturing()", "done-capturing", right_tab,
      polling_interval_msec));

  HangUp(left_tab);
  WaitUntilHangupVerified(left_tab);
  WaitUntilHangupVerified(right_tab);

  AssertNoAsynchronousErrors(left_tab);
  AssertNoAsynchronousErrors(right_tab);

  // TODO(phoglund): (de-dupe later) different from original flow.
  EXPECT_TRUE(PollingWaitUntil(
      "haveMoreFramesToSend()", "no-more-frames", right_tab,
      polling_interval_msec));

  RunARGBtoI420Converter(
      kVgaWidth, kVgaHeight, GetWorkingDir().Append(kCapturedYuvFileName));
  std::string output =
      CompareVideos(kVgaWidth,
                    kVgaHeight,
                    GetWorkingDir().Append(kCapturedYuvFileName),
                    GetWorkingDir().Append(kReferenceYuvFileName),
                    GetWorkingDir().Append(kStatsFileName));

  PrintFramesCountPerfResults(output);
  PrintPsnrAndSsimPerfResults(output);
}