xref: /frameworks/base/tests/DumpRenderTree/assets/run_reliability_tests.py

#!/usr/bin/python2.4

"""Run reliability tests using Android instrumentation.

  A test file consists of list web sites to test is needed as a parameter

  Usage:
    run_reliability_tests.py path/to/url/list
"""

import logging
import optparse
import os
import subprocess
import sys
import time
from Numeric import *

TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt"
TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt"
TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt"
TEST_LOAD_TIME_FILE = "/sdcard/android/reliability_load_time.txt"
HTTP_URL_FILE = "urllist_http"
HTTPS_URL_FILE = "urllist_https"
NUM_URLS = 25


def DumpRenderTreeFinished(adb_cmd):
  """Check if DumpRenderTree finished running.

  Args:
    adb_cmd: adb command string

  Returns:
    True if DumpRenderTree has finished, False otherwise
  """

  # pull test status file and look for "#DONE"
  shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
  adb_output = subprocess.Popen(shell_cmd_str,
                                shell=True, stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE).communicate()[0]
  return adb_output.strip() == "#DONE"


def RemoveDeviceFile(adb_cmd, file_name):
  shell_cmd_str = adb_cmd + " shell rm " + file_name
  subprocess.Popen(shell_cmd_str,
                   shell=True, stdout=subprocess.PIPE,
                   stderr=subprocess.PIPE).communicate()


def Bugreport(url, bugreport_dir, adb_cmd):
  """Pull a bugreport from the device."""
  bugreport_filename = "%s/reliability_bugreport_%d.txt" % (bugreport_dir,
                                                            int(time.time()))

  # prepend the report with url
  handle = open(bugreport_filename, "w")
  handle.writelines("Bugreport for crash in url - %s\n\n" % url)
  handle.close()

  cmd = "%s bugreport >> %s" % (adb_cmd, bugreport_filename)
  os.system(cmd)


def ProcessPageLoadTime(raw_log):
  """Processes the raw page load time logged by test app."""
  log_handle = open(raw_log, "r")
  load_times = {}

  for line in log_handle:
    line = line.strip()
    pair = line.split("|")
    if len(pair) != 2:
      logging.info("Line has more than one '|': " + line)
      continue
    if pair[0] not in load_times:
      load_times[pair[0]] = []
    try:
      pair[1] = int(pair[1])
    except ValueError:
      logging.info("Lins has non-numeric load time: " + line)
      continue
    load_times[pair[0]].append(pair[1])

  log_handle.close()

  # rewrite the average time to file
  log_handle = open(raw_log, "w")
  for url, times in load_times.iteritems():
    # calculate std
    arr = array(times)
    avg = average(arr)
    d = arr - avg
    std = sqrt(sum(d * d) / len(arr))
    output = ("%-70s%-10d%-10d%-12.2f%-12.2f%s\n" %
              (url, min(arr), max(arr), avg, std,
               array2string(arr)))
    log_handle.write(output)
  log_handle.close()


def main(options, args):
  """Send the url list to device and start testing, restart if crashed."""

  # Set up logging format.
  log_level = logging.INFO
  if options.verbose:
    log_level = logging.DEBUG
  logging.basicConfig(level=log_level,
                      format="%(message)s")

  # Include all tests if none are specified.
  if not args:
    print "Missing URL list file"
    sys.exit(1)
  else:
    path = args[0]

  if not options.crash_file:
    print "Missing crash file name, use --crash-file to specify"
    sys.exit(1)
  else:
    crashed_file = options.crash_file

  if not options.timeout_file:
    print "Missing timeout file, use --timeout-file to specify"
    sys.exit(1)
  else:
    timedout_file = options.timeout_file

  if not options.delay:
    manual_delay = 0
  else:
    manual_delay = options.delay

  if not options.bugreport:
    bugreport_dir = "."
  else:
    bugreport_dir = options.bugreport
  if not os.path.exists(bugreport_dir):
    os.makedirs(bugreport_dir)
  if not os.path.isdir(bugreport_dir):
    logging.error("Cannot create results dir: " + bugreport_dir)
    sys.exit(1)

  adb_cmd = "adb "
  if options.adb_options:
    adb_cmd += options.adb_options + " "

  # push url list to device
  test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\""
  proc = subprocess.Popen(test_cmd, shell=True,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE)
  (adb_output, adb_error) = proc.communicate()
  if proc.returncode != 0:
    logging.error("failed to push url list to device.")
    logging.error(adb_output)
    logging.error(adb_error)
    sys.exit(1)

  # clean up previous results
  RemoveDeviceFile(adb_cmd, TEST_STATUS_FILE)
  RemoveDeviceFile(adb_cmd, TEST_TIMEOUT_FILE)
  RemoveDeviceFile(adb_cmd, TEST_LOAD_TIME_FILE)

  logging.info("Running the test ...")

  # Count crashed tests.
  crashed_tests = []

  if options.time_out_ms:
    timeout_ms = options.time_out_ms

  # Run test until it's done
  test_cmd_prefix = adb_cmd + " shell am instrument"
  test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner"

  # Call ReliabilityTestsAutoTest#startReliabilityTests
  test_cmd = (test_cmd_prefix + " -e class "
              "com.android.dumprendertree.ReliabilityTest#"
              "runReliabilityTest -e timeout %s -e delay %s" %
              (str(timeout_ms), str(manual_delay)))

  if options.logtime:
    test_cmd += " -e logtime true"

  test_cmd += test_cmd_postfix

  adb_output = subprocess.Popen(test_cmd, shell=True,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE).communicate()[0]
  while not DumpRenderTreeFinished(adb_cmd):
    logging.error("DumpRenderTree exited before all URLs are visited.")
    shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
    crashed_test = ""
    while not crashed_test:
      (crashed_test, err) = subprocess.Popen(
          shell_cmd_str, shell=True, stdout=subprocess.PIPE,
          stderr=subprocess.PIPE).communicate()
      crashed_test = crashed_test.strip()
      if not crashed_test:
        logging.error('Cannot get crashed test name, device offline?')
        logging.error('stderr: ' + err)
        logging.error('retrying in 10s...')
        time.sleep(10)

    logging.info(crashed_test + " CRASHED")
    crashed_tests.append(crashed_test)
    Bugreport(crashed_test, bugreport_dir, adb_cmd)
    logging.info("Resuming reliability test runner...")

    adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE).communicate()[0]

  if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or
      adb_output.find("Process crashed.") != -1):
    logging.error("Error happened : " + adb_output)
    sys.exit(1)

  logging.info(adb_output)
  logging.info("Done\n")

  if crashed_tests:
    file_handle = open(crashed_file, "w")
    file_handle.writelines("\n".join(crashed_tests))
    logging.info("Crashed URL list stored in: " + crashed_file)
    file_handle.close()
  else:
    logging.info("No crash found.")

  # get timeout file from sdcard
  test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \""
              + timedout_file +  "\"")
  subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
                   stderr=subprocess.PIPE).communicate()

  if options.logtime:
    # get logged page load times from sdcard
    test_cmd = (adb_cmd + "pull \"" + TEST_LOAD_TIME_FILE + "\" \""
                + options.logtime +  "\"")
    subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
                     stderr=subprocess.PIPE).communicate()
    ProcessPageLoadTime(options.logtime)


if "__main__" == __name__:
  option_parser = optparse.OptionParser()
  option_parser.add_option("-t", "--time-out-ms",
                           default=60000,
                           help="set the timeout for each test")
  option_parser.add_option("-v", "--verbose", action="store_true",
                           default=False,
                           help="include debug-level logging")
  option_parser.add_option("-a", "--adb-options",
                           default=None,
                           help="pass options to adb, such as -d -e, etc")
  option_parser.add_option("-c", "--crash-file",
                           default="reliability_crashed_sites.txt",
                           help="the list of sites that cause browser to crash")
  option_parser.add_option("-f", "--timeout-file",
                           default="reliability_timedout_sites.txt",
                           help="the list of sites that timedout during test")
  option_parser.add_option("-d", "--delay",
                           default=0,
                           help="add a manual delay between pages (in ms)")
  option_parser.add_option("-b", "--bugreport",
                           default=".",
                           help="the directory to store bugreport for crashes")
  option_parser.add_option("-l", "--logtime",
                           default=None,
                           help="Logs page load time for each url to the file")
  opts, arguments = option_parser.parse_args()
  main(opts, arguments)