LayoutTestsAutoTest.java revision 9b815d080145f0bc8effc9e011090010ad51f203
1/* 2 * Copyright (C) 2008 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.dumprendertree; 18 19import com.android.dumprendertree.TestShellActivity.DumpDataType; 20import com.android.dumprendertree.forwarder.AdbUtils; 21import com.android.dumprendertree.forwarder.ForwardService; 22 23import android.content.Intent; 24import android.test.ActivityInstrumentationTestCase2; 25import android.util.Log; 26 27import java.io.BufferedOutputStream; 28import java.io.BufferedReader; 29import java.io.File; 30import java.io.FileNotFoundException; 31import java.io.FileOutputStream; 32import java.io.FileReader; 33import java.io.IOException; 34import java.io.InputStream; 35import java.io.OutputStream; 36import java.util.Vector; 37 38// TestRecorder creates four files ... 39// - passing tests 40// - failing tests 41// - tests for which results are ignored 42// - tests with no text results available 43// TestRecorder does not have the ability to clear the results. 44class MyTestRecorder { 45 private BufferedOutputStream mBufferedOutputPassedStream; 46 private BufferedOutputStream mBufferedOutputFailedStream; 47 private BufferedOutputStream mBufferedOutputIgnoreResultStream; 48 private BufferedOutputStream mBufferedOutputNoResultStream; 49 50 public void passed(String layout_file) { 51 try { 52 mBufferedOutputPassedStream.write(layout_file.getBytes()); 53 mBufferedOutputPassedStream.write('\n'); 54 mBufferedOutputPassedStream.flush(); 55 } catch(Exception e) { 56 e.printStackTrace(); 57 } 58 } 59 60 public void failed(String layout_file) { 61 try { 62 mBufferedOutputFailedStream.write(layout_file.getBytes()); 63 mBufferedOutputFailedStream.write('\n'); 64 mBufferedOutputFailedStream.flush(); 65 } catch(Exception e) { 66 e.printStackTrace(); 67 } 68 } 69 70 public void ignoreResult(String layout_file) { 71 try { 72 mBufferedOutputIgnoreResultStream.write(layout_file.getBytes()); 73 mBufferedOutputIgnoreResultStream.write('\n'); 74 mBufferedOutputIgnoreResultStream.flush(); 75 } catch(Exception e) { 76 e.printStackTrace(); 77 } 78 } 79 80 public void noResult(String layout_file) { 81 try { 82 mBufferedOutputNoResultStream.write(layout_file.getBytes()); 83 mBufferedOutputNoResultStream.write('\n'); 84 mBufferedOutputNoResultStream.flush(); 85 } catch(Exception e) { 86 e.printStackTrace(); 87 } 88 } 89 90 public MyTestRecorder(boolean resume) { 91 try { 92 File resultsPassedFile = new File("/sdcard/layout_tests_passed.txt"); 93 File resultsFailedFile = new File("/sdcard/layout_tests_failed.txt"); 94 File resultsIgnoreResultFile = new File("/sdcard/layout_tests_ignored.txt"); 95 File noExpectedResultFile = new File("/sdcard/layout_tests_nontext.txt"); 96 97 mBufferedOutputPassedStream = 98 new BufferedOutputStream(new FileOutputStream(resultsPassedFile, resume)); 99 mBufferedOutputFailedStream = 100 new BufferedOutputStream(new FileOutputStream(resultsFailedFile, resume)); 101 mBufferedOutputIgnoreResultStream = 102 new BufferedOutputStream(new FileOutputStream(resultsIgnoreResultFile, resume)); 103 mBufferedOutputNoResultStream = 104 new BufferedOutputStream(new FileOutputStream(noExpectedResultFile, resume)); 105 } catch (Exception e) { 106 e.printStackTrace(); 107 } 108 } 109 110 public void close() { 111 try { 112 mBufferedOutputPassedStream.close(); 113 mBufferedOutputFailedStream.close(); 114 mBufferedOutputIgnoreResultStream.close(); 115 mBufferedOutputNoResultStream.close(); 116 } catch (Exception e) { 117 e.printStackTrace(); 118 } 119 } 120} 121 122 123public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestShellActivity> { 124 125 private static final String LOGTAG = "LayoutTests"; 126 static final int DEFAULT_TIMEOUT_IN_MILLIS = 5000; 127 128 static final String LAYOUT_TESTS_ROOT = "/sdcard/android/layout_tests/"; 129 static final String LAYOUT_TESTS_RESULT_DIR = "/sdcard/android/layout_tests_results/"; 130 static final String ANDROID_EXPECTED_RESULT_DIR = "/sdcard/android/expected_results/"; 131 static final String LAYOUT_TESTS_LIST_FILE = "/sdcard/android/layout_tests_list.txt"; 132 static final String TEST_STATUS_FILE = "/sdcard/android/running_test.txt"; 133 static final String LAYOUT_TESTS_RESULTS_REFERENCE_FILES[] = { 134 "results/layout_tests_passed.txt", 135 "results/layout_tests_failed.txt", 136 "results/layout_tests_nontext.txt", 137 "results/layout_tests_crashed.txt", 138 "run_layout_tests.py" 139 }; 140 141 static final String LAYOUT_RESULTS_FAILED_RESULT_FILE = "results/layout_tests_failed.txt"; 142 static final String LAYOUT_RESULTS_NONTEXT_RESULT_FILE = "results/layout_tests_nontext.txt"; 143 static final String LAYOUT_RESULTS_CRASHED_RESULT_FILE = "results/layout_tests_crashed.txt"; 144 static final String LAYOUT_TESTS_RUNNER = "run_layout_tests.py"; 145 146 private MyTestRecorder mResultRecorder; 147 private Vector<String> mTestList; 148 // Whether we should ignore the result for the corresponding test. Ordered same as mTestList. 149 private Vector<Boolean> mTestListIgnoreResult; 150 private boolean mRebaselineResults; 151 // The JavaScript engine currently in use. This determines which set of Android-specific 152 // expected test results we use. 153 private String mJsEngine; 154 private String mTestPathPrefix; 155 private boolean mFinished; 156 private int mTestCount; 157 private int mResumeIndex; 158 159 public LayoutTestsAutoTest() { 160 super(TestShellActivity.class); 161 } 162 163 private void getTestList() { 164 // Read test list. 165 try { 166 BufferedReader inReader = new BufferedReader(new FileReader(LAYOUT_TESTS_LIST_FILE)); 167 String line = inReader.readLine(); 168 while (line != null) { 169 if (line.startsWith(mTestPathPrefix)) { 170 String[] components = line.split(" "); 171 mTestList.add(components[0]); 172 mTestListIgnoreResult.add(components.length > 1 && components[1].equals("IGNORE_RESULT")); 173 } 174 line = inReader.readLine(); 175 } 176 inReader.close(); 177 Log.v(LOGTAG, "Test list has " + mTestList.size() + " test(s)."); 178 } catch (Exception e) { 179 Log.e(LOGTAG, "Error while reading test list : " + e.getMessage()); 180 } 181 mTestCount = mTestList.size(); 182 } 183 184 private void resumeTestList() { 185 // read out the test name it stoped last time. 186 try { 187 String line = FsUtils.readTestStatus(TEST_STATUS_FILE); 188 for (int i = 0; i < mTestList.size(); i++) { 189 if (mTestList.elementAt(i).equals(line)) { 190 mTestList = new Vector<String>(mTestList.subList(i+1, mTestList.size())); 191 mTestListIgnoreResult = new Vector<Boolean>(mTestListIgnoreResult.subList(i+1, mTestListIgnoreResult.size())); 192 mResumeIndex = i + 1; 193 break; 194 } 195 } 196 } catch (Exception e) { 197 Log.e(LOGTAG, "Error reading " + TEST_STATUS_FILE); 198 } 199 } 200 201 private void clearTestStatus() { 202 // Delete TEST_STATUS_FILE 203 try { 204 File f = new File(TEST_STATUS_FILE); 205 if (f.delete()) 206 Log.v(LOGTAG, "Deleted " + TEST_STATUS_FILE); 207 else 208 Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE); 209 } catch (Exception e) { 210 Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE + " : " + e.getMessage()); 211 } 212 } 213 214 private String getResultFile(String test) { 215 String shortName = test.substring(0, test.lastIndexOf('.')); 216 // Write actual results to result directory. 217 return shortName.replaceFirst(LAYOUT_TESTS_ROOT, LAYOUT_TESTS_RESULT_DIR) + "-result.txt"; 218 } 219 220 // Gets the file which contains WebKit's expected results for this test. 221 private String getExpectedResultFile(String test) { 222 // The generic result is at <path>/<name>-expected.txt 223 // First try the Android-specific result at 224 // platform/android-<js-engine>/<path>/<name>-expected.txt 225 int pos = test.lastIndexOf('.'); 226 if (pos == -1) 227 return null; 228 String genericExpectedResult = test.substring(0, pos) + "-expected.txt"; 229 String androidExpectedResultsDir = "platform/android-" + mJsEngine + "/"; 230 String androidExpectedResult = 231 genericExpectedResult.replaceFirst(LAYOUT_TESTS_ROOT, LAYOUT_TESTS_ROOT + androidExpectedResultsDir); 232 File f = new File(androidExpectedResult); 233 return f.exists() ? androidExpectedResult : genericExpectedResult; 234 } 235 236 // Gets the file which contains the actual results of running the test on 237 // Android, generated by a previous run which set a new baseline. 238 private String getAndroidExpectedResultFile(String expectedResultFile) { 239 return expectedResultFile.replaceFirst(LAYOUT_TESTS_ROOT, ANDROID_EXPECTED_RESULT_DIR); 240 } 241 242 // Wrap up 243 private void failedCase(String file) { 244 Log.w("Layout test: ", file + " failed"); 245 mResultRecorder.failed(file); 246 } 247 248 private void passedCase(String file) { 249 Log.v("Layout test:", file + " passed"); 250 mResultRecorder.passed(file); 251 } 252 253 private void ignoreResultCase(String file) { 254 Log.v("Layout test:", file + " ignore result"); 255 mResultRecorder.ignoreResult(file); 256 } 257 258 private void noResultCase(String file) { 259 Log.v("Layout test:", file + " no expected result"); 260 mResultRecorder.noResult(file); 261 } 262 263 private void processResult(String testFile, String actualResultFile, String expectedResultFile, boolean ignoreResult) { 264 Log.v(LOGTAG, " Processing result: " + testFile); 265 266 if (ignoreResult) { 267 ignoreResultCase(testFile); 268 return; 269 } 270 271 File actual = new File(actualResultFile); 272 File expected = new File(expectedResultFile); 273 if (actual.exists() && expected.exists()) { 274 try { 275 if (FsUtils.diffIgnoreSpaces(actualResultFile, expectedResultFile)) { 276 passedCase(testFile); 277 } else { 278 failedCase(testFile); 279 } 280 } catch (FileNotFoundException ex) { 281 Log.e(LOGTAG, "File not found : " + ex.getMessage()); 282 } catch (IOException ex) { 283 Log.e(LOGTAG, "IO Error : " + ex.getMessage()); 284 } 285 return; 286 } 287 288 if (!expected.exists()) { 289 noResultCase(testFile); 290 } 291 } 292 293 private void runTestAndWaitUntilDone(TestShellActivity activity, String test, int timeout, boolean ignoreResult, int testNumber) { 294 activity.setCallback(new TestShellCallback() { 295 public void finished() { 296 synchronized (LayoutTestsAutoTest.this) { 297 mFinished = true; 298 LayoutTestsAutoTest.this.notifyAll(); 299 } 300 } 301 302 public void timedOut(String url) { 303 Log.v(LOGTAG, "layout timeout: " + url); 304 } 305 }); 306 307 String resultFile = getResultFile(test); 308 if (resultFile == null) { 309 // Simply ignore this test. 310 return; 311 } 312 if (mRebaselineResults) { 313 String expectedResultFile = getExpectedResultFile(test); 314 File f = new File(expectedResultFile); 315 if (f.exists()) { 316 return; // don't run test and don't overwrite default tests. 317 } 318 319 resultFile = getAndroidExpectedResultFile(expectedResultFile); 320 } 321 322 mFinished = false; 323 Intent intent = new Intent(Intent.ACTION_VIEW); 324 intent.setClass(activity, TestShellActivity.class); 325 intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP); 326 intent.putExtra(TestShellActivity.TEST_URL, FsUtils.getTestUrl(test)); 327 intent.putExtra(TestShellActivity.RESULT_FILE, resultFile); 328 intent.putExtra(TestShellActivity.TIMEOUT_IN_MILLIS, timeout); 329 intent.putExtra(TestShellActivity.TOTAL_TEST_COUNT, mTestCount); 330 intent.putExtra(TestShellActivity.CURRENT_TEST_NUMBER, testNumber); 331 activity.startActivity(intent); 332 333 // Wait until done. 334 synchronized (this) { 335 while(!mFinished){ 336 try { 337 this.wait(); 338 } catch (InterruptedException e) { } 339 } 340 } 341 342 if (!mRebaselineResults) { 343 String expectedResultFile = getExpectedResultFile(test); 344 File f = new File(expectedResultFile); 345 if (!f.exists()) { 346 expectedResultFile = getAndroidExpectedResultFile(expectedResultFile); 347 } 348 349 processResult(test, resultFile, expectedResultFile, ignoreResult); 350 } 351 } 352 353 // Invokes running of layout tests 354 // and waits till it has finished running. 355 public void executeLayoutTests(boolean resume) { 356 LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation(); 357 // A convenient method to be called by another activity. 358 359 if (runner.mTestPath == null) { 360 Log.e(LOGTAG, "No test specified"); 361 return; 362 } 363 364 this.mTestList = new Vector<String>(); 365 this.mTestListIgnoreResult = new Vector<Boolean>(); 366 367 // Read settings 368 mTestPathPrefix = (new File(LAYOUT_TESTS_ROOT + runner.mTestPath)).getAbsolutePath(); 369 mRebaselineResults = runner.mRebaseline; 370 // V8 is the default JavaScript engine. 371 mJsEngine = runner.mJsEngine == null ? "v8" : runner.mJsEngine; 372 373 int timeout = runner.mTimeoutInMillis; 374 if (timeout <= 0) { 375 timeout = DEFAULT_TIMEOUT_IN_MILLIS; 376 } 377 378 this.mResultRecorder = new MyTestRecorder(resume); 379 380 if (!resume) 381 clearTestStatus(); 382 383 getTestList(); 384 if (resume) 385 resumeTestList(); 386 387 TestShellActivity activity = getActivity(); 388 activity.setDefaultDumpDataType(DumpDataType.EXT_REPR); 389 390 // Run tests. 391 int addr = -1; 392 try{ 393 addr = AdbUtils.resolve("android-browser-test.mtv.corp.google.com"); 394 } catch (IOException ioe) { 395 Log.w(LOGTAG, "error while resolving test host name", ioe); 396 } 397 if(addr == -1) { 398 Log.w(LOGTAG, "failed to resolve test host. http tests will fail."); 399 } 400 for (int i = 0; i < mTestList.size(); i++) { 401 String s = mTestList.elementAt(i); 402 boolean ignoreResult = mTestListIgnoreResult.elementAt(i); 403 FsUtils.updateTestStatus(TEST_STATUS_FILE, s); 404 // Run tests 405 // i is 0 based, but test count is 1 based so add 1 to i here. 406 runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis, ignoreResult, 407 i + 1 + mResumeIndex); 408 } 409 410 FsUtils.updateTestStatus(TEST_STATUS_FILE, "#DONE"); 411 ForwardService.getForwardService().stopForwardService(); 412 activity.finish(); 413 } 414 415 private String getTestPath() { 416 LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation(); 417 418 String test_path = LAYOUT_TESTS_ROOT; 419 if (runner.mTestPath != null) { 420 test_path += runner.mTestPath; 421 } 422 test_path = new File(test_path).getAbsolutePath(); 423 Log.v("LayoutTestsAutoTest", " Test path : " + test_path); 424 425 return test_path; 426 } 427 428 public void generateTestList() { 429 try { 430 File tests_list = new File(LAYOUT_TESTS_LIST_FILE); 431 BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tests_list, false)); 432 FsUtils.writeLayoutTestListRecursively(bos, getTestPath(), false); // Don't ignore results 433 bos.flush(); 434 bos.close(); 435 } catch (Exception e) { 436 Log.e(LOGTAG, "Error when creating test list: " + e.getMessage()); 437 } 438 } 439 440 // Running all the layout tests at once sometimes 441 // causes the dumprendertree to run out of memory. 442 // So, additional tests are added to run the tests 443 // in chunks. 444 public void startLayoutTests() { 445 try { 446 File tests_list = new File(LAYOUT_TESTS_LIST_FILE); 447 if (!tests_list.exists()) 448 generateTestList(); 449 } catch (Exception e) { 450 e.printStackTrace(); 451 } 452 453 executeLayoutTests(false); 454 } 455 456 public void resumeLayoutTests() { 457 executeLayoutTests(true); 458 } 459 460 public void copyResultsAndRunnerAssetsToCache() { 461 try { 462 String out_dir = getActivity().getApplicationContext().getCacheDir().getPath() + "/"; 463 464 for( int i=0; i< LAYOUT_TESTS_RESULTS_REFERENCE_FILES.length; i++) { 465 InputStream in = getActivity().getAssets().open(LAYOUT_TESTS_RESULTS_REFERENCE_FILES[i]); 466 OutputStream out = new FileOutputStream(out_dir + LAYOUT_TESTS_RESULTS_REFERENCE_FILES[i]); 467 468 byte[] buf = new byte[2048]; 469 int len; 470 471 while ((len = in.read(buf)) >= 0 ) { 472 out.write(buf, 0, len); 473 } 474 out.close(); 475 in.close(); 476 } 477 }catch (IOException e) { 478 e.printStackTrace(); 479 } 480 481 } 482 483} 484