LayoutTestsAutoTest.java revision 5ac2a3bd697cb86022aa7d4dc0c8fbd6a10b8193
1/* 2 * Copyright (C) 2008 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.dumprendertree; 18 19import com.android.dumprendertree.TestShellActivity.DumpDataType; 20import com.android.dumprendertree.forwarder.AdbUtils; 21import com.android.dumprendertree.forwarder.ForwardServer; 22import com.android.dumprendertree.forwarder.ForwardService; 23 24import android.app.Instrumentation; 25import android.content.Intent; 26import android.os.Bundle; 27import android.test.ActivityInstrumentationTestCase2; 28import android.util.Log; 29 30import java.io.BufferedOutputStream; 31import java.io.BufferedReader; 32import java.io.File; 33import java.io.FileNotFoundException; 34import java.io.FileOutputStream; 35import java.io.FileReader; 36import java.io.IOException; 37import java.io.InputStream; 38import java.io.OutputStream; 39import java.util.Vector; 40 41// TestRecorder creates three files ... 42// - passing tests 43// - failing tests 44// - tests with no text results available 45// TestRecorder does not have the ability to clear the results. 46class MyTestRecorder { 47 private BufferedOutputStream mBufferedOutputPassedStream; 48 private BufferedOutputStream mBufferedOutputFailedStream; 49 private BufferedOutputStream mBufferedOutputNoResultStream; 50 51 public void passed(String layout_file) { 52 try { 53 mBufferedOutputPassedStream.write(layout_file.getBytes()); 54 mBufferedOutputPassedStream.write('\n'); 55 mBufferedOutputPassedStream.flush(); 56 } catch(Exception e) { 57 e.printStackTrace(); 58 } 59 } 60 61 public void failed(String layout_file) { 62 try { 63 mBufferedOutputFailedStream.write(layout_file.getBytes()); 64 mBufferedOutputFailedStream.write('\n'); 65 mBufferedOutputFailedStream.flush(); 66 } catch(Exception e) { 67 e.printStackTrace(); 68 } 69 } 70 71 public void noResult(String layout_file) { 72 try { 73 mBufferedOutputNoResultStream.write(layout_file.getBytes()); 74 mBufferedOutputNoResultStream.write('\n'); 75 mBufferedOutputNoResultStream.flush(); 76 } catch(Exception e) { 77 e.printStackTrace(); 78 } 79 } 80 81 public MyTestRecorder(boolean resume) { 82 try { 83 File resultsPassedFile = new File("/sdcard/layout_tests_passed.txt"); 84 File resultsFailedFile = new File("/sdcard/layout_tests_failed.txt"); 85 File noExpectedResultFile = new File("/sdcard/layout_tests_nontext.txt"); 86 87 mBufferedOutputPassedStream = 88 new BufferedOutputStream(new FileOutputStream(resultsPassedFile, resume)); 89 mBufferedOutputFailedStream = 90 new BufferedOutputStream(new FileOutputStream(resultsFailedFile, resume)); 91 mBufferedOutputNoResultStream = 92 new BufferedOutputStream(new FileOutputStream(noExpectedResultFile, resume)); 93 } catch (Exception e) { 94 e.printStackTrace(); 95 } 96 } 97 98 public void close() { 99 try { 100 mBufferedOutputPassedStream.close(); 101 mBufferedOutputFailedStream.close(); 102 mBufferedOutputNoResultStream.close(); 103 } catch (Exception e) { 104 e.printStackTrace(); 105 } 106 } 107} 108 109 110public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestShellActivity> { 111 112 private static final String LOGTAG = "LayoutTests"; 113 static final int DEFAULT_TIMEOUT_IN_MILLIS = 5000; 114 115 static final String LAYOUT_TESTS_ROOT = "/sdcard/android/layout_tests/"; 116 static final String LAYOUT_TESTS_RESULT_DIR = "/sdcard/android/layout_tests_results/"; 117 static final String ANDROID_EXPECTED_RESULT_DIR = "/sdcard/android/expected_results/"; 118 static final String LAYOUT_TESTS_LIST_FILE = "/sdcard/android/layout_tests_list.txt"; 119 static final String TEST_STATUS_FILE = "/sdcard/android/running_test.txt"; 120 static final String LAYOUT_TESTS_RESULTS_REFERENCE_FILES[] = { 121 "results/layout_tests_passed.txt", 122 "results/layout_tests_failed.txt", 123 "results/layout_tests_nontext.txt", 124 "results/layout_tests_crashed.txt", 125 "run_layout_tests.py" 126 }; 127 128 static final String LAYOUT_RESULTS_FAILED_RESULT_FILE = "results/layout_tests_failed.txt"; 129 static final String LAYOUT_RESULTS_NONTEXT_RESULT_FILE = "results/layout_tests_nontext.txt"; 130 static final String LAYOUT_RESULTS_CRASHED_RESULT_FILE = "results/layout_tests_crashed.txt"; 131 static final String LAYOUT_TESTS_RUNNER = "run_layout_tests.py"; 132 133 private MyTestRecorder mResultRecorder; 134 private Vector<String> mTestList; 135 private boolean mRebaselineResults; 136 // The JavaScript engine currently in use. This determines which set of Android-specific 137 // expected test results we use. 138 private String mJsEngine; 139 private String mTestPathPrefix; 140 private boolean mFinished; 141 142 public LayoutTestsAutoTest() { 143 super("com.android.dumprendertree", TestShellActivity.class); 144 } 145 146 // This function writes the result of the layout test to 147 // Am status so that it can be picked up from a script. 148 private void passOrFailCallback(String file, boolean result) { 149 Instrumentation inst = getInstrumentation(); 150 Bundle bundle = new Bundle(); 151 bundle.putBoolean(file, result); 152 inst.sendStatus(0, bundle); 153 } 154 155 private void getTestList() { 156 // Read test list. 157 try { 158 BufferedReader inReader = new BufferedReader(new FileReader(LAYOUT_TESTS_LIST_FILE)); 159 String line = inReader.readLine(); 160 while (line != null) { 161 if (line.startsWith(mTestPathPrefix)) 162 mTestList.add(line); 163 line = inReader.readLine(); 164 } 165 inReader.close(); 166 Log.v(LOGTAG, "Test list has " + mTestList.size() + " test(s)."); 167 } catch (Exception e) { 168 Log.e(LOGTAG, "Error while reading test list : " + e.getMessage()); 169 } 170 } 171 172 private void resumeTestList() { 173 // read out the test name it stoped last time. 174 try { 175 String line = FsUtils.readTestStatus(TEST_STATUS_FILE); 176 for (int i = 0; i < mTestList.size(); i++) { 177 if (mTestList.elementAt(i).equals(line)) { 178 mTestList = new Vector<String>(mTestList.subList(i+1, mTestList.size())); 179 break; 180 } 181 } 182 } catch (Exception e) { 183 Log.e(LOGTAG, "Error reading " + TEST_STATUS_FILE); 184 } 185 } 186 187 private void clearTestStatus() { 188 // Delete TEST_STATUS_FILE 189 try { 190 File f = new File(TEST_STATUS_FILE); 191 if (f.delete()) 192 Log.v(LOGTAG, "Deleted " + TEST_STATUS_FILE); 193 else 194 Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE); 195 } catch (Exception e) { 196 Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE + " : " + e.getMessage()); 197 } 198 } 199 200 private String getResultFile(String test) { 201 String shortName = test.substring(0, test.lastIndexOf('.')); 202 // Write actual results to result directory. 203 return shortName.replaceFirst(LAYOUT_TESTS_ROOT, LAYOUT_TESTS_RESULT_DIR) + "-result.txt"; 204 } 205 206 // Gets the file which contains WebKit's expected results for this test. 207 private String getExpectedResultFile(String test) { 208 // The generic result is at <path>/<name>-expected.txt 209 // First try the Android-specific result at 210 // platform/android-<js-engine>/<path>/<name>-expected.txt 211 int pos = test.lastIndexOf('.'); 212 if (pos == -1) 213 return null; 214 String genericExpectedResult = test.substring(0, pos) + "-expected.txt"; 215 String androidExpectedResultsDir = "platform/android-" + mJsEngine + "/"; 216 String androidExpectedResult = 217 genericExpectedResult.replaceFirst(LAYOUT_TESTS_ROOT, LAYOUT_TESTS_ROOT + androidExpectedResultsDir); 218 File f = new File(androidExpectedResult); 219 return f.exists() ? androidExpectedResult : genericExpectedResult; 220 } 221 222 // Gets the file which contains the actual results of running the test on 223 // Android, generated by a previous run which set a new baseline. 224 private String getAndroidExpectedResultFile(String expectedResultFile) { 225 return expectedResultFile.replaceFirst(LAYOUT_TESTS_ROOT, ANDROID_EXPECTED_RESULT_DIR); 226 } 227 228 // Wrap up 229 private void failedCase(String file) { 230 Log.w("Layout test: ", file + " failed"); 231 mResultRecorder.failed(file); 232 } 233 234 private void passedCase(String file) { 235 Log.v("Layout test:", file + " passed"); 236 mResultRecorder.passed(file); 237 } 238 239 private void noResultCase(String file) { 240 Log.v("Layout test:", file + " no expected result"); 241 mResultRecorder.noResult(file); 242 } 243 244 private void processResult(String testFile, String actualResultFile, String expectedResultFile) { 245 Log.v(LOGTAG, " Processing result: " + testFile); 246 247 File actual = new File(actualResultFile); 248 File expected = new File(expectedResultFile); 249 if (actual.exists() && expected.exists()) { 250 try { 251 if (FsUtils.diffIgnoreSpaces(actualResultFile, expectedResultFile)) { 252 passedCase(testFile); 253 } else { 254 failedCase(testFile); 255 } 256 } catch (FileNotFoundException ex) { 257 Log.e(LOGTAG, "File not found : " + ex.getMessage()); 258 } catch (IOException ex) { 259 Log.e(LOGTAG, "IO Error : " + ex.getMessage()); 260 } 261 return; 262 } 263 264 if (!expected.exists()) { 265 noResultCase(testFile); 266 } 267 } 268 269 private void runTestAndWaitUntilDone(TestShellActivity activity, String test, int timeout) { 270 activity.setCallback(new TestShellCallback() { 271 public void finished() { 272 synchronized (LayoutTestsAutoTest.this) { 273 mFinished = true; 274 LayoutTestsAutoTest.this.notifyAll(); 275 } 276 } 277 278 public void timedOut(String url) { 279 Log.v(LOGTAG, "layout timeout: " + url); 280 } 281 }); 282 283 String resultFile = getResultFile(test); 284 if (resultFile == null) { 285 // Simply ignore this test. 286 return; 287 } 288 if (mRebaselineResults) { 289 String expectedResultFile = getExpectedResultFile(test); 290 File f = new File(expectedResultFile); 291 if (f.exists()) { 292 return; // don't run test and don't overwrite default tests. 293 } 294 295 resultFile = getAndroidExpectedResultFile(expectedResultFile); 296 } 297 298 mFinished = false; 299 Intent intent = new Intent(Intent.ACTION_VIEW); 300 intent.setClass(activity, TestShellActivity.class); 301 intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP); 302 intent.putExtra(TestShellActivity.TEST_URL, FsUtils.getTestUrl(test)); 303 intent.putExtra(TestShellActivity.RESULT_FILE, resultFile); 304 intent.putExtra(TestShellActivity.TIMEOUT_IN_MILLIS, timeout); 305 activity.startActivity(intent); 306 307 // Wait until done. 308 synchronized (this) { 309 while(!mFinished){ 310 try { 311 this.wait(); 312 } catch (InterruptedException e) { } 313 } 314 } 315 316 if (!mRebaselineResults) { 317 String expectedResultFile = getExpectedResultFile(test); 318 File f = new File(expectedResultFile); 319 if (!f.exists()) { 320 expectedResultFile = getAndroidExpectedResultFile(expectedResultFile); 321 } 322 323 processResult(test, resultFile, expectedResultFile); 324 } 325 } 326 327 // Invokes running of layout tests 328 // and waits till it has finished running. 329 public void executeLayoutTests(boolean resume) { 330 LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation(); 331 // A convenient method to be called by another activity. 332 333 if (runner.mTestPath == null) { 334 Log.e(LOGTAG, "No test specified"); 335 return; 336 } 337 338 this.mTestList = new Vector<String>(); 339 340 // Read settings 341 mTestPathPrefix = (new File(LAYOUT_TESTS_ROOT + runner.mTestPath)).getAbsolutePath(); 342 mRebaselineResults = runner.mRebaseline; 343 // JSC is the default JavaScript engine. 344 mJsEngine = runner.mJsEngine == null ? "jsc" : runner.mJsEngine; 345 346 int timeout = runner.mTimeoutInMillis; 347 if (timeout <= 0) { 348 timeout = DEFAULT_TIMEOUT_IN_MILLIS; 349 } 350 351 this.mResultRecorder = new MyTestRecorder(resume); 352 353 if (!resume) 354 clearTestStatus(); 355 356 getTestList(); 357 if (resume) 358 resumeTestList(); 359 360 TestShellActivity activity = getActivity(); 361 activity.setDefaultDumpDataType(DumpDataType.DUMP_AS_TEXT); 362 363 // Run tests. 364 int addr = -1; 365 try{ 366 addr = AdbUtils.resolve("android-browser-test.mtv.corp.google.com"); 367 } catch (IOException ioe) { 368 Log.w(LOGTAG, "error while resolving test host name", ioe); 369 } 370 if(addr == -1) { 371 Log.w(LOGTAG, "failed to resolve test host. http tests will fail."); 372 } 373 for (int i = 0; i < mTestList.size(); i++) { 374 String s = mTestList.elementAt(i); 375 FsUtils.updateTestStatus(TEST_STATUS_FILE, s); 376 // Run tests 377 runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis); 378 } 379 380 FsUtils.updateTestStatus(TEST_STATUS_FILE, "#DONE"); 381 ForwardService.getForwardService().stopForwardService(); 382 activity.finish(); 383 } 384 385 private String getTestPath() { 386 LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation(); 387 388 String test_path = LAYOUT_TESTS_ROOT; 389 if (runner.mTestPath != null) { 390 test_path += runner.mTestPath; 391 } 392 test_path = new File(test_path).getAbsolutePath(); 393 Log.v("LayoutTestsAutoTest", " Test path : " + test_path); 394 395 return test_path; 396 } 397 398 public void generateTestList() { 399 try { 400 File tests_list = new File(LAYOUT_TESTS_LIST_FILE); 401 BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tests_list, false)); 402 FsUtils.findLayoutTestsRecursively(bos, getTestPath()); 403 bos.flush(); 404 bos.close(); 405 } catch (Exception e) { 406 Log.e(LOGTAG, "Error when creating test list: " + e.getMessage()); 407 } 408 } 409 410 // Running all the layout tests at once sometimes 411 // causes the dumprendertree to run out of memory. 412 // So, additional tests are added to run the tests 413 // in chunks. 414 public void startLayoutTests() { 415 try { 416 File tests_list = new File(LAYOUT_TESTS_LIST_FILE); 417 if (!tests_list.exists()) 418 generateTestList(); 419 } catch (Exception e) { 420 e.printStackTrace(); 421 } 422 423 executeLayoutTests(false); 424 } 425 426 public void resumeLayoutTests() { 427 executeLayoutTests(true); 428 } 429 430 public void copyResultsAndRunnerAssetsToCache() { 431 try { 432 String out_dir = getActivity().getApplicationContext().getCacheDir().getPath() + "/"; 433 434 for( int i=0; i< LAYOUT_TESTS_RESULTS_REFERENCE_FILES.length; i++) { 435 InputStream in = getActivity().getAssets().open(LAYOUT_TESTS_RESULTS_REFERENCE_FILES[i]); 436 OutputStream out = new FileOutputStream(out_dir + LAYOUT_TESTS_RESULTS_REFERENCE_FILES[i]); 437 438 byte[] buf = new byte[2048]; 439 int len; 440 441 while ((len = in.read(buf)) >= 0 ) { 442 out.write(buf, 0, len); 443 } 444 out.close(); 445 in.close(); 446 } 447 }catch (IOException e) { 448 e.printStackTrace(); 449 } 450 451 } 452 453} 454