LayoutTestsAutoTest.java revision 8cd303aec6f11ea9fece70f6e3d82efb607f8637
1/* 2 * Copyright (C) 2008 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.dumprendertree; 18 19import com.android.dumprendertree.TestShellActivity.DumpDataType; 20import com.android.dumprendertree.forwarder.AdbUtils; 21import com.android.dumprendertree.forwarder.ForwardServer; 22import com.android.dumprendertree.forwarder.ForwardService; 23 24import android.app.Instrumentation; 25import android.content.Intent; 26import android.os.Bundle; 27import android.test.ActivityInstrumentationTestCase2; 28import android.util.Log; 29 30import java.io.BufferedOutputStream; 31import java.io.BufferedReader; 32import java.io.File; 33import java.io.FileNotFoundException; 34import java.io.FileOutputStream; 35import java.io.FileReader; 36import java.io.IOException; 37import java.io.InputStream; 38import java.io.OutputStream; 39import java.util.Vector; 40 41//TestRecorder creates two files, one for passing tests 42//and another for failing tests and writes the paths to 43//layout tests one line at a time. TestRecorder does not 44//have ability to clear the results. 45class MyTestRecorder { 46 private BufferedOutputStream mBufferedOutputPassedStream; 47 private BufferedOutputStream mBufferedOutputFailedStream; 48 private BufferedOutputStream mBufferedOutputNoresultStream; 49 private BufferedOutputStream mBufferedOutputTimedoutStream; 50 51 public void passed(String layout_file) { 52 try { 53 mBufferedOutputPassedStream.write(layout_file.getBytes()); 54 mBufferedOutputPassedStream.write('\n'); 55 mBufferedOutputPassedStream.flush(); 56 } catch(Exception e) { 57 e.printStackTrace(); 58 } 59 } 60 61 public void failed(String layout_file) { 62 try { 63 mBufferedOutputFailedStream.write(layout_file.getBytes()); 64 mBufferedOutputFailedStream.write('\n'); 65 mBufferedOutputFailedStream.flush(); 66 } catch(Exception e) { 67 e.printStackTrace(); 68 } 69 } 70 71 public void noresult(String layout_file) { 72 try { 73 mBufferedOutputNoresultStream.write(layout_file.getBytes()); 74 mBufferedOutputNoresultStream.write('\n'); 75 mBufferedOutputNoresultStream.flush(); 76 } catch(Exception e) { 77 e.printStackTrace(); 78 } 79 } 80 81 public void timedout(String url) { 82 try { 83 mBufferedOutputTimedoutStream.write(url.getBytes()); 84 mBufferedOutputTimedoutStream.write('\n'); 85 mBufferedOutputTimedoutStream.flush(); 86 } catch (Exception e) { 87 e.printStackTrace(); 88 } 89 } 90 91 public MyTestRecorder(boolean resume) { 92 try { 93 File resultsPassedFile = new File("/sdcard/layout_tests_passed.txt"); 94 File resultsFailedFile = new File("/sdcard/layout_tests_failed.txt"); 95 File noExpectedResultFile = new File("/sdcard/layout_tests_nontext.txt"); 96 File resultTimedoutFile = new File("/sdcard/layout_tests_timedout.txt"); 97 98 mBufferedOutputPassedStream = 99 new BufferedOutputStream(new FileOutputStream(resultsPassedFile, resume)); 100 mBufferedOutputFailedStream = 101 new BufferedOutputStream(new FileOutputStream(resultsFailedFile, resume)); 102 mBufferedOutputNoresultStream = 103 new BufferedOutputStream(new FileOutputStream(noExpectedResultFile, resume)); 104 mBufferedOutputTimedoutStream = 105 new BufferedOutputStream(new FileOutputStream(resultTimedoutFile, resume)); 106 } catch (Exception e) { 107 e.printStackTrace(); 108 } 109 } 110 111 public void close() { 112 try { 113 mBufferedOutputPassedStream.close(); 114 mBufferedOutputFailedStream.close(); 115 mBufferedOutputNoresultStream.close(); 116 mBufferedOutputTimedoutStream.close(); 117 } catch (Exception e) { 118 e.printStackTrace(); 119 } 120 } 121} 122 123 124public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestShellActivity> { 125 126 private static final String LOGTAG = "LayoutTests"; 127 static final int DEFAULT_TIMEOUT_IN_MILLIS = 5000; 128 129 static final String LAYOUT_TESTS_ROOT = "/sdcard/android/layout_tests/"; 130 static final String LAYOUT_TESTS_RESULT_DIR = "/sdcard/android/layout_tests_results/"; 131 static final String ANDROID_EXPECTED_RESULT_DIR = "/sdcard/android/expected_results/"; 132 static final String LAYOUT_TESTS_LIST_FILE = "/sdcard/android/layout_tests_list.txt"; 133 static final String TEST_STATUS_FILE = "/sdcard/android/running_test.txt"; 134 static final String LAYOUT_TESTS_RESULTS_REFERENCE_FILES[] = { 135 "results/layout_tests_passed.txt", 136 "results/layout_tests_failed.txt", 137 "results/layout_tests_nontext.txt", 138 "results/layout_tests_crashed.txt", 139 "run_layout_tests.py" 140 }; 141 142 static final String LAYOUT_RESULTS_FAILED_RESULT_FILE = "results/layout_tests_failed.txt"; 143 static final String LAYOUT_RESULTS_NONTEXT_RESULT_FILE = "results/layout_tests_nontext.txt"; 144 static final String LAYOUT_RESULTS_CRASHED_RESULT_FILE = "results/layout_tests_crashed.txt"; 145 static final String LAYOUT_TESTS_RUNNER = "run_layout_tests.py"; 146 147 private MyTestRecorder mResultRecorder; 148 private Vector<String> mTestList; 149 private boolean mRebaselineResults; 150 private String mTestPathPrefix; 151 private boolean mFinished; 152 153 public LayoutTestsAutoTest() { 154 super("com.android.dumprendertree", TestShellActivity.class); 155 } 156 157 // This function writes the result of the layout test to 158 // Am status so that it can be picked up from a script. 159 private void passOrFailCallback(String file, boolean result) { 160 Instrumentation inst = getInstrumentation(); 161 Bundle bundle = new Bundle(); 162 bundle.putBoolean(file, result); 163 inst.sendStatus(0, bundle); 164 } 165 166 private void getTestList() { 167 // Read test list. 168 try { 169 BufferedReader inReader = new BufferedReader(new FileReader(LAYOUT_TESTS_LIST_FILE)); 170 String line = inReader.readLine(); 171 while (line != null) { 172 if (line.startsWith(mTestPathPrefix)) 173 mTestList.add(line); 174 line = inReader.readLine(); 175 } 176 inReader.close(); 177 Log.v(LOGTAG, "Test list has " + mTestList.size() + " test(s)."); 178 } catch (Exception e) { 179 Log.e(LOGTAG, "Error while reading test list : " + e.getMessage()); 180 } 181 } 182 183 private void resumeTestList() { 184 // read out the test name it stoped last time. 185 try { 186 String line = FsUtils.readTestStatus(TEST_STATUS_FILE); 187 for (int i = 0; i < mTestList.size(); i++) { 188 if (mTestList.elementAt(i).equals(line)) { 189 mTestList = new Vector<String>(mTestList.subList(i+1, mTestList.size())); 190 break; 191 } 192 } 193 } catch (Exception e) { 194 Log.e(LOGTAG, "Error reading " + TEST_STATUS_FILE); 195 } 196 } 197 198 private void clearTestStatus() { 199 // Delete TEST_STATUS_FILE 200 try { 201 File f = new File(TEST_STATUS_FILE); 202 if (f.delete()) 203 Log.v(LOGTAG, "Deleted " + TEST_STATUS_FILE); 204 else 205 Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE); 206 } catch (Exception e) { 207 Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE + " : " + e.getMessage()); 208 } 209 } 210 211 private String getResultFile(String test) { 212 String shortName = test.substring(0, test.lastIndexOf('.')); 213 // Write actual results to result directory. 214 return shortName.replaceFirst(LAYOUT_TESTS_ROOT, LAYOUT_TESTS_RESULT_DIR) + "-result.txt"; 215 } 216 217 private String getExpectedResultFile(String test) { 218 int pos = test.lastIndexOf('.'); 219 if(pos == -1) 220 return null; 221 String shortName = test.substring(0, pos); 222 return shortName + "-expected.txt"; 223 } 224 225 private String getAndroidExpectedResultFile(String expectedResultFile) { 226 return expectedResultFile.replaceFirst(LAYOUT_TESTS_ROOT, ANDROID_EXPECTED_RESULT_DIR); 227 } 228 229 // Wrap up 230 private void failedCase(String file) { 231 Log.w("Layout test: ", file + " failed"); 232 mResultRecorder.failed(file); 233 } 234 235 private void passedCase(String file) { 236 Log.v("Layout test:", file + " passed"); 237 mResultRecorder.passed(file); 238 } 239 240 private void noresultCase(String file) { 241 Log.v("Layout test:", file + " no expected result"); 242 mResultRecorder.noresult(file); 243 } 244 245 private void processResult(String testFile, String actualResultFile, String expectedResultFile) { 246 Log.v(LOGTAG, " Processing result: " + testFile); 247 248 File actual = new File(actualResultFile); 249 File expected = new File(expectedResultFile); 250 if (actual.exists() && expected.exists()) { 251 try { 252 if (FsUtils.diffIgnoreSpaces(actualResultFile, expectedResultFile)) { 253 passedCase(testFile); 254 } else { 255 failedCase(testFile); 256 } 257 } catch (FileNotFoundException ex) { 258 Log.e(LOGTAG, "File not found : " + ex.getMessage()); 259 } catch (IOException ex) { 260 Log.e(LOGTAG, "IO Error : " + ex.getMessage()); 261 } 262 return; 263 } 264 265 if (!expected.exists()) { 266 noresultCase(testFile); 267 } 268 } 269 270 private void runTestAndWaitUntilDone(TestShellActivity activity, String test, int timeout) { 271 activity.setCallback(new TestShellCallback() { 272 public void finished() { 273 synchronized (LayoutTestsAutoTest.this) { 274 mFinished = true; 275 LayoutTestsAutoTest.this.notifyAll(); 276 } 277 } 278 279 public void timedOut(String url) { 280 Log.v(LOGTAG, "layout timeout: " + url); 281 } 282 }); 283 284 String resultFile = getResultFile(test); 285 if(resultFile == null) { 286 //simply ignore this test 287 return; 288 } 289 if (mRebaselineResults) { 290 String expectedResultFile = getExpectedResultFile(test); 291 File f = new File(expectedResultFile); 292 if (f.exists()) { 293 return; // don't run test and don't overwrite default tests. 294 } 295 296 resultFile = getAndroidExpectedResultFile(expectedResultFile); 297 } 298 299 mFinished = false; 300 Intent intent = new Intent(Intent.ACTION_VIEW); 301 intent.setClass(activity, TestShellActivity.class); 302 intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP); 303 intent.putExtra(TestShellActivity.TEST_URL, FsUtils.getTestUrl(test)); 304 intent.putExtra(TestShellActivity.RESULT_FILE, resultFile); 305 intent.putExtra(TestShellActivity.TIMEOUT_IN_MILLIS, timeout); 306 activity.startActivity(intent); 307 308 // Wait until done. 309 synchronized (this) { 310 while(!mFinished){ 311 try { 312 this.wait(); 313 } catch (InterruptedException e) { } 314 } 315 } 316 317 if (!mRebaselineResults) { 318 String expectedResultFile = getExpectedResultFile(test); 319 File f = new File(expectedResultFile); 320 if (!f.exists()) { 321 expectedResultFile = getAndroidExpectedResultFile(expectedResultFile); 322 } 323 324 processResult(test, resultFile, expectedResultFile); 325 } 326 } 327 328 // Invokes running of layout tests 329 // and waits till it has finished running. 330 public void executeLayoutTests(boolean resume) { 331 LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation(); 332 // A convenient method to be called by another activity. 333 334 if (runner.mTestPath == null) { 335 Log.e(LOGTAG, "No test specified"); 336 return; 337 } 338 339 this.mTestList = new Vector<String>(); 340 341 // Read settings 342 this.mTestPathPrefix = (new File(LAYOUT_TESTS_ROOT + runner.mTestPath)).getAbsolutePath(); 343 this.mRebaselineResults = runner.mRebaseline; 344 345 int timeout = runner.mTimeoutInMillis; 346 if (timeout <= 0) { 347 timeout = DEFAULT_TIMEOUT_IN_MILLIS; 348 } 349 350 this.mResultRecorder = new MyTestRecorder(resume); 351 352 if (!resume) 353 clearTestStatus(); 354 355 getTestList(); 356 if (resume) 357 resumeTestList(); 358 359 TestShellActivity activity = getActivity(); 360 activity.setDefaultDumpDataType(DumpDataType.DUMP_AS_TEXT); 361 362 // Run tests. 363 int addr = -1; 364 try{ 365 addr = AdbUtils.resolve("android-browser-test.mtv.corp.google.com"); 366 } catch (IOException ioe) { 367 Log.w(LOGTAG, "error while resolving test host name", ioe); 368 } 369 if(addr == -1) { 370 Log.w(LOGTAG, "failed to resolve test host. http tests will fail."); 371 } 372 for (int i = 0; i < mTestList.size(); i++) { 373 String s = mTestList.elementAt(i); 374 FsUtils.updateTestStatus(TEST_STATUS_FILE, s); 375 // Run tests 376 runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis); 377 } 378 379 FsUtils.updateTestStatus(TEST_STATUS_FILE, "#DONE"); 380 ForwardService.getForwardService().stopForwardService(); 381 activity.finish(); 382 } 383 384 private String getTestPath() { 385 LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation(); 386 387 String test_path = LAYOUT_TESTS_ROOT; 388 if (runner.mTestPath != null) { 389 test_path += runner.mTestPath; 390 } 391 test_path = new File(test_path).getAbsolutePath(); 392 Log.v("LayoutTestsAutoTest", " Test path : " + test_path); 393 394 return test_path; 395 } 396 397 public void generateTestList() { 398 try { 399 File tests_list = new File(LAYOUT_TESTS_LIST_FILE); 400 BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tests_list, false)); 401 FsUtils.findLayoutTestsRecursively(bos, getTestPath()); 402 bos.flush(); 403 bos.close(); 404 } catch (Exception e) { 405 Log.e(LOGTAG, "Error when creating test list: " + e.getMessage()); 406 } 407 } 408 409 // Running all the layout tests at once sometimes 410 // causes the dumprendertree to run out of memory. 411 // So, additional tests are added to run the tests 412 // in chunks. 413 public void startLayoutTests() { 414 try { 415 File tests_list = new File(LAYOUT_TESTS_LIST_FILE); 416 if (!tests_list.exists()) 417 generateTestList(); 418 } catch (Exception e) { 419 e.printStackTrace(); 420 } 421 422 executeLayoutTests(false); 423 } 424 425 public void resumeLayoutTests() { 426 executeLayoutTests(true); 427 } 428 429 public void copyResultsAndRunnerAssetsToCache() { 430 try { 431 String out_dir = getActivity().getApplicationContext().getCacheDir().getPath() + "/"; 432 433 for( int i=0; i< LAYOUT_TESTS_RESULTS_REFERENCE_FILES.length; i++) { 434 InputStream in = getActivity().getAssets().open(LAYOUT_TESTS_RESULTS_REFERENCE_FILES[i]); 435 OutputStream out = new FileOutputStream(out_dir + LAYOUT_TESTS_RESULTS_REFERENCE_FILES[i]); 436 437 byte[] buf = new byte[2048]; 438 int len; 439 440 while ((len = in.read(buf)) >= 0 ) { 441 out.write(buf, 0, len); 442 } 443 out.close(); 444 in.close(); 445 } 446 }catch (IOException e) { 447 e.printStackTrace(); 448 } 449 450 } 451 452} 453