LayoutTestsAutoTest.java revision 3ae8c42152d890ab771053fa6b16b038ee44326d
19066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project/* 29066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * Copyright (C) 2008 The Android Open Source Project 39066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * 49066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * Licensed under the Apache License, Version 2.0 (the "License"); 59066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * you may not use this file except in compliance with the License. 69066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * You may obtain a copy of the License at 79066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * 89066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * http://www.apache.org/licenses/LICENSE-2.0 99066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * 109066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * Unless required by applicable law or agreed to in writing, software 119066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * distributed under the License is distributed on an "AS IS" BASIS, 129066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 139066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * See the License for the specific language governing permissions and 149066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project * limitations under the License. 159066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project */ 169066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 171d3165f10b12165f02b7015ac1a817c5f60e6399Neal Nguyenpackage com.android.dumprendertree; 189066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 199066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport com.android.dumprendertree.TestShellActivity.DumpDataType; 209066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport com.android.dumprendertree.forwarder.AdbUtils; 21696cba573e651b0e4f18a4718627c8ccecb3bda0Adam Powellimport com.android.dumprendertree.forwarder.ForwardService; 229066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 239066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport android.content.Context; 249066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport android.content.Intent; 259066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport android.os.Environment; 269066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport android.test.ActivityInstrumentationTestCase2; 279066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport android.util.Log; 289066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 299066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport java.io.BufferedOutputStream; 309066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport java.io.BufferedReader; 319066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport java.io.File; 329066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport java.io.FileNotFoundException; 339066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport java.io.FileOutputStream; 349066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport java.io.FileReader; 359066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport java.io.IOException; 369066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport java.io.InputStream; 379066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport java.io.OutputStream; 389066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectimport java.util.Vector; 399066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 409066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project// TestRecorder creates four files ... 419066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project// - passing tests 429066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project// - failing tests 439066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project// - tests for which results are ignored 449066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project// - tests with no text results available 459066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project// TestRecorder does not have the ability to clear the results. 469066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectclass MyTestRecorder { 479066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private BufferedOutputStream mBufferedOutputPassedStream; 489066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private BufferedOutputStream mBufferedOutputFailedStream; 499066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private BufferedOutputStream mBufferedOutputIgnoreResultStream; 509066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private BufferedOutputStream mBufferedOutputNoResultStream; 519066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 529066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public void passed(String layout_file) { 539066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project try { 549066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputPassedStream.write(layout_file.getBytes()); 559066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputPassedStream.write('\n'); 569066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputPassedStream.flush(); 579066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } catch(Exception e) { 589066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project e.printStackTrace(); 599066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 609066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 619066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 629066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public void failed(String layout_file) { 639066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project try { 649066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputFailedStream.write(layout_file.getBytes()); 659066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputFailedStream.write('\n'); 669066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputFailedStream.flush(); 679066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } catch(Exception e) { 689066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project e.printStackTrace(); 699066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 709066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 719066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 729066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public void ignoreResult(String layout_file) { 739066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project try { 749066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputIgnoreResultStream.write(layout_file.getBytes()); 759066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputIgnoreResultStream.write('\n'); 769066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputIgnoreResultStream.flush(); 779066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } catch(Exception e) { 789066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project e.printStackTrace(); 799066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 809066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 819066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 829066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public void noResult(String layout_file) { 839066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project try { 849066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputNoResultStream.write(layout_file.getBytes()); 859066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputNoResultStream.write('\n'); 869066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputNoResultStream.flush(); 879066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } catch(Exception e) { 889066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project e.printStackTrace(); 899066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 909066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 919066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 929066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public MyTestRecorder(boolean resume) { 939066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project try { 949066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project File externalDir = Environment.getExternalStorageDirectory(); 959066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project File resultsPassedFile = new File(externalDir, "layout_tests_passed.txt"); 969066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project File resultsFailedFile = new File(externalDir, "layout_tests_failed.txt"); 979066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project File resultsIgnoreResultFile = new File(externalDir, "layout_tests_ignored.txt"); 989066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project File noExpectedResultFile = new File(externalDir, "layout_tests_nontext.txt"); 999066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 1009066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputPassedStream = 1019066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project new BufferedOutputStream(new FileOutputStream(resultsPassedFile, resume)); 1029066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputFailedStream = 1039066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project new BufferedOutputStream(new FileOutputStream(resultsFailedFile, resume)); 1049066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputIgnoreResultStream = 1059066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project new BufferedOutputStream(new FileOutputStream(resultsIgnoreResultFile, resume)); 1069066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputNoResultStream = 1079066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project new BufferedOutputStream(new FileOutputStream(noExpectedResultFile, resume)); 1089066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } catch (Exception e) { 1099066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project e.printStackTrace(); 1109066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 1119066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 1129066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 1139066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public void close() { 1149066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project try { 1159066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputPassedStream.close(); 1169066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputFailedStream.close(); 1179066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputIgnoreResultStream.close(); 1189066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project mBufferedOutputNoResultStream.close(); 1199066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } catch (Exception e) { 1209066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project e.printStackTrace(); 1219066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 1229066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 1239066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project} 1249066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 1259066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 1269066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Projectpublic class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestShellActivity> { 1279066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 1289066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private static final String LOGTAG = "LayoutTests"; 1299066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project static final int DEFAULT_TIMEOUT_IN_MILLIS = 5000; 1309066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 1319066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project static final String EXTERNAL_DIR = Environment.getExternalStorageDirectory().toString(); 1329066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project static final String LAYOUT_TESTS_ROOT = EXTERNAL_DIR + "/webkit/layout_tests/"; 1339066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project static final String LAYOUT_TESTS_RESULT_DIR = EXTERNAL_DIR + "/webkit/layout_tests_results/"; 1349066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project static final String ANDROID_EXPECTED_RESULT_DIR = EXTERNAL_DIR + "/webkit/expected_results/"; 1359066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project static final String LAYOUT_TESTS_LIST_FILE = EXTERNAL_DIR + "/webkit/layout_tests_list.txt"; 1369066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project static final String TEST_STATUS_FILE = EXTERNAL_DIR + "/webkit/running_test.txt"; 1379066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project static final String LAYOUT_TESTS_RESULTS_REFERENCE_FILES[] = { 1389066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project "results/layout_tests_passed.txt", 1399066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project "results/layout_tests_failed.txt", 1409066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project "results/layout_tests_nontext.txt", 1419066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project "results/layout_tests_crashed.txt", 1429066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project "run_layout_tests.py" 1439066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project }; 1449066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 1459066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project static final String LAYOUT_RESULTS_FAILED_RESULT_FILE = "results/layout_tests_failed.txt"; 1469066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project static final String LAYOUT_RESULTS_NONTEXT_RESULT_FILE = "results/layout_tests_nontext.txt"; 1479066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project static final String LAYOUT_RESULTS_CRASHED_RESULT_FILE = "results/layout_tests_crashed.txt"; 1489066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project static final String LAYOUT_TESTS_RUNNER = "run_layout_tests.py"; 1499066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 1509066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private MyTestRecorder mResultRecorder; 1519066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private Vector<String> mTestList; 1529066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project // Whether we should ignore the result for the corresponding test. Ordered same as mTestList. 1539066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private Vector<Boolean> mTestListIgnoreResult; 1549066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private boolean mRebaselineResults; 1559066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project // The JavaScript engine currently in use. This determines which set of Android-specific 1569066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project // expected test results we use. 1579066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private String mJsEngine; 1589066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private String mTestPathPrefix; 1599066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private boolean mFinished; 1609066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private int mTestCount; 1619066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private int mResumeIndex; 1629066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 1639066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project public LayoutTestsAutoTest() { 1649066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project super(TestShellActivity.class); 1659066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project } 1669066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project 1679066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project private void getTestList() { 1689066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project // Read test list. 1699066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project try { 1709066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project BufferedReader inReader = new BufferedReader(new FileReader(LAYOUT_TESTS_LIST_FILE)); 1719066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project String line = inReader.readLine(); 1729066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project while (line != null) { 1739066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project if (line.startsWith(mTestPathPrefix)) { 1749066cfe9886ac131c34d59ed0e2d287b0e3c0087The Android Open Source Project String[] components = line.split(" "); 175 mTestList.add(components[0]); 176 mTestListIgnoreResult.add(components.length > 1 && components[1].equals("IGNORE_RESULT")); 177 } 178 line = inReader.readLine(); 179 } 180 inReader.close(); 181 Log.v(LOGTAG, "Test list has " + mTestList.size() + " test(s)."); 182 } catch (Exception e) { 183 Log.e(LOGTAG, "Error while reading test list : " + e.getMessage()); 184 } 185 mTestCount = mTestList.size(); 186 } 187 188 private void resumeTestList() { 189 // read out the test name it stoped last time. 190 try { 191 String line = FsUtils.readTestStatus(TEST_STATUS_FILE); 192 for (int i = 0; i < mTestList.size(); i++) { 193 if (mTestList.elementAt(i).equals(line)) { 194 mTestList = new Vector<String>(mTestList.subList(i+1, mTestList.size())); 195 mTestListIgnoreResult = new Vector<Boolean>(mTestListIgnoreResult.subList(i+1, mTestListIgnoreResult.size())); 196 mResumeIndex = i + 1; 197 break; 198 } 199 } 200 } catch (Exception e) { 201 Log.e(LOGTAG, "Error reading " + TEST_STATUS_FILE); 202 } 203 } 204 205 private void clearTestStatus() { 206 // Delete TEST_STATUS_FILE 207 try { 208 File f = new File(TEST_STATUS_FILE); 209 if (f.delete()) 210 Log.v(LOGTAG, "Deleted " + TEST_STATUS_FILE); 211 else 212 Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE); 213 } catch (Exception e) { 214 Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE + " : " + e.getMessage()); 215 } 216 } 217 218 private String getResultFile(String test) { 219 String shortName = test.substring(0, test.lastIndexOf('.')); 220 // Write actual results to result directory. 221 return shortName.replaceFirst(LAYOUT_TESTS_ROOT, LAYOUT_TESTS_RESULT_DIR) + "-result.txt"; 222 } 223 224 // Gets the file which contains WebKit's expected results for this test. 225 private String getExpectedResultFile(String test) { 226 // The generic result is at <path>/<name>-expected.txt 227 // First try the Android-specific result at 228 // platform/android-<js-engine>/<path>/<name>-expected.txt 229 // then 230 // platform/android/<path>/<name>-expected.txt 231 int pos = test.lastIndexOf('.'); 232 if (pos == -1) 233 return null; 234 String genericExpectedResult = test.substring(0, pos) + "-expected.txt"; 235 String androidExpectedResultsDir = "platform/android-" + mJsEngine + "/"; 236 String androidExpectedResult = genericExpectedResult.replaceFirst(LAYOUT_TESTS_ROOT, 237 LAYOUT_TESTS_ROOT + androidExpectedResultsDir); 238 File f = new File(androidExpectedResult); 239 if (f.exists()) 240 return androidExpectedResult; 241 androidExpectedResultsDir = "platform/android/"; 242 androidExpectedResult = genericExpectedResult.replaceFirst(LAYOUT_TESTS_ROOT, 243 LAYOUT_TESTS_ROOT + androidExpectedResultsDir); 244 f = new File(androidExpectedResult); 245 return f.exists() ? androidExpectedResult : genericExpectedResult; 246 } 247 248 // Gets the file which contains the actual results of running the test on 249 // Android, generated by a previous run which set a new baseline. 250 private String getAndroidExpectedResultFile(String expectedResultFile) { 251 return expectedResultFile.replaceFirst(LAYOUT_TESTS_ROOT, ANDROID_EXPECTED_RESULT_DIR); 252 } 253 254 // Wrap up 255 private void failedCase(String file) { 256 Log.w("Layout test: ", file + " failed"); 257 mResultRecorder.failed(file); 258 } 259 260 private void passedCase(String file) { 261 Log.v("Layout test:", file + " passed"); 262 mResultRecorder.passed(file); 263 } 264 265 private void ignoreResultCase(String file) { 266 Log.v("Layout test:", file + " ignore result"); 267 mResultRecorder.ignoreResult(file); 268 } 269 270 private void noResultCase(String file) { 271 Log.v("Layout test:", file + " no expected result"); 272 mResultRecorder.noResult(file); 273 } 274 275 private void processResult(String testFile, String actualResultFile, String expectedResultFile, boolean ignoreResult) { 276 Log.v(LOGTAG, " Processing result: " + testFile); 277 278 if (ignoreResult) { 279 ignoreResultCase(testFile); 280 return; 281 } 282 283 File actual = new File(actualResultFile); 284 File expected = new File(expectedResultFile); 285 if (actual.exists() && expected.exists()) { 286 try { 287 if (FsUtils.diffIgnoreSpaces(actualResultFile, expectedResultFile)) { 288 passedCase(testFile); 289 } else { 290 failedCase(testFile); 291 } 292 } catch (FileNotFoundException ex) { 293 Log.e(LOGTAG, "File not found : " + ex.getMessage()); 294 } catch (IOException ex) { 295 Log.e(LOGTAG, "IO Error : " + ex.getMessage()); 296 } 297 return; 298 } 299 300 if (!expected.exists()) { 301 noResultCase(testFile); 302 } 303 } 304 305 private void runTestAndWaitUntilDone(TestShellActivity activity, String test, int timeout, boolean ignoreResult, int testNumber) { 306 activity.setCallback(new TestShellCallback() { 307 public void finished() { 308 synchronized (LayoutTestsAutoTest.this) { 309 mFinished = true; 310 LayoutTestsAutoTest.this.notifyAll(); 311 } 312 } 313 314 public void timedOut(String url) { 315 Log.v(LOGTAG, "layout timeout: " + url); 316 } 317 318 @Override 319 public void dumpResult(String webViewDump) { 320 } 321 }); 322 323 String resultFile = getResultFile(test); 324 if (resultFile == null) { 325 // Simply ignore this test. 326 return; 327 } 328 if (mRebaselineResults) { 329 String expectedResultFile = getExpectedResultFile(test); 330 File f = new File(expectedResultFile); 331 if (f.exists()) { 332 return; // don't run test and don't overwrite default tests. 333 } 334 335 resultFile = getAndroidExpectedResultFile(expectedResultFile); 336 } 337 338 mFinished = false; 339 Intent intent = new Intent(Intent.ACTION_VIEW); 340 intent.setClass(activity, TestShellActivity.class); 341 intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP); 342 intent.putExtra(TestShellActivity.TEST_URL, FsUtils.getTestUrl(test)); 343 intent.putExtra(TestShellActivity.RESULT_FILE, resultFile); 344 intent.putExtra(TestShellActivity.TIMEOUT_IN_MILLIS, timeout); 345 intent.putExtra(TestShellActivity.TOTAL_TEST_COUNT, mTestCount); 346 intent.putExtra(TestShellActivity.CURRENT_TEST_NUMBER, testNumber); 347 intent.putExtra(TestShellActivity.STOP_ON_REF_ERROR, true); 348 activity.startActivity(intent); 349 350 // Wait until done. 351 synchronized (this) { 352 while(!mFinished){ 353 try { 354 this.wait(); 355 } catch (InterruptedException e) { } 356 } 357 } 358 359 if (!mRebaselineResults) { 360 String expectedResultFile = getExpectedResultFile(test); 361 File f = new File(expectedResultFile); 362 if (!f.exists()) { 363 expectedResultFile = getAndroidExpectedResultFile(expectedResultFile); 364 } 365 366 processResult(test, resultFile, expectedResultFile, ignoreResult); 367 } 368 } 369 370 // Invokes running of layout tests 371 // and waits till it has finished running. 372 public void executeLayoutTests(boolean resume) { 373 LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation(); 374 // A convenient method to be called by another activity. 375 376 if (runner.mTestPath == null) { 377 Log.e(LOGTAG, "No test specified"); 378 return; 379 } 380 381 this.mTestList = new Vector<String>(); 382 this.mTestListIgnoreResult = new Vector<Boolean>(); 383 384 // Read settings 385 mTestPathPrefix = (new File(LAYOUT_TESTS_ROOT + runner.mTestPath)).getAbsolutePath(); 386 mRebaselineResults = runner.mRebaseline; 387 // V8 is the default JavaScript engine. 388 mJsEngine = runner.mJsEngine == null ? "v8" : runner.mJsEngine; 389 390 int timeout = runner.mTimeoutInMillis; 391 if (timeout <= 0) { 392 timeout = DEFAULT_TIMEOUT_IN_MILLIS; 393 } 394 395 this.mResultRecorder = new MyTestRecorder(resume); 396 397 if (!resume) 398 clearTestStatus(); 399 400 getTestList(); 401 if (resume) 402 resumeTestList(); 403 404 TestShellActivity activity = getActivity(); 405 activity.setDefaultDumpDataType(DumpDataType.EXT_REPR); 406 407 // Run tests. 408 for (int i = 0; i < mTestList.size(); i++) { 409 String s = mTestList.elementAt(i); 410 boolean ignoreResult = mTestListIgnoreResult.elementAt(i); 411 FsUtils.updateTestStatus(TEST_STATUS_FILE, s); 412 // Run tests 413 // i is 0 based, but test count is 1 based so add 1 to i here. 414 runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis, ignoreResult, 415 i + 1 + mResumeIndex); 416 } 417 418 FsUtils.updateTestStatus(TEST_STATUS_FILE, "#DONE"); 419 ForwardService.getForwardService().stopForwardService(); 420 activity.finish(); 421 } 422 423 private String getTestPath() { 424 LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation(); 425 426 String test_path = LAYOUT_TESTS_ROOT; 427 if (runner.mTestPath != null) { 428 test_path += runner.mTestPath; 429 } 430 test_path = new File(test_path).getAbsolutePath(); 431 Log.v("LayoutTestsAutoTest", " Test path : " + test_path); 432 433 return test_path; 434 } 435 436 public void generateTestList() { 437 try { 438 File tests_list = new File(LAYOUT_TESTS_LIST_FILE); 439 BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tests_list, false)); 440 FsUtils.writeLayoutTestListRecursively(bos, getTestPath(), false); // Don't ignore results 441 bos.flush(); 442 bos.close(); 443 } catch (Exception e) { 444 Log.e(LOGTAG, "Error when creating test list: " + e.getMessage()); 445 } 446 } 447 448 // Running all the layout tests at once sometimes 449 // causes the dumprendertree to run out of memory. 450 // So, additional tests are added to run the tests 451 // in chunks. 452 public void startLayoutTests() { 453 try { 454 File tests_list = new File(LAYOUT_TESTS_LIST_FILE); 455 if (!tests_list.exists()) 456 generateTestList(); 457 } catch (Exception e) { 458 e.printStackTrace(); 459 } 460 461 executeLayoutTests(false); 462 } 463 464 public void resumeLayoutTests() { 465 executeLayoutTests(true); 466 } 467 468 public void copyResultsAndRunnerAssetsToCache() { 469 try { 470 Context targetContext = getInstrumentation().getTargetContext(); 471 File cacheDir = targetContext.getCacheDir(); 472 473 for( int i=0; i< LAYOUT_TESTS_RESULTS_REFERENCE_FILES.length; i++) { 474 InputStream in = targetContext.getAssets().open( 475 LAYOUT_TESTS_RESULTS_REFERENCE_FILES[i]); 476 OutputStream out = new FileOutputStream(new File(cacheDir, 477 LAYOUT_TESTS_RESULTS_REFERENCE_FILES[i])); 478 479 byte[] buf = new byte[2048]; 480 int len; 481 482 while ((len = in.read(buf)) >= 0 ) { 483 out.write(buf, 0, len); 484 } 485 out.close(); 486 in.close(); 487 } 488 }catch (IOException e) { 489 e.printStackTrace(); 490 } 491 492 } 493 494} 495