1/*
2 * Copyright (C) 2008 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package com.android.dumprendertree;
18
19import com.android.dumprendertree.TestShellActivity.DumpDataType;
20import com.android.dumprendertree.forwarder.AdbUtils;
21import com.android.dumprendertree.forwarder.ForwardServer;
22import com.android.dumprendertree.forwarder.ForwardService;
23
24import android.app.Instrumentation;
25import android.content.Intent;
26import android.os.Bundle;
27import android.test.ActivityInstrumentationTestCase2;
28import android.util.Log;
29
30import java.io.BufferedOutputStream;
31import java.io.BufferedReader;
32import java.io.File;
33import java.io.FileNotFoundException;
34import java.io.FileOutputStream;
35import java.io.FileReader;
36import java.io.IOException;
37import java.io.InputStream;
38import java.io.OutputStream;
39import java.util.Vector;
40
41// TestRecorder creates four files ...
42// - passing tests
43// - failing tests
44// - tests for which results are ignored
45// - tests with no text results available
46// TestRecorder does not have the ability to clear the results.
47class MyTestRecorder {
48    private BufferedOutputStream mBufferedOutputPassedStream;
49    private BufferedOutputStream mBufferedOutputFailedStream;
50    private BufferedOutputStream mBufferedOutputIgnoreResultStream;
51    private BufferedOutputStream mBufferedOutputNoResultStream;
52
53    public void passed(String layout_file) {
54        try {
55            mBufferedOutputPassedStream.write(layout_file.getBytes());
56            mBufferedOutputPassedStream.write('\n');
57            mBufferedOutputPassedStream.flush();
58        } catch(Exception e) {
59            e.printStackTrace();
60        }
61    }
62
63    public void failed(String layout_file) {
64        try {
65            mBufferedOutputFailedStream.write(layout_file.getBytes());
66            mBufferedOutputFailedStream.write('\n');
67            mBufferedOutputFailedStream.flush();
68        } catch(Exception e) {
69            e.printStackTrace();
70        }
71    }
72
73    public void ignoreResult(String layout_file) {
74        try {
75            mBufferedOutputIgnoreResultStream.write(layout_file.getBytes());
76            mBufferedOutputIgnoreResultStream.write('\n');
77            mBufferedOutputIgnoreResultStream.flush();
78        } catch(Exception e) {
79            e.printStackTrace();
80        }
81    }
82
83    public void noResult(String layout_file) {
84        try {
85            mBufferedOutputNoResultStream.write(layout_file.getBytes());
86            mBufferedOutputNoResultStream.write('\n');
87            mBufferedOutputNoResultStream.flush();
88        } catch(Exception e) {
89            e.printStackTrace();
90        }
91    }
92
93    public MyTestRecorder(boolean resume) {
94        try {
95            File resultsPassedFile = new File("/sdcard/layout_tests_passed.txt");
96            File resultsFailedFile = new File("/sdcard/layout_tests_failed.txt");
97            File resultsIgnoreResultFile = new File("/sdcard/layout_tests_ignored.txt");
98            File noExpectedResultFile = new File("/sdcard/layout_tests_nontext.txt");
99
100            mBufferedOutputPassedStream =
101                new BufferedOutputStream(new FileOutputStream(resultsPassedFile, resume));
102            mBufferedOutputFailedStream =
103                new BufferedOutputStream(new FileOutputStream(resultsFailedFile, resume));
104            mBufferedOutputIgnoreResultStream =
105                new BufferedOutputStream(new FileOutputStream(resultsIgnoreResultFile, resume));
106            mBufferedOutputNoResultStream =
107                new BufferedOutputStream(new FileOutputStream(noExpectedResultFile, resume));
108        } catch (Exception e) {
109            e.printStackTrace();
110        }
111    }
112
113    public void close() {
114        try {
115            mBufferedOutputPassedStream.close();
116            mBufferedOutputFailedStream.close();
117            mBufferedOutputIgnoreResultStream.close();
118            mBufferedOutputNoResultStream.close();
119        } catch (Exception e) {
120            e.printStackTrace();
121        }
122    }
123}
124
125
126public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestShellActivity> {
127
128    private static final String LOGTAG = "LayoutTests";
129    static final int DEFAULT_TIMEOUT_IN_MILLIS = 5000;
130
131    static final String LAYOUT_TESTS_ROOT = "/sdcard/android/layout_tests/";
132    static final String LAYOUT_TESTS_RESULT_DIR = "/sdcard/android/layout_tests_results/";
133    static final String ANDROID_EXPECTED_RESULT_DIR = "/sdcard/android/expected_results/";
134    static final String LAYOUT_TESTS_LIST_FILE = "/sdcard/android/layout_tests_list.txt";
135    static final String TEST_STATUS_FILE = "/sdcard/android/running_test.txt";
136    static final String LAYOUT_TESTS_RESULTS_REFERENCE_FILES[] = {
137          "results/layout_tests_passed.txt",
138          "results/layout_tests_failed.txt",
139          "results/layout_tests_nontext.txt",
140          "results/layout_tests_crashed.txt",
141          "run_layout_tests.py"
142    };
143
144    static final String LAYOUT_RESULTS_FAILED_RESULT_FILE = "results/layout_tests_failed.txt";
145    static final String LAYOUT_RESULTS_NONTEXT_RESULT_FILE = "results/layout_tests_nontext.txt";
146    static final String LAYOUT_RESULTS_CRASHED_RESULT_FILE = "results/layout_tests_crashed.txt";
147    static final String LAYOUT_TESTS_RUNNER = "run_layout_tests.py";
148
149    private MyTestRecorder mResultRecorder;
150    private Vector<String> mTestList;
151    // Whether we should ignore the result for the corresponding test. Ordered same as mTestList.
152    private Vector<Boolean> mTestListIgnoreResult;
153    private boolean mRebaselineResults;
154    // The JavaScript engine currently in use. This determines which set of Android-specific
155    // expected test results we use.
156    private String mJsEngine;
157    private String mTestPathPrefix;
158    private boolean mFinished;
159
160    public LayoutTestsAutoTest() {
161      super("com.android.dumprendertree", TestShellActivity.class);
162    }
163
164    // This function writes the result of the layout test to
165    // Am status so that it can be picked up from a script.
166    private void passOrFailCallback(String file, boolean result) {
167      Instrumentation inst = getInstrumentation();
168      Bundle bundle = new Bundle();
169      bundle.putBoolean(file, result);
170      inst.sendStatus(0, bundle);
171    }
172
173    private void getTestList() {
174        // Read test list.
175        try {
176            BufferedReader inReader = new BufferedReader(new FileReader(LAYOUT_TESTS_LIST_FILE));
177            String line = inReader.readLine();
178            while (line != null) {
179                if (line.startsWith(mTestPathPrefix)) {
180                    String[] components = line.split(" ");
181                    mTestList.add(components[0]);
182                    mTestListIgnoreResult.add(components.length > 1 && components[1].equals("IGNORE_RESULT"));
183                }
184                line = inReader.readLine();
185            }
186            inReader.close();
187            Log.v(LOGTAG, "Test list has " + mTestList.size() + " test(s).");
188        } catch (Exception e) {
189            Log.e(LOGTAG, "Error while reading test list : " + e.getMessage());
190        }
191    }
192
193    private void resumeTestList() {
194        // read out the test name it stoped last time.
195        try {
196            String line = FsUtils.readTestStatus(TEST_STATUS_FILE);
197            for (int i = 0; i < mTestList.size(); i++) {
198                if (mTestList.elementAt(i).equals(line)) {
199                    mTestList = new Vector<String>(mTestList.subList(i+1, mTestList.size()));
200                    mTestListIgnoreResult = new Vector<Boolean>(mTestListIgnoreResult.subList(i+1, mTestListIgnoreResult.size()));
201                    break;
202                }
203            }
204        } catch (Exception e) {
205            Log.e(LOGTAG, "Error reading " + TEST_STATUS_FILE);
206        }
207    }
208
209    private void clearTestStatus() {
210        // Delete TEST_STATUS_FILE
211        try {
212            File f = new File(TEST_STATUS_FILE);
213            if (f.delete())
214                Log.v(LOGTAG, "Deleted " + TEST_STATUS_FILE);
215            else
216                Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE);
217        } catch (Exception e) {
218            Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE + " : " + e.getMessage());
219        }
220    }
221
222    private String getResultFile(String test) {
223        String shortName = test.substring(0, test.lastIndexOf('.'));
224        // Write actual results to result directory.
225        return shortName.replaceFirst(LAYOUT_TESTS_ROOT, LAYOUT_TESTS_RESULT_DIR) + "-result.txt";
226    }
227
228    // Gets the file which contains WebKit's expected results for this test.
229    private String getExpectedResultFile(String test) {
230        // The generic result is at <path>/<name>-expected.txt
231        // First try the Android-specific result at
232        // platform/android-<js-engine>/<path>/<name>-expected.txt
233        int pos = test.lastIndexOf('.');
234        if (pos == -1)
235            return null;
236        String genericExpectedResult = test.substring(0, pos) + "-expected.txt";
237        String androidExpectedResultsDir = "platform/android-" + mJsEngine + "/";
238        String androidExpectedResult =
239            genericExpectedResult.replaceFirst(LAYOUT_TESTS_ROOT, LAYOUT_TESTS_ROOT + androidExpectedResultsDir);
240        File f = new File(androidExpectedResult);
241        return f.exists() ? androidExpectedResult : genericExpectedResult;
242    }
243
244    // Gets the file which contains the actual results of running the test on
245    // Android, generated by a previous run which set a new baseline.
246    private String getAndroidExpectedResultFile(String expectedResultFile) {
247        return expectedResultFile.replaceFirst(LAYOUT_TESTS_ROOT, ANDROID_EXPECTED_RESULT_DIR);
248    }
249
250    // Wrap up
251    private void failedCase(String file) {
252        Log.w("Layout test: ", file + " failed");
253        mResultRecorder.failed(file);
254    }
255
256    private void passedCase(String file) {
257        Log.v("Layout test:", file + " passed");
258        mResultRecorder.passed(file);
259    }
260
261    private void ignoreResultCase(String file) {
262        Log.v("Layout test:", file + " ignore result");
263        mResultRecorder.ignoreResult(file);
264    }
265
266    private void noResultCase(String file) {
267        Log.v("Layout test:", file + " no expected result");
268        mResultRecorder.noResult(file);
269    }
270
271    private void processResult(String testFile, String actualResultFile, String expectedResultFile, boolean ignoreResult) {
272        Log.v(LOGTAG, "  Processing result: " + testFile);
273
274        if (ignoreResult) {
275            ignoreResultCase(testFile);
276            return;
277        }
278
279        File actual = new File(actualResultFile);
280        File expected = new File(expectedResultFile);
281        if (actual.exists() && expected.exists()) {
282            try {
283                if (FsUtils.diffIgnoreSpaces(actualResultFile, expectedResultFile)) {
284                    passedCase(testFile);
285                } else {
286                    failedCase(testFile);
287                }
288            } catch (FileNotFoundException ex) {
289                Log.e(LOGTAG, "File not found : " + ex.getMessage());
290            } catch (IOException ex) {
291                Log.e(LOGTAG, "IO Error : " + ex.getMessage());
292            }
293            return;
294        }
295
296        if (!expected.exists()) {
297            noResultCase(testFile);
298        }
299    }
300
301    private void runTestAndWaitUntilDone(TestShellActivity activity, String test, int timeout, boolean ignoreResult) {
302        activity.setCallback(new TestShellCallback() {
303            public void finished() {
304                synchronized (LayoutTestsAutoTest.this) {
305                    mFinished = true;
306                    LayoutTestsAutoTest.this.notifyAll();
307                }
308            }
309
310            public void timedOut(String url) {
311                Log.v(LOGTAG, "layout timeout: " + url);
312            }
313        });
314
315        String resultFile = getResultFile(test);
316        if (resultFile == null) {
317            // Simply ignore this test.
318            return;
319        }
320        if (mRebaselineResults) {
321            String expectedResultFile = getExpectedResultFile(test);
322            File f = new File(expectedResultFile);
323            if (f.exists()) {
324                return;  // don't run test and don't overwrite default tests.
325            }
326
327            resultFile = getAndroidExpectedResultFile(expectedResultFile);
328        }
329
330        mFinished = false;
331        Intent intent = new Intent(Intent.ACTION_VIEW);
332        intent.setClass(activity, TestShellActivity.class);
333        intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP);
334        intent.putExtra(TestShellActivity.TEST_URL, FsUtils.getTestUrl(test));
335        intent.putExtra(TestShellActivity.RESULT_FILE, resultFile);
336        intent.putExtra(TestShellActivity.TIMEOUT_IN_MILLIS, timeout);
337        activity.startActivity(intent);
338
339        // Wait until done.
340        synchronized (this) {
341            while(!mFinished){
342                try {
343                    this.wait();
344                } catch (InterruptedException e) { }
345            }
346        }
347
348        if (!mRebaselineResults) {
349            String expectedResultFile = getExpectedResultFile(test);
350            File f = new File(expectedResultFile);
351            if (!f.exists()) {
352                expectedResultFile = getAndroidExpectedResultFile(expectedResultFile);
353            }
354
355            processResult(test, resultFile, expectedResultFile, ignoreResult);
356        }
357    }
358
359    // Invokes running of layout tests
360    // and waits till it has finished running.
361    public void executeLayoutTests(boolean resume) {
362        LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation();
363        // A convenient method to be called by another activity.
364
365        if (runner.mTestPath == null) {
366            Log.e(LOGTAG, "No test specified");
367            return;
368        }
369
370        this.mTestList = new Vector<String>();
371        this.mTestListIgnoreResult = new Vector<Boolean>();
372
373        // Read settings
374        mTestPathPrefix = (new File(LAYOUT_TESTS_ROOT + runner.mTestPath)).getAbsolutePath();
375        mRebaselineResults = runner.mRebaseline;
376        // JSC is the default JavaScript engine.
377        mJsEngine = runner.mJsEngine == null ? "jsc" : runner.mJsEngine;
378
379        int timeout = runner.mTimeoutInMillis;
380        if (timeout <= 0) {
381            timeout = DEFAULT_TIMEOUT_IN_MILLIS;
382        }
383
384        this.mResultRecorder = new MyTestRecorder(resume);
385
386        if (!resume)
387            clearTestStatus();
388
389        getTestList();
390        if (resume)
391            resumeTestList();
392
393        TestShellActivity activity = getActivity();
394        activity.setDefaultDumpDataType(DumpDataType.DUMP_AS_TEXT);
395
396        // Run tests.
397        int addr = -1;
398        try{
399            addr = AdbUtils.resolve("android-browser-test.mtv.corp.google.com");
400        } catch (IOException ioe) {
401            Log.w(LOGTAG, "error while resolving test host name", ioe);
402        }
403        if(addr == -1) {
404            Log.w(LOGTAG, "failed to resolve test host. http tests will fail.");
405        }
406        for (int i = 0; i < mTestList.size(); i++) {
407            String s = mTestList.elementAt(i);
408            boolean ignoreResult = mTestListIgnoreResult.elementAt(i);
409            FsUtils.updateTestStatus(TEST_STATUS_FILE, s);
410            // Run tests
411            runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis, ignoreResult);
412        }
413
414        FsUtils.updateTestStatus(TEST_STATUS_FILE, "#DONE");
415        ForwardService.getForwardService().stopForwardService();
416        activity.finish();
417    }
418
419    private String getTestPath() {
420        LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation();
421
422        String test_path = LAYOUT_TESTS_ROOT;
423        if (runner.mTestPath != null) {
424            test_path += runner.mTestPath;
425        }
426        test_path = new File(test_path).getAbsolutePath();
427        Log.v("LayoutTestsAutoTest", " Test path : " + test_path);
428
429        return test_path;
430    }
431
432    public void generateTestList() {
433        try {
434            File tests_list = new File(LAYOUT_TESTS_LIST_FILE);
435            BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tests_list, false));
436            FsUtils.findLayoutTestsRecursively(bos, getTestPath(), false); // Don't ignore results
437            bos.flush();
438            bos.close();
439       } catch (Exception e) {
440           Log.e(LOGTAG, "Error when creating test list: " + e.getMessage());
441       }
442    }
443
444    // Running all the layout tests at once sometimes
445    // causes the dumprendertree to run out of memory.
446    // So, additional tests are added to run the tests
447    // in chunks.
448    public void startLayoutTests() {
449        try {
450            File tests_list = new File(LAYOUT_TESTS_LIST_FILE);
451            if (!tests_list.exists())
452              generateTestList();
453        } catch (Exception e) {
454            e.printStackTrace();
455        }
456
457        executeLayoutTests(false);
458    }
459
460    public void resumeLayoutTests() {
461        executeLayoutTests(true);
462    }
463
464    public void copyResultsAndRunnerAssetsToCache() {
465        try {
466            String out_dir = getActivity().getApplicationContext().getCacheDir().getPath() + "/";
467
468            for( int i=0; i< LAYOUT_TESTS_RESULTS_REFERENCE_FILES.length; i++) {
469                InputStream in = getActivity().getAssets().open(LAYOUT_TESTS_RESULTS_REFERENCE_FILES[i]);
470                OutputStream out = new FileOutputStream(out_dir + LAYOUT_TESTS_RESULTS_REFERENCE_FILES[i]);
471
472                byte[] buf = new byte[2048];
473                int len;
474
475                while ((len = in.read(buf)) >= 0 ) {
476                    out.write(buf, 0, len);
477                }
478                out.close();
479                in.close();
480            }
481        }catch (IOException e) {
482          e.printStackTrace();
483        }
484
485    }
486
487}
488