1#!/usr/bin/python2.7
2
3# Copyright 2010, The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17
18"""RenderScript Compiler Test.
19
20Runs subdirectories of tests for the RenderScript compiler.
21"""
22
23import filecmp
24import glob
25import os
26import re
27import shutil
28import subprocess
29import sys
30import unittest
31
32
33__author__ = 'Android'
34
35
36DOTTED_LINE = '................'
37
38
39class SlangTests(unittest.TestCase):
40  """Class to contain all the unittest test cases.
41
42  Tests will be dynamically added to this class as methods.
43  No static tests, so this class is initially empty.
44  See GenerateSlangTest() and AddSlangUnitTests().
45
46  """
47  pass
48
49
50def GenerateSlangTest(dir_name):
51  """Creates a test method that can be added as method to SlangTests."""
52  cwd = os.getcwd()
53  def SlangTest(self):
54    os.chdir(cwd)
55    ExecTest(dir_name, self)
56  return SlangTest
57
58
59def AddSlangUnitTests(test_dirs):
60  """Adds a test to SlangTests for each directory in test_dirs."""
61
62  for t in test_dirs:
63    # Must start with 'test_' according to unittest
64    test_name = 'test_%s' % t
65    test = GenerateSlangTest(t)
66    # Add test as method to SlangTests with test_name as method name
67    setattr(SlangTests, test_name, test)
68
69
70class Options(object):
71  verbose = 0
72  cleanup = 1
73  update_cts = 0
74  zero_return = 0
75
76
77def CompareFiles(actual, expect):
78  """Compares actual and expect for equality."""
79  if not os.path.isfile(actual):
80    if Options.verbose:
81      print 'Could not find %s' % actual
82    return False
83  if not os.path.isfile(expect):
84    if Options.verbose:
85      print 'Could not find %s' % expect
86    return False
87
88  return filecmp.cmp(actual, expect, False)
89
90
91def CopyIfDifferent(src, dst):
92  """Updates dst if it is different from src."""
93  if not CompareFiles(src, dst):
94    if Options.verbose:
95      print 'Copying from %s to %s' % (src, dst)
96    shutil.copyfile(src, dst)
97
98
99def GetCommandLineArgs(filename):
100  """Extracts command line arguments from first comment line in a file."""
101  f = open(filename, 'r')
102  line = f.readline()
103  f.close()
104  if line[0] == '/' and line[1] == '/':
105    return line[2:].strip()
106  else:
107    return ''
108
109
110def ReadFileToStr(filename):
111  """Returns contents of file as a str."""
112  with open(filename, 'r') as f:
113    return f.read()
114
115
116def ReportIfDifferFromExpected(tests, name, file1, file2):
117  """Fails tests if file1 and file2 differ."""
118  if not CompareFiles(file1, file2):
119    if Options.verbose:
120      err_message = ('%s is different:\n'
121                     'expected:\n%s\n%s%s\n\n'
122                     'actual:\n%s\n%s%s\n') % (
123                         name,
124                         DOTTED_LINE, ReadFileToStr(file1), DOTTED_LINE,
125                         DOTTED_LINE, ReadFileToStr(file2), DOTTED_LINE)
126    else:
127      err_message = '%s is different' % name
128    tests.fail(err_message)
129
130
131def GetRSFiles():
132  """Returns a list of files in cwd with extension '.rs' or '.fs'."""
133  rs_files = glob.glob('*.rs')
134  fs_files = glob.glob('*.fs')
135  rs_files += fs_files
136  rs_files.sort()
137  return rs_files
138
139
140def GetOutDir():
141  """Returns the directory with llvm-rs-cc."""
142  # If cache has not yet been calculated, do that
143  if GetOutDir.cache is None:
144    try:
145      # ANDROID_HOST_OUT is set after `lunch` on local builds
146      GetOutDir.cache = os.environ['ANDROID_HOST_OUT']
147    except KeyError:
148      # On build server, we need to get the HOST_OUT Makefile variable
149      # because ANDROID_HOST_OUT is not exported on build server
150      GetOutDir.cache = subprocess.check_output(['bash', '-c',
151                                         'cd ../../../../.. ; '
152                                         'source build/envsetup.sh '
153                                         '1> /dev/null 2> /dev/null ; '
154                                         'realpath `get_build_var HOST_OUT`'])
155      GetOutDir.cache = GetOutDir.cache.strip()
156  return GetOutDir.cache
157
158
159# Declare/define cache variable for GetOutDir to cache results
160# This way we only need to call subprocesses once to get the directory
161GetOutDir.cache = None
162
163
164def CreateCmd():
165  """Creates the test command to run for the current test."""
166  cmd_string = ('%s/bin/llvm-rs-cc -o tmp/ -p tmp/ -MD '
167                '-I ../../../../../frameworks/rs/script_api/include/ '
168                '-I ../../../../../external/clang/lib/Headers/') % GetOutDir()
169  base_args = cmd_string.split()
170  rs_files = GetRSFiles()
171
172  # Extra command line arguments can be placed as // comments at the start of
173  # any .rs file. We automatically bundle up all of these extra args and invoke
174  # llvm-rs-cc with them.
175  extra_args_str = ''
176  for rs_file in rs_files:
177    extra_args_str += GetCommandLineArgs(rs_file)
178  extra_args = extra_args_str.split()
179
180  args = base_args + extra_args + rs_files
181  return args
182
183
184def UpdateCTS():
185  """Copies resulting files to appropriate CTS directory (if different)."""
186  if glob.glob('IN_CTS'):
187    cts_path = '../../../../../cts/'
188    cts_res_raw_path = cts_path + 'tests/tests/renderscriptlegacy/res/raw/'
189    cts_src_path = cts_path + 'tests/tests/renderscript/src/'
190    for bc_src in glob.glob('tmp/*.bc'):
191      bc_dst = re.sub(r'tmp\/', cts_res_raw_path, bc_src, 1)
192      CopyIfDifferent(bc_src, bc_dst)
193    for java_src in glob.glob('tmp/android/renderscript/cts/*.java'):
194      java_dst = re.sub(r'tmp\/', cts_src_path, java_src, 1)
195      CopyIfDifferent(java_src, java_dst)
196
197
198def Cleanup():
199  """Cleans up the cwd of any tmp files created in current test."""
200  try:
201    os.remove('stdout.txt')
202    os.remove('stderr.txt')
203    shutil.rmtree('tmp/')
204  except OSError:
205    pass
206
207
208def CheckTestResult(dir_name, subprocess_ret, tests, args):
209  """Checks the result of the subprocess command to see if it passed/failed.
210
211  If dir_name starts with 'F_', then subprocess is expected to fail.
212  If it instead succeeded, then this test is failed.
213  Vice versa with a dir_name starting with 'P_'.
214
215  Args:
216    dir_name: name of current directory/test name
217    subprocess_ret: return code of subprocess
218    tests: unittest, call tests.fail(reason) when failure
219    args: the arguments for the command that was run
220  """
221  if dir_name[0:2] == 'F_':
222    if subprocess_ret == 0:
223      if Options.verbose:
224        err_message = ('Command (%s) passed on invalid input\n'
225                       'stdout:\n%s\n%s%s\n') % (
226                           ' '.join(args),
227                           DOTTED_LINE, ReadFileToStr('stdout.txt'), DOTTED_LINE
228                       )
229      else:
230        err_message = 'Command passed on invalid input'
231      tests.fail(err_message)
232  elif dir_name[0:2] == 'P_':
233    if subprocess_ret != 0:
234      if Options.verbose:
235        err_message = ('Command (%s) failed on valid input\n'
236                       'stderr:\n%s\n%s%s\n') % (
237                           ' '.join(args),
238                           DOTTED_LINE, ReadFileToStr('stderr.txt'), DOTTED_LINE
239                       )
240      else:
241        err_message = 'Command failed on valid input'
242      tests.fail(err_message)
243  else:
244    tests.fail('Invalid test name: ' + dir_name +
245               ', should start with F_ or P_')
246
247
248def CheckJavaOutput(tests):
249  """Check that the Java output files are as expected.
250
251  Each 'Script*.java.expect' file should have exactly one corresponding file.
252  The two files should match exactly.
253
254  Args:
255    tests: unittest, call tests.fail(reason) when failure
256  """
257  java_expect = glob.glob('Script*.java.expect')
258  for expect in java_expect:
259    expect_base = expect[:-7]  # strip ".expect" suffix
260    find = 'tmp/*/' + expect_base
261    found = glob.glob(find)
262    if len(found) != 1:
263      if not found:
264        tests.fail('%s not found' % find)
265      else:
266        tests.fail('multiple %s found' % find)
267    elif not CompareFiles(found[0], expect):
268      tests.fail('%s and %s are different' % (found[0], expect))
269
270
271def ExecTest(dir_name, tests):
272  """Executes an llvm-rs-cc test from dir_name."""
273
274  os.chdir(dir_name)
275  stdout_file = open('stdout.txt', 'w+')
276  stderr_file = open('stderr.txt', 'w+')
277
278  args = CreateCmd()
279
280  if Options.verbose > 1:
281    print 'Executing:', ' '.join(args)
282
283  # Execute the command and check the resulting shell return value.
284  # All tests that are expected to FAIL have directory names that
285  # start with 'F_'. Other tests that are expected to PASS have
286  # directory names that start with 'P_'.
287  ret = 0
288  try:
289    ret = subprocess.call(args, stdout=stdout_file, stderr=stderr_file)
290  except OSError:
291    tests.fail('subprocess.call failed: ' + ' '.join(args))
292
293  stdout_file.close()
294  stderr_file.close()
295
296  CheckTestResult(dir_name, ret, tests, args)
297
298  ReportIfDifferFromExpected(tests, 'stdout', 'stdout.txt.expect', 'stdout.txt')
299  ReportIfDifferFromExpected(tests, 'stderr', 'stderr.txt.expect', 'stderr.txt')
300
301  CheckJavaOutput(tests)
302
303  if Options.update_cts:
304    UpdateCTS()
305
306  if Options.cleanup:
307    Cleanup()
308
309
310def Usage():
311  """Print out usage information."""
312  print ('Usage: %s [OPTION]... [TESTNAME]...'
313         'Renderscript Compiler Test Harness\n'
314         'Runs TESTNAMEs (all tests by default)\n'
315         'Available Options:\n'
316         '  -h, --help          Help message\n'
317         '  -n, --no-cleanup    Don\'t clean up after running tests\n'
318         '  -u, --update-cts    Update CTS test versions\n'
319         '  -v, --verbose       Verbose output.  Enter multiple -v to get more verbose.\n'
320         '  -z, --zero-return   Return 0 as exit code no matter if tests fail. Required for TreeHugger.\n'
321        ) % (sys.argv[0]),
322  return
323
324
325def main():
326  """Runs the unittest suite.
327
328  Parses command line arguments, adds test directories as tests.
329
330  Returns:
331    0 if '-z' flag is set.
332    Else unittest.main() returns with its own error code.
333  """
334
335  # Chdir to the directory this file is in since tests are in this directory
336  os.chdir(os.path.dirname(os.path.abspath(__file__)))
337  files = []
338  for arg in sys.argv[1:]:
339    if arg in ('-h', '--help'):
340      Usage()
341      return 0
342    elif arg in ('-n', '--no-cleanup'):
343      Options.cleanup = 0
344    elif arg in ('-u', '--update-cts'):
345      Options.update_cts = 1
346    elif arg in ('-v', '--verbose'):
347      Options.verbose += 1
348    elif arg in ('-z', '--zero-return'):
349      Options.zero_return = 1
350    else:
351      # Test list to run
352      if os.path.isdir(arg):
353        files.append(arg)
354      else:
355        print >> sys.stderr, 'Invalid test or option: %s' % arg
356        return 1
357
358  if not files:
359    file_names = os.listdir('.')
360    # Test names must start with 'F_' or 'P_'
361    # 'F_' tests are expected to fail
362    # 'P_' tests are expected to pass
363    for f in file_names:
364      if os.path.isdir(f) and (f[0:2] == 'F_' or f[0:2] == 'P_'):
365        files.append(f)
366    files.sort()
367
368  AddSlangUnitTests(files)
369
370  # verbosity=2 is necessary for PythonUnitTestRunner to parse the results
371  # Otherwise verbosity does not matter
372  # If Options.zero_return is set, do not let unittest.main() exit
373  #  This is necessary in TreeHugger to distinguish between failing tests and
374  #  failing to execute the python script
375  # If Options.zero_return is not set, let unittest.main() exit
376  #  In this case it will return a non-zero code if any tests fail
377  unittest_exit = Options.zero_return == 0
378  unittest.main(verbosity=2,
379                argv=[sys.argv[0]] + ['SlangTests'],
380                exit=unittest_exit)
381
382  return 0
383
384
385if __name__ == '__main__':
386  sys.exit(main())
387
388