create_nmf_test.py revision c5cede9ae108bb15f6b7a8aea21c7e1fefa2834c 
1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5import json
6import os
7import posixpath
8import shutil
9import subprocess
10import sys
11import tempfile
12import unittest
13
14SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
15TOOLS_DIR = os.path.dirname(SCRIPT_DIR)
16DATA_DIR = os.path.join(TOOLS_DIR, 'lib', 'tests', 'data')
17CHROME_SRC = os.path.dirname(os.path.dirname(os.path.dirname(TOOLS_DIR)))
18MOCK_DIR = os.path.join(CHROME_SRC, "third_party", "pymock")
19
20# For the mock library
21sys.path.append(MOCK_DIR)
22sys.path.append(TOOLS_DIR)
23
24import create_nmf
25import getos
26import mock
27
28
29PosixRelPath = create_nmf.PosixRelPath
30
31
32def StripSo(name):
33  """Strip trailing hexidecimal characters from the name of a shared object.
34
35  It strips everything after the last '.' in the name, and checks that the new
36  name ends with .so.
37
38  e.g.
39
40  libc.so.ad6acbfa => libc.so
41  foo.bar.baz => foo.bar.baz
42  """
43  stripped_name = '.'.join(name.split('.')[:-1])
44  if stripped_name.endswith('.so'):
45    return stripped_name
46  return name
47
48
49class TestPosixRelPath(unittest.TestCase):
50  def testBasic(self):
51    # Note that PosixRelPath only converts from native path format to posix
52    # path format, that's why we have to use os.path.join here.
53    path = os.path.join(os.path.sep, 'foo', 'bar', 'baz.blah')
54    start = os.path.sep + 'foo'
55    self.assertEqual(PosixRelPath(path, start), 'bar/baz.blah')
56
57
58class TestDefaultLibpath(unittest.TestCase):
59  def testWithoutNaClSDKRoot(self):
60    """GetDefaultLibPath wihtout NACL_SDK_ROOT set
61
62    In the absence of NACL_SDK_ROOT GetDefaultLibPath should
63    return the empty list."""
64    with mock.patch.dict('os.environ', clear=True):
65      paths = create_nmf.GetDefaultLibPath('Debug')
66    self.assertEqual(paths, [])
67
68  def testHonorNaClSDKRoot(self):
69    with mock.patch.dict('os.environ', {'NACL_SDK_ROOT': '/dummy/path'}):
70      paths = create_nmf.GetDefaultLibPath('Debug')
71    for path in paths:
72      self.assertTrue(path.startswith('/dummy/path'))
73
74  def testIncludesNaClPorts(self):
75    with mock.patch.dict('os.environ', {'NACL_SDK_ROOT': '/dummy/path'}):
76      paths = create_nmf.GetDefaultLibPath('Debug')
77    self.assertTrue(any(os.path.join('ports', 'lib') in p for p in paths),
78                    "naclports libpath missing: %s" % str(paths))
79
80
81class TestNmfUtils(unittest.TestCase):
82  """Tests for the main NmfUtils class in create_nmf."""
83
84  def setUp(self):
85    self.tempdir = None
86    toolchain = os.path.join(CHROME_SRC, 'native_client', 'toolchain')
87    self.toolchain = os.path.join(toolchain, '%s_x86' % getos.GetPlatform(),
88                                  'nacl_x86_glibc')
89    self.objdump = os.path.join(self.toolchain, 'bin', 'i686-nacl-objdump')
90    if os.name == 'nt':
91      self.objdump += '.exe'
92    self._Mktemp()
93
94  def _CreateTestNexe(self, name, arch):
95    """Create an empty test .nexe file for use in create_nmf tests.
96
97    This is used rather than checking in test binaries since the
98    checked in binaries depend on .so files that only exist in the
99    certain SDK that build them.
100    """
101    compiler = os.path.join(self.toolchain, 'bin', '%s-nacl-g++' % arch)
102    if os.name == 'nt':
103      compiler += '.exe'
104      os.environ['CYGWIN'] = 'nodosfilewarning'
105    program = 'int main() { return 0; }'
106    name = os.path.join(self.tempdir, name)
107    dst_dir = os.path.dirname(name)
108    if not os.path.exists(dst_dir):
109      os.makedirs(dst_dir)
110    cmd = [compiler, '-pthread', '-x' , 'c', '-o', name, '-']
111    p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
112    p.communicate(input=program)
113    self.assertEqual(p.returncode, 0)
114    return name
115
116  def tearDown(self):
117    if self.tempdir:
118      shutil.rmtree(self.tempdir)
119
120  def _Mktemp(self):
121    self.tempdir = tempfile.mkdtemp()
122
123  def _CreateNmfUtils(self, nexes, **kwargs):
124    if not kwargs.get('lib_path'):
125      # Use lib instead of lib64 (lib64 is a symlink to lib).
126      kwargs['lib_path'] = [
127          os.path.join(self.toolchain, 'x86_64-nacl', 'lib'),
128          os.path.join(self.toolchain, 'x86_64-nacl', 'lib32')]
129    return create_nmf.NmfUtils(nexes,
130                               objdump=self.objdump,
131                               **kwargs)
132
133  def _CreateStatic(self, arch_path=None, **kwargs):
134    """Copy all static .nexe files from the DATA_DIR to a temporary directory.
135
136    Args:
137      arch_path: A dictionary mapping architecture to the directory to generate
138          the .nexe for the architecture in.
139      kwargs: Keyword arguments to pass through to create_nmf.NmfUtils
140          constructor.
141
142    Returns:
143      A tuple with 2 elements:
144        * The generated NMF as a dictionary (i.e. parsed by json.loads)
145        * A list of the generated .nexe paths
146    """
147    arch_path = arch_path or {}
148    nexes = []
149    for arch in ('x86_64', 'x86_32', 'arm'):
150      nexe_name = 'test_static_%s.nexe' % arch
151      src_nexe = os.path.join(DATA_DIR, nexe_name)
152      dst_nexe = os.path.join(self.tempdir, arch_path.get(arch, ''), nexe_name)
153      dst_dir = os.path.dirname(dst_nexe)
154      if not os.path.exists(dst_dir):
155        os.makedirs(dst_dir)
156      shutil.copy(src_nexe, dst_nexe)
157      nexes.append(dst_nexe)
158
159    nexes.sort()
160    nmf_utils = self._CreateNmfUtils(nexes, **kwargs)
161    nmf = json.loads(nmf_utils.GetJson())
162    return nmf, nexes
163
164  def _CreateDynamicAndStageDeps(self, arch_path=None, **kwargs):
165    """Create dynamic .nexe files and put them in a temporary directory, with
166    their dependencies staged in the same directory.
167
168    Args:
169      arch_path: A dictionary mapping architecture to the directory to generate
170          the .nexe for the architecture in.
171      kwargs: Keyword arguments to pass through to create_nmf.NmfUtils
172          constructor.
173
174    Returns:
175      A tuple with 2 elements:
176        * The generated NMF as a dictionary (i.e. parsed by json.loads)
177        * A list of the generated .nexe paths
178    """
179    arch_path = arch_path or {}
180    nexes = []
181    for arch in ('x86_64', 'x86_32'):
182      nexe_name = 'test_dynamic_%s.nexe' % arch
183      rel_nexe = os.path.join(arch_path.get(arch, ''), nexe_name)
184      arch_alt = 'i686' if arch == 'x86_32' else arch
185      nexe = self._CreateTestNexe(rel_nexe, arch_alt)
186      nexes.append(nexe)
187
188    nexes.sort()
189    nmf_utils = self._CreateNmfUtils(nexes, **kwargs)
190    nmf = json.loads(nmf_utils.GetJson())
191    nmf_utils.StageDependencies(self.tempdir)
192
193    return nmf, nexes
194
195  def _CreatePexe(self, **kwargs):
196    """Copy test.pexe from the DATA_DIR to a temporary directory.
197
198    Args:
199      kwargs: Keyword arguments to pass through to create_nmf.NmfUtils
200          constructor.
201
202    Returns:
203      A tuple with 2 elements:
204        * The generated NMF as a dictionary (i.e. parsed by json.loads)
205        * A list of the generated .pexe paths
206    """
207    pexe_name = 'test.pexe'
208    src_pexe = os.path.join(DATA_DIR, pexe_name)
209    dst_pexe = os.path.join(self.tempdir, pexe_name)
210    shutil.copy(src_pexe, dst_pexe)
211
212    pexes = [dst_pexe]
213    nmf_utils = self._CreateNmfUtils(pexes, **kwargs)
214    nmf = json.loads(nmf_utils.GetJson())
215
216    return nmf, pexes
217
218  def _CreateBitCode(self, **kwargs):
219    """Copy test.bc from the DATA_DIR to a temporary directory.
220
221    Args:
222      kwargs: Keyword arguments to pass through to create_nmf.NmfUtils
223          constructor.
224
225    Returns:
226      A tuple with 2 elements:
227        * The generated NMF as a dictionary (i.e. parsed by json.loads)
228        * A list of the generated .bc paths
229    """
230    bc_name = 'test.bc'
231    src_bc = os.path.join(DATA_DIR, bc_name)
232    dst_bc = os.path.join(self.tempdir, bc_name)
233    shutil.copy(src_bc, dst_bc)
234
235    bcs = [dst_bc]
236    nmf_utils = self._CreateNmfUtils(bcs, **kwargs)
237    nmf = json.loads(nmf_utils.GetJson())
238
239    return nmf, bcs
240
241  def assertManifestEquals(self, manifest, expected):
242    """Compare two manifest dictionaries.
243
244    The input manifest is regenerated with all string keys and values being
245    processed through StripSo, to remove the random hexidecimal characters at
246    the end of shared object names.
247
248    Args:
249      manifest: The generated manifest.
250      expected: The expected manifest.
251    """
252    def StripSoCopyDict(d):
253      new_d = {}
254      for k, v in d.iteritems():
255        new_k = StripSo(k)
256        if isinstance(v, (str, unicode)):
257          new_v = StripSo(v)
258        elif type(v) is list:
259          new_v = v[:]
260        elif type(v) is dict:
261          new_v = StripSoCopyDict(v)
262        else:
263          # Assume that anything else can be copied directly.
264          new_v = v
265
266        new_d[new_k] = new_v
267      return new_d
268
269    self.assertEqual(StripSoCopyDict(manifest), expected)
270
271  def assertStagingEquals(self, expected):
272    """Compare the contents of the temporary directory, to an expected
273    directory layout.
274
275    Args:
276      expected: The expected directory layout.
277    """
278    all_files = []
279    for root, _, files in os.walk(self.tempdir):
280      rel_root_posix = PosixRelPath(root, self.tempdir)
281      for f in files:
282        path = posixpath.join(rel_root_posix, StripSo(f))
283        if path.startswith('./'):
284          path = path[2:]
285        all_files.append(path)
286    self.assertEqual(set(expected), set(all_files))
287
288
289  def testStatic(self):
290    nmf, _ = self._CreateStatic()
291    expected_manifest = {
292      'files': {},
293      'program': {
294        'x86-64': {'url': 'test_static_x86_64.nexe'},
295        'x86-32': {'url': 'test_static_x86_32.nexe'},
296        'arm': {'url': 'test_static_arm.nexe'},
297      }
298    }
299    self.assertManifestEquals(nmf, expected_manifest)
300
301  def testStaticWithPath(self):
302    arch_dir = {'x86_32': 'x86_32', 'x86_64': 'x86_64', 'arm': 'arm'}
303    nmf, _ = self._CreateStatic(arch_dir, nmf_root=self.tempdir)
304    expected_manifest = {
305      'files': {},
306      'program': {
307        'x86-32': {'url': 'x86_32/test_static_x86_32.nexe'},
308        'x86-64': {'url': 'x86_64/test_static_x86_64.nexe'},
309        'arm': {'url': 'arm/test_static_arm.nexe'},
310      }
311    }
312    self.assertManifestEquals(nmf, expected_manifest)
313
314  def testStaticWithPathNoNmfRoot(self):
315    # This case is not particularly useful, but it is similar to how create_nmf
316    # used to work. If there is no nmf_root given, all paths are relative to
317    # the first nexe passed on the commandline. I believe the assumption
318    # previously was that all .nexes would be in the same directory.
319    arch_dir = {'x86_32': 'x86_32', 'x86_64': 'x86_64', 'arm': 'arm'}
320    nmf, _ = self._CreateStatic(arch_dir)
321    expected_manifest = {
322      'files': {},
323      'program': {
324        'x86-32': {'url': '../x86_32/test_static_x86_32.nexe'},
325        'x86-64': {'url': '../x86_64/test_static_x86_64.nexe'},
326        'arm': {'url': 'test_static_arm.nexe'},
327      }
328    }
329    self.assertManifestEquals(nmf, expected_manifest)
330
331  def testStaticWithNexePrefix(self):
332    nmf, _ = self._CreateStatic(nexe_prefix='foo')
333    expected_manifest = {
334      'files': {},
335      'program': {
336        'x86-64': {'url': 'foo/test_static_x86_64.nexe'},
337        'x86-32': {'url': 'foo/test_static_x86_32.nexe'},
338        'arm': {'url': 'foo/test_static_arm.nexe'},
339      }
340    }
341    self.assertManifestEquals(nmf, expected_manifest)
342
343  def testDynamic(self):
344    nmf, nexes = self._CreateDynamicAndStageDeps()
345    expected_manifest = {
346      'files': {
347        'main.nexe': {
348          'x86-32': {'url': 'test_dynamic_x86_32.nexe'},
349          'x86-64': {'url': 'test_dynamic_x86_64.nexe'},
350        },
351        'libc.so': {
352          'x86-32': {'url': 'lib32/libc.so'},
353          'x86-64': {'url': 'lib64/libc.so'},
354        },
355        'libgcc_s.so': {
356          'x86-32': {'url': 'lib32/libgcc_s.so'},
357          'x86-64': {'url': 'lib64/libgcc_s.so'},
358        },
359        'libpthread.so': {
360          'x86-32': {'url': 'lib32/libpthread.so'},
361          'x86-64': {'url': 'lib64/libpthread.so'},
362        },
363      },
364      'program': {
365        'x86-32': {'url': 'lib32/runnable-ld.so'},
366        'x86-64': {'url': 'lib64/runnable-ld.so'},
367      }
368    }
369
370    expected_staging = [os.path.basename(f) for f in nexes]
371    expected_staging.extend([
372      'lib32/libc.so',
373      'lib32/libgcc_s.so',
374      'lib32/libpthread.so',
375      'lib32/runnable-ld.so',
376      'lib64/libc.so',
377      'lib64/libgcc_s.so',
378      'lib64/libpthread.so',
379      'lib64/runnable-ld.so'])
380
381    self.assertManifestEquals(nmf, expected_manifest)
382    self.assertStagingEquals(expected_staging)
383
384  def testDynamicWithPath(self):
385    arch_dir = {'x86_64': 'x86_64', 'x86_32': 'x86_32'}
386    nmf, nexes = self._CreateDynamicAndStageDeps(arch_dir,
387                                                 nmf_root=self.tempdir)
388    expected_manifest = {
389      'files': {
390        'main.nexe': {
391          'x86-32': {'url': 'x86_32/test_dynamic_x86_32.nexe'},
392          'x86-64': {'url': 'x86_64/test_dynamic_x86_64.nexe'},
393        },
394        'libc.so': {
395          'x86-32': {'url': 'x86_32/lib32/libc.so'},
396          'x86-64': {'url': 'x86_64/lib64/libc.so'},
397        },
398        'libgcc_s.so': {
399          'x86-32': {'url': 'x86_32/lib32/libgcc_s.so'},
400          'x86-64': {'url': 'x86_64/lib64/libgcc_s.so'},
401        },
402        'libpthread.so': {
403          'x86-32': {'url': 'x86_32/lib32/libpthread.so'},
404          'x86-64': {'url': 'x86_64/lib64/libpthread.so'},
405        },
406      },
407      'program': {
408        'x86-32': {'url': 'x86_32/lib32/runnable-ld.so'},
409        'x86-64': {'url': 'x86_64/lib64/runnable-ld.so'},
410      }
411    }
412
413    expected_staging = [PosixRelPath(f, self.tempdir) for f in nexes]
414    expected_staging.extend([
415      'x86_32/lib32/libc.so',
416      'x86_32/lib32/libgcc_s.so',
417      'x86_32/lib32/libpthread.so',
418      'x86_32/lib32/runnable-ld.so',
419      'x86_64/lib64/libc.so',
420      'x86_64/lib64/libgcc_s.so',
421      'x86_64/lib64/libpthread.so',
422      'x86_64/lib64/runnable-ld.so'])
423
424    self.assertManifestEquals(nmf, expected_manifest)
425    self.assertStagingEquals(expected_staging)
426
427  def testDynamicWithRelPath(self):
428    """Test that when the nmf root is a relative path that things work."""
429    arch_dir = {'x86_64': 'x86_64', 'x86_32': 'x86_32'}
430    old_path = os.getcwd()
431    try:
432      os.chdir(self.tempdir)
433      nmf, nexes = self._CreateDynamicAndStageDeps(arch_dir, nmf_root='')
434      expected_manifest = {
435        'files': {
436          'main.nexe': {
437            'x86-32': {'url': 'x86_32/test_dynamic_x86_32.nexe'},
438            'x86-64': {'url': 'x86_64/test_dynamic_x86_64.nexe'},
439          },
440          'libc.so': {
441            'x86-32': {'url': 'x86_32/lib32/libc.so'},
442            'x86-64': {'url': 'x86_64/lib64/libc.so'},
443          },
444          'libgcc_s.so': {
445            'x86-32': {'url': 'x86_32/lib32/libgcc_s.so'},
446            'x86-64': {'url': 'x86_64/lib64/libgcc_s.so'},
447          },
448          'libpthread.so': {
449            'x86-32': {'url': 'x86_32/lib32/libpthread.so'},
450            'x86-64': {'url': 'x86_64/lib64/libpthread.so'},
451          },
452        },
453        'program': {
454          'x86-32': {'url': 'x86_32/lib32/runnable-ld.so'},
455          'x86-64': {'url': 'x86_64/lib64/runnable-ld.so'},
456        }
457      }
458
459      expected_staging = [PosixRelPath(f, self.tempdir) for f in nexes]
460      expected_staging.extend([
461        'x86_32/lib32/libc.so',
462        'x86_32/lib32/libgcc_s.so',
463        'x86_32/lib32/libpthread.so',
464        'x86_32/lib32/runnable-ld.so',
465        'x86_64/lib64/libc.so',
466        'x86_64/lib64/libgcc_s.so',
467        'x86_64/lib64/libpthread.so',
468        'x86_64/lib64/runnable-ld.so'])
469
470      self.assertManifestEquals(nmf, expected_manifest)
471      self.assertStagingEquals(expected_staging)
472    finally:
473      os.chdir(old_path)
474
475  def testDynamicWithPathNoArchPrefix(self):
476    arch_dir = {'x86_64': 'x86_64', 'x86_32': 'x86_32'}
477    nmf, nexes = self._CreateDynamicAndStageDeps(arch_dir,
478                                                 nmf_root=self.tempdir,
479                                                 no_arch_prefix=True)
480    expected_manifest = {
481      'files': {
482        'main.nexe': {
483          'x86-32': {'url': 'x86_32/test_dynamic_x86_32.nexe'},
484          'x86-64': {'url': 'x86_64/test_dynamic_x86_64.nexe'},
485        },
486        'libc.so': {
487          'x86-32': {'url': 'x86_32/libc.so'},
488          'x86-64': {'url': 'x86_64/libc.so'},
489        },
490        'libgcc_s.so': {
491          'x86-32': {'url': 'x86_32/libgcc_s.so'},
492          'x86-64': {'url': 'x86_64/libgcc_s.so'},
493        },
494        'libpthread.so': {
495          'x86-32': {'url': 'x86_32/libpthread.so'},
496          'x86-64': {'url': 'x86_64/libpthread.so'},
497        },
498      },
499      'program': {
500        'x86-32': {'url': 'x86_32/runnable-ld.so'},
501        'x86-64': {'url': 'x86_64/runnable-ld.so'},
502      }
503    }
504
505    expected_staging = [PosixRelPath(f, self.tempdir) for f in nexes]
506    expected_staging.extend([
507      'x86_32/libc.so',
508      'x86_32/libgcc_s.so',
509      'x86_32/libpthread.so',
510      'x86_32/runnable-ld.so',
511      'x86_64/libc.so',
512      'x86_64/libgcc_s.so',
513      'x86_64/libpthread.so',
514      'x86_64/runnable-ld.so'])
515
516    self.assertManifestEquals(nmf, expected_manifest)
517    self.assertStagingEquals(expected_staging)
518
519  def testDynamicWithLibPrefix(self):
520    nmf, nexes = self._CreateDynamicAndStageDeps(lib_prefix='foo')
521    expected_manifest = {
522      'files': {
523        'main.nexe': {
524          'x86-32': {'url': 'test_dynamic_x86_32.nexe'},
525          'x86-64': {'url': 'test_dynamic_x86_64.nexe'},
526        },
527        'libc.so': {
528          'x86-32': {'url': 'foo/lib32/libc.so'},
529          'x86-64': {'url': 'foo/lib64/libc.so'},
530        },
531        'libgcc_s.so': {
532          'x86-32': {'url': 'foo/lib32/libgcc_s.so'},
533          'x86-64': {'url': 'foo/lib64/libgcc_s.so'},
534        },
535        'libpthread.so': {
536          'x86-32': {'url': 'foo/lib32/libpthread.so'},
537          'x86-64': {'url': 'foo/lib64/libpthread.so'},
538        },
539      },
540      'program': {
541        'x86-32': {'url': 'foo/lib32/runnable-ld.so'},
542        'x86-64': {'url': 'foo/lib64/runnable-ld.so'},
543      }
544    }
545
546    expected_staging = [PosixRelPath(f, self.tempdir) for f in nexes]
547    expected_staging.extend([
548      'foo/lib32/libc.so',
549      'foo/lib32/libgcc_s.so',
550      'foo/lib32/libpthread.so',
551      'foo/lib32/runnable-ld.so',
552      'foo/lib64/libc.so',
553      'foo/lib64/libgcc_s.so',
554      'foo/lib64/libpthread.so',
555      'foo/lib64/runnable-ld.so'])
556
557    self.assertManifestEquals(nmf, expected_manifest)
558    self.assertStagingEquals(expected_staging)
559
560  def testPexe(self):
561    nmf, _ = self._CreatePexe()
562    expected_manifest = {
563      'program': {
564        'portable': {
565          'pnacl-translate': {
566            'url': 'test.pexe'
567          }
568        }
569      }
570    }
571    self.assertManifestEquals(nmf, expected_manifest)
572
573  def testPexeOptLevel(self):
574    nmf, _ = self._CreatePexe(pnacl_optlevel=2)
575    expected_manifest = {
576      'program': {
577        'portable': {
578          'pnacl-translate': {
579            'url': 'test.pexe',
580            'optlevel': 2,
581          }
582        }
583      }
584    }
585    self.assertManifestEquals(nmf, expected_manifest)
586
587  def testBitCode(self):
588    nmf, _ = self._CreateBitCode(pnacl_debug_optlevel=0)
589    expected_manifest = {
590      'program': {
591        'portable': {
592          'pnacl-debug': {
593            'url': 'test.bc',
594            'optlevel': 0,
595          }
596        }
597      }
598    }
599    self.assertManifestEquals(nmf, expected_manifest)
600
601
602if __name__ == '__main__':
603  unittest.main()
604