1#!/usr/bin/python
2# Copyright 2016 the V8 project authors. All rights reserved.
3# Copyright 2015 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Tests for mb.py."""
8
9import json
10import StringIO
11import os
12import sys
13import unittest
14
15import mb
16
17
18class FakeMBW(mb.MetaBuildWrapper):
19  def __init__(self, win32=False):
20    super(FakeMBW, self).__init__()
21
22    # Override vars for test portability.
23    if win32:
24      self.chromium_src_dir = 'c:\\fake_src'
25      self.default_config = 'c:\\fake_src\\tools\\mb\\mb_config.pyl'
26      self.platform = 'win32'
27      self.executable = 'c:\\python\\python.exe'
28      self.sep = '\\'
29    else:
30      self.chromium_src_dir = '/fake_src'
31      self.default_config = '/fake_src/tools/mb/mb_config.pyl'
32      self.executable = '/usr/bin/python'
33      self.platform = 'linux2'
34      self.sep = '/'
35
36    self.files = {}
37    self.calls = []
38    self.cmds = []
39    self.cross_compile = None
40    self.out = ''
41    self.err = ''
42    self.rmdirs = []
43
44  def ExpandUser(self, path):
45    return '$HOME/%s' % path
46
47  def Exists(self, path):
48    return self.files.get(path) is not None
49
50  def MaybeMakeDirectory(self, path):
51    self.files[path] = True
52
53  def PathJoin(self, *comps):
54    return self.sep.join(comps)
55
56  def ReadFile(self, path):
57    return self.files[path]
58
59  def WriteFile(self, path, contents, force_verbose=False):
60    if self.args.dryrun or self.args.verbose or force_verbose:
61      self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path))
62    self.files[path] = contents
63
64  def Call(self, cmd, env=None, buffer_output=True):
65    if env:
66      self.cross_compile = env.get('GYP_CROSSCOMPILE')
67    self.calls.append(cmd)
68    if self.cmds:
69      return self.cmds.pop(0)
70    return 0, '', ''
71
72  def Print(self, *args, **kwargs):
73    sep = kwargs.get('sep', ' ')
74    end = kwargs.get('end', '\n')
75    f = kwargs.get('file', sys.stdout)
76    if f == sys.stderr:
77      self.err += sep.join(args) + end
78    else:
79      self.out += sep.join(args) + end
80
81  def TempFile(self, mode='w'):
82    return FakeFile(self.files)
83
84  def RemoveFile(self, path):
85    del self.files[path]
86
87  def RemoveDirectory(self, path):
88    self.rmdirs.append(path)
89    files_to_delete = [f for f in self.files if f.startswith(path)]
90    for f in files_to_delete:
91      self.files[f] = None
92
93
94class FakeFile(object):
95  def __init__(self, files):
96    self.name = '/tmp/file'
97    self.buf = ''
98    self.files = files
99
100  def write(self, contents):
101    self.buf += contents
102
103  def close(self):
104     self.files[self.name] = self.buf
105
106
107TEST_CONFIG = """\
108{
109  'masters': {
110    'chromium': {},
111    'fake_master': {
112      'fake_builder': 'gyp_rel_bot',
113      'fake_gn_builder': 'gn_rel_bot',
114      'fake_gyp_crosscompile_builder': 'gyp_crosscompile',
115      'fake_gn_debug_builder': 'gn_debug_goma',
116      'fake_gyp_builder': 'gyp_debug',
117      'fake_gn_args_bot': '//build/args/bots/fake_master/fake_gn_args_bot.gn',
118      'fake_multi_phase': ['gn_phase_1', 'gn_phase_2'],
119    },
120  },
121  'configs': {
122    'gyp_rel_bot': ['gyp', 'rel', 'goma'],
123    'gn_debug_goma': ['gn', 'debug', 'goma'],
124    'gyp_debug': ['gyp', 'debug', 'fake_feature1'],
125    'gn_rel_bot': ['gn', 'rel', 'goma'],
126    'gyp_crosscompile': ['gyp', 'crosscompile'],
127    'gn_phase_1': ['gn', 'phase_1'],
128    'gn_phase_2': ['gn', 'phase_2'],
129  },
130  'mixins': {
131    'crosscompile': {
132      'gyp_crosscompile': True,
133    },
134    'fake_feature1': {
135      'gn_args': 'enable_doom_melon=true',
136      'gyp_defines': 'doom_melon=1',
137    },
138    'gyp': {'type': 'gyp'},
139    'gn': {'type': 'gn'},
140    'goma': {
141      'gn_args': 'use_goma=true',
142      'gyp_defines': 'goma=1',
143    },
144    'phase_1': {
145      'gn_args': 'phase=1',
146      'gyp_args': 'phase=1',
147    },
148    'phase_2': {
149      'gn_args': 'phase=2',
150      'gyp_args': 'phase=2',
151    },
152    'rel': {
153      'gn_args': 'is_debug=false',
154    },
155    'debug': {
156      'gn_args': 'is_debug=true',
157    },
158  },
159}
160"""
161
162
163TEST_BAD_CONFIG = """\
164{
165  'configs': {
166    'gn_rel_bot_1': ['gn', 'rel', 'chrome_with_codecs'],
167    'gn_rel_bot_2': ['gn', 'rel', 'bad_nested_config'],
168  },
169  'masters': {
170    'chromium': {
171      'a': 'gn_rel_bot_1',
172      'b': 'gn_rel_bot_2',
173    },
174  },
175  'mixins': {
176    'gn': {'type': 'gn'},
177    'chrome_with_codecs': {
178      'gn_args': 'proprietary_codecs=true',
179    },
180    'bad_nested_config': {
181      'mixins': ['chrome_with_codecs'],
182    },
183    'rel': {
184      'gn_args': 'is_debug=false',
185    },
186  },
187}
188"""
189
190
191GYP_HACKS_CONFIG = """\
192{
193  'masters': {
194    'chromium': {},
195    'fake_master': {
196      'fake_builder': 'fake_config',
197    },
198  },
199  'configs': {
200    'fake_config': ['fake_mixin'],
201  },
202  'mixins': {
203    'fake_mixin': {
204      'type': 'gyp',
205      'gn_args': '',
206      'gyp_defines':
207         ('foo=bar llvm_force_head_revision=1 '
208          'gyp_link_concurrency=1 baz=1'),
209    },
210  },
211}
212"""
213
214
215class UnitTest(unittest.TestCase):
216  def fake_mbw(self, files=None, win32=False):
217    mbw = FakeMBW(win32=win32)
218    mbw.files.setdefault(mbw.default_config, TEST_CONFIG)
219    mbw.files.setdefault(
220        mbw.ToAbsPath('//build/args/bots/fake_master/fake_gn_args_bot.gn'),
221        'is_debug = false\n')
222    if files:
223      for path, contents in files.items():
224        mbw.files[path] = contents
225    return mbw
226
227  def check(self, args, mbw=None, files=None, out=None, err=None, ret=None):
228    if not mbw:
229      mbw = self.fake_mbw(files)
230
231    actual_ret = mbw.Main(args)
232
233    self.assertEqual(actual_ret, ret)
234    if out is not None:
235      self.assertEqual(mbw.out, out)
236    if err is not None:
237      self.assertEqual(mbw.err, err)
238    return mbw
239
240  def test_clobber(self):
241    files = {
242      '/fake_src/out/Debug': None,
243      '/fake_src/out/Debug/mb_type': None,
244    }
245    mbw = self.fake_mbw(files)
246
247    # The first time we run this, the build dir doesn't exist, so no clobber.
248    self.check(['gen', '-c', 'gn_debug_goma', '//out/Debug'], mbw=mbw, ret=0)
249    self.assertEqual(mbw.rmdirs, [])
250    self.assertEqual(mbw.files['/fake_src/out/Debug/mb_type'], 'gn')
251
252    # The second time we run this, the build dir exists and matches, so no
253    # clobber.
254    self.check(['gen', '-c', 'gn_debug_goma', '//out/Debug'], mbw=mbw, ret=0)
255    self.assertEqual(mbw.rmdirs, [])
256    self.assertEqual(mbw.files['/fake_src/out/Debug/mb_type'], 'gn')
257
258    # Now we switch build types; this should result in a clobber.
259    self.check(['gen', '-c', 'gyp_debug', '//out/Debug'], mbw=mbw, ret=0)
260    self.assertEqual(mbw.rmdirs, ['/fake_src/out/Debug'])
261    self.assertEqual(mbw.files['/fake_src/out/Debug/mb_type'], 'gyp')
262
263    # Now we delete mb_type; this checks the case where the build dir
264    # exists but wasn't populated by mb; this should also result in a clobber.
265    del mbw.files['/fake_src/out/Debug/mb_type']
266    self.check(['gen', '-c', 'gyp_debug', '//out/Debug'], mbw=mbw, ret=0)
267    self.assertEqual(mbw.rmdirs,
268                     ['/fake_src/out/Debug', '/fake_src/out/Debug'])
269    self.assertEqual(mbw.files['/fake_src/out/Debug/mb_type'], 'gyp')
270
271  def test_gn_analyze(self):
272    files = {'/tmp/in.json': """{\
273               "files": ["foo/foo_unittest.cc"],
274               "test_targets": ["foo_unittests", "bar_unittests"],
275               "additional_compile_targets": []
276             }"""}
277
278    mbw = self.fake_mbw(files)
279    mbw.Call = lambda cmd, env=None, buffer_output=True: (
280        0, 'out/Default/foo_unittests\n', '')
281
282    self.check(['analyze', '-c', 'gn_debug_goma', '//out/Default',
283                '/tmp/in.json', '/tmp/out.json'], mbw=mbw, ret=0)
284    out = json.loads(mbw.files['/tmp/out.json'])
285    self.assertEqual(out, {
286      'status': 'Found dependency',
287      'compile_targets': ['foo_unittests'],
288      'test_targets': ['foo_unittests']
289    })
290
291  def test_gn_analyze_fails(self):
292    files = {'/tmp/in.json': """{\
293               "files": ["foo/foo_unittest.cc"],
294               "test_targets": ["foo_unittests", "bar_unittests"],
295               "additional_compile_targets": []
296             }"""}
297
298    mbw = self.fake_mbw(files)
299    mbw.Call = lambda cmd, env=None, buffer_output=True: (1, '', '')
300
301    self.check(['analyze', '-c', 'gn_debug_goma', '//out/Default',
302                '/tmp/in.json', '/tmp/out.json'], mbw=mbw, ret=1)
303
304  def test_gn_analyze_all(self):
305    files = {'/tmp/in.json': """{\
306               "files": ["foo/foo_unittest.cc"],
307               "test_targets": ["bar_unittests"],
308               "additional_compile_targets": ["all"]
309             }"""}
310    mbw = self.fake_mbw(files)
311    mbw.Call = lambda cmd, env=None, buffer_output=True: (
312        0, 'out/Default/foo_unittests\n', '')
313    self.check(['analyze', '-c', 'gn_debug_goma', '//out/Default',
314                '/tmp/in.json', '/tmp/out.json'], mbw=mbw, ret=0)
315    out = json.loads(mbw.files['/tmp/out.json'])
316    self.assertEqual(out, {
317      'status': 'Found dependency (all)',
318      'compile_targets': ['all', 'bar_unittests'],
319      'test_targets': ['bar_unittests'],
320    })
321
322  def test_gn_analyze_missing_file(self):
323    files = {'/tmp/in.json': """{\
324               "files": ["foo/foo_unittest.cc"],
325               "test_targets": ["bar_unittests"],
326               "additional_compile_targets": []
327             }"""}
328    mbw = self.fake_mbw(files)
329    mbw.cmds = [
330        (0, '', ''),
331        (1, 'The input matches no targets, configs, or files\n', ''),
332        (1, 'The input matches no targets, configs, or files\n', ''),
333    ]
334
335    self.check(['analyze', '-c', 'gn_debug_goma', '//out/Default',
336                '/tmp/in.json', '/tmp/out.json'], mbw=mbw, ret=0)
337    out = json.loads(mbw.files['/tmp/out.json'])
338    self.assertEqual(out, {
339      'status': 'No dependency',
340      'compile_targets': [],
341      'test_targets': [],
342    })
343
344  def test_gn_gen(self):
345    mbw = self.fake_mbw()
346    self.check(['gen', '-c', 'gn_debug_goma', '//out/Default', '-g', '/goma'],
347               mbw=mbw, ret=0)
348    self.assertMultiLineEqual(mbw.files['/fake_src/out/Default/args.gn'],
349                              ('goma_dir = "/goma"\n'
350                               'is_debug = true\n'
351                               'use_goma = true\n'))
352
353    # Make sure we log both what is written to args.gn and the command line.
354    self.assertIn('Writing """', mbw.out)
355    self.assertIn('/fake_src/buildtools/linux64/gn gen //out/Default --check',
356                  mbw.out)
357
358    mbw = self.fake_mbw(win32=True)
359    self.check(['gen', '-c', 'gn_debug_goma', '-g', 'c:\\goma', '//out/Debug'],
360               mbw=mbw, ret=0)
361    self.assertMultiLineEqual(mbw.files['c:\\fake_src\\out\\Debug\\args.gn'],
362                              ('goma_dir = "c:\\\\goma"\n'
363                               'is_debug = true\n'
364                               'use_goma = true\n'))
365    self.assertIn('c:\\fake_src\\buildtools\\win\\gn.exe gen //out/Debug '
366                  '--check\n', mbw.out)
367
368    mbw = self.fake_mbw()
369    self.check(['gen', '-m', 'fake_master', '-b', 'fake_gn_args_bot',
370                '//out/Debug'],
371               mbw=mbw, ret=0)
372    self.assertEqual(
373        mbw.files['/fake_src/out/Debug/args.gn'],
374        'import("//build/args/bots/fake_master/fake_gn_args_bot.gn")\n')
375
376
377  def test_gn_gen_fails(self):
378    mbw = self.fake_mbw()
379    mbw.Call = lambda cmd, env=None, buffer_output=True: (1, '', '')
380    self.check(['gen', '-c', 'gn_debug_goma', '//out/Default'], mbw=mbw, ret=1)
381
382  def test_gn_gen_swarming(self):
383    files = {
384      '/tmp/swarming_targets': 'base_unittests\n',
385      '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
386          "{'base_unittests': {"
387          "  'label': '//base:base_unittests',"
388          "  'type': 'raw',"
389          "  'args': [],"
390          "}}\n"
391      ),
392      '/fake_src/out/Default/base_unittests.runtime_deps': (
393          "base_unittests\n"
394      ),
395    }
396    mbw = self.fake_mbw(files)
397    self.check(['gen',
398                '-c', 'gn_debug_goma',
399                '--swarming-targets-file', '/tmp/swarming_targets',
400                '//out/Default'], mbw=mbw, ret=0)
401    self.assertIn('/fake_src/out/Default/base_unittests.isolate',
402                  mbw.files)
403    self.assertIn('/fake_src/out/Default/base_unittests.isolated.gen.json',
404                  mbw.files)
405
406  def test_gn_isolate(self):
407    files = {
408      '/fake_src/out/Default/toolchain.ninja': "",
409      '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
410          "{'base_unittests': {"
411          "  'label': '//base:base_unittests',"
412          "  'type': 'raw',"
413          "  'args': [],"
414          "}}\n"
415      ),
416      '/fake_src/out/Default/base_unittests.runtime_deps': (
417          "base_unittests\n"
418      ),
419    }
420    self.check(['isolate', '-c', 'gn_debug_goma', '//out/Default',
421                'base_unittests'], files=files, ret=0)
422
423    # test running isolate on an existing build_dir
424    files['/fake_src/out/Default/args.gn'] = 'is_debug = True\n'
425    self.check(['isolate', '//out/Default', 'base_unittests'],
426               files=files, ret=0)
427
428    files['/fake_src/out/Default/mb_type'] = 'gn\n'
429    self.check(['isolate', '//out/Default', 'base_unittests'],
430               files=files, ret=0)
431
432  def test_gn_run(self):
433    files = {
434      '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
435          "{'base_unittests': {"
436          "  'label': '//base:base_unittests',"
437          "  'type': 'raw',"
438          "  'args': [],"
439          "}}\n"
440      ),
441      '/fake_src/out/Default/base_unittests.runtime_deps': (
442          "base_unittests\n"
443      ),
444    }
445    self.check(['run', '-c', 'gn_debug_goma', '//out/Default',
446                'base_unittests'], files=files, ret=0)
447
448  def test_gn_lookup(self):
449    self.check(['lookup', '-c', 'gn_debug_goma'], ret=0)
450
451  def test_gn_lookup_goma_dir_expansion(self):
452    self.check(['lookup', '-c', 'gn_rel_bot', '-g', '/foo'], ret=0,
453               out=('\n'
454                    'Writing """\\\n'
455                    'goma_dir = "/foo"\n'
456                    'is_debug = false\n'
457                    'use_goma = true\n'
458                    '""" to _path_/args.gn.\n\n'
459                    '/fake_src/buildtools/linux64/gn gen _path_\n'))
460
461  def test_gyp_analyze(self):
462    mbw = self.check(['analyze', '-c', 'gyp_rel_bot', '//out/Release',
463                      '/tmp/in.json', '/tmp/out.json'], ret=0)
464    self.assertIn('analyzer', mbw.calls[0])
465
466  def test_gyp_crosscompile(self):
467    mbw = self.fake_mbw()
468    self.check(['gen', '-c', 'gyp_crosscompile', '//out/Release'],
469               mbw=mbw, ret=0)
470    self.assertTrue(mbw.cross_compile)
471
472  def test_gyp_gen(self):
473    self.check(['gen', '-c', 'gyp_rel_bot', '-g', '/goma', '//out/Release'],
474               ret=0,
475               out=("GYP_DEFINES='goma=1 gomadir=/goma'\n"
476                    "python build/gyp_chromium -G output_dir=out\n"))
477
478    mbw = self.fake_mbw(win32=True)
479    self.check(['gen', '-c', 'gyp_rel_bot', '-g', 'c:\\goma', '//out/Release'],
480               mbw=mbw, ret=0,
481               out=("set GYP_DEFINES=goma=1 gomadir='c:\\goma'\n"
482                    "python build\\gyp_chromium -G output_dir=out\n"))
483
484  def test_gyp_gen_fails(self):
485    mbw = self.fake_mbw()
486    mbw.Call = lambda cmd, env=None, buffer_output=True: (1, '', '')
487    self.check(['gen', '-c', 'gyp_rel_bot', '//out/Release'], mbw=mbw, ret=1)
488
489  def test_gyp_lookup_goma_dir_expansion(self):
490    self.check(['lookup', '-c', 'gyp_rel_bot', '-g', '/foo'], ret=0,
491               out=("GYP_DEFINES='goma=1 gomadir=/foo'\n"
492                    "python build/gyp_chromium -G output_dir=_path_\n"))
493
494  def test_help(self):
495    orig_stdout = sys.stdout
496    try:
497      sys.stdout = StringIO.StringIO()
498      self.assertRaises(SystemExit, self.check, ['-h'])
499      self.assertRaises(SystemExit, self.check, ['help'])
500      self.assertRaises(SystemExit, self.check, ['help', 'gen'])
501    finally:
502      sys.stdout = orig_stdout
503
504  def test_multiple_phases(self):
505    # Check that not passing a --phase to a multi-phase builder fails.
506    mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_multi_phase'],
507                     ret=1)
508    self.assertIn('Must specify a build --phase', mbw.out)
509
510    # Check that passing a --phase to a single-phase builder fails.
511    mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_gn_builder',
512                      '--phase', '1'],
513                     ret=1)
514    self.assertIn('Must not specify a build --phase', mbw.out)
515
516    # Check different ranges; 0 and 3 are out of bounds, 1 and 2 should work.
517    mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_multi_phase',
518                      '--phase', '0'], ret=1)
519    self.assertIn('Phase 0 out of bounds', mbw.out)
520
521    mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_multi_phase',
522                      '--phase', '1'], ret=0)
523    self.assertIn('phase = 1', mbw.out)
524
525    mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_multi_phase',
526                      '--phase', '2'], ret=0)
527    self.assertIn('phase = 2', mbw.out)
528
529    mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_multi_phase',
530                      '--phase', '3'], ret=1)
531    self.assertIn('Phase 3 out of bounds', mbw.out)
532
533  def test_validate(self):
534    mbw = self.fake_mbw()
535    self.check(['validate'], mbw=mbw, ret=0)
536
537  def test_gyp_env_hacks(self):
538    mbw = self.fake_mbw()
539    mbw.files[mbw.default_config] = GYP_HACKS_CONFIG
540    self.check(['lookup', '-c', 'fake_config'], mbw=mbw,
541               ret=0,
542               out=("GYP_DEFINES='foo=bar baz=1'\n"
543                    "GYP_LINK_CONCURRENCY=1\n"
544                    "LLVM_FORCE_HEAD_REVISION=1\n"
545                    "python build/gyp_chromium -G output_dir=_path_\n"))
546
547
548if __name__ == '__main__':
549  unittest.main()
550
551  def test_validate(self):
552    mbw = self.fake_mbw()
553    self.check(['validate'], mbw=mbw, ret=0)
554
555  def test_bad_validate(self):
556    mbw = self.fake_mbw()
557    mbw.files[mbw.default_config] = TEST_BAD_CONFIG
558    self.check(['validate'], mbw=mbw, ret=1)
559
560  def test_gyp_env_hacks(self):
561    mbw = self.fake_mbw()
562    mbw.files[mbw.default_config] = GYP_HACKS_CONFIG
563    self.check(['lookup', '-c', 'fake_config'], mbw=mbw,
564               ret=0,
565               out=("GYP_DEFINES='foo=bar baz=1'\n"
566                    "GYP_LINK_CONCURRENCY=1\n"
567                    "LLVM_FORCE_HEAD_REVISION=1\n"
568                    "python build/gyp_chromium -G output_dir=_path_\n"))
569
570
571if __name__ == '__main__':
572  unittest.main()
573