1# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
2#
3# Use of this source code is governed by a BSD-style license
4# that can be found in the LICENSE file in the root of the source
5# tree. An additional intellectual property rights grant can be found
6# in the file PATENTS.  All contributing project authors may
7# be found in the AUTHORS file in the root of the source tree.
8
9import json
10import os
11import platform
12import re
13import subprocess
14import sys
15
16
17# Directories that will be scanned by cpplint by the presubmit script.
18CPPLINT_DIRS = [
19  'webrtc/audio',
20  'webrtc/call',
21  'webrtc/common_video',
22  'webrtc/examples',
23  'webrtc/modules/remote_bitrate_estimator',
24  'webrtc/modules/rtp_rtcp',
25  'webrtc/modules/video_coding',
26  'webrtc/modules/video_processing',
27  'webrtc/sound',
28  'webrtc/tools',
29  'webrtc/video',
30]
31
32# List of directories of "supported" native APIs. That means changes to headers
33# will be done in a compatible way following this scheme:
34# 1. Non-breaking changes are made.
35# 2. The old APIs as marked as deprecated (with comments).
36# 3. Deprecation is announced to discuss-webrtc@googlegroups.com and
37#    webrtc-users@google.com (internal list).
38# 4. (later) The deprecated APIs are removed.
39# Directories marked as DEPRECATED should not be used. They're only present in
40# the list to support legacy downstream code.
41NATIVE_API_DIRS = (
42  'talk/app/webrtc',
43  'webrtc',
44  'webrtc/base',  # DEPRECATED.
45  'webrtc/common_audio/include',  # DEPRECATED.
46  'webrtc/modules/audio_coding/include',
47  'webrtc/modules/audio_conference_mixer/include',  # DEPRECATED.
48  'webrtc/modules/audio_device/include',
49  'webrtc/modules/audio_processing/include',
50  'webrtc/modules/bitrate_controller/include',
51  'webrtc/modules/include',
52  'webrtc/modules/remote_bitrate_estimator/include',
53  'webrtc/modules/rtp_rtcp/include',
54  'webrtc/modules/rtp_rtcp/source',  # DEPRECATED.
55  'webrtc/modules/utility/include',
56  'webrtc/modules/video_coding/codecs/h264/include',
57  'webrtc/modules/video_coding/codecs/i420/include',
58  'webrtc/modules/video_coding/codecs/vp8/include',
59  'webrtc/modules/video_coding/codecs/vp9/include',
60  'webrtc/modules/video_coding/include',
61  'webrtc/system_wrappers/include',  # DEPRECATED.
62  'webrtc/voice_engine/include',
63)
64
65
66def _VerifyNativeApiHeadersListIsValid(input_api, output_api):
67  """Ensures the list of native API header directories is up to date."""
68  non_existing_paths = []
69  native_api_full_paths = [
70      input_api.os_path.join(input_api.PresubmitLocalPath(),
71                             *path.split('/')) for path in NATIVE_API_DIRS]
72  for path in native_api_full_paths:
73    if not os.path.isdir(path):
74      non_existing_paths.append(path)
75  if non_existing_paths:
76    return [output_api.PresubmitError(
77        'Directories to native API headers have changed which has made the '
78        'list in PRESUBMIT.py outdated.\nPlease update it to the current '
79        'location of our native APIs.',
80        non_existing_paths)]
81  return []
82
83
84def _CheckNativeApiHeaderChanges(input_api, output_api):
85  """Checks to remind proper changing of native APIs."""
86  files = []
87  for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile):
88    if f.LocalPath().endswith('.h'):
89      for path in NATIVE_API_DIRS:
90        if os.path.dirname(f.LocalPath()) == path:
91          files.append(f)
92
93  if files:
94    return [output_api.PresubmitNotifyResult(
95        'You seem to be changing native API header files. Please make sure '
96        'you:\n'
97        '  1. Make compatible changes that don\'t break existing clients.\n'
98        '  2. Mark the old APIs as deprecated.\n'
99        '  3. Create a timeline and plan for when the deprecated method will '
100        'be removed (preferably 3 months or so).\n'
101        '  4. Update/inform existing downstream code owners to stop using the '
102        'deprecated APIs: \n'
103        'send announcement to discuss-webrtc@googlegroups.com and '
104        'webrtc-users@google.com.\n'
105        '  5. (after ~3 months) remove the deprecated API.\n'
106        'Related files:',
107        files)]
108  return []
109
110
111def _CheckNoIOStreamInHeaders(input_api, output_api):
112  """Checks to make sure no .h files include <iostream>."""
113  files = []
114  pattern = input_api.re.compile(r'^#include\s*<iostream>',
115                                 input_api.re.MULTILINE)
116  for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile):
117    if not f.LocalPath().endswith('.h'):
118      continue
119    contents = input_api.ReadFile(f)
120    if pattern.search(contents):
121      files.append(f)
122
123  if len(files):
124    return [output_api.PresubmitError(
125        'Do not #include <iostream> in header files, since it inserts static ' +
126        'initialization into every file including the header. Instead, ' +
127        '#include <ostream>. See http://crbug.com/94794',
128        files)]
129  return []
130
131
132def _CheckNoFRIEND_TEST(input_api, output_api):
133  """Make sure that gtest's FRIEND_TEST() macro is not used, the
134  FRIEND_TEST_ALL_PREFIXES() macro from testsupport/gtest_prod_util.h should be
135  used instead since that allows for FLAKY_, FAILS_ and DISABLED_ prefixes."""
136  problems = []
137
138  file_filter = lambda f: f.LocalPath().endswith(('.cc', '.h'))
139  for f in input_api.AffectedFiles(file_filter=file_filter):
140    for line_num, line in f.ChangedContents():
141      if 'FRIEND_TEST(' in line:
142        problems.append('    %s:%d' % (f.LocalPath(), line_num))
143
144  if not problems:
145    return []
146  return [output_api.PresubmitPromptWarning('WebRTC\'s code should not use '
147      'gtest\'s FRIEND_TEST() macro. Include testsupport/gtest_prod_util.h and '
148      'use FRIEND_TEST_ALL_PREFIXES() instead.\n' + '\n'.join(problems))]
149
150
151def _IsLintWhitelisted(whitelist_dirs, file_path):
152  """ Checks if a file is whitelisted for lint check."""
153  for path in whitelist_dirs:
154    if os.path.dirname(file_path).startswith(path):
155      return True
156  return False
157
158
159def _CheckApprovedFilesLintClean(input_api, output_api,
160                                 source_file_filter=None):
161  """Checks that all new or whitelisted .cc and .h files pass cpplint.py.
162  This check is based on _CheckChangeLintsClean in
163  depot_tools/presubmit_canned_checks.py but has less filters and only checks
164  added files."""
165  result = []
166
167  # Initialize cpplint.
168  import cpplint
169  # Access to a protected member _XX of a client class
170  # pylint: disable=W0212
171  cpplint._cpplint_state.ResetErrorCounts()
172
173  # Create a platform independent whitelist for the CPPLINT_DIRS.
174  whitelist_dirs = [input_api.os_path.join(*path.split('/'))
175                    for path in CPPLINT_DIRS]
176
177  # Use the strictest verbosity level for cpplint.py (level 1) which is the
178  # default when running cpplint.py from command line.
179  # To make it possible to work with not-yet-converted code, we're only applying
180  # it to new (or moved/renamed) files and files listed in LINT_FOLDERS.
181  verbosity_level = 1
182  files = []
183  for f in input_api.AffectedSourceFiles(source_file_filter):
184    # Note that moved/renamed files also count as added.
185    if f.Action() == 'A' or _IsLintWhitelisted(whitelist_dirs, f.LocalPath()):
186      files.append(f.AbsoluteLocalPath())
187
188  for file_name in files:
189    cpplint.ProcessFile(file_name, verbosity_level)
190
191  if cpplint._cpplint_state.error_count > 0:
192    if input_api.is_committing:
193      # TODO(kjellander): Change back to PresubmitError below when we're
194      # confident with the lint settings.
195      res_type = output_api.PresubmitPromptWarning
196    else:
197      res_type = output_api.PresubmitPromptWarning
198    result = [res_type('Changelist failed cpplint.py check.')]
199
200  return result
201
202def _CheckNoRtcBaseDeps(input_api, gyp_files, output_api):
203  pattern = input_api.re.compile(r"base.gyp:rtc_base\s*'")
204  violating_files = []
205  for f in gyp_files:
206    gyp_exceptions = (
207        'base_tests.gyp',
208        'desktop_capture.gypi',
209        'libjingle.gyp',
210        'libjingle_tests.gyp',
211        'p2p.gyp',
212        'sound.gyp',
213        'webrtc_test_common.gyp',
214        'webrtc_tests.gypi',
215    )
216    if f.LocalPath().endswith(gyp_exceptions):
217      continue
218    contents = input_api.ReadFile(f)
219    if pattern.search(contents):
220      violating_files.append(f)
221  if violating_files:
222    return [output_api.PresubmitError(
223        'Depending on rtc_base is not allowed. Change your dependency to '
224        'rtc_base_approved and possibly sanitize and move the desired source '
225        'file(s) to rtc_base_approved.\nChanged GYP files:',
226        items=violating_files)]
227  return []
228
229def _CheckNoSourcesAboveGyp(input_api, gyp_files, output_api):
230  # Disallow referencing source files with paths above the GYP file location.
231  source_pattern = input_api.re.compile(r'sources.*?\[(.*?)\]',
232                                        re.MULTILINE | re.DOTALL)
233  file_pattern = input_api.re.compile(r"'((\.\./.*?)|(<\(webrtc_root\).*?))'")
234  violating_gyp_files = set()
235  violating_source_entries = []
236  for gyp_file in gyp_files:
237    contents = input_api.ReadFile(gyp_file)
238    for source_block_match in source_pattern.finditer(contents):
239      # Find all source list entries starting with ../ in the source block
240      # (exclude overrides entries).
241      for file_list_match in file_pattern.finditer(source_block_match.group(0)):
242        source_file = file_list_match.group(0)
243        if 'overrides/' not in source_file:
244          violating_source_entries.append(source_file)
245          violating_gyp_files.add(gyp_file)
246  if violating_gyp_files:
247    return [output_api.PresubmitError(
248        'Referencing source files above the directory of the GYP file is not '
249        'allowed. Please introduce new GYP targets and/or GYP files in the '
250        'proper location instead.\n'
251        'Invalid source entries:\n'
252        '%s\n'
253        'Violating GYP files:' % '\n'.join(violating_source_entries),
254        items=violating_gyp_files)]
255  return []
256
257def _CheckGypChanges(input_api, output_api):
258  source_file_filter = lambda x: input_api.FilterSourceFile(
259      x, white_list=(r'.+\.(gyp|gypi)$',))
260
261  gyp_files = []
262  for f in input_api.AffectedSourceFiles(source_file_filter):
263    if f.LocalPath().startswith('webrtc'):
264      gyp_files.append(f)
265
266  result = []
267  if gyp_files:
268    result.append(output_api.PresubmitNotifyResult(
269        'As you\'re changing GYP files: please make sure corresponding '
270        'BUILD.gn files are also updated.\nChanged GYP files:',
271        items=gyp_files))
272    result.extend(_CheckNoRtcBaseDeps(input_api, gyp_files, output_api))
273    result.extend(_CheckNoSourcesAboveGyp(input_api, gyp_files, output_api))
274  return result
275
276def _CheckUnwantedDependencies(input_api, output_api):
277  """Runs checkdeps on #include statements added in this
278  change. Breaking - rules is an error, breaking ! rules is a
279  warning.
280  """
281  # Copied from Chromium's src/PRESUBMIT.py.
282
283  # We need to wait until we have an input_api object and use this
284  # roundabout construct to import checkdeps because this file is
285  # eval-ed and thus doesn't have __file__.
286  original_sys_path = sys.path
287  try:
288    checkdeps_path = input_api.os_path.join(input_api.PresubmitLocalPath(),
289                                            'buildtools', 'checkdeps')
290    if not os.path.exists(checkdeps_path):
291      return [output_api.PresubmitError(
292          'Cannot find checkdeps at %s\nHave you run "gclient sync" to '
293          'download Chromium and setup the symlinks?' % checkdeps_path)]
294    sys.path.append(checkdeps_path)
295    import checkdeps
296    from cpp_checker import CppChecker
297    from rules import Rule
298  finally:
299    # Restore sys.path to what it was before.
300    sys.path = original_sys_path
301
302  added_includes = []
303  for f in input_api.AffectedFiles():
304    if not CppChecker.IsCppFile(f.LocalPath()):
305      continue
306
307    changed_lines = [line for _, line in f.ChangedContents()]
308    added_includes.append([f.LocalPath(), changed_lines])
309
310  deps_checker = checkdeps.DepsChecker(input_api.PresubmitLocalPath())
311
312  error_descriptions = []
313  warning_descriptions = []
314  for path, rule_type, rule_description in deps_checker.CheckAddedCppIncludes(
315      added_includes):
316    description_with_path = '%s\n    %s' % (path, rule_description)
317    if rule_type == Rule.DISALLOW:
318      error_descriptions.append(description_with_path)
319    else:
320      warning_descriptions.append(description_with_path)
321
322  results = []
323  if error_descriptions:
324    results.append(output_api.PresubmitError(
325        'You added one or more #includes that violate checkdeps rules.',
326        error_descriptions))
327  if warning_descriptions:
328    results.append(output_api.PresubmitPromptOrNotify(
329        'You added one or more #includes of files that are temporarily\n'
330        'allowed but being removed. Can you avoid introducing the\n'
331        '#include? See relevant DEPS file(s) for details and contacts.',
332        warning_descriptions))
333  return results
334
335
336def _RunPythonTests(input_api, output_api):
337  def join(*args):
338    return input_api.os_path.join(input_api.PresubmitLocalPath(), *args)
339
340  test_directories = [
341    join('tools', 'autoroller', 'unittests'),
342  ]
343
344  tests = []
345  for directory in test_directories:
346    tests.extend(
347      input_api.canned_checks.GetUnitTestsInDirectory(
348          input_api,
349          output_api,
350          directory,
351          whitelist=[r'.+_test\.py$']))
352  return input_api.RunTests(tests, parallel=True)
353
354
355def _CommonChecks(input_api, output_api):
356  """Checks common to both upload and commit."""
357  results = []
358  # Filter out files that are in objc or ios dirs from being cpplint-ed since
359  # they do not follow C++ lint rules.
360  black_list = input_api.DEFAULT_BLACK_LIST + (
361    r".*\bobjc[\\\/].*",
362  )
363  source_file_filter = lambda x: input_api.FilterSourceFile(x, None, black_list)
364  results.extend(_CheckApprovedFilesLintClean(
365      input_api, output_api, source_file_filter))
366  results.extend(input_api.canned_checks.RunPylint(input_api, output_api,
367      black_list=(r'^.*gviz_api\.py$',
368                  r'^.*gaeunit\.py$',
369                  # Embedded shell-script fakes out pylint.
370                  r'^build[\\\/].*\.py$',
371                  r'^buildtools[\\\/].*\.py$',
372                  r'^chromium[\\\/].*\.py$',
373                  r'^google_apis[\\\/].*\.py$',
374                  r'^net.*[\\\/].*\.py$',
375                  r'^out.*[\\\/].*\.py$',
376                  r'^testing[\\\/].*\.py$',
377                  r'^third_party[\\\/].*\.py$',
378                  r'^tools[\\\/]find_depot_tools.py$',
379                  r'^tools[\\\/]clang[\\\/].*\.py$',
380                  r'^tools[\\\/]generate_library_loader[\\\/].*\.py$',
381                  r'^tools[\\\/]gn[\\\/].*\.py$',
382                  r'^tools[\\\/]gyp[\\\/].*\.py$',
383                  r'^tools[\\\/]isolate_driver.py$',
384                  r'^tools[\\\/]protoc_wrapper[\\\/].*\.py$',
385                  r'^tools[\\\/]python[\\\/].*\.py$',
386                  r'^tools[\\\/]python_charts[\\\/]data[\\\/].*\.py$',
387                  r'^tools[\\\/]refactoring[\\\/].*\.py$',
388                  r'^tools[\\\/]swarming_client[\\\/].*\.py$',
389                  r'^tools[\\\/]vim[\\\/].*\.py$',
390                  # TODO(phoglund): should arguably be checked.
391                  r'^tools[\\\/]valgrind-webrtc[\\\/].*\.py$',
392                  r'^tools[\\\/]valgrind[\\\/].*\.py$',
393                  r'^tools[\\\/]win[\\\/].*\.py$',
394                  r'^xcodebuild.*[\\\/].*\.py$',),
395      disabled_warnings=['F0401',  # Failed to import x
396                         'E0611',  # No package y in x
397                         'W0232',  # Class has no __init__ method
398                        ],
399      pylintrc='pylintrc'))
400  # WebRTC can't use the presubmit_canned_checks.PanProjectChecks function since
401  # we need to have different license checks in talk/ and webrtc/ directories.
402  # Instead, hand-picked checks are included below.
403
404  # Skip long-lines check for DEPS, GN and GYP files.
405  long_lines_sources = lambda x: input_api.FilterSourceFile(x,
406      black_list=(r'.+\.gyp$', r'.+\.gypi$', r'.+\.gn$', r'.+\.gni$', 'DEPS'))
407  results.extend(input_api.canned_checks.CheckLongLines(
408      input_api, output_api, maxlen=80, source_file_filter=long_lines_sources))
409  results.extend(input_api.canned_checks.CheckChangeHasNoTabs(
410      input_api, output_api))
411  results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
412      input_api, output_api))
413  results.extend(input_api.canned_checks.CheckChangeTodoHasOwner(
414      input_api, output_api))
415  results.extend(_CheckNativeApiHeaderChanges(input_api, output_api))
416  results.extend(_CheckNoIOStreamInHeaders(input_api, output_api))
417  results.extend(_CheckNoFRIEND_TEST(input_api, output_api))
418  results.extend(_CheckGypChanges(input_api, output_api))
419  results.extend(_CheckUnwantedDependencies(input_api, output_api))
420  results.extend(_RunPythonTests(input_api, output_api))
421  return results
422
423
424def CheckChangeOnUpload(input_api, output_api):
425  results = []
426  results.extend(_CommonChecks(input_api, output_api))
427  results.extend(
428      input_api.canned_checks.CheckGNFormatted(input_api, output_api))
429  return results
430
431
432def CheckChangeOnCommit(input_api, output_api):
433  results = []
434  results.extend(_CommonChecks(input_api, output_api))
435  results.extend(_VerifyNativeApiHeadersListIsValid(input_api, output_api))
436  results.extend(input_api.canned_checks.CheckOwners(input_api, output_api))
437  results.extend(input_api.canned_checks.CheckChangeWasUploaded(
438      input_api, output_api))
439  results.extend(input_api.canned_checks.CheckChangeHasDescription(
440      input_api, output_api))
441  results.extend(input_api.canned_checks.CheckChangeHasBugField(
442      input_api, output_api))
443  results.extend(input_api.canned_checks.CheckChangeHasTestField(
444      input_api, output_api))
445  results.extend(input_api.canned_checks.CheckTreeIsOpen(
446      input_api, output_api,
447      json_url='http://webrtc-status.appspot.com/current?format=json'))
448  return results
449
450
451# pylint: disable=W0613
452def GetPreferredTryMasters(project, change):
453  cq_config_path = os.path.join(
454      change.RepositoryRoot(), 'infra', 'config', 'cq.cfg')
455  # commit_queue.py below is a script in depot_tools directory, which has a
456  # 'builders' command to retrieve a list of CQ builders from the CQ config.
457  is_win = platform.system() == 'Windows'
458  masters = json.loads(subprocess.check_output(
459      ['commit_queue', 'builders', cq_config_path], shell=is_win))
460
461  try_config = {}
462  for master in masters:
463    try_config.setdefault(master, {})
464    for builder in masters[master]:
465      if 'presubmit' in builder:
466        # Do not trigger presubmit builders, since they're likely to fail
467        # (e.g. OWNERS checks before finished code review), and we're running
468        # local presubmit anyway.
469        pass
470      else:
471        try_config[master][builder] = ['defaulttests']
472
473  return try_config
474