1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5#
6"""Logic to generate lists of DEPS used by various parts of
7the android_webview continuous integration (buildbot) infrastructure.
8
9Note: The root Chromium project (which is not explicitly listed here)
10has a couple of third_party libraries checked in directly into it. This means
11that the list of third parties present in this file is not a comprehensive
12list of third party android_webview dependencies.
13"""
14
15import argparse
16import json
17import logging
18import os
19import sys
20
21# Add android_webview/tools to path to get at known_issues.
22sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'tools'))
23import known_issues
24
25
26class DepsWhitelist(object):
27  def __init__(self):
28    # If a new DEPS entry is needed for the AOSP bot to compile please add it
29    # here first.
30    # This is a staging area for deps that are accepted by the android_webview
31    # team and are in the process of having the required branches being created
32    # in the Android tree.
33    self._compile_but_not_snapshot_dependencies = [
34    ]
35
36    # Dependencies that need to be merged into the Android tree.
37    self._snapshot_into_android_dependencies = [
38      'sdch/open-vcdiff',
39      'testing/gtest',
40      'third_party/WebKit',
41      'third_party/angle',
42      'third_party/boringssl/src',
43      'third_party/brotli/src',
44      ('third_party/eyesfree/src/android/java/src/com/googlecode/eyesfree/'
45       'braille'),
46      'third_party/freetype',
47      'third_party/icu',
48      'third_party/leveldatabase/src',
49      'third_party/libaddressinput/src',
50      'third_party/libjingle/source/talk',
51      'third_party/libjpeg_turbo',
52      'third_party/libphonenumber/src/phonenumbers',
53      'third_party/libphonenumber/src/resources',
54      'third_party/libsrtp',
55      'third_party/libvpx',
56      'third_party/libyuv',
57      'third_party/mesa/src',
58      'third_party/openmax_dl',
59      'third_party/opus/src',
60      'third_party/ots',
61      'third_party/sfntly/cpp/src',
62      'third_party/skia',
63      'third_party/smhasher/src',
64      'third_party/usrsctp/usrsctplib',
65      'third_party/webrtc',
66      'third_party/yasm/source/patched-yasm',
67      'tools/grit',
68      'tools/gyp',
69      'v8',
70    ]
71
72    # We can save some time by not rsyncing code we don't use.
73    self._prune_from_rsync_build = [
74        'third_party/WebKit/LayoutTests',
75    ]
76
77    # Dependencies required to build android_webview.
78    self._compile_dependencies = (self._snapshot_into_android_dependencies +
79                                  self._compile_but_not_snapshot_dependencies)
80
81    # Dependencies required to run android_webview tests but not required to
82    # compile.
83    self._test_data_dependencies = [
84      'chrome/test/data/perf/third_party/octane',
85    ]
86
87  @staticmethod
88  def _read_deps_file(deps_file_path):
89    class FileImplStub(object):
90      """Stub for the File syntax."""
91      def __init__(self, file_location):
92        pass
93
94      @staticmethod
95      def GetPath():
96        return ''
97
98      @staticmethod
99      def GetFilename():
100        return ''
101
102      @staticmethod
103      def GetRevision():
104        return None
105
106    def from_stub(__, _=None):
107      """Stub for the From syntax."""
108      return ''
109
110    class VarImpl(object):
111      def __init__(self, custom_vars, local_scope):
112        self._custom_vars = custom_vars
113        self._local_scope = local_scope
114
115      def Lookup(self, var_name):
116        """Implements the Var syntax."""
117        if var_name in self._custom_vars:
118          return self._custom_vars[var_name]
119        elif var_name in self._local_scope.get("vars", {}):
120          return self._local_scope["vars"][var_name]
121        raise Exception("Var is not defined: %s" % var_name)
122
123    local_scope = {}
124    var = VarImpl({}, local_scope)
125    global_scope = {
126        'File': FileImplStub,
127        'From': from_stub,
128        'Var': var.Lookup,
129        'deps_os': {},
130    }
131    execfile(deps_file_path, global_scope, local_scope)
132    deps = local_scope.get('deps', {})
133    deps_os = local_scope.get('deps_os', {})
134    for os_specific_deps in deps_os.itervalues():
135      deps.update(os_specific_deps)
136    return deps.keys()
137
138  def _get_known_issues(self):
139    issues = []
140    for root, paths in known_issues.KNOWN_INCOMPATIBLE.items():
141      for path in paths:
142        issues.append(os.path.normpath(os.path.join(root, path)))
143    return issues
144
145  def _make_gclient_blacklist(self, deps_file_path, whitelisted_deps):
146    """Calculates the list of deps that need to be excluded from the deps_file
147    so that the only deps left are the one in the whitelist."""
148    all_deps = self._read_deps_file(deps_file_path)
149    # The list of deps read from the DEPS file are prefixed with the source
150    # tree root, which is 'src' for Chromium.
151    def prepend_root(path):
152      return os.path.join('src', path)
153    whitelisted_deps = map(prepend_root, whitelisted_deps)
154    deps_blacklist = set(all_deps).difference(set(whitelisted_deps))
155    return dict(map(lambda(x): (x, None), deps_blacklist))
156
157  def _make_blacklist(self, deps_file_path, whitelisted_deps):
158    """Calculates the list of paths we should exclude """
159    all_deps = self._read_deps_file(deps_file_path)
160    def remove_src_prefix(path):
161      return path.replace('src/', '', 1)
162    all_deps = map(remove_src_prefix, all_deps)
163    # Ignore all deps except those whitelisted.
164    blacklist = set(all_deps).difference(whitelisted_deps)
165    # Ignore the 'known issues'. Typically these are the licence incompatible
166    # things checked directly into Chromium.
167    blacklist = blacklist.union(self._get_known_issues())
168    # Ignore any other non-deps, non-licence paths we don't like.
169    blacklist = blacklist.union(self._prune_from_rsync_build)
170    return list(blacklist)
171
172  def get_deps_for_android_build(self, deps_file_path):
173    """This is used to calculate the custom_deps list for the Android bot.
174    """
175    if not deps_file_path:
176      raise Exception('You need to specify a DEPS file path.')
177    return self._make_gclient_blacklist(deps_file_path,
178                                        self._compile_dependencies)
179
180  def get_deps_for_android_build_and_test(self, deps_file_path):
181    """This is used to calculate the custom_deps list for the Android perf bot.
182    """
183    if not deps_file_path:
184      raise Exception('You need to specify a DEPS file path.')
185    return self._make_gclient_blacklist(deps_file_path,
186                                        self._compile_dependencies +
187                                        self._test_data_dependencies)
188
189  def get_blacklist_for_android_rsync_build(self, deps_file_path):
190    """Calculates the list of paths we should exclude when building Android
191    either because of license compatibility or because they are large and
192    uneeded.
193    """
194    if not deps_file_path:
195      raise Exception('You need to specify a DEPS file path.')
196    return self._make_blacklist(deps_file_path, self._compile_dependencies)
197
198  def get_deps_for_android_merge(self, _):
199    """Calculates the list of deps that need to be merged into the Android tree
200    in order to build the C++ and Java android_webview code."""
201    return self._snapshot_into_android_dependencies
202
203  def get_deps_for_license_check(self, _):
204    """Calculates the list of deps that need to be checked for Android license
205    compatibility"""
206    return self._compile_dependencies
207
208
209  def execute_method(self, method_name, deps_file_path):
210    methods = {
211      'android_build': self.get_deps_for_android_build,
212      'android_build_and_test':
213        self.get_deps_for_android_build_and_test,
214      'android_merge': self.get_deps_for_android_merge,
215      'license_check': self.get_deps_for_license_check,
216      'android_rsync_build': self.get_blacklist_for_android_rsync_build,
217    }
218    if not method_name in methods:
219      raise Exception('Method name %s is not valid. Valid choices are %s' %
220                      (method_name, methods.keys()))
221    return methods[method_name](deps_file_path)
222
223def main():
224  parser = argparse.ArgumentParser()
225  parser.add_argument('--method', help='Method to use to fetch from whitelist.',
226                      required=True)
227  parser.add_argument('--path-to-deps', help='Path to DEPS file.')
228  parser.add_argument('--output-json', help='Name of file to write output to.')
229  parser.add_argument('verbose', action='store_true', default=False)
230  opts = parser.parse_args()
231
232  logging.getLogger().setLevel(logging.DEBUG if opts.verbose else logging.WARN)
233
234  deps_whitelist = DepsWhitelist()
235  blacklist = deps_whitelist.execute_method(opts.method, opts.path_to_deps)
236
237  if (opts.output_json):
238    output_dict = {
239        'blacklist' : blacklist
240    }
241    with open(opts.output_json, 'w') as output_json_file:
242      json.dump(output_dict, output_json_file)
243  else:
244    print blacklist
245
246  return 0
247
248
249if __name__ == '__main__':
250  sys.exit(main())
251