1#!/usr/bin/python
2#
3# This tool is used to compare headers between Bionic and NDK
4# script should be in development/ndk/tools for correct roots autodetection
5#
6
7import sys, os, os.path
8import subprocess
9import argparse, textwrap
10
11class FileCollector:
12    """Collect headers from Bionic and sysroot
13
14    sysincludes data format:
15    sysincludes                     -- dict with arch as key
16    sysincludes[arch]               -- dict with includes root as key
17    sysincludes[arch][root]         -- dict with header name as key
18    sysincludes[arch][root][header] -- list [last_platform, ..., first_platform]
19    """
20
21    def __init__(self, platforms_root, archs):
22        """Init platform roots and structures before collecting"""
23        self.platforms = []
24        self.archs = archs
25        self.sysincludes = {}
26        for arch in self.archs:
27            self.sysincludes[arch] = {}
28
29        ## scaning available platforms ##
30        for dirname in os.listdir(platforms_root):
31            path = os.path.join(platforms_root, dirname)
32            if os.path.isdir(path) and ('android' in dirname):
33                self.platforms.append(dirname)
34        try:
35            self.platforms.sort(key = lambda s: int(s.split('-')[1]))
36            self.root = platforms_root
37        except Exception:
38            print 'Wrong platforms list \n{0}'.format(str(self.platforms))
39
40    def scan_dir(self, root):
41        """Non-recursive file scan in directory"""
42        files = []
43        for filename in os.listdir(root):
44            if os.path.isfile(os.path.join(root, filename)):
45                files.append(filename)
46        return files
47
48    def scan_includes(self, root):
49        """Recursive includes scan in given root"""
50        includes = []
51        includes_root = os.path.join(root, 'include')
52        if not os.path.isdir(includes_root):
53            return includes
54
55        ## recursive scanning ##
56        includes.append(('', self.scan_dir(includes_root)))
57        for dirname, dirnames, filenames in os.walk(includes_root):
58            for subdirname in dirnames:
59                path = os.path.join(dirname, subdirname)
60                relpath = os.path.relpath(path, includes_root)
61                includes.append((relpath, self.scan_dir(path)))
62
63        return includes
64
65    def scan_archs_includes(self, root):
66        """Scan includes for all defined archs in given root"""
67        includes = {}
68        includes['common'] = self.scan_includes(root)
69
70        for arch in [a for a in self.archs if a != 'common']:
71            arch_root = os.path.join(root, arch)
72            includes[arch] = self.scan_includes(arch_root)
73
74        return includes
75
76    def scan_platform_includes(self, platform):
77        """Scan all platform includes of one layer"""
78        platform_root = os.path.join(self.root, platform)
79        return self.scan_archs_includes(platform_root)
80
81    def scan_bionic_includes(self, bionic_root):
82        """Scan Bionic's libc includes"""
83        self.bionic_root = bionic_root
84        self.bionic_includes = self.scan_archs_includes(bionic_root)
85
86    def append_sysincludes(self, arch, root, headers, platform):
87        """Merge new platform includes layer with current sysincludes"""
88        if not (root in self.sysincludes[arch]):
89            self.sysincludes[arch][root] = {}
90
91        for include in headers:
92            if include in self.sysincludes[arch][root]:
93                last_platform = self.sysincludes[arch][root][include][0]
94                if platform != last_platform:
95                    self.sysincludes[arch][root][include].insert(0, platform)
96            else:
97                self.sysincludes[arch][root][include] = [platform]
98
99    def update_to_platform(self, platform):
100        """Update sysincludes state by applying new platform layer"""
101        new_includes = self.scan_platform_includes(platform)
102        for arch in self.archs:
103            for pack in new_includes[arch]:
104                self.append_sysincludes(arch, pack[0], pack[1], platform)
105
106    def scan_sysincludes(self, target_platform):
107        """Fully automated sysincludes collector upto specified platform"""
108        version = int(target_platform.split('-')[1])
109        layers = filter(lambda s: int(s.split('-')[1]) <= version, self.platforms)
110        for platform in layers:
111            self.update_to_platform(platform)
112
113
114class BionicSysincludes:
115    def set_roots(self):
116        """Automated roots initialization (AOSP oriented)"""
117        script_root = os.path.dirname(os.path.realpath(__file__))
118        self.aosp_root      = os.path.normpath(os.path.join(script_root, '../../..'))
119        self.platforms_root = os.path.join(self.aosp_root, 'development/ndk/platforms')
120        self.bionic_root    = os.path.join(self.aosp_root, 'bionic/libc')
121
122    def scan_includes(self):
123        """Scan all required includes"""
124        self.collector = FileCollector(self.platforms_root, self.archs)
125        ## detecting latest platform ##
126        self.platforms = self.collector.platforms
127        latest_platform = self.platforms[-1:][0]
128        ## scanning both includes repositories ##
129        self.collector.scan_sysincludes(latest_platform)
130        self.collector.scan_bionic_includes(self.bionic_root)
131        ## scan results ##
132        self.sysincludes     = self.collector.sysincludes
133        self.bionic_includes = self.collector.bionic_includes
134
135    def git_diff(self, file_origin, file_probe):
136        """Difference routine based on git diff"""
137        try:
138            subprocess.check_output(['git', 'diff', '--no-index', file_origin, file_probe])
139        except subprocess.CalledProcessError as error:
140            return error.output
141        return None
142
143    def match_with_bionic_includes(self):
144        """Compare headers between Bionic and sysroot"""
145        self.diffs = {}
146        ## for every arch ##
147        for arch in self.archs:
148            arch_root = (lambda s: s if s != 'common' else '')(arch)
149            ## for every includes directory ##
150            for pack in self.bionic_includes[arch]:
151                root = pack[0]
152                path_bionic = os.path.join(self.bionic_root, arch_root, 'include', root)
153                ## for every header that both in Bionic and sysroot ##
154                for include in pack[1]:
155                    if include in self.sysincludes[arch][root]:
156                        ## completing paths ##
157                        platform = self.sysincludes[arch][root][include][0]
158                        file_origin = os.path.join(path_bionic, include)
159                        file_probe  = os.path.join(self.platforms_root, platform, arch_root, 'include', root, include)
160                        ## comparison by git diff ##
161                        output = self.git_diff(file_origin, file_probe)
162                        if output is not None:
163                            if arch not in self.diffs:
164                                self.diffs[arch] = {}
165                            if root not in self.diffs[arch]:
166                                self.diffs[arch][root] = {}
167                            ## storing git diff ##
168                            self.diffs[arch][root][include] = output
169
170    def print_history(self, arch, root, header):
171        """Print human-readable list header updates across platforms"""
172        history = self.sysincludes[arch][root][header]
173        for platform in self.platforms:
174            entry = (lambda s: s.split('-')[1] if s in history else '-')(platform)
175            print '{0:3}'.format(entry),
176        print ''
177
178    def show_and_store_results(self):
179        """Print summary list of headers and write diff-report to file"""
180        try:
181            diff_fd = open(self.diff_file, 'w')
182            for arch in self.archs:
183                if arch not in self.diffs:
184                    continue
185                print '{0}/'.format(arch)
186                roots = self.diffs[arch].keys()
187                roots.sort()
188                for root in roots:
189                    print '    {0}/'.format((lambda s: s if s != '' else '../include')(root))
190                    includes = self.diffs[arch][root].keys()
191                    includes.sort()
192                    for include in includes:
193                        print '        {0:32}'.format(include),
194                        self.print_history(arch, root, include)
195                        diff = self.diffs[arch][root][include]
196                        diff_fd.write(diff)
197                        diff_fd.write('\n\n')
198                    print ''
199                print ''
200
201        finally:
202            diff_fd.close()
203
204    def main(self):
205        self.set_roots()
206        self.scan_includes()
207        self.match_with_bionic_includes()
208        self.show_and_store_results()
209
210if __name__ == '__main__':
211    ## configuring command line parser ##
212    parser = argparse.ArgumentParser(formatter_class = argparse.RawTextHelpFormatter,
213                                     description = 'Headers comparison tool between bionic and NDK platforms')
214    parser.epilog = textwrap.dedent('''
215    output format:
216    {architecture}/
217        {directory}/
218            {header name}.h  {platforms history}
219
220    platforms history format:
221        number X means header has been changed in android-X
222        `-\' means it is the same
223
224    diff-report format:
225        git diff output for all headers
226        use --diff option to specify filename
227    ''')
228
229    parser.add_argument('--archs', metavar = 'A', nargs = '+',
230                        default = ['common', 'arm', 'x86', 'mips'],
231                        help = 'list of architectures\n(default: common arm x86 mips)')
232    parser.add_argument('--diff', metavar = 'FILE', nargs = 1,
233                        default = ['headers-diff-bionic-vs-ndk.diff'],
234                        help = 'diff-report filename\n(default: `bionic-vs-sysincludes_report.diff\')')
235
236    ## parsing arguments ##
237    args = parser.parse_args()
238
239    ## doing work ##
240    app = BionicSysincludes()
241    app.archs = map((lambda s: 'arch-{0}'.format(s) if s != 'common' else s), args.archs)
242    app.diff_file = args.diff[0]
243    app.main()
244
245    print 'Headers listed above are DIFFERENT in Bionic and NDK platforms'
246    print 'See `{0}\' for details'.format(app.diff_file)
247    print 'See --help for format description.'
248    print ''
249