1#!/usr/bin/env python
2# A tool to parse ASTMatchers.h and update the documentation in
3# ../LibASTMatchersReference.html automatically. Run from the
4# directory in which this file is located to update the docs.
5
6import collections
7import re
8import urllib2
9
10MATCHERS_FILE = '../../include/clang/ASTMatchers/ASTMatchers.h'
11
12# Each matcher is documented in one row of the form:
13#   result | name | argA
14# The subsequent row contains the documentation and is hidden by default,
15# becoming visible via javascript when the user clicks the matcher name.
16TD_TEMPLATE="""
17<tr><td>%(result)s</td><td class="name" onclick="toggle('%(id)s')"><a name="%(id)sAnchor">%(name)s</a></td><td>%(args)s</td></tr>
18<tr><td colspan="4" class="doc" id="%(id)s"><pre>%(comment)s</pre></td></tr>
19"""
20
21# We categorize the matchers into these three categories in the reference:
22node_matchers = {}
23narrowing_matchers = {}
24traversal_matchers = {}
25
26# We output multiple rows per matcher if the matcher can be used on multiple
27# node types. Thus, we need a new id per row to control the documentation
28# pop-up. ids[name] keeps track of those ids.
29ids = collections.defaultdict(int)
30
31# Cache for doxygen urls we have already verified.
32doxygen_probes = {}
33
34def esc(text):
35  """Escape any html in the given text."""
36  text = re.sub(r'&', '&amp;', text)
37  text = re.sub(r'<', '&lt;', text)
38  text = re.sub(r'>', '&gt;', text)
39  def link_if_exists(m):
40    name = m.group(1)
41    url = 'http://clang.llvm.org/doxygen/classclang_1_1%s.html' % name
42    if url not in doxygen_probes:
43      try:
44        print 'Probing %s...' % url
45        urllib2.urlopen(url)
46        doxygen_probes[url] = True
47      except:
48        doxygen_probes[url] = False
49    if doxygen_probes[url]:
50      return r'Matcher&lt<a href="%s">%s</a>&gt;' % (url, name)
51    else:
52      return m.group(0)
53  text = re.sub(
54    r'Matcher&lt;([^\*&]+)&gt;', link_if_exists, text)
55  return text
56
57def extract_result_types(comment):
58  """Extracts a list of result types from the given comment.
59
60     We allow annotations in the comment of the matcher to specify what
61     nodes a matcher can match on. Those comments have the form:
62       Usable as: Any Matcher | (Matcher<T1>[, Matcher<t2>[, ...]])
63
64     Returns ['*'] in case of 'Any Matcher', or ['T1', 'T2', ...].
65     Returns the empty list if no 'Usable as' specification could be
66     parsed.
67  """
68  result_types = []
69  m = re.search(r'Usable as: Any Matcher[\s\n]*$', comment, re.S)
70  if m:
71    return ['*']
72  while True:
73    m = re.match(r'^(.*)Matcher<([^>]+)>\s*,?[\s\n]*$', comment, re.S)
74    if not m:
75      if re.search(r'Usable as:\s*$', comment):
76        return result_types
77      else:
78        return None
79    result_types += [m.group(2)]
80    comment = m.group(1)
81
82def strip_doxygen(comment):
83  """Returns the given comment without \-escaped words."""
84  # If there is only a doxygen keyword in the line, delete the whole line.
85  comment = re.sub(r'^\\[^\s]+\n', r'', comment, flags=re.M)
86  # Delete the doxygen command and the following whitespace.
87  comment = re.sub(r'\\[^\s]+\s+', r'', comment)
88  return comment
89
90def unify_arguments(args):
91  """Gets rid of anything the user doesn't care about in the argument list."""
92  args = re.sub(r'internal::', r'', args)
93  args = re.sub(r'const\s+', r'', args)
94  args = re.sub(r'&', r' ', args)
95  args = re.sub(r'(^|\s)M\d?(\s)', r'\1Matcher<*>\2', args)
96  return args
97
98def add_matcher(result_type, name, args, comment, is_dyncast=False):
99  """Adds a matcher to one of our categories."""
100  if name == 'id':
101     # FIXME: Figure out whether we want to support the 'id' matcher.
102     return
103  matcher_id = '%s%d' % (name, ids[name])
104  ids[name] += 1
105  args = unify_arguments(args)
106  matcher_html = TD_TEMPLATE % {
107    'result': esc('Matcher<%s>' % result_type),
108    'name': name,
109    'args': esc(args),
110    'comment': esc(strip_doxygen(comment)),
111    'id': matcher_id,
112  }
113  if is_dyncast:
114    node_matchers[result_type + name] = matcher_html
115  # Use a heuristic to figure out whether a matcher is a narrowing or
116  # traversal matcher. By default, matchers that take other matchers as
117  # arguments (and are not node matchers) do traversal. We specifically
118  # exclude known narrowing matchers that also take other matchers as
119  # arguments.
120  elif ('Matcher<' not in args or
121        name in ['allOf', 'anyOf', 'anything', 'unless']):
122    narrowing_matchers[result_type + name] = matcher_html
123  else:
124    traversal_matchers[result_type + name] = matcher_html
125
126def act_on_decl(declaration, comment, allowed_types):
127  """Parse the matcher out of the given declaration and comment.
128
129     If 'allowed_types' is set, it contains a list of node types the matcher
130     can match on, as extracted from the static type asserts in the matcher
131     definition.
132  """
133  if declaration.strip():
134    # Node matchers are defined by writing:
135    #   VariadicDynCastAllOfMatcher<ResultType, ArgumentType> name;
136    m = re.match(r""".*VariadicDynCastAllOfMatcher\s*<
137                       \s*([^\s,]+)\s*,
138                       \s*([^\s>]+)\s*>
139                       \s*([^\s;]+)\s*;\s*$""", declaration, flags=re.X)
140    if m:
141      result, inner, name = m.groups()
142      add_matcher(result, name, 'Matcher<%s>...' % inner,
143                  comment, is_dyncast=True)
144      return
145
146    # Parse the various matcher definition macros.
147    m = re.match(r"""^\s*AST_(POLYMORPHIC_)?MATCHER(_P)?(.?)\(
148                       (?:\s*([^\s,]+)\s*,)?
149                          \s*([^\s,]+)\s*
150                       (?:,\s*([^\s,]+)\s*
151                          ,\s*([^\s,]+)\s*)?
152                       (?:,\s*([^\s,]+)\s*
153                          ,\s*([^\s,]+)\s*)?
154                      \)\s*{\s*$""", declaration, flags=re.X)
155    if m:
156      p, n, result, name = m.groups()[1:5]
157      args = m.groups()[5:]
158      if not result:
159        if not allowed_types:
160          raise Exception('Did not find allowed result types for: %s' % name)
161        result_types = allowed_types
162      else:
163        result_types = [result]
164      if n not in ['', '2']:
165        raise Exception('Cannot parse "%s"' % declaration)
166      args = ', '.join('%s %s' % (args[i], args[i+1])
167                       for i in range(0, len(args), 2) if args[i])
168      for result_type in result_types:
169        add_matcher(result_type, name, args, comment)
170      return
171
172    # Parse free standing matcher functions, like:
173    #   Matcher<ResultType> Name(Matcher<ArgumentType> InnerMatcher) {
174    m = re.match(r"""^\s*(.*)\s+
175                     ([^\s\(]+)\s*\(
176                     (.*)
177                     \)\s*{""", declaration, re.X)
178    if m:
179      result, name, args = m.groups()
180      args = ', '.join(p.strip() for p in args.split(','))
181      m = re.match(r'.*\s+internal::Matcher<([^>]+)>$', result)
182      if m:
183        result_types = [m.group(1)]
184      else:
185        result_types = extract_result_types(comment)
186      if not result_types:
187        if not comment:
188          # Only overloads don't have their own doxygen comments; ignore those.
189          print 'Ignoring "%s"' % name
190        else:
191          print 'Cannot determine result type for "%s"' % name
192      else:
193        for result_type in result_types:
194          add_matcher(result_type, name, args, comment)
195    else:
196      print '*** Unparsable: "' + declaration + '" ***'
197
198def sort_table(matcher_type, matcher_map):
199  """Returns the sorted html table for the given row map."""
200  table = ''
201  for key in sorted(matcher_map.keys()):
202    table += matcher_map[key] + '\n'
203  return ('<!-- START_%(type)s_MATCHERS -->\n' +
204          '%(table)s' +
205          '<!--END_%(type)s_MATCHERS -->') % {
206    'type': matcher_type,
207    'table': table,
208  }
209
210# Parse the ast matchers.
211# We alternate between two modes:
212# body = True: We parse the definition of a matcher. We need
213#   to parse the full definition before adding a matcher, as the
214#   definition might contain static asserts that specify the result
215#   type.
216# body = False: We parse the comments and declaration of the matcher.
217comment = ''
218declaration = ''
219allowed_types = []
220body = False
221for line in open(MATCHERS_FILE).read().splitlines():
222  if body:
223    if line.strip() and line[0] == '}':
224      if declaration:
225        act_on_decl(declaration, comment, allowed_types)
226        comment = ''
227        declaration = ''
228        allowed_types = []
229      body = False
230    else:
231      m = re.search(r'is_base_of<([^,]+), NodeType>', line)
232      if m and m.group(1):
233        allowed_types += [m.group(1)]
234    continue
235  if line.strip() and line.lstrip()[0] == '/':
236    comment += re.sub(r'/+\s?', '', line) + '\n'
237  else:
238    declaration += ' ' + line
239    if ((not line.strip()) or
240        line.rstrip()[-1] == ';' or
241        line.rstrip()[-1] == '{'):
242      if line.strip() and line.rstrip()[-1] == '{':
243        body = True
244      else:
245        act_on_decl(declaration, comment, allowed_types)
246        comment = ''
247        declaration = ''
248        allowed_types = []
249
250node_matcher_table = sort_table('DECL', node_matchers)
251narrowing_matcher_table = sort_table('NARROWING', narrowing_matchers)
252traversal_matcher_table = sort_table('TRAVERSAL', traversal_matchers)
253
254reference = open('../LibASTMatchersReference.html').read()
255reference = re.sub(r'<!-- START_DECL_MATCHERS.*END_DECL_MATCHERS -->',
256                   '%s', reference, flags=re.S) % node_matcher_table
257reference = re.sub(r'<!-- START_NARROWING_MATCHERS.*END_NARROWING_MATCHERS -->',
258                   '%s', reference, flags=re.S) % narrowing_matcher_table
259reference = re.sub(r'<!-- START_TRAVERSAL_MATCHERS.*END_TRAVERSAL_MATCHERS -->',
260                   '%s', reference, flags=re.S) % traversal_matcher_table
261
262with open('../LibASTMatchersReference.html', 'w') as output:
263  output.write(reference)
264
265