subversion_file_system.py revision a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7 
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import logging
6import posixpath
7import traceback
8import xml.dom.minidom as xml
9from xml.parsers.expat import ExpatError
10
11from appengine_url_fetcher import AppEngineUrlFetcher
12from docs_server_utils import StringIdentity
13from file_system import (
14    FileNotFoundError, FileSystem, FileSystemError, StatInfo)
15from future import Future
16import url_constants
17
18
19def _ParseHTML(html):
20  '''Unfortunately, the viewvc page has a stray </div> tag, so this takes care
21  of all mismatched tags.
22  '''
23  try:
24    return xml.parseString(html)
25  except ExpatError as e:
26    return _ParseHTML('\n'.join(
27        line for (i, line) in enumerate(html.split('\n'))
28        if e.lineno != i + 1))
29
30def _InnerText(node):
31  '''Like node.innerText in JS DOM, but strips surrounding whitespace.
32  '''
33  text = []
34  if node.nodeValue:
35    text.append(node.nodeValue)
36  if hasattr(node, 'childNodes'):
37    for child_node in node.childNodes:
38      text.append(_InnerText(child_node))
39  return ''.join(text).strip()
40
41def _CreateStatInfo(html):
42  parent_version = None
43  child_versions = {}
44
45  # Try all of the tables until we find the ones that contain the data (the
46  # directory and file versions are in different tables).
47  for table in _ParseHTML(html).getElementsByTagName('table'):
48    # Within the table there is a list of files. However, there may be some
49    # things beforehand; a header, "parent directory" list, etc. We will deal
50    # with that below by being generous and just ignoring such rows.
51    rows = table.getElementsByTagName('tr')
52
53    for row in rows:
54      cells = row.getElementsByTagName('td')
55
56      # The version of the directory will eventually appear in the soup of
57      # table rows, like this:
58      #
59      # <tr>
60      #   <td>Directory revision:</td>
61      #   <td><a href=... title="Revision 214692">214692</a> (of...)</td>
62      # </tr>
63      #
64      # So look out for that.
65      if len(cells) == 2 and _InnerText(cells[0]) == 'Directory revision:':
66        links = cells[1].getElementsByTagName('a')
67        if len(links) != 2:
68          raise FileSystemError('ViewVC assumption invalid: directory ' +
69                                'revision content did not have 2 <a> ' +
70                                ' elements, instead %s' % _InnerText(cells[1]))
71        this_parent_version = _InnerText(links[0])
72        int(this_parent_version)  # sanity check
73        if parent_version is not None:
74          raise FileSystemError('There was already a parent version %s, and ' +
75                                ' we just found a second at %s' %
76                                (parent_version, this_parent_version))
77        parent_version = this_parent_version
78
79      # The version of each file is a list of rows with 5 cells: name, version,
80      # age, author, and last log entry. Maybe the columns will change; we're
81      # at the mercy viewvc, but this constant can be easily updated.
82      if len(cells) != 5:
83        continue
84      name_element, version_element, _, __, ___ = cells
85
86      name = _InnerText(name_element)  # note: will end in / for directories
87      try:
88        version = int(_InnerText(version_element))
89      except StandardError:
90        continue
91      child_versions[name] = str(version)
92
93    if parent_version and child_versions:
94      break
95
96  return StatInfo(parent_version, child_versions)
97
98class _AsyncFetchFuture(object):
99  def __init__(self, paths, fetcher, args=None):
100    def apply_args(path):
101      return path if args is None else '%s?%s' % (path, args)
102    # A list of tuples of the form (path, Future).
103    self._fetches = [(path, fetcher.FetchAsync(apply_args(path)))
104                     for path in paths]
105    self._value = {}
106    self._error = None
107
108  def _ListDir(self, directory):
109    dom = xml.parseString(directory)
110    files = [elem.childNodes[0].data for elem in dom.getElementsByTagName('a')]
111    if '..' in files:
112      files.remove('..')
113    return files
114
115  def Get(self):
116    for path, future in self._fetches:
117      try:
118        result = future.Get()
119      except Exception as e:
120        raise FileSystemError('Error fetching %s for Get: %s' %
121            (path, traceback.format_exc()))
122
123      if result.status_code == 404:
124        raise FileNotFoundError('Got 404 when fetching %s for Get, content %s' %
125            (path, result.content))
126      if result.status_code != 200:
127        raise FileSystemError('Got %s when fetching %s for Get, content %s' %
128            (result.status_code, path, result.content))
129
130      if path.endswith('/'):
131        self._value[path] = self._ListDir(result.content)
132      else:
133        self._value[path] = result.content
134    if self._error is not None:
135      raise self._error
136    return self._value
137
138class SubversionFileSystem(FileSystem):
139  '''Class to fetch resources from src.chromium.org.
140  '''
141  @staticmethod
142  def Create(branch='trunk', revision=None):
143    if branch == 'trunk':
144      svn_path = 'trunk/src'
145    else:
146      svn_path = 'branches/%s/src' % branch
147    return SubversionFileSystem(
148        AppEngineUrlFetcher('%s/%s' % (url_constants.SVN_URL, svn_path)),
149        AppEngineUrlFetcher('%s/%s' % (url_constants.VIEWVC_URL, svn_path)),
150        svn_path,
151        revision=revision)
152
153  def __init__(self, file_fetcher, stat_fetcher, svn_path, revision=None):
154    self._file_fetcher = file_fetcher
155    self._stat_fetcher = stat_fetcher
156    self._svn_path = svn_path
157    self._revision = revision
158
159  def Read(self, paths):
160    args = None
161    if self._revision is not None:
162      # |fetcher| gets from svn.chromium.org which uses p= for version.
163      args = 'p=%s' % self._revision
164    return Future(delegate=_AsyncFetchFuture(paths,
165                                             self._file_fetcher,
166                                             args=args))
167
168  def Refresh(self):
169    return Future(value=())
170
171  def Stat(self, path):
172    directory, filename = posixpath.split(path)
173    if self._revision is not None:
174      # |stat_fetch| uses viewvc which uses pathrev= for version.
175      directory += '?pathrev=%s' % self._revision
176
177    try:
178      result = self._stat_fetcher.Fetch(directory)
179    except Exception as e:
180      raise FileSystemError('Error fetching %s for Stat: %s' %
181          (path, traceback.format_exc()))
182
183    if result.status_code == 404:
184      raise FileNotFoundError('Got 404 when fetching %s for Stat, content %s' %
185          (path, result.content))
186    if result.status_code != 200:
187      raise FileNotFoundError('Got %s when fetching %s for Stat, content %s' %
188          (result.status_code, path, result.content))
189
190    stat_info = _CreateStatInfo(result.content)
191    if stat_info.version is None:
192      raise FileSystemError('Failed to find version of dir %s' % directory)
193    if path == '' or path.endswith('/'):
194      return stat_info
195    if filename not in stat_info.child_versions:
196      raise FileNotFoundError(
197          '%s from %s was not in child versions for Stat' % (filename, path))
198    return StatInfo(stat_info.child_versions[filename])
199
200  def GetIdentity(self):
201    # NOTE: no revision here, since it would mess up the caching of reads. It
202    # probably doesn't matter since all the caching classes will use the result
203    # of Stat to decide whether to re-read - and Stat has a ceiling of the
204    # revision - so when the revision changes, so might Stat. That is enough.
205    return '@'.join((self.__class__.__name__, StringIdentity(self._svn_path)))
206