repo_to_repo.py revision f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbe 
1#!/usr/bin/python
2#
3# Copyright 2010 Google Inc. All Rights Reserved.
4
5__author__ = 'asharif@google.com (Ahmad Sharif)'
6
7import datetime
8import optparse
9import os
10import re
11import socket
12import sys
13import tempfile
14
15from automation.clients.helper import perforce
16from utils import command_executer
17from utils import logger
18from utils import misc
19
20
21def GetCanonicalMappings(mappings):
22  canonical_mappings = []
23  for mapping in mappings:
24    remote_path, local_path = mapping.split()
25    if local_path.endswith('/') and not remote_path.endswith('/'):
26      local_path = os.path.join(local_path, os.path.basename(remote_path))
27    remote_path = remote_path.lstrip('/').split('/', 1)[1]
28    canonical_mappings.append(perforce.PathMapping(remote_path, local_path))
29  return canonical_mappings
30
31
32def SplitMapping(mapping):
33  parts = mapping.split()
34  assert len(parts) <= 2, 'Mapping %s invalid' % mapping
35  remote_path = parts[0]
36  if len(parts) == 2:
37    local_path = parts[1]
38  else:
39    local_path = '.'
40  return remote_path, local_path
41
42
43class Repo(object):
44
45  def __init__(self, no_create_tmp_dir=False):
46    self.repo_type = None
47    self.address = None
48    self.mappings = None
49    self.revision = None
50    self.ignores = ['.gitignore', '.p4config', 'README.google']
51    if no_create_tmp_dir:
52      self._root_dir = None
53    else:
54      self._root_dir = tempfile.mkdtemp()
55    self._ce = command_executer.GetCommandExecuter()
56    self._logger = logger.GetLogger()
57
58  def PullSources(self):
59    """Pull all sources into an internal dir."""
60    pass
61
62  def SetupForPush(self):
63    """Setup a repository for pushing later."""
64    pass
65
66  def PushSources(self, commit_message=None, dry_run=False, message_file=None):
67    """Push to the external repo with the commit message."""
68    pass
69
70  def _RsyncExcludingRepoDirs(self, source_dir, dest_dir):
71    for f in os.listdir(source_dir):
72      if f in ['.git', '.svn', '.p4config']:
73        continue
74      dest_file = os.path.join(dest_dir, f)
75      source_file = os.path.join(source_dir, f)
76      if os.path.exists(dest_file):
77        command = 'rm -rf %s' % dest_file
78        self._ce.RunCommand(command)
79      command = 'rsync -a %s %s' % (source_file, dest_dir)
80      self._ce.RunCommand(command)
81    return 0
82
83  def MapSources(self, dest_dir):
84    """Copy sources from the internal dir to root_dir."""
85    return self._RsyncExcludingRepoDirs(self._root_dir, dest_dir)
86
87  def GetRoot(self):
88    return self._root_dir
89
90  def CleanupRoot(self):
91    command = 'rm -rf %s' % self._root_dir
92    return self._ce.RunCommand(command)
93
94  def __str__(self):
95    return '\n'.join(str(s)
96                     for s in [self.repo_type, self.address, self.mappings])
97
98
99# Note - this type of repo is used only for "readonly", in other words, this
100# only serves as a incoming repo.
101class FileRepo(Repo):
102
103  def __init__(self, address, ignores=None):
104    Repo.__init__(self, no_create_tmp_dir=True)
105    self.repo_type = 'file'
106    self.address = address
107    self.mappings = None
108    self.branch = None
109    self.revision = '{0} (as of "{1}")'.format(address, datetime.datetime.now())
110    self.gerrit = None
111    self._root_dir = self.address
112
113  def CleanupRoot(self):
114    """Override to prevent deletion."""
115    pass
116
117
118class P4Repo(Repo):
119
120  def __init__(self, address, mappings, revision=None):
121    Repo.__init__(self)
122    self.repo_type = 'p4'
123    self.address = address
124    self.mappings = mappings
125    self.revision = revision
126
127  def PullSources(self):
128    client_name = socket.gethostname()
129    client_name += tempfile.mkstemp()[1].replace('/', '-')
130    mappings = self.mappings
131    p4view = perforce.View('depot2', GetCanonicalMappings(mappings))
132    p4client = perforce.CommandsFactory(self._root_dir,
133                                        p4view,
134                                        name=client_name)
135    command = p4client.SetupAndDo(p4client.Sync(self.revision))
136    ret = self._ce.RunCommand(command)
137    assert ret == 0, 'Could not setup client.'
138    command = p4client.InCheckoutDir(p4client.SaveCurrentCLNumber())
139    ret, o, _ = self._ce.RunCommandWOutput(command)
140    assert ret == 0, 'Could not get version from client.'
141    self.revision = re.search('^\d+$', o.strip(), re.MULTILINE).group(0)
142    command = p4client.InCheckoutDir(p4client.Remove())
143    ret = self._ce.RunCommand(command)
144    assert ret == 0, 'Could not delete client.'
145    return 0
146
147
148class SvnRepo(Repo):
149
150  def __init__(self, address, mappings):
151    Repo.__init__(self)
152    self.repo_type = 'svn'
153    self.address = address
154    self.mappings = mappings
155
156  def PullSources(self):
157    with misc.WorkingDirectory(self._root_dir):
158      for mapping in self.mappings:
159        remote_path, local_path = SplitMapping(mapping)
160        command = 'svn co %s/%s %s' % (self.address, remote_path, local_path)
161      ret = self._ce.RunCommand(command)
162      if ret:
163        return ret
164
165      self.revision = ''
166      for mapping in self.mappings:
167        remote_path, local_path = SplitMapping(mapping)
168        command = 'cd %s && svnversion -c .' % (local_path)
169        ret, o, _ = self._ce.RunCommandWOutput(command)
170        self.revision += o.strip().split(':')[-1]
171        if ret:
172          return ret
173    return 0
174
175
176class GitRepo(Repo):
177
178  def __init__(self, address, branch, mappings=None, ignores=None, gerrit=None):
179    Repo.__init__(self)
180    self.repo_type = 'git'
181    self.address = address
182    self.branch = branch or 'master'
183    if ignores:
184      self.ignores += ignores
185    self.mappings = mappings
186    self.gerrit = gerrit
187
188  def _CloneSources(self):
189    with misc.WorkingDirectory(self._root_dir):
190      command = 'git clone %s .' % (self.address)
191      return self._ce.RunCommand(command)
192
193  def PullSources(self):
194    with misc.WorkingDirectory(self._root_dir):
195      ret = self._CloneSources()
196      if ret:
197        return ret
198
199      command = 'git checkout %s' % self.branch
200      ret = self._ce.RunCommand(command)
201      if ret:
202        return ret
203
204      command = 'git describe --always'
205      ret, o, _ = self._ce.RunCommandWOutput(command)
206      self.revision = o.strip()
207      return ret
208
209  def SetupForPush(self):
210    with misc.WorkingDirectory(self._root_dir):
211      ret = self._CloneSources()
212      logger.GetLogger().LogFatalIf(ret, 'Could not clone git repo %s.' %
213                                    self.address)
214
215      command = 'git branch -a | grep -wq %s' % self.branch
216      ret = self._ce.RunCommand(command)
217
218      if ret == 0:
219        if self.branch != 'master':
220          command = ('git branch --track %s remotes/origin/%s' %
221                     (self.branch, self.branch))
222        else:
223          command = 'pwd'
224        command += '&& git checkout %s' % self.branch
225      else:
226        command = 'git symbolic-ref HEAD refs/heads/%s' % self.branch
227      command += '&& rm -rf *'
228      ret = self._ce.RunCommand(command)
229      return ret
230
231  def CommitLocally(self, commit_message=None, message_file=None):
232    with misc.WorkingDirectory(self._root_dir):
233      command = 'pwd'
234      for ignore in self.ignores:
235        command += '&& echo \'%s\' >> .git/info/exclude' % ignore
236      command += '&& git add -Av .'
237      if message_file:
238        message_arg = '-F %s' % message_file
239      elif commit_message:
240        message_arg = '-m \'%s\'' % commit_message
241      else:
242        raise Exception('No commit message given!')
243      command += '&& git commit -v %s' % message_arg
244      return self._ce.RunCommand(command)
245
246  def PushSources(self, commit_message=None, dry_run=False, message_file=None):
247    ret = self.CommitLocally(commit_message, message_file)
248    if ret:
249      return ret
250    push_args = ''
251    if dry_run:
252      push_args += ' -n '
253    with misc.WorkingDirectory(self._root_dir):
254      if self.gerrit:
255        label = 'somelabel'
256        command = 'git remote add %s %s' % (label, self.address)
257        command += ('&& git push %s %s HEAD:refs/for/master' %
258                    (push_args, label))
259      else:
260        command = 'git push -v %s origin %s:%s' % (push_args, self.branch,
261                                                   self.branch)
262      ret = self._ce.RunCommand(command)
263    return ret
264
265  def MapSources(self, root_dir):
266    if not self.mappings:
267      self._RsyncExcludingRepoDirs(self._root_dir, root_dir)
268      return
269    with misc.WorkingDirectory(self._root_dir):
270      for mapping in self.mappings:
271        remote_path, local_path = SplitMapping(mapping)
272        remote_path.rstrip('...')
273        local_path.rstrip('...')
274        full_local_path = os.path.join(root_dir, local_path)
275        ret = self._RsyncExcludingRepoDirs(remote_path, full_local_path)
276        if ret:
277          return ret
278    return 0
279
280
281class RepoReader(object):
282
283  def __init__(self, filename):
284    self.filename = filename
285    self.main_dict = {}
286    self.input_repos = []
287    self.output_repos = []
288
289  def ParseFile(self):
290    with open(self.filename) as f:
291      self.main_dict = eval(f.read())
292      self.CreateReposFromDict(self.main_dict)
293    return [self.input_repos, self.output_repos]
294
295  def CreateReposFromDict(self, main_dict):
296    for key, repo_list in main_dict.items():
297      for repo_dict in repo_list:
298        repo = self.CreateRepoFromDict(repo_dict)
299        if key == 'input':
300          self.input_repos.append(repo)
301        elif key == 'output':
302          self.output_repos.append(repo)
303        else:
304          logger.GetLogger().LogFatal('Unknown key: %s found' % key)
305
306  def CreateRepoFromDict(self, repo_dict):
307    repo_type = repo_dict.get('type', None)
308    repo_address = repo_dict.get('address', None)
309    repo_mappings = repo_dict.get('mappings', None)
310    repo_ignores = repo_dict.get('ignores', None)
311    repo_branch = repo_dict.get('branch', None)
312    gerrit = repo_dict.get('gerrit', None)
313    revision = repo_dict.get('revision', None)
314
315    if repo_type == 'p4':
316      repo = P4Repo(repo_address, repo_mappings, revision=revision)
317    elif repo_type == 'svn':
318      repo = SvnRepo(repo_address, repo_mappings)
319    elif repo_type == 'git':
320      repo = GitRepo(repo_address,
321                     repo_branch,
322                     mappings=repo_mappings,
323                     ignores=repo_ignores,
324                     gerrit=gerrit)
325    elif repo_type == 'file':
326      repo = FileRepo(repo_address)
327    else:
328      logger.GetLogger().LogFatal('Unknown repo type: %s' % repo_type)
329    return repo
330
331
332@logger.HandleUncaughtExceptions
333def Main(argv):
334  parser = optparse.OptionParser()
335  parser.add_option('-i',
336                    '--input_file',
337                    dest='input_file',
338                    help='The input file that contains repo descriptions.')
339
340  parser.add_option('-n',
341                    '--dry_run',
342                    dest='dry_run',
343                    action='store_true',
344                    default=False,
345                    help='Do a dry run of the push.')
346
347  parser.add_option('-F',
348                    '--message_file',
349                    dest='message_file',
350                    default=None,
351                    help='Use contents of the log file as the commit message.')
352
353  options = parser.parse_args(argv)[0]
354  if not options.input_file:
355    parser.print_help()
356    return 1
357  rr = RepoReader(options.input_file)
358  [input_repos, output_repos] = rr.ParseFile()
359
360  # Make sure FileRepo is not used as output destination.
361  for output_repo in output_repos:
362    if output_repo.repo_type == 'file':
363      logger.GetLogger().LogFatal(
364          'FileRepo is only supported as an input repo.')
365
366  for output_repo in output_repos:
367    ret = output_repo.SetupForPush()
368    if ret:
369      return ret
370
371  input_revisions = []
372  for input_repo in input_repos:
373    ret = input_repo.PullSources()
374    if ret:
375      return ret
376    input_revisions.append(input_repo.revision)
377
378  for input_repo in input_repos:
379    for output_repo in output_repos:
380      ret = input_repo.MapSources(output_repo.GetRoot())
381      if ret:
382        return ret
383
384  commit_message = 'Synced repos to: %s' % ','.join(input_revisions)
385  for output_repo in output_repos:
386    ret = output_repo.PushSources(commit_message=commit_message,
387                                  dry_run=options.dry_run,
388                                  message_file=options.message_file)
389    if ret:
390      return ret
391
392  if not options.dry_run:
393    for output_repo in output_repos:
394      output_repo.CleanupRoot()
395    for input_repo in input_repos:
396      input_repo.CleanupRoot()
397
398  return ret
399
400
401if __name__ == '__main__':
402  retval = Main(sys.argv)
403  sys.exit(retval)
404