repo_to_repo.py revision f81680c018729fd4499e1e200d04b48c4b90127c 
1#!/usr/bin/python
2#
3# Copyright 2010 Google Inc. All Rights Reserved.
4
5__author__ = 'asharif@google.com (Ahmad Sharif)'
6
7import optparse
8import os
9import re
10import socket
11import sys
12import tempfile
13
14from automation.clients.helper import perforce
15from utils import command_executer
16from utils import logger
17from utils import misc
18
19
20def GetCanonicalMappings(mappings):
21  canonical_mappings = []
22  for mapping in mappings:
23    remote_path, local_path = mapping.split()
24    if local_path.endswith('/') and not remote_path.endswith('/'):
25      local_path = os.path.join(local_path, os.path.basename(remote_path))
26    remote_path = remote_path.lstrip('/').split('/', 1)[1]
27    canonical_mappings.append(perforce.PathMapping(remote_path, local_path))
28  return canonical_mappings
29
30
31def SplitMapping(mapping):
32  parts = mapping.split()
33  assert len(parts) <= 2, 'Mapping %s invalid' % mapping
34  remote_path = parts[0]
35  if len(parts) == 2:
36    local_path = parts[1]
37  else:
38    local_path = '.'
39  return remote_path, local_path
40
41
42class Repo(object):
43  def __init__(self):
44    self.repo_type = None
45    self.address = None
46    self.mappings = None
47    self.revision = None
48    self.ignores = ['.gitignore', '.p4config', 'README.google']
49    self._root_dir = tempfile.mkdtemp()
50    self._ce = command_executer.GetCommandExecuter()
51    self._logger = logger.GetLogger()
52
53  def PullSources(self):
54    """Pull all sources into an internal dir."""
55    pass
56
57  def SetupForPush(self):
58    """Setup a repository for pushing later."""
59    pass
60
61  def PushSources(self, commit_message=None, dry_run=False, message_file=None):
62    """Push to the external repo with the commit message."""
63    pass
64
65  def _RsyncExcludingRepoDirs(self, source_dir, dest_dir):
66    for f in os.listdir(source_dir):
67      if f in [".git", ".svn", ".p4config"]:
68        continue
69      dest_file = os.path.join(dest_dir, f)
70      source_file = os.path.join(source_dir, f)
71      if os.path.exists(dest_file):
72        command = "rm -rf %s" % dest_file
73        self._ce.RunCommand(command)
74      command = "rsync -a %s %s" % (source_file, dest_dir)
75      self._ce.RunCommand(command)
76    return 0
77
78  def MapSources(self, dest_dir):
79    """Copy sources from the internal dir to root_dir."""
80    return self._RsyncExcludingRepoDirs(self._root_dir, dest_dir)
81
82  def GetRoot(self):
83    return self._root_dir
84
85  def CleanupRoot(self):
86    command = 'rm -rf %s' % self._root_dir
87    return self._ce.RunCommand(command)
88
89  def __str__(self):
90    return '\n'.join(str(s) for s in [self.repo_type,
91                                      self.address,
92                                      self.mappings])
93
94
95class P4Repo(Repo):
96  def __init__(self, address, mappings, revision=None):
97    Repo.__init__(self)
98    self.repo_type = 'p4'
99    self.address = address
100    self.mappings = mappings
101    self.revision = revision
102
103  def PullSources(self):
104    client_name = socket.gethostname()
105    client_name += tempfile.mkstemp()[1].replace('/', '-')
106    mappings = self.mappings
107    p4view = perforce.View('depot2',
108                           GetCanonicalMappings(mappings))
109    p4client = perforce.CommandsFactory(self._root_dir, p4view,
110                                        name=client_name)
111    command = p4client.SetupAndDo(p4client.Sync(self.revision))
112    ret = self._ce.RunCommand(command)
113    assert ret == 0, 'Could not setup client.'
114    command = p4client.InCheckoutDir(p4client.SaveCurrentCLNumber())
115    ret, o, _ = self._ce.RunCommand(command, return_output=True)
116    assert ret == 0, 'Could not get version from client.'
117    self.revision = re.search('^\d+$', o.strip(), re.MULTILINE).group(0)
118    command = p4client.InCheckoutDir(p4client.Remove())
119    ret = self._ce.RunCommand(command)
120    assert ret == 0, 'Could not delete client.'
121    return 0
122
123
124class SvnRepo(Repo):
125  def __init__(self, address, mappings):
126    Repo.__init__(self)
127    self.repo_type = 'svn'
128    self.address = address
129    self.mappings = mappings
130
131  def PullSources(self):
132    with misc.WorkingDirectory(self._root_dir):
133      for mapping in self.mappings:
134        remote_path, local_path = SplitMapping(mapping)
135        command = 'svn co %s/%s %s' % (self.address, remote_path, local_path)
136      ret = self._ce.RunCommand(command)
137      if ret: return ret
138
139      self.revision = ''
140      for mapping in self.mappings:
141        remote_path, local_path = SplitMapping(mapping)
142        command = 'cd %s && svnversion -c .' % (local_path)
143        ret, o, _ = self._ce.RunCommand(command, return_output=True)
144        self.revision += o.strip().split(":")[-1]
145        if ret: return ret
146    return 0
147
148
149class GitRepo(Repo):
150  def __init__(self, address, branch, mappings=None, ignores=None, gerrit=None):
151    Repo.__init__(self)
152    self.repo_type = 'git'
153    self.address = address
154    self.branch = branch or 'master'
155    if ignores:
156      self.ignores += ignores
157    self.mappings = mappings
158    self.gerrit = gerrit
159
160  def _CloneSources(self):
161    with misc.WorkingDirectory(self._root_dir):
162      command = 'git clone %s .' % (self.address)
163      return self._ce.RunCommand(command)
164
165  def PullSources(self):
166    with misc.WorkingDirectory(self._root_dir):
167      ret = self._CloneSources()
168      if ret: return ret
169
170      command = 'git checkout %s' % self.branch
171      ret = self._ce.RunCommand(command)
172      if ret: return ret
173
174      command = 'git describe --always'
175      ret, o, _ = self._ce.RunCommand(command, return_output=True)
176      self.revision = o.strip()
177      return ret
178
179  def SetupForPush(self):
180    with misc.WorkingDirectory(self._root_dir):
181      ret = self._CloneSources()
182      logger.GetLogger().LogFatalIf(ret, 'Could not clone git repo %s.' %
183                                    self.address)
184
185      command = 'git branch -a | grep -wq %s' % self.branch
186      ret = self._ce.RunCommand(command)
187
188      if ret == 0:
189        if self.branch != 'master':
190          command = ('git branch --track %s remotes/origin/%s' %
191                     (self.branch, self.branch))
192        else:
193          command = 'pwd'
194        command += '&& git checkout %s' % self.branch
195      else:
196        command = 'git symbolic-ref HEAD refs/heads/%s' % self.branch
197      command += '&& rm -rf *'
198      ret = self._ce.RunCommand(command)
199      return ret
200
201  def PushSources(self, commit_message=None, dry_run=False, message_file=None):
202    with misc.WorkingDirectory(self._root_dir):
203      push_args = ''
204      if dry_run:
205        push_args += ' -n '
206
207      command = 'pwd'
208      for ignore in self.ignores:
209        command += '&& echo \'%s\' >> .git/info/exclude' % ignore
210      command += '&& git add -Av .'
211      if message_file:
212        message_arg = '-F %s' % message_file
213      elif commit_message:
214        message_arg = '-m \'%s\'' % commit_message
215      else:
216        raise Exception("No commit message given!")
217      command += '&& git commit -v %s' % message_arg
218      ret = self._ce.RunCommand(command)
219      if ret: return ret
220      if self.gerrit:
221        label = 'somelabel'
222        command = 'git remote add %s %s' % (label, self.address)
223        command += ('&& git push %s %s HEAD:refs/for/master' %
224                    (push_args,label))
225      else:
226        command = 'git push -v %s origin %s:%s' % (push_args, self.branch,
227                                                   self.branch)
228      ret = self._ce.RunCommand(command)
229
230  def MapSources(self, root_dir):
231    if not self.mappings:
232      self._RsyncExcludingRepoDirs(self._root_dir, root_dir)
233      return
234    with misc.WorkingDirectory(self._root_dir):
235      for mapping in self.mappings:
236        remote_path, local_path = SplitMapping(mapping)
237        remote_path.rstrip('...')
238        local_path.rstrip('...')
239        full_local_path = os.path.join(root_dir, local_path)
240        ret = self._RsyncExcludingRepoDirs(remote_path, full_local_path)
241        if ret: return ret
242    return 0
243
244
245class RepoReader(object):
246  def __init__(self, filename):
247    self.filename = filename
248    self.main_dict = {}
249    self.input_repos = []
250    self.output_repos = []
251
252  def ParseFile(self):
253    with open(self.filename) as f:
254      self.main_dict = eval(f.read())
255      self.CreateReposFromDict(self.main_dict)
256    return [self.input_repos, self.output_repos]
257
258  def CreateReposFromDict(self, main_dict):
259    for key, repo_list in main_dict.items():
260      for repo_dict in repo_list:
261        repo = self.CreateRepoFromDict(repo_dict)
262        if key == 'input':
263          self.input_repos.append(repo)
264        elif key == 'output':
265          self.output_repos.append(repo)
266        else:
267          logger.GetLogger().LogFatal('Unknown key: %s found' % key)
268
269  def CreateRepoFromDict(self, repo_dict):
270    repo_type = repo_dict.get('type', None)
271    repo_address = repo_dict.get('address', None)
272    repo_mappings = repo_dict.get('mappings', None)
273    repo_ignores = repo_dict.get('ignores', None)
274    repo_branch = repo_dict.get('branch', None)
275    gerrit = repo_dict.get('gerrit', None)
276    revision = repo_dict.get('revision', None)
277
278    if repo_type == 'p4':
279      repo = P4Repo(repo_address,
280                    repo_mappings,
281                    revision=revision)
282    elif repo_type == 'svn':
283      repo = SvnRepo(repo_address,
284                     repo_mappings)
285    elif repo_type == 'git':
286      repo = GitRepo(repo_address,
287                     repo_branch,
288                     mappings=repo_mappings,
289                     ignores=repo_ignores,
290                     gerrit=gerrit)
291    else:
292      logger.GetLogger().LogFatal('Unknown repo type: %s' % repo_type)
293    return repo
294
295
296@logger.HandleUncaughtExceptions
297def Main(argv):
298  parser = optparse.OptionParser()
299  parser.add_option('-i',
300                    '--input_file',
301                    dest='input_file',
302                    help='The input file that contains repo descriptions.')
303
304  parser.add_option('-n',
305                    '--dry_run',
306                    dest='dry_run',
307                    action='store_true',
308                    default=False,
309                    help='Do a dry run of the push.')
310
311  parser.add_option('-F',
312                    '--message_file',
313                    dest='message_file',
314                    default=None,
315                    help='Use contents of the log file as the commit message.')
316
317  options = parser.parse_args(argv)[0]
318  if not options.input_file:
319    parser.print_help()
320    return 1
321  rr = RepoReader(options.input_file)
322  [input_repos, output_repos] = rr.ParseFile()
323
324  for output_repo in output_repos:
325    ret = output_repo.SetupForPush()
326    if ret: return ret
327
328  input_revisions = []
329  for input_repo in input_repos:
330    ret = input_repo.PullSources()
331    if ret: return ret
332    input_revisions.append(input_repo.revision)
333
334  for input_repo in input_repos:
335    for output_repo in output_repos:
336      ret = input_repo.MapSources(output_repo.GetRoot())
337      if ret: return ret
338
339  commit_message = 'Synced repos to: %s' % ','.join(input_revisions)
340  for output_repo in output_repos:
341    ret = output_repo.PushSources(commit_message=commit_message,
342                                  dry_run=options.dry_run,
343                                  message_file=options.message_file)
344    if ret: return ret
345
346  if not options.dry_run:
347    for output_repo in output_repos:
348      output_repo.CleanupRoot()
349    for input_repo in input_repos:
350      input_repo.CleanupRoot()
351
352  return ret
353
354
355if __name__ == '__main__':
356  retval = Main(sys.argv)
357  sys.exit(retval)
358