repo_to_repo.py revision a8af9a7a2462b00e72deff99327bdb452a715277 
1#!/usr/bin/python2
2#
3# Copyright 2010 Google Inc. All Rights Reserved.
4"""Module for transferring files between various types of repositories."""
5
6from __future__ import print_function
7
8__author__ = 'asharif@google.com (Ahmad Sharif)'
9
10import argparse
11import datetime
12import json
13import os
14import re
15import socket
16import sys
17import tempfile
18
19from automation.clients.helper import perforce
20from cros_utils import command_executer
21from cros_utils import logger
22from cros_utils import misc
23
24# pylint: disable=anomalous-backslash-in-string
25
26def GetCanonicalMappings(mappings):
27  canonical_mappings = []
28  for mapping in mappings:
29    remote_path, local_path = mapping.split()
30    if local_path.endswith('/') and not remote_path.endswith('/'):
31      local_path = os.path.join(local_path, os.path.basename(remote_path))
32    remote_path = remote_path.lstrip('/').split('/', 1)[1]
33    canonical_mappings.append(perforce.PathMapping(remote_path, local_path))
34  return canonical_mappings
35
36
37def SplitMapping(mapping):
38  parts = mapping.split()
39  assert len(parts) <= 2, 'Mapping %s invalid' % mapping
40  remote_path = parts[0]
41  if len(parts) == 2:
42    local_path = parts[1]
43  else:
44    local_path = '.'
45  return remote_path, local_path
46
47
48class Repo(object):
49  """Basic repository base class."""
50
51  def __init__(self, no_create_tmp_dir=False):
52    self.repo_type = None
53    self.address = None
54    self.mappings = None
55    self.revision = None
56    self.ignores = ['.gitignore', '.p4config', 'README.google']
57    if no_create_tmp_dir:
58      self._root_dir = None
59    else:
60      self._root_dir = tempfile.mkdtemp()
61    self._ce = command_executer.GetCommandExecuter()
62    self._logger = logger.GetLogger()
63
64  def PullSources(self):
65    """Pull all sources into an internal dir."""
66    pass
67
68  def SetupForPush(self):
69    """Setup a repository for pushing later."""
70    pass
71
72  def PushSources(self, commit_message=None, dry_run=False, message_file=None):
73    """Push to the external repo with the commit message."""
74    pass
75
76  def _RsyncExcludingRepoDirs(self, source_dir, dest_dir):
77    for f in os.listdir(source_dir):
78      if f in ['.git', '.svn', '.p4config']:
79        continue
80      dest_file = os.path.join(dest_dir, f)
81      source_file = os.path.join(source_dir, f)
82      if os.path.exists(dest_file):
83        command = 'rm -rf %s' % dest_file
84        self._ce.RunCommand(command)
85      command = 'rsync -a %s %s' % (source_file, dest_dir)
86      self._ce.RunCommand(command)
87    return 0
88
89  def MapSources(self, dest_dir):
90    """Copy sources from the internal dir to root_dir."""
91    return self._RsyncExcludingRepoDirs(self._root_dir, dest_dir)
92
93  def GetRoot(self):
94    return self._root_dir
95
96  def SetRoot(self, directory):
97    self._root_dir = directory
98
99  def CleanupRoot(self):
100    command = 'rm -rf %s' % self._root_dir
101    return self._ce.RunCommand(command)
102
103  def __str__(self):
104    return '\n'.join(str(s)
105                     for s in [self.repo_type, self.address, self.mappings])
106
107
108# Note - this type of repo is used only for "readonly", in other words, this
109# only serves as a incoming repo.
110class FileRepo(Repo):
111  """Class for file repositories."""
112
113  def __init__(self, address, ignores=None):
114    Repo.__init__(self, no_create_tmp_dir=True)
115    self.repo_type = 'file'
116    self.address = address
117    self.mappings = None
118    self.branch = None
119    self.revision = '{0} (as of "{1}")'.format(address, datetime.datetime.now())
120    self.gerrit = None
121    self._root_dir = self.address
122    if ignores:
123      self.ignores += ignores
124
125  def CleanupRoot(self):
126    """Override to prevent deletion."""
127    pass
128
129
130class P4Repo(Repo):
131  """Class for P4 repositories."""
132
133
134  def __init__(self, address, mappings, revision=None):
135    Repo.__init__(self)
136    self.repo_type = 'p4'
137    self.address = address
138    self.mappings = mappings
139    self.revision = revision
140
141  def PullSources(self):
142    client_name = socket.gethostname()
143    client_name += tempfile.mkstemp()[1].replace('/', '-')
144    mappings = self.mappings
145    p4view = perforce.View('depot2', GetCanonicalMappings(mappings))
146    p4client = perforce.CommandsFactory(self._root_dir,
147                                        p4view,
148                                        name=client_name)
149    command = p4client.SetupAndDo(p4client.Sync(self.revision))
150    ret = self._ce.RunCommand(command)
151    assert ret == 0, 'Could not setup client.'
152    command = p4client.InCheckoutDir(p4client.SaveCurrentCLNumber())
153    ret, o, _ = self._ce.RunCommandWOutput(command)
154    assert ret == 0, 'Could not get version from client.'
155    self.revision = re.search('^\d+$', o.strip(), re.MULTILINE).group(0)
156    command = p4client.InCheckoutDir(p4client.Remove())
157    ret = self._ce.RunCommand(command)
158    assert ret == 0, 'Could not delete client.'
159    return 0
160
161
162class SvnRepo(Repo):
163  """Class for svn repositories."""
164
165  def __init__(self, address, mappings):
166    Repo.__init__(self)
167    self.repo_type = 'svn'
168    self.address = address
169    self.mappings = mappings
170
171  def PullSources(self):
172    with misc.WorkingDirectory(self._root_dir):
173      for mapping in self.mappings:
174        remote_path, local_path = SplitMapping(mapping)
175        command = 'svn co %s/%s %s' % (self.address, remote_path, local_path)
176      ret = self._ce.RunCommand(command)
177      if ret:
178        return ret
179
180      self.revision = ''
181      for mapping in self.mappings:
182        remote_path, local_path = SplitMapping(mapping)
183        command = 'cd %s && svnversion -c .' % (local_path)
184        ret, o, _ = self._ce.RunCommandWOutput(command)
185        self.revision += o.strip().split(':')[-1]
186        if ret:
187          return ret
188    return 0
189
190
191class GitRepo(Repo):
192  """Class for git repositories."""
193
194  def __init__(self, address, branch, mappings=None, ignores=None, gerrit=None):
195    Repo.__init__(self)
196    self.repo_type = 'git'
197    self.address = address
198    self.branch = branch or 'master'
199    if ignores:
200      self.ignores += ignores
201    self.mappings = mappings
202    self.gerrit = gerrit
203
204  def _CloneSources(self):
205    with misc.WorkingDirectory(self._root_dir):
206      command = 'git clone %s .' % (self.address)
207      return self._ce.RunCommand(command)
208
209  def PullSources(self):
210    with misc.WorkingDirectory(self._root_dir):
211      ret = self._CloneSources()
212      if ret:
213        return ret
214
215      command = 'git checkout %s' % self.branch
216      ret = self._ce.RunCommand(command)
217      if ret:
218        return ret
219
220      command = 'git describe --always'
221      ret, o, _ = self._ce.RunCommandWOutput(command)
222      self.revision = o.strip()
223      return ret
224
225  def SetupForPush(self):
226    with misc.WorkingDirectory(self._root_dir):
227      ret = self._CloneSources()
228      logger.GetLogger().LogFatalIf(ret, 'Could not clone git repo %s.' %
229                                    self.address)
230
231      command = 'git branch -a | grep -wq %s' % self.branch
232      ret = self._ce.RunCommand(command)
233
234      if ret == 0:
235        if self.branch != 'master':
236          command = ('git branch --track %s remotes/origin/%s' %
237                     (self.branch, self.branch))
238        else:
239          command = 'pwd'
240        command += '&& git checkout %s' % self.branch
241      else:
242        command = 'git symbolic-ref HEAD refs/heads/%s' % self.branch
243      command += '&& rm -rf *'
244      ret = self._ce.RunCommand(command)
245      return ret
246
247  def CommitLocally(self, commit_message=None, message_file=None):
248    with misc.WorkingDirectory(self._root_dir):
249      command = 'pwd'
250      for ignore in self.ignores:
251        command += '&& echo \'%s\' >> .git/info/exclude' % ignore
252      command += '&& git add -Av .'
253      if message_file:
254        message_arg = '-F %s' % message_file
255      elif commit_message:
256        message_arg = '-m \'%s\'' % commit_message
257      else:
258        raise Exception('No commit message given!')
259      command += '&& git commit -v %s' % message_arg
260      return self._ce.RunCommand(command)
261
262  def PushSources(self, commit_message=None, dry_run=False, message_file=None):
263    ret = self.CommitLocally(commit_message, message_file)
264    if ret:
265      return ret
266    push_args = ''
267    if dry_run:
268      push_args += ' -n '
269    with misc.WorkingDirectory(self._root_dir):
270      if self.gerrit:
271        label = 'somelabel'
272        command = 'git remote add %s %s' % (label, self.address)
273        command += ('&& git push %s %s HEAD:refs/for/master' %
274                    (push_args, label))
275      else:
276        command = 'git push -v %s origin %s:%s' % (push_args, self.branch,
277                                                   self.branch)
278      ret = self._ce.RunCommand(command)
279    return ret
280
281  def MapSources(self, root_dir):
282    if not self.mappings:
283      self._RsyncExcludingRepoDirs(self._root_dir, root_dir)
284      return
285    with misc.WorkingDirectory(self._root_dir):
286      for mapping in self.mappings:
287        remote_path, local_path = SplitMapping(mapping)
288        remote_path.rstrip('...')
289        local_path.rstrip('...')
290        full_local_path = os.path.join(root_dir, local_path)
291        ret = self._RsyncExcludingRepoDirs(remote_path, full_local_path)
292        if ret:
293          return ret
294    return 0
295
296
297class RepoReader(object):
298  """Class for reading repositories."""
299
300  def __init__(self, filename):
301    self.filename = filename
302    self.main_dict = {}
303    self.input_repos = []
304    self.output_repos = []
305
306  def ParseFile(self):
307    with open(self.filename) as f:
308      self.main_dict = json.load(f)
309      self.CreateReposFromDict(self.main_dict)
310    return [self.input_repos, self.output_repos]
311
312  def CreateReposFromDict(self, main_dict):
313    for key, repo_list in main_dict.items():
314      for repo_dict in repo_list:
315        repo = self.CreateRepoFromDict(repo_dict)
316        if key == 'input':
317          self.input_repos.append(repo)
318        elif key == 'output':
319          self.output_repos.append(repo)
320        else:
321          logger.GetLogger().LogFatal('Unknown key: %s found' % key)
322
323  def CreateRepoFromDict(self, repo_dict):
324    repo_type = repo_dict.get('type', None)
325    repo_address = repo_dict.get('address', None)
326    repo_mappings = repo_dict.get('mappings', None)
327    repo_ignores = repo_dict.get('ignores', None)
328    repo_branch = repo_dict.get('branch', None)
329    gerrit = repo_dict.get('gerrit', None)
330    revision = repo_dict.get('revision', None)
331
332    if repo_type == 'p4':
333      repo = P4Repo(repo_address, repo_mappings, revision=revision)
334    elif repo_type == 'svn':
335      repo = SvnRepo(repo_address, repo_mappings)
336    elif repo_type == 'git':
337      repo = GitRepo(repo_address,
338                     repo_branch,
339                     mappings=repo_mappings,
340                     ignores=repo_ignores,
341                     gerrit=gerrit)
342    elif repo_type == 'file':
343      repo = FileRepo(repo_address)
344    else:
345      logger.GetLogger().LogFatal('Unknown repo type: %s' % repo_type)
346    return repo
347
348
349@logger.HandleUncaughtExceptions
350def Main(argv):
351  parser = argparse.ArgumentParser()
352  parser.add_argument('-i',
353                      '--input_file',
354                      dest='input_file',
355                      help='The input file that contains repo descriptions.')
356
357  parser.add_argument('-n',
358                      '--dry_run',
359                      dest='dry_run',
360                      action='store_true',
361                      default=False,
362                      help='Do a dry run of the push.')
363
364  parser.add_argument('-F',
365                      '--message_file',
366                      dest='message_file',
367                      default=None,
368                      help=('Use contents of the log file as the commit '
369                            'message.'))
370
371  options = parser.parse_args(argv)
372  if not options.input_file:
373    parser.print_help()
374    return 1
375  rr = RepoReader(options.input_file)
376  [input_repos, output_repos] = rr.ParseFile()
377
378  # Make sure FileRepo is not used as output destination.
379  for output_repo in output_repos:
380    if output_repo.repo_type == 'file':
381      logger.GetLogger().LogFatal(
382          'FileRepo is only supported as an input repo.')
383
384  for output_repo in output_repos:
385    ret = output_repo.SetupForPush()
386    if ret:
387      return ret
388
389  input_revisions = []
390  for input_repo in input_repos:
391    ret = input_repo.PullSources()
392    if ret:
393      return ret
394    input_revisions.append(input_repo.revision)
395
396  for input_repo in input_repos:
397    for output_repo in output_repos:
398      ret = input_repo.MapSources(output_repo.GetRoot())
399      if ret:
400        return ret
401
402  commit_message = 'Synced repos to: %s' % ','.join(input_revisions)
403  for output_repo in output_repos:
404    ret = output_repo.PushSources(commit_message=commit_message,
405                                  dry_run=options.dry_run,
406                                  message_file=options.message_file)
407    if ret:
408      return ret
409
410  if not options.dry_run:
411    for output_repo in output_repos:
412      output_repo.CleanupRoot()
413    for input_repo in input_repos:
414      input_repo.CleanupRoot()
415
416  return ret
417
418
419if __name__ == '__main__':
420  retval = Main(sys.argv[1:])
421  sys.exit(retval)
422