1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import fnmatch
7import glob
8import optparse
9import os
10import posixpath
11import shutil
12import stat
13import sys
14import time
15import zipfile
16
17if sys.version_info < (2, 6, 0):
18  sys.stderr.write("python 2.6 or later is required run this script\n")
19  sys.exit(1)
20
21
22def IncludeFiles(filters, files):
23  """Filter files based on inclusion lists
24
25  Return a list of files which match and of the Unix shell-style wildcards
26  provided, or return all the files if no filter is provided."""
27  if not filters:
28    return files
29  match = set()
30  for file_filter in filters:
31    match |= set(fnmatch.filter(files, file_filter))
32  return [name for name in files if name in match]
33
34
35def ExcludeFiles(filters, files):
36  """Filter files based on exclusions lists
37
38  Return a list of files which do not match any of the Unix shell-style
39  wildcards provided, or return all the files if no filter is provided."""
40  if not filters:
41    return files
42  match = set()
43  for file_filter in filters:
44    excludes = set(fnmatch.filter(files, file_filter))
45    match |= excludes
46  return [name for name in files if name not in match]
47
48
49def CopyPath(options, src, dst):
50  """CopyPath from src to dst
51
52  Copy a fully specified src to a fully specified dst.  If src and dst are
53  both files, the dst file is removed first to prevent error.  If and include
54  or exclude list are provided, the destination is first matched against that
55  filter."""
56  if options.includes:
57    if not IncludeFiles(options.includes, [src]):
58      return
59
60  if options.excludes:
61    if not ExcludeFiles(options.excludes, [src]):
62      return
63
64  if options.verbose:
65    print 'cp %s %s' % (src, dst)
66
67  # If the source is a single file, copy it individually
68  if os.path.isfile(src):
69    # We can not copy over a directory with a file.
70    if os.path.exists(dst):
71      if not os.path.isfile(dst):
72        msg = "cp: cannot overwrite non-file '%s' with file." % dst
73        raise OSError(msg)
74      # If the destination exists as a file, remove it before copying to avoid
75      # 'readonly' issues.
76      os.remove(dst)
77
78    # Now copy to the non-existent fully qualified target
79    shutil.copy(src, dst)
80    return
81
82  # Otherwise it's a directory, ignore it unless allowed
83  if os.path.isdir(src):
84    if not options.recursive:
85      print "cp: omitting directory '%s'" % src
86      return
87
88    # We can not copy over a file with a directory.
89    if os.path.exists(dst):
90      if not os.path.isdir(dst):
91        msg = "cp: cannot overwrite non-directory '%s' with directory." % dst
92        raise OSError(msg)
93    else:
94      # if it didn't exist, create the directory
95      os.makedirs(dst)
96
97    # Now copy all members
98    for filename in os.listdir(src):
99      srcfile = os.path.join(src, filename)
100      dstfile = os.path.join(dst, filename)
101      CopyPath(options, srcfile, dstfile)
102  return
103
104
105def Copy(args):
106  """A Unix cp style copy.
107
108  Copies multiple sources to a single destination using the normal cp
109  semantics.  In addition, it support inclusion and exclusion filters which
110  allows the copy to skip certain types of files."""
111  parser = optparse.OptionParser(usage='usage: cp [Options] sources... dest')
112  parser.add_option(
113      '-R', '-r', '--recursive', dest='recursive', action='store_true',
114      default=False,
115      help='copy directories recursively.')
116  parser.add_option(
117      '-v', '--verbose', dest='verbose', action='store_true',
118      default=False,
119      help='verbose output.')
120  parser.add_option(
121      '--include', dest='includes', action='append', default=[],
122      help='include files matching this expression.')
123  parser.add_option(
124      '--exclude', dest='excludes', action='append', default=[],
125      help='exclude files matching this expression.')
126  options, files = parser.parse_args(args)
127  if len(files) < 2:
128    parser.error('ERROR: expecting SOURCE(s) and DEST.')
129
130  srcs = files[:-1]
131  dst = files[-1]
132
133  src_list = []
134  for src in srcs:
135    files = glob.glob(src)
136    if not files:
137      raise OSError('cp: no such file or directory: ' + src)
138    if files:
139      src_list.extend(files)
140
141  for src in src_list:
142    # If the destination is a directory, then append the basename of the src
143    # to the destination.
144    if os.path.isdir(dst):
145      CopyPath(options, src, os.path.join(dst, os.path.basename(src)))
146    else:
147      CopyPath(options, src, dst)
148
149
150def Mkdir(args):
151  """A Unix style mkdir"""
152  parser = optparse.OptionParser(usage='usage: mkdir [Options] DIRECTORY...')
153  parser.add_option(
154      '-p', '--parents', dest='parents', action='store_true',
155      default=False,
156      help='ignore existing parents, create parents as needed.')
157  parser.add_option(
158      '-v', '--verbose', dest='verbose', action='store_true',
159      default=False,
160      help='verbose output.')
161
162  options, dsts = parser.parse_args(args)
163  if len(dsts) < 1:
164    parser.error('ERROR: expecting DIRECTORY...')
165
166  for dst in dsts:
167    if options.verbose:
168      print 'mkdir ' + dst
169    try:
170      os.makedirs(dst)
171    except OSError:
172      if os.path.isdir(dst):
173        if options.parents:
174          continue
175        raise OSError('mkdir: Already exists: ' + dst)
176      else:
177        raise OSError('mkdir: Failed to create: ' + dst)
178  return 0
179
180
181def MovePath(options, src, dst):
182  """MovePath from src to dst
183
184  Moves the src to the dst much like the Unix style mv command, except it
185  only handles one source at a time.  Because of possible temporary failures
186  do to locks (such as anti-virus software on Windows), the function will retry
187  up to five times."""
188  # if the destination is not an existing directory, then overwrite it
189  if os.path.isdir(dst):
190    dst = os.path.join(dst, os.path.basename(src))
191
192  # If the destination exists, the remove it
193  if os.path.exists(dst):
194    if options.force:
195      Remove(['-vfr', dst])
196      if os.path.exists(dst):
197        raise OSError('mv: FAILED TO REMOVE ' + dst)
198    else:
199      raise OSError('mv: already exists ' + dst)
200  for _ in range(5):
201    try:
202      os.rename(src, dst)
203      return
204    except OSError as error:
205      print 'Failed on %s with %s, retrying' % (src, error)
206      time.sleep(5)
207  print 'Gave up.'
208  raise OSError('mv: ' + error)
209
210
211def Move(args):
212  parser = optparse.OptionParser(usage='usage: mv [Options] sources... dest')
213  parser.add_option(
214      '-v', '--verbose', dest='verbose', action='store_true',
215      default=False,
216      help='verbose output.')
217  parser.add_option(
218      '-f', '--force', dest='force', action='store_true',
219      default=False,
220      help='force, do not error it files already exist.')
221  options, files = parser.parse_args(args)
222  if len(files) < 2:
223    parser.error('ERROR: expecting SOURCE... and DEST.')
224
225  srcs = files[:-1]
226  dst = files[-1]
227
228  if options.verbose:
229    print 'mv %s %s' % (' '.join(srcs), dst)
230
231  for src in srcs:
232    MovePath(options, src, dst)
233  return 0
234
235
236def Remove(args):
237  """A Unix style rm.
238
239  Removes the list of paths.  Because of possible temporary failures do to locks
240  (such as anti-virus software on Windows), the function will retry up to five
241  times."""
242  parser = optparse.OptionParser(usage='usage: rm [Options] PATHS...')
243  parser.add_option(
244      '-R', '-r', '--recursive', dest='recursive', action='store_true',
245      default=False,
246      help='remove directories recursively.')
247  parser.add_option(
248      '-v', '--verbose', dest='verbose', action='store_true',
249      default=False,
250      help='verbose output.')
251  parser.add_option(
252      '-f', '--force', dest='force', action='store_true',
253      default=False,
254      help='force, do not error it files does not exist.')
255  options, files = parser.parse_args(args)
256  if len(files) < 1:
257    parser.error('ERROR: expecting FILE...')
258
259  try:
260    for pattern in files:
261      dst_files = glob.glob(pattern)
262      if not dst_files:
263        # Ignore non existing files when using force
264        if options.force:
265          continue
266        raise OSError('rm: no such file or directory: ' + pattern)
267
268      for dst in dst_files:
269        if options.verbose:
270          print 'rm ' + dst
271
272        if os.path.isfile(dst) or os.path.islink(dst):
273          for i in range(5):
274            try:
275              # Check every time, since it may have been deleted after the
276              # previous failed attempt.
277              if os.path.isfile(dst) or os.path.islink(dst):
278                os.remove(dst)
279              break
280            except OSError as error:
281              if i == 5:
282                print 'Gave up.'
283                raise OSError('rm: ' + str(error))
284              print 'Failed remove with %s, retrying' % error
285              time.sleep(5)
286
287        if options.recursive:
288          for i in range(5):
289            try:
290              if os.path.isdir(dst):
291                shutil.rmtree(dst)
292              break
293            except OSError as error:
294              if i == 5:
295                print 'Gave up.'
296                raise OSError('rm: ' + str(error))
297              print 'Failed rmtree with %s, retrying' % error
298              time.sleep(5)
299
300  except OSError as error:
301    print error
302  return 0
303
304
305def MakeZipPath(os_path, isdir, iswindows):
306  """Changes a path into zipfile format.
307
308  # doctest doesn't seem to honor r'' strings, so the backslashes need to be
309  # escaped.
310  >>> MakeZipPath(r'C:\\users\\foobar\\blah', False, True)
311  'users/foobar/blah'
312  >>> MakeZipPath('/tmp/tmpfoobar/something', False, False)
313  'tmp/tmpfoobar/something'
314  >>> MakeZipPath('./somefile.txt', False, False)
315  'somefile.txt'
316  >>> MakeZipPath('somedir', True, False)
317  'somedir/'
318  >>> MakeZipPath('../dir/filename.txt', False, False)
319  '../dir/filename.txt'
320  >>> MakeZipPath('dir/../filename.txt', False, False)
321  'filename.txt'
322  """
323  zip_path = os_path
324  if iswindows:
325    import ntpath
326    # zipfile paths are always posix-style. They also have the drive
327    # letter and leading slashes removed.
328    zip_path = ntpath.splitdrive(os_path)[1].replace('\\', '/')
329  if zip_path.startswith('/'):
330    zip_path = zip_path[1:]
331  zip_path = posixpath.normpath(zip_path)
332  # zipfile also always appends a slash to a directory name.
333  if isdir:
334    zip_path += '/'
335  return zip_path
336
337
338def OSMakeZipPath(os_path):
339  return MakeZipPath(os_path, os.path.isdir(os_path), sys.platform == 'win32')
340
341
342def Zip(args):
343  """A Unix style zip.
344
345  Compresses the listed files."""
346  parser = optparse.OptionParser(usage='usage: zip [Options] zipfile list')
347  parser.add_option(
348      '-r', dest='recursive', action='store_true',
349      default=False,
350      help='recurse into directories')
351  parser.add_option(
352      '-q', dest='quiet', action='store_true',
353      default=False,
354      help='quiet operation')
355  options, files = parser.parse_args(args)
356  if len(files) < 2:
357    parser.error('ERROR: expecting ZIPFILE and LIST.')
358
359  dest_zip = files[0]
360  src_args = files[1:]
361
362  src_files = []
363  for src_arg in src_args:
364    globbed_src_args = glob.glob(src_arg)
365    if not globbed_src_args:
366      if not options.quiet:
367        print 'zip warning: name not matched: %s' % (src_arg,)
368
369    for src_file in globbed_src_args:
370      src_file = os.path.normpath(src_file)
371      src_files.append(src_file)
372      if options.recursive and os.path.isdir(src_file):
373        for root, dirs, files in os.walk(src_file):
374          for dirname in dirs:
375            src_files.append(os.path.join(root, dirname))
376          for filename in files:
377            src_files.append(os.path.join(root, filename))
378
379  zip_stream = None
380  # zip_data represents a list of the data to be written or appended to the
381  # zip_stream. It is a list of tuples:
382  #   (OS file path, zip path/zip file info, and file data)
383  # In all cases one of the |os path| or the |file data| will be None.
384  # |os path| is None when there is no OS file to write to the archive (i.e.
385  # the file data already existed in the archive). |file data| is None when the
386  # file is new (never existed in the archive) or being updated.
387  zip_data = []
388  new_files_to_add = [OSMakeZipPath(src_file) for src_file in src_files]
389  zip_path_to_os_path_dict = dict((new_files_to_add[i], src_files[i])
390                                  for i in range(len(src_files)))
391  write_mode = 'a'
392  try:
393    zip_stream = zipfile.ZipFile(dest_zip, 'r')
394    files_to_update = set(new_files_to_add).intersection(
395        set(zip_stream.namelist()))
396    if files_to_update:
397      # As far as I can tell, there is no way to update a zip entry using
398      # zipfile; the best you can do is rewrite the archive.
399      # Iterate through the zipfile to maintain file order.
400      write_mode = 'w'
401      for zip_path in zip_stream.namelist():
402        if zip_path in files_to_update:
403          os_path = zip_path_to_os_path_dict[zip_path]
404          zip_data.append((os_path, zip_path, None))
405          new_files_to_add.remove(zip_path)
406        else:
407          file_bytes = zip_stream.read(zip_path)
408          file_info = zip_stream.getinfo(zip_path)
409          zip_data.append((None, file_info, file_bytes))
410  except IOError:
411    pass
412  finally:
413    if zip_stream:
414      zip_stream.close()
415
416  for zip_path in new_files_to_add:
417    zip_data.append((zip_path_to_os_path_dict[zip_path], zip_path, None))
418
419  if not zip_data:
420    print 'zip error: Nothing to do! (%s)' % (dest_zip,)
421    return 1
422
423  try:
424    zip_stream = zipfile.ZipFile(dest_zip, write_mode, zipfile.ZIP_DEFLATED)
425    for os_path, file_info_or_zip_path, file_bytes in zip_data:
426      if isinstance(file_info_or_zip_path, zipfile.ZipInfo):
427        zip_path = file_info_or_zip_path.filename
428      else:
429        zip_path = file_info_or_zip_path
430
431      if os_path:
432        st = os.stat(os_path)
433        if stat.S_ISDIR(st.st_mode):
434          # Python 2.6 on the buildbots doesn't support writing directories to
435          # zip files. This was resolved in a later version of Python 2.6.
436          # We'll work around it by writing an empty file with the correct
437          # path. (This is basically what later versions do anyway.)
438          zip_info = zipfile.ZipInfo()
439          zip_info.filename = zip_path
440          zip_info.date_time = time.localtime(st.st_mtime)[0:6]
441          zip_info.compress_type = zip_stream.compression
442          zip_info.flag_bits = 0x00
443          zip_info.external_attr = (st[0] & 0xFFFF) << 16L
444          zip_info.CRC = 0
445          zip_info.compress_size = 0
446          zip_info.file_size = 0
447          zip_stream.writestr(zip_info, '')
448        else:
449          zip_stream.write(os_path, zip_path)
450      else:
451        zip_stream.writestr(file_info_or_zip_path, file_bytes)
452
453      if not options.quiet:
454        if zip_path in new_files_to_add:
455          operation = 'adding'
456        else:
457          operation = 'updating'
458        zip_info = zip_stream.getinfo(zip_path)
459        if (zip_info.compress_type == zipfile.ZIP_STORED or
460            zip_info.file_size == 0):
461          print '  %s: %s (stored 0%%)' % (operation, zip_path)
462        elif zip_info.compress_type == zipfile.ZIP_DEFLATED:
463          print '  %s: %s (deflated %d%%)' % (operation, zip_path,
464              100 - zip_info.compress_size * 100 / zip_info.file_size)
465  finally:
466    zip_stream.close()
467
468  return 0
469
470
471def FindExeInPath(filename):
472  env_path = os.environ.get('PATH', '')
473  paths = env_path.split(os.pathsep)
474
475  def IsExecutableFile(path):
476    return os.path.isfile(path) and os.access(path, os.X_OK)
477
478  if os.path.sep in filename:
479    if IsExecutableFile(filename):
480      return filename
481
482  for path in paths:
483    filepath = os.path.join(path, filename)
484    if IsExecutableFile(filepath):
485      return os.path.abspath(os.path.join(path, filename))
486
487
488def Which(args):
489  """A Unix style which.
490
491  Looks for all arguments in the PATH environment variable, and prints their
492  path if they are executable files.
493
494  Note: If you pass an argument with a path to which, it will just test if it
495  is executable, not if it is in the path.
496  """
497  parser = optparse.OptionParser(usage='usage: which args...')
498  _, files = parser.parse_args(args)
499  if not files:
500    return 0
501
502  retval = 0
503  for filename in files:
504    fullname = FindExeInPath(filename)
505    if fullname:
506      print fullname
507    else:
508      retval = 1
509
510  return retval
511
512
513FuncMap = {
514  'cp': Copy,
515  'mkdir': Mkdir,
516  'mv': Move,
517  'rm': Remove,
518  'zip': Zip,
519  'which': Which,
520}
521
522
523def main(args):
524  if not args:
525    print 'No command specified'
526    print 'Available commands: %s' % ' '.join(FuncMap)
527    return 1
528  func_name = args[0]
529  func = FuncMap.get(func_name)
530  if not func:
531    print 'Do not recognize command: %s' % func_name
532    print 'Available commands: %s' % ' '.join(FuncMap)
533    return 1
534  try:
535    return func(args[1:])
536  except KeyboardInterrupt:
537    print '%s: interrupted' % func_name
538    return 1
539
540if __name__ == '__main__':
541  sys.exit(main(sys.argv[1:]))
542