1#!/usr/bin/env python2
2"""A crontab script to delete night test data."""
3
4from __future__ import print_function
5
6__author__ = 'shenhan@google.com (Han Shen)'
7
8import argparse
9import datetime
10import os
11import re
12import sys
13
14from cros_utils import command_executer
15from cros_utils import constants
16from cros_utils import misc
17
18DIR_BY_WEEKDAY = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')
19
20
21def CleanNumberedDir(s, dry_run=False):
22  """Deleted directories under each dated_dir."""
23  chromeos_dirs = [
24      os.path.join(s, x) for x in os.listdir(s)
25      if misc.IsChromeOsTree(os.path.join(s, x))
26  ]
27  ce = command_executer.GetCommandExecuter(log_level='none')
28  all_succeeded = True
29  for cd in chromeos_dirs:
30    if misc.DeleteChromeOsTree(cd, dry_run=dry_run):
31      print('Successfully removed chromeos tree "{0}".'.format(cd))
32    else:
33      all_succeeded = False
34      print('Failed to remove chromeos tree "{0}", please check.'.format(cd))
35
36  if not all_succeeded:
37    print('Failed to delete at least one chromeos tree, please check.')
38    return False
39
40  ## Now delete the numbered dir Before forcibly removing the directory, just
41  ## check 's' to make sure it is sane.  A valid dir to be removed must be
42  ## '/usr/local/google/crostc/(SUN|MON|TUE...|SAT)'.
43  valid_dir_pattern = (
44      '^' + constants.CROSTC_WORKSPACE + '/(' + '|'.join(DIR_BY_WEEKDAY) + ')')
45  if not re.search(valid_dir_pattern, s):
46    print('Trying to delete an invalid dir "{0}" (must match "{1}"), '
47          'please check.'.format(s, valid_dir_pattern))
48    return False
49
50  cmd = 'rm -fr {0}'.format(s)
51  if dry_run:
52    print(cmd)
53  else:
54    if ce.RunCommand(cmd, print_to_console=False, terminated_timeout=480) == 0:
55      print('Successfully removed "{0}".'.format(s))
56    else:
57      all_succeeded = False
58      print('Failed to remove "{0}", please check.'.format(s))
59  return all_succeeded
60
61
62def CleanDatedDir(dated_dir, dry_run=False):
63  # List subdirs under dir
64  subdirs = [
65      os.path.join(dated_dir, x) for x in os.listdir(dated_dir)
66      if os.path.isdir(os.path.join(dated_dir, x))
67  ]
68  all_succeeded = True
69  for s in subdirs:
70    if not CleanNumberedDir(s, dry_run):
71      all_succeeded = False
72  return all_succeeded
73
74
75def ProcessArguments(argv):
76  """Process arguments."""
77  parser = argparse.ArgumentParser(
78      description='Automatically delete nightly test data directories.',
79      usage='auto_delete_nightly_test_data.py options')
80  parser.add_argument(
81      '-d',
82      '--dry_run',
83      dest='dry_run',
84      default=False,
85      action='store_true',
86      help='Only print command line, do not execute anything.')
87  parser.add_argument(
88      '--days_to_preserve',
89      dest='days_to_preserve',
90      default=3,
91      help=('Specify the number of days (not including today),'
92            ' test data generated on these days will *NOT* be '
93            'deleted. Defaults to 3.'))
94  options = parser.parse_args(argv)
95  return options
96
97
98def CleanChromeOsTmpFiles(chroot_tmp, days_to_preserve, dry_run):
99  rv = 0
100  ce = command_executer.GetCommandExecuter()
101  # Clean chroot/tmp/test_that_* and chroot/tmp/tmpxxxxxx, that were last
102  # accessed more than specified time.
103  minutes = 1440 * days_to_preserve
104  cmd = (r'find {0} -maxdepth 1 -type d '
105         r'\( -name "test_that_*"  -amin +{1} -o '
106         r'   -name "cros-update*" -amin +{1} -o '
107         r' -regex "{0}/tmp......" -amin +{1} \) '
108         r'-exec bash -c "echo rm -fr {{}}" \; '
109         r'-exec bash -c "rm -fr {{}}" \;').format(chroot_tmp, minutes)
110  if dry_run:
111    print('Going to execute:\n%s' % cmd)
112  else:
113    rv = ce.RunCommand(cmd, print_to_console=False)
114    if rv == 0:
115      print('Successfully cleaned chromeos tree tmp directory '
116            '"{0}".'.format(chroot_tmp))
117    else:
118      print('Some directories were not removed under chromeos tree '
119            'tmp directory -"{0}".'.format(chroot_tmp))
120
121  return rv
122
123
124def CleanChromeOsImageFiles(chroot_tmp, subdir_suffix, days_to_preserve,
125                            dry_run):
126  rv = 0
127  rv2 = 0
128  ce = command_executer.GetCommandExecuter()
129  minutes = 1440 * days_to_preserve
130  # Clean image tar files, which were last accessed 1 hour ago and clean image
131  # bin files that were last accessed more than specified time.
132  cmd = ('find {0}/*{1} -type f '
133         r'\( -name "chromiumos_test_image.tar"    -amin +60 -o '
134         r'   -name "chromiumos_test_image.tar.xz" -amin +60 -o '
135         r'   -name "chromiumos_test_image.bin"    -amin +{2} \) '
136         r'-exec bash -c "echo rm -f {{}}" \; '
137         r'-exec bash -c "rm -f {{}}" \;').format(chroot_tmp, subdir_suffix,
138                                                  minutes)
139
140  if dry_run:
141    print('Going to execute:\n%s' % cmd)
142  else:
143    rv2 = ce.RunCommand(cmd, print_to_console=False)
144    if rv2 == 0:
145      print('Successfully cleaned chromeos images from '
146            '"{0}/*{1}".'.format(chroot_tmp, subdir_suffix))
147    else:
148      print('Some chromeos images were not removed from '
149            '"{0}/*{1}".'.format(chroot_tmp, subdir_suffix))
150
151  rv += rv2
152
153  # Clean autotest files that were last accessed more than specified time.
154  rv2 = 0
155  cmd = (r'find {0}/*{1} -maxdepth 2 -type d '
156         r'\( -name "autotest_files" \) '
157         r'-amin +{2} '
158         r'-exec bash -c "echo rm -fr {{}}" \; '
159         r'-exec bash -c "rm -fr {{}}" \;').format(chroot_tmp, subdir_suffix,
160                                                   minutes)
161  if dry_run:
162    print('Going to execute:\n%s' % cmd)
163  else:
164    rv2 = ce.RunCommand(cmd, print_to_console=False)
165    if rv2 == 0:
166      print('Successfully cleaned chromeos image autotest directories from '
167            '"{0}/*{1}".'.format(chroot_tmp, subdir_suffix))
168    else:
169      print('Some image autotest directories were not removed from '
170            '"{0}/*{1}".'.format(chroot_tmp, subdir_suffix))
171
172  rv += rv2
173  return rv
174
175
176def CleanChromeOsTmpAndImages(days_to_preserve=1, dry_run=False):
177  """Delete temporaries, images under crostc/chromeos."""
178  chromeos_chroot_tmp = os.path.join(constants.CROSTC_WORKSPACE, 'chromeos',
179                                     'chroot', 'tmp')
180  # Clean files in tmp directory
181  rv = CleanChromeOsTmpFiles(chromeos_chroot_tmp, days_to_preserve, dry_run)
182  # Clean image files in *-release directories
183  rv += CleanChromeOsImageFiles(chromeos_chroot_tmp, '-release',
184                                days_to_preserve, dry_run)
185  # Clean image files in *-pfq directories
186  rv += CleanChromeOsImageFiles(chromeos_chroot_tmp, '-pfq', days_to_preserve,
187                                dry_run)
188
189  return rv
190
191
192def Main(argv):
193  """Delete nightly test data directories, tmps and test images."""
194  options = ProcessArguments(argv)
195  # Function 'isoweekday' returns 1(Monday) - 7 (Sunday).
196  d = datetime.datetime.today().isoweekday()
197  # We go back 1 week, delete from that day till we are
198  # options.days_to_preserve away from today.
199  s = d - 7
200  e = d - int(options.days_to_preserve)
201  rv = 0
202  for i in range(s + 1, e):
203    if i <= 0:
204      ## Wrap around if index is negative.  6 is from i + 7 - 1, because
205      ## DIR_BY_WEEKDAY starts from 0, while isoweekday is from 1-7.
206      dated_dir = DIR_BY_WEEKDAY[i + 6]
207    else:
208      dated_dir = DIR_BY_WEEKDAY[i - 1]
209
210    rv += 0 if CleanDatedDir(
211        os.path.join(constants.CROSTC_WORKSPACE, dated_dir),
212        options.dry_run) else 1
213
214## Finally clean temporaries, images under crostc/chromeos
215  rv2 = CleanChromeOsTmpAndImages(
216      int(options.days_to_preserve), options.dry_run)
217
218  return rv + rv2
219
220if __name__ == '__main__':
221  retval = Main(sys.argv[1:])
222  sys.exit(retval)
223