machine_manager.py revision 0d39863d8578fb1c6e7077b2c7cc5ca79edcf4dd 
1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Machine Manager module."""
5
6from __future__ import print_function
7
8import hashlib
9import image_chromeos
10import file_lock_machine
11import math
12import os.path
13import re
14import sys
15import threading
16import time
17
18import test_flag
19from cros_utils import command_executer
20from cros_utils import logger
21
22CHECKSUM_FILE = '/usr/local/osimage_checksum_file'
23
24
25class BadChecksum(Exception):
26  """Raised if all machines for a label don't have the same checksum."""
27  pass
28
29
30class BadChecksumString(Exception):
31  """Raised if all machines for a label don't have the same checksum string."""
32  pass
33
34
35class MissingLocksDirectory(Exception):
36  """Raised when cannot find/access the machine locks directory."""
37
38
39class CrosCommandError(Exception):
40  """Raised when an error occurs running command on DUT."""
41
42
43class CrosMachine(object):
44  """The machine class."""
45
46  def __init__(self, name, chromeos_root, log_level, cmd_exec=None):
47    self.name = name
48    self.image = None
49    # We relate a dut with a label if we reimage the dut using label or we
50    # detect at the very beginning that the dut is running this label.
51    self.label = None
52    self.checksum = None
53    self.locked = False
54    self.released_time = time.time()
55    self.test_run = None
56    self.chromeos_root = chromeos_root
57    self.log_level = log_level
58    self.cpuinfo = None
59    self.machine_id = None
60    self.checksum_string = None
61    self.meminfo = None
62    self.phys_kbytes = None
63    self.ce = cmd_exec or command_executer.GetCommandExecuter(
64        log_level=self.log_level)
65    self.SetUpChecksumInfo()
66
67  def SetUpChecksumInfo(self):
68    if not self.IsReachable():
69      self.machine_checksum = None
70      return
71    self._GetMemoryInfo()
72    self._GetCPUInfo()
73    self._ComputeMachineChecksumString()
74    self._GetMachineID()
75    self.machine_checksum = self._GetMD5Checksum(self.checksum_string)
76    self.machine_id_checksum = self._GetMD5Checksum(self.machine_id)
77
78  def IsReachable(self):
79    command = 'ls'
80    ret = self.ce.CrosRunCommand(command,
81                                 machine=self.name,
82                                 chromeos_root=self.chromeos_root)
83    if ret:
84      return False
85    return True
86
87  def _ParseMemoryInfo(self):
88    line = self.meminfo.splitlines()[0]
89    usable_kbytes = int(line.split()[1])
90    # This code is from src/third_party/test/files/client/bin/base_utils.py
91    # usable_kbytes is system's usable DRAM in kbytes,
92    #   as reported by memtotal() from device /proc/meminfo memtotal
93    #   after Linux deducts 1.5% to 9.5% for system table overhead
94    # Undo the unknown actual deduction by rounding up
95    #   to next small multiple of a big power-of-two
96    #   eg  12GB - 5.1% gets rounded back up to 12GB
97    mindeduct = 0.005  # 0.5 percent
98    maxdeduct = 0.095  # 9.5 percent
99    # deduction range 1.5% .. 9.5% supports physical mem sizes
100    #    6GB .. 12GB in steps of .5GB
101    #   12GB .. 24GB in steps of 1 GB
102    #   24GB .. 48GB in steps of 2 GB ...
103    # Finer granularity in physical mem sizes would require
104    #   tighter spread between min and max possible deductions
105
106    # increase mem size by at least min deduction, without rounding
107    min_kbytes = int(usable_kbytes / (1.0 - mindeduct))
108    # increase mem size further by 2**n rounding, by 0..roundKb or more
109    round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes
110    # find least binary roundup 2**n that covers worst-cast roundKb
111    mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2)))
112    # have round_kbytes <= mod2n < round_kbytes*2
113    # round min_kbytes up to next multiple of mod2n
114    phys_kbytes = min_kbytes + mod2n - 1
115    phys_kbytes -= phys_kbytes % mod2n  # clear low bits
116    self.phys_kbytes = phys_kbytes
117
118  def _GetMemoryInfo(self):
119    #TODO yunlian: when the machine in rebooting, it will not return
120    #meminfo, the assert does not catch it either
121    command = 'cat /proc/meminfo'
122    ret, self.meminfo, _ = self.ce.CrosRunCommandWOutput(
123        command,
124        machine=self.name,
125        chromeos_root=self.chromeos_root)
126    assert ret == 0, 'Could not get meminfo from machine: %s' % self.name
127    if ret == 0:
128      self._ParseMemoryInfo()
129
130  def _GetCPUInfo(self):
131    command = 'cat /proc/cpuinfo'
132    ret, self.cpuinfo, _ = self.ce.CrosRunCommandWOutput(
133        command,
134        machine=self.name,
135        chromeos_root=self.chromeos_root)
136    assert ret == 0, 'Could not get cpuinfo from machine: %s' % self.name
137
138  def _ComputeMachineChecksumString(self):
139    self.checksum_string = ''
140    exclude_lines_list = ['MHz', 'BogoMIPS', 'bogomips']
141    for line in self.cpuinfo.splitlines():
142      if not any([e in line for e in exclude_lines_list]):
143        self.checksum_string += line
144    self.checksum_string += ' ' + str(self.phys_kbytes)
145
146  def _GetMD5Checksum(self, ss):
147    if ss:
148      return hashlib.md5(ss).hexdigest()
149    else:
150      return ''
151
152  def _GetMachineID(self):
153    command = 'dump_vpd_log --full --stdout'
154    _, if_out, _ = self.ce.CrosRunCommandWOutput(
155        command,
156        machine=self.name,
157        chromeos_root=self.chromeos_root)
158    b = if_out.splitlines()
159    a = [l for l in b if 'Product' in l]
160    if len(a):
161      self.machine_id = a[0]
162      return
163    command = 'ifconfig'
164    _, if_out, _ = self.ce.CrosRunCommandWOutput(
165        command,
166        machine=self.name,
167        chromeos_root=self.chromeos_root)
168    b = if_out.splitlines()
169    a = [l for l in b if 'HWaddr' in l]
170    if len(a):
171      self.machine_id = '_'.join(a)
172      return
173    a = [l for l in b if 'ether' in l]
174    if len(a):
175      self.machine_id = '_'.join(a)
176      return
177    assert 0, 'Could not get machine_id from machine: %s' % self.name
178
179  def __str__(self):
180    l = []
181    l.append(self.name)
182    l.append(str(self.image))
183    l.append(str(self.checksum))
184    l.append(str(self.locked))
185    l.append(str(self.released_time))
186    return ', '.join(l)
187
188
189class MachineManager(object):
190  """Lock, image and unlock machines locally for benchmark runs.
191
192  This class contains methods and calls to lock, unlock and image
193  machines and distribute machines to each benchmark run.  The assumption is
194  that all of the machines for the experiment have been globally locked
195  (using an AFE server) in the ExperimentRunner, but the machines still need
196  to be locally locked/unlocked (allocated to benchmark runs) to prevent
197  multiple benchmark runs within the same experiment from trying to use the
198  same machine at the same time.
199  """
200
201  def __init__(self,
202               chromeos_root,
203               acquire_timeout,
204               log_level,
205               locks_dir,
206               cmd_exec=None,
207               lgr=None):
208    self._lock = threading.RLock()
209    self._all_machines = []
210    self._machines = []
211    self.image_lock = threading.Lock()
212    self.num_reimages = 0
213    self.chromeos_root = None
214    self.machine_checksum = {}
215    self.machine_checksum_string = {}
216    self.acquire_timeout = acquire_timeout
217    self.log_level = log_level
218    self.locks_dir = locks_dir
219    self.ce = cmd_exec or command_executer.GetCommandExecuter(
220        log_level=self.log_level)
221    self.logger = lgr or logger.GetLogger()
222
223    if self.locks_dir and not os.path.isdir(self.locks_dir):
224      raise MissingLocksDirectory('Cannot access locks directory: %s' %
225                                  self.locks_dir)
226
227    self._initialized_machines = []
228    self.chromeos_root = chromeos_root
229
230  def RemoveNonLockedMachines(self, locked_machines):
231    for m in self._all_machines:
232      if m.name not in locked_machines:
233        self._all_machines.remove(m)
234
235    for m in self._machines:
236      if m.name not in locked_machines:
237        self._machines.remove(m)
238
239  def GetChromeVersion(self, machine):
240    """Get the version of Chrome running on the DUT."""
241
242    cmd = '/opt/google/chrome/chrome --version'
243    ret, version, _ = self.ce.CrosRunCommandWOutput(
244        cmd,
245        machine=machine.name,
246        chromeos_root=self.chromeos_root)
247    if ret != 0:
248      raise CrosCommandError("Couldn't get Chrome version from %s." %
249                             machine.name)
250
251    if ret != 0:
252      version = ''
253    return version.rstrip()
254
255  def ImageMachine(self, machine, label):
256    checksum = label.checksum
257
258    if checksum and (machine.checksum == checksum):
259      return
260    chromeos_root = label.chromeos_root
261    if not chromeos_root:
262      chromeos_root = self.chromeos_root
263    image_chromeos_args = [image_chromeos.__file__, '--no_lock',
264                           '--chromeos_root=%s' % chromeos_root,
265                           '--image=%s' % label.chromeos_image,
266                           '--image_args=%s' % label.image_args, '--remote=%s' %
267                           machine.name, '--logging_level=%s' % self.log_level]
268    if label.board:
269      image_chromeos_args.append('--board=%s' % label.board)
270
271    # Currently can't image two machines at once.
272    # So have to serialized on this lock.
273    save_ce_log_level = self.ce.log_level
274    if self.log_level != 'verbose':
275      self.ce.log_level = 'average'
276
277    with self.image_lock:
278      if self.log_level != 'verbose':
279        self.logger.LogOutput('Pushing image onto machine.')
280        self.logger.LogOutput('Running image_chromeos.DoImage with %s' %
281                              ' '.join(image_chromeos_args))
282      retval = 0
283      if not test_flag.GetTestMode():
284        retval = image_chromeos.DoImage(image_chromeos_args)
285      if retval:
286        cmd = 'reboot && exit'
287        if self.log_level != 'verbose':
288          self.logger.LogOutput('reboot & exit.')
289        self.ce.CrosRunCommand(cmd,
290                               machine=machine.name,
291                               chromeos_root=self.chromeos_root)
292        time.sleep(60)
293        if self.log_level != 'verbose':
294          self.logger.LogOutput('Pushing image onto machine.')
295          self.logger.LogOutput('Running image_chromeos.DoImage with %s' %
296                                ' '.join(image_chromeos_args))
297        retval = image_chromeos.DoImage(image_chromeos_args)
298      if retval:
299        raise Exception("Could not image machine: '%s'." % machine.name)
300      else:
301        self.num_reimages += 1
302      machine.checksum = checksum
303      machine.image = label.chromeos_image
304      machine.label = label
305
306    if not label.chrome_version:
307      label.chrome_version = self.GetChromeVersion(machine)
308
309    self.ce.log_level = save_ce_log_level
310    return retval
311
312  def ComputeCommonCheckSum(self, label):
313    # Since this is used for cache lookups before the machines have been
314    # compared/verified, check here to make sure they all have the same
315    # checksum (otherwise the cache lookup may not be valid).
316    common_checksum = None
317    for machine in self.GetMachines(label):
318      # Make sure the machine's checksums are calculated.
319      if not machine.machine_checksum:
320        machine.SetUpChecksumInfo()
321      cs = machine.machine_checksum
322      # If this is the first machine we've examined, initialize
323      # common_checksum.
324      if not common_checksum:
325        common_checksum = cs
326      # Make sure this machine's checksum matches our 'common' checksum.
327      if cs != common_checksum:
328        raise BadChecksum('Machine checksums do not match!')
329    self.machine_checksum[label.name] = common_checksum
330
331  def ComputeCommonCheckSumString(self, label):
332    # The assumption is that this function is only called AFTER
333    # ComputeCommonCheckSum, so there is no need to verify the machines
334    # are the same here.  If this is ever changed, this function should be
335    # modified to verify that all the machines for a given label are the
336    # same.
337    for machine in self.GetMachines(label):
338      if machine.checksum_string:
339        self.machine_checksum_string[label.name] = machine.checksum_string
340        break
341
342  def _TryToLockMachine(self, cros_machine):
343    with self._lock:
344      assert cros_machine, "Machine can't be None"
345      for m in self._machines:
346        if m.name == cros_machine.name:
347          return
348      locked = True
349      if self.locks_dir:
350        locked = file_lock_machine.Machine(cros_machine.name,
351                                           self.locks_dir).Lock(True,
352                                                                sys.argv[0])
353      if locked:
354        self._machines.append(cros_machine)
355        command = 'cat %s' % CHECKSUM_FILE
356        ret, out, _ = self.ce.CrosRunCommandWOutput(
357            command,
358            chromeos_root=self.chromeos_root,
359            machine=cros_machine.name)
360        if ret == 0:
361          cros_machine.checksum = out.strip()
362      elif self.locks_dir:
363        self.logger.LogOutput("Couldn't lock: %s" % cros_machine.name)
364
365  # This is called from single threaded mode.
366  def AddMachine(self, machine_name):
367    with self._lock:
368      for m in self._all_machines:
369        assert m.name != machine_name, 'Tried to double-add %s' % machine_name
370
371      if self.log_level != 'verbose':
372        self.logger.LogOutput('Setting up remote access to %s' % machine_name)
373        self.logger.LogOutput('Checking machine characteristics for %s' %
374                              machine_name)
375      cm = CrosMachine(machine_name, self.chromeos_root, self.log_level)
376      if cm.machine_checksum:
377        self._all_machines.append(cm)
378
379  def RemoveMachine(self, machine_name):
380    with self._lock:
381      self._machines = [m for m in self._machines if m.name != machine_name]
382      if self.locks_dir:
383        res = file_lock_machine.Machine(machine_name,
384                                        self.locks_dir).Unlock(True)
385        if not res:
386          self.logger.LogError("Could not unlock machine: '%s'." % machine_name)
387
388  def ForceSameImageToAllMachines(self, label):
389    machines = self.GetMachines(label)
390    for m in machines:
391      self.ImageMachine(m, label)
392      m.SetUpChecksumInfo()
393
394  def AcquireMachine(self, label):
395    image_checksum = label.checksum
396    machines = self.GetMachines(label)
397    check_interval_time = 120
398    with self._lock:
399      # Lazily external lock machines
400      while self.acquire_timeout >= 0:
401        for m in machines:
402          new_machine = m not in self._all_machines
403          self._TryToLockMachine(m)
404          if new_machine:
405            m.released_time = time.time()
406        if self.GetAvailableMachines(label):
407          break
408        else:
409          sleep_time = max(1, min(self.acquire_timeout, check_interval_time))
410          time.sleep(sleep_time)
411          self.acquire_timeout -= sleep_time
412
413      if self.acquire_timeout < 0:
414        machine_names = []
415        for machine in machines:
416          machine_names.append(machine.name)
417        self.logger.LogFatal('Could not acquire any of the '
418                             "following machines: '%s'" %
419                             ', '.join(machine_names))
420
421###      for m in self._machines:
422###        if (m.locked and time.time() - m.released_time < 10 and
423###            m.checksum == image_checksum):
424###          return None
425      for m in [machine
426                for machine in self.GetAvailableMachines(label)
427                if not machine.locked]:
428        if image_checksum and (m.checksum == image_checksum):
429          m.locked = True
430          m.test_run = threading.current_thread()
431          return m
432      for m in [machine
433                for machine in self.GetAvailableMachines(label)
434                if not machine.locked]:
435        if not m.checksum:
436          m.locked = True
437          m.test_run = threading.current_thread()
438          return m
439      # This logic ensures that threads waiting on a machine will get a machine
440      # with a checksum equal to their image over other threads. This saves time
441      # when crosperf initially assigns the machines to threads by minimizing
442      # the number of re-images.
443      # TODO(asharif): If we centralize the thread-scheduler, we wont need this
444      # code and can implement minimal reimaging code more cleanly.
445      for m in [machine
446                for machine in self.GetAvailableMachines(label)
447                if not machine.locked]:
448        if time.time() - m.released_time > 15:
449          # The release time gap is too large, so it is probably in the start
450          # stage, we need to reset the released_time.
451          m.released_time = time.time()
452        elif time.time() - m.released_time > 8:
453          m.locked = True
454          m.test_run = threading.current_thread()
455          return m
456    return None
457
458  def GetAvailableMachines(self, label=None):
459    if not label:
460      return self._machines
461    return [m for m in self._machines if m.name in label.remote]
462
463  def GetMachines(self, label=None):
464    if not label:
465      return self._all_machines
466    return [m for m in self._all_machines if m.name in label.remote]
467
468  def ReleaseMachine(self, machine):
469    with self._lock:
470      for m in self._machines:
471        if machine.name == m.name:
472          assert m.locked == True, 'Tried to double-release %s' % m.name
473          m.released_time = time.time()
474          m.locked = False
475          m.status = 'Available'
476          break
477
478  def Cleanup(self):
479    with self._lock:
480      # Unlock all machines (via file lock)
481      for m in self._machines:
482        res = file_lock_machine.Machine(m.name, self.locks_dir).Unlock(True)
483
484        if not res:
485          self.logger.LogError("Could not unlock machine: '%s'." % m.name)
486
487  def __str__(self):
488    with self._lock:
489      l = ['MachineManager Status:']
490      for m in self._machines:
491        l.append(str(m))
492      return '\n'.join(l)
493
494  def AsString(self):
495    with self._lock:
496      stringify_fmt = '%-30s %-10s %-4s %-25s %-32s'
497      header = stringify_fmt % ('Machine', 'Thread', 'Lock', 'Status',
498                                'Checksum')
499      table = [header]
500      for m in self._machines:
501        if m.test_run:
502          test_name = m.test_run.name
503          test_status = m.test_run.timeline.GetLastEvent()
504        else:
505          test_name = ''
506          test_status = ''
507
508        try:
509          machine_string = stringify_fmt % (m.name, test_name, m.locked,
510                                            test_status, m.checksum)
511        except ValueError:
512          machine_string = ''
513        table.append(machine_string)
514      return 'Machine Status:\n%s' % '\n'.join(table)
515
516  def GetAllCPUInfo(self, labels):
517    """Get cpuinfo for labels, merge them if their cpuinfo are the same."""
518    dic = {}
519    for label in labels:
520      for machine in self._all_machines:
521        if machine.name in label.remote:
522          if machine.cpuinfo not in dic:
523            dic[machine.cpuinfo] = [label.name]
524          else:
525            dic[machine.cpuinfo].append(label.name)
526          break
527    output = ''
528    for key, v in dic.items():
529      output += ' '.join(v)
530      output += '\n-------------------\n'
531      output += key
532      output += '\n\n\n'
533    return output
534
535  def GetAllMachines(self):
536    return self._all_machines
537
538
539class MockCrosMachine(CrosMachine):
540  """Mock cros machine class."""
541  # pylint: disable=super-init-not-called
542
543  MEMINFO_STRING = """MemTotal:        3990332 kB
544MemFree:         2608396 kB
545Buffers:          147168 kB
546Cached:           811560 kB
547SwapCached:            0 kB
548Active:           503480 kB
549Inactive:         628572 kB
550Active(anon):     174532 kB
551Inactive(anon):    88576 kB
552Active(file):     328948 kB
553Inactive(file):   539996 kB
554Unevictable:           0 kB
555Mlocked:               0 kB
556SwapTotal:       5845212 kB
557SwapFree:        5845212 kB
558Dirty:              9384 kB
559Writeback:             0 kB
560AnonPages:        173408 kB
561Mapped:           146268 kB
562Shmem:             89676 kB
563Slab:             188260 kB
564SReclaimable:     169208 kB
565SUnreclaim:        19052 kB
566KernelStack:        2032 kB
567PageTables:         7120 kB
568NFS_Unstable:          0 kB
569Bounce:                0 kB
570WritebackTmp:          0 kB
571CommitLimit:     7840376 kB
572Committed_AS:    1082032 kB
573VmallocTotal:   34359738367 kB
574VmallocUsed:      364980 kB
575VmallocChunk:   34359369407 kB
576DirectMap4k:       45824 kB
577DirectMap2M:     4096000 kB
578"""
579
580  CPUINFO_STRING = """processor: 0
581vendor_id: GenuineIntel
582cpu family: 6
583model: 42
584model name: Intel(R) Celeron(R) CPU 867 @ 1.30GHz
585stepping: 7
586microcode: 0x25
587cpu MHz: 1300.000
588cache size: 2048 KB
589physical id: 0
590siblings: 2
591core id: 0
592cpu cores: 2
593apicid: 0
594initial apicid: 0
595fpu: yes
596fpu_exception: yes
597cpuid level: 13
598wp: yes
599flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer xsave lahf_lm arat epb xsaveopt pln pts dts tpr_shadow vnmi flexpriority ept vpid
600bogomips: 2594.17
601clflush size: 64
602cache_alignment: 64
603address sizes: 36 bits physical, 48 bits virtual
604power management:
605
606processor: 1
607vendor_id: GenuineIntel
608cpu family: 6
609model: 42
610model name: Intel(R) Celeron(R) CPU 867 @ 1.30GHz
611stepping: 7
612microcode: 0x25
613cpu MHz: 1300.000
614cache size: 2048 KB
615physical id: 0
616siblings: 2
617core id: 1
618cpu cores: 2
619apicid: 2
620initial apicid: 2
621fpu: yes
622fpu_exception: yes
623cpuid level: 13
624wp: yes
625flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer xsave lahf_lm arat epb xsaveopt pln pts dts tpr_shadow vnmi flexpriority ept vpid
626bogomips: 2594.17
627clflush size: 64
628cache_alignment: 64
629address sizes: 36 bits physical, 48 bits virtual
630power management:
631"""
632
633  def __init__(self, name, chromeos_root, log_level):
634    self.name = name
635    self.image = None
636    self.checksum = None
637    self.locked = False
638    self.released_time = time.time()
639    self.test_run = None
640    self.chromeos_root = chromeos_root
641    self.checksum_string = re.sub(r'\d', '', name)
642    #In test, we assume "lumpy1", "lumpy2" are the same machine.
643    self.machine_checksum = self._GetMD5Checksum(self.checksum_string)
644    self.log_level = log_level
645    self.label = None
646    self.ce = command_executer.GetCommandExecuter(log_level=self.log_level)
647
648  def IsReachable(self):
649    return True
650
651  def _GetMemoryInfo(self):
652    self.meminfo = self.MEMINFO_STRING
653    self._ParseMemoryInfo()
654
655  def _GetCPUInfo(self):
656    self.cpuinfo = self.CPUINFO_STRING
657
658
659class MockMachineManager(MachineManager):
660  """Mock machine manager class."""
661
662  def __init__(self, chromeos_root, acquire_timeout, log_level, locks_dir):
663    super(MockMachineManager, self).__init__(
664        chromeos_root, acquire_timeout, log_level, locks_dir)
665
666  def _TryToLockMachine(self, cros_machine):
667    self._machines.append(cros_machine)
668    cros_machine.checksum = ''
669
670  def AddMachine(self, machine_name):
671    with self._lock:
672      for m in self._all_machines:
673        assert m.name != machine_name, 'Tried to double-add %s' % machine_name
674      cm = MockCrosMachine(machine_name, self.chromeos_root, self.log_level)
675      assert cm.machine_checksum, ('Could not find checksum for machine %s' %
676                                   machine_name)
677      # In Original MachineManager, the test is 'if cm.machine_checksum:' - if a
678      # machine is unreachable, then its machine_checksum is None. Here we
679      # cannot do this, because machine_checksum is always faked, so we directly
680      # test cm.IsReachable, which is properly mocked.
681      if cm.IsReachable():
682        self._all_machines.append(cm)
683
684  def GetChromeVersion(self, machine):
685    return 'Mock Chrome Version R50'
686
687  def AcquireMachine(self, label):
688    for machine in self._all_machines:
689      if not machine.locked:
690        machine.locked = True
691        return machine
692    return None
693
694  def ImageMachine(self, machine_name, label):
695    if machine_name or label:
696      return 0
697    return 1
698
699  def ReleaseMachine(self, machine):
700    machine.locked = False
701
702  def GetMachines(self, label=None):
703    return self._all_machines
704
705  def GetAvailableMachines(self, label=None):
706    return self._all_machines
707
708  def ForceSameImageToAllMachines(self, label=None):
709    return 0
710
711  def ComputeCommonCheckSum(self, label=None):
712    common_checksum = 12345
713    for machine in self.GetMachines(label):
714      machine.machine_checksum = common_checksum
715    self.machine_checksum[label.name] = common_checksum
716
717  def GetAllMachines(self):
718    return self._all_machines
719