machine_manager.py revision 63f13489b1a02bbbc75b46fec6ae7a817442df94 
1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Machine Manager module."""
5
6from __future__ import print_function
7
8import collections
9import file_lock_machine
10import hashlib
11import image_chromeos
12import math
13import os.path
14import re
15import sys
16import threading
17import time
18
19import test_flag
20from cros_utils import command_executer
21from cros_utils import logger
22
23CHECKSUM_FILE = '/usr/local/osimage_checksum_file'
24
25
26class BadChecksum(Exception):
27  """Raised if all machines for a label don't have the same checksum."""
28  pass
29
30
31class BadChecksumString(Exception):
32  """Raised if all machines for a label don't have the same checksum string."""
33  pass
34
35
36class MissingLocksDirectory(Exception):
37  """Raised when cannot find/access the machine locks directory."""
38
39
40class CrosCommandError(Exception):
41  """Raised when an error occurs running command on DUT."""
42
43
44class CrosMachine(object):
45  """The machine class."""
46
47  def __init__(self, name, chromeos_root, log_level, cmd_exec=None):
48    self.name = name
49    self.image = None
50    # We relate a dut with a label if we reimage the dut using label or we
51    # detect at the very beginning that the dut is running this label.
52    self.label = None
53    self.checksum = None
54    self.locked = False
55    self.released_time = time.time()
56    self.test_run = None
57    self.chromeos_root = chromeos_root
58    self.log_level = log_level
59    self.cpuinfo = None
60    self.machine_id = None
61    self.checksum_string = None
62    self.meminfo = None
63    self.phys_kbytes = None
64    self.ce = cmd_exec or command_executer.GetCommandExecuter(
65        log_level=self.log_level)
66    self.SetUpChecksumInfo()
67
68  def SetUpChecksumInfo(self):
69    if not self.IsReachable():
70      self.machine_checksum = None
71      return
72    self._GetMemoryInfo()
73    self._GetCPUInfo()
74    self._ComputeMachineChecksumString()
75    self._GetMachineID()
76    self.machine_checksum = self._GetMD5Checksum(self.checksum_string)
77    self.machine_id_checksum = self._GetMD5Checksum(self.machine_id)
78
79  def IsReachable(self):
80    command = 'ls'
81    ret = self.ce.CrosRunCommand(command,
82                                 machine=self.name,
83                                 chromeos_root=self.chromeos_root)
84    if ret:
85      return False
86    return True
87
88  def _ParseMemoryInfo(self):
89    line = self.meminfo.splitlines()[0]
90    usable_kbytes = int(line.split()[1])
91    # This code is from src/third_party/test/files/client/bin/base_utils.py
92    # usable_kbytes is system's usable DRAM in kbytes,
93    #   as reported by memtotal() from device /proc/meminfo memtotal
94    #   after Linux deducts 1.5% to 9.5% for system table overhead
95    # Undo the unknown actual deduction by rounding up
96    #   to next small multiple of a big power-of-two
97    #   eg  12GB - 5.1% gets rounded back up to 12GB
98    mindeduct = 0.005  # 0.5 percent
99    maxdeduct = 0.095  # 9.5 percent
100    # deduction range 1.5% .. 9.5% supports physical mem sizes
101    #    6GB .. 12GB in steps of .5GB
102    #   12GB .. 24GB in steps of 1 GB
103    #   24GB .. 48GB in steps of 2 GB ...
104    # Finer granularity in physical mem sizes would require
105    #   tighter spread between min and max possible deductions
106
107    # increase mem size by at least min deduction, without rounding
108    min_kbytes = int(usable_kbytes / (1.0 - mindeduct))
109    # increase mem size further by 2**n rounding, by 0..roundKb or more
110    round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes
111    # find least binary roundup 2**n that covers worst-cast roundKb
112    mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2)))
113    # have round_kbytes <= mod2n < round_kbytes*2
114    # round min_kbytes up to next multiple of mod2n
115    phys_kbytes = min_kbytes + mod2n - 1
116    phys_kbytes -= phys_kbytes % mod2n  # clear low bits
117    self.phys_kbytes = phys_kbytes
118
119  def _GetMemoryInfo(self):
120    #TODO yunlian: when the machine in rebooting, it will not return
121    #meminfo, the assert does not catch it either
122    command = 'cat /proc/meminfo'
123    ret, self.meminfo, _ = self.ce.CrosRunCommandWOutput(
124        command,
125        machine=self.name,
126        chromeos_root=self.chromeos_root)
127    assert ret == 0, 'Could not get meminfo from machine: %s' % self.name
128    if ret == 0:
129      self._ParseMemoryInfo()
130
131  def _GetCPUInfo(self):
132    command = 'cat /proc/cpuinfo'
133    ret, self.cpuinfo, _ = self.ce.CrosRunCommandWOutput(
134        command,
135        machine=self.name,
136        chromeos_root=self.chromeos_root)
137    assert ret == 0, 'Could not get cpuinfo from machine: %s' % self.name
138
139  def _ComputeMachineChecksumString(self):
140    self.checksum_string = ''
141    exclude_lines_list = ['MHz', 'BogoMIPS', 'bogomips']
142    for line in self.cpuinfo.splitlines():
143      if not any([e in line for e in exclude_lines_list]):
144        self.checksum_string += line
145    self.checksum_string += ' ' + str(self.phys_kbytes)
146
147  def _GetMD5Checksum(self, ss):
148    if ss:
149      return hashlib.md5(ss).hexdigest()
150    else:
151      return ''
152
153  def _GetMachineID(self):
154    command = 'dump_vpd_log --full --stdout'
155    _, if_out, _ = self.ce.CrosRunCommandWOutput(
156        command,
157        machine=self.name,
158        chromeos_root=self.chromeos_root)
159    b = if_out.splitlines()
160    a = [l for l in b if 'Product' in l]
161    if len(a):
162      self.machine_id = a[0]
163      return
164    command = 'ifconfig'
165    _, if_out, _ = self.ce.CrosRunCommandWOutput(
166        command,
167        machine=self.name,
168        chromeos_root=self.chromeos_root)
169    b = if_out.splitlines()
170    a = [l for l in b if 'HWaddr' in l]
171    if len(a):
172      self.machine_id = '_'.join(a)
173      return
174    a = [l for l in b if 'ether' in l]
175    if len(a):
176      self.machine_id = '_'.join(a)
177      return
178    assert 0, 'Could not get machine_id from machine: %s' % self.name
179
180  def __str__(self):
181    l = []
182    l.append(self.name)
183    l.append(str(self.image))
184    l.append(str(self.checksum))
185    l.append(str(self.locked))
186    l.append(str(self.released_time))
187    return ', '.join(l)
188
189
190class MachineManager(object):
191  """Lock, image and unlock machines locally for benchmark runs.
192
193  This class contains methods and calls to lock, unlock and image
194  machines and distribute machines to each benchmark run.  The assumption is
195  that all of the machines for the experiment have been globally locked
196  (using an AFE server) in the ExperimentRunner, but the machines still need
197  to be locally locked/unlocked (allocated to benchmark runs) to prevent
198  multiple benchmark runs within the same experiment from trying to use the
199  same machine at the same time.
200  """
201
202  def __init__(self,
203               chromeos_root,
204               acquire_timeout,
205               log_level,
206               locks_dir,
207               cmd_exec=None,
208               lgr=None):
209    self._lock = threading.RLock()
210    self._all_machines = []
211    self._machines = []
212    self.image_lock = threading.Lock()
213    self.num_reimages = 0
214    self.chromeos_root = None
215    self.machine_checksum = {}
216    self.machine_checksum_string = {}
217    self.acquire_timeout = acquire_timeout
218    self.log_level = log_level
219    self.locks_dir = locks_dir
220    self.ce = cmd_exec or command_executer.GetCommandExecuter(
221        log_level=self.log_level)
222    self.logger = lgr or logger.GetLogger()
223
224    if self.locks_dir and not os.path.isdir(self.locks_dir):
225      raise MissingLocksDirectory('Cannot access locks directory: %s' %
226                                  self.locks_dir)
227
228    self._initialized_machines = []
229    self.chromeos_root = chromeos_root
230
231  def RemoveNonLockedMachines(self, locked_machines):
232    for m in self._all_machines:
233      if m.name not in locked_machines:
234        self._all_machines.remove(m)
235
236    for m in self._machines:
237      if m.name not in locked_machines:
238        self._machines.remove(m)
239
240  def GetChromeVersion(self, machine):
241    """Get the version of Chrome running on the DUT."""
242
243    cmd = '/opt/google/chrome/chrome --version'
244    ret, version, _ = self.ce.CrosRunCommandWOutput(
245        cmd,
246        machine=machine.name,
247        chromeos_root=self.chromeos_root)
248    if ret != 0:
249      raise CrosCommandError("Couldn't get Chrome version from %s." %
250                             machine.name)
251
252    if ret != 0:
253      version = ''
254    return version.rstrip()
255
256  def ImageMachine(self, machine, label):
257    checksum = label.checksum
258
259    if checksum and (machine.checksum == checksum):
260      return
261    chromeos_root = label.chromeos_root
262    if not chromeos_root:
263      chromeos_root = self.chromeos_root
264    image_chromeos_args = [image_chromeos.__file__, '--no_lock',
265                           '--chromeos_root=%s' % chromeos_root,
266                           '--image=%s' % label.chromeos_image,
267                           '--image_args=%s' % label.image_args, '--remote=%s' %
268                           machine.name, '--logging_level=%s' % self.log_level]
269    if label.board:
270      image_chromeos_args.append('--board=%s' % label.board)
271
272    # Currently can't image two machines at once.
273    # So have to serialized on this lock.
274    save_ce_log_level = self.ce.log_level
275    if self.log_level != 'verbose':
276      self.ce.log_level = 'average'
277
278    with self.image_lock:
279      if self.log_level != 'verbose':
280        self.logger.LogOutput('Pushing image onto machine.')
281        self.logger.LogOutput('Running image_chromeos.DoImage with %s' %
282                              ' '.join(image_chromeos_args))
283      retval = 0
284      if not test_flag.GetTestMode():
285        retval = image_chromeos.DoImage(image_chromeos_args)
286      if retval:
287        cmd = 'reboot && exit'
288        if self.log_level != 'verbose':
289          self.logger.LogOutput('reboot & exit.')
290        self.ce.CrosRunCommand(cmd,
291                               machine=machine.name,
292                               chromeos_root=self.chromeos_root)
293        time.sleep(60)
294        if self.log_level != 'verbose':
295          self.logger.LogOutput('Pushing image onto machine.')
296          self.logger.LogOutput('Running image_chromeos.DoImage with %s' %
297                                ' '.join(image_chromeos_args))
298        retval = image_chromeos.DoImage(image_chromeos_args)
299      if retval:
300        raise Exception("Could not image machine: '%s'." % machine.name)
301      else:
302        self.num_reimages += 1
303      machine.checksum = checksum
304      machine.image = label.chromeos_image
305      machine.label = label
306
307    if not label.chrome_version:
308      label.chrome_version = self.GetChromeVersion(machine)
309
310    self.ce.log_level = save_ce_log_level
311    return retval
312
313  def ComputeCommonCheckSum(self, label):
314    # Since this is used for cache lookups before the machines have been
315    # compared/verified, check here to make sure they all have the same
316    # checksum (otherwise the cache lookup may not be valid).
317    common_checksum = None
318    for machine in self.GetMachines(label):
319      # Make sure the machine's checksums are calculated.
320      if not machine.machine_checksum:
321        machine.SetUpChecksumInfo()
322      cs = machine.machine_checksum
323      # If this is the first machine we've examined, initialize
324      # common_checksum.
325      if not common_checksum:
326        common_checksum = cs
327      # Make sure this machine's checksum matches our 'common' checksum.
328      if cs != common_checksum:
329        raise BadChecksum('Machine checksums do not match!')
330    self.machine_checksum[label.name] = common_checksum
331
332  def ComputeCommonCheckSumString(self, label):
333    # The assumption is that this function is only called AFTER
334    # ComputeCommonCheckSum, so there is no need to verify the machines
335    # are the same here.  If this is ever changed, this function should be
336    # modified to verify that all the machines for a given label are the
337    # same.
338    for machine in self.GetMachines(label):
339      if machine.checksum_string:
340        self.machine_checksum_string[label.name] = machine.checksum_string
341        break
342
343  def _TryToLockMachine(self, cros_machine):
344    with self._lock:
345      assert cros_machine, "Machine can't be None"
346      for m in self._machines:
347        if m.name == cros_machine.name:
348          return
349      locked = True
350      if self.locks_dir:
351        locked = file_lock_machine.Machine(cros_machine.name,
352                                           self.locks_dir).Lock(True,
353                                                                sys.argv[0])
354      if locked:
355        self._machines.append(cros_machine)
356        command = 'cat %s' % CHECKSUM_FILE
357        ret, out, _ = self.ce.CrosRunCommandWOutput(
358            command,
359            chromeos_root=self.chromeos_root,
360            machine=cros_machine.name)
361        if ret == 0:
362          cros_machine.checksum = out.strip()
363      elif self.locks_dir:
364        self.logger.LogOutput("Couldn't lock: %s" % cros_machine.name)
365
366  # This is called from single threaded mode.
367  def AddMachine(self, machine_name):
368    with self._lock:
369      for m in self._all_machines:
370        assert m.name != machine_name, 'Tried to double-add %s' % machine_name
371
372      if self.log_level != 'verbose':
373        self.logger.LogOutput('Setting up remote access to %s' % machine_name)
374        self.logger.LogOutput('Checking machine characteristics for %s' %
375                              machine_name)
376      cm = CrosMachine(machine_name, self.chromeos_root, self.log_level)
377      if cm.machine_checksum:
378        self._all_machines.append(cm)
379
380  def RemoveMachine(self, machine_name):
381    with self._lock:
382      self._machines = [m for m in self._machines if m.name != machine_name]
383      if self.locks_dir:
384        res = file_lock_machine.Machine(machine_name,
385                                        self.locks_dir).Unlock(True)
386        if not res:
387          self.logger.LogError("Could not unlock machine: '%s'." % machine_name)
388
389  def ForceSameImageToAllMachines(self, label):
390    machines = self.GetMachines(label)
391    for m in machines:
392      self.ImageMachine(m, label)
393      m.SetUpChecksumInfo()
394
395  def AcquireMachine(self, label):
396    image_checksum = label.checksum
397    machines = self.GetMachines(label)
398    check_interval_time = 120
399    with self._lock:
400      # Lazily external lock machines
401      while self.acquire_timeout >= 0:
402        for m in machines:
403          new_machine = m not in self._all_machines
404          self._TryToLockMachine(m)
405          if new_machine:
406            m.released_time = time.time()
407        if self.GetAvailableMachines(label):
408          break
409        else:
410          sleep_time = max(1, min(self.acquire_timeout, check_interval_time))
411          time.sleep(sleep_time)
412          self.acquire_timeout -= sleep_time
413
414      if self.acquire_timeout < 0:
415        machine_names = []
416        for machine in machines:
417          machine_names.append(machine.name)
418        self.logger.LogFatal('Could not acquire any of the '
419                             "following machines: '%s'" %
420                             ', '.join(machine_names))
421
422###      for m in self._machines:
423###        if (m.locked and time.time() - m.released_time < 10 and
424###            m.checksum == image_checksum):
425###          return None
426      for m in [machine
427                for machine in self.GetAvailableMachines(label)
428                if not machine.locked]:
429        if image_checksum and (m.checksum == image_checksum):
430          m.locked = True
431          m.test_run = threading.current_thread()
432          return m
433      for m in [machine
434                for machine in self.GetAvailableMachines(label)
435                if not machine.locked]:
436        if not m.checksum:
437          m.locked = True
438          m.test_run = threading.current_thread()
439          return m
440      # This logic ensures that threads waiting on a machine will get a machine
441      # with a checksum equal to their image over other threads. This saves time
442      # when crosperf initially assigns the machines to threads by minimizing
443      # the number of re-images.
444      # TODO(asharif): If we centralize the thread-scheduler, we wont need this
445      # code and can implement minimal reimaging code more cleanly.
446      for m in [machine
447                for machine in self.GetAvailableMachines(label)
448                if not machine.locked]:
449        if time.time() - m.released_time > 15:
450          # The release time gap is too large, so it is probably in the start
451          # stage, we need to reset the released_time.
452          m.released_time = time.time()
453        elif time.time() - m.released_time > 8:
454          m.locked = True
455          m.test_run = threading.current_thread()
456          return m
457    return None
458
459  def GetAvailableMachines(self, label=None):
460    if not label:
461      return self._machines
462    return [m for m in self._machines if m.name in label.remote]
463
464  def GetMachines(self, label=None):
465    if not label:
466      return self._all_machines
467    return [m for m in self._all_machines if m.name in label.remote]
468
469  def ReleaseMachine(self, machine):
470    with self._lock:
471      for m in self._machines:
472        if machine.name == m.name:
473          assert m.locked == True, 'Tried to double-release %s' % m.name
474          m.released_time = time.time()
475          m.locked = False
476          m.status = 'Available'
477          break
478
479  def Cleanup(self):
480    with self._lock:
481      # Unlock all machines (via file lock)
482      for m in self._machines:
483        res = file_lock_machine.Machine(m.name, self.locks_dir).Unlock(True)
484
485        if not res:
486          self.logger.LogError("Could not unlock machine: '%s'." % m.name)
487
488  def __str__(self):
489    with self._lock:
490      l = ['MachineManager Status:']
491      for m in self._machines:
492        l.append(str(m))
493      return '\n'.join(l)
494
495  def AsString(self):
496    with self._lock:
497      stringify_fmt = '%-30s %-10s %-4s %-25s %-32s'
498      header = stringify_fmt % ('Machine', 'Thread', 'Lock', 'Status',
499                                'Checksum')
500      table = [header]
501      for m in self._machines:
502        if m.test_run:
503          test_name = m.test_run.name
504          test_status = m.test_run.timeline.GetLastEvent()
505        else:
506          test_name = ''
507          test_status = ''
508
509        try:
510          machine_string = stringify_fmt % (m.name, test_name, m.locked,
511                                            test_status, m.checksum)
512        except ValueError:
513          machine_string = ''
514        table.append(machine_string)
515      return 'Machine Status:\n%s' % '\n'.join(table)
516
517  def GetAllCPUInfo(self, labels):
518    """Get cpuinfo for labels, merge them if their cpuinfo are the same."""
519    dic = collections.defaultdict(list)
520    for label in labels:
521      for machine in self._all_machines:
522        if machine.name in label.remote:
523          dic[machine.cpuinfo].append(label.name)
524          break
525    output_segs = []
526    for key, v in dic.iteritems():
527      output = ' '.join(v)
528      output += '\n-------------------\n'
529      output += key
530      output += '\n\n\n'
531      output_segs.append(output)
532    return ''.join(output_segs)
533
534  def GetAllMachines(self):
535    return self._all_machines
536
537
538class MockCrosMachine(CrosMachine):
539  """Mock cros machine class."""
540  # pylint: disable=super-init-not-called
541
542  MEMINFO_STRING = """MemTotal:        3990332 kB
543MemFree:         2608396 kB
544Buffers:          147168 kB
545Cached:           811560 kB
546SwapCached:            0 kB
547Active:           503480 kB
548Inactive:         628572 kB
549Active(anon):     174532 kB
550Inactive(anon):    88576 kB
551Active(file):     328948 kB
552Inactive(file):   539996 kB
553Unevictable:           0 kB
554Mlocked:               0 kB
555SwapTotal:       5845212 kB
556SwapFree:        5845212 kB
557Dirty:              9384 kB
558Writeback:             0 kB
559AnonPages:        173408 kB
560Mapped:           146268 kB
561Shmem:             89676 kB
562Slab:             188260 kB
563SReclaimable:     169208 kB
564SUnreclaim:        19052 kB
565KernelStack:        2032 kB
566PageTables:         7120 kB
567NFS_Unstable:          0 kB
568Bounce:                0 kB
569WritebackTmp:          0 kB
570CommitLimit:     7840376 kB
571Committed_AS:    1082032 kB
572VmallocTotal:   34359738367 kB
573VmallocUsed:      364980 kB
574VmallocChunk:   34359369407 kB
575DirectMap4k:       45824 kB
576DirectMap2M:     4096000 kB
577"""
578
579  CPUINFO_STRING = """processor: 0
580vendor_id: GenuineIntel
581cpu family: 6
582model: 42
583model name: Intel(R) Celeron(R) CPU 867 @ 1.30GHz
584stepping: 7
585microcode: 0x25
586cpu MHz: 1300.000
587cache size: 2048 KB
588physical id: 0
589siblings: 2
590core id: 0
591cpu cores: 2
592apicid: 0
593initial apicid: 0
594fpu: yes
595fpu_exception: yes
596cpuid level: 13
597wp: yes
598flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer xsave lahf_lm arat epb xsaveopt pln pts dts tpr_shadow vnmi flexpriority ept vpid
599bogomips: 2594.17
600clflush size: 64
601cache_alignment: 64
602address sizes: 36 bits physical, 48 bits virtual
603power management:
604
605processor: 1
606vendor_id: GenuineIntel
607cpu family: 6
608model: 42
609model name: Intel(R) Celeron(R) CPU 867 @ 1.30GHz
610stepping: 7
611microcode: 0x25
612cpu MHz: 1300.000
613cache size: 2048 KB
614physical id: 0
615siblings: 2
616core id: 1
617cpu cores: 2
618apicid: 2
619initial apicid: 2
620fpu: yes
621fpu_exception: yes
622cpuid level: 13
623wp: yes
624flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer xsave lahf_lm arat epb xsaveopt pln pts dts tpr_shadow vnmi flexpriority ept vpid
625bogomips: 2594.17
626clflush size: 64
627cache_alignment: 64
628address sizes: 36 bits physical, 48 bits virtual
629power management:
630"""
631
632  def __init__(self, name, chromeos_root, log_level):
633    self.name = name
634    self.image = None
635    self.checksum = None
636    self.locked = False
637    self.released_time = time.time()
638    self.test_run = None
639    self.chromeos_root = chromeos_root
640    self.checksum_string = re.sub(r'\d', '', name)
641    #In test, we assume "lumpy1", "lumpy2" are the same machine.
642    self.machine_checksum = self._GetMD5Checksum(self.checksum_string)
643    self.log_level = log_level
644    self.label = None
645    self.ce = command_executer.GetCommandExecuter(log_level=self.log_level)
646    self._GetCPUInfo()
647
648  def IsReachable(self):
649    return True
650
651  def _GetMemoryInfo(self):
652    self.meminfo = self.MEMINFO_STRING
653    self._ParseMemoryInfo()
654
655  def _GetCPUInfo(self):
656    self.cpuinfo = self.CPUINFO_STRING
657
658
659class MockMachineManager(MachineManager):
660  """Mock machine manager class."""
661
662  def __init__(self, chromeos_root, acquire_timeout, log_level, locks_dir):
663    super(MockMachineManager, self).__init__(
664        chromeos_root, acquire_timeout, log_level, locks_dir)
665
666  def _TryToLockMachine(self, cros_machine):
667    self._machines.append(cros_machine)
668    cros_machine.checksum = ''
669
670  def AddMachine(self, machine_name):
671    with self._lock:
672      for m in self._all_machines:
673        assert m.name != machine_name, 'Tried to double-add %s' % machine_name
674      cm = MockCrosMachine(machine_name, self.chromeos_root, self.log_level)
675      assert cm.machine_checksum, ('Could not find checksum for machine %s' %
676                                   machine_name)
677      # In Original MachineManager, the test is 'if cm.machine_checksum:' - if a
678      # machine is unreachable, then its machine_checksum is None. Here we
679      # cannot do this, because machine_checksum is always faked, so we directly
680      # test cm.IsReachable, which is properly mocked.
681      if cm.IsReachable():
682        self._all_machines.append(cm)
683
684  def GetChromeVersion(self, machine):
685    return 'Mock Chrome Version R50'
686
687  def AcquireMachine(self, label):
688    for machine in self._all_machines:
689      if not machine.locked:
690        machine.locked = True
691        return machine
692    return None
693
694  def ImageMachine(self, machine_name, label):
695    if machine_name or label:
696      return 0
697    return 1
698
699  def ReleaseMachine(self, machine):
700    machine.locked = False
701
702  def GetMachines(self, label=None):
703    return self._all_machines
704
705  def GetAvailableMachines(self, label=None):
706    return self._all_machines
707
708  def ForceSameImageToAllMachines(self, label=None):
709    return 0
710
711  def ComputeCommonCheckSum(self, label=None):
712    common_checksum = 12345
713    for machine in self.GetMachines(label):
714      machine.machine_checksum = common_checksum
715    self.machine_checksum[label.name] = common_checksum
716
717  def GetAllMachines(self):
718    return self._all_machines
719