machine_manager.py revision e56ceb4e7387a7f7ccb6d8d3b03e3e9692453cbd 
1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Machine Manager module."""
5
6from __future__ import print_function
7
8import collections
9import file_lock_machine
10import hashlib
11import image_chromeos
12import math
13import os.path
14import re
15import sys
16import threading
17import time
18
19import test_flag
20from cros_utils import command_executer
21from cros_utils import logger
22
23CHECKSUM_FILE = '/usr/local/osimage_checksum_file'
24
25
26class BadChecksum(Exception):
27  """Raised if all machines for a label don't have the same checksum."""
28  pass
29
30
31class BadChecksumString(Exception):
32  """Raised if all machines for a label don't have the same checksum string."""
33  pass
34
35
36class MissingLocksDirectory(Exception):
37  """Raised when cannot find/access the machine locks directory."""
38
39
40class CrosCommandError(Exception):
41  """Raised when an error occurs running command on DUT."""
42
43
44class CrosMachine(object):
45  """The machine class."""
46
47  def __init__(self, name, chromeos_root, log_level, cmd_exec=None):
48    self.name = name
49    self.image = None
50    # We relate a dut with a label if we reimage the dut using label or we
51    # detect at the very beginning that the dut is running this label.
52    self.label = None
53    self.checksum = None
54    self.locked = False
55    self.released_time = time.time()
56    self.test_run = None
57    self.chromeos_root = chromeos_root
58    self.log_level = log_level
59    self.cpuinfo = None
60    self.machine_id = None
61    self.checksum_string = None
62    self.meminfo = None
63    self.phys_kbytes = None
64    self.ce = cmd_exec or command_executer.GetCommandExecuter(
65        log_level=self.log_level)
66    self.SetUpChecksumInfo()
67
68  def SetUpChecksumInfo(self):
69    if not self.IsReachable():
70      self.machine_checksum = None
71      return
72    self._GetMemoryInfo()
73    self._GetCPUInfo()
74    self._ComputeMachineChecksumString()
75    self._GetMachineID()
76    self.machine_checksum = self._GetMD5Checksum(self.checksum_string)
77    self.machine_id_checksum = self._GetMD5Checksum(self.machine_id)
78
79  def IsReachable(self):
80    command = 'ls'
81    ret = self.ce.CrosRunCommand(command,
82                                 machine=self.name,
83                                 chromeos_root=self.chromeos_root)
84    if ret:
85      return False
86    return True
87
88  def _ParseMemoryInfo(self):
89    line = self.meminfo.splitlines()[0]
90    usable_kbytes = int(line.split()[1])
91    # This code is from src/third_party/test/files/client/bin/base_utils.py
92    # usable_kbytes is system's usable DRAM in kbytes,
93    #   as reported by memtotal() from device /proc/meminfo memtotal
94    #   after Linux deducts 1.5% to 9.5% for system table overhead
95    # Undo the unknown actual deduction by rounding up
96    #   to next small multiple of a big power-of-two
97    #   eg  12GB - 5.1% gets rounded back up to 12GB
98    mindeduct = 0.005  # 0.5 percent
99    maxdeduct = 0.095  # 9.5 percent
100    # deduction range 1.5% .. 9.5% supports physical mem sizes
101    #    6GB .. 12GB in steps of .5GB
102    #   12GB .. 24GB in steps of 1 GB
103    #   24GB .. 48GB in steps of 2 GB ...
104    # Finer granularity in physical mem sizes would require
105    #   tighter spread between min and max possible deductions
106
107    # increase mem size by at least min deduction, without rounding
108    min_kbytes = int(usable_kbytes / (1.0 - mindeduct))
109    # increase mem size further by 2**n rounding, by 0..roundKb or more
110    round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes
111    # find least binary roundup 2**n that covers worst-cast roundKb
112    mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2)))
113    # have round_kbytes <= mod2n < round_kbytes*2
114    # round min_kbytes up to next multiple of mod2n
115    phys_kbytes = min_kbytes + mod2n - 1
116    phys_kbytes -= phys_kbytes % mod2n  # clear low bits
117    self.phys_kbytes = phys_kbytes
118
119  def _GetMemoryInfo(self):
120    #TODO yunlian: when the machine in rebooting, it will not return
121    #meminfo, the assert does not catch it either
122    command = 'cat /proc/meminfo'
123    ret, self.meminfo, _ = self.ce.CrosRunCommandWOutput(
124        command,
125        machine=self.name,
126        chromeos_root=self.chromeos_root)
127    assert ret == 0, 'Could not get meminfo from machine: %s' % self.name
128    if ret == 0:
129      self._ParseMemoryInfo()
130
131  def _GetCPUInfo(self):
132    command = 'cat /proc/cpuinfo'
133    ret, self.cpuinfo, _ = self.ce.CrosRunCommandWOutput(
134        command,
135        machine=self.name,
136        chromeos_root=self.chromeos_root)
137    assert ret == 0, 'Could not get cpuinfo from machine: %s' % self.name
138
139  def _ComputeMachineChecksumString(self):
140    self.checksum_string = ''
141    exclude_lines_list = ['MHz', 'BogoMIPS', 'bogomips']
142    for line in self.cpuinfo.splitlines():
143      if not any(e in line for e in exclude_lines_list):
144        self.checksum_string += line
145    self.checksum_string += ' ' + str(self.phys_kbytes)
146
147  def _GetMD5Checksum(self, ss):
148    if ss:
149      return hashlib.md5(ss).hexdigest()
150    else:
151      return ''
152
153  def _GetMachineID(self):
154    command = 'dump_vpd_log --full --stdout'
155    _, if_out, _ = self.ce.CrosRunCommandWOutput(
156        command,
157        machine=self.name,
158        chromeos_root=self.chromeos_root)
159    b = if_out.splitlines()
160    a = [l for l in b if 'Product' in l]
161    if len(a):
162      self.machine_id = a[0]
163      return
164    command = 'ifconfig'
165    _, if_out, _ = self.ce.CrosRunCommandWOutput(
166        command,
167        machine=self.name,
168        chromeos_root=self.chromeos_root)
169    b = if_out.splitlines()
170    a = [l for l in b if 'HWaddr' in l]
171    if len(a):
172      self.machine_id = '_'.join(a)
173      return
174    a = [l for l in b if 'ether' in l]
175    if len(a):
176      self.machine_id = '_'.join(a)
177      return
178    assert 0, 'Could not get machine_id from machine: %s' % self.name
179
180  def __str__(self):
181    l = []
182    l.append(self.name)
183    l.append(str(self.image))
184    l.append(str(self.checksum))
185    l.append(str(self.locked))
186    l.append(str(self.released_time))
187    return ', '.join(l)
188
189
190class MachineManager(object):
191  """Lock, image and unlock machines locally for benchmark runs.
192
193  This class contains methods and calls to lock, unlock and image
194  machines and distribute machines to each benchmark run.  The assumption is
195  that all of the machines for the experiment have been globally locked
196  (using an AFE server) in the ExperimentRunner, but the machines still need
197  to be locally locked/unlocked (allocated to benchmark runs) to prevent
198  multiple benchmark runs within the same experiment from trying to use the
199  same machine at the same time.
200  """
201
202  def __init__(self,
203               chromeos_root,
204               acquire_timeout,
205               log_level,
206               locks_dir,
207               cmd_exec=None,
208               lgr=None):
209    self._lock = threading.RLock()
210    self._all_machines = []
211    self._machines = []
212    self.image_lock = threading.Lock()
213    self.num_reimages = 0
214    self.chromeos_root = None
215    self.machine_checksum = {}
216    self.machine_checksum_string = {}
217    self.acquire_timeout = acquire_timeout
218    self.log_level = log_level
219    self.locks_dir = locks_dir
220    self.ce = cmd_exec or command_executer.GetCommandExecuter(
221        log_level=self.log_level)
222    self.logger = lgr or logger.GetLogger()
223
224    if self.locks_dir and not os.path.isdir(self.locks_dir):
225      raise MissingLocksDirectory('Cannot access locks directory: %s' %
226                                  self.locks_dir)
227
228    self._initialized_machines = []
229    self.chromeos_root = chromeos_root
230
231  def RemoveNonLockedMachines(self, locked_machines):
232    for m in self._all_machines:
233      if m.name not in locked_machines:
234        self._all_machines.remove(m)
235
236    for m in self._machines:
237      if m.name not in locked_machines:
238        self._machines.remove(m)
239
240  def GetChromeVersion(self, machine):
241    """Get the version of Chrome running on the DUT."""
242
243    cmd = '/opt/google/chrome/chrome --version'
244    ret, version, _ = self.ce.CrosRunCommandWOutput(
245        cmd,
246        machine=machine.name,
247        chromeos_root=self.chromeos_root)
248    if ret != 0:
249      raise CrosCommandError("Couldn't get Chrome version from %s." %
250                             machine.name)
251
252    if ret != 0:
253      version = ''
254    return version.rstrip()
255
256  def ImageMachine(self, machine, label):
257    checksum = label.checksum
258
259    if checksum and (machine.checksum == checksum):
260      return
261    chromeos_root = label.chromeos_root
262    if not chromeos_root:
263      chromeos_root = self.chromeos_root
264    image_chromeos_args = [image_chromeos.__file__, '--no_lock',
265                           '--chromeos_root=%s' % chromeos_root,
266                           '--image=%s' % label.chromeos_image,
267                           '--image_args=%s' % label.image_args, '--remote=%s' %
268                           machine.name, '--logging_level=%s' % self.log_level]
269    if label.board:
270      image_chromeos_args.append('--board=%s' % label.board)
271
272    # Currently can't image two machines at once.
273    # So have to serialized on this lock.
274    save_ce_log_level = self.ce.log_level
275    if self.log_level != 'verbose':
276      self.ce.log_level = 'average'
277
278    with self.image_lock:
279      if self.log_level != 'verbose':
280        self.logger.LogOutput('Pushing image onto machine.')
281        self.logger.LogOutput('Running image_chromeos.DoImage with %s' %
282                              ' '.join(image_chromeos_args))
283      retval = 0
284      if not test_flag.GetTestMode():
285        retval = image_chromeos.DoImage(image_chromeos_args)
286      if retval:
287        cmd = 'reboot && exit'
288        if self.log_level != 'verbose':
289          self.logger.LogOutput('reboot & exit.')
290        self.ce.CrosRunCommand(cmd,
291                               machine=machine.name,
292                               chromeos_root=self.chromeos_root)
293        time.sleep(60)
294        if self.log_level != 'verbose':
295          self.logger.LogOutput('Pushing image onto machine.')
296          self.logger.LogOutput('Running image_chromeos.DoImage with %s' %
297                                ' '.join(image_chromeos_args))
298        retval = image_chromeos.DoImage(image_chromeos_args)
299      if retval:
300        raise RuntimeError("Could not image machine: '%s'." % machine.name)
301      else:
302        self.num_reimages += 1
303      machine.checksum = checksum
304      machine.image = label.chromeos_image
305      machine.label = label
306
307    if not label.chrome_version:
308      label.chrome_version = self.GetChromeVersion(machine)
309
310    self.ce.log_level = save_ce_log_level
311    return retval
312
313  def ComputeCommonCheckSum(self, label):
314    # Since this is used for cache lookups before the machines have been
315    # compared/verified, check here to make sure they all have the same
316    # checksum (otherwise the cache lookup may not be valid).
317    common_checksum = None
318    for machine in self.GetMachines(label):
319      # Make sure the machine's checksums are calculated.
320      if not machine.machine_checksum:
321        machine.SetUpChecksumInfo()
322      cs = machine.machine_checksum
323      # If this is the first machine we've examined, initialize
324      # common_checksum.
325      if not common_checksum:
326        common_checksum = cs
327      # Make sure this machine's checksum matches our 'common' checksum.
328      if cs != common_checksum:
329        raise BadChecksum('Machine checksums do not match!')
330    self.machine_checksum[label.name] = common_checksum
331
332  def ComputeCommonCheckSumString(self, label):
333    # The assumption is that this function is only called AFTER
334    # ComputeCommonCheckSum, so there is no need to verify the machines
335    # are the same here.  If this is ever changed, this function should be
336    # modified to verify that all the machines for a given label are the
337    # same.
338    for machine in self.GetMachines(label):
339      if machine.checksum_string:
340        self.machine_checksum_string[label.name] = machine.checksum_string
341        break
342
343  def _TryToLockMachine(self, cros_machine):
344    with self._lock:
345      assert cros_machine, "Machine can't be None"
346      for m in self._machines:
347        if m.name == cros_machine.name:
348          return
349      locked = True
350      if self.locks_dir:
351        locked = file_lock_machine.Machine(cros_machine.name,
352                                           self.locks_dir).Lock(True,
353                                                                sys.argv[0])
354      if locked:
355        self._machines.append(cros_machine)
356        command = 'cat %s' % CHECKSUM_FILE
357        ret, out, _ = self.ce.CrosRunCommandWOutput(
358            command,
359            chromeos_root=self.chromeos_root,
360            machine=cros_machine.name)
361        if ret == 0:
362          cros_machine.checksum = out.strip()
363      elif self.locks_dir:
364        self.logger.LogOutput("Couldn't lock: %s" % cros_machine.name)
365
366  # This is called from single threaded mode.
367  def AddMachine(self, machine_name):
368    with self._lock:
369      for m in self._all_machines:
370        assert m.name != machine_name, 'Tried to double-add %s' % machine_name
371
372      if self.log_level != 'verbose':
373        self.logger.LogOutput('Setting up remote access to %s' % machine_name)
374        self.logger.LogOutput('Checking machine characteristics for %s' %
375                              machine_name)
376      cm = CrosMachine(machine_name, self.chromeos_root, self.log_level)
377      if cm.machine_checksum:
378        self._all_machines.append(cm)
379
380  def RemoveMachine(self, machine_name):
381    with self._lock:
382      self._machines = [m for m in self._machines if m.name != machine_name]
383      if self.locks_dir:
384        res = file_lock_machine.Machine(machine_name,
385                                        self.locks_dir).Unlock(True)
386        if not res:
387          self.logger.LogError("Could not unlock machine: '%s'." % machine_name)
388
389  def ForceSameImageToAllMachines(self, label):
390    machines = self.GetMachines(label)
391    for m in machines:
392      self.ImageMachine(m, label)
393      m.SetUpChecksumInfo()
394
395  def AcquireMachine(self, label):
396    image_checksum = label.checksum
397    machines = self.GetMachines(label)
398    check_interval_time = 120
399    with self._lock:
400      # Lazily external lock machines
401      while self.acquire_timeout >= 0:
402        for m in machines:
403          new_machine = m not in self._all_machines
404          self._TryToLockMachine(m)
405          if new_machine:
406            m.released_time = time.time()
407        if self.GetAvailableMachines(label):
408          break
409        sleep_time = max(1, min(self.acquire_timeout, check_interval_time))
410        time.sleep(sleep_time)
411        self.acquire_timeout -= sleep_time
412
413      if self.acquire_timeout < 0:
414        self.logger.LogFatal('Could not acquire any of the '
415                             "following machines: '%s'" %
416                             ', '.join(machine.name for machine in machines))
417
418###      for m in self._machines:
419###        if (m.locked and time.time() - m.released_time < 10 and
420###            m.checksum == image_checksum):
421###          return None
422      unlocked_machines = [machine
423                           for machine in self.GetAvailableMachines(label)
424                           if not machine.locked]
425      for m in unlocked_machines:
426        if image_checksum and m.checksum == image_checksum:
427          m.locked = True
428          m.test_run = threading.current_thread()
429          return m
430      for m in unlocked_machines:
431        if not m.checksum:
432          m.locked = True
433          m.test_run = threading.current_thread()
434          return m
435      # This logic ensures that threads waiting on a machine will get a machine
436      # with a checksum equal to their image over other threads. This saves time
437      # when crosperf initially assigns the machines to threads by minimizing
438      # the number of re-images.
439      # TODO(asharif): If we centralize the thread-scheduler, we wont need this
440      # code and can implement minimal reimaging code more cleanly.
441      for m in unlocked_machines:
442        if time.time() - m.released_time > 15:
443          # The release time gap is too large, so it is probably in the start
444          # stage, we need to reset the released_time.
445          m.released_time = time.time()
446        elif time.time() - m.released_time > 8:
447          m.locked = True
448          m.test_run = threading.current_thread()
449          return m
450    return None
451
452  def GetAvailableMachines(self, label=None):
453    if not label:
454      return self._machines
455    return [m for m in self._machines if m.name in label.remote]
456
457  def GetMachines(self, label=None):
458    if not label:
459      return self._all_machines
460    return [m for m in self._all_machines if m.name in label.remote]
461
462  def ReleaseMachine(self, machine):
463    with self._lock:
464      for m in self._machines:
465        if machine.name == m.name:
466          assert m.locked, 'Tried to double-release %s' % m.name
467          m.released_time = time.time()
468          m.locked = False
469          m.status = 'Available'
470          break
471
472  def Cleanup(self):
473    with self._lock:
474      # Unlock all machines (via file lock)
475      for m in self._machines:
476        res = file_lock_machine.Machine(m.name, self.locks_dir).Unlock(True)
477
478        if not res:
479          self.logger.LogError("Could not unlock machine: '%s'." % m.name)
480
481  def __str__(self):
482    with self._lock:
483      l = ['MachineManager Status:'] + [str(m) for m in self._machines]
484      return '\n'.join(l)
485
486  def AsString(self):
487    with self._lock:
488      stringify_fmt = '%-30s %-10s %-4s %-25s %-32s'
489      header = stringify_fmt % ('Machine', 'Thread', 'Lock', 'Status',
490                                'Checksum')
491      table = [header]
492      for m in self._machines:
493        if m.test_run:
494          test_name = m.test_run.name
495          test_status = m.test_run.timeline.GetLastEvent()
496        else:
497          test_name = ''
498          test_status = ''
499
500        try:
501          machine_string = stringify_fmt % (m.name, test_name, m.locked,
502                                            test_status, m.checksum)
503        except ValueError:
504          machine_string = ''
505        table.append(machine_string)
506      return 'Machine Status:\n%s' % '\n'.join(table)
507
508  def GetAllCPUInfo(self, labels):
509    """Get cpuinfo for labels, merge them if their cpuinfo are the same."""
510    dic = collections.defaultdict(list)
511    for label in labels:
512      for machine in self._all_machines:
513        if machine.name in label.remote:
514          dic[machine.cpuinfo].append(label.name)
515          break
516    output_segs = []
517    for key, v in dic.iteritems():
518      output = ' '.join(v)
519      output += '\n-------------------\n'
520      output += key
521      output += '\n\n\n'
522      output_segs.append(output)
523    return ''.join(output_segs)
524
525  def GetAllMachines(self):
526    return self._all_machines
527
528
529class MockCrosMachine(CrosMachine):
530  """Mock cros machine class."""
531  # pylint: disable=super-init-not-called
532
533  MEMINFO_STRING = """MemTotal:        3990332 kB
534MemFree:         2608396 kB
535Buffers:          147168 kB
536Cached:           811560 kB
537SwapCached:            0 kB
538Active:           503480 kB
539Inactive:         628572 kB
540Active(anon):     174532 kB
541Inactive(anon):    88576 kB
542Active(file):     328948 kB
543Inactive(file):   539996 kB
544Unevictable:           0 kB
545Mlocked:               0 kB
546SwapTotal:       5845212 kB
547SwapFree:        5845212 kB
548Dirty:              9384 kB
549Writeback:             0 kB
550AnonPages:        173408 kB
551Mapped:           146268 kB
552Shmem:             89676 kB
553Slab:             188260 kB
554SReclaimable:     169208 kB
555SUnreclaim:        19052 kB
556KernelStack:        2032 kB
557PageTables:         7120 kB
558NFS_Unstable:          0 kB
559Bounce:                0 kB
560WritebackTmp:          0 kB
561CommitLimit:     7840376 kB
562Committed_AS:    1082032 kB
563VmallocTotal:   34359738367 kB
564VmallocUsed:      364980 kB
565VmallocChunk:   34359369407 kB
566DirectMap4k:       45824 kB
567DirectMap2M:     4096000 kB
568"""
569
570  CPUINFO_STRING = """processor: 0
571vendor_id: GenuineIntel
572cpu family: 6
573model: 42
574model name: Intel(R) Celeron(R) CPU 867 @ 1.30GHz
575stepping: 7
576microcode: 0x25
577cpu MHz: 1300.000
578cache size: 2048 KB
579physical id: 0
580siblings: 2
581core id: 0
582cpu cores: 2
583apicid: 0
584initial apicid: 0
585fpu: yes
586fpu_exception: yes
587cpuid level: 13
588wp: yes
589flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer xsave lahf_lm arat epb xsaveopt pln pts dts tpr_shadow vnmi flexpriority ept vpid
590bogomips: 2594.17
591clflush size: 64
592cache_alignment: 64
593address sizes: 36 bits physical, 48 bits virtual
594power management:
595
596processor: 1
597vendor_id: GenuineIntel
598cpu family: 6
599model: 42
600model name: Intel(R) Celeron(R) CPU 867 @ 1.30GHz
601stepping: 7
602microcode: 0x25
603cpu MHz: 1300.000
604cache size: 2048 KB
605physical id: 0
606siblings: 2
607core id: 1
608cpu cores: 2
609apicid: 2
610initial apicid: 2
611fpu: yes
612fpu_exception: yes
613cpuid level: 13
614wp: yes
615flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer xsave lahf_lm arat epb xsaveopt pln pts dts tpr_shadow vnmi flexpriority ept vpid
616bogomips: 2594.17
617clflush size: 64
618cache_alignment: 64
619address sizes: 36 bits physical, 48 bits virtual
620power management:
621"""
622
623  def __init__(self, name, chromeos_root, log_level):
624    self.name = name
625    self.image = None
626    self.checksum = None
627    self.locked = False
628    self.released_time = time.time()
629    self.test_run = None
630    self.chromeos_root = chromeos_root
631    self.checksum_string = re.sub(r'\d', '', name)
632    #In test, we assume "lumpy1", "lumpy2" are the same machine.
633    self.machine_checksum = self._GetMD5Checksum(self.checksum_string)
634    self.log_level = log_level
635    self.label = None
636    self.ce = command_executer.GetCommandExecuter(log_level=self.log_level)
637    self._GetCPUInfo()
638
639  def IsReachable(self):
640    return True
641
642  def _GetMemoryInfo(self):
643    self.meminfo = self.MEMINFO_STRING
644    self._ParseMemoryInfo()
645
646  def _GetCPUInfo(self):
647    self.cpuinfo = self.CPUINFO_STRING
648
649
650class MockMachineManager(MachineManager):
651  """Mock machine manager class."""
652
653  def __init__(self, chromeos_root, acquire_timeout, log_level, locks_dir):
654    super(MockMachineManager, self).__init__(
655        chromeos_root, acquire_timeout, log_level, locks_dir)
656
657  def _TryToLockMachine(self, cros_machine):
658    self._machines.append(cros_machine)
659    cros_machine.checksum = ''
660
661  def AddMachine(self, machine_name):
662    with self._lock:
663      for m in self._all_machines:
664        assert m.name != machine_name, 'Tried to double-add %s' % machine_name
665      cm = MockCrosMachine(machine_name, self.chromeos_root, self.log_level)
666      assert cm.machine_checksum, ('Could not find checksum for machine %s' %
667                                   machine_name)
668      # In Original MachineManager, the test is 'if cm.machine_checksum:' - if a
669      # machine is unreachable, then its machine_checksum is None. Here we
670      # cannot do this, because machine_checksum is always faked, so we directly
671      # test cm.IsReachable, which is properly mocked.
672      if cm.IsReachable():
673        self._all_machines.append(cm)
674
675  def GetChromeVersion(self, machine):
676    return 'Mock Chrome Version R50'
677
678  def AcquireMachine(self, label):
679    for machine in self._all_machines:
680      if not machine.locked:
681        machine.locked = True
682        return machine
683    return None
684
685  def ImageMachine(self, machine_name, label):
686    if machine_name or label:
687      return 0
688    return 1
689
690  def ReleaseMachine(self, machine):
691    machine.locked = False
692
693  def GetMachines(self, label=None):
694    return self._all_machines
695
696  def GetAvailableMachines(self, label=None):
697    return self._all_machines
698
699  def ForceSameImageToAllMachines(self, label=None):
700    return 0
701
702  def ComputeCommonCheckSum(self, label=None):
703    common_checksum = 12345
704    for machine in self.GetMachines(label):
705      machine.machine_checksum = common_checksum
706    self.machine_checksum[label.name] = common_checksum
707
708  def GetAllMachines(self):
709    return self._all_machines
710