1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""
6This module helps to deploy config files and shared folders from host to
7container. It reads the settings from a setting file (ssp_deploy_config), and
8deploy the config files based on the settings. The setting file has a json
9string of a list of deployment settings. For example:
10[{
11    "source": "/etc/resolv.conf",
12    "target": "/etc/resolv.conf",
13    "append": true,
14    "permission": 400
15 },
16 {
17    "source": "ssh",
18    "target": "/root/.ssh",
19    "append": false,
20    "permission": 400
21 },
22 {
23    "source": "/usr/local/autotest/results/shared",
24    "target": "/usr/local/autotest/results/shared",
25    "mount": true,
26    "readonly": false,
27    "force_create": true
28 }
29]
30
31Definition of each attribute for config files are as follows:
32source: config file in host to be copied to container.
33target: config file's location inside container.
34append: true to append the content of config file to existing file inside
35        container. If it's set to false, the existing file inside container will
36        be overwritten.
37permission: Permission to set to the config file inside container.
38
39Example:
40{
41    "source": "/etc/resolv.conf",
42    "target": "/etc/resolv.conf",
43    "append": true,
44    "permission": 400
45}
46The above example will:
471. Append the content of /etc/resolv.conf in host machine to file
48   /etc/resolv.conf inside container.
492. Copy all files in ssh to /root/.ssh in container.
503. Change all these files' permission to 400
51
52Definition of each attribute for sharing folders are as follows:
53source: a folder in host to be mounted in container.
54target: the folder's location inside container.
55mount: true to mount the source folder onto the target inside container.
56       A setting with false value of mount is invalid.
57readonly: true if the mounted folder inside container should be readonly.
58force_create: true to create the source folder if it doesn't exist.
59
60Example:
61 {
62    "source": "/usr/local/autotest/results/shared",
63    "target": "/usr/local/autotest/results/shared",
64    "mount": true,
65    "readonly": false,
66    "force_create": true
67 }
68The above example will mount folder "/usr/local/autotest/results/shared" in the
69host to path "/usr/local/autotest/results/shared" inside the container. The
70folder can be written to inside container. If the source folder doesn't exist,
71it will be created as `force_create` is set to true.
72
73The setting file (ssp_deploy_config) lives in AUTOTEST_DIR folder.
74For relative file path specified in ssp_deploy_config, AUTOTEST_DIR/containers
75is the parent folder.
76The setting file can be overridden by a shadow config, ssp_deploy_shadow_config.
77For lab servers, puppet should be used to deploy ssp_deploy_shadow_config to
78AUTOTEST_DIR and the configure files to AUTOTEST_DIR/containers.
79
80The default setting file (ssp_deploy_config) contains
81For SSP to work with none-lab servers, e.g., moblab and developer's workstation,
82the module still supports copy over files like ssh config and autotest
83shadow_config to container when AUTOTEST_DIR/containers/ssp_deploy_config is not
84presented.
85
86"""
87
88import collections
89import getpass
90import json
91import os
92import socket
93
94import common
95from autotest_lib.client.common_lib import global_config
96from autotest_lib.client.common_lib import utils
97from autotest_lib.site_utils.lxc import constants
98from autotest_lib.site_utils.lxc import utils as lxc_utils
99
100
101config = global_config.global_config
102
103# Path to ssp_deploy_config and ssp_deploy_shadow_config.
104SSP_DEPLOY_CONFIG_FILE = os.path.join(common.autotest_dir,
105                                      'ssp_deploy_config.json')
106SSP_DEPLOY_SHADOW_CONFIG_FILE = os.path.join(common.autotest_dir,
107                                             'ssp_deploy_shadow_config.json')
108# A temp folder used to store files to be appended to the files inside
109# container.
110_APPEND_FOLDER = '/usr/local/ssp_append'
111
112DeployConfig = collections.namedtuple(
113        'DeployConfig', ['source', 'target', 'append', 'permission'])
114MountConfig = collections.namedtuple(
115        'MountConfig', ['source', 'target', 'mount', 'readonly',
116                        'force_create'])
117
118
119class SSPDeployError(Exception):
120    """Exception raised if any error occurs when setting up test container."""
121
122
123class DeployConfigManager(object):
124    """An object to deploy config to container.
125
126    The manager retrieves deploy configs from ssp_deploy_config or
127    ssp_deploy_shadow_config, and sets up the container accordingly.
128    For example:
129    1. Copy given config files to specified location inside container.
130    2. Append the content of given config files to specific files inside
131       container.
132    3. Make sure the config files have proper permission inside container.
133
134    """
135
136    @staticmethod
137    def validate_path(deploy_config):
138        """Validate the source and target in deploy_config dict.
139
140        @param deploy_config: A dictionary of deploy config to be validated.
141
142        @raise SSPDeployError: If any path in deploy config is invalid.
143        """
144        target = deploy_config['target']
145        source = deploy_config['source']
146        if not os.path.isabs(target):
147            raise SSPDeployError('Target path must be absolute path: %s' %
148                                 target)
149        if not os.path.isabs(source):
150            if source.startswith('~'):
151                # This is to handle the case that the script is run with sudo.
152                inject_user_path = ('~%s%s' % (utils.get_real_user(),
153                                               source[1:]))
154                source = os.path.expanduser(inject_user_path)
155            else:
156                source = os.path.join(common.autotest_dir, source)
157            # Update the source setting in deploy config with the updated path.
158            deploy_config['source'] = source
159
160
161    @staticmethod
162    def validate(deploy_config):
163        """Validate the deploy config.
164
165        Deploy configs need to be validated and pre-processed, e.g.,
166        1. Target must be an absolute path.
167        2. Source must be updated to be an absolute path.
168
169        @param deploy_config: A dictionary of deploy config to be validated.
170
171        @return: A DeployConfig object that contains the deploy config.
172
173        @raise SSPDeployError: If the deploy config is invalid.
174
175        """
176        DeployConfigManager.validate_path(deploy_config)
177        return DeployConfig(**deploy_config)
178
179
180    @staticmethod
181    def validate_mount(deploy_config):
182        """Validate the deploy config for mounting a directory.
183
184        Deploy configs need to be validated and pre-processed, e.g.,
185        1. Target must be an absolute path.
186        2. Source must be updated to be an absolute path.
187        3. Mount must be true.
188
189        @param deploy_config: A dictionary of deploy config to be validated.
190
191        @return: A DeployConfig object that contains the deploy config.
192
193        @raise SSPDeployError: If the deploy config is invalid.
194
195        """
196        DeployConfigManager.validate_path(deploy_config)
197        c = MountConfig(**deploy_config)
198        if not c.mount:
199            raise SSPDeployError('`mount` must be true.')
200        if not c.force_create and not os.path.exists(c.source):
201            raise SSPDeployError('`source` does not exist.')
202        return c
203
204
205    def __init__(self, container, config_file=None):
206        """Initialize the deploy config manager.
207
208        @param container: The container needs to deploy config.
209        @param config_file: An optional config file.  For testing.
210        """
211        self.container = container
212        # If shadow config is used, the deployment procedure will skip some
213        # special handling of config file, e.g.,
214        # 1. Set enable_master_ssh to False in autotest shadow config.
215        # 2. Set ssh logleve to ERROR for all hosts.
216        if config_file is None:
217            self.is_shadow_config = os.path.exists(
218                    SSP_DEPLOY_SHADOW_CONFIG_FILE)
219            config_file = (
220                    SSP_DEPLOY_SHADOW_CONFIG_FILE if self.is_shadow_config
221                    else SSP_DEPLOY_CONFIG_FILE)
222        else:
223            self.is_shadow_config = False
224
225        with open(config_file) as f:
226            deploy_configs = json.load(f)
227        self.deploy_configs = [self.validate(c) for c in deploy_configs
228                               if 'append' in c]
229        self.mount_configs = [self.validate_mount(c) for c in deploy_configs
230                              if 'mount' in c]
231        tmp_append = os.path.join(self.container.rootfs,
232                                  _APPEND_FOLDER.lstrip(os.path.sep))
233        if lxc_utils.path_exists(tmp_append):
234            utils.run('sudo rm -rf "%s"' % tmp_append)
235        utils.run('sudo mkdir -p "%s"' % tmp_append)
236
237
238    def _deploy_config_pre_start(self, deploy_config):
239        """Deploy a config before container is started.
240
241        Most configs can be deployed before the container is up. For configs
242        require a reboot to take effective, they must be deployed in this
243        function.
244
245        @param deploy_config: Config to be deployed.
246        """
247        if not lxc_utils.path_exists(deploy_config.source):
248            return
249        # Path to the target file relative to host.
250        if deploy_config.append:
251            target = os.path.join(_APPEND_FOLDER,
252                                  os.path.basename(deploy_config.target))
253        else:
254            target = deploy_config.target
255
256        self.container.copy(deploy_config.source, target)
257
258
259    def _deploy_config_post_start(self, deploy_config):
260        """Deploy a config after container is started.
261
262        For configs to be appended after the existing config files in container,
263        they must be copied to a temp location before container is up (deployed
264        in function _deploy_config_pre_start). After the container is up, calls
265        can be made to append the content of such configs to existing config
266        files.
267
268        @param deploy_config: Config to be deployed.
269
270        """
271        if deploy_config.append:
272            source = os.path.join(_APPEND_FOLDER,
273                                  os.path.basename(deploy_config.target))
274            self.container.attach_run('cat \'%s\' >> \'%s\'' %
275                                      (source, deploy_config.target))
276        self.container.attach_run(
277                'chmod -R %s \'%s\'' %
278                (deploy_config.permission, deploy_config.target))
279
280
281    def _modify_shadow_config(self):
282        """Update the shadow config used in container with correct values.
283
284        This only applies when no shadow SSP deploy config is applied. For
285        default SSP deploy config, autotest shadow_config.ini is from autotest
286        directory, which requires following modification to be able to work in
287        container. If one chooses to use a shadow SSP deploy config file, the
288        autotest shadow_config.ini must be from a source with following
289        modification:
290        1. Disable master ssh connection in shadow config, as it is not working
291           properly in container yet, and produces noise in the log.
292        2. Update AUTOTEST_WEB/host and SERVER/hostname to be the IP of the host
293           if any is set to localhost or 127.0.0.1. Otherwise, set it to be the
294           FQDN of the config value.
295        3. Update SSP/user, which is used as the user makes RPC inside the
296           container. This allows the RPC to pass ACL check as if the call is
297           made in the host.
298
299        """
300        shadow_config = os.path.join(constants.CONTAINER_AUTOTEST_DIR,
301                                     'shadow_config.ini')
302
303        # Inject "AUTOSERV/enable_master_ssh: False" in shadow config as
304        # container does not support master ssh connection yet.
305        self.container.attach_run(
306                'echo $\'\n[AUTOSERV]\nenable_master_ssh: False\n\' >> %s' %
307                shadow_config)
308
309        host_ip = lxc_utils.get_host_ip()
310        local_names = ['localhost', '127.0.0.1']
311
312        db_host = config.get_config_value('AUTOTEST_WEB', 'host')
313        if db_host.lower() in local_names:
314            new_host = host_ip
315        else:
316            new_host = socket.getfqdn(db_host)
317        self.container.attach_run('echo $\'\n[AUTOTEST_WEB]\nhost: %s\n\' >> %s'
318                                  % (new_host, shadow_config))
319
320        afe_host = config.get_config_value('SERVER', 'hostname')
321        if afe_host.lower() in local_names:
322            new_host = host_ip
323        else:
324            new_host = socket.getfqdn(afe_host)
325        self.container.attach_run('echo $\'\n[SERVER]\nhostname: %s\n\' >> %s' %
326                                  (new_host, shadow_config))
327
328        # Update configurations in SSP section:
329        # user: The user running current process.
330        # is_moblab: True if the autotest server is a Moblab instance.
331        # host_container_ip: IP address of the lxcbr0 interface. Process running
332        #     inside container can make RPC through this IP.
333        self.container.attach_run(
334                'echo $\'\n[SSP]\nuser: %s\nis_moblab: %s\n'
335                'host_container_ip: %s\n\' >> %s' %
336                (getpass.getuser(), bool(utils.is_moblab()),
337                 lxc_utils.get_host_ip(), shadow_config))
338
339
340    def _modify_ssh_config(self):
341        """Modify ssh config for it to work inside container.
342
343        This is only called when default ssp_deploy_config is used. If shadow
344        deploy config is manually set up, this function will not be called.
345        Therefore, the source of ssh config must be properly updated to be able
346        to work inside container.
347
348        """
349        # Remove domain specific flags.
350        ssh_config = '/root/.ssh/config'
351        self.container.attach_run('sed -i \'s/UseProxyIf=false//g\' \'%s\'' %
352                                  ssh_config)
353        # TODO(dshi): crbug.com/451622 ssh connection loglevel is set to
354        # ERROR in container before master ssh connection works. This is
355        # to avoid logs being flooded with warning `Permanently added
356        # '[hostname]' (RSA) to the list of known hosts.` (crbug.com/478364)
357        # The sed command injects following at the beginning of .ssh/config
358        # used in config. With such change, ssh command will not post
359        # warnings.
360        # Host *
361        #   LogLevel Error
362        self.container.attach_run(
363                'sed -i \'1s/^/Host *\\n  LogLevel ERROR\\n\\n/\' \'%s\'' %
364                ssh_config)
365
366        # Inject ssh config for moblab to ssh to dut from container.
367        if utils.is_moblab():
368            # ssh to moblab itself using moblab user.
369            self.container.attach_run(
370                    'echo $\'\nHost 192.168.231.1\n  User moblab\n  '
371                    'IdentityFile %%d/.ssh/testing_rsa\' >> %s' %
372                    '/root/.ssh/config')
373            # ssh to duts using root user.
374            self.container.attach_run(
375                    'echo $\'\nHost *\n  User root\n  '
376                    'IdentityFile %%d/.ssh/testing_rsa\' >> %s' %
377                    '/root/.ssh/config')
378
379
380    def deploy_pre_start(self):
381        """Deploy configs before the container is started.
382        """
383        for deploy_config in self.deploy_configs:
384            self._deploy_config_pre_start(deploy_config)
385        for mount_config in self.mount_configs:
386            if (mount_config.force_create and
387                not os.path.exists(mount_config.source)):
388                utils.run('mkdir -p %s' % mount_config.source)
389            self.container.mount_dir(mount_config.source,
390                                     mount_config.target,
391                                     mount_config.readonly)
392
393
394    def deploy_post_start(self):
395        """Deploy configs after the container is started.
396        """
397        for deploy_config in self.deploy_configs:
398            self._deploy_config_post_start(deploy_config)
399        # Autotest shadow config requires special handling to update hostname
400        # of `localhost` with host IP. Shards always use `localhost` as value
401        # of SERVER\hostname and AUTOTEST_WEB\host.
402        self._modify_shadow_config()
403        # Only apply special treatment for files deployed by the default
404        # ssp_deploy_config
405        if not self.is_shadow_config:
406            self._modify_ssh_config()
407