1# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import logging
6import os
7import re
8import time
9
10import common
11from autotest_lib.client.common_lib import error, global_config
12from autotest_lib.client.common_lib.cros import retry
13from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
14from autotest_lib.server.hosts import cros_host
15from autotest_lib.server.hosts import cros_repair
16
17
18AUTOTEST_INSTALL_DIR = global_config.global_config.get_config_value(
19        'SCHEDULER', 'drone_installation_directory')
20
21ENABLE_SSH_TUNNEL_FOR_MOBLAB = global_config.global_config.get_config_value(
22        'CROS', 'enable_ssh_tunnel_for_moblab', type=bool, default=False)
23
24#'/usr/local/autotest'
25SHADOW_CONFIG_PATH = '%s/shadow_config.ini' % AUTOTEST_INSTALL_DIR
26ATEST_PATH = '%s/cli/atest' % AUTOTEST_INSTALL_DIR
27SUBNET_DUT_SEARCH_RE = (
28        r'/?.*\((?P<ip>192.168.231.*)\) at '
29        '(?P<mac>[0-9a-fA-F][0-9a-fA-F]:){5}([0-9a-fA-F][0-9a-fA-F])')
30MOBLAB_IMAGE_STORAGE = '/mnt/moblab/static'
31MOBLAB_HOME = '/home/moblab'
32MOBLAB_BOTO_LOCATION = '%s/.boto' % MOBLAB_HOME
33MOBLAB_LAUNCH_CONTROL_KEY_LOCATION = '%s/.launch_control_key' % MOBLAB_HOME
34MOBLAB_SERVICE_ACCOUNT_LOCATION = '%s/.service_account.json' % MOBLAB_HOME
35MOBLAB_AUTODIR = '/usr/local/autodir'
36DHCPD_LEASE_FILE = '/var/lib/dhcp/dhcpd.leases'
37MOBLAB_SERVICES = ['moblab-scheduler-init',
38                   'moblab-database-init',
39                   'moblab-devserver-init',
40                   'moblab-gsoffloader-init',
41                   'moblab-gsoffloader_s-init']
42MOBLAB_PROCESSES = ['apache2', 'dhcpd']
43DUT_VERIFY_SLEEP_SECS = 5
44DUT_VERIFY_TIMEOUT = 15 * 60
45MOBLAB_TMP_DIR = '/mnt/moblab/tmp'
46MOBLAB_PORT = 80
47
48
49class MoblabHost(cros_host.CrosHost):
50    """Moblab specific host class."""
51
52
53    def _initialize_frontend_rpcs(self, timeout_min):
54        """Initialize frontends for AFE and TKO for a moblab host.
55
56        AFE and TKO are initialized differently based on |_use_tunnel|,
57        which indicates that whether to use ssh tunnel to connect to moblab.
58
59        @param timeout_min: The timeout minuties for AFE services.
60        """
61        if self._use_tunnel:
62            self.web_address = self.rpc_server_tracker.tunnel_connect(
63                    MOBLAB_PORT)
64        # Pass timeout_min to self.afe
65        self.afe = frontend_wrappers.RetryingAFE(timeout_min=timeout_min,
66                                                 user='moblab',
67                                                 server=self.web_address)
68        # Use default timeout_min of MoblabHost for self.tko
69        self.tko = frontend_wrappers.RetryingTKO(timeout_min=self.timeout_min,
70                                                 user='moblab',
71                                                 server=self.web_address)
72
73
74    def _initialize(self, *args, **dargs):
75        super(MoblabHost, self)._initialize(*args, **dargs)
76        # TODO(jrbarnette):  Our superclass already initialized
77        # _repair_strategy, and now we're re-initializing it here.
78        # That's awkward, if not actually wrong.
79        self._repair_strategy = cros_repair.create_moblab_repair_strategy()
80
81        # Clear the Moblab Image Storage so that staging an image is properly
82        # tested.
83        if dargs.get('retain_image_storage') is not True:
84            self.run('rm -rf %s/*' % MOBLAB_IMAGE_STORAGE)
85        self.web_address = dargs.get('web_address', self.hostname)
86        self._use_tunnel = (ENABLE_SSH_TUNNEL_FOR_MOBLAB and
87                            self.web_address == self.hostname)
88        self.timeout_min = dargs.get('rpc_timeout_min', 1)
89        self._initialize_frontend_rpcs(self.timeout_min)
90
91
92    @staticmethod
93    def check_host(host, timeout=10):
94        """
95        Check if the given host is an moblab host.
96
97        @param host: An ssh host representing a device.
98        @param timeout: The timeout for the run command.
99
100
101        @return: True if the host device has adb.
102
103        @raises AutoservRunError: If the command failed.
104        @raises AutoservSSHTimeout: Ssh connection has timed out.
105        """
106        try:
107            result = host.run(
108                    'grep -q moblab /etc/lsb-release && '
109                    '! test -f /mnt/stateful_partition/.android_tester',
110                    ignore_status=True, timeout=timeout)
111        except (error.AutoservRunError, error.AutoservSSHTimeout):
112            return False
113        return result.exit_status == 0
114
115
116    def install_boto_file(self, boto_path=''):
117        """Install a boto file on the Moblab device.
118
119        @param boto_path: Path to the boto file to install. If None, sends the
120                          boto file in the current HOME directory.
121
122        @raises error.TestError if the boto file does not exist.
123        """
124        if not boto_path:
125            boto_path = os.path.join(os.getenv('HOME'), '.boto')
126        if not os.path.exists(boto_path):
127            raise error.TestError('Boto File:%s does not exist.' % boto_path)
128        self.send_file(boto_path, MOBLAB_BOTO_LOCATION)
129        self.run('chown moblab:moblab %s' % MOBLAB_BOTO_LOCATION)
130
131
132    def get_autodir(self):
133        """Return the directory to install autotest for client side tests."""
134        return self.autodir or MOBLAB_AUTODIR
135
136
137    def run_as_moblab(self, command, **kwargs):
138        """Moblab commands should be ran as the moblab user not root.
139
140        @param command: Command to run as user moblab.
141        """
142        command = "su - moblab -c '%s'" % command
143        return self.run(command, **kwargs)
144
145
146    def reboot(self, **dargs):
147        """Reboot the Moblab Host and wait for its services to restart."""
148        super(MoblabHost, self).reboot(**dargs)
149        # In general after a reboot, we want to wait till the web frontend
150        # and other Autotest services are up before executing. However should
151        # something be wrong with these services, repair needs to be able
152        # to continue and reimage the device.
153        try:
154            self.wait_afe_up()
155        except Exception as e:
156            logging.error('DUT has rebooted but AFE has failed to load.: %s',
157                          e)
158
159
160    def wait_afe_up(self, timeout_min=5):
161        """Wait till the AFE is up and loaded.
162
163        Attempt to reach the Moblab's AFE and database through its RPC
164        interface.
165
166        @param timeout_min: Minutes to wait for the AFE to respond. Default is
167                            5 minutes.
168
169        @raises urllib2.HTTPError if AFE does not respond within the timeout.
170        """
171        # Use moblabhost's own AFE object with a longer timeout to wait for the
172        # AFE to load. Also re-create the ssh tunnel for connections to moblab.
173        # Set the timeout_min to be longer than self.timeout_min for rebooting.
174        self._initialize_frontend_rpcs(timeout_min)
175        # Verify the AFE can handle a simple request.
176        self._check_afe()
177        # Reset the timeout_min after rebooting checks for afe services.
178        self.afe.set_timeout(self.timeout_min)
179
180
181    def _wake_devices(self):
182        """Search the subnet and attempt to ping any available duts.
183
184        Fills up the arp table with entries about devices on the subnet.
185
186        Either uses fping or directly pings devices listed in the dhcpd lease
187        file.
188        """
189        fping_result = self.run('fping -g 192.168.231.100 192.168.231.120',
190                                ignore_status=True)
191        # If fping is not on the system, ping entries in the dhcpd lease file.
192        if fping_result.exit_status == 127:
193            leases = set(self.run('grep ^lease %s' % DHCPD_LEASE_FILE,
194                                  ignore_status=True).stdout.splitlines())
195            for lease in leases:
196                ip = re.match('lease (?P<ip>.*) {', lease).groups('ip')
197                self.run('ping %s -w 1' % ip, ignore_status=True)
198
199
200    def add_dut(self, hostname):
201        """Add a DUT hostname to the AFE.
202
203        @param hostname: DUT hostname to add.
204        """
205        result = self.run_as_moblab('%s host create %s' % (ATEST_PATH,
206                                                           hostname))
207        logging.debug('atest host create output for host %s:\n%s',
208                      hostname, result.stdout)
209
210
211    def find_and_add_duts(self):
212        """Discover DUTs on the testing subnet and add them to the AFE.
213
214        Runs 'arp -a' on the Moblab host and parses the output to discover DUTs
215        and if they are not already in the AFE, adds them.
216        """
217        self._wake_devices()
218        existing_hosts = [host.hostname for host in self.afe.get_hosts()]
219        arp_command = self.run('arp -a')
220        for line in arp_command.stdout.splitlines():
221            match = re.match(SUBNET_DUT_SEARCH_RE, line)
222            if match:
223                dut_hostname = match.group('ip')
224                if dut_hostname in existing_hosts:
225                    break
226                self.add_dut(dut_hostname)
227
228
229    def verify_software(self):
230        """Verify working software on a Chrome OS system.
231
232        Tests for the following conditions:
233         1. All conditions tested by the parent version of this
234            function.
235         2. Ensures that Moblab services are running.
236         3. Ensures that both DUTs successfully run Verify.
237
238        """
239        # In case cleanup or powerwash wiped the autodir, create an empty
240        # directory.
241        self.run('mkdir -p %s' % MOBLAB_AUTODIR)
242        super(MoblabHost, self).verify_software()
243        self._verify_moblab_services()
244        self._verify_duts()
245
246
247    @retry.retry(error.AutoservError, timeout_min=2, delay_sec=10)
248    def _verify_upstart_service(self, service):
249        """Retry to verify the required moblab services are up and running.
250
251        Regarding crbug.com/649811, moblab services takes longer to restart
252        under the new provision framework. This is a fix to retry the service
253        check until all services are successfully restarted.
254
255        @param service: the moblab upstart service.
256
257        @return True if this service is started and running, otherwise False.
258        """
259        return self.upstart_status(service)
260
261
262    def _verify_moblab_services(self):
263        """Verify the required Moblab services are up and running.
264
265        @raises AutoservError if any moblab service is not running.
266        """
267        for service in MOBLAB_SERVICES:
268            if not self._verify_upstart_service(service):
269                raise error.AutoservError('Moblab service: %s is not running.'
270                                          % service)
271        for process in MOBLAB_PROCESSES:
272            try:
273                self.run('pgrep %s' % process)
274            except error.AutoservRunError:
275                raise error.AutoservError('Moblab process: %s is not running.'
276                                          % process)
277
278
279    def _check_afe(self):
280        """Verify whether afe of moblab works before verify its DUTs.
281
282        Verifying moblab sometimes happens after a successful provision, in
283        which case moblab is restarted but tunnel of afe is not re-connected.
284        This func is used to check whether afe is working now.
285
286        @return True if afe works, otherwise, raise urllib2.HTTPError.
287        """
288        try:
289            self.afe.get_hosts()
290        except:
291            logging.debug('AFE is not responding')
292            raise
293
294        return True
295
296
297    def _verify_duts(self):
298        """Verify the Moblab DUTs are up and running.
299
300        @raises AutoservError if no DUTs are in the Ready State.
301        """
302        # Check whether afe is well connected, if not, restart it.
303        try:
304            self._check_afe()
305        except:
306            self.wait_afe_up()
307
308        # Add the DUTs if they have not yet been added.
309        self.find_and_add_duts()
310        # Ensure a boto file is installed in case this Moblab was wiped in
311        # repair.
312        self.install_boto_file()
313        hosts = self.afe.reverify_hosts()
314        logging.debug('DUTs scheduled for reverification: %s', hosts)
315        # Wait till all pending special tasks are completed.
316        total_time = 0
317        while (self.afe.get_special_tasks(is_complete=False) and
318               total_time < DUT_VERIFY_TIMEOUT):
319            total_time = total_time + DUT_VERIFY_SLEEP_SECS
320            time.sleep(DUT_VERIFY_SLEEP_SECS)
321        if not self.afe.get_hosts(status='Ready'):
322            for host in self.afe.get_hosts():
323                logging.error('DUT: %s Status: %s', host, host.status)
324            raise error.AutoservError('Moblab has 0 Ready DUTs')
325
326
327    def get_platform(self):
328        """Determine the correct platform label for this host.
329
330        For Moblab devices '_moblab' is appended.
331
332        @returns a string representing this host's platform.
333        """
334        return super(MoblabHost, self).get_platform() + '_moblab'
335
336
337    def make_tmp_dir(self, base=MOBLAB_TMP_DIR):
338        """Creates a temporary directory.
339
340        @param base: The directory where it should be created.
341
342        @return Path to a newly created temporary directory.
343        """
344        self.run('mkdir -p %s' % base)
345        return self.run('mktemp -d -p %s' % base).stdout.strip()
346
347
348    def get_os_type(self):
349        return 'moblab'
350