• Home
  • History
  • Annotate
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 #
2 # Copyright 2007 Google Inc. Released under the GPL v2
3 
4 """
5 This module defines the SSHHost class.
6 
7 Implementation details:
8 You should import the "hosts" package instead of importing each type of host.
9 
10         SSHHost: a remote machine with a ssh access
11 """
12 
13 import inspect
14 import logging
15 import re
16 from autotest_lib.client.common_lib import error
17 from autotest_lib.client.common_lib import pxssh
18 from autotest_lib.server import utils
19 from autotest_lib.server.hosts import abstract_ssh
20 
21 # In case cros_host is being ran via SSP on an older Moblab version with an
22 # older chromite version.
23 try:
24     from chromite.lib import metrics
25 except ImportError:
26     metrics = utils.metrics_mock
27 
28 
29 class SSHHost(abstract_ssh.AbstractSSHHost):
30     """
31     This class represents a remote machine controlled through an ssh
32     session on which you can run programs.
33 
34     It is not the machine autoserv is running on. The machine must be
35     configured for password-less login, for example through public key
36     authentication.
37 
38     It includes support for controlling the machine through a serial
39     console on which you can run programs. If such a serial console is
40     set up on the machine then capabilities such as hard reset and
41     boot strap monitoring are available. If the machine does not have a
42     serial console available then ordinary SSH-based commands will
43     still be available, but attempts to use extensions such as
44     console logging or hard reset will fail silently.
45 
46     Implementation details:
47     This is a leaf class in an abstract class hierarchy, it must
48     implement the unimplemented methods in parent classes.
49     """
50 
51     def _initialize(self, hostname, *args, **dargs):
52         """
53         Construct a SSHHost object
54 
55         Args:
56                 hostname: network hostname or address of remote machine
57         """
58         super(SSHHost, self)._initialize(hostname=hostname, *args, **dargs)
59         self.setup_ssh()
60 
61 
62     def ssh_command(self, connect_timeout=30, options='', alive_interval=300):
63         """
64         Construct an ssh command with proper args for this host.
65 
66         @param connect_timeout: connection timeout (in seconds)
67         @param options: SSH options
68         @param alive_interval: SSH Alive interval.
69         """
70         options = "%s %s" % (options, self.master_ssh_option)
71         base_cmd = self.make_ssh_command(user=self.user, port=self.port,
72                                          opts=options,
73                                          hosts_file=self.known_hosts_file,
74                                          connect_timeout=connect_timeout,
75                                          alive_interval=alive_interval)
76         return "%s %s" % (base_cmd, self.hostname)
77 
78 
79     def _verbose_logger_command(self, command):
80         """
81         Prepend the command for the client with information about the ssh command
82         to be executed and the server stack state.
83 
84         @param command: the ssh command to be executed.
85         """
86         stack_frames = inspect.stack()
87         stack = ''
88         # The last 2 frames on the stack are boring. Print 5-2=3 stack frames.
89         count = min(5, len(stack_frames))
90         if count >= 3:
91             stack = inspect.getframeinfo(stack_frames[2][0]).function
92             for frame in stack_frames[3:count]:
93                 function_name = inspect.getframeinfo(frame[0]).function
94                 stack = '%s|%s' % (function_name, stack)
95         del stack_frames
96         # If "logger" executable exists on the DUT use it to respew |command|.
97         # Then regardless of "logger" run |command| as usual.
98         command = ('if type "logger" > /dev/null 2>&1; then'
99                    ' logger -tag "autotest" "server[stack::%s] -> ssh_run(%s)";'
100                    'fi; '
101                    '%s' % (stack, utils.sh_escape(command), command))
102         return command
103 
104 
105     def _run(self, command, timeout, ignore_status,
106              stdout, stderr, connect_timeout, env, options, stdin, args,
107              ignore_timeout, ssh_failure_retry_ok):
108         """Helper function for run()."""
109         ssh_cmd = self.ssh_command(connect_timeout, options)
110         if not env.strip():
111             env = ""
112         else:
113             env = "export %s;" % env
114         for arg in args:
115             command += ' "%s"' % utils.sh_escape(arg)
116         full_cmd = '%s "%s %s"' % (ssh_cmd, env, utils.sh_escape(command))
117 
118         # TODO(jrbarnette):  crbug.com/484726 - When we're in an SSP
119         # container, sometimes shortly after reboot we will see DNS
120         # resolution errors on ssh commands; the problem never
121         # occurs more than once in a row.  This especially affects
122         # the autoupdate_Rollback test, but other cases have been
123         # affected, too.
124         #
125         # We work around it by detecting the first DNS resolution error
126         # and retrying exactly one time.
127         dns_error_retry_count = 1
128 
129         def counters_inc(counter_name, failure_name):
130             """Helper function to increment metrics counters.
131             @param counter_name: string indicating which counter to use
132             @param failure_name: string indentifying an error, or 'success'
133             """
134             if counter_name == 'call':
135                 # ssh_counter records the outcome of each ssh invocation
136                 # inside _run(), including exceptions.
137                 ssh_counter = metrics.Counter('chromeos/autotest/ssh/calls')
138                 fields = {'error' : failure_name or 'success',
139                           'attempt' : ssh_call_count}
140                 ssh_counter.increment(fields=fields)
141 
142             if counter_name == 'run':
143                 # run_counter records each call to _run() with its result
144                 # and how many tries were made.  Calls are recorded when
145                 # _run() exits (including exiting with an exception)
146                 run_counter = metrics.Counter('chromeos/autotest/ssh/runs')
147                 fields = {'error' : failure_name or 'success',
148                           'attempt' : ssh_call_count}
149                 run_counter.increment(fields=fields)
150 
151         # If ssh_failure_retry_ok is True, retry twice on timeouts and generic
152         # error 255: if a simple retry doesn't work, kill the ssh master
153         # connection and try again.  (Note that either error could come from
154         # the command running in the DUT, in which case the retry may be
155         # useless but, in theory, also harmless.)
156         if ssh_failure_retry_ok:
157             # Ignore ssh command timeout, even though it could be a timeout due
158             # to the command executing in the remote host.  Note that passing
159             # ignore_timeout = True makes utils.run() return None on timeouts
160             # (and only on timeouts).
161             original_ignore_timeout = ignore_timeout
162             ignore_timeout = True
163             ssh_failure_retry_count = 2
164         else:
165             ssh_failure_retry_count = 0
166 
167         ssh_call_count = 0
168 
169         while True:
170             try:
171                 # Increment call count first, in case utils.run() throws an
172                 # exception.
173                 ssh_call_count += 1
174                 result = utils.run(full_cmd, timeout, True, stdout, stderr,
175                                    verbose=False, stdin=stdin,
176                                    stderr_is_expected=ignore_status,
177                                    ignore_timeout=ignore_timeout)
178             except Exception as e:
179                 # No retries on exception.
180                 counters_inc('call', 'exception')
181                 counters_inc('run', 'exception')
182                 raise e
183 
184             failure_name = None
185 
186             if result:
187                 if result.exit_status == 255:
188                     if re.search(r'^ssh: .*: Name or service not known',
189                                  result.stderr):
190                         failure_name = 'dns_failure'
191                     else:
192                         failure_name = 'error_255'
193                 elif result.exit_status > 0:
194                     failure_name = 'nonzero_status'
195             else:
196                 # result == None
197                 failure_name = 'timeout'
198 
199             # Record the outcome of the ssh invocation.
200             counters_inc('call', failure_name)
201 
202             if failure_name:
203                 # There was a failure: decide whether to retry.
204                 if failure_name == 'dns_failure':
205                     if dns_error_retry_count > 0:
206                         logging.debug('retrying ssh because of DNS failure')
207                         dns_error_retry_count -= 1
208                         continue
209                 else:
210                     if ssh_failure_retry_count == 2:
211                         logging.debug('retrying ssh command after %s',
212                                        failure_name)
213                         ssh_failure_retry_count -= 1
214                         continue
215                     elif ssh_failure_retry_count == 1:
216                         # After two failures, restart the master connection
217                         # before the final try.
218                         logging.debug('retry 2: restarting master connection')
219                         self.restart_master_ssh()
220                         # Last retry: reinstate timeout behavior.
221                         ignore_timeout = original_ignore_timeout
222                         ssh_failure_retry_count -= 1
223                         continue
224 
225             # No retry conditions occurred.  Exit the loop.
226             break
227 
228         # The outcomes of ssh invocations have been recorded.  Now record
229         # the outcome of this function.
230 
231         if ignore_timeout and not result:
232             counters_inc('run', 'ignored_timeout')
233             return None
234 
235         # The error messages will show up in band (indistinguishable
236         # from stuff sent through the SSH connection), so we have the
237         # remote computer echo the message "Connected." before running
238         # any command.  Since the following 2 errors have to do with
239         # connecting, it's safe to do these checks.
240         if result.exit_status == 255:
241             if re.search(r'^ssh: connect to host .* port .*: '
242                          r'Connection timed out\r$', result.stderr):
243                 counters_inc('run', 'final_timeout')
244                 raise error.AutoservSSHTimeout("ssh timed out", result)
245             if "Permission denied." in result.stderr:
246                 msg = "ssh permission denied"
247                 counters_inc('run', 'final_eperm')
248                 raise error.AutoservSshPermissionDeniedError(msg, result)
249 
250         if not ignore_status and result.exit_status > 0:
251             counters_inc('run', 'final_run_error')
252             raise error.AutoservRunError("command execution error", result)
253 
254         counters_inc('run', failure_name)
255         return result
256 
257 
258     def run(self, command, timeout=3600, ignore_status=False,
259             stdout_tee=utils.TEE_TO_LOGS, stderr_tee=utils.TEE_TO_LOGS,
260             connect_timeout=30, options='', stdin=None, verbose=True, args=(),
261             ignore_timeout=False, ssh_failure_retry_ok=False):
262         """
263         Run a command on the remote host.
264         @see common_lib.hosts.host.run()
265 
266         @param timeout: command execution timeout
267         @param connect_timeout: ssh connection timeout (in seconds)
268         @param options: string with additional ssh command options
269         @param verbose: log the commands
270         @param ignore_timeout: bool True if SSH command timeouts should be
271                 ignored.  Will return None on command timeout.
272         @param ssh_failure_retry_ok: True if the command may be retried on
273                 probable ssh failure (error 255 or timeout).  When true,
274                 the command may be executed up to three times, the second
275                 time after restarting the ssh master connection.  Use only for
276                 commands that are idempotent, because when a "probable
277                 ssh failure" occurs, we cannot tell if the command executed
278                 or not.
279 
280         @raises AutoservRunError: if the command failed
281         @raises AutoservSSHTimeout: ssh connection has timed out
282         """
283         if verbose:
284             logging.debug("Running (ssh) '%s'", command)
285             command = self._verbose_logger_command(command)
286 
287         # Start a master SSH connection if necessary.
288         self.start_master_ssh()
289 
290         env = " ".join("=".join(pair) for pair in self.env.iteritems())
291         try:
292             return self._run(command, timeout, ignore_status,
293                              stdout_tee, stderr_tee, connect_timeout, env,
294                              options, stdin, args, ignore_timeout,
295                              ssh_failure_retry_ok)
296         except error.CmdError, cmderr:
297             # We get a CmdError here only if there is timeout of that command.
298             # Catch that and stuff it into AutoservRunError and raise it.
299             timeout_message = str('Timeout encountered: %s' % cmderr.args[0])
300             raise error.AutoservRunError(timeout_message, cmderr.args[1])
301 
302 
303     def run_background(self, command, verbose=True):
304         """Start a command on the host in the background.
305 
306         The command is started on the host in the background, and
307         this method call returns immediately without waiting for the
308         command's completion.  The PID of the process on the host is
309         returned as a string.
310 
311         The command may redirect its stdin, stdout, or stderr as
312         necessary.  Without redirection, all input and output will
313         use /dev/null.
314 
315         @param command The command to run in the background
316         @param verbose As for `self.run()`
317 
318         @return Returns the PID of the remote background process
319                 as a string.
320         """
321         # Redirection here isn't merely hygienic; it's a functional
322         # requirement.  sshd won't terminate until stdin, stdout,
323         # and stderr are all closed.
324         #
325         # The subshell is needed to do the right thing in case the
326         # passed in command has its own I/O redirections.
327         cmd_fmt = '( %s ) </dev/null >/dev/null 2>&1 & echo -n $!'
328         return self.run(cmd_fmt % command, verbose=verbose).stdout
329 
330 
331     def run_short(self, command, **kwargs):
332         """
333         Calls the run() command with a short default timeout.
334 
335         Takes the same arguments as does run(),
336         with the exception of the timeout argument which
337         here is fixed at 60 seconds.
338         It returns the result of run.
339 
340         @param command: the command line string
341 
342         """
343         return self.run(command, timeout=60, **kwargs)
344 
345 
346     def run_grep(self, command, timeout=30, ignore_status=False,
347                  stdout_ok_regexp=None, stdout_err_regexp=None,
348                  stderr_ok_regexp=None, stderr_err_regexp=None,
349                  connect_timeout=30):
350         """
351         Run a command on the remote host and look for regexp
352         in stdout or stderr to determine if the command was
353         successul or not.
354 
355 
356         @param command: the command line string
357         @param timeout: time limit in seconds before attempting to
358                         kill the running process. The run() function
359                         will take a few seconds longer than 'timeout'
360                         to complete if it has to kill the process.
361         @param ignore_status: do not raise an exception, no matter
362                               what the exit code of the command is.
363         @param stdout_ok_regexp: regexp that should be in stdout
364                                  if the command was successul.
365         @param stdout_err_regexp: regexp that should be in stdout
366                                   if the command failed.
367         @param stderr_ok_regexp: regexp that should be in stderr
368                                  if the command was successul.
369         @param stderr_err_regexp: regexp that should be in stderr
370                                  if the command failed.
371         @param connect_timeout: connection timeout (in seconds)
372 
373         Returns:
374                 if the command was successul, raises an exception
375                 otherwise.
376 
377         Raises:
378                 AutoservRunError:
379                 - the exit code of the command execution was not 0.
380                 - If stderr_err_regexp is found in stderr,
381                 - If stdout_err_regexp is found in stdout,
382                 - If stderr_ok_regexp is not found in stderr.
383                 - If stdout_ok_regexp is not found in stdout,
384         """
385 
386         # We ignore the status, because we will handle it at the end.
387         result = self.run(command, timeout, ignore_status=True,
388                           connect_timeout=connect_timeout)
389 
390         # Look for the patterns, in order
391         for (regexp, stream) in ((stderr_err_regexp, result.stderr),
392                                  (stdout_err_regexp, result.stdout)):
393             if regexp and stream:
394                 err_re = re.compile (regexp)
395                 if err_re.search(stream):
396                     raise error.AutoservRunError(
397                         '%s failed, found error pattern: "%s"' % (command,
398                                                                 regexp), result)
399 
400         for (regexp, stream) in ((stderr_ok_regexp, result.stderr),
401                                  (stdout_ok_regexp, result.stdout)):
402             if regexp and stream:
403                 ok_re = re.compile (regexp)
404                 if ok_re.search(stream):
405                     if ok_re.search(stream):
406                         return
407 
408         if not ignore_status and result.exit_status > 0:
409             raise error.AutoservRunError("command execution error", result)
410 
411 
412     def setup_ssh_key(self):
413         """Setup SSH Key"""
414         logging.debug('Performing SSH key setup on %s:%d as %s.',
415                       self.hostname, self.port, self.user)
416 
417         try:
418             host = pxssh.pxssh()
419             host.login(self.hostname, self.user, self.password,
420                         port=self.port)
421             public_key = utils.get_public_key()
422 
423             host.sendline('mkdir -p ~/.ssh')
424             host.prompt()
425             host.sendline('chmod 700 ~/.ssh')
426             host.prompt()
427             host.sendline("echo '%s' >> ~/.ssh/authorized_keys; " %
428                             public_key)
429             host.prompt()
430             host.sendline('chmod 600 ~/.ssh/authorized_keys')
431             host.prompt()
432             host.logout()
433 
434             logging.debug('SSH key setup complete.')
435 
436         except:
437             logging.debug('SSH key setup has failed.')
438             try:
439                 host.logout()
440             except:
441                 pass
442 
443 
444     def setup_ssh(self):
445         """Setup SSH"""
446         if self.password:
447             try:
448                 self.ssh_ping()
449             except error.AutoservSshPingHostError:
450                 self.setup_ssh_key()
451