1#!/usr/bin/env python3
2"""Run Trusty under QEMU in different configurations"""
3import enum
4import errno
5import fcntl
6import json
7import os
8from textwrap import dedent
9import re
10import select
11import socket
12import subprocess
13import shutil
14import sys
15import tempfile
16import time
17import threading
18
19from typing import Optional, List
20
21import qemu_options
22from qemu_error import AdbFailure, ConfigError, RunnerGenericError, Timeout
23
24
25# ADB expects its first console on 5554, and control on 5555
26ADB_BASE_PORT = 5554
27
28
29def find_android_build_dir(android):
30    if os.path.exists(f"{android}/target/product/trusty"):
31        return android
32    if os.path.exists(f"{android}/out/target/product/trusty"):
33        return f"{android}/out"
34
35    print(f"{android} not an Android source or build directory")
36    sys.exit(1)
37
38
39class Config(object):
40    """Stores a QEMU configuration for use with the runner
41
42    Attributes:
43        android:          Path to a built Android tree or prebuilt.
44        linux:            Path to a built Linux kernel tree or prebuilt.
45        linux_arch:       Architecture of Linux kernel.
46        atf:              Path to the ATF build to use.
47        qemu:             Path to the emulator to use.
48        arch:             Architecture definition.
49        rpmbd:            Path to the rpmb daemon to use.
50        extra_qemu_flags: Extra flags to pass to QEMU.
51    Setting android or linux to None will result in a QEMU which starts
52    without those components.
53    """
54
55    def __init__(self, config=None):
56        """Qemu Configuration
57
58        If config is passed in, it should be a file containing a json
59        specification fields described in the docs.
60        Unspecified fields will be defaulted.
61
62        If you do not pass in a config, you will almost always need to
63        override these values; the default is not especially useful.
64        """
65        config_dict = {}
66        if config:
67            config_dict = json.load(config)
68
69        self.script_dir = os.path.dirname(os.path.realpath(__file__))
70
71        def abspath(config_key, default_value=None):
72            if config_value := config_dict.get(config_key, default_value):
73                return os.path.join(self.script_dir, config_value)
74            return None
75
76        if android_path := abspath("android"):
77            self.android = find_android_build_dir(android_path)
78        else:
79            self.android = None
80        self.linux = abspath("linux")
81        self.linux_arch = config_dict.get("linux_arch")
82        self.atf = abspath("atf")
83        self.qemu = abspath("qemu", "qemu-system-aarch64")
84        self.rpmbd = abspath("rpmbd")
85        self.arch = config_dict.get("arch")
86        self.extra_qemu_flags = config_dict.get("extra_qemu_flags", [])
87
88    def check_config(self, interactive: bool, boot_tests=(),
89                     android_tests=()):
90        """Checks the runner/qemu config to make sure they are compatible"""
91        # If we have any android tests, we need a linux dir and android dir
92        if android_tests:
93            if not self.linux:
94                raise ConfigError("Need Linux to run android tests")
95            if not self.android:
96                raise ConfigError("Need Android to run android tests")
97
98        # For now, we can't run boot tests and android tests at the same time,
99        # because test-runner reports its exit code by terminating the
100        # emulator.
101        if android_tests:
102            if boot_tests:
103                raise ConfigError("Cannot run Android tests and boot"
104                                  " tests from same runner")
105
106        # Since boot test utilizes virtio serial console port for communication
107        # between QEMU guest and current process, it is not compatible with
108        # interactive mode.
109        if boot_tests:
110            if interactive:
111                raise ConfigError("Cannot run boot tests interactively")
112
113        if self.android:
114            if not self.linux:
115                raise ConfigError("Cannot run Android without Linux")
116
117
118def alloc_ports():
119    """Allocates 2 sequential ports above 5554 for adb"""
120    # adb uses ports in pairs
121    port_width = 2
122
123    # We can't actually reserve ports atomically for QEMU, but we can at
124    # least scan and find two that are not currently in use.
125    min_port = ADB_BASE_PORT
126    while True:
127        alloced_ports = []
128        for port in range(min_port, min_port + port_width):
129            # If the port is already in use, don't hand it out
130            try:
131                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
132                sock.connect(("localhost", port))
133                break
134            except IOError:
135                alloced_ports += [port]
136        if len(alloced_ports) == port_width:
137            return alloced_ports
138
139        # We could increment by only 1, but if we are competing with other
140        # adb sessions for ports, this will be more polite
141        min_port += port_width
142
143
144def forward_ports(ports):
145    """Generates arguments to forward ports in QEMU on a virtio network"""
146    forwards = ""
147    remap_port = ADB_BASE_PORT
148    for port in ports:
149        forwards += f",hostfwd=tcp::{port}-:{remap_port}"
150        remap_port = remap_port + 1
151    return [
152        "-device", "virtio-net,netdev=adbnet0", "-netdev",
153        "user,id=adbnet0" + forwards
154    ]
155
156
157class QEMUCommandPipe(object):
158    """Communicate with QEMU."""
159
160    def __init__(self):
161        """Produces pipes for talking to QEMU and args to enable them."""
162        self.command_dir = tempfile.mkdtemp()
163        os.mkfifo(f"{self.command_dir}/com.in")
164        os.mkfifo(f"{self.command_dir}/com.out")
165        self.command_args = [
166            "-chardev",
167            f"pipe,id=command0,path={self.command_dir}/com", "-mon",
168            "chardev=command0,mode=control"
169        ]
170        self.com_pipe_in = None
171        self.com_pipe_out = None
172
173    def open(self):
174        # pylint: disable=consider-using-with
175        self.com_pipe_in = open(f"{self.command_dir}/com.in", "w",
176                                encoding="utf-8")
177        self.com_pipe_out = open(f"{self.command_dir}/com.out", "r",
178                                 encoding="utf-8")
179        self.qmp_command({"execute": "qmp_capabilities"})
180
181    def close(self):
182        """Close and clean up command pipes."""
183
184        def try_close(pipe):
185            try:
186                pipe.close()
187            except IOError as e:
188                print("close error ignored", e)
189
190        try_close(self.com_pipe_in)
191        try_close(self.com_pipe_out)
192
193        # Onerror callback function to handle errors when we try to remove
194        # command pipe directory, since we sleep one second if QEMU doesn't
195        # die immediately, command pipe directory might has been removed
196        # already during sleep period.
197        def cb_handle_error(func, path, exc_info):
198            if not os.access(path, os.F_OK):
199                # Command pipe directory already removed, this case is
200                # expected, pass this case.
201                pass
202            else:
203                raise RunnerGenericError("Failed to clean up command pipe.")
204
205        # Clean up our command pipe
206        shutil.rmtree(self.command_dir, onerror=cb_handle_error)
207
208    def qmp_command(self, qmp_command):
209        """Send a qmp command and return result."""
210
211        try:
212            json.dump(qmp_command, self.com_pipe_in)
213            self.com_pipe_in.flush()
214            for line in iter(self.com_pipe_out.readline, ""):
215                res = json.loads(line)
216
217                if err := res.get("error"):
218                    sys.stderr.write(f"Command {qmp_command} failed: {err}\n")
219                    return res
220
221                if "return" in res:
222                    return res
223
224                if "QMP" not in res and "event" not in res:
225                    # Print unexpected extra lines
226                    sys.stderr.write("ignored:" + line)
227        except IOError as e:
228            print("qmp_command error ignored", e)
229
230        return None
231
232    def qmp_execute(self, execute, arguments=None):
233        """Send a qmp execute command and return result."""
234        cmp_command = {"execute": execute}
235        if arguments:
236            cmp_command["arguments"] = arguments
237        return self.qmp_command(cmp_command)
238
239    def monitor_command(self, monitor_command):
240        """Send a monitor command and write result to stderr."""
241
242        res = self.qmp_execute("human-monitor-command",
243                               {"command-line": monitor_command})
244        if res and "return" in res:
245            sys.stderr.write(res["return"])
246
247
248def qemu_handle_error(command_pipe, debug_on_error):
249    """Dump registers and/or wait for debugger."""
250
251    sys.stdout.flush()
252
253    sys.stderr.write("QEMU register dump:\n")
254    command_pipe.monitor_command("info registers -a")
255    sys.stderr.write("\n")
256
257    if debug_on_error:
258        command_pipe.monitor_command("gdbserver")
259        print("Connect gdb, press enter when done ")
260        select.select([sys.stdin], [], [])
261        input("\n")  # pylint: disable=bad-builtin
262
263
264def qemu_exit(command_pipe, qemu_proc, has_error, debug_on_error):
265    """Ensures QEMU is terminated"""
266    unclean_exit = False
267
268    if command_pipe:
269        # Ask QEMU to quit
270        if qemu_proc and (qemu_proc.poll() is None):
271            try:
272                if has_error:
273                    qemu_handle_error(command_pipe=command_pipe,
274                                      debug_on_error=debug_on_error)
275                command_pipe.qmp_execute("quit")
276            except OSError:
277                pass
278
279            # If it doesn't die immediately, wait a second
280            if qemu_proc.poll() is None:
281                time.sleep(1)
282                # If it's still not dead, take it out
283                if qemu_proc.poll() is None:
284                    qemu_proc.kill()
285                    print("QEMU refused quit")
286                    unclean_exit = True
287            qemu_proc.wait()
288
289        command_pipe.close()
290
291    elif qemu_proc and (qemu_proc.poll() is None):
292        # This was an interactive run or a boot test
293        # QEMU should not be running at this point
294        print("QEMU still running with no command channel")
295        qemu_proc.kill()
296        qemu_proc.wait()
297        unclean_exit = True
298    return unclean_exit
299
300class RunnerSession:
301    """Hold shared state between runner launch and shutdown."""
302
303    def __init__(self):
304        self.has_error = False
305        self.command_pipe = None
306        self.qemu_proc = None
307        self.ports = None
308        # stores the arguments used to start qemu iff performing a boot test
309        self.args = []
310        self.temp_files = []
311
312    def get_qemu_arg_temp_file(self):
313        """Returns a temp file that will be deleted after qemu exits."""
314        tmp = tempfile.NamedTemporaryFile(delete=False)  # pylint: disable=consider-using-with
315        self.temp_files.append(tmp.name)
316        return tmp
317
318
319class RunnerState(enum.Enum):
320    OFF = 0
321    BOOTLOADER = 1
322    ANDROID = 2
323
324
325class Runner(object):
326    """Executes tests in QEMU"""
327
328    def __init__(self,
329                 config,
330                 interactive=False,
331                 verbose=False,
332                 rpmb=True,
333                 debug=False,
334                 debug_on_error=False):
335        """Initializes the runner with provided settings.
336
337        See .run() for the meanings of these.
338        """
339        self.config = config
340        self.interactive = interactive
341        self.debug = debug
342        self.verbose = verbose
343        self.adb_transport = None
344        self.use_rpmb = rpmb
345        self.rpmb_proc = None
346        self.rpmb_sock_dir = None
347        self.msg_sock = None
348        self.msg_sock_conn = None
349        self.msg_sock_dir = None
350        self.debug_on_error = debug_on_error
351        self.dump_stdout_on_error = False
352        self.qemu_arch_options = None
353        self.default_timeout = 60 * 10  # 10 Minutes
354        self.session: Optional[RunnerSession] = None
355        self.state = RunnerState.OFF
356
357        # If we're not verbose or interactive, squelch command output
358        if verbose or self.interactive:
359            self.stdout = None
360            self.stderr = None
361        else:
362            self.stdout = tempfile.TemporaryFile()  # pylint: disable=consider-using-with
363            self.stderr = subprocess.STDOUT
364            self.dump_stdout_on_error = True
365
366        # If we're interactive connect stdin to the user
367        if self.interactive:
368            self.stdin = None
369        else:
370            self.stdin = subprocess.DEVNULL
371
372        if self.config.arch in ("arm64", "arm"):
373            self.qemu_arch_options = qemu_options.QemuArm64Options(self.config)
374        elif self.config.arch == "x86_64":
375            # pylint: disable=no-member
376            self.qemu_arch_options = qemu_options.QemuX86_64Options(self.config)
377        else:
378            raise ConfigError("Architecture unspecified or unsupported!")
379
380    def error_dump_output(self):
381        if self.dump_stdout_on_error:
382            sys.stdout.flush()
383            sys.stderr.write("System log:\n")
384            self.stdout.seek(0)
385            sys.stderr.buffer.write(self.stdout.read())
386
387    def get_qemu_arg_temp_file(self):
388        """Returns a temp file that will be deleted after qemu exits."""
389        # pylint: disable=consider-using-with
390        tmp = tempfile.NamedTemporaryFile(delete=False)
391        self.session.temp_files.append(tmp.name)
392        return tmp
393
394    def rpmb_up(self):
395        """Brings up the rpmb daemon, returning QEMU args to connect"""
396        rpmb_data = self.qemu_arch_options.rpmb_data_path()
397
398        self.rpmb_sock_dir = tempfile.mkdtemp()
399        rpmb_sock = f"{self.rpmb_sock_dir}/rpmb"
400        # pylint: disable=consider-using-with
401        rpmb_proc = subprocess.Popen([self.config.rpmbd,
402                                      "-d", rpmb_data,
403                                      "--sock", rpmb_sock])
404        self.rpmb_proc = rpmb_proc
405
406        # Wait for RPMB socket to appear to avoid a race with QEMU
407        test_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
408        tries = 0
409        max_tries = 10
410        while True:
411            tries += 1
412            try:
413                test_sock.connect(rpmb_sock)
414                break
415            except socket.error as exn:
416                if tries >= max_tries:
417                    raise exn
418                time.sleep(1)
419
420        return self.qemu_arch_options.rpmb_options(rpmb_sock)
421
422    def rpmb_down(self):
423        """Kills the running rpmb daemon, cleaning up its socket directory"""
424        if self.rpmb_proc:
425            self.rpmb_proc.kill()
426            self.rpmb_proc = None
427        if self.rpmb_sock_dir:
428            shutil.rmtree(self.rpmb_sock_dir)
429            self.rpmb_sock_dir = None
430
431    def msg_channel_up(self):
432        """Create message channel between host and QEMU guest
433
434        Virtual serial console port 'testrunner0' is introduced as socket
435        communication channel for QEMU guest and current process. Testrunner
436        enumerates this port, reads test case which to be executed from
437        testrunner0 port, sends output log message and test result to
438        testrunner0 port.
439        """
440
441        self.msg_sock_dir = tempfile.mkdtemp()
442        msg_sock_file = f"{self.msg_sock_dir}/msg"
443        self.msg_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
444        self.msg_sock.bind(msg_sock_file)
445
446        # Listen on message socket
447        self.msg_sock.listen(1)
448
449        return ["-device",
450                "virtserialport,chardev=testrunner0,name=testrunner0",
451                "-chardev", f"socket,id=testrunner0,path={msg_sock_file}"]
452
453    def msg_channel_down(self):
454        if self.msg_sock_conn:
455            self.msg_sock_conn.close()
456            self.msg_sock_conn = None
457        if self.msg_sock_dir:
458            shutil.rmtree(self.msg_sock_dir)
459            self.msg_sock_dir = None
460
461    def msg_channel_wait_for_connection(self):
462        """wait for testrunner to connect."""
463
464        # Accept testrunner's connection request
465        self.msg_sock_conn, _ = self.msg_sock.accept()
466
467    def msg_channel_send_msg(self, msg):
468        """Send message to testrunner via testrunner0 port
469
470        Testrunner tries to connect port while message with following format
471        "boottest your.port.here". Currently, we utilize this format to execute
472        cases in boot test.
473        If message does not comply above format, testrunner starts to launch
474        secondary OS.
475
476        """
477        if self.msg_sock_conn:
478            self.msg_sock_conn.send(msg.encode())
479        else:
480            sys.stderr.write("Connection has not been established yet!")
481
482    def msg_channel_recv(self):
483        if self.msg_sock_conn:
484            return self.msg_sock_conn.recv(64)
485
486        # error cases: channel not yet initialized or channel torn down
487        return bytes()
488
489    def msg_channel_close(self):
490        if self.msg_sock_conn:
491            self.msg_sock_conn.close()
492
493    def boottest_run(self, boot_tests, timeout=(60 * 2)):
494        """Run boot test cases"""
495        args = self.session.args
496        has_error = False
497        result = 2
498
499        if self.debug:
500            warning = """\
501                      Warning: Test selection does not work when --debug is set.
502                      To run a test in test runner, run in GDB:
503
504                      target remote :1234
505                      break host_get_cmdline
506                      c
507                      next 6
508                      set cmdline="boottest your.port.here"
509                      set cmdline_len=sizeof("boottest your.port.here")-1
510                      c
511                      """
512            print(dedent(warning))
513
514        if self.interactive:
515            args = ["-serial", "mon:stdio"] + args
516        elif self.verbose:
517            # This still leaves stdin connected, but doesn't connect a monitor
518            args = ["-serial", "stdio", "-monitor", "none"] + args
519        else:
520            # Silence debugging output
521            args = ["-serial", "null", "-monitor", "none"] + args
522
523        # Create command channel which used to quit QEMU after case execution
524        command_pipe = QEMUCommandPipe()
525        args += command_pipe.command_args
526        cmd = [self.config.qemu] + args
527
528        # pylint: disable=consider-using-with
529        qemu_proc = subprocess.Popen(cmd, cwd=self.config.atf)
530
531        command_pipe.open()
532        self.msg_channel_wait_for_connection()
533
534        def kill_testrunner():
535            self.msg_channel_down()
536            qemu_exit(command_pipe, qemu_proc, has_error=True,
537                      debug_on_error=self.debug_on_error)
538            raise Timeout("Wait for boottest to complete", timeout)
539
540        kill_timer = threading.Timer(timeout, kill_testrunner)
541        if not self.debug:
542            kill_timer.start()
543
544        testcase = "boottest " + "".join(boot_tests)
545        try:
546            self.msg_channel_send_msg(testcase)
547
548            while True:
549                ret = self.msg_channel_recv()
550
551                # If connection is disconnected accidently by peer, for
552                # instance child QEMU process crashed, a message with length
553                # 0 would be received. We should drop this message, and
554                # indicate test framework that something abnormal happened.
555                if len(ret) == 0:
556                    has_error = True
557                    break
558
559                # Print message to STDOUT. Since we might meet EAGAIN IOError
560                # when writting to STDOUT, use try except loop to catch EAGAIN
561                # and waiting STDOUT to be available, then try to write again.
562                def print_msg(msg):
563                    while True:
564                        try:
565                            sys.stdout.write(msg)
566                            break
567                        except IOError as e:
568                            if e.errno != errno.EAGAIN:
569                                RunnerGenericError("Failed to print message")
570                            select.select([], [sys.stdout], [])
571
572                # Please align message structure definition in testrunner.
573                if ret[0] == 0:
574                    msg_len = ret[1]
575                    msg = ret[2 : 2 + msg_len].decode()
576                    print_msg(msg)
577                elif ret[0] == 1:
578                    result = ret[1]
579                    break
580                else:
581                    # Unexpected type, return test result:TEST_FAILED
582                    has_error = True
583                    result = 1
584                    break
585        finally:
586            kill_timer.cancel()
587            self.msg_channel_down()
588            unclean_exit = qemu_exit(command_pipe, qemu_proc,
589                                     has_error=has_error,
590                                     debug_on_error=self.debug_on_error)
591
592        if unclean_exit:
593            raise RunnerGenericError("QEMU did not exit cleanly")
594
595        return result
596
597    def androidtest_run(self, cmd, test_timeout=None):
598        """Run android test cases"""
599        session: RunnerSession = self.session
600        assert session, "No session; must call launch before running any tests."
601
602        try:
603            if not test_timeout:
604                test_timeout = self.default_timeout
605
606            def on_adb_timeout():
607                print(f"adb Timed out ({test_timeout} s)")
608                qemu_handle_error(command_pipe=session.command_pipe,
609                                  debug_on_error=self.debug_on_error)
610
611            test_result = self.adb(["shell"] + cmd, timeout=test_timeout,
612                                   on_timeout=on_adb_timeout, force_output=True)
613            if test_result:
614                session.has_error = True
615
616            return test_result
617        except:
618            session.has_error = True
619            raise
620
621    def adb_bin(self):
622        """Returns location of adb"""
623        return f"{self.config.android}/host/linux-x86/bin/adb"
624
625    def adb(self,
626            args,
627            timeout=60,
628            on_timeout=lambda timeout: print(f"Timed out ({timeout} s)"),
629            force_output=False):
630        """Runs an adb command
631
632        If self.adb_transport is set, specializes the command to that
633        transport to allow for multiple simultaneous tests.
634
635        Timeout specifies a timeout for the command in seconds.
636
637        If force_output is set true, will send results to stdout and
638        stderr regardless of the runner's preferences.
639        """
640        if self.adb_transport:
641            args = ["-t", str(self.adb_transport)] + args
642
643        if force_output:
644            stdout = None
645            stderr = None
646        else:
647            stdout = self.stdout
648            stderr = self.stderr
649
650        adb_proc = subprocess.Popen(  # pylint: disable=consider-using-with
651            [self.adb_bin()] + args, stdin=self.stdin, stdout=stdout,
652            stderr=stderr)
653
654        status = 1
655        try:
656            status = adb_proc.wait(timeout)
657        except subprocess.TimeoutExpired:
658            if on_timeout:
659                on_timeout()
660
661            try:
662                adb_proc.kill()
663            except OSError:
664                pass
665
666        return status
667
668
669    def check_adb(self, args, **kwargs):
670        """As .adb(), but throws an exception if the command fails"""
671        code = self.adb(args, **kwargs)
672        if code != 0:
673            raise AdbFailure(args, code)
674
675    def adb_root(self):
676        """Restarts adbd with root permissions and waits until it's back up"""
677        max_tries = 10
678        num_tries = 0
679
680        # Ensure device is up else adb root can fail
681        self.adb(["wait-for-device"])
682        self.check_adb(["root"])
683
684        while True:
685            # adbd might not be down by this point yet
686            self.adb(["wait-for-device"])
687
688            # Check that adbd is up and running with root permissions
689            code = self.adb(["shell",
690                             "if [[ $(id -u) -ne 0 ]] ; then exit 1; fi"])
691            if code == 0:
692                return
693
694            num_tries += 1
695            if num_tries >= max_tries:
696                raise AdbFailure(["root"], code)
697            time.sleep(1)
698
699    def scan_transport(self, port, expect_none=False):
700        """Given a port and `adb devices -l`, find the transport id"""
701        output = subprocess.check_output([self.adb_bin(), "devices", "-l"],
702                                         universal_newlines=True)
703        match = re.search(fr"localhost:{port}.*transport_id:(\d+)", output)
704        if not match:
705            if expect_none:
706                self.adb_transport = None
707                return
708            raise RunnerGenericError(
709                f"Failed to find transport for port {port} in \n{output}")
710        self.adb_transport = int(match.group(1))
711
712    def adb_up(self, port):
713        """Ensures adb is connected to adbd on the selected port"""
714        # Wait until we can connect to the target port
715        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
716        connect_max_tries = 15
717        connect_tries = 0
718        while True:
719            try:
720                sock.connect(("localhost", port))
721                break
722            except IOError as ioe:
723                connect_tries += 1
724                if connect_tries >= connect_max_tries:
725                    raise Timeout("Wait for adbd socket",
726                                  connect_max_tries) from ioe
727                time.sleep(1)
728        sock.close()
729        self.check_adb(["connect", f"localhost:{port}"])
730        self.scan_transport(port)
731
732        # Sometimes adb can get stuck and will never connect. Using multiple
733        # shorter timeouts works better than one longer timeout in such cases.
734        adb_exception = None
735        for _ in range(10):
736            try:
737                self.check_adb(["wait-for-device"], timeout=30, on_timeout=None)
738                break
739            except AdbFailure as e:
740                adb_exception = e
741                continue
742        else:
743            print("'adb wait-for-device' Timed out")
744            raise adb_exception
745
746        self.adb_root()
747
748        # Files put onto the data partition in the Android build will not
749        # actually be populated into userdata.img when make dist is used.
750        # To work around this, we manually update /data once the device is
751        # booted by pushing it the files that would have been there.
752        userdata = self.qemu_arch_options.android_trusty_user_data()
753        self.check_adb(["push", userdata, "/"])
754
755    def adb_down(self, port):
756        """Cleans up after adb connection to adbd on selected port"""
757        self.check_adb(["disconnect", f"localhost:{port}"])
758
759        # Wait until QEMU's forward has expired
760        connect_max_tries = 300
761        connect_tries = 0
762        while True:
763            try:
764                self.scan_transport(port, expect_none=True)
765                if not self.adb_transport:
766                    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
767                    sock.connect(("localhost", port))
768                    sock.close()
769                connect_tries += 1
770                if connect_tries >= connect_max_tries:
771                    raise Timeout("Wait for port forward to go away",
772                                  connect_max_tries)
773                time.sleep(1)
774            except IOError:
775                break
776
777    def universal_args(self):
778        """Generates arguments used in all qemu invocations"""
779        args = self.qemu_arch_options.basic_options()
780        args += self.qemu_arch_options.bios_options()
781
782        if self.config.linux:
783            args += self.qemu_arch_options.linux_options()
784
785        if self.config.android:
786            args += self.qemu_arch_options.android_drives_args()
787
788        # Append configured extra flags
789        args += self.config.extra_qemu_flags
790
791        return args
792
793    def launch(self, target_state):
794        """Launches the QEMU execution.
795
796        If interactive is specified, it will leave the user connected
797        to the serial console/monitor, and they are responsible for
798        terminating execution.
799
800        If debug is on, the main QEMU instance will be launched with -S and
801        -s, which pause the CPU rather than booting, and starts a gdb server
802        on port 1234 respectively.
803
804        It is the responsibility of callers to ensure that shutdown gets called
805        after launch - regardless of whether the launch succeeded or not.
806
807        Limitations:
808          If the adb port range is already in use, port forwarding may fail.
809
810        TODO: For boot tests, the emulator isn't actually launched here but in
811              boottest_run. Eventually, we want to unify the way boot tests and
812              android tests are launched. Specifically, we might stop execution
813              in the bootloader and have it wait for a command to boot android.
814        """
815        assert self.state == RunnerState.OFF
816        assert target_state in [RunnerState.BOOTLOADER,
817                                RunnerState.ANDROID], target_state
818
819        self.session = RunnerSession()
820        args = self.universal_args()
821
822        try:
823            if self.use_rpmb:
824                args += self.rpmb_up()
825
826            if self.config.linux:
827                args += self.qemu_arch_options.gen_dtb(
828                    args,
829                    self.session.get_qemu_arg_temp_file())
830
831            # Prepend the machine since we don't need to edit it as in gen_dtb
832            args = self.qemu_arch_options.machine_options() + args
833
834            if self.debug:
835                args += ["-s", "-S"]
836
837            # Create socket for communication channel
838            args += self.msg_channel_up()
839
840            if target_state == RunnerState.BOOTLOADER:
841                self.session.args = args
842                self.state = target_state
843                return
844
845            # Logging and terminal monitor
846            # Prepend so that it is the *first* serial port and avoid
847            # conflicting with rpmb0.
848            args = ["-serial", "mon:stdio"] + args
849
850            # If we're noninteractive (e.g. testing) we need a command channel
851            # to tell the guest to exit
852            if not self.interactive:
853                self.session.command_pipe = QEMUCommandPipe()
854                args += self.session.command_pipe.command_args
855
856            # Reserve ADB ports
857            self.session.ports = alloc_ports()
858
859            # Write expected serial number (as given in adb) to stdout.
860            sys.stdout.write(
861                f"DEVICE_SERIAL: localhost:{self.session.ports[1]}\n")
862            sys.stdout.flush()
863
864            # Forward ADB ports in qemu
865            args += forward_ports(self.session.ports)
866
867            qemu_cmd = [self.config.qemu] + args
868            self.session.qemu_proc = subprocess.Popen(  # pylint: disable=consider-using-with
869                qemu_cmd,
870                cwd=self.config.atf,
871                stdin=self.stdin,
872                stdout=self.stdout,
873                stderr=self.stderr)
874
875            if self.session.command_pipe:
876                self.session.command_pipe.open()
877            self.msg_channel_wait_for_connection()
878
879            if self.debug:
880                script_dir = self.config.script_dir
881                if script_dir.endswith("/build-qemu-generic-arm64-test-debug"):
882                    print(f"Debug with: lldb --source {script_dir}/lldbinit")
883                else:
884                    print("Debug with: lldb --one-line 'gdb-remote 1234'")
885
886            # Send request to boot secondary OS
887            self.msg_channel_send_msg("Boot Secondary OS")
888
889            # Bring ADB up talking to the command port
890            self.adb_up(self.session.ports[1])
891
892            self.state = target_state
893        except:
894            self.session.has_error = True
895            raise
896
897    def shutdown(self):
898        """Shut down emulator after test cases have run
899
900        The launch and shutdown methods store shared state in a session object.
901        Calls to launch and shutdown must be correctly paired no matter whether
902        the launch steps and calls to adb succeed or fail.
903        """
904        if self.state == RunnerState.OFF:
905            return
906
907        assert self.session is not None
908
909        # Clean up generated device tree
910        for temp_file in self.session.temp_files:
911            os.remove(temp_file)
912
913        if self.session.has_error:
914            self.error_dump_output()
915
916        unclean_exit = qemu_exit(self.session.command_pipe,
917                                 self.session.qemu_proc,
918                                 has_error=self.session.has_error,
919                                 debug_on_error=self.debug_on_error)
920
921        fcntl.fcntl(0, fcntl.F_SETFL,
922                    fcntl.fcntl(0, fcntl.F_GETFL) & ~os.O_NONBLOCK)
923
924        self.rpmb_down()
925
926        self.msg_channel_down()
927
928        if self.adb_transport:
929            # Disconnect ADB and wait for our port to be released by qemu
930            self.adb_down(self.session.ports[1])
931
932        self.session = None
933        self.state = RunnerState.OFF
934
935        if unclean_exit:
936            raise RunnerGenericError("QEMU did not exit cleanly")
937
938    def reboot(self, target_state):
939        self.shutdown()
940
941        try:
942            self.launch(target_state)
943        except:
944            self.shutdown()
945            raise
946
947    def run(self, boot_tests: Optional[List] = None,
948            android_tests: Optional[List] = None,
949            timeout: Optional[int] = None) -> List[int]:
950        """Run boot or android tests.
951
952        Runs boot_tests through test_runner, android_tests through ADB,
953        returning aggregated test return codes in a list.
954
955        Returns:
956          A list of return codes for the provided tests.
957          A negative return code indicates an internal tool failure.
958
959        Limitations:
960          Until test_runner is updated, only one of android_tests or boot_tests
961          may be provided.
962          Similarly, while boot_tests is a list, test_runner only knows how to
963          correctly run a single test at a time.
964          Again due to test_runner's current state, if boot_tests are
965          specified, interactive will be ignored since the machine will
966          terminate itself.
967
968          If android_tests is provided, a Linux and Android dir must be
969          provided in the config.
970        """
971        assert self.state == RunnerState.OFF
972        self.config.check_config(self.interactive, boot_tests, android_tests)
973
974        if boot_tests and android_tests:
975            raise RunnerGenericError(
976                "Cannot run boot tests and android tests in the same "
977                "QEMU instance")
978
979        if boot_tests and len(boot_tests) > 1:
980            raise RunnerGenericError(
981                "Can only run a single boot test at a time")
982
983        timeout = timeout if timeout else self.default_timeout
984        try:
985            self.launch(RunnerState.BOOTLOADER if boot_tests else
986                        RunnerState.ANDROID)
987            test_results = []
988
989            if boot_tests:
990                test_results.append(self.boottest_run(boot_tests, timeout))
991
992            if android_tests:
993                for android_test in android_tests:
994                    test_result = self.androidtest_run([android_test], timeout)
995                    test_results.append(test_result)
996                    if test_result:
997                        break
998
999            return test_results
1000        finally:
1001            # The wait on QEMU is done here to ensure that ADB failures do not
1002            # take away the user's serial console in interactive mode.
1003            if self.interactive and self.session:
1004                # The user is responsible for quitting QEMU
1005                self.session.qemu_proc.wait()
1006            self.shutdown()
1007