1# Copyright 2019 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import json
6import logging
7import os
8import re
9
10from autotest_lib.client.common_lib import error
11from autotest_lib.client.common_lib import utils as cutils
12from autotest_lib.client.common_lib.cros import kernel_utils
13from autotest_lib.client.cros import constants
14from autotest_lib.server import utils
15from autotest_lib.server.cros import provisioner
16from autotest_lib.server.cros.update_engine import update_engine_test
17
18
19class autoupdate_StatefulCompatibility(update_engine_test.UpdateEngineTest):
20    """Tests autoupdating to/from kernel-next images."""
21    version = 1
22
23    _LOGIN_TEST = 'login_LoginSuccess'
24
25
26    def cleanup(self):
27        """Save the logs from stateful_partition's preserved/log dir."""
28        stateful_preserved_logs = os.path.join(self.resultsdir,
29                                               '~stateful_preserved_logs')
30        os.makedirs(stateful_preserved_logs)
31        self._host.get_file(
32                constants.AUTOUPDATE_PRESERVE_LOG,
33                stateful_preserved_logs,
34                safe_symlinks=True,
35                preserve_perm=False)
36        super(autoupdate_StatefulCompatibility, self).cleanup()
37
38
39    def _get_target_uri(self, target_board, version_regex, max_image_checks):
40        """Checks through all valid builds for the latest green build
41
42        @param target_board: the name of the board to test against
43        @param version_regex: the version regex to test against
44        @param max_image_checks: the number of images to check for stability
45
46        @return the URI for the most recent passing build to test against
47
48        """
49        candidate_uris = self._get_candidate_uris(target_board, version_regex)
50        candidate_uris = candidate_uris[:max_image_checks]
51
52        metadata_uri = None
53        most_recent_version = None
54        most_recent_channel = None
55
56        for uri in candidate_uris:
57            uri = self._to_real_path(uri)
58            metadata = self._get_metadata_dict(uri)
59            chan = self._get_image_channel(metadata)
60            version = cutils.parse_gs_uri_version(uri)
61
62            if not self._stateful_payload_exists(chan, target_board, version):
63                continue
64
65            # Keep track of the first image found that has an available payload
66            if most_recent_version is None:
67                most_recent_version = version
68                most_recent_channel = chan
69
70            if self._is_build_green(metadata):
71                metadata_uri = uri
72                break
73
74        if most_recent_version is None:
75            raise error.TestError('Could not find an acceptable image for %s.' %
76                                  target_board)
77
78        if metadata_uri is None:
79            logging.warning('No image met quality criteria. Checked %d images',
80                            len(candidate_uris))
81            # At this point we've checked as many images as possible up to the
82            # specified maximum, and none of them have qualified with our pass/
83            # fail criteria. Any image is as good as any other, so we might as
84            # well continue with the most recent image. The only other option is
85            # to fail this test
86            version = most_recent_version
87            chan = most_recent_channel
88
89        payload = self._get_payload_uri(chan, target_board, version)
90        if payload is not None:
91            return payload
92
93        raise error.TestError('Could not find an acceptable payload for %s.' %
94                              target_board)
95
96
97    def _get_candidate_uris(self, target_board, version_regex):
98        """Retrieves a list of GS URIs that match the target board and version
99
100        @param target_board: the name of the board to get image URIs for
101        @param version_regex: a regex passed to 'gsutil ls' to match GS URIs
102
103        @return: a list of boards that match the target_board and version_regex
104
105        """
106        logging.info('Going to find candidate image for %s.', target_board)
107
108        payload_uri = 'gs://chromeos-image-archive/%s-release/%s/' % (
109            target_board, version_regex)
110
111        candidate_uris = utils.system_output('gsutil ls -d %s' %
112                                             payload_uri).splitlines()
113        candidate_uris.sort(cutils.compare_gs_uri_build_versions, reverse=True)
114        return candidate_uris
115
116
117    @staticmethod
118    def _to_real_path(uri):
119        """Converts a target image URI from the form LATEST-* to R##-*
120
121        Target images can be referenced by matching against LATEST-* rather than
122        the actual milestone. The LATEST-* files are actually text files that
123        contain the name of the actual GS bucket that contains the image data.
124
125        @param uri: the GS bucket URI of the LATEST-* bucket path
126
127        @return the URI of the dereferenced GS bucket
128
129        """
130        latest_pos = uri.find('LATEST')
131        if latest_pos < 0:
132            # Path is not in the form 'gs://../../LATEST-*'
133            return uri
134
135        relative_path = utils.system_output('gsutil cat %s' % uri).strip()
136        return uri[:latest_pos] + relative_path
137
138
139    @staticmethod
140    def _stateful_payload_exists(channel, target_board, version):
141        """Checks that stateful.tgz exists for the given board and version
142
143        @param channel: The release channel (canary, dev, beta, or stable)
144        @param target_board: The name of the target board
145        @param version: A string containing the build version ('12345.6.7')
146
147        @return True if stateful.gz exists for this image, otherwise False
148
149        """
150
151        if channel is None:
152            return False
153
154        channel_payload_uri = 'gs://chromeos-releases/%s-channel/%s/%s' % (
155                channel, target_board, version)
156        exists = not utils.system('gsutil -q stat %s/stateful.tgz' %
157                                  channel_payload_uri, ignore_status=True)
158        return exists
159
160
161    @staticmethod
162    def _get_payload_uri(channel, board, version):
163        """Gets the location of the update payload for staging on the dev server
164
165        For a given release channel, board, and release version this will return
166        the location for the full signed payload (as opposed to delta payloads).
167
168        @param channel: The release channel (canary, dev, beta, or stable)
169        @param board: The name of the target board
170        @param version: A string containing the build version ('12345.6.7')
171
172        @return The GS URI for the full payload to be staged on the devserver
173
174        """
175        payload_uri = 'gs://chromeos-releases/%s-channel/%s/%s/payloads' % (
176            channel, board, version)
177
178        payloads = utils.system_output('gsutil ls -d %s/*%s*full_test*' % (
179            payload_uri, version)).splitlines()
180        logging.debug('Payloads: %s', str(payloads))
181
182        for payload in payloads:
183            if re.match('.*-[a-z|0-9]{32}$', payload) is not None:
184                return payload
185        return None
186
187
188    @staticmethod
189    def _get_metadata_dict(payload_uri):
190        """Fetches the build metadata from the associated GS bucket
191
192        @param payload_uri: the URI for the GS bucket the image is from.
193
194        @return a dictionary of values representing the metadata json values
195
196        """
197        metadata_uri = payload_uri.strip('/') + '/metadata.json'
198        logging.info('Going to fetch image metadata (%s)', metadata_uri)
199        cat_result = utils.run('gsutil cat %s' % metadata_uri,
200                               ignore_status=True)
201
202        if cat_result.exit_status != 0:
203            logging.info('''Couldn't find metadata at %s.''', metadata_uri)
204            return None
205
206        metadata = json.loads(cat_result.stdout)
207        return metadata
208
209
210    @staticmethod
211    def _get_image_channel(metadata):
212        """Returns the release channel from the image metadata
213
214        @param metadata: A dict of values representing the image metadata
215
216        @return the release channel for the image (canary, dev, beta, stable)
217
218        """
219
220        all_channels = ['Stable', 'Beta', 'Dev', 'Canary']
221
222        if 'tags' not in metadata:
223            return None
224
225        # The metadata tags contains the status for paygen stages on all
226        # channels paygen was run for. This should tell us what channels the
227        # payload is available under.
228        # These tags use the form 'stage_status:PaygenBuild<Channel>'
229        paygen_tags = [t for t in metadata['tags'] if 'PaygenBuild' in t]
230
231        # Find all the channels paygen was run for on this image
232        channels = [c for c in all_channels for t in paygen_tags if c in t]
233
234        if not channels:
235            return None
236
237        # The channels list contains some subset of the elements in the
238        # all_channels list, presented in the same order. If both the Beta and
239        # Stable channels are available, this will return "stable", for example.
240        return channels[0].lower()
241
242
243    @staticmethod
244    def _is_build_green(metadata):
245        """Inspects the image metadata to see if the build is "green"
246
247        @param metadata A dict of values representing the image metadata
248
249        @return True if the image appears to be good enough to test against.
250
251        """
252        if metadata is None:
253            return False
254
255        if not ('tags' in metadata and 'status' in metadata['tags']):
256            return False
257
258        return metadata['tags']['status'] == 'pass'
259
260
261    def run_once(self, test_conf, max_image_checks):
262        """Main entry point of the test."""
263        logging.debug("Using test_conf: %s", test_conf)
264
265        self._source_payload_uri = test_conf['source_payload_uri']
266        self._target_payload_uri = test_conf['target_payload_uri']
267
268        if self._target_payload_uri is None:
269            target_board = test_conf['target_board']
270            target_version_regex = test_conf['target_version_regex']
271
272            self._target_payload_uri = self._get_target_uri(
273                target_board, target_version_regex, max_image_checks)
274
275        logging.debug('Using source image %s', self._source_payload_uri)
276        logging.debug('Using target image %s', self._target_payload_uri)
277
278        self._autotest_devserver = self._get_devserver_for_test(
279            {'target_payload_uri': self._target_payload_uri})
280
281        self._stage_payloads(self._source_payload_uri, None)
282        self._stage_payloads(self._target_payload_uri, None)
283
284        if self._source_payload_uri is not None:
285            build_name, _ = self._get_update_parameters_from_uri(
286                    self._source_payload_uri)
287            update_url = self._autotest_devserver.get_update_url(build_name)
288            logging.info('Installing source image with update url: %s',
289                         update_url)
290
291            provisioner.ChromiumOSProvisioner(
292                    update_url, host=self._host,
293                    is_release_bucket=True).run_provision()
294
295            self._run_client_test_and_check_result(self._LOGIN_TEST,
296                                                   tag='source')
297
298        # Record the active root partition.
299        active, inactive = kernel_utils.get_kernel_state(self._host)
300        logging.info('Source active slot: %s', active)
301
302        # Get the source and target versions for verifying hostlog update events.
303        source_release = self._host.get_release_version()
304        target_release, _ = self._get_update_parameters_from_uri(
305                self._target_payload_uri)
306        target_release = target_release.split('/')[-1]
307
308        logging.debug('Going to install target image on DUT.')
309        self.update_device(
310                self._target_payload_uri, tag='target', ignore_appid=True)
311
312        # Compare hostlog events from the update to the expected ones.
313        rootfs, reboot = self._create_hostlog_files()
314        self.verify_update_events(source_release, rootfs)
315        self.verify_update_events(source_release, reboot, target_release)
316        kernel_utils.verify_boot_expectations(inactive, host=self._host)
317
318        self._run_client_test_and_check_result(self._LOGIN_TEST, tag='target')
319