1# Copyright (C) 2020 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import copy
16import itertools
17import logging
18import os
19import zipfile
20
21import ota_metadata_pb2
22from common import (ZipDelete, ZipClose, OPTIONS, MakeTempFile,
23                    ZipWriteStr, BuildInfo, LoadDictionaryFromFile,
24                    SignFile, PARTITIONS_WITH_BUILD_PROP, PartitionBuildProps)
25
26logger = logging.getLogger(__name__)
27
28OPTIONS.no_signing = False
29OPTIONS.force_non_ab = False
30OPTIONS.wipe_user_data = False
31OPTIONS.downgrade = False
32OPTIONS.key_passwords = {}
33OPTIONS.package_key = None
34OPTIONS.incremental_source = None
35OPTIONS.retrofit_dynamic_partitions = False
36OPTIONS.output_metadata_path = None
37OPTIONS.boot_variable_file = None
38
39METADATA_NAME = 'META-INF/com/android/metadata'
40METADATA_PROTO_NAME = 'META-INF/com/android/metadata.pb'
41UNZIP_PATTERN = ['IMAGES/*', 'META/*', 'OTA/*', 'RADIO/*']
42SECURITY_PATCH_LEVEL_PROP_NAME = "ro.build.version.security_patch"
43
44
45def FinalizeMetadata(metadata, input_file, output_file, needed_property_files):
46  """Finalizes the metadata and signs an A/B OTA package.
47
48  In order to stream an A/B OTA package, we need 'ota-streaming-property-files'
49  that contains the offsets and sizes for the ZIP entries. An example
50  property-files string is as follows.
51
52    "payload.bin:679:343,payload_properties.txt:378:45,metadata:69:379"
53
54  OTA server can pass down this string, in addition to the package URL, to the
55  system update client. System update client can then fetch individual ZIP
56  entries (ZIP_STORED) directly at the given offset of the URL.
57
58  Args:
59    metadata: The metadata dict for the package.
60    input_file: The input ZIP filename that doesn't contain the package METADATA
61        entry yet.
62    output_file: The final output ZIP filename.
63    needed_property_files: The list of PropertyFiles' to be generated.
64  """
65
66  def ComputeAllPropertyFiles(input_file, needed_property_files):
67    # Write the current metadata entry with placeholders.
68    with zipfile.ZipFile(input_file, allowZip64=True) as input_zip:
69      for property_files in needed_property_files:
70        metadata.property_files[property_files.name] = property_files.Compute(
71            input_zip)
72      namelist = input_zip.namelist()
73
74    if METADATA_NAME in namelist or METADATA_PROTO_NAME in namelist:
75      ZipDelete(input_file, [METADATA_NAME, METADATA_PROTO_NAME])
76    output_zip = zipfile.ZipFile(input_file, 'a', allowZip64=True)
77    WriteMetadata(metadata, output_zip)
78    ZipClose(output_zip)
79
80    if OPTIONS.no_signing:
81      return input_file
82
83    prelim_signing = MakeTempFile(suffix='.zip')
84    SignOutput(input_file, prelim_signing)
85    return prelim_signing
86
87  def FinalizeAllPropertyFiles(prelim_signing, needed_property_files):
88    with zipfile.ZipFile(prelim_signing, allowZip64=True) as prelim_signing_zip:
89      for property_files in needed_property_files:
90        metadata.property_files[property_files.name] = property_files.Finalize(
91            prelim_signing_zip,
92            len(metadata.property_files[property_files.name]))
93
94  # SignOutput(), which in turn calls signapk.jar, will possibly reorder the ZIP
95  # entries, as well as padding the entry headers. We do a preliminary signing
96  # (with an incomplete metadata entry) to allow that to happen. Then compute
97  # the ZIP entry offsets, write back the final metadata and do the final
98  # signing.
99  prelim_signing = ComputeAllPropertyFiles(input_file, needed_property_files)
100  try:
101    FinalizeAllPropertyFiles(prelim_signing, needed_property_files)
102  except PropertyFiles.InsufficientSpaceException:
103    # Even with the preliminary signing, the entry orders may change
104    # dramatically, which leads to insufficiently reserved space during the
105    # first call to ComputeAllPropertyFiles(). In that case, we redo all the
106    # preliminary signing works, based on the already ordered ZIP entries, to
107    # address the issue.
108    prelim_signing = ComputeAllPropertyFiles(
109        prelim_signing, needed_property_files)
110    FinalizeAllPropertyFiles(prelim_signing, needed_property_files)
111
112  # Replace the METADATA entry.
113  ZipDelete(prelim_signing, [METADATA_NAME, METADATA_PROTO_NAME])
114  output_zip = zipfile.ZipFile(prelim_signing, 'a', allowZip64=True)
115  WriteMetadata(metadata, output_zip)
116  ZipClose(output_zip)
117
118  # Re-sign the package after updating the metadata entry.
119  if OPTIONS.no_signing:
120    output_file = prelim_signing
121  else:
122    SignOutput(prelim_signing, output_file)
123
124  # Reopen the final signed zip to double check the streaming metadata.
125  with zipfile.ZipFile(output_file, allowZip64=True) as output_zip:
126    for property_files in needed_property_files:
127      property_files.Verify(
128          output_zip, metadata.property_files[property_files.name].strip())
129
130  # If requested, dump the metadata to a separate file.
131  output_metadata_path = OPTIONS.output_metadata_path
132  if output_metadata_path:
133    WriteMetadata(metadata, output_metadata_path)
134
135
136def WriteMetadata(metadata_proto, output):
137  """Writes the metadata to the zip archive or a file.
138
139  Args:
140    metadata_proto: The metadata protobuf for the package.
141    output: A ZipFile object or a string of the output file path. If a string
142      path is given, the metadata in the protobuf format will be written to
143      {output}.pb, e.g. ota_metadata.pb
144  """
145
146  metadata_dict = BuildLegacyOtaMetadata(metadata_proto)
147  legacy_metadata = "".join(["%s=%s\n" % kv for kv in
148                             sorted(metadata_dict.items())])
149  if isinstance(output, zipfile.ZipFile):
150    ZipWriteStr(output, METADATA_PROTO_NAME, metadata_proto.SerializeToString(),
151                compress_type=zipfile.ZIP_STORED)
152    ZipWriteStr(output, METADATA_NAME, legacy_metadata,
153                compress_type=zipfile.ZIP_STORED)
154    return
155
156  with open('{}.pb'.format(output), 'w') as f:
157    f.write(metadata_proto.SerializeToString())
158  with open(output, 'w') as f:
159    f.write(legacy_metadata)
160
161
162def UpdateDeviceState(device_state, build_info, boot_variable_values,
163                      is_post_build):
164  """Update the fields of the DeviceState proto with build info."""
165
166  def UpdatePartitionStates(partition_states):
167    """Update the per-partition state according to its build.prop"""
168    if not build_info.is_ab:
169      return
170    build_info_set = ComputeRuntimeBuildInfos(build_info,
171                                              boot_variable_values)
172    assert "ab_partitions" in build_info.info_dict,\
173        "ab_partitions property required for ab update."
174    ab_partitions = set(build_info.info_dict.get("ab_partitions"))
175
176    # delta_generator will error out on unused timestamps,
177    # so only generate timestamps for dynamic partitions
178    # used in OTA update.
179    for partition in sorted(set(PARTITIONS_WITH_BUILD_PROP) & ab_partitions):
180      partition_prop = build_info.info_dict.get(
181          '{}.build.prop'.format(partition))
182      # Skip if the partition is missing, or it doesn't have a build.prop
183      if not partition_prop or not partition_prop.build_props:
184        continue
185
186      partition_state = partition_states.add()
187      partition_state.partition_name = partition
188      # Update the partition's runtime device names and fingerprints
189      partition_devices = set()
190      partition_fingerprints = set()
191      for runtime_build_info in build_info_set:
192        partition_devices.add(
193            runtime_build_info.GetPartitionBuildProp('ro.product.device',
194                                                     partition))
195        partition_fingerprints.add(
196            runtime_build_info.GetPartitionFingerprint(partition))
197
198      partition_state.device.extend(sorted(partition_devices))
199      partition_state.build.extend(sorted(partition_fingerprints))
200
201      # TODO(xunchang) set the boot image's version with kmi. Note the boot
202      # image doesn't have a file map.
203      partition_state.version = build_info.GetPartitionBuildProp(
204          'ro.build.date.utc', partition)
205
206  # TODO(xunchang), we can save a call to ComputeRuntimeBuildInfos.
207  build_devices, build_fingerprints = \
208      CalculateRuntimeDevicesAndFingerprints(build_info, boot_variable_values)
209  device_state.device.extend(sorted(build_devices))
210  device_state.build.extend(sorted(build_fingerprints))
211  device_state.build_incremental = build_info.GetBuildProp(
212      'ro.build.version.incremental')
213
214  UpdatePartitionStates(device_state.partition_state)
215
216  if is_post_build:
217    device_state.sdk_level = build_info.GetBuildProp(
218        'ro.build.version.sdk')
219    device_state.security_patch_level = build_info.GetBuildProp(
220        'ro.build.version.security_patch')
221    # Use the actual post-timestamp, even for a downgrade case.
222    device_state.timestamp = int(build_info.GetBuildProp('ro.build.date.utc'))
223
224
225def GetPackageMetadata(target_info, source_info=None):
226  """Generates and returns the metadata proto.
227
228  It generates a ota_metadata protobuf that contains the info to be written
229  into an OTA package (META-INF/com/android/metadata.pb). It also handles the
230  detection of downgrade / data wipe based on the global options.
231
232  Args:
233    target_info: The BuildInfo instance that holds the target build info.
234    source_info: The BuildInfo instance that holds the source build info, or
235        None if generating full OTA.
236
237  Returns:
238    A protobuf to be written into package metadata entry.
239  """
240  assert isinstance(target_info, BuildInfo)
241  assert source_info is None or isinstance(source_info, BuildInfo)
242
243  boot_variable_values = {}
244  if OPTIONS.boot_variable_file:
245    d = LoadDictionaryFromFile(OPTIONS.boot_variable_file)
246    for key, values in d.items():
247      boot_variable_values[key] = [val.strip() for val in values.split(',')]
248
249  metadata_proto = ota_metadata_pb2.OtaMetadata()
250  # TODO(xunchang) some fields, e.g. post-device isn't necessary. We can
251  # consider skipping them if they aren't used by clients.
252  UpdateDeviceState(metadata_proto.postcondition, target_info,
253                    boot_variable_values, True)
254
255  if target_info.is_ab and not OPTIONS.force_non_ab:
256    metadata_proto.type = ota_metadata_pb2.OtaMetadata.AB
257    metadata_proto.required_cache = 0
258  else:
259    metadata_proto.type = ota_metadata_pb2.OtaMetadata.BLOCK
260    # cache requirement will be updated by the non-A/B codes.
261
262  if OPTIONS.wipe_user_data:
263    metadata_proto.wipe = True
264
265  if OPTIONS.retrofit_dynamic_partitions:
266    metadata_proto.retrofit_dynamic_partitions = True
267
268  is_incremental = source_info is not None
269  if is_incremental:
270    UpdateDeviceState(metadata_proto.precondition, source_info,
271                      boot_variable_values, False)
272  else:
273    metadata_proto.precondition.device.extend(
274        metadata_proto.postcondition.device)
275
276  # Detect downgrades and set up downgrade flags accordingly.
277  if is_incremental:
278    HandleDowngradeMetadata(metadata_proto, target_info, source_info)
279
280  return metadata_proto
281
282
283def BuildLegacyOtaMetadata(metadata_proto):
284  """Converts the metadata proto to a legacy metadata dict.
285
286  This metadata dict is used to build the legacy metadata text file for
287  backward compatibility. We won't add new keys to the legacy metadata format.
288  If new information is needed, we should add it as a new field in OtaMetadata
289  proto definition.
290  """
291
292  separator = '|'
293
294  metadata_dict = {}
295  if metadata_proto.type == ota_metadata_pb2.OtaMetadata.AB:
296    metadata_dict['ota-type'] = 'AB'
297  elif metadata_proto.type == ota_metadata_pb2.OtaMetadata.BLOCK:
298    metadata_dict['ota-type'] = 'BLOCK'
299  if metadata_proto.wipe:
300    metadata_dict['ota-wipe'] = 'yes'
301  if metadata_proto.retrofit_dynamic_partitions:
302    metadata_dict['ota-retrofit-dynamic-partitions'] = 'yes'
303  if metadata_proto.downgrade:
304    metadata_dict['ota-downgrade'] = 'yes'
305
306  metadata_dict['ota-required-cache'] = str(metadata_proto.required_cache)
307
308  post_build = metadata_proto.postcondition
309  metadata_dict['post-build'] = separator.join(post_build.build)
310  metadata_dict['post-build-incremental'] = post_build.build_incremental
311  metadata_dict['post-sdk-level'] = post_build.sdk_level
312  metadata_dict['post-security-patch-level'] = post_build.security_patch_level
313  metadata_dict['post-timestamp'] = str(post_build.timestamp)
314
315  pre_build = metadata_proto.precondition
316  metadata_dict['pre-device'] = separator.join(pre_build.device)
317  # incremental updates
318  if len(pre_build.build) != 0:
319    metadata_dict['pre-build'] = separator.join(pre_build.build)
320    metadata_dict['pre-build-incremental'] = pre_build.build_incremental
321
322  if metadata_proto.spl_downgrade:
323    metadata_dict['spl-downgrade'] = 'yes'
324  metadata_dict.update(metadata_proto.property_files)
325
326  return metadata_dict
327
328
329def HandleDowngradeMetadata(metadata_proto, target_info, source_info):
330  # Only incremental OTAs are allowed to reach here.
331  assert OPTIONS.incremental_source is not None
332
333  post_timestamp = target_info.GetBuildProp("ro.build.date.utc")
334  pre_timestamp = source_info.GetBuildProp("ro.build.date.utc")
335  is_downgrade = int(post_timestamp) < int(pre_timestamp)
336
337  if OPTIONS.spl_downgrade:
338    metadata_proto.spl_downgrade = True
339
340  if OPTIONS.downgrade:
341    if not is_downgrade:
342      raise RuntimeError(
343          "--downgrade or --override_timestamp specified but no downgrade "
344          "detected: pre: %s, post: %s" % (pre_timestamp, post_timestamp))
345    metadata_proto.downgrade = True
346  else:
347    if is_downgrade:
348      raise RuntimeError(
349          "Downgrade detected based on timestamp check: pre: %s, post: %s. "
350          "Need to specify --override_timestamp OR --downgrade to allow "
351          "building the incremental." % (pre_timestamp, post_timestamp))
352
353
354def ComputeRuntimeBuildInfos(default_build_info, boot_variable_values):
355  """Returns a set of build info objects that may exist during runtime."""
356
357  build_info_set = {default_build_info}
358  if not boot_variable_values:
359    return build_info_set
360
361  # Calculate all possible combinations of the values for the boot variables.
362  keys = boot_variable_values.keys()
363  value_list = boot_variable_values.values()
364  combinations = [dict(zip(keys, values))
365                  for values in itertools.product(*value_list)]
366  for placeholder_values in combinations:
367    # Reload the info_dict as some build properties may change their values
368    # based on the value of ro.boot* properties.
369    info_dict = copy.deepcopy(default_build_info.info_dict)
370    for partition in PARTITIONS_WITH_BUILD_PROP:
371      partition_prop_key = "{}.build.prop".format(partition)
372      input_file = info_dict[partition_prop_key].input_file
373      if isinstance(input_file, zipfile.ZipFile):
374        with zipfile.ZipFile(input_file.filename, allowZip64=True) as input_zip:
375          info_dict[partition_prop_key] = \
376              PartitionBuildProps.FromInputFile(input_zip, partition,
377                                                placeholder_values)
378      else:
379        info_dict[partition_prop_key] = \
380            PartitionBuildProps.FromInputFile(input_file, partition,
381                                              placeholder_values)
382    info_dict["build.prop"] = info_dict["system.build.prop"]
383    build_info_set.add(BuildInfo(info_dict, default_build_info.oem_dicts))
384
385  return build_info_set
386
387
388def CalculateRuntimeDevicesAndFingerprints(default_build_info,
389                                           boot_variable_values):
390  """Returns a tuple of sets for runtime devices and fingerprints"""
391
392  device_names = set()
393  fingerprints = set()
394  build_info_set = ComputeRuntimeBuildInfos(default_build_info,
395                                            boot_variable_values)
396  for runtime_build_info in build_info_set:
397    device_names.add(runtime_build_info.device)
398    fingerprints.add(runtime_build_info.fingerprint)
399  return device_names, fingerprints
400
401
402class PropertyFiles(object):
403  """A class that computes the property-files string for an OTA package.
404
405  A property-files string is a comma-separated string that contains the
406  offset/size info for an OTA package. The entries, which must be ZIP_STORED,
407  can be fetched directly with the package URL along with the offset/size info.
408  These strings can be used for streaming A/B OTAs, or allowing an updater to
409  download package metadata entry directly, without paying the cost of
410  downloading entire package.
411
412  Computing the final property-files string requires two passes. Because doing
413  the whole package signing (with signapk.jar) will possibly reorder the ZIP
414  entries, which may in turn invalidate earlier computed ZIP entry offset/size
415  values.
416
417  This class provides functions to be called for each pass. The general flow is
418  as follows.
419
420    property_files = PropertyFiles()
421    # The first pass, which writes placeholders before doing initial signing.
422    property_files.Compute()
423    SignOutput()
424
425    # The second pass, by replacing the placeholders with actual data.
426    property_files.Finalize()
427    SignOutput()
428
429  And the caller can additionally verify the final result.
430
431    property_files.Verify()
432  """
433
434  def __init__(self):
435    self.name = None
436    self.required = ()
437    self.optional = ()
438
439  def Compute(self, input_zip):
440    """Computes and returns a property-files string with placeholders.
441
442    We reserve extra space for the offset and size of the metadata entry itself,
443    although we don't know the final values until the package gets signed.
444
445    Args:
446      input_zip: The input ZIP file.
447
448    Returns:
449      A string with placeholders for the metadata offset/size info, e.g.
450      "payload.bin:679:343,payload_properties.txt:378:45,metadata:        ".
451    """
452    return self.GetPropertyFilesString(input_zip, reserve_space=True)
453
454  class InsufficientSpaceException(Exception):
455    pass
456
457  def Finalize(self, input_zip, reserved_length):
458    """Finalizes a property-files string with actual METADATA offset/size info.
459
460    The input ZIP file has been signed, with the ZIP entries in the desired
461    place (signapk.jar will possibly reorder the ZIP entries). Now we compute
462    the ZIP entry offsets and construct the property-files string with actual
463    data. Note that during this process, we must pad the property-files string
464    to the reserved length, so that the METADATA entry size remains the same.
465    Otherwise the entries' offsets and sizes may change again.
466
467    Args:
468      input_zip: The input ZIP file.
469      reserved_length: The reserved length of the property-files string during
470          the call to Compute(). The final string must be no more than this
471          size.
472
473    Returns:
474      A property-files string including the metadata offset/size info, e.g.
475      "payload.bin:679:343,payload_properties.txt:378:45,metadata:69:379  ".
476
477    Raises:
478      InsufficientSpaceException: If the reserved length is insufficient to hold
479          the final string.
480    """
481    result = self.GetPropertyFilesString(input_zip, reserve_space=False)
482    if len(result) > reserved_length:
483      raise self.InsufficientSpaceException(
484          'Insufficient reserved space: reserved={}, actual={}'.format(
485              reserved_length, len(result)))
486
487    result += ' ' * (reserved_length - len(result))
488    return result
489
490  def Verify(self, input_zip, expected):
491    """Verifies the input ZIP file contains the expected property-files string.
492
493    Args:
494      input_zip: The input ZIP file.
495      expected: The property-files string that's computed from Finalize().
496
497    Raises:
498      AssertionError: On finding a mismatch.
499    """
500    actual = self.GetPropertyFilesString(input_zip)
501    assert actual == expected, \
502        "Mismatching streaming metadata: {} vs {}.".format(actual, expected)
503
504  def GetPropertyFilesString(self, zip_file, reserve_space=False):
505    """
506    Constructs the property-files string per request.
507
508    Args:
509      zip_file: The input ZIP file.
510      reserved_length: The reserved length of the property-files string.
511
512    Returns:
513      A property-files string including the metadata offset/size info, e.g.
514      "payload.bin:679:343,payload_properties.txt:378:45,metadata:     ".
515    """
516
517    def ComputeEntryOffsetSize(name):
518      """Computes the zip entry offset and size."""
519      info = zip_file.getinfo(name)
520      offset = info.header_offset
521      offset += zipfile.sizeFileHeader
522      offset += len(info.extra) + len(info.filename)
523      size = info.file_size
524      return '%s:%d:%d' % (os.path.basename(name), offset, size)
525
526    tokens = []
527    tokens.extend(self._GetPrecomputed(zip_file))
528    for entry in self.required:
529      tokens.append(ComputeEntryOffsetSize(entry))
530    for entry in self.optional:
531      if entry in zip_file.namelist():
532        tokens.append(ComputeEntryOffsetSize(entry))
533
534    # 'META-INF/com/android/metadata' is required. We don't know its actual
535    # offset and length (as well as the values for other entries). So we reserve
536    # 15-byte as a placeholder ('offset:length'), which is sufficient to cover
537    # the space for metadata entry. Because 'offset' allows a max of 10-digit
538    # (i.e. ~9 GiB), with a max of 4-digit for the length. Note that all the
539    # reserved space serves the metadata entry only.
540    if reserve_space:
541      tokens.append('metadata:' + ' ' * 15)
542      tokens.append('metadata.pb:' + ' ' * 15)
543    else:
544      tokens.append(ComputeEntryOffsetSize(METADATA_NAME))
545      tokens.append(ComputeEntryOffsetSize(METADATA_PROTO_NAME))
546
547    return ','.join(tokens)
548
549  def _GetPrecomputed(self, input_zip):
550    """Computes the additional tokens to be included into the property-files.
551
552    This applies to tokens without actual ZIP entries, such as
553    payload_metadata.bin. We want to expose the offset/size to updaters, so
554    that they can download the payload metadata directly with the info.
555
556    Args:
557      input_zip: The input zip file.
558
559    Returns:
560      A list of strings (tokens) to be added to the property-files string.
561    """
562    # pylint: disable=no-self-use
563    # pylint: disable=unused-argument
564    return []
565
566
567def SignOutput(temp_zip_name, output_zip_name):
568  pw = OPTIONS.key_passwords[OPTIONS.package_key]
569
570  SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw,
571           whole_file=True)
572