1#!/usr/bin/env python
2#
3# Copyright (C) 2008 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Given a target-files zipfile, produces an OTA package that installs
19that build.  An incremental OTA is produced if -i is given, otherwise
20a full OTA is produced.
21
22Usage:  ota_from_target_files [flags] input_target_files output_ota_package
23
24  --board_config  <file>
25      Deprecated.
26
27  -k (--package_key) <key> Key to use to sign the package (default is
28      the value of default_system_dev_certificate from the input
29      target-files's META/misc_info.txt, or
30      "build/target/product/security/testkey" if that value is not
31      specified).
32
33      For incremental OTAs, the default value is based on the source
34      target-file, not the target build.
35
36  -i  (--incremental_from)  <file>
37      Generate an incremental OTA using the given target-files zip as
38      the starting build.
39
40  -v  (--verify)
41      Remount and verify the checksums of the files written to the
42      system and vendor (if used) partitions.  Incremental builds only.
43
44  -o  (--oem_settings)  <file>
45      Use the file to specify the expected OEM-specific properties
46      on the OEM partition of the intended device.
47
48  -w  (--wipe_user_data)
49      Generate an OTA package that will wipe the user data partition
50      when installed.
51
52  -n  (--no_prereq)
53      Omit the timestamp prereq check normally included at the top of
54      the build scripts (used for developer OTA packages which
55      legitimately need to go back and forth).
56
57  -e  (--extra_script)  <file>
58      Insert the contents of file at the end of the update script.
59
60  -a  (--aslr_mode)  <on|off>
61      Specify whether to turn on ASLR for the package (on by default).
62
63  -2  (--two_step)
64      Generate a 'two-step' OTA package, where recovery is updated
65      first, so that any changes made to the system partition are done
66      using the new recovery (new kernel, etc.).
67
68  --block
69      Generate a block-based OTA if possible.  Will fall back to a
70      file-based OTA if the target_files is older and doesn't support
71      block-based OTAs.
72
73  -b  (--binary)  <file>
74      Use the given binary as the update-binary in the output package,
75      instead of the binary in the build's target_files.  Use for
76      development only.
77
78  -t  (--worker_threads) <int>
79      Specifies the number of worker-threads that will be used when
80      generating patches for incremental updates (defaults to 3).
81
82"""
83
84import sys
85
86if sys.hexversion < 0x02070000:
87  print >> sys.stderr, "Python 2.7 or newer is required."
88  sys.exit(1)
89
90import copy
91import errno
92import multiprocessing
93import os
94import re
95import subprocess
96import tempfile
97import time
98import zipfile
99
100from hashlib import sha1 as sha1
101
102import common
103import edify_generator
104import build_image
105import blockimgdiff
106import sparse_img
107
108OPTIONS = common.OPTIONS
109OPTIONS.package_key = None
110OPTIONS.incremental_source = None
111OPTIONS.verify = False
112OPTIONS.require_verbatim = set()
113OPTIONS.prohibit_verbatim = set(("system/build.prop",))
114OPTIONS.patch_threshold = 0.95
115OPTIONS.wipe_user_data = False
116OPTIONS.omit_prereq = False
117OPTIONS.extra_script = None
118OPTIONS.aslr_mode = True
119OPTIONS.worker_threads = multiprocessing.cpu_count() // 2
120if OPTIONS.worker_threads == 0:
121  OPTIONS.worker_threads = 1
122OPTIONS.two_step = False
123OPTIONS.no_signing = False
124OPTIONS.block_based = False
125OPTIONS.updater_binary = None
126OPTIONS.oem_source = None
127OPTIONS.fallback_to_full = True
128
129def MostPopularKey(d, default):
130  """Given a dict, return the key corresponding to the largest
131  value.  Returns 'default' if the dict is empty."""
132  x = [(v, k) for (k, v) in d.iteritems()]
133  if not x: return default
134  x.sort()
135  return x[-1][1]
136
137
138def IsSymlink(info):
139  """Return true if the zipfile.ZipInfo object passed in represents a
140  symlink."""
141  return (info.external_attr >> 16) == 0120777
142
143def IsRegular(info):
144  """Return true if the zipfile.ZipInfo object passed in represents a
145  symlink."""
146  return (info.external_attr >> 28) == 010
147
148def ClosestFileMatch(src, tgtfiles, existing):
149  """Returns the closest file match between a source file and list
150     of potential matches.  The exact filename match is preferred,
151     then the sha1 is searched for, and finally a file with the same
152     basename is evaluated.  Rename support in the updater-binary is
153     required for the latter checks to be used."""
154
155  result = tgtfiles.get("path:" + src.name)
156  if result is not None:
157    return result
158
159  if not OPTIONS.target_info_dict.get("update_rename_support", False):
160    return None
161
162  if src.size < 1000:
163    return None
164
165  result = tgtfiles.get("sha1:" + src.sha1)
166  if result is not None and existing.get(result.name) is None:
167    return result
168  result = tgtfiles.get("file:" + src.name.split("/")[-1])
169  if result is not None and existing.get(result.name) is None:
170    return result
171  return None
172
173class ItemSet:
174  def __init__(self, partition, fs_config):
175    self.partition = partition
176    self.fs_config = fs_config
177    self.ITEMS = {}
178
179  def Get(self, name, dir=False):
180    if name not in self.ITEMS:
181      self.ITEMS[name] = Item(self, name, dir=dir)
182    return self.ITEMS[name]
183
184  def GetMetadata(self, input_zip):
185    # The target_files contains a record of what the uid,
186    # gid, and mode are supposed to be.
187    output = input_zip.read(self.fs_config)
188
189    for line in output.split("\n"):
190      if not line: continue
191      columns = line.split()
192      name, uid, gid, mode = columns[:4]
193      selabel = None
194      capabilities = None
195
196      # After the first 4 columns, there are a series of key=value
197      # pairs. Extract out the fields we care about.
198      for element in columns[4:]:
199        key, value = element.split("=")
200        if key == "selabel":
201          selabel = value
202        if key == "capabilities":
203          capabilities = value
204
205      i = self.ITEMS.get(name, None)
206      if i is not None:
207        i.uid = int(uid)
208        i.gid = int(gid)
209        i.mode = int(mode, 8)
210        i.selabel = selabel
211        i.capabilities = capabilities
212        if i.dir:
213          i.children.sort(key=lambda i: i.name)
214
215    # set metadata for the files generated by this script.
216    i = self.ITEMS.get("system/recovery-from-boot.p", None)
217    if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0644, None, None
218    i = self.ITEMS.get("system/etc/install-recovery.sh", None)
219    if i: i.uid, i.gid, i.mode, i.selabel, i.capabilities = 0, 0, 0544, None, None
220
221
222class Item:
223  """Items represent the metadata (user, group, mode) of files and
224  directories in the system image."""
225  def __init__(self, itemset, name, dir=False):
226    self.itemset = itemset
227    self.name = name
228    self.uid = None
229    self.gid = None
230    self.mode = None
231    self.selabel = None
232    self.capabilities = None
233    self.dir = dir
234
235    if name:
236      self.parent = itemset.Get(os.path.dirname(name), dir=True)
237      self.parent.children.append(self)
238    else:
239      self.parent = None
240    if dir:
241      self.children = []
242
243  def Dump(self, indent=0):
244    if self.uid is not None:
245      print "%s%s %d %d %o" % ("  "*indent, self.name, self.uid, self.gid, self.mode)
246    else:
247      print "%s%s %s %s %s" % ("  "*indent, self.name, self.uid, self.gid, self.mode)
248    if self.dir:
249      print "%s%s" % ("  "*indent, self.descendants)
250      print "%s%s" % ("  "*indent, self.best_subtree)
251      for i in self.children:
252        i.Dump(indent=indent+1)
253
254  def CountChildMetadata(self):
255    """Count up the (uid, gid, mode, selabel, capabilities) tuples for
256    all children and determine the best strategy for using set_perm_recursive and
257    set_perm to correctly chown/chmod all the files to their desired
258    values.  Recursively calls itself for all descendants.
259
260    Returns a dict of {(uid, gid, dmode, fmode, selabel, capabilities): count} counting up
261    all descendants of this node.  (dmode or fmode may be None.)  Also
262    sets the best_subtree of each directory Item to the (uid, gid,
263    dmode, fmode, selabel, capabilities) tuple that will match the most
264    descendants of that Item.
265    """
266
267    assert self.dir
268    d = self.descendants = {(self.uid, self.gid, self.mode, None, self.selabel, self.capabilities): 1}
269    for i in self.children:
270      if i.dir:
271        for k, v in i.CountChildMetadata().iteritems():
272          d[k] = d.get(k, 0) + v
273      else:
274        k = (i.uid, i.gid, None, i.mode, i.selabel, i.capabilities)
275        d[k] = d.get(k, 0) + 1
276
277    # Find the (uid, gid, dmode, fmode, selabel, capabilities)
278    # tuple that matches the most descendants.
279
280    # First, find the (uid, gid) pair that matches the most
281    # descendants.
282    ug = {}
283    for (uid, gid, _, _, _, _), count in d.iteritems():
284      ug[(uid, gid)] = ug.get((uid, gid), 0) + count
285    ug = MostPopularKey(ug, (0, 0))
286
287    # Now find the dmode, fmode, selabel, and capabilities that match
288    # the most descendants with that (uid, gid), and choose those.
289    best_dmode = (0, 0755)
290    best_fmode = (0, 0644)
291    best_selabel = (0, None)
292    best_capabilities = (0, None)
293    for k, count in d.iteritems():
294      if k[:2] != ug: continue
295      if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2])
296      if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3])
297      if k[4] is not None and count >= best_selabel[0]: best_selabel = (count, k[4])
298      if k[5] is not None and count >= best_capabilities[0]: best_capabilities = (count, k[5])
299    self.best_subtree = ug + (best_dmode[1], best_fmode[1], best_selabel[1], best_capabilities[1])
300
301    return d
302
303  def SetPermissions(self, script):
304    """Append set_perm/set_perm_recursive commands to 'script' to
305    set all permissions, users, and groups for the tree of files
306    rooted at 'self'."""
307
308    self.CountChildMetadata()
309
310    def recurse(item, current):
311      # current is the (uid, gid, dmode, fmode, selabel, capabilities) tuple that the current
312      # item (and all its children) have already been set to.  We only
313      # need to issue set_perm/set_perm_recursive commands if we're
314      # supposed to be something different.
315      if item.dir:
316        if current != item.best_subtree:
317          script.SetPermissionsRecursive("/"+item.name, *item.best_subtree)
318          current = item.best_subtree
319
320        if item.uid != current[0] or item.gid != current[1] or \
321           item.mode != current[2] or item.selabel != current[4] or \
322           item.capabilities != current[5]:
323          script.SetPermissions("/"+item.name, item.uid, item.gid,
324                                item.mode, item.selabel, item.capabilities)
325
326        for i in item.children:
327          recurse(i, current)
328      else:
329        if item.uid != current[0] or item.gid != current[1] or \
330               item.mode != current[3] or item.selabel != current[4] or \
331               item.capabilities != current[5]:
332          script.SetPermissions("/"+item.name, item.uid, item.gid,
333                                item.mode, item.selabel, item.capabilities)
334
335    recurse(self, (-1, -1, -1, -1, None, None))
336
337
338def CopyPartitionFiles(itemset, input_zip, output_zip=None, substitute=None):
339  """Copies files for the partition in the input zip to the output
340  zip.  Populates the Item class with their metadata, and returns a
341  list of symlinks.  output_zip may be None, in which case the copy is
342  skipped (but the other side effects still happen).  substitute is an
343  optional dict of {output filename: contents} to be output instead of
344  certain input files.
345  """
346
347  symlinks = []
348
349  partition = itemset.partition
350
351  for info in input_zip.infolist():
352    if info.filename.startswith(partition.upper() + "/"):
353      basefilename = info.filename[7:]
354      if IsSymlink(info):
355        symlinks.append((input_zip.read(info.filename),
356                         "/" + partition + "/" + basefilename))
357      else:
358        info2 = copy.copy(info)
359        fn = info2.filename = partition + "/" + basefilename
360        if substitute and fn in substitute and substitute[fn] is None:
361          continue
362        if output_zip is not None:
363          if substitute and fn in substitute:
364            data = substitute[fn]
365          else:
366            data = input_zip.read(info.filename)
367          output_zip.writestr(info2, data)
368        if fn.endswith("/"):
369          itemset.Get(fn[:-1], dir=True)
370        else:
371          itemset.Get(fn, dir=False)
372
373  symlinks.sort()
374  return symlinks
375
376
377def SignOutput(temp_zip_name, output_zip_name):
378  key_passwords = common.GetKeyPasswords([OPTIONS.package_key])
379  pw = key_passwords[OPTIONS.package_key]
380
381  common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw,
382                  whole_file=True)
383
384
385def AppendAssertions(script, info_dict, oem_dict = None):
386  oem_props = info_dict.get("oem_fingerprint_properties")
387  if oem_props is None or len(oem_props) == 0:
388    device = GetBuildProp("ro.product.device", info_dict)
389    script.AssertDevice(device)
390  else:
391    if oem_dict is None:
392      raise common.ExternalError("No OEM file provided to answer expected assertions")
393    for prop in oem_props.split():
394      if oem_dict.get(prop) is None:
395        raise common.ExternalError("The OEM file is missing the property %s" % prop)
396      script.AssertOemProperty(prop, oem_dict.get(prop))
397
398
399def HasRecoveryPatch(target_files_zip):
400  try:
401    target_files_zip.getinfo("SYSTEM/recovery-from-boot.p")
402    return True
403  except KeyError:
404    return False
405
406def HasVendorPartition(target_files_zip):
407  try:
408    target_files_zip.getinfo("VENDOR/")
409    return True
410  except KeyError:
411    return False
412
413def GetOemProperty(name, oem_props, oem_dict, info_dict):
414  if oem_props is not None and name in oem_props:
415    return oem_dict[name]
416  return GetBuildProp(name, info_dict)
417
418
419def CalculateFingerprint(oem_props, oem_dict, info_dict):
420  if oem_props is None:
421    return GetBuildProp("ro.build.fingerprint", info_dict)
422  return "%s/%s/%s:%s" % (
423    GetOemProperty("ro.product.brand", oem_props, oem_dict, info_dict),
424    GetOemProperty("ro.product.name", oem_props, oem_dict, info_dict),
425    GetOemProperty("ro.product.device", oem_props, oem_dict, info_dict),
426    GetBuildProp("ro.build.thumbprint", info_dict))
427
428
429def GetImage(which, tmpdir, info_dict):
430  # Return an image object (suitable for passing to BlockImageDiff)
431  # for the 'which' partition (most be "system" or "vendor").  If a
432  # prebuilt image and file map are found in tmpdir they are used,
433  # otherwise they are reconstructed from the individual files.
434
435  assert which in ("system", "vendor")
436
437  path = os.path.join(tmpdir, "IMAGES", which + ".img")
438  mappath = os.path.join(tmpdir, "IMAGES", which + ".map")
439  if os.path.exists(path) and os.path.exists(mappath):
440    print "using %s.img from target-files" % (which,)
441    # This is a 'new' target-files, which already has the image in it.
442
443  else:
444    print "building %s.img from target-files" % (which,)
445
446    # This is an 'old' target-files, which does not contain images
447    # already built.  Build them.
448
449    mappath = tempfile.mkstemp()[1]
450    OPTIONS.tempfiles.append(mappath)
451
452    import add_img_to_target_files
453    if which == "system":
454      path = add_img_to_target_files.BuildSystem(
455          tmpdir, info_dict, block_list=mappath)
456    elif which == "vendor":
457      path = add_img_to_target_files.BuildVendor(
458          tmpdir, info_dict, block_list=mappath)
459
460  return sparse_img.SparseImage(path, mappath)
461
462
463def WriteFullOTAPackage(input_zip, output_zip):
464  # TODO: how to determine this?  We don't know what version it will
465  # be installed on top of.  For now, we expect the API just won't
466  # change very often.
467  script = edify_generator.EdifyGenerator(3, OPTIONS.info_dict)
468
469  oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties")
470  recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options")
471  oem_dict = None
472  if oem_props is not None and len(oem_props) > 0:
473    if OPTIONS.oem_source is None:
474      raise common.ExternalError("OEM source required for this build")
475    script.Mount("/oem", recovery_mount_options)
476    oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines())
477
478  metadata = {"post-build": CalculateFingerprint(
479                               oem_props, oem_dict, OPTIONS.info_dict),
480              "pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict,
481                                         OPTIONS.info_dict),
482              "post-timestamp": GetBuildProp("ro.build.date.utc",
483                                             OPTIONS.info_dict),
484              }
485
486  device_specific = common.DeviceSpecificParams(
487      input_zip=input_zip,
488      input_version=OPTIONS.info_dict["recovery_api_version"],
489      output_zip=output_zip,
490      script=script,
491      input_tmp=OPTIONS.input_tmp,
492      metadata=metadata,
493      info_dict=OPTIONS.info_dict)
494
495  has_recovery_patch = HasRecoveryPatch(input_zip)
496  block_based = OPTIONS.block_based and has_recovery_patch
497
498  if not OPTIONS.omit_prereq:
499    ts = GetBuildProp("ro.build.date.utc", OPTIONS.info_dict)
500    ts_text = GetBuildProp("ro.build.date", OPTIONS.info_dict)
501    script.AssertOlderBuild(ts, ts_text)
502
503  AppendAssertions(script, OPTIONS.info_dict, oem_dict)
504  device_specific.FullOTA_Assertions()
505
506  # Two-step package strategy (in chronological order, which is *not*
507  # the order in which the generated script has things):
508  #
509  # if stage is not "2/3" or "3/3":
510  #    write recovery image to boot partition
511  #    set stage to "2/3"
512  #    reboot to boot partition and restart recovery
513  # else if stage is "2/3":
514  #    write recovery image to recovery partition
515  #    set stage to "3/3"
516  #    reboot to recovery partition and restart recovery
517  # else:
518  #    (stage must be "3/3")
519  #    set stage to ""
520  #    do normal full package installation:
521  #       wipe and install system, boot image, etc.
522  #       set up system to update recovery partition on first boot
523  #    complete script normally (allow recovery to mark itself finished and reboot)
524
525  recovery_img = common.GetBootableImage("recovery.img", "recovery.img",
526                                         OPTIONS.input_tmp, "RECOVERY")
527  if OPTIONS.two_step:
528    if not OPTIONS.info_dict.get("multistage_support", None):
529      assert False, "two-step packages not supported by this build"
530    fs = OPTIONS.info_dict["fstab"]["/misc"]
531    assert fs.fs_type.upper() == "EMMC", \
532        "two-step packages only supported on devices with EMMC /misc partitions"
533    bcb_dev = {"bcb_dev": fs.device}
534    common.ZipWriteStr(output_zip, "recovery.img", recovery_img.data)
535    script.AppendExtra("""
536if get_stage("%(bcb_dev)s") == "2/3" then
537""" % bcb_dev)
538    script.WriteRawImage("/recovery", "recovery.img")
539    script.AppendExtra("""
540set_stage("%(bcb_dev)s", "3/3");
541reboot_now("%(bcb_dev)s", "recovery");
542else if get_stage("%(bcb_dev)s") == "3/3" then
543""" % bcb_dev)
544
545  device_specific.FullOTA_InstallBegin()
546
547  system_progress = 0.75
548
549  if OPTIONS.wipe_user_data:
550    system_progress -= 0.1
551  if HasVendorPartition(input_zip):
552    system_progress -= 0.1
553
554  if "selinux_fc" in OPTIONS.info_dict:
555    WritePolicyConfig(OPTIONS.info_dict["selinux_fc"], output_zip)
556
557  recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options")
558
559  system_items = ItemSet("system", "META/filesystem_config.txt")
560  script.ShowProgress(system_progress, 0)
561
562  if block_based:
563    # Full OTA is done as an "incremental" against an empty source
564    # image.  This has the effect of writing new data from the package
565    # to the entire partition, but lets us reuse the updater code that
566    # writes incrementals to do it.
567    system_tgt = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict)
568    system_tgt.ResetFileMap()
569    system_diff = common.BlockDifference("system", system_tgt, src=None)
570    system_diff.WriteScript(script, output_zip)
571  else:
572    script.FormatPartition("/system")
573    script.Mount("/system", recovery_mount_options)
574    if not has_recovery_patch:
575      script.UnpackPackageDir("recovery", "/system")
576    script.UnpackPackageDir("system", "/system")
577
578    symlinks = CopyPartitionFiles(system_items, input_zip, output_zip)
579    script.MakeSymlinks(symlinks)
580
581  boot_img = common.GetBootableImage("boot.img", "boot.img",
582                                     OPTIONS.input_tmp, "BOOT")
583
584  if not block_based:
585    def output_sink(fn, data):
586      common.ZipWriteStr(output_zip, "recovery/" + fn, data)
587      system_items.Get("system/" + fn, dir=False)
588
589    common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink,
590                             recovery_img, boot_img)
591
592    system_items.GetMetadata(input_zip)
593    system_items.Get("system").SetPermissions(script)
594
595  if HasVendorPartition(input_zip):
596    vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt")
597    script.ShowProgress(0.1, 0)
598
599    if block_based:
600      vendor_tgt = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict)
601      vendor_tgt.ResetFileMap()
602      vendor_diff = common.BlockDifference("vendor", vendor_tgt)
603      vendor_diff.WriteScript(script, output_zip)
604    else:
605      script.FormatPartition("/vendor")
606      script.Mount("/vendor", recovery_mount_options)
607      script.UnpackPackageDir("vendor", "/vendor")
608
609      symlinks = CopyPartitionFiles(vendor_items, input_zip, output_zip)
610      script.MakeSymlinks(symlinks)
611
612      vendor_items.GetMetadata(input_zip)
613      vendor_items.Get("vendor").SetPermissions(script)
614
615  common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict)
616  common.ZipWriteStr(output_zip, "boot.img", boot_img.data)
617
618  script.ShowProgress(0.05, 5)
619  script.WriteRawImage("/boot", "boot.img")
620
621  script.ShowProgress(0.2, 10)
622  device_specific.FullOTA_InstallEnd()
623
624  if OPTIONS.extra_script is not None:
625    script.AppendExtra(OPTIONS.extra_script)
626
627  script.UnmountAll()
628
629  if OPTIONS.wipe_user_data:
630    script.ShowProgress(0.1, 10)
631    script.FormatPartition("/data")
632
633  if OPTIONS.two_step:
634    script.AppendExtra("""
635set_stage("%(bcb_dev)s", "");
636""" % bcb_dev)
637    script.AppendExtra("else\n")
638    script.WriteRawImage("/boot", "recovery.img")
639    script.AppendExtra("""
640set_stage("%(bcb_dev)s", "2/3");
641reboot_now("%(bcb_dev)s", "");
642endif;
643endif;
644""" % bcb_dev)
645  script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary)
646  WriteMetadata(metadata, output_zip)
647
648
649def WritePolicyConfig(file_context, output_zip):
650  f = open(file_context, 'r');
651  basename = os.path.basename(file_context)
652  common.ZipWriteStr(output_zip, basename, f.read())
653
654
655def WriteMetadata(metadata, output_zip):
656  common.ZipWriteStr(output_zip, "META-INF/com/android/metadata",
657                     "".join(["%s=%s\n" % kv
658                              for kv in sorted(metadata.iteritems())]))
659
660
661def LoadPartitionFiles(z, partition):
662  """Load all the files from the given partition in a given target-files
663  ZipFile, and return a dict of {filename: File object}."""
664  out = {}
665  prefix = partition.upper() + "/"
666  for info in z.infolist():
667    if info.filename.startswith(prefix) and not IsSymlink(info):
668      basefilename = info.filename[7:]
669      fn = partition + "/" + basefilename
670      data = z.read(info.filename)
671      out[fn] = common.File(fn, data)
672  return out
673
674
675def GetBuildProp(prop, info_dict):
676  """Return the fingerprint of the build of a given target-files info_dict."""
677  try:
678    return info_dict.get("build.prop", {})[prop]
679  except KeyError:
680    raise common.ExternalError("couldn't find %s in build.prop" % (prop,))
681
682
683def AddToKnownPaths(filename, known_paths):
684  if filename[-1] == "/":
685    return
686  dirs = filename.split("/")[:-1]
687  while len(dirs) > 0:
688    path = "/".join(dirs)
689    if path in known_paths:
690      break;
691    known_paths.add(path)
692    dirs.pop()
693
694
695def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip):
696  source_version = OPTIONS.source_info_dict["recovery_api_version"]
697  target_version = OPTIONS.target_info_dict["recovery_api_version"]
698
699  if source_version == 0:
700    print ("WARNING: generating edify script for a source that "
701           "can't install it.")
702  script = edify_generator.EdifyGenerator(source_version,
703                                          OPTIONS.target_info_dict)
704
705  metadata = {"pre-device": GetBuildProp("ro.product.device",
706                                         OPTIONS.source_info_dict),
707              "post-timestamp": GetBuildProp("ro.build.date.utc",
708                                             OPTIONS.target_info_dict),
709              }
710
711  device_specific = common.DeviceSpecificParams(
712      source_zip=source_zip,
713      source_version=source_version,
714      target_zip=target_zip,
715      target_version=target_version,
716      output_zip=output_zip,
717      script=script,
718      metadata=metadata,
719      info_dict=OPTIONS.info_dict)
720
721  source_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.source_info_dict)
722  target_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.target_info_dict)
723  metadata["pre-build"] = source_fp
724  metadata["post-build"] = target_fp
725
726  source_boot = common.GetBootableImage(
727      "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT",
728      OPTIONS.source_info_dict)
729  target_boot = common.GetBootableImage(
730      "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT")
731  updating_boot = (not OPTIONS.two_step and
732                   (source_boot.data != target_boot.data))
733
734  source_recovery = common.GetBootableImage(
735      "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY",
736      OPTIONS.source_info_dict)
737  target_recovery = common.GetBootableImage(
738      "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY")
739  updating_recovery = (source_recovery.data != target_recovery.data)
740
741  system_src = GetImage("system", OPTIONS.source_tmp, OPTIONS.source_info_dict)
742  system_tgt = GetImage("system", OPTIONS.target_tmp, OPTIONS.target_info_dict)
743  system_diff = common.BlockDifference("system", system_tgt, system_src,
744                                       check_first_block=True)
745
746  if HasVendorPartition(target_zip):
747    if not HasVendorPartition(source_zip):
748      raise RuntimeError("can't generate incremental that adds /vendor")
749    vendor_src = GetImage("vendor", OPTIONS.source_tmp, OPTIONS.source_info_dict)
750    vendor_tgt = GetImage("vendor", OPTIONS.target_tmp, OPTIONS.target_info_dict)
751    vendor_diff = common.BlockDifference("vendor", vendor_tgt, vendor_src,
752                                         check_first_block=True)
753  else:
754    vendor_diff = None
755
756  oem_props = OPTIONS.target_info_dict.get("oem_fingerprint_properties")
757  recovery_mount_options = OPTIONS.target_info_dict.get("recovery_mount_options")
758  oem_dict = None
759  if oem_props is not None and len(oem_props) > 0:
760    if OPTIONS.oem_source is None:
761      raise common.ExternalError("OEM source required for this build")
762    script.Mount("/oem", recovery_mount_options)
763    oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines())
764
765  AppendAssertions(script, OPTIONS.target_info_dict, oem_dict)
766  device_specific.IncrementalOTA_Assertions()
767
768  # Two-step incremental package strategy (in chronological order,
769  # which is *not* the order in which the generated script has
770  # things):
771  #
772  # if stage is not "2/3" or "3/3":
773  #    do verification on current system
774  #    write recovery image to boot partition
775  #    set stage to "2/3"
776  #    reboot to boot partition and restart recovery
777  # else if stage is "2/3":
778  #    write recovery image to recovery partition
779  #    set stage to "3/3"
780  #    reboot to recovery partition and restart recovery
781  # else:
782  #    (stage must be "3/3")
783  #    perform update:
784  #       patch system files, etc.
785  #       force full install of new boot image
786  #       set up system to update recovery partition on first boot
787  #    complete script normally (allow recovery to mark itself finished and reboot)
788
789  if OPTIONS.two_step:
790    if not OPTIONS.info_dict.get("multistage_support", None):
791      assert False, "two-step packages not supported by this build"
792    fs = OPTIONS.info_dict["fstab"]["/misc"]
793    assert fs.fs_type.upper() == "EMMC", \
794        "two-step packages only supported on devices with EMMC /misc partitions"
795    bcb_dev = {"bcb_dev": fs.device}
796    common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data)
797    script.AppendExtra("""
798if get_stage("%(bcb_dev)s") == "2/3" then
799""" % bcb_dev)
800    script.AppendExtra("sleep(20);\n");
801    script.WriteRawImage("/recovery", "recovery.img")
802    script.AppendExtra("""
803set_stage("%(bcb_dev)s", "3/3");
804reboot_now("%(bcb_dev)s", "recovery");
805else if get_stage("%(bcb_dev)s") != "3/3" then
806""" % bcb_dev)
807
808  script.Print("Verifying current system...")
809
810  device_specific.IncrementalOTA_VerifyBegin()
811
812  if oem_props is None:
813    script.AssertSomeFingerprint(source_fp, target_fp)
814  else:
815    script.AssertSomeThumbprint(
816        GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict),
817        GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict))
818
819  if updating_boot:
820    boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict)
821    d = common.Difference(target_boot, source_boot)
822    _, _, d = d.ComputePatch()
823    if d is None:
824      include_full_boot = True
825      common.ZipWriteStr(output_zip, "boot.img", target_boot.data)
826    else:
827      include_full_boot = False
828
829      print "boot      target: %d  source: %d  diff: %d" % (
830          target_boot.size, source_boot.size, len(d))
831
832      common.ZipWriteStr(output_zip, "patch/boot.img.p", d)
833
834      script.PatchCheck("%s:%s:%d:%s:%d:%s" %
835                        (boot_type, boot_device,
836                         source_boot.size, source_boot.sha1,
837                         target_boot.size, target_boot.sha1))
838
839  device_specific.IncrementalOTA_VerifyEnd()
840
841  if OPTIONS.two_step:
842    script.WriteRawImage("/boot", "recovery.img")
843    script.AppendExtra("""
844set_stage("%(bcb_dev)s", "2/3");
845reboot_now("%(bcb_dev)s", "");
846else
847""" % bcb_dev)
848
849  # Verify the existing partitions.
850  system_diff.WriteVerifyScript(script)
851  if vendor_diff:
852    vendor_diff.WriteVerifyScript(script)
853
854  script.Comment("---- start making changes here ----")
855
856  device_specific.IncrementalOTA_InstallBegin()
857
858  system_diff.WriteScript(script, output_zip,
859                          progress=0.8 if vendor_diff else 0.9)
860  if vendor_diff:
861    vendor_diff.WriteScript(script, output_zip, progress=0.1)
862
863  if OPTIONS.two_step:
864    common.ZipWriteStr(output_zip, "boot.img", target_boot.data)
865    script.WriteRawImage("/boot", "boot.img")
866    print "writing full boot image (forced by two-step mode)"
867
868  if not OPTIONS.two_step:
869    if updating_boot:
870      if include_full_boot:
871        print "boot image changed; including full."
872        script.Print("Installing boot image...")
873        script.WriteRawImage("/boot", "boot.img")
874      else:
875        # Produce the boot image by applying a patch to the current
876        # contents of the boot partition, and write it back to the
877        # partition.
878        print "boot image changed; including patch."
879        script.Print("Patching boot image...")
880        script.ShowProgress(0.1, 10)
881        script.ApplyPatch("%s:%s:%d:%s:%d:%s"
882                          % (boot_type, boot_device,
883                             source_boot.size, source_boot.sha1,
884                             target_boot.size, target_boot.sha1),
885                          "-",
886                          target_boot.size, target_boot.sha1,
887                          source_boot.sha1, "patch/boot.img.p")
888    else:
889      print "boot image unchanged; skipping."
890
891  # Do device-specific installation (eg, write radio image).
892  device_specific.IncrementalOTA_InstallEnd()
893
894  if OPTIONS.extra_script is not None:
895    script.AppendExtra(OPTIONS.extra_script)
896
897  if OPTIONS.wipe_user_data:
898    script.Print("Erasing user data...")
899    script.FormatPartition("/data")
900
901  if OPTIONS.two_step:
902    script.AppendExtra("""
903set_stage("%(bcb_dev)s", "");
904endif;
905endif;
906""" % bcb_dev)
907
908  script.SetProgress(1)
909  script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary)
910  WriteMetadata(metadata, output_zip)
911
912
913class FileDifference:
914  def __init__(self, partition, source_zip, target_zip, output_zip):
915    print "Loading target..."
916    self.target_data = target_data = LoadPartitionFiles(target_zip, partition)
917    print "Loading source..."
918    self.source_data = source_data = LoadPartitionFiles(source_zip, partition)
919
920    self.verbatim_targets = verbatim_targets = []
921    self.patch_list = patch_list = []
922    diffs = []
923    self.renames = renames = {}
924    known_paths = set()
925    largest_source_size = 0
926
927    matching_file_cache = {}
928    for fn, sf in source_data.items():
929      assert fn == sf.name
930      matching_file_cache["path:" + fn] = sf
931      if fn in target_data.keys():
932        AddToKnownPaths(fn, known_paths)
933      # Only allow eligibility for filename/sha matching
934      # if there isn't a perfect path match.
935      if target_data.get(sf.name) is None:
936        matching_file_cache["file:" + fn.split("/")[-1]] = sf
937        matching_file_cache["sha:" + sf.sha1] = sf
938
939    for fn in sorted(target_data.keys()):
940      tf = target_data[fn]
941      assert fn == tf.name
942      sf = ClosestFileMatch(tf, matching_file_cache, renames)
943      if sf is not None and sf.name != tf.name:
944        print "File has moved from " + sf.name + " to " + tf.name
945        renames[sf.name] = tf
946
947      if sf is None or fn in OPTIONS.require_verbatim:
948        # This file should be included verbatim
949        if fn in OPTIONS.prohibit_verbatim:
950          raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,))
951        print "send", fn, "verbatim"
952        tf.AddToZip(output_zip)
953        verbatim_targets.append((fn, tf.size, tf.sha1))
954        if fn in target_data.keys():
955          AddToKnownPaths(fn, known_paths)
956      elif tf.sha1 != sf.sha1:
957        # File is different; consider sending as a patch
958        diffs.append(common.Difference(tf, sf))
959      else:
960        # Target file data identical to source (may still be renamed)
961        pass
962
963    common.ComputeDifferences(diffs)
964
965    for diff in diffs:
966      tf, sf, d = diff.GetPatch()
967      path = "/".join(tf.name.split("/")[:-1])
968      if d is None or len(d) > tf.size * OPTIONS.patch_threshold or \
969          path not in known_paths:
970        # patch is almost as big as the file; don't bother patching
971        # or a patch + rename cannot take place due to the target
972        # directory not existing
973        tf.AddToZip(output_zip)
974        verbatim_targets.append((tf.name, tf.size, tf.sha1))
975        if sf.name in renames:
976          del renames[sf.name]
977        AddToKnownPaths(tf.name, known_paths)
978      else:
979        common.ZipWriteStr(output_zip, "patch/" + sf.name + ".p", d)
980        patch_list.append((tf, sf, tf.size, common.sha1(d).hexdigest()))
981        largest_source_size = max(largest_source_size, sf.size)
982
983    self.largest_source_size = largest_source_size
984
985  def EmitVerification(self, script):
986    so_far = 0
987    for tf, sf, size, patch_sha in self.patch_list:
988      if tf.name != sf.name:
989        script.SkipNextActionIfTargetExists(tf.name, tf.sha1)
990      script.PatchCheck("/"+sf.name, tf.sha1, sf.sha1)
991      so_far += sf.size
992    return so_far
993
994  def EmitExplicitTargetVerification(self, script):
995    for fn, size, sha1 in self.verbatim_targets:
996      if (fn[-1] != "/"):
997        script.FileCheck("/"+fn, sha1)
998    for tf, _, _, _ in self.patch_list:
999      script.FileCheck(tf.name, tf.sha1)
1000
1001  def RemoveUnneededFiles(self, script, extras=()):
1002    script.DeleteFiles(["/"+i[0] for i in self.verbatim_targets] +
1003                       ["/"+i for i in sorted(self.source_data)
1004                              if i not in self.target_data and
1005                              i not in self.renames] +
1006                       list(extras))
1007
1008  def TotalPatchSize(self):
1009    return sum(i[1].size for i in self.patch_list)
1010
1011  def EmitPatches(self, script, total_patch_size, so_far):
1012    self.deferred_patch_list = deferred_patch_list = []
1013    for item in self.patch_list:
1014      tf, sf, size, _ = item
1015      if tf.name == "system/build.prop":
1016        deferred_patch_list.append(item)
1017        continue
1018      if (sf.name != tf.name):
1019        script.SkipNextActionIfTargetExists(tf.name, tf.sha1)
1020      script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, "patch/"+sf.name+".p")
1021      so_far += tf.size
1022      script.SetProgress(so_far / total_patch_size)
1023    return so_far
1024
1025  def EmitDeferredPatches(self, script):
1026    for item in self.deferred_patch_list:
1027      tf, sf, size, _ = item
1028      script.ApplyPatch("/"+sf.name, "-", tf.size, tf.sha1, sf.sha1, "patch/"+sf.name+".p")
1029    script.SetPermissions("/system/build.prop", 0, 0, 0644, None, None)
1030
1031  def EmitRenames(self, script):
1032    if len(self.renames) > 0:
1033      script.Print("Renaming files...")
1034      for src, tgt in self.renames.iteritems():
1035        print "Renaming " + src + " to " + tgt.name
1036        script.RenameFile(src, tgt.name)
1037
1038
1039
1040
1041def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
1042  target_has_recovery_patch = HasRecoveryPatch(target_zip)
1043  source_has_recovery_patch = HasRecoveryPatch(source_zip)
1044
1045  if (OPTIONS.block_based and
1046      target_has_recovery_patch and
1047      source_has_recovery_patch):
1048    return WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip)
1049
1050  source_version = OPTIONS.source_info_dict["recovery_api_version"]
1051  target_version = OPTIONS.target_info_dict["recovery_api_version"]
1052
1053  if source_version == 0:
1054    print ("WARNING: generating edify script for a source that "
1055           "can't install it.")
1056  script = edify_generator.EdifyGenerator(source_version,
1057                                          OPTIONS.target_info_dict)
1058
1059  oem_props = OPTIONS.info_dict.get("oem_fingerprint_properties")
1060  recovery_mount_options = OPTIONS.info_dict.get("recovery_mount_options")
1061  oem_dict = None
1062  if oem_props is not None and len(oem_props) > 0:
1063    if OPTIONS.oem_source is None:
1064      raise common.ExternalError("OEM source required for this build")
1065    script.Mount("/oem", recovery_mount_options)
1066    oem_dict = common.LoadDictionaryFromLines(open(OPTIONS.oem_source).readlines())
1067
1068  metadata = {"pre-device": GetOemProperty("ro.product.device", oem_props, oem_dict,
1069                                         OPTIONS.source_info_dict),
1070              "post-timestamp": GetBuildProp("ro.build.date.utc",
1071                                             OPTIONS.target_info_dict),
1072              }
1073
1074  device_specific = common.DeviceSpecificParams(
1075      source_zip=source_zip,
1076      source_version=source_version,
1077      target_zip=target_zip,
1078      target_version=target_version,
1079      output_zip=output_zip,
1080      script=script,
1081      metadata=metadata,
1082      info_dict=OPTIONS.info_dict)
1083
1084  system_diff = FileDifference("system", source_zip, target_zip, output_zip)
1085  script.Mount("/system", recovery_mount_options)
1086  if HasVendorPartition(target_zip):
1087    vendor_diff = FileDifference("vendor", source_zip, target_zip, output_zip)
1088    script.Mount("/vendor", recovery_mount_options)
1089  else:
1090    vendor_diff = None
1091
1092  target_fp = CalculateFingerprint(oem_props, oem_dict, OPTIONS.target_info_dict)
1093  source_fp = CalculateFingerprint(oem_props, oem_dict, OPTIONS.source_info_dict)
1094
1095  if oem_props is None:
1096    script.AssertSomeFingerprint(source_fp, target_fp)
1097  else:
1098    script.AssertSomeThumbprint(
1099        GetBuildProp("ro.build.thumbprint", OPTIONS.target_info_dict),
1100        GetBuildProp("ro.build.thumbprint", OPTIONS.source_info_dict))
1101
1102  metadata["pre-build"] = source_fp
1103  metadata["post-build"] = target_fp
1104
1105  source_boot = common.GetBootableImage(
1106      "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT",
1107      OPTIONS.source_info_dict)
1108  target_boot = common.GetBootableImage(
1109      "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT")
1110  updating_boot = (not OPTIONS.two_step and
1111                   (source_boot.data != target_boot.data))
1112
1113  source_recovery = common.GetBootableImage(
1114      "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY",
1115      OPTIONS.source_info_dict)
1116  target_recovery = common.GetBootableImage(
1117      "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY")
1118  updating_recovery = (source_recovery.data != target_recovery.data)
1119
1120  # Here's how we divide up the progress bar:
1121  #  0.1 for verifying the start state (PatchCheck calls)
1122  #  0.8 for applying patches (ApplyPatch calls)
1123  #  0.1 for unpacking verbatim files, symlinking, and doing the
1124  #      device-specific commands.
1125
1126  AppendAssertions(script, OPTIONS.target_info_dict, oem_dict)
1127  device_specific.IncrementalOTA_Assertions()
1128
1129  # Two-step incremental package strategy (in chronological order,
1130  # which is *not* the order in which the generated script has
1131  # things):
1132  #
1133  # if stage is not "2/3" or "3/3":
1134  #    do verification on current system
1135  #    write recovery image to boot partition
1136  #    set stage to "2/3"
1137  #    reboot to boot partition and restart recovery
1138  # else if stage is "2/3":
1139  #    write recovery image to recovery partition
1140  #    set stage to "3/3"
1141  #    reboot to recovery partition and restart recovery
1142  # else:
1143  #    (stage must be "3/3")
1144  #    perform update:
1145  #       patch system files, etc.
1146  #       force full install of new boot image
1147  #       set up system to update recovery partition on first boot
1148  #    complete script normally (allow recovery to mark itself finished and reboot)
1149
1150  if OPTIONS.two_step:
1151    if not OPTIONS.info_dict.get("multistage_support", None):
1152      assert False, "two-step packages not supported by this build"
1153    fs = OPTIONS.info_dict["fstab"]["/misc"]
1154    assert fs.fs_type.upper() == "EMMC", \
1155        "two-step packages only supported on devices with EMMC /misc partitions"
1156    bcb_dev = {"bcb_dev": fs.device}
1157    common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data)
1158    script.AppendExtra("""
1159if get_stage("%(bcb_dev)s") == "2/3" then
1160""" % bcb_dev)
1161    script.AppendExtra("sleep(20);\n");
1162    script.WriteRawImage("/recovery", "recovery.img")
1163    script.AppendExtra("""
1164set_stage("%(bcb_dev)s", "3/3");
1165reboot_now("%(bcb_dev)s", "recovery");
1166else if get_stage("%(bcb_dev)s") != "3/3" then
1167""" % bcb_dev)
1168
1169  script.Print("Verifying current system...")
1170
1171  device_specific.IncrementalOTA_VerifyBegin()
1172
1173  script.ShowProgress(0.1, 0)
1174  so_far = system_diff.EmitVerification(script)
1175  if vendor_diff:
1176    so_far += vendor_diff.EmitVerification(script)
1177
1178  if updating_boot:
1179    d = common.Difference(target_boot, source_boot)
1180    _, _, d = d.ComputePatch()
1181    print "boot      target: %d  source: %d  diff: %d" % (
1182        target_boot.size, source_boot.size, len(d))
1183
1184    common.ZipWriteStr(output_zip, "patch/boot.img.p", d)
1185
1186    boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict)
1187
1188    script.PatchCheck("%s:%s:%d:%s:%d:%s" %
1189                      (boot_type, boot_device,
1190                       source_boot.size, source_boot.sha1,
1191                       target_boot.size, target_boot.sha1))
1192    so_far += source_boot.size
1193
1194  size = []
1195  if system_diff.patch_list: size.append(system_diff.largest_source_size)
1196  if vendor_diff:
1197    if vendor_diff.patch_list: size.append(vendor_diff.largest_source_size)
1198  if size or updating_recovery or updating_boot:
1199    script.CacheFreeSpaceCheck(max(size))
1200
1201  device_specific.IncrementalOTA_VerifyEnd()
1202
1203  if OPTIONS.two_step:
1204    script.WriteRawImage("/boot", "recovery.img")
1205    script.AppendExtra("""
1206set_stage("%(bcb_dev)s", "2/3");
1207reboot_now("%(bcb_dev)s", "");
1208else
1209""" % bcb_dev)
1210
1211  script.Comment("---- start making changes here ----")
1212
1213  device_specific.IncrementalOTA_InstallBegin()
1214
1215  if OPTIONS.two_step:
1216    common.ZipWriteStr(output_zip, "boot.img", target_boot.data)
1217    script.WriteRawImage("/boot", "boot.img")
1218    print "writing full boot image (forced by two-step mode)"
1219
1220  script.Print("Removing unneeded files...")
1221  system_diff.RemoveUnneededFiles(script, ("/system/recovery.img",))
1222  if vendor_diff:
1223    vendor_diff.RemoveUnneededFiles(script)
1224
1225  script.ShowProgress(0.8, 0)
1226  total_patch_size = 1.0 + system_diff.TotalPatchSize()
1227  if vendor_diff:
1228    total_patch_size += vendor_diff.TotalPatchSize()
1229  if updating_boot:
1230    total_patch_size += target_boot.size
1231
1232  script.Print("Patching system files...")
1233  so_far = system_diff.EmitPatches(script, total_patch_size, 0)
1234  if vendor_diff:
1235    script.Print("Patching vendor files...")
1236    so_far = vendor_diff.EmitPatches(script, total_patch_size, so_far)
1237
1238  if not OPTIONS.two_step:
1239    if updating_boot:
1240      # Produce the boot image by applying a patch to the current
1241      # contents of the boot partition, and write it back to the
1242      # partition.
1243      script.Print("Patching boot image...")
1244      script.ApplyPatch("%s:%s:%d:%s:%d:%s"
1245                        % (boot_type, boot_device,
1246                           source_boot.size, source_boot.sha1,
1247                           target_boot.size, target_boot.sha1),
1248                        "-",
1249                        target_boot.size, target_boot.sha1,
1250                        source_boot.sha1, "patch/boot.img.p")
1251      so_far += target_boot.size
1252      script.SetProgress(so_far / total_patch_size)
1253      print "boot image changed; including."
1254    else:
1255      print "boot image unchanged; skipping."
1256
1257  system_items = ItemSet("system", "META/filesystem_config.txt")
1258  if vendor_diff:
1259    vendor_items = ItemSet("vendor", "META/vendor_filesystem_config.txt")
1260
1261  if updating_recovery:
1262    # Recovery is generated as a patch using both the boot image
1263    # (which contains the same linux kernel as recovery) and the file
1264    # /system/etc/recovery-resource.dat (which contains all the images
1265    # used in the recovery UI) as sources.  This lets us minimize the
1266    # size of the patch, which must be included in every OTA package.
1267    #
1268    # For older builds where recovery-resource.dat is not present, we
1269    # use only the boot image as the source.
1270
1271    if not target_has_recovery_patch:
1272      def output_sink(fn, data):
1273        common.ZipWriteStr(output_zip, "recovery/" + fn, data)
1274        system_items.Get("system/" + fn, dir=False)
1275
1276      common.MakeRecoveryPatch(OPTIONS.target_tmp, output_sink,
1277                               target_recovery, target_boot)
1278      script.DeleteFiles(["/system/recovery-from-boot.p",
1279                          "/system/etc/install-recovery.sh"])
1280    print "recovery image changed; including as patch from boot."
1281  else:
1282    print "recovery image unchanged; skipping."
1283
1284  script.ShowProgress(0.1, 10)
1285
1286  target_symlinks = CopyPartitionFiles(system_items, target_zip, None)
1287  if vendor_diff:
1288    target_symlinks.extend(CopyPartitionFiles(vendor_items, target_zip, None))
1289
1290  temp_script = script.MakeTemporary()
1291  system_items.GetMetadata(target_zip)
1292  system_items.Get("system").SetPermissions(temp_script)
1293  if vendor_diff:
1294    vendor_items.GetMetadata(target_zip)
1295    vendor_items.Get("vendor").SetPermissions(temp_script)
1296
1297  # Note that this call will mess up the trees of Items, so make sure
1298  # we're done with them.
1299  source_symlinks = CopyPartitionFiles(system_items, source_zip, None)
1300  if vendor_diff:
1301    source_symlinks.extend(CopyPartitionFiles(vendor_items, source_zip, None))
1302
1303  target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks])
1304  source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks])
1305
1306  # Delete all the symlinks in source that aren't in target.  This
1307  # needs to happen before verbatim files are unpacked, in case a
1308  # symlink in the source is replaced by a real file in the target.
1309  to_delete = []
1310  for dest, link in source_symlinks:
1311    if link not in target_symlinks_d:
1312      to_delete.append(link)
1313  script.DeleteFiles(to_delete)
1314
1315  if system_diff.verbatim_targets:
1316    script.Print("Unpacking new system files...")
1317    script.UnpackPackageDir("system", "/system")
1318  if vendor_diff and vendor_diff.verbatim_targets:
1319    script.Print("Unpacking new vendor files...")
1320    script.UnpackPackageDir("vendor", "/vendor")
1321
1322  if updating_recovery and not target_has_recovery_patch:
1323    script.Print("Unpacking new recovery...")
1324    script.UnpackPackageDir("recovery", "/system")
1325
1326  system_diff.EmitRenames(script)
1327  if vendor_diff:
1328    vendor_diff.EmitRenames(script)
1329
1330  script.Print("Symlinks and permissions...")
1331
1332  # Create all the symlinks that don't already exist, or point to
1333  # somewhere different than what we want.  Delete each symlink before
1334  # creating it, since the 'symlink' command won't overwrite.
1335  to_create = []
1336  for dest, link in target_symlinks:
1337    if link in source_symlinks_d:
1338      if dest != source_symlinks_d[link]:
1339        to_create.append((dest, link))
1340    else:
1341      to_create.append((dest, link))
1342  script.DeleteFiles([i[1] for i in to_create])
1343  script.MakeSymlinks(to_create)
1344
1345  # Now that the symlinks are created, we can set all the
1346  # permissions.
1347  script.AppendScript(temp_script)
1348
1349  # Do device-specific installation (eg, write radio image).
1350  device_specific.IncrementalOTA_InstallEnd()
1351
1352  if OPTIONS.extra_script is not None:
1353    script.AppendExtra(OPTIONS.extra_script)
1354
1355  # Patch the build.prop file last, so if something fails but the
1356  # device can still come up, it appears to be the old build and will
1357  # get set the OTA package again to retry.
1358  script.Print("Patching remaining system files...")
1359  system_diff.EmitDeferredPatches(script)
1360
1361  if OPTIONS.wipe_user_data:
1362    script.Print("Erasing user data...")
1363    script.FormatPartition("/data")
1364
1365  if OPTIONS.two_step:
1366    script.AppendExtra("""
1367set_stage("%(bcb_dev)s", "");
1368endif;
1369endif;
1370""" % bcb_dev)
1371
1372  if OPTIONS.verify and system_diff:
1373    script.Print("Remounting and verifying system partition files...")
1374    script.Unmount("/system")
1375    script.Mount("/system")
1376    system_diff.EmitExplicitTargetVerification(script)
1377
1378  if OPTIONS.verify and vendor_diff:
1379    script.Print("Remounting and verifying vendor partition files...")
1380    script.Unmount("/vendor")
1381    script.Mount("/vendor")
1382    vendor_diff.EmitExplicitTargetVerification(script)
1383  script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary)
1384
1385  WriteMetadata(metadata, output_zip)
1386
1387
1388def main(argv):
1389
1390  def option_handler(o, a):
1391    if o == "--board_config":
1392      pass   # deprecated
1393    elif o in ("-k", "--package_key"):
1394      OPTIONS.package_key = a
1395    elif o in ("-i", "--incremental_from"):
1396      OPTIONS.incremental_source = a
1397    elif o in ("-w", "--wipe_user_data"):
1398      OPTIONS.wipe_user_data = True
1399    elif o in ("-n", "--no_prereq"):
1400      OPTIONS.omit_prereq = True
1401    elif o in ("-o", "--oem_settings"):
1402      OPTIONS.oem_source = a
1403    elif o in ("-e", "--extra_script"):
1404      OPTIONS.extra_script = a
1405    elif o in ("-a", "--aslr_mode"):
1406      if a in ("on", "On", "true", "True", "yes", "Yes"):
1407        OPTIONS.aslr_mode = True
1408      else:
1409        OPTIONS.aslr_mode = False
1410    elif o in ("-t", "--worker_threads"):
1411      if a.isdigit():
1412        OPTIONS.worker_threads = int(a)
1413      else:
1414        raise ValueError("Cannot parse value %r for option %r - only "
1415                         "integers are allowed." % (a, o))
1416    elif o in ("-2", "--two_step"):
1417      OPTIONS.two_step = True
1418    elif o == "--no_signing":
1419      OPTIONS.no_signing = True
1420    elif o in ("--verify"):
1421      OPTIONS.verify = True
1422    elif o == "--block":
1423      OPTIONS.block_based = True
1424    elif o in ("-b", "--binary"):
1425      OPTIONS.updater_binary = a
1426    elif o in ("--no_fallback_to_full",):
1427      OPTIONS.fallback_to_full = False
1428    else:
1429      return False
1430    return True
1431
1432  args = common.ParseOptions(argv, __doc__,
1433                             extra_opts="b:k:i:d:wne:t:a:2o:",
1434                             extra_long_opts=["board_config=",
1435                                              "package_key=",
1436                                              "incremental_from=",
1437                                              "wipe_user_data",
1438                                              "no_prereq",
1439                                              "extra_script=",
1440                                              "worker_threads=",
1441                                              "aslr_mode=",
1442                                              "two_step",
1443                                              "no_signing",
1444                                              "block",
1445                                              "binary=",
1446                                              "oem_settings=",
1447                                              "verify",
1448                                              "no_fallback_to_full",
1449                                              ],
1450                             extra_option_handler=option_handler)
1451
1452  if len(args) != 2:
1453    common.Usage(__doc__)
1454    sys.exit(1)
1455
1456  if OPTIONS.extra_script is not None:
1457    OPTIONS.extra_script = open(OPTIONS.extra_script).read()
1458
1459  print "unzipping target target-files..."
1460  OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0])
1461
1462  OPTIONS.target_tmp = OPTIONS.input_tmp
1463  OPTIONS.info_dict = common.LoadInfoDict(input_zip)
1464
1465  # If this image was originally labelled with SELinux contexts, make sure we
1466  # also apply the labels in our new image. During building, the "file_contexts"
1467  # is in the out/ directory tree, but for repacking from target-files.zip it's
1468  # in the root directory of the ramdisk.
1469  if "selinux_fc" in OPTIONS.info_dict:
1470    OPTIONS.info_dict["selinux_fc"] = os.path.join(OPTIONS.input_tmp, "BOOT", "RAMDISK",
1471        "file_contexts")
1472
1473  if OPTIONS.verbose:
1474    print "--- target info ---"
1475    common.DumpInfoDict(OPTIONS.info_dict)
1476
1477  # If the caller explicitly specified the device-specific extensions
1478  # path via -s/--device_specific, use that.  Otherwise, use
1479  # META/releasetools.py if it is present in the target target_files.
1480  # Otherwise, take the path of the file from 'tool_extensions' in the
1481  # info dict and look for that in the local filesystem, relative to
1482  # the current directory.
1483
1484  if OPTIONS.device_specific is None:
1485    from_input = os.path.join(OPTIONS.input_tmp, "META", "releasetools.py")
1486    if os.path.exists(from_input):
1487      print "(using device-specific extensions from target_files)"
1488      OPTIONS.device_specific = from_input
1489    else:
1490      OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None)
1491
1492  if OPTIONS.device_specific is not None:
1493    OPTIONS.device_specific = os.path.abspath(OPTIONS.device_specific)
1494
1495  while True:
1496
1497    if OPTIONS.no_signing:
1498      if os.path.exists(args[1]): os.unlink(args[1])
1499      output_zip = zipfile.ZipFile(args[1], "w", compression=zipfile.ZIP_DEFLATED)
1500    else:
1501      temp_zip_file = tempfile.NamedTemporaryFile()
1502      output_zip = zipfile.ZipFile(temp_zip_file, "w",
1503                                   compression=zipfile.ZIP_DEFLATED)
1504
1505    if OPTIONS.incremental_source is None:
1506      WriteFullOTAPackage(input_zip, output_zip)
1507      if OPTIONS.package_key is None:
1508        OPTIONS.package_key = OPTIONS.info_dict.get(
1509            "default_system_dev_certificate",
1510            "build/target/product/security/testkey")
1511      break
1512
1513    else:
1514      print "unzipping source target-files..."
1515      OPTIONS.source_tmp, source_zip = common.UnzipTemp(OPTIONS.incremental_source)
1516      OPTIONS.target_info_dict = OPTIONS.info_dict
1517      OPTIONS.source_info_dict = common.LoadInfoDict(source_zip)
1518      if "selinux_fc" in OPTIONS.source_info_dict:
1519        OPTIONS.source_info_dict["selinux_fc"] = os.path.join(OPTIONS.source_tmp, "BOOT", "RAMDISK",
1520                                                              "file_contexts")
1521      if OPTIONS.package_key is None:
1522        OPTIONS.package_key = OPTIONS.source_info_dict.get(
1523            "default_system_dev_certificate",
1524            "build/target/product/security/testkey")
1525      if OPTIONS.verbose:
1526        print "--- source info ---"
1527        common.DumpInfoDict(OPTIONS.source_info_dict)
1528      try:
1529        WriteIncrementalOTAPackage(input_zip, source_zip, output_zip)
1530        break
1531      except ValueError:
1532        if not OPTIONS.fallback_to_full: raise
1533        print "--- failed to build incremental; falling back to full ---"
1534        OPTIONS.incremental_source = None
1535        output_zip.close()
1536
1537  output_zip.close()
1538
1539  if not OPTIONS.no_signing:
1540    SignOutput(temp_zip_file.name, args[1])
1541    temp_zip_file.close()
1542
1543  print "done."
1544
1545
1546if __name__ == '__main__':
1547  try:
1548    common.CloseInheritedPipes()
1549    main(sys.argv[1:])
1550  except common.ExternalError, e:
1551    print
1552    print "   ERROR: %s" % (e,)
1553    print
1554    sys.exit(1)
1555  finally:
1556    common.Cleanup()
1557