1 /*
2  * Copyright (C) 2024 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "dexopt_chroot_setup.h"
18 
19 #include <linux/mount.h>
20 #include <sched.h>
21 #include <sys/mount.h>
22 #include <sys/stat.h>
23 #include <sys/types.h>
24 
25 #include <chrono>
26 #include <cstring>
27 #include <filesystem>
28 #include <mutex>
29 #include <optional>
30 #include <string>
31 #include <string_view>
32 #include <system_error>
33 #include <vector>
34 
35 #include "aidl/com/android/server/art/BnDexoptChrootSetup.h"
36 #include "android-base/errors.h"
37 #include "android-base/file.h"
38 #include "android-base/logging.h"
39 #include "android-base/no_destructor.h"
40 #include "android-base/properties.h"
41 #include "android-base/result.h"
42 #include "android-base/strings.h"
43 #include "android/binder_auto_utils.h"
44 #include "android/binder_manager.h"
45 #include "android/binder_process.h"
46 #include "base/file_utils.h"
47 #include "base/macros.h"
48 #include "base/os.h"
49 #include "exec_utils.h"
50 #include "fstab/fstab.h"
51 #include "tools/binder_utils.h"
52 #include "tools/cmdline_builder.h"
53 #include "tools/tools.h"
54 
55 namespace art {
56 namespace dexopt_chroot_setup {
57 
58 namespace {
59 
60 using ::android::base::ConsumePrefix;
61 using ::android::base::Error;
62 using ::android::base::Join;
63 using ::android::base::NoDestructor;
64 using ::android::base::ReadFileToString;
65 using ::android::base::Result;
66 using ::android::base::SetProperty;
67 using ::android::base::Split;
68 using ::android::base::Tokenize;
69 using ::android::base::WaitForProperty;
70 using ::android::base::WriteStringToFile;
71 using ::android::fs_mgr::FstabEntry;
72 using ::art::tools::CmdlineBuilder;
73 using ::art::tools::Fatal;
74 using ::art::tools::GetProcMountsDescendantsOfPath;
75 using ::art::tools::NonFatal;
76 using ::art::tools::PathStartsWith;
77 using ::ndk::ScopedAStatus;
78 
79 constexpr const char* kServiceName = "dexopt_chroot_setup";
80 const NoDestructor<std::string> kBindMountTmpDir(
81     std::string(DexoptChrootSetup::PRE_REBOOT_DEXOPT_DIR) + "/mount_tmp");
82 const NoDestructor<std::string> kOtaSlotFile(std::string(DexoptChrootSetup::PRE_REBOOT_DEXOPT_DIR) +
83                                              "/ota_slot");
84 const NoDestructor<std::string> kSnapshotMappedFile(
85     std::string(DexoptChrootSetup::PRE_REBOOT_DEXOPT_DIR) + "/snapshot_mapped");
86 constexpr mode_t kChrootDefaultMode = 0755;
87 constexpr std::chrono::milliseconds kSnapshotCtlTimeout = std::chrono::seconds(60);
88 
IsOtaUpdate(const std::optional<std::string> & ota_slot)89 bool IsOtaUpdate(const std::optional<std::string>& ota_slot) { return ota_slot.has_value(); }
90 
Run(std::string_view log_name,const std::vector<std::string> & args)91 Result<void> Run(std::string_view log_name, const std::vector<std::string>& args) {
92   LOG(INFO) << "Running " << log_name << ": " << Join(args, /*separator=*/" ");
93 
94   std::string error_msg;
95   if (!Exec(args, &error_msg)) {
96     return Errorf("Failed to run {}: {}", log_name, error_msg);
97   }
98 
99   LOG(INFO) << log_name << " returned code 0";
100   return {};
101 }
102 
GetArtExecCmdlineBuilder()103 Result<CmdlineBuilder> GetArtExecCmdlineBuilder() {
104   std::string error_msg;
105   std::string art_root = GetArtRootSafe(&error_msg);
106   if (!error_msg.empty()) {
107     return Error() << error_msg;
108   }
109   CmdlineBuilder args;
110   args.Add(art_root + "/bin/art_exec")
111       .Add("--chroot=%s", DexoptChrootSetup::CHROOT_DIR)
112       .Add("--process-name-suffix=Pre-reboot Dexopt chroot");
113   return args;
114 }
115 
CreateDir(const std::string & path)116 Result<void> CreateDir(const std::string& path) {
117   std::error_code ec;
118   std::filesystem::create_directory(path, ec);
119   if (ec) {
120     return Errorf("Failed to create dir '{}': {}", path, ec.message());
121   }
122   return {};
123 }
124 
Unmount(const std::string & target)125 Result<void> Unmount(const std::string& target) {
126   if (umount2(target.c_str(), UMOUNT_NOFOLLOW) == 0) {
127     return {};
128   }
129   LOG(WARNING) << ART_FORMAT(
130       "Failed to umount2 '{}': {}. Retrying with MNT_DETACH", target, strerror(errno));
131   if (umount2(target.c_str(), UMOUNT_NOFOLLOW | MNT_DETACH) == 0) {
132     return {};
133   }
134   return ErrnoErrorf("Failed to umount2 '{}'", target);
135 }
136 
BindMount(const std::string & source,const std::string & target)137 Result<void> BindMount(const std::string& source, const std::string& target) {
138   // Don't bind-mount repeatedly.
139   CHECK(!PathStartsWith(source, DexoptChrootSetup::CHROOT_DIR));
140   // system_server has a different mount namespace from init, and it uses slave mounts. E.g:
141   //
142   //    a: init mount ns: shared(1):          /foo
143   //    b: init mount ns: shared(2):          /mnt
144   //    c: SS mount ns:   slave(1):           /foo
145   //    d: SS mount ns:   slave(2):           /mnt
146   //
147   // We create our chroot setup in the init namespace but also want it to appear inside the
148   // system_server one, since we need to access some files in it from system_server (in particular
149   // service-art.jar).
150   //
151   // Hence we want the mount propagation type to be "slave+shared": Slave of the init namespace so
152   // that unmounts in the chroot doesn't affect the rest of the system, while at the same time
153   // shared with the system_server namespace so that it gets the same mounts recursively in the
154   // chroot tree. This can be achieved in 4 steps:
155   //
156   // 1. Bind-mount /foo at a temp mount point /mnt/pre_reboot_dexopt/mount_tmp.
157   //    a: init mount ns: shared(1):          /foo
158   //    b: init mount ns: shared(2):          /mnt
159   //    e: init mount ns: shared(1):          /mnt/pre_reboot_dexopt/mount_tmp
160   //    c: SS mount ns:   slave(1):           /foo
161   //    d: SS mount ns:   slave(2):           /mnt
162   //    f: SS mount ns:   slave(1):           /mnt/pre_reboot_dexopt/mount_tmp
163   //
164   // 2. Make the temp mount point slave.
165   //    a: init mount ns: shared(1):          /foo
166   //    b: init mount ns: shared(2):          /mnt
167   //    e: init mount ns: slave(1):           /mnt/pre_reboot_dexopt/mount_tmp
168   //    c: SS mount ns:   slave(1):           /foo
169   //    d: SS mount ns:   slave(2):           /mnt
170   //    f: SS mount ns:   slave(1):           /mnt/pre_reboot_dexopt/mount_tmp
171   //
172   // 3. Bind-mount the temp mount point at /mnt/pre_reboot_dexopt/chroot/foo. (The new mount point
173   //    gets "slave+shared". It gets "slave" because the source (`e`) is "slave", and it gets
174   //    "shared" because the dest (`b`) is "shared".)
175   //    a: init mount ns: shared(1):          /foo
176   //    b: init mount ns: shared(2):          /mnt
177   //    e: init mount ns: slave(1):           /mnt/pre_reboot_dexopt/mount_tmp
178   //    g: init mount ns: slave(1),shared(3): /mnt/pre_reboot_dexopt/chroot/foo
179   //    b: SS mount ns:   slave(1):           /foo
180   //    d: SS mount ns:   slave(2):           /mnt
181   //    f: SS mount ns:   slave(1):           /mnt/pre_reboot_dexopt/mount_tmp
182   //    h: SS mount ns:   slave(3):           /mnt/pre_reboot_dexopt/chroot/foo
183   //
184   // 4. Unmount the temp mount point.
185   //    a: init mount ns: shared(1):          /foo
186   //    b: init mount ns: shared(2):          /mnt
187   //    g: init mount ns: slave(1),shared(3): /mnt/pre_reboot_dexopt/chroot/foo
188   //    b: SS mount ns:   slave(1):           /foo
189   //    d: SS mount ns:   slave(2):           /mnt
190   //    h: SS mount ns:   slave(3):           /mnt/pre_reboot_dexopt/chroot/foo
191   //
192   // At this point, we have achieved what we want. `g` is a slave of `a` so that unmounts in `g`
193   // doesn't affect `a`, and `g` is shared with `h` so that mounts in `g` are propagated to `h`.
194   OR_RETURN(CreateDir(*kBindMountTmpDir));
195   if (mount(source.c_str(),
196             kBindMountTmpDir->c_str(),
197             /*fs_type=*/nullptr,
198             MS_BIND,
199             /*data=*/nullptr) != 0) {
200     return ErrnoErrorf("Failed to bind-mount '{}' at '{}'", source, *kBindMountTmpDir);
201   }
202   if (mount(/*source=*/nullptr,
203             kBindMountTmpDir->c_str(),
204             /*fs_type=*/nullptr,
205             MS_SLAVE,
206             /*data=*/nullptr) != 0) {
207     return ErrnoErrorf("Failed to make mount slave for '{}'", *kBindMountTmpDir);
208   }
209   if (mount(kBindMountTmpDir->c_str(),
210             target.c_str(),
211             /*fs_type=*/nullptr,
212             MS_BIND,
213             /*data=*/nullptr) != 0) {
214     return ErrnoErrorf("Failed to bind-mount '{}' at '{}'", *kBindMountTmpDir, target);
215   }
216   OR_RETURN(Unmount(*kBindMountTmpDir));
217   LOG(INFO) << ART_FORMAT("Bind-mounted '{}' at '{}'", source, target);
218   return {};
219 }
220 
BindMountRecursive(const std::string & source,const std::string & target)221 Result<void> BindMountRecursive(const std::string& source, const std::string& target) {
222   CHECK(!source.ends_with('/'));
223   OR_RETURN(BindMount(source, target));
224 
225   // Mount and make slave one by one. Do not use MS_REC because we don't want to mount a child if
226   // the parent cannot be slave (i.e., is shared). Otherwise, unmount events will be undesirably
227   // propagated to the source. For example, if "/dev" and "/dev/pts" are mounted at "/chroot/dev"
228   // and "/chroot/dev/pts" respectively, and "/chroot/dev" is shared, then unmounting
229   // "/chroot/dev/pts" will also unmount "/dev/pts".
230   //
231   // The list is in mount order.
232   std::vector<FstabEntry> entries = OR_RETURN(GetProcMountsDescendantsOfPath(source));
233   for (const FstabEntry& entry : entries) {
234     CHECK(!entry.mount_point.ends_with('/'));
235     std::string_view sub_dir = entry.mount_point;
236     CHECK(ConsumePrefix(&sub_dir, source));
237     if (sub_dir.empty()) {
238       // `source` itself. Already mounted.
239       continue;
240     }
241     OR_RETURN(BindMount(entry.mount_point, std::string(target).append(sub_dir)));
242   }
243   return {};
244 }
245 
GetBlockDeviceName(const std::string & partition,const std::string & slot)246 std::string GetBlockDeviceName(const std::string& partition, const std::string& slot) {
247   return ART_FORMAT("/dev/block/mapper/{}{}", partition, slot);
248 }
249 
GetSupportedFilesystems()250 Result<std::vector<std::string>> GetSupportedFilesystems() {
251   std::string content;
252   if (!ReadFileToString("/proc/filesystems", &content)) {
253     return ErrnoErrorf("Failed to read '/proc/filesystems'");
254   }
255   std::vector<std::string> filesystems;
256   for (const std::string& line : Split(content, "\n")) {
257     std::vector<std::string> tokens = Tokenize(line, " \t");
258     // If there are two tokens, the first token is a "nodev" mark, meaning it's not for a block
259     // device, so we skip it.
260     if (tokens.size() == 1) {
261       filesystems.push_back(tokens[0]);
262     }
263   }
264   // Prioritize the filesystems that are known to behave correctly, just in case some bad
265   // filesystems are unexpectedly happy to mount volumes that aren't of their types. We have never
266   // seen this case in practice though.
267   constexpr const char* kWellKnownFilesystems[] = {"erofs", "ext4"};
268   for (const char* well_known_fs : kWellKnownFilesystems) {
269     auto it = std::find(filesystems.begin(), filesystems.end(), well_known_fs);
270     if (it != filesystems.end()) {
271       filesystems.erase(it);
272       filesystems.insert(filesystems.begin(), well_known_fs);
273     }
274   }
275   return filesystems;
276 }
277 
Mount(const std::string & block_device,const std::string & target,bool is_optional)278 Result<void> Mount(const std::string& block_device, const std::string& target, bool is_optional) {
279   static const NoDestructor<Result<std::vector<std::string>>> supported_filesystems(
280       GetSupportedFilesystems());
281   if (!supported_filesystems->ok()) {
282     return supported_filesystems->error();
283   }
284   std::vector<std::string> error_msgs;
285   for (const std::string& filesystem : supported_filesystems->value()) {
286     if (mount(block_device.c_str(),
287               target.c_str(),
288               filesystem.c_str(),
289               MS_RDONLY,
290               /*data=*/nullptr) == 0) {
291       // Success.
292       LOG(INFO) << ART_FORMAT(
293           "Mounted '{}' at '{}' with type '{}'", block_device, target, filesystem);
294       return {};
295     } else {
296       if (errno == ENOENT && is_optional) {
297         LOG(INFO) << ART_FORMAT("Skipped non-existing block device '{}'", block_device);
298         return {};
299       }
300       error_msgs.push_back(ART_FORMAT("Tried '{}': {}", filesystem, strerror(errno)));
301       if (errno != EINVAL && errno != EBUSY) {
302         // If the filesystem type is wrong, `errno` must be either `EINVAL` or `EBUSY`. For example,
303         // we've seen that trying to mount a device with a wrong filesystem type yields `EBUSY` if
304         // the device is also mounted elsewhere, though we can't find any document about this
305         // behavior.
306         break;
307       }
308     }
309   }
310   return Errorf("Failed to mount '{}' at '{}':\n{}", block_device, target, Join(error_msgs, '\n'));
311 }
312 
MountTmpfs(const std::string & target,std::string_view se_context)313 Result<void> MountTmpfs(const std::string& target, std::string_view se_context) {
314   if (mount(/*source=*/"tmpfs",
315             target.c_str(),
316             /*fs_type=*/"tmpfs",
317             MS_NODEV | MS_NOEXEC | MS_NOSUID,
318             ART_FORMAT("mode={:#o},rootcontext={}", kChrootDefaultMode, se_context).c_str()) != 0) {
319     return ErrnoErrorf("Failed to mount tmpfs at '{}'", target);
320   }
321   return {};
322 }
323 
LoadOtaSlotFile()324 Result<std::optional<std::string>> LoadOtaSlotFile() {
325   std::string content;
326   if (!ReadFileToString(*kOtaSlotFile, &content)) {
327     return ErrnoErrorf("Failed to read '{}'", *kOtaSlotFile);
328   }
329   if (content == "_a" || content == "_b") {
330     return content;
331   }
332   if (content.empty()) {
333     return std::nullopt;
334   }
335   return Errorf("Invalid content of '{}': '{}'", *kOtaSlotFile, content);
336 }
337 
338 }  // namespace
339 
setUp(const std::optional<std::string> & in_otaSlot,bool in_mapSnapshotsForOta)340 ScopedAStatus DexoptChrootSetup::setUp(const std::optional<std::string>& in_otaSlot,
341                                        bool in_mapSnapshotsForOta) {
342   if (!mu_.try_lock()) {
343     return Fatal("Unexpected concurrent calls");
344   }
345   std::lock_guard<std::mutex> lock(mu_, std::adopt_lock);
346 
347   if (in_otaSlot.has_value() && (in_otaSlot.value() != "_a" && in_otaSlot.value() != "_b")) {
348     return Fatal(ART_FORMAT("Invalid OTA slot '{}'", in_otaSlot.value()));
349   }
350   OR_RETURN_NON_FATAL(SetUpChroot(in_otaSlot, in_mapSnapshotsForOta));
351   return ScopedAStatus::ok();
352 }
353 
init()354 ScopedAStatus DexoptChrootSetup::init() {
355   if (!mu_.try_lock()) {
356     return Fatal("Unexpected concurrent calls");
357   }
358   std::lock_guard<std::mutex> lock(mu_, std::adopt_lock);
359 
360   OR_RETURN_NON_FATAL(InitChroot());
361   return ScopedAStatus::ok();
362 }
363 
tearDown()364 ScopedAStatus DexoptChrootSetup::tearDown() {
365   if (!mu_.try_lock()) {
366     return Fatal("Unexpected concurrent calls");
367   }
368   std::lock_guard<std::mutex> lock(mu_, std::adopt_lock);
369 
370   OR_RETURN_NON_FATAL(TearDownChroot());
371   return ScopedAStatus::ok();
372 }
373 
Start()374 Result<void> DexoptChrootSetup::Start() {
375   ScopedAStatus status = ScopedAStatus::fromStatus(
376       AServiceManager_registerLazyService(this->asBinder().get(), kServiceName));
377   if (!status.isOk()) {
378     return Error() << status.getDescription();
379   }
380 
381   ABinderProcess_startThreadPool();
382 
383   return {};
384 }
385 
SetUpChroot(const std::optional<std::string> & ota_slot,bool map_snapshots_for_ota) const386 Result<void> DexoptChrootSetup::SetUpChroot(const std::optional<std::string>& ota_slot,
387                                             bool map_snapshots_for_ota) const {
388   // Set the default permission mode for new files and dirs to be `kChrootDefaultMode`.
389   umask(~kChrootDefaultMode & 0777);
390 
391   // In case there is some leftover.
392   OR_RETURN(TearDownChroot());
393 
394   // Prepare the root dir of chroot. The parent directory has been created by init (see `init.rc`).
395   OR_RETURN(CreateDir(CHROOT_DIR));
396   LOG(INFO) << ART_FORMAT("Created '{}'", CHROOT_DIR);
397 
398   std::vector<std::string> additional_system_partitions = {
399       "system_ext",
400       "vendor",
401       "product",
402   };
403 
404   if (!IsOtaUpdate(ota_slot)) {  // Mainline update
405     OR_RETURN(BindMount("/", CHROOT_DIR));
406     for (const std::string& partition : additional_system_partitions) {
407       // Some additional partitions are optional, but that's okay. The root filesystem (mounted at
408       // `/`) has empty directories for additional partitions. If additional partitions don't exist,
409       // we'll just be bind-mounting empty directories.
410       OR_RETURN(BindMount("/" + partition, PathInChroot("/" + partition)));
411     }
412   } else {
413     CHECK(ota_slot.value() == "_a" || ota_slot.value() == "_b");
414 
415     if (map_snapshots_for_ota) {
416       // Write the file early in case `snapshotctl map` fails in the middle, leaving some devices
417       // mapped. We don't assume that `snapshotctl map` is transactional.
418       if (!WriteStringToFile("", *kSnapshotMappedFile)) {
419         return ErrnoErrorf("Failed to write '{}'", *kSnapshotMappedFile);
420       }
421 
422       // Run `snapshotctl map` through init to map block devices. We can't run it ourselves because
423       // it requires the UID to be 0. See `sys.snapshotctl.map` in `init.rc`.
424       if (!SetProperty("sys.snapshotctl.map", "requested")) {
425         return Errorf("Failed to request snapshotctl map");
426       }
427       if (!WaitForProperty("sys.snapshotctl.map", "finished", kSnapshotCtlTimeout)) {
428         return Errorf("snapshotctl timed out");
429       }
430 
431       // We don't know whether snapshotctl succeeded or not, but if it failed, the mount operation
432       // below will fail with `ENOENT`.
433       OR_RETURN(
434           Mount(GetBlockDeviceName("system", ota_slot.value()), CHROOT_DIR, /*is_optional=*/false));
435     } else {
436       // update_engine has mounted `system` at `/postinstall` for us.
437       OR_RETURN(BindMount("/postinstall", CHROOT_DIR));
438     }
439 
440     for (const std::string& partition : additional_system_partitions) {
441       OR_RETURN(Mount(GetBlockDeviceName(partition, ota_slot.value()),
442                       PathInChroot("/" + partition),
443                       /*is_optional=*/true));
444     }
445   }
446 
447   OR_RETURN(MountTmpfs(PathInChroot("/apex"), "u:object_r:apex_mnt_dir:s0"));
448   OR_RETURN(MountTmpfs(PathInChroot("/linkerconfig"), "u:object_r:linkerconfig_file:s0"));
449   OR_RETURN(MountTmpfs(PathInChroot("/mnt"), "u:object_r:pre_reboot_dexopt_file:s0"));
450   OR_RETURN(CreateDir(PathInChroot("/mnt/artd_tmp")));
451   OR_RETURN(MountTmpfs(PathInChroot("/mnt/artd_tmp"), "u:object_r:pre_reboot_dexopt_artd_file:s0"));
452   OR_RETURN(CreateDir(PathInChroot("/mnt/expand")));
453 
454   std::vector<std::string> bind_mount_srcs = {
455       // Data partitions.
456       "/data",
457       "/mnt/expand",
458       // Linux API filesystems.
459       "/dev",
460       "/proc",
461       "/sys",
462       // For apexd to query staged APEX sessions.
463       "/metadata",
464   };
465 
466   for (const std::string& src : bind_mount_srcs) {
467     OR_RETURN(BindMountRecursive(src, PathInChroot(src)));
468   }
469 
470   if (!WriteStringToFile(ota_slot.value_or(""), *kOtaSlotFile)) {
471     return ErrnoErrorf("Failed to write '{}'", *kOtaSlotFile);
472   }
473 
474   return {};
475 }
476 
InitChroot() const477 Result<void> DexoptChrootSetup::InitChroot() const {
478   std::optional<std::string> ota_slot = OR_RETURN(LoadOtaSlotFile());
479 
480   // Generate empty linker config to suppress warnings.
481   if (!android::base::WriteStringToFile("", PathInChroot("/linkerconfig/ld.config.txt"))) {
482     PLOG(WARNING) << "Failed to generate empty linker config to suppress warnings";
483   }
484 
485   CmdlineBuilder args = OR_RETURN(GetArtExecCmdlineBuilder());
486   args.Add("--")
487       .Add("/system/bin/apexd")
488       .Add("--otachroot-bootstrap")
489       .AddIf(!IsOtaUpdate(ota_slot), "--also-include-staged-apexes");
490   OR_RETURN(Run("apexd", args.Get()));
491 
492   args = OR_RETURN(GetArtExecCmdlineBuilder());
493   args.Add("--drop-capabilities")
494       .Add("--")
495       .Add("/apex/com.android.runtime/bin/linkerconfig")
496       .Add("--target")
497       .Add("/linkerconfig");
498   OR_RETURN(Run("linkerconfig", args.Get()));
499 
500   return {};
501 }
502 
TearDownChroot() const503 Result<void> DexoptChrootSetup::TearDownChroot() const {
504   std::vector<FstabEntry> apex_entries =
505       OR_RETURN(GetProcMountsDescendantsOfPath(PathInChroot("/apex")));
506   // If there is only one entry, it's /apex itself.
507   bool has_apex = apex_entries.size() > 1;
508 
509   if (has_apex && OS::FileExists(PathInChroot("/system/bin/apexd").c_str())) {
510     // Delegate to apexd to unmount all APEXes. It also cleans up loop devices.
511     CmdlineBuilder args = OR_RETURN(GetArtExecCmdlineBuilder());
512     args.Add("--")
513         .Add("/system/bin/apexd")
514         .Add("--unmount-all")
515         .Add("--also-include-staged-apexes");
516     OR_RETURN(Run("apexd", args.Get()));
517   }
518 
519   // Double check to make sure all APEXes are unmounted, just in case apexd incorrectly reported
520   // success.
521   apex_entries = OR_RETURN(GetProcMountsDescendantsOfPath(PathInChroot("/apex")));
522   for (const FstabEntry& entry : apex_entries) {
523     if (entry.mount_point != PathInChroot("/apex")) {
524       return Errorf("apexd didn't unmount '{}'. See logs for details", entry.mount_point);
525     }
526   }
527 
528   // The list is in mount order.
529   std::vector<FstabEntry> entries = OR_RETURN(GetProcMountsDescendantsOfPath(CHROOT_DIR));
530   for (auto it = entries.rbegin(); it != entries.rend(); it++) {
531     OR_RETURN(Unmount(it->mount_point));
532     LOG(INFO) << ART_FORMAT("Unmounted '{}'", it->mount_point);
533   }
534 
535   std::error_code ec;
536   std::uintmax_t removed = std::filesystem::remove_all(CHROOT_DIR, ec);
537   if (ec) {
538     return Errorf("Failed to remove dir '{}': {}", CHROOT_DIR, ec.message());
539   }
540   if (removed > 0) {
541     LOG(INFO) << ART_FORMAT("Removed '{}'", CHROOT_DIR);
542   }
543 
544   if (!OR_RETURN(GetProcMountsDescendantsOfPath(*kBindMountTmpDir)).empty()) {
545     OR_RETURN(Unmount(*kBindMountTmpDir));
546   }
547 
548   std::filesystem::remove_all(*kBindMountTmpDir, ec);
549   if (ec) {
550     return Errorf("Failed to remove dir '{}': {}", *kBindMountTmpDir, ec.message());
551   }
552 
553   std::filesystem::remove(*kOtaSlotFile, ec);
554   if (ec) {
555     return Errorf("Failed to remove file '{}': {}", *kOtaSlotFile, ec.message());
556   }
557 
558   if (OS::FileExists(kSnapshotMappedFile->c_str())) {
559     if (!SetProperty("sys.snapshotctl.unmap", "requested")) {
560       return Errorf("Failed to request snapshotctl unmap");
561     }
562     if (!WaitForProperty("sys.snapshotctl.unmap", "finished", kSnapshotCtlTimeout)) {
563       return Errorf("snapshotctl timed out");
564     }
565     std::filesystem::remove(*kSnapshotMappedFile, ec);
566     if (ec) {
567       return Errorf("Failed to remove file '{}': {}", *kSnapshotMappedFile, ec.message());
568     }
569   }
570 
571   return {};
572 }
573 
PathInChroot(std::string_view path)574 std::string PathInChroot(std::string_view path) {
575   return std::string(DexoptChrootSetup::CHROOT_DIR).append(path);
576 }
577 
578 }  // namespace dexopt_chroot_setup
579 }  // namespace art
580