1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #define LOG_TAG "incfs"
18 
19 #include "incfs.h"
20 
21 #include <IncrementalProperties.sysprop.h>
22 #include <android-base/file.h>
23 #include <android-base/logging.h>
24 #include <android-base/no_destructor.h>
25 #include <android-base/parsebool.h>
26 #include <android-base/stringprintf.h>
27 #include <android-base/strings.h>
28 #include <android-base/unique_fd.h>
29 #include <dirent.h>
30 #include <errno.h>
31 #include <fcntl.h>
32 #include <libgen.h>
33 #include <openssl/sha.h>
34 #include <selinux/android.h>
35 #include <selinux/selinux.h>
36 #include <sys/mount.h>
37 #include <sys/poll.h>
38 #include <sys/stat.h>
39 #include <sys/syscall.h>
40 #include <sys/types.h>
41 #include <sys/vfs.h>
42 #include <sys/xattr.h>
43 #include <unistd.h>
44 
45 #include <chrono>
46 #include <fstream>
47 #include <iterator>
48 #include <mutex>
49 #include <optional>
50 #include <string_view>
51 
52 #include "MountRegistry.h"
53 #include "path.h"
54 
55 using namespace std::literals;
56 using namespace android::incfs;
57 using namespace android::sysprop;
58 namespace ab = android::base;
59 
60 struct IncFsControl final {
61     IncFsFd cmd;
62     IncFsFd pendingReads;
63     IncFsFd logs;
64     constexpr IncFsControl(IncFsFd cmd, IncFsFd pendingReads, IncFsFd logs)
65           : cmd(cmd), pendingReads(pendingReads), logs(logs) {}
66 };
67 
68 static MountRegistry& registry() {
69     static ab::NoDestructor<MountRegistry> instance{};
70     return *instance;
71 }
72 
73 static ab::unique_fd openRaw(std::string_view file) {
74     auto fd = ab::unique_fd(::open(details::c_str(file), O_RDONLY | O_CLOEXEC));
75     if (fd < 0) {
76         return ab::unique_fd{-errno};
77     }
78     return fd;
79 }
80 
81 static ab::unique_fd openRaw(std::string_view dir, std::string_view name) {
82     return openRaw(path::join(dir, name));
83 }
84 
85 static std::string rootForCmd(int fd) {
86     auto cmdFile = path::fromFd(fd);
87     if (cmdFile.empty()) {
88         LOG(INFO) << __func__ << "(): name empty for " << fd;
89         return {};
90     }
91     auto res = path::dirName(cmdFile);
92     if (res.empty()) {
93         LOG(INFO) << __func__ << "(): dirname empty for " << cmdFile;
94         return {};
95     }
96     if (!path::endsWith(cmdFile, INCFS_PENDING_READS_FILENAME)) {
97         LOG(INFO) << __func__ << "(): invalid file name " << cmdFile;
98         return {};
99     }
100     if (cmdFile.data() == res.data() || cmdFile.starts_with(res)) {
101         cmdFile.resize(res.size());
102         return cmdFile;
103     }
104     return std::string(res);
105 }
106 
107 static Features readIncFsFeatures() {
108     static const char kSysfsFeaturesDir[] = "/sys/fs/" INCFS_NAME "/features";
109     const auto dir = path::openDir(kSysfsFeaturesDir);
110     if (!dir) {
111         return Features::none;
112     }
113 
114     int res = Features::none;
115     while (auto entry = ::readdir(dir.get())) {
116         if (entry->d_type != DT_REG) {
117             continue;
118         }
119         if (entry->d_name == "corefs"sv) {
120             res |= Features::core;
121         }
122     }
123 
124     return Features(res);
125 }
126 
127 IncFsFeatures IncFs_Features() {
128     return IncFsFeatures(readIncFsFeatures());
129 }
130 
131 static bool isFsAvailable() {
132     static const char kProcFilesystems[] = "/proc/filesystems";
133     std::string filesystems;
134     if (!ab::ReadFileToString(kProcFilesystems, &filesystems)) {
135         return false;
136     }
137     return filesystems.find("\t" INCFS_NAME "\n") != std::string::npos;
138 }
139 
140 static std::string_view incFsPropertyValue() {
141     static const ab::NoDestructor<std::string> kValue{IncrementalProperties::enable().value_or("")};
142     return *kValue;
143 }
144 
145 static std::pair<bool, std::string_view> parseProperty(std::string_view property) {
146     auto boolVal = ab::ParseBool(property);
147     if (boolVal == ab::ParseBoolResult::kTrue) {
148         return {isFsAvailable(), {}};
149     }
150     if (boolVal == ab::ParseBoolResult::kFalse) {
151         return {false, {}};
152     }
153 
154     // Don't load the module at once, but instead only check if it is loadable.
155     static const auto kModulePrefix = "module:"sv;
156     if (property.starts_with(kModulePrefix)) {
157         const auto modulePath = property.substr(kModulePrefix.size());
158         return {::access(details::c_str(modulePath), R_OK | X_OK), modulePath};
159     }
160     return {false, {}};
161 }
162 
163 namespace {
164 
165 class IncFsInit {
166 public:
167     IncFsInit() {
168         auto [featureEnabled, moduleName] = parseProperty(incFsPropertyValue());
169         featureEnabled_ = featureEnabled;
170         moduleName_ = moduleName;
171         loaded_ = featureEnabled_ && isFsAvailable();
172     }
173 
174     constexpr ~IncFsInit() = default;
175 
176     bool enabled() const { return featureEnabled_; }
177     bool enabledAndReady() const {
178         if (!featureEnabled_) {
179             return false;
180         }
181         if (moduleName_.empty()) {
182             return true;
183         }
184         if (loaded_) {
185             return true;
186         }
187         std::call_once(loadedFlag_, [this] {
188             const ab::unique_fd fd(TEMP_FAILURE_RETRY(
189                     ::open(details::c_str(moduleName_), O_RDONLY | O_NOFOLLOW | O_CLOEXEC)));
190             if (fd < 0) {
191                 PLOG(ERROR) << "could not open IncFs kernel module \"" << moduleName_ << '"';
192                 return;
193             }
194 
195             const auto rc = syscall(__NR_finit_module, fd.get(), "", 0);
196             if (rc < 0) {
197                 PLOG(ERROR) << "finit_module for IncFs \"" << moduleName_ << "\" failed";
198                 return;
199             }
200             if (!isFsAvailable()) {
201                 LOG(ERROR) << "loaded IncFs kernel module \"" << moduleName_
202                            << "\" but incremental-fs is still not available";
203             }
204             loaded_ = true;
205             LOG(INFO) << "successfully loaded IncFs kernel module \"" << moduleName_ << '"';
206         });
207         return loaded_;
208     }
209 
210 private:
211     bool featureEnabled_;
212     std::string_view moduleName_;
213     mutable std::once_flag loadedFlag_;
214     mutable bool loaded_;
215 };
216 
217 } // namespace
218 
219 static IncFsInit& init() {
220     static IncFsInit initer;
221     return initer;
222 }
223 
224 bool IncFs_IsEnabled() {
225     return init().enabled();
226 }
227 
228 bool isIncFsPath(const char* path) {
229     struct statfs fs = {};
230     if (::statfs(path, &fs) != 0) {
231         PLOG(WARNING) << __func__ << "(): could not statfs " << path;
232         return false;
233     }
234 
235     return fs.f_type == (decltype(fs.f_type))INCFS_MAGIC_NUMBER;
236 }
237 
238 static int isDir(const char* path) {
239     struct stat st;
240     if (::stat(path, &st) != 0) {
241         return -errno;
242     }
243     if (!S_ISDIR(st.st_mode)) {
244         return -ENOTDIR;
245     }
246     return 0;
247 }
248 
249 static bool isAbsolute(const char* path) {
250     return path && path[0] == '/';
251 }
252 
253 static int isValidMountTarget(const char* path) {
254     if (!isAbsolute(path)) {
255         return -EINVAL;
256     }
257     if (isIncFsPath(path)) {
258         LOG(ERROR) << "[incfs] mounting over existing incfs mount is not allowed";
259         return -EINVAL;
260     }
261     if (const auto err = isDir(path); err != 0) {
262         return err;
263     }
264     if (const auto err = path::isEmptyDir(path); err != 0) {
265         return err;
266     }
267     return 0;
268 }
269 
270 static int rmDirContent(const char* path) {
271     auto dir = path::openDir(path);
272     if (!dir) {
273         return -EINVAL;
274     }
275     while (auto entry = ::readdir(dir.get())) {
276         if (entry->d_name == "."sv || entry->d_name == ".."sv) {
277             continue;
278         }
279         auto fullPath = ab::StringPrintf("%s/%s", path, entry->d_name);
280         if (entry->d_type == DT_DIR) {
281             if (const auto err = rmDirContent(fullPath.c_str()); err != 0) {
282                 return err;
283             }
284             if (const auto err = ::rmdir(fullPath.c_str()); err != 0) {
285                 return err;
286             }
287         } else {
288             if (const auto err = ::unlink(fullPath.c_str()); err != 0) {
289                 return err;
290             }
291         }
292     }
293     return 0;
294 }
295 
296 static std::string makeMountOptionsString(IncFsMountOptions options) {
297     return ab::StringPrintf("read_timeout_ms=%u,readahead=0,rlog_pages=%u,rlog_wakeup_cnt=1",
298                             unsigned(options.defaultReadTimeoutMs),
299                             unsigned(options.readLogBufferPages < 0
300                                              ? INCFS_DEFAULT_PAGE_READ_BUFFER_PAGES
301                                              : options.readLogBufferPages));
302 }
303 
304 static IncFsControl* makeControl(const char* root) {
305     auto cmd = openRaw(root, INCFS_PENDING_READS_FILENAME);
306     if (!cmd.ok()) {
307         return nullptr;
308     }
309     ab::unique_fd pendingReads(fcntl(cmd.get(), F_DUPFD_CLOEXEC, cmd.get()));
310     if (!pendingReads.ok()) {
311         return nullptr;
312     }
313     auto logs = openRaw(root, INCFS_LOG_FILENAME);
314     // logs may be absent, that's fine
315     auto control = IncFs_CreateControl(cmd.get(), pendingReads.get(), logs.get());
316     if (control) {
317         (void)cmd.release();
318         (void)pendingReads.release();
319         (void)logs.release();
320     }
321     return control;
322 }
323 
324 static std::string makeCommandPath(std::string_view root, std::string_view item) {
325     auto [itemRoot, subpath] = registry().rootAndSubpathFor(item);
326     if (itemRoot != root) {
327         return {};
328     }
329     // TODO: add "/.cmd/" if we decide to use a separate control tree.
330     return path::join(itemRoot, subpath);
331 }
332 
333 static void toString(IncFsFileId id, char* out) {
334     // Make sure this function matches the one in the kernel (e.g. same case for a-f digits).
335     static constexpr char kHexChar[] = "0123456789abcdef";
336 
337     for (auto item = std::begin(id.data); item != std::end(id.data); ++item, out += 2) {
338         out[0] = kHexChar[(*item & 0xf0) >> 4];
339         out[1] = kHexChar[(*item & 0x0f)];
340     }
341 }
342 
343 static std::string toStringImpl(IncFsFileId id) {
344     std::string res(kIncFsFileIdStringLength, '\0');
345     toString(id, res.data());
346     return res;
347 }
348 
349 static IncFsFileId toFileIdImpl(std::string_view str) {
350     if (str.size() != kIncFsFileIdStringLength) {
351         return kIncFsInvalidFileId;
352     }
353 
354     IncFsFileId res;
355     auto out = (char*)&res;
356     for (auto it = str.begin(); it != str.end(); it += 2, ++out) {
357         static const auto fromChar = [](char src) -> char {
358             if (src >= '0' && src <= '9') {
359                 return src - '0';
360             }
361             if (src >= 'a' && src <= 'f') {
362                 return src - 'a' + 10;
363             }
364             return -1;
365         };
366 
367         const char c[2] = {fromChar(it[0]), fromChar(it[1])};
368         if (c[0] == -1 || c[1] == -1) {
369             errno = EINVAL;
370             return kIncFsInvalidFileId;
371         }
372         *out = (c[0] << 4) | c[1];
373     }
374     return res;
375 }
376 
377 int IncFs_FileIdToString(IncFsFileId id, char* out) {
378     if (!out) {
379         return -EINVAL;
380     }
381     toString(id, out);
382     return 0;
383 }
384 
385 IncFsFileId IncFs_FileIdFromString(const char* in) {
386     return toFileIdImpl({in, kIncFsFileIdStringLength});
387 }
388 
389 IncFsFileId IncFs_FileIdFromMetadata(IncFsSpan metadata) {
390     IncFsFileId id = {};
391     if (size_t(metadata.size) <= sizeof(id)) {
392         memcpy(&id, metadata.data, metadata.size);
393     } else {
394         uint8_t buffer[SHA_DIGEST_LENGTH];
395         static_assert(sizeof(buffer) >= sizeof(id));
396 
397         SHA_CTX ctx;
398         SHA1_Init(&ctx);
399         SHA1_Update(&ctx, metadata.data, metadata.size);
400         SHA1_Final(buffer, &ctx);
401         memcpy(&id, buffer, sizeof(id));
402     }
403     return id;
404 }
405 
406 static bool restoreconControlFiles(std::string_view targetDir) {
407     const std::string controlFilePaths[] = {path::join(targetDir, INCFS_PENDING_READS_FILENAME),
408                                             path::join(targetDir, INCFS_LOG_FILENAME)};
409     for (size_t i = 0; i < std::size(controlFilePaths); i++) {
410         if (const auto err = selinux_android_restorecon(controlFilePaths[i].c_str(),
411                                                         SELINUX_ANDROID_RESTORECON_FORCE);
412             err != 0) {
413             PLOG(ERROR) << "[incfs] Failed to restorecon: " << controlFilePaths[i]
414                         << " error code: " << err;
415             errno = -err;
416             return false;
417         }
418     }
419     return true;
420 }
421 
422 IncFsControl* IncFs_Mount(const char* backingPath, const char* targetDir,
423                           IncFsMountOptions options) {
424     if (!init().enabledAndReady()) {
425         LOG(WARNING) << "[incfs] Feature is not enabled";
426         errno = ENOTSUP;
427         return nullptr;
428     }
429 
430     if (auto err = isValidMountTarget(targetDir); err != 0) {
431         errno = -err;
432         return nullptr;
433     }
434     if (!isAbsolute(backingPath)) {
435         errno = EINVAL;
436         return nullptr;
437     }
438 
439     if (options.flags & createOnly) {
440         if (const auto err = path::isEmptyDir(backingPath); err != 0) {
441             errno = -err;
442             return nullptr;
443         }
444     } else if (options.flags & android::incfs::truncate) {
445         if (const auto err = rmDirContent(backingPath); err != 0) {
446             errno = -err;
447             return nullptr;
448         }
449     }
450 
451     const auto opts = makeMountOptionsString(options);
452     if (::mount(backingPath, targetDir, INCFS_NAME, MS_NOSUID | MS_NODEV | MS_NOATIME,
453                 opts.c_str())) {
454         PLOG(ERROR) << "[incfs] Failed to mount IncFS filesystem: " << targetDir
455                     << " errno: " << errno;
456         return nullptr;
457     }
458 
459     if (!restoreconControlFiles(targetDir)) {
460         return nullptr;
461     }
462 
463     auto control = makeControl(targetDir);
464     if (control == nullptr) {
465         return nullptr;
466     }
467     return control;
468 }
469 
470 IncFsControl* IncFs_Open(const char* dir) {
471     auto root = registry().rootFor(dir);
472     if (root.empty()) {
473         errno = EINVAL;
474         return nullptr;
475     }
476     return makeControl(details::c_str(root));
477 }
478 
479 IncFsFd IncFs_GetControlFd(const IncFsControl* control, IncFsFdType type) {
480     if (!control) {
481         return -EINVAL;
482     }
483     switch (type) {
484         case CMD:
485             return control->cmd;
486         case PENDING_READS:
487             return control->pendingReads;
488         case LOGS:
489             return control->logs;
490         default:
491             return -EINVAL;
492     }
493 }
494 
495 IncFsSize IncFs_ReleaseControlFds(IncFsControl* control, IncFsFd out[], IncFsSize outSize) {
496     if (!control || !out) {
497         return -EINVAL;
498     }
499     if (outSize < IncFsFdType::FDS_COUNT) {
500         return -ERANGE;
501     }
502     out[CMD] = std::exchange(control->cmd, -1);
503     out[PENDING_READS] = std::exchange(control->pendingReads, -1);
504     out[LOGS] = std::exchange(control->logs, -1);
505     return IncFsFdType::FDS_COUNT;
506 }
507 
508 IncFsControl* IncFs_CreateControl(IncFsFd cmd, IncFsFd pendingReads, IncFsFd logs) {
509     return new IncFsControl(cmd, pendingReads, logs);
510 }
511 
512 void IncFs_DeleteControl(IncFsControl* control) {
513     if (control) {
514         if (control->cmd >= 0) {
515             close(control->cmd);
516         }
517         if (control->pendingReads >= 0) {
518             close(control->pendingReads);
519         }
520         if (control->logs >= 0) {
521             close(control->logs);
522         }
523         delete control;
524     }
525 }
526 
527 IncFsErrorCode IncFs_SetOptions(const IncFsControl* control, IncFsMountOptions options) {
528     if (!control) {
529         return -EINVAL;
530     }
531     auto root = rootForCmd(control->cmd);
532     if (root.empty()) {
533         return -EINVAL;
534     }
535     auto opts = makeMountOptionsString(options);
536     if (::mount(nullptr, root.c_str(), nullptr, MS_REMOUNT | MS_NOSUID | MS_NODEV | MS_NOATIME,
537                 opts.c_str()) != 0) {
538         const auto error = errno;
539         PLOG(ERROR) << "[incfs] Failed to remount IncFS filesystem: " << root;
540         return -error;
541     }
542     return 0;
543 }
544 
545 IncFsErrorCode IncFs_Root(const IncFsControl* control, char buffer[], size_t* bufferSize) {
546     if (!control) {
547         return -EINVAL;
548     }
549     std::string result = rootForCmd(control->cmd);
550     if (*bufferSize <= result.size()) {
551         *bufferSize = result.size() + 1;
552         return -EOVERFLOW;
553     }
554     result.copy(buffer, result.size());
555     buffer[result.size()] = '\0';
556     *bufferSize = result.size();
557     return 0;
558 }
559 
560 template <class T>
561 std::optional<T> read(IncFsSpan& data) {
562     if (data.size < (int32_t)sizeof(T)) {
563         return {};
564     }
565     T res;
566     memcpy(&res, data.data, sizeof(res));
567     data.data += sizeof(res);
568     data.size -= sizeof(res);
569     return res;
570 }
571 
572 static IncFsErrorCode validateSignatureFormat(IncFsSpan signature) {
573     if (signature.data == nullptr && signature.size == 0) {
574         return 0; // it's fine to have unverified files too
575     }
576     if ((signature.data == nullptr) != (signature.size == 0)) {
577         return -EINVAL;
578     }
579 
580     // These structs are here purely for checking the minimum size. Maybe will use them for
581     // parsing later.
582     struct __attribute__((packed)) Hashing {
583         int32_t size;
584         int32_t algorithm;
585         int8_t log2_blocksize;
586         int32_t salt_size;
587         int32_t raw_root_hash_size;
588     };
589     struct __attribute__((packed)) Signing {
590         int32_t size;
591         int32_t apk_digest_size;
592         int32_t certificate_size;
593         int32_t addl_data_size;
594         int32_t public_key_size;
595         int32_t algorithm;
596         int32_t signature_size;
597     };
598     struct __attribute__((packed)) MinSignature {
599         int32_t version;
600         Hashing hashing_info;
601         Signing signing_info;
602     };
603 
604     if (signature.size < (int32_t)sizeof(MinSignature)) {
605         return -ERANGE;
606     }
607     if (signature.size > INCFS_MAX_SIGNATURE_SIZE) {
608         return -ERANGE;
609     }
610 
611     auto version = read<int32_t>(signature);
612     if (version.value_or(-1) != INCFS_SIGNATURE_VERSION) {
613         return -EINVAL;
614     }
615     auto hashSize = read<int32_t>(signature);
616     if (!hashSize || signature.size < *hashSize) {
617         return -EINVAL;
618     }
619     auto hashAlgo = read<int32_t>(signature);
620     if (hashAlgo.value_or(-1) != INCFS_HASH_TREE_SHA256) {
621         return -EINVAL;
622     }
623     auto logBlockSize = read<int8_t>(signature);
624     if (logBlockSize.value_or(-1) != 12 /* 2^12 == 4096 */) {
625         return -EINVAL;
626     }
627     auto saltSize = read<int32_t>(signature);
628     if (saltSize.value_or(-1) != 0) {
629         return -EINVAL;
630     }
631     auto rootHashSize = read<int32_t>(signature);
632     if (rootHashSize.value_or(-1) != INCFS_MAX_HASH_SIZE) {
633         return -EINVAL;
634     }
635     if (signature.size < *rootHashSize) {
636         return -EINVAL;
637     }
638     signature.data += *rootHashSize;
639     signature.size -= *rootHashSize;
640     auto signingSize = read<int32_t>(signature);
641     // everything remaining has to be in the signing info
642     if (signingSize.value_or(-1) != signature.size) {
643         return -EINVAL;
644     }
645 
646     // TODO: validate the signature part too.
647     return 0;
648 }
649 
650 IncFsErrorCode IncFs_MakeFile(const IncFsControl* control, const char* path, int32_t mode,
651                               IncFsFileId id, IncFsNewFileParams params) {
652     if (!control) {
653         return -EINVAL;
654     }
655 
656     auto [root, subpath] = registry().rootAndSubpathFor(path);
657     if (root.empty()) {
658         PLOG(WARNING) << "[incfs] makeFile failed for path " << path << ", root is empty.";
659         return -EINVAL;
660     }
661     if (params.size < 0) {
662         LOG(WARNING) << "[incfs] makeFile failed for path " << path
663                      << ", size is invalid: " << params.size;
664         return -ERANGE;
665     }
666 
667     const auto [subdir, name] = path::splitDirBase(subpath);
668     incfs_new_file_args args = {
669             .size = (uint64_t)params.size,
670             .mode = (uint16_t)mode,
671             .directory_path = (uint64_t)subdir.data(),
672             .file_name = (uint64_t)name.data(),
673             .file_attr = (uint64_t)params.metadata.data,
674             .file_attr_len = (uint32_t)params.metadata.size,
675     };
676     static_assert(sizeof(args.file_id.bytes) == sizeof(id.data));
677     memcpy(args.file_id.bytes, id.data, sizeof(args.file_id.bytes));
678 
679     if (auto err = validateSignatureFormat(params.signature)) {
680         return err;
681     }
682     args.signature_info = (uint64_t)(uintptr_t)params.signature.data;
683     args.signature_size = (uint64_t)params.signature.size;
684 
685     if (::ioctl(control->cmd, INCFS_IOC_CREATE_FILE, &args)) {
686         PLOG(WARNING) << "[incfs] makeFile failed for " << root << " / " << subdir << " / " << name
687                       << " of " << params.size << " bytes";
688         return -errno;
689     }
690     if (::chmod(path::join(root, subpath).c_str(), mode)) {
691         PLOG(WARNING) << "[incfs] couldn't change file mode to 0" << std::oct << mode;
692     }
693 
694     return 0;
695 }
696 
697 static IncFsErrorCode makeDir(const char* commandPath, int32_t mode, bool allowExisting) {
698     if (!::mkdir(commandPath, mode)) {
699         if (::chmod(commandPath, mode)) {
700             PLOG(WARNING) << "[incfs] couldn't change directory mode to 0" << std::oct << mode;
701         }
702         return 0;
703     }
704     // don't touch the existing dir's mode - mkdir(1) works that way.
705     return (allowExisting && errno == EEXIST) ? 0 : -errno;
706 }
707 
708 static IncFsErrorCode makeDirs(std::string_view commandPath, std::string_view path,
709                                std::string_view root, int32_t mode) {
710     auto commandCPath = details::c_str(commandPath);
711     const auto mkdirRes = makeDir(commandCPath, mode, true);
712     if (!mkdirRes) {
713         return 0;
714     }
715     if (mkdirRes != -ENOENT) {
716         LOG(ERROR) << __func__ << "(): mkdir failed for " << path << " - " << mkdirRes;
717         return mkdirRes;
718     }
719 
720     const auto parent = path::dirName(commandPath);
721     if (!path::startsWith(parent, root)) {
722         // went too far, already out of the root mount
723         return -EINVAL;
724     }
725 
726     if (auto parentMkdirRes = makeDirs(parent, path::dirName(path), root, mode)) {
727         return parentMkdirRes;
728     }
729     return makeDir(commandCPath, mode, true);
730 }
731 
732 IncFsErrorCode IncFs_MakeDir(const IncFsControl* control, const char* path, int32_t mode) {
733     if (!control) {
734         return -EINVAL;
735     }
736     const auto root = rootForCmd(control->cmd);
737     if (root.empty()) {
738         LOG(ERROR) << __func__ << "(): root is empty for " << path;
739         return -EINVAL;
740     }
741     auto commandPath = makeCommandPath(root, path);
742     if (commandPath.empty()) {
743         LOG(ERROR) << __func__ << "(): commandPath is empty for " << path;
744         return -EINVAL;
745     }
746     if (auto res = makeDir(commandPath.c_str(), mode, false)) {
747         LOG(ERROR) << __func__ << "(): mkdir failed for " << commandPath << " - " << res;
748         return res;
749     }
750     return 0;
751 }
752 
753 IncFsErrorCode IncFs_MakeDirs(const IncFsControl* control, const char* path, int32_t mode) {
754     if (!control) {
755         return -EINVAL;
756     }
757     const auto root = rootForCmd(control->cmd);
758     if (root.empty()) {
759         LOG(ERROR) << __func__ << "(): root is empty for " << path;
760         return -EINVAL;
761     }
762     auto commandPath = makeCommandPath(root, path);
763     if (commandPath.empty()) {
764         LOG(ERROR) << __func__ << "(): commandPath is empty for " << path;
765         return -EINVAL;
766     }
767     return makeDirs(commandPath, path, root, mode);
768 }
769 
770 static IncFsErrorCode getMetadata(const char* path, char buffer[], size_t* bufferSize) {
771     const auto res = ::getxattr(path, kMetadataAttrName, buffer, *bufferSize);
772     if (res < 0) {
773         if (errno == ERANGE) {
774             auto neededSize = ::getxattr(path, kMetadataAttrName, buffer, 0);
775             if (neededSize >= 0) {
776                 *bufferSize = neededSize;
777                 return 0;
778             }
779         }
780         return -errno;
781     }
782     *bufferSize = res;
783     return 0;
784 }
785 
786 IncFsErrorCode IncFs_GetMetadataById(const IncFsControl* control, IncFsFileId fileId, char buffer[],
787                                      size_t* bufferSize) {
788     if (!control) {
789         return -EINVAL;
790     }
791 
792     const auto root = rootForCmd(control->cmd);
793     if (root.empty()) {
794         return -EINVAL;
795     }
796     auto name = path::join(root, kIndexDir, toStringImpl(fileId));
797     return getMetadata(details::c_str(name), buffer, bufferSize);
798 }
799 
800 IncFsErrorCode IncFs_GetMetadataByPath(const IncFsControl* control, const char* path, char buffer[],
801                                        size_t* bufferSize) {
802     if (!control) {
803         return -EINVAL;
804     }
805     const auto pathRoot = registry().rootFor(path);
806     const auto root = rootForCmd(control->cmd);
807     if (root.empty() || root != pathRoot) {
808         return -EINVAL;
809     }
810 
811     return getMetadata(path, buffer, bufferSize);
812 }
813 
814 IncFsFileId IncFs_GetId(const IncFsControl* control, const char* path) {
815     if (!control) {
816         return kIncFsInvalidFileId;
817     }
818     const auto pathRoot = registry().rootFor(path);
819     const auto root = rootForCmd(control->cmd);
820     if (root.empty() || root != pathRoot) {
821         errno = EINVAL;
822         return kIncFsInvalidFileId;
823     }
824     char buffer[kIncFsFileIdStringLength];
825     const auto res = ::getxattr(path, kIdAttrName, buffer, sizeof(buffer));
826     if (res != sizeof(buffer)) {
827         return kIncFsInvalidFileId;
828     }
829     return toFileIdImpl({buffer, std::size(buffer)});
830 }
831 
832 static IncFsErrorCode getSignature(int fd, char buffer[], size_t* bufferSize) {
833     incfs_get_file_sig_args args = {
834             .file_signature = (uint64_t)buffer,
835             .file_signature_buf_size = (uint32_t)*bufferSize,
836     };
837 
838     auto res = ::ioctl(fd, INCFS_IOC_READ_FILE_SIGNATURE, &args);
839     if (res < 0) {
840         if (errno == E2BIG) {
841             *bufferSize = INCFS_MAX_SIGNATURE_SIZE;
842         }
843         return -errno;
844     }
845     *bufferSize = args.file_signature_len_out;
846     return 0;
847 }
848 
849 IncFsErrorCode IncFs_GetSignatureById(const IncFsControl* control, IncFsFileId fileId,
850                                       char buffer[], size_t* bufferSize) {
851     if (!control) {
852         return -EINVAL;
853     }
854 
855     const auto root = rootForCmd(control->cmd);
856     if (root.empty()) {
857         return -EINVAL;
858     }
859     auto file = path::join(root, kIndexDir, toStringImpl(fileId));
860     auto fd = openRaw(file);
861     if (fd < 0) {
862         return fd.get();
863     }
864     return getSignature(fd, buffer, bufferSize);
865 }
866 
867 IncFsErrorCode IncFs_GetSignatureByPath(const IncFsControl* control, const char* path,
868                                         char buffer[], size_t* bufferSize) {
869     if (!control) {
870         return -EINVAL;
871     }
872 
873     const auto pathRoot = registry().rootFor(path);
874     const auto root = rootForCmd(control->cmd);
875     if (root.empty() || root != pathRoot) {
876         return -EINVAL;
877     }
878     return IncFs_UnsafeGetSignatureByPath(path, buffer, bufferSize);
879 }
880 
881 IncFsErrorCode IncFs_UnsafeGetSignatureByPath(const char* path, char buffer[], size_t* bufferSize) {
882     if (!isIncFsPath(path)) {
883         return -EINVAL;
884     }
885     auto fd = openRaw(path);
886     if (fd < 0) {
887         return fd.get();
888     }
889     return getSignature(fd, buffer, bufferSize);
890 }
891 
892 IncFsErrorCode IncFs_Link(const IncFsControl* control, const char* fromPath,
893                           const char* wherePath) {
894     if (!control) {
895         return -EINVAL;
896     }
897 
898     auto root = rootForCmd(control->cmd);
899     if (root.empty()) {
900         return -EINVAL;
901     }
902     auto cmdFrom = makeCommandPath(root, fromPath);
903     if (cmdFrom.empty()) {
904         return -EINVAL;
905     }
906     auto cmdWhere = makeCommandPath(root, wherePath);
907     if (cmdWhere.empty()) {
908         return -EINVAL;
909     }
910     if (::link(cmdFrom.c_str(), cmdWhere.c_str())) {
911         return -errno;
912     }
913     return 0;
914 }
915 
916 IncFsErrorCode IncFs_Unlink(const IncFsControl* control, const char* path) {
917     if (!control) {
918         return -EINVAL;
919     }
920 
921     auto root = rootForCmd(control->cmd);
922     if (root.empty()) {
923         return -EINVAL;
924     }
925     auto cmdPath = makeCommandPath(root, path);
926     if (cmdPath.empty()) {
927         return -EINVAL;
928     }
929     if (::unlink(cmdPath.c_str())) {
930         if (errno == EISDIR) {
931             if (!::rmdir(cmdPath.c_str())) {
932                 return 0;
933             }
934         }
935         return -errno;
936     }
937     return 0;
938 }
939 
940 static int waitForReads(int fd, int32_t timeoutMs, incfs_pending_read_info pendingReadsBuffer[],
941                         size_t* pendingReadsBufferSize) {
942     using namespace std::chrono;
943     auto hrTimeout = steady_clock::duration(milliseconds(timeoutMs));
944 
945     while (hrTimeout > hrTimeout.zero() || (!pendingReadsBuffer && hrTimeout == hrTimeout.zero())) {
946         const auto startTs = steady_clock::now();
947 
948         pollfd pfd = {fd, POLLIN, 0};
949         const auto res = ::poll(&pfd, 1, duration_cast<milliseconds>(hrTimeout).count());
950         if (res > 0) {
951             break;
952         }
953         if (res == 0) {
954             if (pendingReadsBufferSize) {
955                 *pendingReadsBufferSize = 0;
956             }
957             return -ETIMEDOUT;
958         }
959         const auto error = errno;
960         if (error != EINTR) {
961             PLOG(ERROR) << "poll() failed";
962             return -error;
963         }
964         hrTimeout -= steady_clock::now() - startTs;
965     }
966     if (!pendingReadsBuffer) {
967         return hrTimeout < hrTimeout.zero() ? -ETIMEDOUT : 0;
968     }
969 
970     auto res =
971             ::read(fd, pendingReadsBuffer, *pendingReadsBufferSize * sizeof(*pendingReadsBuffer));
972     if (res < 0) {
973         const auto error = errno;
974         PLOG(ERROR) << "read() failed";
975         return -error;
976     }
977     if (res == 0) {
978         *pendingReadsBufferSize = 0;
979         return -ETIMEDOUT;
980     }
981     if ((res % sizeof(*pendingReadsBuffer)) != 0) {
982         PLOG(ERROR) << "read() returned half of a struct??";
983         return -EFAULT;
984     }
985     *pendingReadsBufferSize = res / sizeof(*pendingReadsBuffer);
986     return 0;
987 }
988 
989 IncFsErrorCode IncFs_WaitForPendingReads(const IncFsControl* control, int32_t timeoutMs,
990                                          IncFsReadInfo buffer[], size_t* bufferSize) {
991     if (!control || control->pendingReads < 0) {
992         return -EINVAL;
993     }
994 
995     std::vector<incfs_pending_read_info> pendingReads;
996     pendingReads.resize(*bufferSize);
997     if (const auto res =
998                 waitForReads(control->pendingReads, timeoutMs, pendingReads.data(), bufferSize)) {
999         return res;
1000     }
1001     for (size_t i = 0; i != *bufferSize; ++i) {
1002         buffer[i] = IncFsReadInfo{
1003                 .bootClockTsUs = pendingReads[i].timestamp_us,
1004                 .block = (IncFsBlockIndex)pendingReads[i].block_index,
1005                 .serialNo = pendingReads[i].serial_number,
1006         };
1007         memcpy(&buffer[i].id.data, pendingReads[i].file_id.bytes, sizeof(buffer[i].id.data));
1008     }
1009     return 0;
1010 }
1011 
1012 IncFsErrorCode IncFs_WaitForPageReads(const IncFsControl* control, int32_t timeoutMs,
1013                                       IncFsReadInfo buffer[], size_t* bufferSize) {
1014     if (!control) {
1015         return -EINVAL;
1016     }
1017 
1018     auto logsFd = control->logs;
1019     if (logsFd < 0) {
1020         return -EINVAL;
1021     }
1022     std::vector<incfs_pending_read_info> pendingReads;
1023     pendingReads.resize(*bufferSize);
1024     if (const auto res = waitForReads(logsFd, timeoutMs, pendingReads.data(), bufferSize)) {
1025         return res;
1026     }
1027     for (size_t i = 0; i != *bufferSize; ++i) {
1028         buffer[i] = IncFsReadInfo{
1029                 .bootClockTsUs = pendingReads[i].timestamp_us,
1030                 .block = (IncFsBlockIndex)pendingReads[i].block_index,
1031                 .serialNo = pendingReads[i].serial_number,
1032         };
1033         memcpy(&buffer[i].id.data, pendingReads[i].file_id.bytes, sizeof(buffer[i].id.data));
1034     }
1035     return 0;
1036 }
1037 
1038 static IncFsFd openForSpecialOps(int cmd, const char* path) {
1039     ab::unique_fd fd(::open(path, O_RDONLY | O_CLOEXEC));
1040     if (fd < 0) {
1041         return -errno;
1042     }
1043     struct incfs_permit_fill args = {.file_descriptor = (uint32_t)fd.get()};
1044     auto err = ::ioctl(cmd, INCFS_IOC_PERMIT_FILL, &args);
1045     if (err < 0) {
1046         return -errno;
1047     }
1048     return fd.release();
1049 }
1050 
1051 IncFsFd IncFs_OpenForSpecialOpsByPath(const IncFsControl* control, const char* path) {
1052     if (!control) {
1053         return -EINVAL;
1054     }
1055 
1056     const auto pathRoot = registry().rootFor(path);
1057     const auto cmd = control->cmd;
1058     const auto root = rootForCmd(cmd);
1059     if (root.empty() || root != pathRoot) {
1060         return -EINVAL;
1061     }
1062     return openForSpecialOps(cmd, makeCommandPath(root, path).c_str());
1063 }
1064 
1065 IncFsFd IncFs_OpenForSpecialOpsById(const IncFsControl* control, IncFsFileId id) {
1066     if (!control) {
1067         return -EINVAL;
1068     }
1069 
1070     const auto cmd = control->cmd;
1071     const auto root = rootForCmd(cmd);
1072     if (root.empty()) {
1073         return -EINVAL;
1074     }
1075     auto name = path::join(root, kIndexDir, toStringImpl(id));
1076     return openForSpecialOps(cmd, makeCommandPath(root, name).c_str());
1077 }
1078 
1079 static int writeBlocks(int fd, const incfs_fill_block blocks[], int blocksCount) {
1080     if (fd < 0 || blocksCount == 0) {
1081         return 0;
1082     }
1083     if (blocksCount < 0) {
1084         return -EINVAL;
1085     }
1086 
1087     auto ptr = blocks;
1088     const auto end = blocks + blocksCount;
1089     do {
1090         struct incfs_fill_blocks args = {.count = uint64_t(end - ptr),
1091                                          .fill_blocks = (uint64_t)(uintptr_t)ptr};
1092         const auto written = ::ioctl(fd, INCFS_IOC_FILL_BLOCKS, &args);
1093         if (written < 0) {
1094             if (errno == EINTR) {
1095                 continue;
1096             }
1097             const auto error = errno;
1098             PLOG(WARNING) << "writing IncFS blocks failed";
1099             if (ptr == blocks) {
1100                 return -error;
1101             }
1102             // something has been written, return a success here and let the
1103             // next call handle the error.
1104             break;
1105         }
1106         ptr += written;
1107     } while (ptr < end);
1108     return ptr - blocks;
1109 }
1110 
1111 IncFsErrorCode IncFs_WriteBlocks(const IncFsDataBlock blocks[], size_t blocksCount) {
1112     incfs_fill_block incfsBlocks[128];
1113     int writtenCount = 0;
1114     int incfsBlocksUsed = 0;
1115     int lastBlockFd = -1;
1116     for (size_t i = 0; i < blocksCount; ++i) {
1117         if (lastBlockFd != blocks[i].fileFd || incfsBlocksUsed == std::size(incfsBlocks)) {
1118             auto count = writeBlocks(lastBlockFd, incfsBlocks, incfsBlocksUsed);
1119             if (count > 0) {
1120                 writtenCount += count;
1121             }
1122             if (count != incfsBlocksUsed) {
1123                 return writtenCount ? writtenCount : count;
1124             }
1125             lastBlockFd = blocks[i].fileFd;
1126             incfsBlocksUsed = 0;
1127         }
1128         incfsBlocks[incfsBlocksUsed] = incfs_fill_block{
1129                 .block_index = (uint32_t)blocks[i].pageIndex,
1130                 .data_len = blocks[i].dataSize,
1131                 .data = (uint64_t)blocks[i].data,
1132                 .compression = (uint8_t)blocks[i].compression,
1133                 .flags = uint8_t(blocks[i].kind == INCFS_BLOCK_KIND_HASH ? INCFS_BLOCK_FLAGS_HASH
1134                                                                          : 0),
1135         };
1136         ++incfsBlocksUsed;
1137     }
1138     auto count = writeBlocks(lastBlockFd, incfsBlocks, incfsBlocksUsed);
1139     if (count > 0) {
1140         writtenCount += count;
1141     }
1142     return writtenCount ? writtenCount : count;
1143 }
1144 
1145 IncFsErrorCode IncFs_BindMount(const char* sourceDir, const char* targetDir) {
1146     if (!enabled()) {
1147         return -ENOTSUP;
1148     }
1149 
1150     auto [sourceRoot, subpath] = registry().rootAndSubpathFor(sourceDir);
1151     if (sourceRoot.empty()) {
1152         return -EINVAL;
1153     }
1154     if (subpath.empty()) {
1155         LOG(WARNING) << "[incfs] Binding the root mount '" << sourceRoot << "' is not allowed";
1156         return -EINVAL;
1157     }
1158 
1159     if (auto err = isValidMountTarget(targetDir); err != 0) {
1160         return err;
1161     }
1162 
1163     if (::mount(sourceDir, targetDir, nullptr, MS_BIND, nullptr)) {
1164         PLOG(ERROR) << "[incfs] Failed to bind mount '" << sourceDir << "' to '" << targetDir
1165                     << '\'';
1166         return -errno;
1167     }
1168     return 0;
1169 }
1170 
1171 IncFsErrorCode IncFs_Unmount(const char* dir) {
1172     if (!enabled()) {
1173         return -ENOTSUP;
1174     }
1175 
1176     errno = 0;
1177     if (::umount2(dir, MNT_FORCE) == 0 || errno == EINVAL || errno == ENOENT) {
1178         // EINVAL - not a mount point, ENOENT - doesn't exist at all
1179         return -errno;
1180     }
1181     PLOG(WARNING) << __func__ << ": umount(force) failed, detaching '" << dir << '\'';
1182     errno = 0;
1183     if (!::umount2(dir, MNT_DETACH)) {
1184         return 0;
1185     }
1186     PLOG(WARNING) << __func__ << ": umount(detach) returned non-zero for '" << dir << '\'';
1187     return 0;
1188 }
1189 
1190 bool IncFs_IsIncFsPath(const char* path) {
1191     return isIncFsPath(path);
1192 }
1193 
1194 IncFsErrorCode IncFs_GetFilledRanges(int fd, IncFsSpan outBuffer, IncFsFilledRanges* filledRanges) {
1195     return IncFs_GetFilledRangesStartingFrom(fd, 0, outBuffer, filledRanges);
1196 }
1197 
1198 IncFsErrorCode IncFs_GetFilledRangesStartingFrom(int fd, int startBlockIndex, IncFsSpan outBuffer,
1199                                                  IncFsFilledRanges* filledRanges) {
1200     if (fd < 0) {
1201         return -EBADF;
1202     }
1203     if (startBlockIndex < 0) {
1204         return -EINVAL;
1205     }
1206     if (!outBuffer.data && outBuffer.size > 0) {
1207         return -EINVAL;
1208     }
1209     if (!filledRanges) {
1210         return -EINVAL;
1211     }
1212     // Use this to optimize the incfs call and have the same buffer for both the incfs and the
1213     // public structs.
1214     static_assert(sizeof(IncFsBlockRange) == sizeof(incfs_filled_range));
1215 
1216     *filledRanges = {};
1217 
1218     auto outStart = (IncFsBlockRange*)outBuffer.data;
1219     auto outEnd = outStart + outBuffer.size / sizeof(*outStart);
1220 
1221     auto outPtr = outStart;
1222     int error = 0;
1223     int dataBlocks;
1224     incfs_get_filled_blocks_args args = {};
1225     for (;;) {
1226         auto start = args.index_out ? args.index_out : startBlockIndex;
1227         args = incfs_get_filled_blocks_args{
1228                 .range_buffer = (uint64_t)(uintptr_t)outPtr,
1229                 .range_buffer_size = uint32_t((outEnd - outPtr) * sizeof(*outPtr)),
1230                 .start_index = start,
1231         };
1232         errno = 0;
1233         auto res = ::ioctl(fd, INCFS_IOC_GET_FILLED_BLOCKS, &args);
1234         error = errno;
1235         if (res && error != EINTR && error != ERANGE) {
1236             return -error;
1237         }
1238 
1239         dataBlocks = args.data_blocks_out;
1240         outPtr += args.range_buffer_size_out / sizeof(incfs_filled_range);
1241         if (!res || error == ERANGE) {
1242             break;
1243         }
1244         // in case of EINTR we want to continue calling the function
1245     }
1246 
1247     if (outPtr > outEnd) {
1248         outPtr = outEnd;
1249         error = ERANGE;
1250     }
1251 
1252     filledRanges->endIndex = args.index_out;
1253     auto hashStartPtr = outPtr;
1254     if (outPtr != outStart) {
1255         // figure out the ranges for data block and hash blocks in the output
1256         for (; hashStartPtr != outStart; --hashStartPtr) {
1257             if ((hashStartPtr - 1)->begin < dataBlocks) {
1258                 break;
1259             }
1260         }
1261         auto lastDataPtr = hashStartPtr - 1;
1262         // here we go, this is the first block that's before or at the hashes
1263         if (lastDataPtr->end <= dataBlocks) {
1264             ; // we're good, the boundary is between the ranges - |hashStartPtr| is correct
1265         } else {
1266             // the hard part: split the |lastDataPtr| range into the data and the hash pieces
1267             if (outPtr == outEnd) {
1268                 // the buffer turned out to be too small, even though it actually wasn't
1269                 error = ERANGE;
1270                 if (hashStartPtr == outEnd) {
1271                     // this is even worse: there's no room to put even a single hash block into.
1272                     filledRanges->endIndex = lastDataPtr->end = dataBlocks;
1273                 } else {
1274                     std::copy_backward(lastDataPtr, outPtr - 1, outPtr);
1275                     lastDataPtr->end = hashStartPtr->begin = dataBlocks;
1276                     filledRanges->endIndex = (outPtr - 1)->end;
1277                 }
1278             } else {
1279                 std::copy_backward(lastDataPtr, outPtr, outPtr + 1);
1280                 lastDataPtr->end = hashStartPtr->begin = dataBlocks;
1281                 ++outPtr;
1282             }
1283         }
1284         // now fix the indices of all hash blocks - no one should know they're simply past the
1285         // regular data blocks in the file!
1286         for (auto ptr = hashStartPtr; ptr != outPtr; ++ptr) {
1287             ptr->begin -= dataBlocks;
1288             ptr->end -= dataBlocks;
1289         }
1290     }
1291 
1292     filledRanges->dataRanges = outStart;
1293     filledRanges->dataRangesCount = hashStartPtr - outStart;
1294     filledRanges->hashRanges = hashStartPtr;
1295     filledRanges->hashRangesCount = outPtr - hashStartPtr;
1296 
1297     return -error;
1298 }
1299 
1300 IncFsErrorCode IncFs_IsFullyLoaded(int fd) {
1301     char buffer[2 * sizeof(IncFsBlockRange)];
1302     IncFsFilledRanges ranges;
1303     auto res = IncFs_GetFilledRanges(fd, IncFsSpan{.data = buffer, .size = std::size(buffer)},
1304                                      &ranges);
1305     if (res == -ERANGE) {
1306         // need room for more than two ranges - definitely not fully loaded
1307         return -ENODATA;
1308     }
1309     if (res != 0) {
1310         return res;
1311     }
1312     // empty file
1313     if (ranges.endIndex == 0) {
1314         return 0;
1315     }
1316     // file with no hash tree
1317     if (ranges.dataRangesCount == 1 && ranges.hashRangesCount == 0) {
1318         return (ranges.dataRanges[0].begin == 0 && ranges.dataRanges[0].end == ranges.endIndex)
1319                 ? 0
1320                 : -ENODATA;
1321     }
1322     // file with a hash tree
1323     if (ranges.dataRangesCount == 1 && ranges.hashRangesCount == 1) {
1324         // calculate the expected data size from the size of the hash range and |endIndex|, which is
1325         // the total number of blocks in the file, both data and hash blocks together.
1326         if (ranges.hashRanges[0].begin != 0) {
1327             return -ENODATA;
1328         }
1329         const auto expectedDataBlocks =
1330                 ranges.endIndex - (ranges.hashRanges[0].end - ranges.hashRanges[0].begin);
1331         return (ranges.dataRanges[0].begin == 0 && ranges.dataRanges[0].end == expectedDataBlocks)
1332                 ? 0
1333                 : -ENODATA;
1334     }
1335     return -ENODATA;
1336 }
1337 
1338 MountRegistry& android::incfs::defaultMountRegistry() {
1339     return registry();
1340 }
1341