1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <fcntl.h>
18 #include <inttypes.h>
19 #include <stdint.h>
20 #include <stdlib.h>
21 #include <string.h>
22 #include <sys/mount.h>
23 #include <sys/stat.h>
24 #include <sys/types.h>
25 #include <sys/vfs.h>
26 #include <unistd.h>
27
28 #include <string>
29
30 #include <android-base/file.h>
31 #include <android-base/logging.h>
32 #include <android-base/stringprintf.h>
33 #include <android-base/unique_fd.h>
34 #include <gtest/gtest.h>
35 #include <libdm/loop_control.h>
36 #include <libfiemap/fiemap_writer.h>
37 #include <libfiemap/split_fiemap_writer.h>
38 #include <libgsi/libgsi.h>
39
40 #include "utility.h"
41
42 namespace android {
43 namespace fiemap {
44
45 using namespace std;
46 using namespace std::string_literals;
47 using namespace android::fiemap;
48 using unique_fd = android::base::unique_fd;
49 using LoopDevice = android::dm::LoopDevice;
50
51 std::string gTestDir;
52 uint64_t testfile_size = 536870912; // default of 512MiB
53 size_t gBlockSize = 0;
54
55 class FiemapWriterTest : public ::testing::Test {
56 protected:
SetUp()57 void SetUp() override {
58 const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
59 testfile = gTestDir + "/"s + tinfo->name();
60 }
61
TearDown()62 void TearDown() override { unlink(testfile.c_str()); }
63
64 // name of the file we use for testing
65 std::string testfile;
66 };
67
68 class SplitFiemapTest : public ::testing::Test {
69 protected:
SetUp()70 void SetUp() override {
71 const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
72 testfile = gTestDir + "/"s + tinfo->name();
73 }
74
TearDown()75 void TearDown() override {
76 std::string message;
77 if (!SplitFiemap::RemoveSplitFiles(testfile, &message)) {
78 cerr << "Could not remove all split files: " << message;
79 }
80 }
81
82 // name of the file we use for testing
83 std::string testfile;
84 };
85
TEST_F(FiemapWriterTest,CreateImpossiblyLargeFile)86 TEST_F(FiemapWriterTest, CreateImpossiblyLargeFile) {
87 // Try creating a file of size ~100TB but aligned to
88 // 512 byte to make sure block alignment tests don't
89 // fail.
90 FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1099511627997184);
91 EXPECT_EQ(fptr, nullptr);
92 EXPECT_EQ(access(testfile.c_str(), F_OK), -1);
93 EXPECT_EQ(errno, ENOENT);
94 }
95
TEST_F(FiemapWriterTest,CreateUnalignedFile)96 TEST_F(FiemapWriterTest, CreateUnalignedFile) {
97 // Try creating a file of size 4097 bytes which is guaranteed
98 // to be unaligned to all known block sizes.
99 FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize + 1);
100 ASSERT_NE(fptr, nullptr);
101 ASSERT_EQ(fptr->size(), gBlockSize * 2);
102 }
103
TEST_F(FiemapWriterTest,CheckFilePath)104 TEST_F(FiemapWriterTest, CheckFilePath) {
105 FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
106 ASSERT_NE(fptr, nullptr);
107 EXPECT_EQ(fptr->size(), gBlockSize);
108 EXPECT_EQ(fptr->file_path(), testfile);
109 EXPECT_EQ(access(testfile.c_str(), F_OK), 0);
110 }
111
TEST_F(FiemapWriterTest,CheckFileSize)112 TEST_F(FiemapWriterTest, CheckFileSize) {
113 // Create a large-ish file and test that the expected size matches.
114 FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1024 * 1024 * 16);
115 ASSERT_NE(fptr, nullptr);
116
117 struct stat s;
118 ASSERT_EQ(stat(testfile.c_str(), &s), 0);
119 EXPECT_EQ(static_cast<uint64_t>(s.st_size), fptr->size());
120 }
121
TEST_F(FiemapWriterTest,CheckProgress)122 TEST_F(FiemapWriterTest, CheckProgress) {
123 std::vector<uint64_t> expected;
124 size_t invocations = 0;
125 auto callback = [&](uint64_t done, uint64_t total) -> bool {
126 if (invocations >= expected.size()) {
127 return false;
128 }
129 EXPECT_EQ(done, expected[invocations]);
130 EXPECT_EQ(total, gBlockSize);
131 invocations++;
132 return true;
133 };
134
135 expected.push_back(gBlockSize);
136
137 auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback));
138 EXPECT_NE(ptr, nullptr);
139 EXPECT_EQ(invocations, expected.size());
140 }
141
TEST_F(FiemapWriterTest,CheckPinning)142 TEST_F(FiemapWriterTest, CheckPinning) {
143 auto ptr = FiemapWriter::Open(testfile, 4096);
144 ASSERT_NE(ptr, nullptr);
145 EXPECT_TRUE(FiemapWriter::HasPinnedExtents(testfile));
146 }
147
TEST_F(FiemapWriterTest,CheckBlockDevicePath)148 TEST_F(FiemapWriterTest, CheckBlockDevicePath) {
149 FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
150 EXPECT_EQ(fptr->size(), gBlockSize);
151 EXPECT_EQ(fptr->bdev_path().find("/dev/block/"), size_t(0));
152
153 if (!android::gsi::IsGsiRunning()) {
154 EXPECT_EQ(fptr->bdev_path().find("/dev/block/dm-"), string::npos);
155 }
156 }
157
TEST_F(FiemapWriterTest,CheckFileCreated)158 TEST_F(FiemapWriterTest, CheckFileCreated) {
159 FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 32768);
160 ASSERT_NE(fptr, nullptr);
161 unique_fd fd(open(testfile.c_str(), O_RDONLY));
162 EXPECT_GT(fd, -1);
163 }
164
TEST_F(FiemapWriterTest,CheckFileSizeActual)165 TEST_F(FiemapWriterTest, CheckFileSizeActual) {
166 FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size);
167 ASSERT_NE(fptr, nullptr);
168
169 struct stat sb;
170 ASSERT_EQ(stat(testfile.c_str(), &sb), 0);
171 EXPECT_GE(sb.st_size, testfile_size);
172 }
173
TEST_F(FiemapWriterTest,CheckFileExtents)174 TEST_F(FiemapWriterTest, CheckFileExtents) {
175 FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size);
176 ASSERT_NE(fptr, nullptr);
177 EXPECT_GT(fptr->extents().size(), 0);
178 }
179
TEST_F(FiemapWriterTest,ExistingFile)180 TEST_F(FiemapWriterTest, ExistingFile) {
181 // Create the file.
182 { ASSERT_NE(FiemapWriter::Open(testfile, gBlockSize), nullptr); }
183 // Test that we can still open it.
184 {
185 auto ptr = FiemapWriter::Open(testfile, 0, false);
186 ASSERT_NE(ptr, nullptr);
187 EXPECT_GT(ptr->extents().size(), 0);
188 }
189 }
190
TEST_F(FiemapWriterTest,FileDeletedOnError)191 TEST_F(FiemapWriterTest, FileDeletedOnError) {
192 auto callback = [](uint64_t, uint64_t) -> bool { return false; };
193 auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback));
194 EXPECT_EQ(ptr, nullptr);
195 EXPECT_EQ(access(testfile.c_str(), F_OK), -1);
196 EXPECT_EQ(errno, ENOENT);
197 }
198
TEST_F(FiemapWriterTest,MaxBlockSize)199 TEST_F(FiemapWriterTest, MaxBlockSize) {
200 uint64_t max_piece_size = 0;
201 ASSERT_TRUE(DetermineMaximumFileSize(testfile, &max_piece_size));
202 ASSERT_GT(max_piece_size, 0);
203 }
204
TEST_F(FiemapWriterTest,FibmapBlockAddressing)205 TEST_F(FiemapWriterTest, FibmapBlockAddressing) {
206 FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
207 ASSERT_NE(fptr, nullptr);
208
209 switch (fptr->fs_type()) {
210 case F2FS_SUPER_MAGIC:
211 case EXT4_SUPER_MAGIC:
212 // Skip the test for FIEMAP supported filesystems. This is really
213 // because f2fs/ext4 have caches that seem to defeat reading back
214 // directly from the block device, and writing directly is too
215 // dangerous.
216 std::cout << "Skipping test, filesystem does not use FIBMAP\n";
217 return;
218 }
219
220 bool uses_dm;
221 std::string bdev_path;
222 ASSERT_TRUE(FiemapWriter::GetBlockDeviceForFile(testfile, &bdev_path, &uses_dm));
223
224 if (uses_dm) {
225 // We could use a device-mapper wrapper here to bypass encryption, but
226 // really this test is for FIBMAP correctness on VFAT (where encryption
227 // is never used), so we don't bother.
228 std::cout << "Skipping test, block device is metadata encrypted\n";
229 return;
230 }
231
232 std::string data(fptr->size(), '\0');
233 for (size_t i = 0; i < data.size(); i++) {
234 data[i] = 'A' + static_cast<char>(data.size() % 26);
235 }
236
237 {
238 unique_fd fd(open(testfile.c_str(), O_WRONLY | O_CLOEXEC));
239 ASSERT_GE(fd, 0);
240 ASSERT_TRUE(android::base::WriteFully(fd, data.data(), data.size()));
241 ASSERT_EQ(fsync(fd), 0);
242 }
243
244 ASSERT_FALSE(fptr->extents().empty());
245 const auto& first_extent = fptr->extents()[0];
246
247 unique_fd bdev(open(fptr->bdev_path().c_str(), O_RDONLY | O_CLOEXEC));
248 ASSERT_GE(bdev, 0);
249
250 off_t where = first_extent.fe_physical;
251 ASSERT_EQ(lseek(bdev, where, SEEK_SET), where);
252
253 // Note: this will fail on encrypted folders.
254 std::string actual(data.size(), '\0');
255 ASSERT_GE(first_extent.fe_length, data.size());
256 ASSERT_TRUE(android::base::ReadFully(bdev, actual.data(), actual.size()));
257 EXPECT_EQ(memcmp(actual.data(), data.data(), data.size()), 0);
258 }
259
TEST_F(SplitFiemapTest,Create)260 TEST_F(SplitFiemapTest, Create) {
261 auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32);
262 ASSERT_NE(ptr, nullptr);
263
264 auto extents = ptr->extents();
265
266 // Destroy the fiemap, closing file handles. This should not delete them.
267 ptr = nullptr;
268
269 std::vector<std::string> files;
270 ASSERT_TRUE(SplitFiemap::GetSplitFileList(testfile, &files));
271 for (const auto& path : files) {
272 EXPECT_EQ(access(path.c_str(), F_OK), 0);
273 }
274
275 ASSERT_GE(extents.size(), files.size());
276 }
277
TEST_F(SplitFiemapTest,Open)278 TEST_F(SplitFiemapTest, Open) {
279 {
280 auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32);
281 ASSERT_NE(ptr, nullptr);
282 }
283
284 auto ptr = SplitFiemap::Open(testfile);
285 ASSERT_NE(ptr, nullptr);
286
287 auto extents = ptr->extents();
288 ASSERT_GE(extents.size(), 24);
289 }
290
TEST_F(SplitFiemapTest,DeleteOnFail)291 TEST_F(SplitFiemapTest, DeleteOnFail) {
292 auto ptr = SplitFiemap::Create(testfile, 1024 * 1024 * 100, 1);
293 ASSERT_EQ(ptr, nullptr);
294
295 std::string first_file = testfile + ".0001";
296 ASSERT_NE(access(first_file.c_str(), F_OK), 0);
297 ASSERT_EQ(errno, ENOENT);
298 ASSERT_NE(access(testfile.c_str(), F_OK), 0);
299 ASSERT_EQ(errno, ENOENT);
300 }
301
ReadSplitFiles(const std::string & base_path,size_t num_files)302 static string ReadSplitFiles(const std::string& base_path, size_t num_files) {
303 std::string result;
304 for (int i = 0; i < num_files; i++) {
305 std::string path = base_path + android::base::StringPrintf(".%04d", i);
306 std::string data;
307 if (!android::base::ReadFileToString(path, &data)) {
308 return {};
309 }
310 result += data;
311 }
312 return result;
313 }
314
TEST_F(SplitFiemapTest,WriteWholeFile)315 TEST_F(SplitFiemapTest, WriteWholeFile) {
316 static constexpr size_t kChunkSize = 32768;
317 static constexpr size_t kSize = kChunkSize * 3;
318 auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
319 ASSERT_NE(ptr, nullptr);
320
321 auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
322 for (size_t i = 0; i < kSize / sizeof(int); i++) {
323 buffer[i] = i;
324 }
325 ASSERT_TRUE(ptr->Write(buffer.get(), kSize));
326
327 std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
328 auto actual = ReadSplitFiles(testfile, 3);
329 ASSERT_EQ(expected.size(), actual.size());
330 EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
331 }
332
TEST_F(SplitFiemapTest,WriteFileInChunks1)333 TEST_F(SplitFiemapTest, WriteFileInChunks1) {
334 static constexpr size_t kChunkSize = 32768;
335 static constexpr size_t kSize = kChunkSize * 3;
336 auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
337 ASSERT_NE(ptr, nullptr);
338
339 auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
340 for (size_t i = 0; i < kSize / sizeof(int); i++) {
341 buffer[i] = i;
342 }
343
344 // Write in chunks of 1000 (so some writes straddle the boundary of two
345 // files).
346 size_t bytes_written = 0;
347 while (bytes_written < kSize) {
348 size_t to_write = std::min(kSize - bytes_written, (size_t)1000);
349 char* data = reinterpret_cast<char*>(buffer.get()) + bytes_written;
350 ASSERT_TRUE(ptr->Write(data, to_write));
351 bytes_written += to_write;
352 }
353
354 std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
355 auto actual = ReadSplitFiles(testfile, 3);
356 ASSERT_EQ(expected.size(), actual.size());
357 EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
358 }
359
TEST_F(SplitFiemapTest,WriteFileInChunks2)360 TEST_F(SplitFiemapTest, WriteFileInChunks2) {
361 static constexpr size_t kChunkSize = 32768;
362 static constexpr size_t kSize = kChunkSize * 3;
363 auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
364 ASSERT_NE(ptr, nullptr);
365
366 auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
367 for (size_t i = 0; i < kSize / sizeof(int); i++) {
368 buffer[i] = i;
369 }
370
371 // Write in chunks of 32KiB so every write is exactly at the end of the
372 // current file.
373 size_t bytes_written = 0;
374 while (bytes_written < kSize) {
375 size_t to_write = std::min(kSize - bytes_written, kChunkSize);
376 char* data = reinterpret_cast<char*>(buffer.get()) + bytes_written;
377 ASSERT_TRUE(ptr->Write(data, to_write));
378 bytes_written += to_write;
379 }
380
381 std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
382 auto actual = ReadSplitFiles(testfile, 3);
383 ASSERT_EQ(expected.size(), actual.size());
384 EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
385 }
386
TEST_F(SplitFiemapTest,WritePastEnd)387 TEST_F(SplitFiemapTest, WritePastEnd) {
388 static constexpr size_t kChunkSize = 32768;
389 static constexpr size_t kSize = kChunkSize * 3;
390 auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
391 ASSERT_NE(ptr, nullptr);
392
393 auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
394 for (size_t i = 0; i < kSize / sizeof(int); i++) {
395 buffer[i] = i;
396 }
397 ASSERT_TRUE(ptr->Write(buffer.get(), kSize));
398 ASSERT_FALSE(ptr->Write(buffer.get(), kSize));
399 }
400
401 class VerifyBlockWritesExt4 : public ::testing::Test {
402 // 2GB Filesystem and 4k block size by default
403 static constexpr uint64_t block_size = 4096;
404 static constexpr uint64_t fs_size = 2147483648;
405
406 protected:
SetUp()407 void SetUp() override {
408 fs_path = std::string(getenv("TMPDIR")) + "/ext4_2G.img";
409 uint64_t count = fs_size / block_size;
410 std::string dd_cmd =
411 ::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64
412 " count=%" PRIu64 " > /dev/null 2>&1",
413 fs_path.c_str(), block_size, count);
414 std::string mkfs_cmd =
415 ::android::base::StringPrintf("/system/bin/mkfs.ext4 -q %s", fs_path.c_str());
416 // create mount point
417 mntpoint = std::string(getenv("TMPDIR")) + "/fiemap_mnt";
418 ASSERT_EQ(mkdir(mntpoint.c_str(), S_IRWXU), 0);
419 // create file for the file system
420 int ret = system(dd_cmd.c_str());
421 ASSERT_EQ(ret, 0);
422 // Get and attach a loop device to the filesystem we created
423 LoopDevice loop_dev(fs_path, 10s);
424 ASSERT_TRUE(loop_dev.valid());
425 // create file system
426 ret = system(mkfs_cmd.c_str());
427 ASSERT_EQ(ret, 0);
428
429 // mount the file system
430 ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint.c_str(), "ext4", 0, nullptr), 0);
431 }
432
TearDown()433 void TearDown() override {
434 umount(mntpoint.c_str());
435 rmdir(mntpoint.c_str());
436 unlink(fs_path.c_str());
437 }
438
439 std::string mntpoint;
440 std::string fs_path;
441 };
442
443 class VerifyBlockWritesF2fs : public ::testing::Test {
444 // 2GB Filesystem and 4k block size by default
445 static constexpr uint64_t block_size = 4096;
446 static constexpr uint64_t fs_size = 2147483648;
447
448 protected:
SetUp()449 void SetUp() override {
450 fs_path = std::string(getenv("TMPDIR")) + "/f2fs_2G.img";
451 uint64_t count = fs_size / block_size;
452 std::string dd_cmd =
453 ::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64
454 " count=%" PRIu64 " > /dev/null 2>&1",
455 fs_path.c_str(), block_size, count);
456 std::string mkfs_cmd =
457 ::android::base::StringPrintf("/system/bin/make_f2fs -q %s", fs_path.c_str());
458 // create mount point
459 mntpoint = std::string(getenv("TMPDIR")) + "/fiemap_mnt";
460 ASSERT_EQ(mkdir(mntpoint.c_str(), S_IRWXU), 0);
461 // create file for the file system
462 int ret = system(dd_cmd.c_str());
463 ASSERT_EQ(ret, 0);
464 // Get and attach a loop device to the filesystem we created
465 LoopDevice loop_dev(fs_path, 10s);
466 ASSERT_TRUE(loop_dev.valid());
467 // create file system
468 ret = system(mkfs_cmd.c_str());
469 ASSERT_EQ(ret, 0);
470
471 // mount the file system
472 ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint.c_str(), "f2fs", 0, nullptr), 0);
473 }
474
TearDown()475 void TearDown() override {
476 umount(mntpoint.c_str());
477 rmdir(mntpoint.c_str());
478 unlink(fs_path.c_str());
479 }
480
481 std::string mntpoint;
482 std::string fs_path;
483 };
484
DetermineBlockSize()485 bool DetermineBlockSize() {
486 struct statfs s;
487 if (statfs(gTestDir.c_str(), &s)) {
488 std::cerr << "Could not call statfs: " << strerror(errno) << "\n";
489 return false;
490 }
491 if (!s.f_bsize) {
492 std::cerr << "Invalid block size: " << s.f_bsize << "\n";
493 return false;
494 }
495
496 gBlockSize = s.f_bsize;
497 return true;
498 }
499
500 } // namespace fiemap
501 } // namespace android
502
503 using namespace android::fiemap;
504
main(int argc,char ** argv)505 int main(int argc, char** argv) {
506 ::testing::InitGoogleTest(&argc, argv);
507 if (argc > 1 && argv[1] == "-h"s) {
508 cerr << "Usage: [test_dir] [file_size]\n";
509 cerr << "\n";
510 cerr << "Note: test_dir must be a writable, unencrypted directory.\n";
511 exit(EXIT_FAILURE);
512 }
513 ::android::base::InitLogging(argv, ::android::base::StderrLogger);
514
515 std::string root_dir = "/data/local/unencrypted";
516 if (access(root_dir.c_str(), F_OK)) {
517 root_dir = "/data";
518 }
519
520 std::string tempdir = root_dir + "/XXXXXX"s;
521 if (!mkdtemp(tempdir.data())) {
522 cerr << "unable to create tempdir on " << root_dir << "\n";
523 exit(EXIT_FAILURE);
524 }
525 if (!android::base::Realpath(tempdir, &gTestDir)) {
526 cerr << "unable to find realpath for " << tempdir;
527 exit(EXIT_FAILURE);
528 }
529
530 if (argc > 2) {
531 testfile_size = strtoull(argv[2], NULL, 0);
532 if (testfile_size == ULLONG_MAX) {
533 testfile_size = 512 * 1024 * 1024;
534 }
535 }
536
537 if (!DetermineBlockSize()) {
538 exit(EXIT_FAILURE);
539 }
540
541 auto result = RUN_ALL_TESTS();
542
543 std::string cmd = "rm -rf " + gTestDir;
544 system(cmd.c_str());
545
546 return result;
547 }
548