1 /* 2 * Copyright (C) 2007 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "client/file_sync_client.h" 18 19 #include <dirent.h> 20 #include <inttypes.h> 21 #include <limits.h> 22 #include <stdio.h> 23 #include <stdlib.h> 24 #include <sys/stat.h> 25 #include <sys/time.h> 26 #include <sys/types.h> 27 #include <time.h> 28 #include <unistd.h> 29 #include <utime.h> 30 31 #include <chrono> 32 #include <deque> 33 #include <functional> 34 #include <memory> 35 #include <sstream> 36 #include <string> 37 #include <variant> 38 #include <vector> 39 40 #include "sysdeps.h" 41 42 #include "adb.h" 43 #include "adb_client.h" 44 #include "adb_io.h" 45 #include "adb_utils.h" 46 #include "compression_utils.h" 47 #include "file_sync_protocol.h" 48 #include "line_printer.h" 49 #include "sysdeps/errno.h" 50 #include "sysdeps/stat.h" 51 52 #include "client/commandline.h" 53 54 #include <android-base/file.h> 55 #include <android-base/strings.h> 56 #include <android-base/stringprintf.h> 57 58 using namespace std::literals; 59 60 typedef void(sync_ls_cb)(unsigned mode, uint64_t size, uint64_t time, const char* name); 61 62 struct syncsendbuf { 63 unsigned id; 64 unsigned size; 65 char data[SYNC_DATA_MAX]; 66 }; 67 68 static void ensure_trailing_separators(std::string& local_path, std::string& remote_path) { 69 if (!adb_is_separator(local_path.back())) { 70 local_path.push_back(OS_PATH_SEPARATOR); 71 } 72 if (remote_path.back() != '/') { 73 remote_path.push_back('/'); 74 } 75 } 76 77 static bool should_pull_file(mode_t mode) { 78 return S_ISREG(mode) || S_ISBLK(mode) || S_ISCHR(mode); 79 } 80 81 static bool should_push_file(mode_t mode) { 82 return S_ISREG(mode) || S_ISLNK(mode); 83 } 84 85 struct copyinfo { 86 std::string lpath; 87 std::string rpath; 88 int64_t time = 0; 89 uint32_t mode; 90 uint64_t size = 0; 91 bool skip = false; 92 93 copyinfo(const std::string& local_path, 94 const std::string& remote_path, 95 const std::string& name, 96 unsigned int mode) 97 : lpath(local_path), rpath(remote_path), mode(mode) { 98 ensure_trailing_separators(lpath, rpath); 99 lpath.append(name); 100 rpath.append(name); 101 if (S_ISDIR(mode)) { 102 ensure_trailing_separators(lpath, rpath); 103 } 104 } 105 }; 106 107 enum class TransferDirection { 108 push, 109 pull, 110 }; 111 112 struct TransferLedger { 113 std::chrono::steady_clock::time_point start_time; 114 uint64_t files_transferred; 115 uint64_t files_skipped; 116 uint64_t bytes_transferred; 117 uint64_t bytes_expected; 118 bool expect_multiple_files; 119 120 private: 121 std::string last_progress_str; 122 std::chrono::steady_clock::time_point last_progress_time; 123 124 public: 125 TransferLedger() { 126 Reset(); 127 } 128 129 bool operator==(const TransferLedger& other) const { 130 return files_transferred == other.files_transferred && 131 files_skipped == other.files_skipped && bytes_transferred == other.bytes_transferred; 132 } 133 134 bool operator!=(const TransferLedger& other) const { 135 return !(*this == other); 136 } 137 138 void Reset() { 139 start_time = std::chrono::steady_clock::now(); 140 files_transferred = 0; 141 files_skipped = 0; 142 bytes_transferred = 0; 143 bytes_expected = 0; 144 last_progress_str.clear(); 145 last_progress_time = {}; 146 } 147 148 std::string TransferRate() { 149 if (bytes_transferred == 0) return ""; 150 151 std::chrono::duration<double> duration; 152 duration = std::chrono::steady_clock::now() - start_time; 153 154 double s = duration.count(); 155 if (s == 0) { 156 return ""; 157 } 158 double rate = (static_cast<double>(bytes_transferred) / s) / (1024 * 1024); 159 return android::base::StringPrintf(" %.1f MB/s (%" PRIu64 " bytes in %.3fs)", rate, 160 bytes_transferred, s); 161 } 162 163 void ReportProgress(LinePrinter& lp, const std::string& file, uint64_t file_copied_bytes, 164 uint64_t file_total_bytes) { 165 static constexpr auto kProgressReportInterval = 100ms; 166 167 auto now = std::chrono::steady_clock::now(); 168 if (now < last_progress_time + kProgressReportInterval) { 169 return; 170 } 171 char overall_percentage_str[5] = "?"; 172 if (bytes_expected != 0 && bytes_transferred <= bytes_expected) { 173 int overall_percentage = static_cast<int>(bytes_transferred * 100 / bytes_expected); 174 // If we're pulling symbolic links, we'll pull the target of the link rather than 175 // just create a local link, and that will cause us to go over 100%. 176 if (overall_percentage <= 100) { 177 snprintf(overall_percentage_str, sizeof(overall_percentage_str), "%d%%", 178 overall_percentage); 179 } 180 } 181 182 std::string output; 183 if (file_copied_bytes > file_total_bytes || file_total_bytes == 0) { 184 // This case can happen if we're racing against something that wrote to the file 185 // between our stat and our read, or if we're reading a magic file that lies about 186 // its size. Just show how much we've copied. 187 output = android::base::StringPrintf("[%4s] %s: %" PRId64 "/?", overall_percentage_str, 188 file.c_str(), file_copied_bytes); 189 } else { 190 // If we're transferring multiple files, we want to know how far through the current 191 // file we are, as well as the overall percentage. 192 if (expect_multiple_files) { 193 int file_percentage = static_cast<int>(file_copied_bytes * 100 / file_total_bytes); 194 output = android::base::StringPrintf("[%4s] %s: %d%%", overall_percentage_str, 195 file.c_str(), file_percentage); 196 } else { 197 output = 198 android::base::StringPrintf("[%4s] %s", overall_percentage_str, file.c_str()); 199 } 200 } 201 if (output != last_progress_str) { 202 lp.Print(output, LinePrinter::LineType::INFO); 203 last_progress_str = std::move(output); 204 last_progress_time = now; 205 } 206 } 207 208 void ReportTransferRate(LinePrinter& lp, const std::string& name, TransferDirection direction) { 209 const char* direction_str = (direction == TransferDirection::push) ? "pushed" : "pulled"; 210 std::stringstream ss; 211 if (!name.empty()) { 212 std::string_view display_name(name); 213 char* out = getenv("ANDROID_PRODUCT_OUT"); 214 if (out) android::base::ConsumePrefix(&display_name, out); 215 ss << display_name << ": "; 216 } 217 ss << files_transferred << " file" << ((files_transferred == 1) ? "" : "s") << " " 218 << direction_str << ", " << files_skipped << " skipped."; 219 ss << TransferRate(); 220 221 lp.Print(ss.str(), LinePrinter::LineType::INFO); 222 lp.KeepInfoLine(); 223 } 224 }; 225 226 class SyncConnection { 227 public: 228 SyncConnection() : acknowledgement_buffer_(sizeof(sync_status) + SYNC_DATA_MAX) { 229 acknowledgement_buffer_.resize(0); 230 max = SYNC_DATA_MAX; // TODO: decide at runtime. 231 232 std::string error; 233 auto&& features = adb_get_feature_set(&error); 234 if (!features) { 235 Error("failed to get feature set: %s", error.c_str()); 236 } else { 237 features_ = &*features; 238 have_stat_v2_ = CanUseFeature(*features, kFeatureStat2); 239 have_ls_v2_ = CanUseFeature(*features, kFeatureLs2); 240 have_sendrecv_v2_ = CanUseFeature(*features, kFeatureSendRecv2); 241 have_sendrecv_v2_brotli_ = CanUseFeature(*features, kFeatureSendRecv2Brotli); 242 have_sendrecv_v2_lz4_ = CanUseFeature(*features, kFeatureSendRecv2LZ4); 243 have_sendrecv_v2_zstd_ = CanUseFeature(*features, kFeatureSendRecv2Zstd); 244 have_sendrecv_v2_dry_run_send_ = CanUseFeature(*features, kFeatureSendRecv2DryRunSend); 245 std::string error; 246 fd.reset(adb_connect("sync:", &error)); 247 if (fd < 0) { 248 Error("connect failed: %s", error.c_str()); 249 } 250 } 251 } 252 253 ~SyncConnection() { 254 if (!IsValid()) return; 255 256 if (SendQuit()) { 257 // We sent a quit command, so the server should be doing orderly 258 // shutdown soon. But if we encountered an error while we were using 259 // the connection, the server might still be sending data (before 260 // doing orderly shutdown), in which case we won't wait for all of 261 // the data nor the coming orderly shutdown. In the common success 262 // case, this will wait for the server to do orderly shutdown. 263 ReadOrderlyShutdown(fd); 264 } 265 266 line_printer_.KeepInfoLine(); 267 } 268 269 bool HaveSendRecv2() const { return have_sendrecv_v2_; } 270 bool HaveSendRecv2Brotli() const { return have_sendrecv_v2_brotli_; } 271 bool HaveSendRecv2LZ4() const { return have_sendrecv_v2_lz4_; } 272 bool HaveSendRecv2Zstd() const { return have_sendrecv_v2_zstd_; } 273 bool HaveSendRecv2DryRunSend() const { return have_sendrecv_v2_dry_run_send_; } 274 275 // Resolve a compression type which might be CompressionType::Any to a specific compression 276 // algorithm. 277 CompressionType ResolveCompressionType(CompressionType compression) const { 278 if (compression == CompressionType::Any) { 279 if (HaveSendRecv2Zstd()) { 280 return CompressionType::Zstd; 281 } else if (HaveSendRecv2LZ4()) { 282 return CompressionType::LZ4; 283 } else if (HaveSendRecv2Brotli()) { 284 return CompressionType::Brotli; 285 } 286 return CompressionType::None; 287 } 288 return compression; 289 } 290 291 const FeatureSet& Features() const { return *features_; } 292 293 bool IsValid() { return fd >= 0; } 294 295 void NewTransfer() { 296 current_ledger_.Reset(); 297 } 298 299 void RecordBytesTransferred(size_t bytes) { 300 current_ledger_.bytes_transferred += bytes; 301 global_ledger_.bytes_transferred += bytes; 302 } 303 304 void RecordFileSent(std::string from, std::string to) { 305 RecordFilesTransferred(1); 306 deferred_acknowledgements_.emplace_back(std::move(from), std::move(to)); 307 } 308 309 void RecordFilesTransferred(size_t files) { 310 current_ledger_.files_transferred += files; 311 global_ledger_.files_transferred += files; 312 } 313 314 void RecordFilesSkipped(size_t files) { 315 current_ledger_.files_skipped += files; 316 global_ledger_.files_skipped += files; 317 } 318 319 void ReportProgress(const std::string& file, uint64_t file_copied_bytes, 320 uint64_t file_total_bytes) { 321 current_ledger_.ReportProgress(line_printer_, file, file_copied_bytes, file_total_bytes); 322 } 323 324 void ReportTransferRate(const std::string& file, TransferDirection direction) { 325 current_ledger_.ReportTransferRate(line_printer_, file, direction); 326 } 327 328 void ReportOverallTransferRate(TransferDirection direction) { 329 if (current_ledger_ != global_ledger_) { 330 global_ledger_.ReportTransferRate(line_printer_, "", direction); 331 } 332 } 333 334 bool SendRequest(int id, const std::string& path) { 335 if (path.length() > 1024) { 336 Error("SendRequest failed: path too long: %zu", path.length()); 337 errno = ENAMETOOLONG; 338 return false; 339 } 340 341 // Sending header and payload in a single write makes a noticeable 342 // difference to "adb sync" performance. 343 std::vector<char> buf(sizeof(SyncRequest) + path.length()); 344 SyncRequest* req = reinterpret_cast<SyncRequest*>(&buf[0]); 345 req->id = id; 346 req->path_length = path.length(); 347 char* data = reinterpret_cast<char*>(req + 1); 348 memcpy(data, path.data(), path.length()); 349 return WriteFdExactly(fd, buf.data(), buf.size()); 350 } 351 352 bool SendSend2(std::string_view path, mode_t mode, CompressionType compression, bool dry_run) { 353 if (path.length() > 1024) { 354 Error("SendRequest failed: path too long: %zu", path.length()); 355 errno = ENAMETOOLONG; 356 return false; 357 } 358 359 Block buf; 360 361 SyncRequest req; 362 req.id = ID_SEND_V2; 363 req.path_length = path.length(); 364 365 syncmsg msg; 366 msg.send_v2_setup.id = ID_SEND_V2; 367 msg.send_v2_setup.mode = mode; 368 msg.send_v2_setup.flags = 0; 369 switch (compression) { 370 case CompressionType::None: 371 break; 372 373 case CompressionType::Brotli: 374 msg.send_v2_setup.flags = kSyncFlagBrotli; 375 break; 376 377 case CompressionType::LZ4: 378 msg.send_v2_setup.flags = kSyncFlagLZ4; 379 break; 380 381 case CompressionType::Zstd: 382 msg.send_v2_setup.flags = kSyncFlagZstd; 383 break; 384 385 case CompressionType::Any: 386 LOG(FATAL) << "unexpected CompressionType::Any"; 387 } 388 389 if (dry_run) { 390 msg.send_v2_setup.flags |= kSyncFlagDryRun; 391 } 392 393 buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.send_v2_setup)); 394 395 void* p = buf.data(); 396 397 p = mempcpy(p, &req, sizeof(SyncRequest)); 398 p = mempcpy(p, path.data(), path.length()); 399 p = mempcpy(p, &msg.send_v2_setup, sizeof(msg.send_v2_setup)); 400 401 return WriteFdExactly(fd, buf.data(), buf.size()); 402 } 403 404 bool SendRecv2(const std::string& path, CompressionType compression) { 405 if (path.length() > 1024) { 406 Error("SendRequest failed: path too long: %zu", path.length()); 407 errno = ENAMETOOLONG; 408 return false; 409 } 410 411 Block buf; 412 413 SyncRequest req; 414 req.id = ID_RECV_V2; 415 req.path_length = path.length(); 416 417 syncmsg msg; 418 msg.recv_v2_setup.id = ID_RECV_V2; 419 msg.recv_v2_setup.flags = 0; 420 switch (compression) { 421 case CompressionType::None: 422 break; 423 424 case CompressionType::Brotli: 425 msg.recv_v2_setup.flags |= kSyncFlagBrotli; 426 break; 427 428 case CompressionType::LZ4: 429 msg.recv_v2_setup.flags |= kSyncFlagLZ4; 430 break; 431 432 case CompressionType::Zstd: 433 msg.recv_v2_setup.flags |= kSyncFlagZstd; 434 break; 435 436 case CompressionType::Any: 437 LOG(FATAL) << "unexpected CompressionType::Any"; 438 } 439 440 buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.recv_v2_setup)); 441 442 void* p = buf.data(); 443 444 p = mempcpy(p, &req, sizeof(SyncRequest)); 445 p = mempcpy(p, path.data(), path.length()); 446 p = mempcpy(p, &msg.recv_v2_setup, sizeof(msg.recv_v2_setup)); 447 448 return WriteFdExactly(fd, buf.data(), buf.size()); 449 } 450 451 bool SendStat(const std::string& path) { 452 if (!have_stat_v2_) { 453 errno = ENOTSUP; 454 return false; 455 } 456 return SendRequest(ID_STAT_V2, path); 457 } 458 459 bool SendLstat(const std::string& path) { 460 if (have_stat_v2_) { 461 return SendRequest(ID_LSTAT_V2, path); 462 } else { 463 return SendRequest(ID_LSTAT_V1, path); 464 } 465 } 466 467 bool FinishStat(struct stat* st) { 468 syncmsg msg; 469 470 memset(st, 0, sizeof(*st)); 471 if (have_stat_v2_) { 472 if (!ReadFdExactly(fd.get(), &msg.stat_v2, sizeof(msg.stat_v2))) { 473 PLOG(FATAL) << "protocol fault: failed to read stat response"; 474 } 475 476 if (msg.stat_v2.id != ID_LSTAT_V2 && msg.stat_v2.id != ID_STAT_V2) { 477 PLOG(FATAL) << "protocol fault: stat response has wrong message id: " 478 << msg.stat_v2.id; 479 } 480 481 if (msg.stat_v2.error != 0) { 482 errno = errno_from_wire(msg.stat_v2.error); 483 return false; 484 } 485 486 st->st_dev = msg.stat_v2.dev; 487 st->st_ino = msg.stat_v2.ino; 488 st->st_mode = msg.stat_v2.mode; 489 st->st_nlink = msg.stat_v2.nlink; 490 st->st_uid = msg.stat_v2.uid; 491 st->st_gid = msg.stat_v2.gid; 492 st->st_size = msg.stat_v2.size; 493 st->st_atime = msg.stat_v2.atime; 494 st->st_mtime = msg.stat_v2.mtime; 495 st->st_ctime = msg.stat_v2.ctime; 496 return true; 497 } else { 498 if (!ReadFdExactly(fd.get(), &msg.stat_v1, sizeof(msg.stat_v1))) { 499 PLOG(FATAL) << "protocol fault: failed to read stat response"; 500 } 501 502 if (msg.stat_v1.id != ID_LSTAT_V1) { 503 LOG(FATAL) << "protocol fault: stat response has wrong message id: " 504 << msg.stat_v1.id; 505 } 506 507 if (msg.stat_v1.mode == 0 && msg.stat_v1.size == 0 && msg.stat_v1.mtime == 0) { 508 // There's no way for us to know what the error was. 509 errno = ENOPROTOOPT; 510 return false; 511 } 512 513 st->st_mode = msg.stat_v1.mode; 514 st->st_size = msg.stat_v1.size; 515 st->st_ctime = msg.stat_v1.mtime; 516 st->st_mtime = msg.stat_v1.mtime; 517 } 518 519 return true; 520 } 521 522 bool SendLs(const std::string& path) { 523 return SendRequest(have_ls_v2_ ? ID_LIST_V2 : ID_LIST_V1, path); 524 } 525 526 private: 527 template <bool v2> 528 static bool FinishLsImpl(borrowed_fd fd, const std::function<sync_ls_cb>& callback) { 529 using dent_type = 530 std::conditional_t<v2, decltype(syncmsg::dent_v2), decltype(syncmsg::dent_v1)>; 531 532 while (true) { 533 dent_type dent; 534 if (!ReadFdExactly(fd, &dent, sizeof(dent))) return false; 535 536 uint32_t expected_id = v2 ? ID_DENT_V2 : ID_DENT_V1; 537 if (dent.id == ID_DONE) return true; 538 if (dent.id != expected_id) return false; 539 540 // Maximum length of a file name excluding null terminator (NAME_MAX) on Linux is 255. 541 char buf[256]; 542 size_t len = dent.namelen; 543 if (len > 255) return false; 544 545 if (!ReadFdExactly(fd, buf, len)) return false; 546 buf[len] = 0; 547 548 callback(dent.mode, dent.size, dent.mtime, buf); 549 } 550 } 551 552 public: 553 bool FinishLs(const std::function<sync_ls_cb>& callback) { 554 if (have_ls_v2_) { 555 return FinishLsImpl<true>(this->fd, callback); 556 } else { 557 return FinishLsImpl<false>(this->fd, callback); 558 } 559 } 560 561 // Sending header, payload, and footer in a single write makes a huge 562 // difference to "adb sync" performance. 563 bool SendSmallFile(const std::string& path, mode_t mode, const std::string& lpath, 564 const std::string& rpath, unsigned mtime, const char* data, 565 size_t data_length, bool dry_run) { 566 if (dry_run) { 567 // We need to use send v2 for dry run. 568 return SendLargeFile(path, mode, lpath, rpath, mtime, CompressionType::None, dry_run); 569 } 570 571 std::string path_and_mode = android::base::StringPrintf("%s,%d", path.c_str(), mode); 572 if (path_and_mode.length() > 1024) { 573 Error("SendSmallFile failed: path too long: %zu", path_and_mode.length()); 574 errno = ENAMETOOLONG; 575 return false; 576 } 577 578 std::vector<char> buf(sizeof(SyncRequest) + path_and_mode.length() + sizeof(SyncRequest) + 579 data_length + sizeof(SyncRequest)); 580 char* p = &buf[0]; 581 582 SyncRequest* req_send = reinterpret_cast<SyncRequest*>(p); 583 req_send->id = ID_SEND_V1; 584 req_send->path_length = path_and_mode.length(); 585 p += sizeof(SyncRequest); 586 memcpy(p, path_and_mode.data(), path_and_mode.size()); 587 p += path_and_mode.length(); 588 589 SyncRequest* req_data = reinterpret_cast<SyncRequest*>(p); 590 req_data->id = ID_DATA; 591 req_data->path_length = data_length; 592 p += sizeof(SyncRequest); 593 memcpy(p, data, data_length); 594 p += data_length; 595 596 SyncRequest* req_done = reinterpret_cast<SyncRequest*>(p); 597 req_done->id = ID_DONE; 598 req_done->path_length = mtime; 599 p += sizeof(SyncRequest); 600 601 WriteOrDie(lpath, rpath, &buf[0], (p - &buf[0])); 602 603 RecordFileSent(lpath, rpath); 604 RecordBytesTransferred(data_length); 605 ReportProgress(rpath, data_length, data_length); 606 return true; 607 } 608 609 bool SendLargeFile(const std::string& path, mode_t mode, const std::string& lpath, 610 const std::string& rpath, unsigned mtime, CompressionType compression, 611 bool dry_run) { 612 if (dry_run && !HaveSendRecv2DryRunSend()) { 613 Error("dry-run not supported by the device"); 614 return false; 615 } 616 617 if (!HaveSendRecv2()) { 618 return SendLargeFileLegacy(path, mode, lpath, rpath, mtime); 619 } 620 621 compression = ResolveCompressionType(compression); 622 623 if (!SendSend2(path, mode, compression, dry_run)) { 624 Error("failed to send ID_SEND_V2 message '%s': %s", path.c_str(), strerror(errno)); 625 return false; 626 } 627 628 struct stat st; 629 if (stat(lpath.c_str(), &st) == -1) { 630 Error("cannot stat '%s': %s", lpath.c_str(), strerror(errno)); 631 return false; 632 } 633 634 uint64_t total_size = st.st_size; 635 uint64_t bytes_copied = 0; 636 637 unique_fd lfd(adb_open(lpath.c_str(), O_RDONLY | O_CLOEXEC)); 638 if (lfd < 0) { 639 Error("opening '%s' locally failed: %s", lpath.c_str(), strerror(errno)); 640 return false; 641 } 642 643 syncsendbuf sbuf; 644 sbuf.id = ID_DATA; 645 646 std::variant<std::monostate, NullEncoder, BrotliEncoder, LZ4Encoder, ZstdEncoder> 647 encoder_storage; 648 Encoder* encoder = nullptr; 649 switch (compression) { 650 case CompressionType::None: 651 encoder = &encoder_storage.emplace<NullEncoder>(SYNC_DATA_MAX); 652 break; 653 654 case CompressionType::Brotli: 655 encoder = &encoder_storage.emplace<BrotliEncoder>(SYNC_DATA_MAX); 656 break; 657 658 case CompressionType::LZ4: 659 encoder = &encoder_storage.emplace<LZ4Encoder>(SYNC_DATA_MAX); 660 break; 661 662 case CompressionType::Zstd: 663 encoder = &encoder_storage.emplace<ZstdEncoder>(SYNC_DATA_MAX); 664 break; 665 666 case CompressionType::Any: 667 LOG(FATAL) << "unexpected CompressionType::Any"; 668 } 669 670 bool sending = true; 671 while (sending) { 672 Block input(SYNC_DATA_MAX); 673 int r = adb_read(lfd.get(), input.data(), input.size()); 674 if (r < 0) { 675 Error("reading '%s' locally failed: %s", lpath.c_str(), strerror(errno)); 676 return false; 677 } 678 679 if (r == 0) { 680 encoder->Finish(); 681 } else { 682 input.resize(r); 683 encoder->Append(std::move(input)); 684 RecordBytesTransferred(r); 685 bytes_copied += r; 686 ReportProgress(rpath, bytes_copied, total_size); 687 } 688 689 while (true) { 690 Block output; 691 EncodeResult result = encoder->Encode(&output); 692 if (result == EncodeResult::Error) { 693 Error("compressing '%s' locally failed", lpath.c_str()); 694 return false; 695 } 696 697 if (!output.empty()) { 698 sbuf.size = output.size(); 699 memcpy(sbuf.data, output.data(), output.size()); 700 WriteOrDie(lpath, rpath, &sbuf, sizeof(SyncRequest) + output.size()); 701 } 702 703 if (result == EncodeResult::Done) { 704 sending = false; 705 break; 706 } else if (result == EncodeResult::NeedInput) { 707 break; 708 } else if (result == EncodeResult::MoreOutput) { 709 continue; 710 } 711 } 712 } 713 714 syncmsg msg; 715 msg.data.id = ID_DONE; 716 msg.data.size = mtime; 717 RecordFileSent(lpath, rpath); 718 return WriteOrDie(lpath, rpath, &msg.data, sizeof(msg.data)); 719 } 720 721 bool SendLargeFileLegacy(const std::string& path, mode_t mode, const std::string& lpath, 722 const std::string& rpath, unsigned mtime) { 723 std::string path_and_mode = android::base::StringPrintf("%s,%d", path.c_str(), mode); 724 if (!SendRequest(ID_SEND_V1, path_and_mode)) { 725 Error("failed to send ID_SEND_V1 message '%s': %s", path_and_mode.c_str(), 726 strerror(errno)); 727 return false; 728 } 729 730 struct stat st; 731 if (stat(lpath.c_str(), &st) == -1) { 732 Error("cannot stat '%s': %s", lpath.c_str(), strerror(errno)); 733 return false; 734 } 735 736 uint64_t total_size = st.st_size; 737 uint64_t bytes_copied = 0; 738 739 unique_fd lfd(adb_open(lpath.c_str(), O_RDONLY | O_CLOEXEC)); 740 if (lfd < 0) { 741 Error("opening '%s' locally failed: %s", lpath.c_str(), strerror(errno)); 742 return false; 743 } 744 745 syncsendbuf sbuf; 746 sbuf.id = ID_DATA; 747 748 while (true) { 749 int bytes_read = adb_read(lfd, sbuf.data, max); 750 if (bytes_read == -1) { 751 Error("reading '%s' locally failed: %s", lpath.c_str(), strerror(errno)); 752 return false; 753 } else if (bytes_read == 0) { 754 break; 755 } 756 757 sbuf.size = bytes_read; 758 WriteOrDie(lpath, rpath, &sbuf, sizeof(SyncRequest) + bytes_read); 759 760 RecordBytesTransferred(bytes_read); 761 bytes_copied += bytes_read; 762 ReportProgress(rpath, bytes_copied, total_size); 763 } 764 765 syncmsg msg; 766 msg.data.id = ID_DONE; 767 msg.data.size = mtime; 768 RecordFileSent(lpath, rpath); 769 return WriteOrDie(lpath, rpath, &msg.data, sizeof(msg.data)); 770 } 771 772 bool ReportCopyFailure(const std::string& from, const std::string& to, const syncmsg& msg) { 773 std::vector<char> buf(msg.status.msglen + 1); 774 if (!ReadFdExactly(fd, &buf[0], msg.status.msglen)) { 775 Error("failed to copy '%s' to '%s'; failed to read reason (!): %s", from.c_str(), 776 to.c_str(), strerror(errno)); 777 return false; 778 } 779 buf[msg.status.msglen] = 0; 780 Error("failed to copy '%s' to '%s': remote %s", from.c_str(), to.c_str(), &buf[0]); 781 return false; 782 } 783 784 void CopyDone() { deferred_acknowledgements_.pop_front(); } 785 786 void ReportDeferredCopyFailure(const std::string& msg) { 787 auto& [from, to] = deferred_acknowledgements_.front(); 788 Error("failed to copy '%s' to '%s': remote %s", from.c_str(), to.c_str(), msg.c_str()); 789 deferred_acknowledgements_.pop_front(); 790 } 791 792 bool ReadAcknowledgements(bool read_all = false) { 793 // We need to read enough such that adbd's intermediate socket's write buffer can't be 794 // full. The default buffer on Linux is 212992 bytes, but there's 576 bytes of bookkeeping 795 // overhead per write. The worst case scenario is a continuous string of failures, since 796 // each logical packet is divided into two writes. If our packet size if conservatively 512 797 // bytes long, this leaves us with space for 128 responses. 798 constexpr size_t max_deferred_acks = 128; 799 auto& buf = acknowledgement_buffer_; 800 adb_pollfd pfd = {.fd = fd.get(), .events = POLLIN}; 801 while (!deferred_acknowledgements_.empty()) { 802 bool should_block = read_all || deferred_acknowledgements_.size() >= max_deferred_acks; 803 804 ssize_t rc = adb_poll(&pfd, 1, should_block ? -1 : 0); 805 if (rc == 0) { 806 CHECK(!should_block); 807 return true; 808 } 809 810 if (acknowledgement_buffer_.size() < sizeof(sync_status)) { 811 const ssize_t header_bytes_left = sizeof(sync_status) - buf.size(); 812 ssize_t rc = adb_read(fd, buf.end(), header_bytes_left); 813 if (rc <= 0) { 814 Error("failed to read copy response"); 815 return false; 816 } 817 818 buf.resize(buf.size() + rc); 819 if (rc != header_bytes_left) { 820 // Early exit if we run out of data in the socket. 821 return true; 822 } 823 824 if (!should_block) { 825 // We don't want to read again yet, because the socket might be empty. 826 continue; 827 } 828 } 829 830 auto* hdr = reinterpret_cast<sync_status*>(buf.data()); 831 if (hdr->id == ID_OKAY) { 832 buf.resize(0); 833 if (hdr->msglen != 0) { 834 Error("received ID_OKAY with msg_len (%" PRIu32 " != 0", hdr->msglen); 835 return false; 836 } 837 CopyDone(); 838 continue; 839 } else if (hdr->id != ID_FAIL) { 840 Error("unexpected response from daemon: id = %#" PRIx32, hdr->id); 841 return false; 842 } else if (hdr->msglen > SYNC_DATA_MAX) { 843 Error("too-long message length from daemon: msglen = %" PRIu32, hdr->msglen); 844 return false; 845 } 846 847 const ssize_t msg_bytes_left = hdr->msglen + sizeof(sync_status) - buf.size(); 848 CHECK_GE(msg_bytes_left, 0); 849 if (msg_bytes_left > 0) { 850 ssize_t rc = adb_read(fd, buf.end(), msg_bytes_left); 851 if (rc <= 0) { 852 Error("failed to read copy failure message"); 853 return false; 854 } 855 856 buf.resize(buf.size() + rc); 857 if (rc != msg_bytes_left) { 858 if (should_block) { 859 continue; 860 } else { 861 return true; 862 } 863 } 864 865 std::string msg(buf.begin() + sizeof(sync_status), buf.end()); 866 ReportDeferredCopyFailure(msg); 867 buf.resize(0); 868 return false; 869 } 870 } 871 872 return true; 873 } 874 875 void Printf(const char* fmt, ...) __attribute__((__format__(__printf__, 2, 3))) { 876 std::string s; 877 878 va_list ap; 879 va_start(ap, fmt); 880 android::base::StringAppendV(&s, fmt, ap); 881 va_end(ap); 882 883 line_printer_.Print(s, LinePrinter::INFO); 884 } 885 886 void Println(const char* fmt, ...) __attribute__((__format__(__printf__, 2, 3))) { 887 std::string s; 888 889 va_list ap; 890 va_start(ap, fmt); 891 android::base::StringAppendV(&s, fmt, ap); 892 va_end(ap); 893 894 line_printer_.Print(s, LinePrinter::INFO); 895 line_printer_.KeepInfoLine(); 896 } 897 898 void Error(const char* fmt, ...) __attribute__((__format__(__printf__, 2, 3))) { 899 std::string s = "adb: error: "; 900 901 va_list ap; 902 va_start(ap, fmt); 903 android::base::StringAppendV(&s, fmt, ap); 904 va_end(ap); 905 906 line_printer_.Print(s, LinePrinter::ERROR); 907 } 908 909 void Warning(const char* fmt, ...) __attribute__((__format__(__printf__, 2, 3))) { 910 std::string s = "adb: warning: "; 911 912 va_list ap; 913 va_start(ap, fmt); 914 android::base::StringAppendV(&s, fmt, ap); 915 va_end(ap); 916 917 line_printer_.Print(s, LinePrinter::WARNING); 918 } 919 920 void ComputeExpectedTotalBytes(const std::vector<copyinfo>& file_list) { 921 current_ledger_.bytes_expected = 0; 922 for (const copyinfo& ci : file_list) { 923 // Unfortunately, this doesn't work for symbolic links, because we'll copy the 924 // target of the link rather than just creating a link. (But ci.size is the link size.) 925 if (!ci.skip) current_ledger_.bytes_expected += ci.size; 926 } 927 current_ledger_.expect_multiple_files = true; 928 } 929 930 void SetExpectedTotalBytes(uint64_t expected_total_bytes) { 931 current_ledger_.bytes_expected = expected_total_bytes; 932 current_ledger_.expect_multiple_files = false; 933 } 934 935 // TODO: add a char[max] buffer here, to replace syncsendbuf... 936 unique_fd fd; 937 size_t max; 938 939 private: 940 std::deque<std::pair<std::string, std::string>> deferred_acknowledgements_; 941 Block acknowledgement_buffer_; 942 const FeatureSet* features_ = nullptr; 943 bool have_stat_v2_; 944 bool have_ls_v2_; 945 bool have_sendrecv_v2_; 946 bool have_sendrecv_v2_brotli_; 947 bool have_sendrecv_v2_lz4_; 948 bool have_sendrecv_v2_zstd_; 949 bool have_sendrecv_v2_dry_run_send_; 950 951 TransferLedger global_ledger_; 952 TransferLedger current_ledger_; 953 LinePrinter line_printer_; 954 955 bool SendQuit() { 956 return SendRequest(ID_QUIT, ""); // TODO: add a SendResponse? 957 } 958 959 bool WriteOrDie(const std::string& from, const std::string& to, const void* data, 960 size_t data_length) { 961 if (!WriteFdExactly(fd, data, data_length)) { 962 if (errno == ECONNRESET) { 963 // Assume adbd told us why it was closing the connection, and 964 // try to read failure reason from adbd. 965 syncmsg msg; 966 if (!ReadFdExactly(fd, &msg.status, sizeof(msg.status))) { 967 Error("failed to copy '%s' to '%s': no response: %s", from.c_str(), to.c_str(), 968 strerror(errno)); 969 } else if (msg.status.id != ID_FAIL) { 970 Error("failed to copy '%s' to '%s': not ID_FAIL: %d", from.c_str(), to.c_str(), 971 msg.status.id); 972 } else { 973 ReportCopyFailure(from, to, msg); 974 } 975 } else { 976 Error("%zu-byte write failed: %s", data_length, strerror(errno)); 977 } 978 _exit(1); 979 } 980 return true; 981 } 982 }; 983 984 static bool sync_ls(SyncConnection& sc, const std::string& path, 985 const std::function<sync_ls_cb>& func) { 986 return sc.SendLs(path) && sc.FinishLs(func); 987 } 988 989 static bool sync_stat(SyncConnection& sc, const std::string& path, struct stat* st) { 990 return sc.SendStat(path) && sc.FinishStat(st); 991 } 992 993 static bool sync_lstat(SyncConnection& sc, const std::string& path, struct stat* st) { 994 return sc.SendLstat(path) && sc.FinishStat(st); 995 } 996 997 static bool sync_stat_fallback(SyncConnection& sc, const std::string& path, struct stat* st) { 998 if (sync_stat(sc, path, st)) { 999 return true; 1000 } 1001 1002 if (errno != ENOTSUP) { 1003 return false; 1004 } 1005 1006 // Try to emulate the parts we can when talking to older adbds. 1007 bool lstat_result = sync_lstat(sc, path, st); 1008 if (!lstat_result) { 1009 return false; 1010 } 1011 1012 if (S_ISLNK(st->st_mode)) { 1013 // If the target is a symlink, figure out whether it's a file or a directory. 1014 // Also, zero out the st_size field, since no one actually cares what the path length is. 1015 st->st_size = 0; 1016 std::string dir_path = path; 1017 dir_path.push_back('/'); 1018 struct stat tmp_st; 1019 1020 st->st_mode &= ~S_IFMT; 1021 if (sync_lstat(sc, dir_path, &tmp_st)) { 1022 st->st_mode |= S_IFDIR; 1023 } else { 1024 st->st_mode |= S_IFREG; 1025 } 1026 } 1027 return true; 1028 } 1029 1030 static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::string& rpath, 1031 unsigned mtime, mode_t mode, bool sync, CompressionType compression, 1032 bool dry_run) { 1033 if (sync) { 1034 struct stat st; 1035 if (sync_lstat(sc, rpath, &st)) { 1036 if (st.st_mtime == static_cast<time_t>(mtime)) { 1037 sc.RecordFilesSkipped(1); 1038 return true; 1039 } 1040 } 1041 } 1042 1043 if (S_ISLNK(mode)) { 1044 #if !defined(_WIN32) 1045 char buf[PATH_MAX]; 1046 ssize_t data_length = readlink(lpath.c_str(), buf, PATH_MAX - 1); 1047 if (data_length == -1) { 1048 sc.Error("readlink '%s' failed: %s", lpath.c_str(), strerror(errno)); 1049 return false; 1050 } 1051 buf[data_length++] = '\0'; 1052 1053 if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, buf, data_length, dry_run)) { 1054 return false; 1055 } 1056 return sc.ReadAcknowledgements(sync); 1057 #endif 1058 } 1059 1060 struct stat st; 1061 if (stat(lpath.c_str(), &st) == -1) { 1062 sc.Error("failed to stat local file '%s': %s", lpath.c_str(), strerror(errno)); 1063 return false; 1064 } 1065 if (st.st_size < SYNC_DATA_MAX) { 1066 std::string data; 1067 if (!android::base::ReadFileToString(lpath, &data, true)) { 1068 sc.Error("failed to read all of '%s': %s", lpath.c_str(), strerror(errno)); 1069 return false; 1070 } 1071 if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, data.data(), data.size(), 1072 dry_run)) { 1073 return false; 1074 } 1075 } else { 1076 if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compression, dry_run)) { 1077 return false; 1078 } 1079 } 1080 return sc.ReadAcknowledgements(sync); 1081 } 1082 1083 static bool sync_recv_v1(SyncConnection& sc, const char* rpath, const char* lpath, const char* name, 1084 uint64_t expected_size) { 1085 if (!sc.SendRequest(ID_RECV_V1, rpath)) return false; 1086 1087 adb_unlink(lpath); 1088 unique_fd lfd(adb_creat(lpath, 0644)); 1089 if (lfd < 0) { 1090 sc.Error("cannot create '%s': %s", lpath, strerror(errno)); 1091 return false; 1092 } 1093 1094 uint64_t bytes_copied = 0; 1095 while (true) { 1096 syncmsg msg; 1097 if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) { 1098 adb_unlink(lpath); 1099 return false; 1100 } 1101 1102 if (msg.data.id == ID_DONE) break; 1103 1104 if (msg.data.id != ID_DATA) { 1105 adb_unlink(lpath); 1106 sc.ReportCopyFailure(rpath, lpath, msg); 1107 return false; 1108 } 1109 1110 if (msg.data.size > sc.max) { 1111 sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max); 1112 adb_unlink(lpath); 1113 return false; 1114 } 1115 1116 char buffer[SYNC_DATA_MAX]; 1117 if (!ReadFdExactly(sc.fd, buffer, msg.data.size)) { 1118 adb_unlink(lpath); 1119 return false; 1120 } 1121 1122 if (!WriteFdExactly(lfd, buffer, msg.data.size)) { 1123 sc.Error("cannot write '%s': %s", lpath, strerror(errno)); 1124 adb_unlink(lpath); 1125 return false; 1126 } 1127 1128 bytes_copied += msg.data.size; 1129 1130 sc.RecordBytesTransferred(msg.data.size); 1131 sc.ReportProgress(name != nullptr ? name : rpath, bytes_copied, expected_size); 1132 } 1133 1134 sc.RecordFilesTransferred(1); 1135 return true; 1136 } 1137 1138 static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpath, const char* name, 1139 uint64_t expected_size, CompressionType compression) { 1140 compression = sc.ResolveCompressionType(compression); 1141 1142 if (!sc.SendRecv2(rpath, compression)) return false; 1143 1144 adb_unlink(lpath); 1145 unique_fd lfd(adb_creat(lpath, 0644)); 1146 if (lfd < 0) { 1147 sc.Error("cannot create '%s': %s", lpath, strerror(errno)); 1148 return false; 1149 } 1150 1151 uint64_t bytes_copied = 0; 1152 1153 Block buffer(SYNC_DATA_MAX); 1154 std::variant<std::monostate, NullDecoder, BrotliDecoder, LZ4Decoder, ZstdDecoder> 1155 decoder_storage; 1156 Decoder* decoder = nullptr; 1157 1158 std::span buffer_span(buffer.data(), buffer.size()); 1159 switch (compression) { 1160 case CompressionType::None: 1161 decoder = &decoder_storage.emplace<NullDecoder>(buffer_span); 1162 break; 1163 1164 case CompressionType::Brotli: 1165 decoder = &decoder_storage.emplace<BrotliDecoder>(buffer_span); 1166 break; 1167 1168 case CompressionType::LZ4: 1169 decoder = &decoder_storage.emplace<LZ4Decoder>(buffer_span); 1170 break; 1171 1172 case CompressionType::Zstd: 1173 decoder = &decoder_storage.emplace<ZstdDecoder>(buffer_span); 1174 break; 1175 1176 case CompressionType::Any: 1177 LOG(FATAL) << "unexpected CompressionType::Any"; 1178 } 1179 1180 while (true) { 1181 syncmsg msg; 1182 if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) { 1183 adb_unlink(lpath); 1184 return false; 1185 } 1186 1187 if (msg.data.id == ID_DONE) { 1188 if (!decoder->Finish()) { 1189 sc.Error("unexpected ID_DONE"); 1190 return false; 1191 } 1192 } else if (msg.data.id != ID_DATA) { 1193 adb_unlink(lpath); 1194 sc.ReportCopyFailure(rpath, lpath, msg); 1195 return false; 1196 } else { 1197 if (msg.data.size > sc.max) { 1198 sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max); 1199 adb_unlink(lpath); 1200 return false; 1201 } 1202 1203 Block block(msg.data.size); 1204 if (!ReadFdExactly(sc.fd, block.data(), msg.data.size)) { 1205 adb_unlink(lpath); 1206 return false; 1207 } 1208 decoder->Append(std::move(block)); 1209 } 1210 1211 while (true) { 1212 std::span<char> output; 1213 DecodeResult result = decoder->Decode(&output); 1214 1215 if (result == DecodeResult::Error) { 1216 sc.Error("decompress failed"); 1217 adb_unlink(lpath); 1218 return false; 1219 } 1220 1221 if (!output.empty()) { 1222 if (!WriteFdExactly(lfd, output.data(), output.size())) { 1223 sc.Error("cannot write '%s': %s", lpath, strerror(errno)); 1224 adb_unlink(lpath); 1225 return false; 1226 } 1227 } 1228 1229 bytes_copied += output.size(); 1230 sc.RecordBytesTransferred(output.size()); 1231 sc.ReportProgress(name != nullptr ? name : rpath, bytes_copied, expected_size); 1232 1233 if (result == DecodeResult::NeedInput) { 1234 break; 1235 } else if (result == DecodeResult::MoreOutput) { 1236 continue; 1237 } else if (result == DecodeResult::Done) { 1238 sc.RecordFilesTransferred(1); 1239 return true; 1240 } else { 1241 LOG(FATAL) << "invalid DecodeResult: " << static_cast<int>(result); 1242 } 1243 } 1244 } 1245 } 1246 1247 static bool sync_recv(SyncConnection& sc, const char* rpath, const char* lpath, const char* name, 1248 uint64_t expected_size, CompressionType compression) { 1249 if (sc.HaveSendRecv2()) { 1250 return sync_recv_v2(sc, rpath, lpath, name, expected_size, compression); 1251 } else { 1252 return sync_recv_v1(sc, rpath, lpath, name, expected_size); 1253 } 1254 } 1255 1256 bool do_sync_ls(const char* path) { 1257 SyncConnection sc; 1258 if (!sc.IsValid()) return false; 1259 1260 return sync_ls(sc, path, [](unsigned mode, uint64_t size, uint64_t time, const char* name) { 1261 printf("%08x %08" PRIx64 " %08" PRIx64 " %s\n", mode, size, time, name); 1262 }); 1263 } 1264 1265 static bool IsDotOrDotDot(const char* name) { 1266 return name[0] == '.' && (name[1] == '\0' || (name[1] == '.' && name[2] == '\0')); 1267 } 1268 1269 static bool local_build_list(SyncConnection& sc, std::vector<copyinfo>* file_list, 1270 std::vector<std::string>* directory_list, const std::string& lpath, 1271 const std::string& rpath) { 1272 std::vector<copyinfo> dirlist; 1273 std::unique_ptr<DIR, int (*)(DIR*)> dir(opendir(lpath.c_str()), closedir); 1274 if (!dir) { 1275 sc.Error("cannot open '%s': %s", lpath.c_str(), strerror(errno)); 1276 return false; 1277 } 1278 1279 bool empty_dir = true; 1280 dirent* de; 1281 while ((de = readdir(dir.get()))) { 1282 if (IsDotOrDotDot(de->d_name)) { 1283 continue; 1284 } 1285 1286 empty_dir = false; 1287 std::string stat_path = lpath + de->d_name; 1288 1289 struct stat st; 1290 if (lstat(stat_path.c_str(), &st) == -1) { 1291 sc.Error("cannot lstat '%s': %s", stat_path.c_str(), 1292 strerror(errno)); 1293 continue; 1294 } 1295 1296 copyinfo ci(lpath, rpath, de->d_name, st.st_mode); 1297 if (S_ISDIR(st.st_mode)) { 1298 dirlist.push_back(ci); 1299 } else { 1300 if (!should_push_file(st.st_mode)) { 1301 sc.Warning("skipping special file '%s' (mode = 0o%o)", lpath.c_str(), st.st_mode); 1302 ci.skip = true; 1303 } 1304 ci.time = st.st_mtime; 1305 ci.size = st.st_size; 1306 file_list->push_back(ci); 1307 } 1308 } 1309 1310 // Close this directory and recurse. 1311 dir.reset(); 1312 1313 for (const copyinfo& ci : dirlist) { 1314 directory_list->push_back(ci.rpath); 1315 local_build_list(sc, file_list, directory_list, ci.lpath, ci.rpath); 1316 } 1317 1318 return true; 1319 } 1320 1321 // dirname("//foo") returns "//", so we can't do the obvious `path == "/"`. 1322 static bool is_root_dir(std::string_view path) { 1323 for (char c : path) { 1324 if (c != '/') { 1325 return false; 1326 } 1327 } 1328 return true; 1329 } 1330 1331 static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::string rpath, 1332 bool check_timestamps, bool list_only, 1333 CompressionType compression, bool dry_run) { 1334 sc.NewTransfer(); 1335 1336 // Make sure that both directory paths end in a slash. 1337 // Both paths are known to be nonempty, so we don't need to check. 1338 ensure_trailing_separators(lpath, rpath); 1339 1340 // Recursively build the list of files to copy. 1341 std::vector<copyinfo> file_list; 1342 std::vector<std::string> directory_list; 1343 1344 for (std::string path = rpath; !is_root_dir(path); path = android::base::Dirname(path)) { 1345 directory_list.push_back(path); 1346 } 1347 std::reverse(directory_list.begin(), directory_list.end()); 1348 1349 int skipped = 0; 1350 if (!local_build_list(sc, &file_list, &directory_list, lpath, rpath)) { 1351 return false; 1352 } 1353 1354 // b/110953234: 1355 // P shipped with a bug that causes directory creation as a side-effect of a push to fail. 1356 // Work around this by explicitly doing a mkdir via shell. 1357 // 1358 // Devices that don't support shell_v2 are unhappy if we try to send a too-long packet to them, 1359 // but they're not affected by this bug, so only apply the workaround if we have shell_v2. 1360 // 1361 // TODO(b/25457350): We don't preserve permissions on directories. 1362 // TODO: Find all of the leaves and `mkdir -p` them instead? 1363 if (!CanUseFeature(sc.Features(), kFeatureFixedPushMkdir) && 1364 CanUseFeature(sc.Features(), kFeatureShell2)) { 1365 SilentStandardStreamsCallbackInterface cb; 1366 std::string cmd = "mkdir"; 1367 for (const auto& dir : directory_list) { 1368 std::string escaped_path = escape_arg(dir); 1369 if (escaped_path.size() > 16384) { 1370 // Somewhat arbitrarily limit that probably won't ever happen. 1371 sc.Error("path too long: %s", escaped_path.c_str()); 1372 return false; 1373 } 1374 1375 // The maximum should be 64kiB, but that's not including other stuff that gets tacked 1376 // onto the command line, so let's be a bit conservative. 1377 if (cmd.size() + escaped_path.size() > 32768) { 1378 // Dispatch the command, ignoring failure (since the directory might already exist). 1379 send_shell_command(cmd, false, &cb); 1380 cmd = "mkdir"; 1381 } 1382 cmd += " "; 1383 cmd += escaped_path; 1384 } 1385 1386 if (cmd != "mkdir") { 1387 send_shell_command(cmd, false, &cb); 1388 } 1389 } 1390 1391 if (check_timestamps) { 1392 for (const copyinfo& ci : file_list) { 1393 if (!sc.SendLstat(ci.rpath)) { 1394 sc.Error("failed to send lstat"); 1395 return false; 1396 } 1397 } 1398 for (copyinfo& ci : file_list) { 1399 struct stat st; 1400 if (sc.FinishStat(&st)) { 1401 if (st.st_size == static_cast<off_t>(ci.size) && st.st_mtime == ci.time) { 1402 ci.skip = true; 1403 } 1404 } 1405 } 1406 } 1407 1408 sc.ComputeExpectedTotalBytes(file_list); 1409 1410 for (const copyinfo& ci : file_list) { 1411 if (!ci.skip) { 1412 if (list_only) { 1413 sc.Println("would push: %s -> %s", ci.lpath.c_str(), ci.rpath.c_str()); 1414 } else { 1415 if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compression, 1416 dry_run)) { 1417 return false; 1418 } 1419 } 1420 } else { 1421 skipped++; 1422 } 1423 } 1424 1425 sc.RecordFilesSkipped(skipped); 1426 bool success = sc.ReadAcknowledgements(true); 1427 sc.ReportTransferRate(lpath, TransferDirection::push); 1428 return success; 1429 } 1430 1431 bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync, 1432 CompressionType compression, bool dry_run) { 1433 SyncConnection sc; 1434 if (!sc.IsValid()) return false; 1435 1436 bool success = true; 1437 bool dst_exists; 1438 bool dst_isdir; 1439 1440 struct stat st; 1441 if (sync_stat_fallback(sc, dst, &st)) { 1442 dst_exists = true; 1443 dst_isdir = S_ISDIR(st.st_mode); 1444 } else { 1445 if (errno == ENOENT || errno == ENOPROTOOPT) { 1446 dst_exists = false; 1447 dst_isdir = false; 1448 } else { 1449 sc.Error("stat failed when trying to push to %s: %s", dst, strerror(errno)); 1450 return false; 1451 } 1452 } 1453 1454 if (!dst_isdir) { 1455 if (srcs.size() > 1) { 1456 sc.Error("target '%s' is not a directory", dst); 1457 return false; 1458 } else { 1459 size_t dst_len = strlen(dst); 1460 1461 // A path that ends with a slash doesn't have to be a directory if 1462 // it doesn't exist yet. 1463 if (dst[dst_len - 1] == '/' && dst_exists) { 1464 sc.Error("failed to access '%s': Not a directory", dst); 1465 return false; 1466 } 1467 } 1468 } 1469 1470 for (const char* src_path : srcs) { 1471 const char* dst_path = dst; 1472 struct stat st; 1473 if (stat(src_path, &st) == -1) { 1474 sc.Error("cannot stat '%s': %s", src_path, strerror(errno)); 1475 success = false; 1476 continue; 1477 } 1478 1479 if (S_ISDIR(st.st_mode)) { 1480 std::string dst_dir = dst; 1481 1482 // If the destination path existed originally, the source directory 1483 // should be copied as a child of the destination. 1484 if (dst_exists) { 1485 if (!dst_isdir) { 1486 sc.Error("target '%s' is not a directory", dst); 1487 return false; 1488 } 1489 // dst is a POSIX path, so we don't want to use the sysdeps 1490 // helpers here. 1491 if (dst_dir.back() != '/') { 1492 dst_dir.push_back('/'); 1493 } 1494 dst_dir.append(android::base::Basename(src_path)); 1495 } 1496 1497 success &= 1498 copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compression, dry_run); 1499 continue; 1500 } else if (!should_push_file(st.st_mode)) { 1501 sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, st.st_mode); 1502 continue; 1503 } 1504 1505 std::string path_holder; 1506 if (dst_isdir) { 1507 // If we're copying a local file to a remote directory, 1508 // we really want to copy to remote_dir + "/" + local_filename. 1509 path_holder = dst_path; 1510 if (path_holder.back() != '/') { 1511 path_holder.push_back('/'); 1512 } 1513 path_holder += android::base::Basename(src_path); 1514 dst_path = path_holder.c_str(); 1515 } 1516 1517 sc.NewTransfer(); 1518 sc.SetExpectedTotalBytes(st.st_size); 1519 success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compression, 1520 dry_run); 1521 sc.ReportTransferRate(src_path, TransferDirection::push); 1522 } 1523 1524 success &= sc.ReadAcknowledgements(true); 1525 sc.ReportOverallTransferRate(TransferDirection::push); 1526 return success; 1527 } 1528 1529 static bool remote_build_list(SyncConnection& sc, std::vector<copyinfo>* file_list, 1530 const std::string& rpath, const std::string& lpath) { 1531 std::vector<copyinfo> dirlist; 1532 std::vector<copyinfo> linklist; 1533 1534 // Add an entry for the current directory to ensure it gets created before pulling its contents. 1535 copyinfo ci(android::base::Dirname(lpath), android::base::Dirname(rpath), 1536 android::base::Basename(lpath), S_IFDIR); 1537 file_list->push_back(ci); 1538 1539 // Put the files/dirs in rpath on the lists. 1540 auto callback = [&](unsigned mode, uint64_t size, uint64_t time, const char* name) { 1541 if (IsDotOrDotDot(name)) { 1542 return; 1543 } 1544 1545 copyinfo ci(lpath, rpath, name, mode); 1546 if (S_ISDIR(mode)) { 1547 dirlist.push_back(ci); 1548 } else if (S_ISLNK(mode)) { 1549 linklist.push_back(ci); 1550 } else { 1551 if (!should_pull_file(ci.mode)) { 1552 sc.Warning("skipping special file '%s' (mode = 0o%o)", ci.rpath.c_str(), ci.mode); 1553 ci.skip = true; 1554 } 1555 ci.time = time; 1556 ci.size = size; 1557 file_list->push_back(ci); 1558 } 1559 }; 1560 1561 if (!sync_ls(sc, rpath, callback)) { 1562 return false; 1563 } 1564 1565 // Check each symlink we found to see whether it's a file or directory. 1566 for (copyinfo& link_ci : linklist) { 1567 struct stat st; 1568 if (!sync_stat_fallback(sc, link_ci.rpath, &st)) { 1569 sc.Warning("stat failed for path %s: %s", link_ci.rpath.c_str(), strerror(errno)); 1570 continue; 1571 } 1572 1573 if (S_ISDIR(st.st_mode)) { 1574 dirlist.emplace_back(std::move(link_ci)); 1575 } else { 1576 file_list->emplace_back(std::move(link_ci)); 1577 } 1578 } 1579 1580 // Recurse into each directory we found. 1581 while (!dirlist.empty()) { 1582 copyinfo current = dirlist.back(); 1583 dirlist.pop_back(); 1584 if (!remote_build_list(sc, file_list, current.rpath, current.lpath)) { 1585 return false; 1586 } 1587 } 1588 1589 return true; 1590 } 1591 1592 static int set_time_and_mode(const std::string& lpath, time_t time, 1593 unsigned int mode) { 1594 struct utimbuf times = { time, time }; 1595 int r1 = utime(lpath.c_str(), ×); 1596 1597 /* use umask for permissions */ 1598 mode_t mask = umask(0000); 1599 umask(mask); 1600 int r2 = chmod(lpath.c_str(), mode & ~mask); 1601 1602 return r1 ? r1 : r2; 1603 } 1604 1605 static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::string lpath, 1606 bool copy_attrs, CompressionType compression) { 1607 sc.NewTransfer(); 1608 1609 // Make sure that both directory paths end in a slash. 1610 // Both paths are known to be nonempty, so we don't need to check. 1611 ensure_trailing_separators(lpath, rpath); 1612 1613 // Recursively build the list of files to copy. 1614 sc.Printf("pull: building file list..."); 1615 std::vector<copyinfo> file_list; 1616 if (!remote_build_list(sc, &file_list, rpath, lpath)) { 1617 return false; 1618 } 1619 1620 sc.ComputeExpectedTotalBytes(file_list); 1621 1622 int skipped = 0; 1623 for (const copyinfo &ci : file_list) { 1624 if (!ci.skip) { 1625 if (S_ISDIR(ci.mode)) { 1626 // Entry is for an empty directory, create it and continue. 1627 // TODO(b/25457350): We don't preserve permissions on directories. 1628 if (!mkdirs(ci.lpath)) { 1629 sc.Error("failed to create directory '%s': %s", 1630 ci.lpath.c_str(), strerror(errno)); 1631 return false; 1632 } 1633 continue; 1634 } 1635 1636 if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size, compression)) { 1637 return false; 1638 } 1639 1640 if (copy_attrs && set_time_and_mode(ci.lpath, ci.time, ci.mode)) { 1641 return false; 1642 } 1643 } else { 1644 skipped++; 1645 } 1646 } 1647 1648 sc.RecordFilesSkipped(skipped); 1649 sc.ReportTransferRate(rpath, TransferDirection::pull); 1650 return true; 1651 } 1652 1653 bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs, 1654 CompressionType compression, const char* name) { 1655 SyncConnection sc; 1656 if (!sc.IsValid()) return false; 1657 1658 bool success = true; 1659 struct stat st; 1660 bool dst_exists = true; 1661 1662 if (stat(dst, &st) == -1) { 1663 dst_exists = false; 1664 1665 // If we're only pulling one path, the destination path might point to 1666 // a path that doesn't exist yet. 1667 if (srcs.size() == 1 && errno == ENOENT) { 1668 // However, its parent must exist. 1669 struct stat parent_st; 1670 if (stat(android::base::Dirname(dst).c_str(), &parent_st) == -1) { 1671 sc.Error("cannot create file/directory '%s': %s", dst, strerror(errno)); 1672 return false; 1673 } 1674 } else { 1675 sc.Error("failed to access '%s': %s", dst, strerror(errno)); 1676 return false; 1677 } 1678 } 1679 1680 bool dst_isdir = dst_exists && S_ISDIR(st.st_mode); 1681 if (!dst_isdir) { 1682 if (srcs.size() > 1) { 1683 sc.Error("target '%s' is not a directory", dst); 1684 return false; 1685 } else { 1686 size_t dst_len = strlen(dst); 1687 1688 // A path that ends with a slash doesn't have to be a directory if 1689 // it doesn't exist yet. 1690 if (adb_is_separator(dst[dst_len - 1]) && dst_exists) { 1691 sc.Error("failed to access '%s': Not a directory", dst); 1692 return false; 1693 } 1694 } 1695 } 1696 1697 for (const char* src_path : srcs) { 1698 const char* dst_path = dst; 1699 struct stat src_st; 1700 if (!sync_stat_fallback(sc, src_path, &src_st)) { 1701 if (errno == ENOPROTOOPT) { 1702 sc.Error("remote object '%s' does not exist", src_path); 1703 } else { 1704 sc.Error("failed to stat remote object '%s': %s", src_path, strerror(errno)); 1705 } 1706 1707 success = false; 1708 continue; 1709 } 1710 1711 bool src_isdir = S_ISDIR(src_st.st_mode); 1712 if (src_isdir) { 1713 std::string dst_dir = dst; 1714 1715 // If the destination path existed originally, the source directory 1716 // should be copied as a child of the destination. 1717 if (dst_exists) { 1718 if (!dst_isdir) { 1719 sc.Error("target '%s' is not a directory", dst); 1720 return false; 1721 } 1722 if (!adb_is_separator(dst_dir.back())) { 1723 dst_dir.push_back(OS_PATH_SEPARATOR); 1724 } 1725 dst_dir.append(android::base::Basename(src_path)); 1726 } 1727 1728 success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs, compression); 1729 continue; 1730 } else if (!should_pull_file(src_st.st_mode)) { 1731 sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, src_st.st_mode); 1732 continue; 1733 } 1734 1735 std::string path_holder; 1736 if (dst_isdir) { 1737 // If we're copying a remote file to a local directory, we 1738 // really want to copy to local_dir + OS_PATH_SEPARATOR + 1739 // basename(remote). 1740 path_holder = android::base::StringPrintf("%s%c%s", dst_path, OS_PATH_SEPARATOR, 1741 android::base::Basename(src_path).c_str()); 1742 dst_path = path_holder.c_str(); 1743 } 1744 1745 sc.NewTransfer(); 1746 sc.SetExpectedTotalBytes(src_st.st_size); 1747 if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size, compression)) { 1748 success = false; 1749 continue; 1750 } 1751 1752 if (copy_attrs && set_time_and_mode(dst_path, src_st.st_mtime, src_st.st_mode) != 0) { 1753 success = false; 1754 continue; 1755 } 1756 sc.ReportTransferRate(src_path, TransferDirection::pull); 1757 } 1758 1759 sc.ReportOverallTransferRate(TransferDirection::pull); 1760 return success; 1761 } 1762 1763 bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only, 1764 CompressionType compression, bool dry_run) { 1765 SyncConnection sc; 1766 if (!sc.IsValid()) return false; 1767 1768 bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compression, dry_run); 1769 if (!list_only) { 1770 sc.ReportOverallTransferRate(TransferDirection::push); 1771 } 1772 return success; 1773 } 1774