1/* 2 * Copyright (C) 2007 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include <dirent.h> 18#include <inttypes.h> 19#include <limits.h> 20#include <stdio.h> 21#include <stdlib.h> 22#include <sys/stat.h> 23#include <sys/time.h> 24#include <sys/types.h> 25#include <time.h> 26#include <unistd.h> 27#include <utime.h> 28 29#include <chrono> 30#include <functional> 31#include <memory> 32#include <sstream> 33#include <string> 34#include <vector> 35 36#include "sysdeps.h" 37 38#include "adb.h" 39#include "adb_client.h" 40#include "adb_io.h" 41#include "adb_utils.h" 42#include "file_sync_service.h" 43#include "line_printer.h" 44#include "sysdeps/errno.h" 45#include "sysdeps/stat.h" 46 47#include <android-base/file.h> 48#include <android-base/strings.h> 49#include <android-base/stringprintf.h> 50 51struct syncsendbuf { 52 unsigned id; 53 unsigned size; 54 char data[SYNC_DATA_MAX]; 55}; 56 57static void ensure_trailing_separators(std::string& local_path, std::string& remote_path) { 58 if (!adb_is_separator(local_path.back())) { 59 local_path.push_back(OS_PATH_SEPARATOR); 60 } 61 if (remote_path.back() != '/') { 62 remote_path.push_back('/'); 63 } 64} 65 66static bool should_pull_file(mode_t mode) { 67 return S_ISREG(mode) || S_ISBLK(mode) || S_ISCHR(mode); 68} 69 70static bool should_push_file(mode_t mode) { 71 return S_ISREG(mode) || S_ISLNK(mode); 72} 73 74struct copyinfo { 75 std::string lpath; 76 std::string rpath; 77 int64_t time = 0; 78 uint32_t mode; 79 uint64_t size = 0; 80 bool skip = false; 81 82 copyinfo(const std::string& local_path, 83 const std::string& remote_path, 84 const std::string& name, 85 unsigned int mode) 86 : lpath(local_path), rpath(remote_path), mode(mode) { 87 ensure_trailing_separators(lpath, rpath); 88 lpath.append(name); 89 rpath.append(name); 90 if (S_ISDIR(mode)) { 91 ensure_trailing_separators(lpath, rpath); 92 } 93 } 94}; 95 96enum class TransferDirection { 97 push, 98 pull, 99}; 100 101struct TransferLedger { 102 std::chrono::steady_clock::time_point start_time; 103 uint64_t files_transferred; 104 uint64_t files_skipped; 105 uint64_t bytes_transferred; 106 uint64_t bytes_expected; 107 bool expect_multiple_files; 108 109 TransferLedger() { 110 Reset(); 111 } 112 113 bool operator==(const TransferLedger& other) const { 114 return files_transferred == other.files_transferred && 115 files_skipped == other.files_skipped && bytes_transferred == other.bytes_transferred; 116 } 117 118 bool operator!=(const TransferLedger& other) const { 119 return !(*this == other); 120 } 121 122 void Reset() { 123 start_time = std::chrono::steady_clock::now(); 124 files_transferred = 0; 125 files_skipped = 0; 126 bytes_transferred = 0; 127 bytes_expected = 0; 128 } 129 130 std::string TransferRate() { 131 if (bytes_transferred == 0) return ""; 132 133 std::chrono::duration<double> duration; 134 duration = std::chrono::steady_clock::now() - start_time; 135 136 double s = duration.count(); 137 if (s == 0) { 138 return ""; 139 } 140 double rate = (static_cast<double>(bytes_transferred) / s) / (1024 * 1024); 141 return android::base::StringPrintf(" %.1f MB/s (%" PRIu64 " bytes in %.3fs)", rate, 142 bytes_transferred, s); 143 } 144 145 void ReportProgress(LinePrinter& lp, const std::string& file, uint64_t file_copied_bytes, 146 uint64_t file_total_bytes) { 147 char overall_percentage_str[5] = "?"; 148 if (bytes_expected != 0 && bytes_transferred <= bytes_expected) { 149 int overall_percentage = static_cast<int>(bytes_transferred * 100 / bytes_expected); 150 // If we're pulling symbolic links, we'll pull the target of the link rather than 151 // just create a local link, and that will cause us to go over 100%. 152 if (overall_percentage <= 100) { 153 snprintf(overall_percentage_str, sizeof(overall_percentage_str), "%d%%", 154 overall_percentage); 155 } 156 } 157 158 std::string output; 159 if (file_copied_bytes > file_total_bytes || file_total_bytes == 0) { 160 // This case can happen if we're racing against something that wrote to the file 161 // between our stat and our read, or if we're reading a magic file that lies about 162 // its size. Just show how much we've copied. 163 output = android::base::StringPrintf("[%4s] %s: %" PRId64 "/?", overall_percentage_str, 164 file.c_str(), file_copied_bytes); 165 } else { 166 // If we're transferring multiple files, we want to know how far through the current 167 // file we are, as well as the overall percentage. 168 if (expect_multiple_files) { 169 int file_percentage = static_cast<int>(file_copied_bytes * 100 / file_total_bytes); 170 output = android::base::StringPrintf("[%4s] %s: %d%%", overall_percentage_str, 171 file.c_str(), file_percentage); 172 } else { 173 output = 174 android::base::StringPrintf("[%4s] %s", overall_percentage_str, file.c_str()); 175 } 176 } 177 lp.Print(output, LinePrinter::LineType::INFO); 178 } 179 180 void ReportTransferRate(LinePrinter& lp, const std::string& name, TransferDirection direction) { 181 const char* direction_str = (direction == TransferDirection::push) ? "pushed" : "pulled"; 182 std::stringstream ss; 183 if (!name.empty()) { 184 ss << name << ": "; 185 } 186 ss << files_transferred << " file" << ((files_transferred == 1) ? "" : "s") << " " 187 << direction_str << "."; 188 if (files_skipped > 0) { 189 ss << " " << files_skipped << " file" << ((files_skipped == 1) ? "" : "s") 190 << " skipped."; 191 } 192 ss << TransferRate(); 193 194 lp.Print(ss.str(), LinePrinter::LineType::INFO); 195 lp.KeepInfoLine(); 196 } 197}; 198 199class SyncConnection { 200 public: 201 SyncConnection() : expect_done_(false) { 202 max = SYNC_DATA_MAX; // TODO: decide at runtime. 203 204 std::string error; 205 FeatureSet features; 206 if (!adb_get_feature_set(&features, &error)) { 207 fd = -1; 208 Error("failed to get feature set: %s", error.c_str()); 209 } else { 210 have_stat_v2_ = CanUseFeature(features, kFeatureStat2); 211 fd = adb_connect("sync:", &error); 212 if (fd < 0) { 213 Error("connect failed: %s", error.c_str()); 214 } 215 } 216 } 217 218 ~SyncConnection() { 219 if (!IsValid()) return; 220 221 if (SendQuit()) { 222 // We sent a quit command, so the server should be doing orderly 223 // shutdown soon. But if we encountered an error while we were using 224 // the connection, the server might still be sending data (before 225 // doing orderly shutdown), in which case we won't wait for all of 226 // the data nor the coming orderly shutdown. In the common success 227 // case, this will wait for the server to do orderly shutdown. 228 ReadOrderlyShutdown(fd); 229 } 230 adb_close(fd); 231 232 line_printer_.KeepInfoLine(); 233 } 234 235 bool IsValid() { return fd >= 0; } 236 237 bool ReceivedError(const char* from, const char* to) { 238 adb_pollfd pfd = {.fd = fd, .events = POLLIN}; 239 int rc = adb_poll(&pfd, 1, 0); 240 if (rc < 0) { 241 Error("failed to poll: %s", strerror(errno)); 242 return true; 243 } 244 return rc != 0; 245 } 246 247 void NewTransfer() { 248 current_ledger_.Reset(); 249 } 250 251 void RecordBytesTransferred(size_t bytes) { 252 current_ledger_.bytes_transferred += bytes; 253 global_ledger_.bytes_transferred += bytes; 254 } 255 256 void RecordFilesTransferred(size_t files) { 257 current_ledger_.files_transferred += files; 258 global_ledger_.files_transferred += files; 259 } 260 261 void RecordFilesSkipped(size_t files) { 262 current_ledger_.files_skipped += files; 263 global_ledger_.files_skipped += files; 264 } 265 266 void ReportProgress(const std::string& file, uint64_t file_copied_bytes, 267 uint64_t file_total_bytes) { 268 current_ledger_.ReportProgress(line_printer_, file, file_copied_bytes, file_total_bytes); 269 } 270 271 void ReportTransferRate(const std::string& file, TransferDirection direction) { 272 current_ledger_.ReportTransferRate(line_printer_, file, direction); 273 } 274 275 void ReportOverallTransferRate(TransferDirection direction) { 276 if (current_ledger_ != global_ledger_) { 277 global_ledger_.ReportTransferRate(line_printer_, "", direction); 278 } 279 } 280 281 bool SendRequest(int id, const char* path_and_mode) { 282 size_t path_length = strlen(path_and_mode); 283 if (path_length > 1024) { 284 Error("SendRequest failed: path too long: %zu", path_length); 285 errno = ENAMETOOLONG; 286 return false; 287 } 288 289 // Sending header and payload in a single write makes a noticeable 290 // difference to "adb sync" performance. 291 std::vector<char> buf(sizeof(SyncRequest) + path_length); 292 SyncRequest* req = reinterpret_cast<SyncRequest*>(&buf[0]); 293 req->id = id; 294 req->path_length = path_length; 295 char* data = reinterpret_cast<char*>(req + 1); 296 memcpy(data, path_and_mode, path_length); 297 298 return WriteFdExactly(fd, &buf[0], buf.size()); 299 } 300 301 bool SendStat(const char* path_and_mode) { 302 if (!have_stat_v2_) { 303 errno = ENOTSUP; 304 return false; 305 } 306 return SendRequest(ID_STAT_V2, path_and_mode); 307 } 308 309 bool SendLstat(const char* path_and_mode) { 310 if (have_stat_v2_) { 311 return SendRequest(ID_LSTAT_V2, path_and_mode); 312 } else { 313 return SendRequest(ID_LSTAT_V1, path_and_mode); 314 } 315 } 316 317 bool FinishStat(struct stat* st) { 318 syncmsg msg; 319 320 memset(st, 0, sizeof(*st)); 321 if (have_stat_v2_) { 322 if (!ReadFdExactly(fd, &msg.stat_v2, sizeof(msg.stat_v2))) { 323 fatal_errno("protocol fault: failed to read stat response"); 324 } 325 326 if (msg.stat_v2.id != ID_LSTAT_V2 && msg.stat_v2.id != ID_STAT_V2) { 327 fatal_errno("protocol fault: stat response has wrong message id: %" PRIx32, 328 msg.stat_v2.id); 329 } 330 331 if (msg.stat_v2.error != 0) { 332 errno = errno_from_wire(msg.stat_v2.error); 333 return false; 334 } 335 336 st->st_dev = msg.stat_v2.dev; 337 st->st_ino = msg.stat_v2.ino; 338 st->st_mode = msg.stat_v2.mode; 339 st->st_nlink = msg.stat_v2.nlink; 340 st->st_uid = msg.stat_v2.uid; 341 st->st_gid = msg.stat_v2.gid; 342 st->st_size = msg.stat_v2.size; 343 st->st_atime = msg.stat_v2.atime; 344 st->st_mtime = msg.stat_v2.mtime; 345 st->st_ctime = msg.stat_v2.ctime; 346 return true; 347 } else { 348 if (!ReadFdExactly(fd, &msg.stat_v1, sizeof(msg.stat_v1))) { 349 fatal_errno("protocol fault: failed to read stat response"); 350 } 351 352 if (msg.stat_v1.id != ID_LSTAT_V1) { 353 fatal_errno("protocol fault: stat response has wrong message id: %" PRIx32, 354 msg.stat_v1.id); 355 } 356 357 if (msg.stat_v1.mode == 0 && msg.stat_v1.size == 0 && msg.stat_v1.time == 0) { 358 // There's no way for us to know what the error was. 359 errno = ENOPROTOOPT; 360 return false; 361 } 362 363 st->st_mode = msg.stat_v1.mode; 364 st->st_size = msg.stat_v1.size; 365 st->st_ctime = msg.stat_v1.time; 366 st->st_mtime = msg.stat_v1.time; 367 } 368 369 return true; 370 } 371 372 // Sending header, payload, and footer in a single write makes a huge 373 // difference to "adb sync" performance. 374 bool SendSmallFile(const char* path_and_mode, 375 const char* lpath, const char* rpath, 376 unsigned mtime, 377 const char* data, size_t data_length) { 378 size_t path_length = strlen(path_and_mode); 379 if (path_length > 1024) { 380 Error("SendSmallFile failed: path too long: %zu", path_length); 381 errno = ENAMETOOLONG; 382 return false; 383 } 384 385 std::vector<char> buf(sizeof(SyncRequest) + path_length + 386 sizeof(SyncRequest) + data_length + 387 sizeof(SyncRequest)); 388 char* p = &buf[0]; 389 390 SyncRequest* req_send = reinterpret_cast<SyncRequest*>(p); 391 req_send->id = ID_SEND; 392 req_send->path_length = path_length; 393 p += sizeof(SyncRequest); 394 memcpy(p, path_and_mode, path_length); 395 p += path_length; 396 397 SyncRequest* req_data = reinterpret_cast<SyncRequest*>(p); 398 req_data->id = ID_DATA; 399 req_data->path_length = data_length; 400 p += sizeof(SyncRequest); 401 memcpy(p, data, data_length); 402 p += data_length; 403 404 SyncRequest* req_done = reinterpret_cast<SyncRequest*>(p); 405 req_done->id = ID_DONE; 406 req_done->path_length = mtime; 407 p += sizeof(SyncRequest); 408 409 WriteOrDie(lpath, rpath, &buf[0], (p - &buf[0])); 410 expect_done_ = true; 411 412 // RecordFilesTransferred gets called in CopyDone. 413 RecordBytesTransferred(data_length); 414 ReportProgress(rpath, data_length, data_length); 415 return true; 416 } 417 418 bool SendLargeFile(const char* path_and_mode, 419 const char* lpath, const char* rpath, 420 unsigned mtime) { 421 if (!SendRequest(ID_SEND, path_and_mode)) { 422 Error("failed to send ID_SEND message '%s': %s", path_and_mode, strerror(errno)); 423 return false; 424 } 425 426 struct stat st; 427 if (stat(lpath, &st) == -1) { 428 Error("cannot stat '%s': %s", lpath, strerror(errno)); 429 return false; 430 } 431 432 uint64_t total_size = st.st_size; 433 uint64_t bytes_copied = 0; 434 435 int lfd = adb_open(lpath, O_RDONLY); 436 if (lfd < 0) { 437 Error("opening '%s' locally failed: %s", lpath, strerror(errno)); 438 return false; 439 } 440 441 syncsendbuf sbuf; 442 sbuf.id = ID_DATA; 443 while (true) { 444 int bytes_read = adb_read(lfd, sbuf.data, max); 445 if (bytes_read == -1) { 446 Error("reading '%s' locally failed: %s", lpath, strerror(errno)); 447 adb_close(lfd); 448 return false; 449 } else if (bytes_read == 0) { 450 break; 451 } 452 453 sbuf.size = bytes_read; 454 WriteOrDie(lpath, rpath, &sbuf, sizeof(SyncRequest) + bytes_read); 455 456 RecordBytesTransferred(bytes_read); 457 bytes_copied += bytes_read; 458 459 // Check to see if we've received an error from the other side. 460 if (ReceivedError(lpath, rpath)) { 461 break; 462 } 463 464 ReportProgress(rpath, bytes_copied, total_size); 465 } 466 467 adb_close(lfd); 468 469 syncmsg msg; 470 msg.data.id = ID_DONE; 471 msg.data.size = mtime; 472 expect_done_ = true; 473 474 // RecordFilesTransferred gets called in CopyDone. 475 return WriteOrDie(lpath, rpath, &msg.data, sizeof(msg.data)); 476 } 477 478 bool CopyDone(const char* from, const char* to) { 479 syncmsg msg; 480 if (!ReadFdExactly(fd, &msg.status, sizeof(msg.status))) { 481 Error("failed to copy '%s' to '%s': couldn't read from device", from, to); 482 return false; 483 } 484 if (msg.status.id == ID_OKAY) { 485 if (expect_done_) { 486 expect_done_ = false; 487 RecordFilesTransferred(1); 488 return true; 489 } else { 490 Error("failed to copy '%s' to '%s': received premature success", from, to); 491 return true; 492 } 493 } 494 if (msg.status.id != ID_FAIL) { 495 Error("failed to copy '%s' to '%s': unknown reason %d", from, to, msg.status.id); 496 return false; 497 } 498 return ReportCopyFailure(from, to, msg); 499 } 500 501 bool ReportCopyFailure(const char* from, const char* to, const syncmsg& msg) { 502 std::vector<char> buf(msg.status.msglen + 1); 503 if (!ReadFdExactly(fd, &buf[0], msg.status.msglen)) { 504 Error("failed to copy '%s' to '%s'; failed to read reason (!): %s", 505 from, to, strerror(errno)); 506 return false; 507 } 508 buf[msg.status.msglen] = 0; 509 Error("failed to copy '%s' to '%s': remote %s", from, to, &buf[0]); 510 return false; 511 } 512 513 514 void Printf(const char* fmt, ...) __attribute__((__format__(ADB_FORMAT_ARCHETYPE, 2, 3))) { 515 std::string s; 516 517 va_list ap; 518 va_start(ap, fmt); 519 android::base::StringAppendV(&s, fmt, ap); 520 va_end(ap); 521 522 line_printer_.Print(s, LinePrinter::INFO); 523 } 524 525 void Println(const char* fmt, ...) __attribute__((__format__(ADB_FORMAT_ARCHETYPE, 2, 3))) { 526 std::string s; 527 528 va_list ap; 529 va_start(ap, fmt); 530 android::base::StringAppendV(&s, fmt, ap); 531 va_end(ap); 532 533 line_printer_.Print(s, LinePrinter::INFO); 534 line_printer_.KeepInfoLine(); 535 } 536 537 void Error(const char* fmt, ...) __attribute__((__format__(ADB_FORMAT_ARCHETYPE, 2, 3))) { 538 std::string s = "adb: error: "; 539 540 va_list ap; 541 va_start(ap, fmt); 542 android::base::StringAppendV(&s, fmt, ap); 543 va_end(ap); 544 545 line_printer_.Print(s, LinePrinter::ERROR); 546 } 547 548 void Warning(const char* fmt, ...) __attribute__((__format__(ADB_FORMAT_ARCHETYPE, 2, 3))) { 549 std::string s = "adb: warning: "; 550 551 va_list ap; 552 va_start(ap, fmt); 553 android::base::StringAppendV(&s, fmt, ap); 554 va_end(ap); 555 556 line_printer_.Print(s, LinePrinter::WARNING); 557 } 558 559 void ComputeExpectedTotalBytes(const std::vector<copyinfo>& file_list) { 560 current_ledger_.bytes_expected = 0; 561 for (const copyinfo& ci : file_list) { 562 // Unfortunately, this doesn't work for symbolic links, because we'll copy the 563 // target of the link rather than just creating a link. (But ci.size is the link size.) 564 if (!ci.skip) current_ledger_.bytes_expected += ci.size; 565 } 566 current_ledger_.expect_multiple_files = true; 567 } 568 569 void SetExpectedTotalBytes(uint64_t expected_total_bytes) { 570 current_ledger_.bytes_expected = expected_total_bytes; 571 current_ledger_.expect_multiple_files = false; 572 } 573 574 // TODO: add a char[max] buffer here, to replace syncsendbuf... 575 int fd; 576 size_t max; 577 578 private: 579 bool expect_done_; 580 bool have_stat_v2_; 581 582 TransferLedger global_ledger_; 583 TransferLedger current_ledger_; 584 LinePrinter line_printer_; 585 586 bool SendQuit() { 587 return SendRequest(ID_QUIT, ""); // TODO: add a SendResponse? 588 } 589 590 bool WriteOrDie(const char* from, const char* to, const void* data, size_t data_length) { 591 if (!WriteFdExactly(fd, data, data_length)) { 592 if (errno == ECONNRESET) { 593 // Assume adbd told us why it was closing the connection, and 594 // try to read failure reason from adbd. 595 syncmsg msg; 596 if (!ReadFdExactly(fd, &msg.status, sizeof(msg.status))) { 597 Error("failed to copy '%s' to '%s': no response: %s", from, to, strerror(errno)); 598 } else if (msg.status.id != ID_FAIL) { 599 Error("failed to copy '%s' to '%s': not ID_FAIL: %d", from, to, msg.status.id); 600 } else { 601 ReportCopyFailure(from, to, msg); 602 } 603 } else { 604 Error("%zu-byte write failed: %s", data_length, strerror(errno)); 605 } 606 _exit(1); 607 } 608 return true; 609 } 610}; 611 612typedef void (sync_ls_cb)(unsigned mode, unsigned size, unsigned time, const char* name); 613 614static bool sync_ls(SyncConnection& sc, const char* path, 615 const std::function<sync_ls_cb>& func) { 616 if (!sc.SendRequest(ID_LIST, path)) return false; 617 618 while (true) { 619 syncmsg msg; 620 if (!ReadFdExactly(sc.fd, &msg.dent, sizeof(msg.dent))) return false; 621 622 if (msg.dent.id == ID_DONE) return true; 623 if (msg.dent.id != ID_DENT) return false; 624 625 size_t len = msg.dent.namelen; 626 if (len > 256) return false; // TODO: resize buffer? continue? 627 628 char buf[257]; 629 if (!ReadFdExactly(sc.fd, buf, len)) return false; 630 buf[len] = 0; 631 632 func(msg.dent.mode, msg.dent.size, msg.dent.time, buf); 633 } 634} 635 636static bool sync_stat(SyncConnection& sc, const char* path, struct stat* st) { 637 return sc.SendStat(path) && sc.FinishStat(st); 638} 639 640static bool sync_lstat(SyncConnection& sc, const char* path, struct stat* st) { 641 return sc.SendLstat(path) && sc.FinishStat(st); 642} 643 644static bool sync_stat_fallback(SyncConnection& sc, const char* path, struct stat* st) { 645 if (sync_stat(sc, path, st)) { 646 return true; 647 } 648 649 if (errno != ENOTSUP) { 650 return false; 651 } 652 653 // Try to emulate the parts we can when talking to older adbds. 654 bool lstat_result = sync_lstat(sc, path, st); 655 if (!lstat_result) { 656 return false; 657 } 658 659 if (S_ISLNK(st->st_mode)) { 660 // If the target is a symlink, figure out whether it's a file or a directory. 661 // Also, zero out the st_size field, since no one actually cares what the path length is. 662 st->st_size = 0; 663 std::string dir_path = path; 664 dir_path.push_back('/'); 665 struct stat tmp_st; 666 667 st->st_mode &= ~S_IFMT; 668 if (sync_lstat(sc, dir_path.c_str(), &tmp_st)) { 669 st->st_mode |= S_IFDIR; 670 } else { 671 st->st_mode |= S_IFREG; 672 } 673 } 674 return true; 675} 676 677static bool sync_send(SyncConnection& sc, const char* lpath, const char* rpath, unsigned mtime, 678 mode_t mode, bool sync) { 679 std::string path_and_mode = android::base::StringPrintf("%s,%d", rpath, mode); 680 681 if (sync) { 682 struct stat st; 683 if (sync_lstat(sc, rpath, &st)) { 684 // For links, we cannot update the atime/mtime. 685 if ((S_ISREG(mode & st.st_mode) && st.st_mtime == static_cast<time_t>(mtime)) || 686 (S_ISLNK(mode & st.st_mode) && st.st_mtime >= static_cast<time_t>(mtime))) { 687 sc.RecordFilesSkipped(1); 688 return true; 689 } 690 } 691 } 692 693 if (S_ISLNK(mode)) { 694#if !defined(_WIN32) 695 char buf[PATH_MAX]; 696 ssize_t data_length = readlink(lpath, buf, PATH_MAX - 1); 697 if (data_length == -1) { 698 sc.Error("readlink '%s' failed: %s", lpath, strerror(errno)); 699 return false; 700 } 701 buf[data_length++] = '\0'; 702 703 if (!sc.SendSmallFile(path_and_mode.c_str(), lpath, rpath, mtime, buf, data_length)) { 704 return false; 705 } 706 return sc.CopyDone(lpath, rpath); 707#endif 708 } 709 710 struct stat st; 711 if (stat(lpath, &st) == -1) { 712 sc.Error("failed to stat local file '%s': %s", lpath, strerror(errno)); 713 return false; 714 } 715 if (st.st_size < SYNC_DATA_MAX) { 716 std::string data; 717 if (!android::base::ReadFileToString(lpath, &data, true)) { 718 sc.Error("failed to read all of '%s': %s", lpath, strerror(errno)); 719 return false; 720 } 721 if (!sc.SendSmallFile(path_and_mode.c_str(), lpath, rpath, mtime, 722 data.data(), data.size())) { 723 return false; 724 } 725 } else { 726 if (!sc.SendLargeFile(path_and_mode.c_str(), lpath, rpath, mtime)) { 727 return false; 728 } 729 } 730 return sc.CopyDone(lpath, rpath); 731} 732 733static bool sync_recv(SyncConnection& sc, const char* rpath, const char* lpath, 734 const char* name, uint64_t expected_size) { 735 if (!sc.SendRequest(ID_RECV, rpath)) return false; 736 737 adb_unlink(lpath); 738 int lfd = adb_creat(lpath, 0644); 739 if (lfd < 0) { 740 sc.Error("cannot create '%s': %s", lpath, strerror(errno)); 741 return false; 742 } 743 744 uint64_t bytes_copied = 0; 745 while (true) { 746 syncmsg msg; 747 if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) { 748 adb_close(lfd); 749 adb_unlink(lpath); 750 return false; 751 } 752 753 if (msg.data.id == ID_DONE) break; 754 755 if (msg.data.id != ID_DATA) { 756 adb_close(lfd); 757 adb_unlink(lpath); 758 sc.ReportCopyFailure(rpath, lpath, msg); 759 return false; 760 } 761 762 if (msg.data.size > sc.max) { 763 sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max); 764 adb_close(lfd); 765 adb_unlink(lpath); 766 return false; 767 } 768 769 char buffer[SYNC_DATA_MAX]; 770 if (!ReadFdExactly(sc.fd, buffer, msg.data.size)) { 771 adb_close(lfd); 772 adb_unlink(lpath); 773 return false; 774 } 775 776 if (!WriteFdExactly(lfd, buffer, msg.data.size)) { 777 sc.Error("cannot write '%s': %s", lpath, strerror(errno)); 778 adb_close(lfd); 779 adb_unlink(lpath); 780 return false; 781 } 782 783 bytes_copied += msg.data.size; 784 785 sc.RecordBytesTransferred(msg.data.size); 786 sc.ReportProgress(name != nullptr ? name : rpath, bytes_copied, expected_size); 787 } 788 789 sc.RecordFilesTransferred(1); 790 adb_close(lfd); 791 return true; 792} 793 794bool do_sync_ls(const char* path) { 795 SyncConnection sc; 796 if (!sc.IsValid()) return false; 797 798 return sync_ls(sc, path, [](unsigned mode, unsigned size, unsigned time, 799 const char* name) { 800 printf("%08x %08x %08x %s\n", mode, size, time, name); 801 }); 802} 803 804static bool IsDotOrDotDot(const char* name) { 805 return name[0] == '.' && (name[1] == '\0' || (name[1] == '.' && name[2] == '\0')); 806} 807 808static bool local_build_list(SyncConnection& sc, std::vector<copyinfo>* file_list, 809 const std::string& lpath, 810 const std::string& rpath) { 811 std::vector<copyinfo> dirlist; 812 std::unique_ptr<DIR, int (*)(DIR*)> dir(opendir(lpath.c_str()), closedir); 813 if (!dir) { 814 sc.Error("cannot open '%s': %s", lpath.c_str(), strerror(errno)); 815 return false; 816 } 817 818 bool empty_dir = true; 819 dirent* de; 820 while ((de = readdir(dir.get()))) { 821 if (IsDotOrDotDot(de->d_name)) { 822 continue; 823 } 824 825 empty_dir = false; 826 std::string stat_path = lpath + de->d_name; 827 828 struct stat st; 829 if (lstat(stat_path.c_str(), &st) == -1) { 830 sc.Error("cannot lstat '%s': %s", stat_path.c_str(), 831 strerror(errno)); 832 continue; 833 } 834 835 copyinfo ci(lpath, rpath, de->d_name, st.st_mode); 836 if (S_ISDIR(st.st_mode)) { 837 dirlist.push_back(ci); 838 } else { 839 if (!should_push_file(st.st_mode)) { 840 sc.Warning("skipping special file '%s' (mode = 0o%o)", lpath.c_str(), st.st_mode); 841 ci.skip = true; 842 } 843 ci.time = st.st_mtime; 844 ci.size = st.st_size; 845 file_list->push_back(ci); 846 } 847 } 848 849 // Close this directory and recurse. 850 dir.reset(); 851 852 // Add the current directory to the list if it was empty, to ensure that 853 // it gets created. 854 if (empty_dir) { 855 // TODO(b/25566053): Make pushing empty directories work. 856 // TODO(b/25457350): We don't preserve permissions on directories. 857 sc.Warning("skipping empty directory '%s'", lpath.c_str()); 858 copyinfo ci(android::base::Dirname(lpath), android::base::Dirname(rpath), 859 android::base::Basename(lpath), S_IFDIR); 860 ci.skip = true; 861 file_list->push_back(ci); 862 return true; 863 } 864 865 for (const copyinfo& ci : dirlist) { 866 local_build_list(sc, file_list, ci.lpath, ci.rpath); 867 } 868 869 return true; 870} 871 872static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, 873 std::string rpath, bool check_timestamps, 874 bool list_only) { 875 sc.NewTransfer(); 876 877 // Make sure that both directory paths end in a slash. 878 // Both paths are known to be nonempty, so we don't need to check. 879 ensure_trailing_separators(lpath, rpath); 880 881 // Recursively build the list of files to copy. 882 std::vector<copyinfo> file_list; 883 int skipped = 0; 884 if (!local_build_list(sc, &file_list, lpath, rpath)) { 885 return false; 886 } 887 888 if (check_timestamps) { 889 for (const copyinfo& ci : file_list) { 890 if (!sc.SendLstat(ci.rpath.c_str())) { 891 sc.Error("failed to send lstat"); 892 return false; 893 } 894 } 895 for (copyinfo& ci : file_list) { 896 struct stat st; 897 if (sc.FinishStat(&st)) { 898 if (st.st_size == static_cast<off_t>(ci.size)) { 899 // For links, we cannot update the atime/mtime. 900 if ((S_ISREG(ci.mode & st.st_mode) && st.st_mtime == ci.time) || 901 (S_ISLNK(ci.mode & st.st_mode) && st.st_mtime >= ci.time)) { 902 ci.skip = true; 903 } 904 } 905 } 906 } 907 } 908 909 sc.ComputeExpectedTotalBytes(file_list); 910 911 for (const copyinfo& ci : file_list) { 912 if (!ci.skip) { 913 if (list_only) { 914 sc.Println("would push: %s -> %s", ci.lpath.c_str(), ci.rpath.c_str()); 915 } else { 916 if (!sync_send(sc, ci.lpath.c_str(), ci.rpath.c_str(), ci.time, ci.mode, false)) { 917 return false; 918 } 919 } 920 } else { 921 skipped++; 922 } 923 } 924 925 sc.RecordFilesSkipped(skipped); 926 sc.ReportTransferRate(lpath, TransferDirection::push); 927 return true; 928} 929 930bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync) { 931 SyncConnection sc; 932 if (!sc.IsValid()) return false; 933 934 bool success = true; 935 bool dst_exists; 936 bool dst_isdir; 937 938 struct stat st; 939 if (sync_stat_fallback(sc, dst, &st)) { 940 dst_exists = true; 941 dst_isdir = S_ISDIR(st.st_mode); 942 } else { 943 if (errno == ENOENT || errno == ENOPROTOOPT) { 944 dst_exists = false; 945 dst_isdir = false; 946 } else { 947 sc.Error("stat failed when trying to push to %s: %s", dst, strerror(errno)); 948 return false; 949 } 950 } 951 952 if (!dst_isdir) { 953 if (srcs.size() > 1) { 954 sc.Error("target '%s' is not a directory", dst); 955 return false; 956 } else { 957 size_t dst_len = strlen(dst); 958 959 // A path that ends with a slash doesn't have to be a directory if 960 // it doesn't exist yet. 961 if (dst[dst_len - 1] == '/' && dst_exists) { 962 sc.Error("failed to access '%s': Not a directory", dst); 963 return false; 964 } 965 } 966 } 967 968 for (const char* src_path : srcs) { 969 const char* dst_path = dst; 970 struct stat st; 971 if (stat(src_path, &st) == -1) { 972 sc.Error("cannot stat '%s': %s", src_path, strerror(errno)); 973 success = false; 974 continue; 975 } 976 977 if (S_ISDIR(st.st_mode)) { 978 std::string dst_dir = dst; 979 980 // If the destination path existed originally, the source directory 981 // should be copied as a child of the destination. 982 if (dst_exists) { 983 if (!dst_isdir) { 984 sc.Error("target '%s' is not a directory", dst); 985 return false; 986 } 987 // dst is a POSIX path, so we don't want to use the sysdeps 988 // helpers here. 989 if (dst_dir.back() != '/') { 990 dst_dir.push_back('/'); 991 } 992 dst_dir.append(android::base::Basename(src_path)); 993 } 994 995 success &= copy_local_dir_remote(sc, src_path, dst_dir.c_str(), sync, false); 996 continue; 997 } else if (!should_push_file(st.st_mode)) { 998 sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, st.st_mode); 999 continue; 1000 } 1001 1002 std::string path_holder; 1003 if (dst_isdir) { 1004 // If we're copying a local file to a remote directory, 1005 // we really want to copy to remote_dir + "/" + local_filename. 1006 path_holder = dst_path; 1007 if (path_holder.back() != '/') { 1008 path_holder.push_back('/'); 1009 } 1010 path_holder += android::base::Basename(src_path); 1011 dst_path = path_holder.c_str(); 1012 } 1013 1014 sc.NewTransfer(); 1015 sc.SetExpectedTotalBytes(st.st_size); 1016 success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync); 1017 sc.ReportTransferRate(src_path, TransferDirection::push); 1018 } 1019 1020 sc.ReportOverallTransferRate(TransferDirection::push); 1021 return success; 1022} 1023 1024static bool remote_build_list(SyncConnection& sc, std::vector<copyinfo>* file_list, 1025 const std::string& rpath, const std::string& lpath) { 1026 std::vector<copyinfo> dirlist; 1027 std::vector<copyinfo> linklist; 1028 1029 // Add an entry for the current directory to ensure it gets created before pulling its contents. 1030 copyinfo ci(android::base::Dirname(lpath), android::base::Dirname(rpath), 1031 android::base::Basename(lpath), S_IFDIR); 1032 file_list->push_back(ci); 1033 1034 // Put the files/dirs in rpath on the lists. 1035 auto callback = [&](unsigned mode, unsigned size, unsigned time, const char* name) { 1036 if (IsDotOrDotDot(name)) { 1037 return; 1038 } 1039 1040 copyinfo ci(lpath, rpath, name, mode); 1041 if (S_ISDIR(mode)) { 1042 dirlist.push_back(ci); 1043 } else if (S_ISLNK(mode)) { 1044 linklist.push_back(ci); 1045 } else { 1046 if (!should_pull_file(ci.mode)) { 1047 sc.Warning("skipping special file '%s' (mode = 0o%o)", ci.rpath.c_str(), ci.mode); 1048 ci.skip = true; 1049 } 1050 ci.time = time; 1051 ci.size = size; 1052 file_list->push_back(ci); 1053 } 1054 }; 1055 1056 if (!sync_ls(sc, rpath.c_str(), callback)) { 1057 return false; 1058 } 1059 1060 // Check each symlink we found to see whether it's a file or directory. 1061 for (copyinfo& link_ci : linklist) { 1062 struct stat st; 1063 if (!sync_stat_fallback(sc, link_ci.rpath.c_str(), &st)) { 1064 sc.Warning("stat failed for path %s: %s", link_ci.rpath.c_str(), strerror(errno)); 1065 continue; 1066 } 1067 1068 if (S_ISDIR(st.st_mode)) { 1069 dirlist.emplace_back(std::move(link_ci)); 1070 } else { 1071 file_list->emplace_back(std::move(link_ci)); 1072 } 1073 } 1074 1075 // Recurse into each directory we found. 1076 while (!dirlist.empty()) { 1077 copyinfo current = dirlist.back(); 1078 dirlist.pop_back(); 1079 if (!remote_build_list(sc, file_list, current.rpath, current.lpath)) { 1080 return false; 1081 } 1082 } 1083 1084 return true; 1085} 1086 1087static int set_time_and_mode(const std::string& lpath, time_t time, 1088 unsigned int mode) { 1089 struct utimbuf times = { time, time }; 1090 int r1 = utime(lpath.c_str(), ×); 1091 1092 /* use umask for permissions */ 1093 mode_t mask = umask(0000); 1094 umask(mask); 1095 int r2 = chmod(lpath.c_str(), mode & ~mask); 1096 1097 return r1 ? r1 : r2; 1098} 1099 1100static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, 1101 std::string lpath, bool copy_attrs) { 1102 sc.NewTransfer(); 1103 1104 // Make sure that both directory paths end in a slash. 1105 // Both paths are known to be nonempty, so we don't need to check. 1106 ensure_trailing_separators(lpath, rpath); 1107 1108 // Recursively build the list of files to copy. 1109 sc.Printf("pull: building file list..."); 1110 std::vector<copyinfo> file_list; 1111 if (!remote_build_list(sc, &file_list, rpath.c_str(), lpath.c_str())) { 1112 return false; 1113 } 1114 1115 sc.ComputeExpectedTotalBytes(file_list); 1116 1117 int skipped = 0; 1118 for (const copyinfo &ci : file_list) { 1119 if (!ci.skip) { 1120 if (S_ISDIR(ci.mode)) { 1121 // Entry is for an empty directory, create it and continue. 1122 // TODO(b/25457350): We don't preserve permissions on directories. 1123 if (!mkdirs(ci.lpath)) { 1124 sc.Error("failed to create directory '%s': %s", 1125 ci.lpath.c_str(), strerror(errno)); 1126 return false; 1127 } 1128 continue; 1129 } 1130 1131 if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size)) { 1132 return false; 1133 } 1134 1135 if (copy_attrs && set_time_and_mode(ci.lpath, ci.time, ci.mode)) { 1136 return false; 1137 } 1138 } else { 1139 skipped++; 1140 } 1141 } 1142 1143 sc.RecordFilesSkipped(skipped); 1144 sc.ReportTransferRate(rpath, TransferDirection::pull); 1145 return true; 1146} 1147 1148bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, 1149 bool copy_attrs, const char* name) { 1150 SyncConnection sc; 1151 if (!sc.IsValid()) return false; 1152 1153 bool success = true; 1154 struct stat st; 1155 bool dst_exists = true; 1156 1157 if (stat(dst, &st) == -1) { 1158 dst_exists = false; 1159 1160 // If we're only pulling one path, the destination path might point to 1161 // a path that doesn't exist yet. 1162 if (srcs.size() == 1 && errno == ENOENT) { 1163 // However, its parent must exist. 1164 struct stat parent_st; 1165 if (stat(android::base::Dirname(dst).c_str(), &parent_st) == -1) { 1166 sc.Error("cannot create file/directory '%s': %s", dst, strerror(errno)); 1167 return false; 1168 } 1169 } else { 1170 sc.Error("failed to access '%s': %s", dst, strerror(errno)); 1171 return false; 1172 } 1173 } 1174 1175 bool dst_isdir = dst_exists && S_ISDIR(st.st_mode); 1176 if (!dst_isdir) { 1177 if (srcs.size() > 1) { 1178 sc.Error("target '%s' is not a directory", dst); 1179 return false; 1180 } else { 1181 size_t dst_len = strlen(dst); 1182 1183 // A path that ends with a slash doesn't have to be a directory if 1184 // it doesn't exist yet. 1185 if (adb_is_separator(dst[dst_len - 1]) && dst_exists) { 1186 sc.Error("failed to access '%s': Not a directory", dst); 1187 return false; 1188 } 1189 } 1190 } 1191 1192 for (const char* src_path : srcs) { 1193 const char* dst_path = dst; 1194 struct stat src_st; 1195 if (!sync_stat_fallback(sc, src_path, &src_st)) { 1196 if (errno == ENOPROTOOPT) { 1197 sc.Error("remote object '%s' does not exist", src_path); 1198 } else { 1199 sc.Error("failed to stat remote object '%s': %s", src_path, strerror(errno)); 1200 } 1201 1202 success = false; 1203 continue; 1204 } 1205 1206 bool src_isdir = S_ISDIR(src_st.st_mode); 1207 if (src_isdir) { 1208 std::string dst_dir = dst; 1209 1210 // If the destination path existed originally, the source directory 1211 // should be copied as a child of the destination. 1212 if (dst_exists) { 1213 if (!dst_isdir) { 1214 sc.Error("target '%s' is not a directory", dst); 1215 return false; 1216 } 1217 if (!adb_is_separator(dst_dir.back())) { 1218 dst_dir.push_back(OS_PATH_SEPARATOR); 1219 } 1220 dst_dir.append(android::base::Basename(src_path)); 1221 } 1222 1223 success &= copy_remote_dir_local(sc, src_path, dst_dir.c_str(), copy_attrs); 1224 continue; 1225 } else if (!should_pull_file(src_st.st_mode)) { 1226 sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, src_st.st_mode); 1227 continue; 1228 } 1229 1230 std::string path_holder; 1231 if (dst_isdir) { 1232 // If we're copying a remote file to a local directory, we 1233 // really want to copy to local_dir + OS_PATH_SEPARATOR + 1234 // basename(remote). 1235 path_holder = android::base::StringPrintf("%s%c%s", dst_path, OS_PATH_SEPARATOR, 1236 android::base::Basename(src_path).c_str()); 1237 dst_path = path_holder.c_str(); 1238 } 1239 1240 sc.NewTransfer(); 1241 sc.SetExpectedTotalBytes(src_st.st_size); 1242 if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size)) { 1243 success = false; 1244 continue; 1245 } 1246 1247 if (copy_attrs && set_time_and_mode(dst_path, src_st.st_mtime, src_st.st_mode) != 0) { 1248 success = false; 1249 continue; 1250 } 1251 sc.ReportTransferRate(src_path, TransferDirection::pull); 1252 } 1253 1254 sc.ReportOverallTransferRate(TransferDirection::pull); 1255 return success; 1256} 1257 1258bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only) { 1259 SyncConnection sc; 1260 if (!sc.IsValid()) return false; 1261 1262 bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only); 1263 if (!list_only) { 1264 sc.ReportOverallTransferRate(TransferDirection::push); 1265 } 1266 return success; 1267} 1268