1 // 2 // Copyright (C) 2009 The Android Open Source Project 3 // 4 // Licensed under the Apache License, Version 2.0 (the "License"); 5 // you may not use this file except in compliance with the License. 6 // You may obtain a copy of the License at 7 // 8 // http://www.apache.org/licenses/LICENSE-2.0 9 // 10 // Unless required by applicable law or agreed to in writing, software 11 // distributed under the License is distributed on an "AS IS" BASIS, 12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 // See the License for the specific language governing permissions and 14 // limitations under the License. 15 // 16 17 #include "update_engine/libcurl_http_fetcher.h" 18 19 #include <netinet/in.h> 20 #include <resolv.h> 21 #include <sys/types.h> 22 #include <unistd.h> 23 24 #include <algorithm> 25 #include <string> 26 27 #include <base/bind.h> 28 #include <base/format_macros.h> 29 #include <base/location.h> 30 #include <base/logging.h> 31 #include <base/strings/string_split.h> 32 #include <base/strings/string_util.h> 33 #include <base/strings/stringprintf.h> 34 #include <base/threading/thread_task_runner_handle.h> 35 36 #ifdef __ANDROID__ 37 #include <cutils/qtaguid.h> 38 #include <private/android_filesystem_config.h> 39 #endif // __ANDROID__ 40 41 #include "update_engine/certificate_checker.h" 42 #include "update_engine/common/hardware_interface.h" 43 #include "update_engine/common/platform_constants.h" 44 45 using base::TimeDelta; 46 using brillo::MessageLoop; 47 using std::max; 48 using std::string; 49 50 // This is a concrete implementation of HttpFetcher that uses libcurl to do the 51 // http work. 52 53 namespace chromeos_update_engine { 54 55 namespace { 56 57 const int kNoNetworkRetrySeconds = 10; 58 59 // libcurl's CURLOPT_SOCKOPTFUNCTION callback function. Called after the socket 60 // is created but before it is connected. This callback tags the created socket 61 // so the network usage can be tracked in Android. 62 int LibcurlSockoptCallback(void* /* clientp */, 63 curl_socket_t curlfd, 64 curlsocktype /* purpose */) { 65 #ifdef __ANDROID__ 66 // Socket tag used by all network sockets. See qtaguid kernel module for 67 // stats. 68 const int kUpdateEngineSocketTag = 0x55417243; // "CrAU" in little-endian. 69 qtaguid_tagSocket(curlfd, kUpdateEngineSocketTag, AID_OTA_UPDATE); 70 #endif // __ANDROID__ 71 return CURL_SOCKOPT_OK; 72 } 73 74 } // namespace 75 76 // static 77 int LibcurlHttpFetcher::LibcurlCloseSocketCallback(void* clientp, 78 curl_socket_t item) { 79 #ifdef __ANDROID__ 80 qtaguid_untagSocket(item); 81 #endif // __ANDROID__ 82 83 LibcurlHttpFetcher* fetcher = static_cast<LibcurlHttpFetcher*>(clientp); 84 // Stop watching the socket before closing it. 85 for (size_t t = 0; t < base::size(fetcher->fd_controller_maps_); ++t) { 86 fetcher->fd_controller_maps_[t].erase(item); 87 } 88 89 // Documentation for this callback says to return 0 on success or 1 on error. 90 if (!IGNORE_EINTR(close(item))) 91 return 0; 92 return 1; 93 } 94 95 LibcurlHttpFetcher::LibcurlHttpFetcher(ProxyResolver* proxy_resolver, 96 HardwareInterface* hardware) 97 : HttpFetcher(proxy_resolver), hardware_(hardware) { 98 // Dev users want a longer timeout (180 seconds) because they may 99 // be waiting on the dev server to build an image. 100 if (!hardware_->IsOfficialBuild()) 101 low_speed_time_seconds_ = kDownloadDevModeLowSpeedTimeSeconds; 102 if (hardware_->IsOOBEEnabled() && !hardware_->IsOOBEComplete(nullptr)) 103 max_retry_count_ = kDownloadMaxRetryCountOobeNotComplete; 104 } 105 106 LibcurlHttpFetcher::~LibcurlHttpFetcher() { 107 LOG_IF(ERROR, transfer_in_progress_) 108 << "Destroying the fetcher while a transfer is in progress."; 109 CancelProxyResolution(); 110 CleanUp(); 111 } 112 113 bool LibcurlHttpFetcher::GetProxyType(const string& proxy, 114 curl_proxytype* out_type) { 115 if (base::StartsWith( 116 proxy, "socks5://", base::CompareCase::INSENSITIVE_ASCII) || 117 base::StartsWith( 118 proxy, "socks://", base::CompareCase::INSENSITIVE_ASCII)) { 119 *out_type = CURLPROXY_SOCKS5_HOSTNAME; 120 return true; 121 } 122 if (base::StartsWith( 123 proxy, "socks4://", base::CompareCase::INSENSITIVE_ASCII)) { 124 *out_type = CURLPROXY_SOCKS4A; 125 return true; 126 } 127 if (base::StartsWith( 128 proxy, "http://", base::CompareCase::INSENSITIVE_ASCII) || 129 base::StartsWith( 130 proxy, "https://", base::CompareCase::INSENSITIVE_ASCII)) { 131 *out_type = CURLPROXY_HTTP; 132 return true; 133 } 134 if (base::StartsWith(proxy, kNoProxy, base::CompareCase::INSENSITIVE_ASCII)) { 135 // known failure case. don't log. 136 return false; 137 } 138 LOG(INFO) << "Unknown proxy type: " << proxy; 139 return false; 140 } 141 142 void LibcurlHttpFetcher::ResumeTransfer(const string& url) { 143 LOG(INFO) << "Starting/Resuming transfer"; 144 CHECK(!transfer_in_progress_); 145 url_ = url; 146 curl_multi_handle_ = curl_multi_init(); 147 CHECK(curl_multi_handle_); 148 149 curl_handle_ = curl_easy_init(); 150 CHECK(curl_handle_); 151 ignore_failure_ = false; 152 153 // Tag and untag the socket for network usage stats. 154 curl_easy_setopt( 155 curl_handle_, CURLOPT_SOCKOPTFUNCTION, LibcurlSockoptCallback); 156 curl_easy_setopt( 157 curl_handle_, CURLOPT_CLOSESOCKETFUNCTION, LibcurlCloseSocketCallback); 158 curl_easy_setopt(curl_handle_, CURLOPT_CLOSESOCKETDATA, this); 159 160 CHECK(HasProxy()); 161 bool is_direct = (GetCurrentProxy() == kNoProxy); 162 LOG(INFO) << "Using proxy: " << (is_direct ? "no" : "yes"); 163 if (is_direct) { 164 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROXY, ""), CURLE_OK); 165 } else { 166 CHECK_EQ(curl_easy_setopt( 167 curl_handle_, CURLOPT_PROXY, GetCurrentProxy().c_str()), 168 CURLE_OK); 169 // Curl seems to require us to set the protocol 170 curl_proxytype type; 171 if (GetProxyType(GetCurrentProxy(), &type)) { 172 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROXYTYPE, type), 173 CURLE_OK); 174 } 175 } 176 177 if (post_data_set_) { 178 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_POST, 1), CURLE_OK); 179 CHECK_EQ( 180 curl_easy_setopt(curl_handle_, CURLOPT_POSTFIELDS, post_data_.data()), 181 CURLE_OK); 182 CHECK_EQ(curl_easy_setopt( 183 curl_handle_, CURLOPT_POSTFIELDSIZE, post_data_.size()), 184 CURLE_OK); 185 } 186 187 // Setup extra HTTP headers. 188 if (curl_http_headers_) { 189 curl_slist_free_all(curl_http_headers_); 190 curl_http_headers_ = nullptr; 191 } 192 for (const auto& header : extra_headers_) { 193 // curl_slist_append() copies the string. 194 curl_http_headers_ = 195 curl_slist_append(curl_http_headers_, header.second.c_str()); 196 } 197 if (post_data_set_) { 198 // Set the Content-Type HTTP header, if one was specifically set. 199 if (post_content_type_ != kHttpContentTypeUnspecified) { 200 const string content_type_attr = base::StringPrintf( 201 "Content-Type: %s", GetHttpContentTypeString(post_content_type_)); 202 curl_http_headers_ = 203 curl_slist_append(curl_http_headers_, content_type_attr.c_str()); 204 } else { 205 LOG(WARNING) << "no content type set, using libcurl default"; 206 } 207 } 208 CHECK_EQ( 209 curl_easy_setopt(curl_handle_, CURLOPT_HTTPHEADER, curl_http_headers_), 210 CURLE_OK); 211 212 if (bytes_downloaded_ > 0 || download_length_) { 213 // Resume from where we left off. 214 resume_offset_ = bytes_downloaded_; 215 CHECK_GE(resume_offset_, 0); 216 217 // Compute end offset, if one is specified. As per HTTP specification, this 218 // is an inclusive boundary. Make sure it doesn't overflow. 219 size_t end_offset = 0; 220 if (download_length_) { 221 end_offset = static_cast<size_t>(resume_offset_) + download_length_ - 1; 222 CHECK_LE((size_t)resume_offset_, end_offset); 223 } 224 225 // Create a string representation of the desired range. 226 string range_str = base::StringPrintf( 227 "%" PRIu64 "-", static_cast<uint64_t>(resume_offset_)); 228 if (end_offset) 229 range_str += std::to_string(end_offset); 230 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_RANGE, range_str.c_str()), 231 CURLE_OK); 232 } 233 234 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_WRITEDATA, this), CURLE_OK); 235 CHECK_EQ( 236 curl_easy_setopt(curl_handle_, CURLOPT_WRITEFUNCTION, StaticLibcurlWrite), 237 CURLE_OK); 238 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_URL, url_.c_str()), CURLE_OK); 239 240 // If the connection drops under |low_speed_limit_bps_| (10 241 // bytes/sec by default) for |low_speed_time_seconds_| (90 seconds, 242 // 180 on non-official builds), reconnect. 243 CHECK_EQ(curl_easy_setopt( 244 curl_handle_, CURLOPT_LOW_SPEED_LIMIT, low_speed_limit_bps_), 245 CURLE_OK); 246 CHECK_EQ(curl_easy_setopt( 247 curl_handle_, CURLOPT_LOW_SPEED_TIME, low_speed_time_seconds_), 248 CURLE_OK); 249 CHECK_EQ(curl_easy_setopt( 250 curl_handle_, CURLOPT_CONNECTTIMEOUT, connect_timeout_seconds_), 251 CURLE_OK); 252 253 // By default, libcurl doesn't follow redirections. Allow up to 254 // |kDownloadMaxRedirects| redirections. 255 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_FOLLOWLOCATION, 1), CURLE_OK); 256 CHECK_EQ( 257 curl_easy_setopt(curl_handle_, CURLOPT_MAXREDIRS, kDownloadMaxRedirects), 258 CURLE_OK); 259 260 // Lock down the appropriate curl options for HTTP or HTTPS depending on 261 // the url. 262 if (hardware_->IsOfficialBuild()) { 263 if (base::StartsWith( 264 url_, "http://", base::CompareCase::INSENSITIVE_ASCII)) { 265 SetCurlOptionsForHttp(); 266 } else if (base::StartsWith( 267 url_, "https://", base::CompareCase::INSENSITIVE_ASCII)) { 268 SetCurlOptionsForHttps(); 269 #ifdef __ANDROID__ 270 } else if (base::StartsWith( 271 url_, "file://", base::CompareCase::INSENSITIVE_ASCII)) { 272 SetCurlOptionsForFile(); 273 #endif // __ANDROID__ 274 } else { 275 LOG(ERROR) << "Received invalid URI: " << url_; 276 // Lock down to no protocol supported for the transfer. 277 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROTOCOLS, 0), CURLE_OK); 278 } 279 } else { 280 LOG(INFO) << "Not setting http(s) curl options because we are " 281 << "running a dev/test image"; 282 } 283 284 CHECK_EQ(curl_multi_add_handle(curl_multi_handle_, curl_handle_), CURLM_OK); 285 transfer_in_progress_ = true; 286 } 287 288 // Lock down only the protocol in case of HTTP. 289 void LibcurlHttpFetcher::SetCurlOptionsForHttp() { 290 LOG(INFO) << "Setting up curl options for HTTP"; 291 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROTOCOLS, CURLPROTO_HTTP), 292 CURLE_OK); 293 CHECK_EQ( 294 curl_easy_setopt(curl_handle_, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP), 295 CURLE_OK); 296 } 297 298 // Security lock-down in official builds: makes sure that peer certificate 299 // verification is enabled, restricts the set of trusted certificates, 300 // restricts protocols to HTTPS, restricts ciphers to HIGH. 301 void LibcurlHttpFetcher::SetCurlOptionsForHttps() { 302 LOG(INFO) << "Setting up curl options for HTTPS"; 303 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_VERIFYPEER, 1), CURLE_OK); 304 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_VERIFYHOST, 2), CURLE_OK); 305 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_CAINFO, nullptr), CURLE_OK); 306 CHECK_EQ(curl_easy_setopt( 307 curl_handle_, CURLOPT_CAPATH, constants::kCACertificatesPath), 308 CURLE_OK); 309 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS), 310 CURLE_OK); 311 CHECK_EQ( 312 curl_easy_setopt(curl_handle_, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS), 313 CURLE_OK); 314 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_CIPHER_LIST, "HIGH:!ADH"), 315 CURLE_OK); 316 if (server_to_check_ != ServerToCheck::kNone) { 317 CHECK_EQ( 318 curl_easy_setopt(curl_handle_, CURLOPT_SSL_CTX_DATA, &server_to_check_), 319 CURLE_OK); 320 CHECK_EQ(curl_easy_setopt(curl_handle_, 321 CURLOPT_SSL_CTX_FUNCTION, 322 CertificateChecker::ProcessSSLContext), 323 CURLE_OK); 324 } 325 } 326 327 // Lock down only the protocol in case of a local file. 328 void LibcurlHttpFetcher::SetCurlOptionsForFile() { 329 LOG(INFO) << "Setting up curl options for FILE"; 330 CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROTOCOLS, CURLPROTO_FILE), 331 CURLE_OK); 332 CHECK_EQ( 333 curl_easy_setopt(curl_handle_, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_FILE), 334 CURLE_OK); 335 } 336 337 // Begins the transfer, which must not have already been started. 338 void LibcurlHttpFetcher::BeginTransfer(const string& url) { 339 CHECK(!transfer_in_progress_); 340 url_ = url; 341 auto closure = 342 base::Bind(&LibcurlHttpFetcher::ProxiesResolved, base::Unretained(this)); 343 ResolveProxiesForUrl(url_, closure); 344 } 345 346 void LibcurlHttpFetcher::ProxiesResolved() { 347 transfer_size_ = -1; 348 resume_offset_ = 0; 349 retry_count_ = 0; 350 no_network_retry_count_ = 0; 351 http_response_code_ = 0; 352 terminate_requested_ = false; 353 sent_byte_ = false; 354 355 // If we are paused, we delay these two operations until Unpause is called. 356 if (transfer_paused_) { 357 restart_transfer_on_unpause_ = true; 358 return; 359 } 360 ResumeTransfer(url_); 361 CurlPerformOnce(); 362 } 363 364 void LibcurlHttpFetcher::ForceTransferTermination() { 365 CancelProxyResolution(); 366 CleanUp(); 367 if (delegate_) { 368 // Note that after the callback returns this object may be destroyed. 369 delegate_->TransferTerminated(this); 370 } 371 } 372 373 void LibcurlHttpFetcher::TerminateTransfer() { 374 if (in_write_callback_) { 375 terminate_requested_ = true; 376 } else { 377 ForceTransferTermination(); 378 } 379 } 380 381 void LibcurlHttpFetcher::SetHeader(const string& header_name, 382 const string& header_value) { 383 string header_line = header_name + ": " + header_value; 384 // Avoid the space if no data on the right side of the semicolon. 385 if (header_value.empty()) 386 header_line = header_name + ":"; 387 TEST_AND_RETURN(header_line.find('\n') == string::npos); 388 TEST_AND_RETURN(header_name.find(':') == string::npos); 389 extra_headers_[base::ToLowerASCII(header_name)] = header_line; 390 } 391 392 // Inputs: header_name, header_value 393 // Example: 394 // extra_headers_ = { {"foo":"foo: 123"}, {"bar":"bar:"} } 395 // string tmp = "gibberish"; 396 // Case 1: 397 // GetHeader("foo", &tmp) -> tmp = "123", return true. 398 // Case 2: 399 // GetHeader("bar", &tmp) -> tmp = "", return true. 400 // Case 3: 401 // GetHeader("moo", &tmp) -> tmp = "", return false. 402 bool LibcurlHttpFetcher::GetHeader(const string& header_name, 403 string* header_value) const { 404 // Initially clear |header_value| to handle both success and failures without 405 // leaving |header_value| in a unclear state. 406 header_value->clear(); 407 auto header_key = base::ToLowerASCII(header_name); 408 auto header_line_itr = extra_headers_.find(header_key); 409 // If the |header_name| was never set, indicate so by returning false. 410 if (header_line_itr == extra_headers_.end()) 411 return false; 412 // From |SetHeader()| the check for |header_name| to not include ":" is 413 // verified, so finding the first index of ":" is a safe operation. 414 auto header_line = header_line_itr->second; 415 *header_value = header_line.substr(header_line.find(':') + 1); 416 // The following is neccessary to remove the leading ' ' before the header 417 // value that was place only if |header_value| passed to |SetHeader()| was 418 // a non-empty string. 419 header_value->erase(0, 1); 420 return true; 421 } 422 423 void LibcurlHttpFetcher::CurlPerformOnce() { 424 CHECK(transfer_in_progress_); 425 int running_handles = 0; 426 CURLMcode retcode = CURLM_CALL_MULTI_PERFORM; 427 428 // libcurl may request that we immediately call curl_multi_perform after it 429 // returns, so we do. libcurl promises that curl_multi_perform will not block. 430 while (CURLM_CALL_MULTI_PERFORM == retcode) { 431 retcode = curl_multi_perform(curl_multi_handle_, &running_handles); 432 if (terminate_requested_) { 433 ForceTransferTermination(); 434 return; 435 } 436 } 437 438 // When retcode is not |CURLM_OK| at this point, libcurl has an internal error 439 // that it is less likely to recover from (libcurl bug, out-of-memory, etc.). 440 // In case of an update check, we send UMA metrics and log the error. 441 if (is_update_check_ && 442 (retcode == CURLM_OUT_OF_MEMORY || retcode == CURLM_INTERNAL_ERROR)) { 443 auxiliary_error_code_ = ErrorCode::kInternalLibCurlError; 444 LOG(ERROR) << "curl_multi_perform is in an unrecoverable error condition: " 445 << retcode; 446 } else if (retcode != CURLM_OK) { 447 LOG(ERROR) << "curl_multi_perform returns error: " << retcode; 448 } 449 450 // If the transfer completes while paused, we should ignore the failure once 451 // the fetcher is unpaused. 452 if (running_handles == 0 && transfer_paused_ && !ignore_failure_) { 453 LOG(INFO) << "Connection closed while paused, ignoring failure."; 454 ignore_failure_ = true; 455 } 456 457 if (running_handles != 0 || transfer_paused_) { 458 // There's either more work to do or we are paused, so we just keep the 459 // file descriptors to watch up to date and exit, until we are done with the 460 // work and we are not paused. 461 // 462 // When there's no |base::SingleThreadTaskRunner| on current thread, it's 463 // not possible to watch file descriptors. Just poll it later. This usually 464 // happens if |brillo::FakeMessageLoop| is used. 465 if (!base::ThreadTaskRunnerHandle::IsSet()) { 466 MessageLoop::current()->PostDelayedTask( 467 FROM_HERE, 468 base::Bind(&LibcurlHttpFetcher::CurlPerformOnce, 469 base::Unretained(this)), 470 TimeDelta::FromSeconds(1)); 471 return; 472 } 473 SetupMessageLoopSources(); 474 return; 475 } 476 477 // At this point, the transfer was completed in some way (error, connection 478 // closed or download finished). 479 480 GetHttpResponseCode(); 481 if (http_response_code_) { 482 LOG(INFO) << "HTTP response code: " << http_response_code_; 483 no_network_retry_count_ = 0; 484 unresolved_host_state_machine_.UpdateState(false); 485 } else { 486 LOG(ERROR) << "Unable to get http response code."; 487 CURLcode curl_code = GetCurlCode(); 488 LOG(ERROR) << "Return code for the transfer: " << curl_code; 489 if (curl_code == CURLE_COULDNT_RESOLVE_HOST) { 490 LOG(ERROR) << "libcurl can not resolve host."; 491 unresolved_host_state_machine_.UpdateState(true); 492 auxiliary_error_code_ = ErrorCode::kUnresolvedHostError; 493 } 494 } 495 496 // we're done! 497 CleanUp(); 498 499 if (unresolved_host_state_machine_.GetState() == 500 UnresolvedHostStateMachine::State::kRetry) { 501 // Based on 502 // https://curl.haxx.se/docs/todo.html#updated_DNS_server_while_running, 503 // update_engine process should call res_init() and unconditionally retry. 504 res_init(); 505 no_network_max_retries_++; 506 LOG(INFO) << "Will retry after reloading resolv.conf because last attempt " 507 "failed to resolve host."; 508 } else if (unresolved_host_state_machine_.GetState() == 509 UnresolvedHostStateMachine::State::kRetriedSuccess) { 510 auxiliary_error_code_ = ErrorCode::kUnresolvedHostRecovered; 511 } 512 513 // TODO(petkov): This temporary code tries to deal with the case where the 514 // update engine performs an update check while the network is not ready 515 // (e.g., right after resume). Longer term, we should check if the network 516 // is online/offline and return an appropriate error code. 517 if (!sent_byte_ && http_response_code_ == 0 && 518 no_network_retry_count_ < no_network_max_retries_) { 519 no_network_retry_count_++; 520 retry_task_id_ = MessageLoop::current()->PostDelayedTask( 521 FROM_HERE, 522 base::Bind(&LibcurlHttpFetcher::RetryTimeoutCallback, 523 base::Unretained(this)), 524 TimeDelta::FromSeconds(kNoNetworkRetrySeconds)); 525 LOG(INFO) << "No HTTP response, retry " << no_network_retry_count_; 526 } else if ((!sent_byte_ && !IsHttpResponseSuccess()) || 527 IsHttpResponseError()) { 528 // The transfer completed w/ error and we didn't get any bytes. 529 // If we have another proxy to try, try that. 530 // 531 // TODO(garnold) in fact there are two separate cases here: one case is an 532 // other-than-success return code (including no return code) and no 533 // received bytes, which is necessary due to the way callbacks are 534 // currently processing error conditions; the second is an explicit HTTP 535 // error code, where some data may have been received (as in the case of a 536 // semi-successful multi-chunk fetch). This is a confusing behavior and 537 // should be unified into a complete, coherent interface. 538 LOG(INFO) << "Transfer resulted in an error (" << http_response_code_ 539 << "), " << bytes_downloaded_ << " bytes downloaded"; 540 541 PopProxy(); // Delete the proxy we just gave up on. 542 543 if (HasProxy()) { 544 // We have another proxy. Retry immediately. 545 LOG(INFO) << "Retrying with next proxy setting"; 546 retry_task_id_ = MessageLoop::current()->PostTask( 547 FROM_HERE, 548 base::Bind(&LibcurlHttpFetcher::RetryTimeoutCallback, 549 base::Unretained(this))); 550 } else { 551 // Out of proxies. Give up. 552 LOG(INFO) << "No further proxies, indicating transfer complete"; 553 if (delegate_) 554 delegate_->TransferComplete(this, false); // signal fail 555 return; 556 } 557 } else if ((transfer_size_ >= 0) && (bytes_downloaded_ < transfer_size_)) { 558 if (!ignore_failure_) 559 retry_count_++; 560 LOG(INFO) << "Transfer interrupted after downloading " << bytes_downloaded_ 561 << " of " << transfer_size_ << " bytes. " 562 << transfer_size_ - bytes_downloaded_ << " bytes remaining " 563 << "after " << retry_count_ << " attempt(s)"; 564 565 if (retry_count_ > max_retry_count_) { 566 LOG(INFO) << "Reached max attempts (" << retry_count_ << ")"; 567 if (delegate_) 568 delegate_->TransferComplete(this, false); // signal fail 569 return; 570 } 571 // Need to restart transfer 572 LOG(INFO) << "Restarting transfer to download the remaining bytes"; 573 retry_task_id_ = MessageLoop::current()->PostDelayedTask( 574 FROM_HERE, 575 base::Bind(&LibcurlHttpFetcher::RetryTimeoutCallback, 576 base::Unretained(this)), 577 TimeDelta::FromSeconds(retry_seconds_)); 578 } else { 579 LOG(INFO) << "Transfer completed (" << http_response_code_ << "), " 580 << bytes_downloaded_ << " bytes downloaded"; 581 if (delegate_) { 582 bool success = IsHttpResponseSuccess(); 583 delegate_->TransferComplete(this, success); 584 } 585 return; 586 } 587 // If we reach this point is because TransferComplete() was not called in any 588 // of the previous branches. The delegate is allowed to destroy the object 589 // once TransferComplete is called so this would be illegal. 590 ignore_failure_ = false; 591 } 592 593 size_t LibcurlHttpFetcher::LibcurlWrite(void* ptr, size_t size, size_t nmemb) { 594 // Update HTTP response first. 595 GetHttpResponseCode(); 596 const size_t payload_size = size * nmemb; 597 598 // Do nothing if no payload or HTTP response is an error. 599 if (payload_size == 0 || !IsHttpResponseSuccess()) { 600 LOG(INFO) << "HTTP response unsuccessful (" << http_response_code_ 601 << ") or no payload (" << payload_size << "), nothing to do"; 602 return 0; 603 } 604 605 sent_byte_ = true; 606 { 607 double transfer_size_double; 608 CHECK_EQ(curl_easy_getinfo(curl_handle_, 609 CURLINFO_CONTENT_LENGTH_DOWNLOAD, 610 &transfer_size_double), 611 CURLE_OK); 612 off_t new_transfer_size = static_cast<off_t>(transfer_size_double); 613 if (new_transfer_size > 0) { 614 transfer_size_ = resume_offset_ + new_transfer_size; 615 } 616 } 617 bytes_downloaded_ += payload_size; 618 if (delegate_) { 619 in_write_callback_ = true; 620 auto should_terminate = !delegate_->ReceivedBytes(this, ptr, payload_size); 621 in_write_callback_ = false; 622 if (should_terminate) { 623 LOG(INFO) << "Requesting libcurl to terminate transfer."; 624 // Returning an amount that differs from the received size signals an 625 // error condition to libcurl, which will cause the transfer to be 626 // aborted. 627 return 0; 628 } 629 } 630 return payload_size; 631 } 632 633 void LibcurlHttpFetcher::Pause() { 634 if (transfer_paused_) { 635 LOG(ERROR) << "Fetcher already paused."; 636 return; 637 } 638 transfer_paused_ = true; 639 if (!transfer_in_progress_) { 640 // If pause before we started a connection, we don't need to notify curl 641 // about that, we will simply not start the connection later. 642 return; 643 } 644 CHECK(curl_handle_); 645 CHECK_EQ(curl_easy_pause(curl_handle_, CURLPAUSE_ALL), CURLE_OK); 646 } 647 648 void LibcurlHttpFetcher::Unpause() { 649 if (!transfer_paused_) { 650 LOG(ERROR) << "Resume attempted when fetcher not paused."; 651 return; 652 } 653 transfer_paused_ = false; 654 if (restart_transfer_on_unpause_) { 655 restart_transfer_on_unpause_ = false; 656 ResumeTransfer(url_); 657 CurlPerformOnce(); 658 return; 659 } 660 if (!transfer_in_progress_) { 661 // If resumed before starting the connection, there's no need to notify 662 // anybody. We will simply start the connection once it is time. 663 return; 664 } 665 CHECK(curl_handle_); 666 CHECK_EQ(curl_easy_pause(curl_handle_, CURLPAUSE_CONT), CURLE_OK); 667 // Since the transfer is in progress, we need to dispatch a CurlPerformOnce() 668 // now to let the connection continue, otherwise it would be called by the 669 // TimeoutCallback but with a delay. 670 CurlPerformOnce(); 671 } 672 673 // This method sets up callbacks with the MessageLoop. 674 void LibcurlHttpFetcher::SetupMessageLoopSources() { 675 fd_set fd_read; 676 fd_set fd_write; 677 fd_set fd_exc; 678 679 FD_ZERO(&fd_read); 680 FD_ZERO(&fd_write); 681 FD_ZERO(&fd_exc); 682 683 int fd_max = 0; 684 685 // Ask libcurl for the set of file descriptors we should track on its 686 // behalf. 687 CHECK_EQ(curl_multi_fdset( 688 curl_multi_handle_, &fd_read, &fd_write, &fd_exc, &fd_max), 689 CURLM_OK); 690 691 // We should iterate through all file descriptors up to libcurl's fd_max or 692 // the highest one we're tracking, whichever is larger. 693 for (size_t t = 0; t < base::size(fd_controller_maps_); ++t) { 694 if (!fd_controller_maps_[t].empty()) 695 fd_max = max(fd_max, fd_controller_maps_[t].rbegin()->first); 696 } 697 698 // For each fd, if we're not tracking it, track it. If we are tracking it, but 699 // libcurl doesn't care about it anymore, stop tracking it. After this loop, 700 // there should be exactly as many tasks scheduled in 701 // fd_controller_maps_[0|1] as there are read/write fds that we're tracking. 702 for (int fd = 0; fd <= fd_max; ++fd) { 703 // Note that fd_exc is unused in the current version of libcurl so is_exc 704 // should always be false. 705 bool is_exc = FD_ISSET(fd, &fd_exc) != 0; 706 bool must_track[2] = { 707 is_exc || (FD_ISSET(fd, &fd_read) != 0), // track 0 -- read 708 is_exc || (FD_ISSET(fd, &fd_write) != 0) // track 1 -- write 709 }; 710 711 for (size_t t = 0; t < base::size(fd_controller_maps_); ++t) { 712 bool tracked = 713 fd_controller_maps_[t].find(fd) != fd_controller_maps_[t].end(); 714 715 if (!must_track[t]) { 716 // If we have an outstanding io_channel, remove it. 717 fd_controller_maps_[t].erase(fd); 718 continue; 719 } 720 721 // If we are already tracking this fd, continue -- nothing to do. 722 if (tracked) 723 continue; 724 725 // Track a new fd. 726 switch (t) { 727 case 0: // Read 728 fd_controller_maps_[t][fd] = 729 base::FileDescriptorWatcher::WatchReadable( 730 fd, 731 base::BindRepeating(&LibcurlHttpFetcher::CurlPerformOnce, 732 base::Unretained(this))); 733 break; 734 case 1: // Write 735 fd_controller_maps_[t][fd] = 736 base::FileDescriptorWatcher::WatchWritable( 737 fd, 738 base::BindRepeating(&LibcurlHttpFetcher::CurlPerformOnce, 739 base::Unretained(this))); 740 } 741 static int io_counter = 0; 742 io_counter++; 743 if (io_counter % 50 == 0) { 744 LOG(INFO) << "io_counter = " << io_counter; 745 } 746 } 747 } 748 749 // Set up a timeout callback for libcurl. 750 if (timeout_id_ == MessageLoop::kTaskIdNull) { 751 VLOG(1) << "Setting up timeout source: " << idle_seconds_ << " seconds."; 752 timeout_id_ = MessageLoop::current()->PostDelayedTask( 753 FROM_HERE, 754 base::Bind(&LibcurlHttpFetcher::TimeoutCallback, 755 base::Unretained(this)), 756 TimeDelta::FromSeconds(idle_seconds_)); 757 } 758 } 759 760 void LibcurlHttpFetcher::RetryTimeoutCallback() { 761 retry_task_id_ = MessageLoop::kTaskIdNull; 762 if (transfer_paused_) { 763 restart_transfer_on_unpause_ = true; 764 return; 765 } 766 ResumeTransfer(url_); 767 CurlPerformOnce(); 768 } 769 770 void LibcurlHttpFetcher::TimeoutCallback() { 771 // We always re-schedule the callback, even if we don't want to be called 772 // anymore. We will remove the event source separately if we don't want to 773 // be called back. 774 timeout_id_ = MessageLoop::current()->PostDelayedTask( 775 FROM_HERE, 776 base::Bind(&LibcurlHttpFetcher::TimeoutCallback, base::Unretained(this)), 777 TimeDelta::FromSeconds(idle_seconds_)); 778 779 // CurlPerformOnce() may call CleanUp(), so we need to schedule our callback 780 // first, since it could be canceled by this call. 781 if (transfer_in_progress_) 782 CurlPerformOnce(); 783 } 784 785 void LibcurlHttpFetcher::CleanUp() { 786 MessageLoop::current()->CancelTask(retry_task_id_); 787 retry_task_id_ = MessageLoop::kTaskIdNull; 788 789 MessageLoop::current()->CancelTask(timeout_id_); 790 timeout_id_ = MessageLoop::kTaskIdNull; 791 792 for (size_t t = 0; t < base::size(fd_controller_maps_); ++t) { 793 fd_controller_maps_[t].clear(); 794 } 795 796 if (curl_http_headers_) { 797 curl_slist_free_all(curl_http_headers_); 798 curl_http_headers_ = nullptr; 799 } 800 if (curl_handle_) { 801 if (curl_multi_handle_) { 802 CHECK_EQ(curl_multi_remove_handle(curl_multi_handle_, curl_handle_), 803 CURLM_OK); 804 } 805 curl_easy_cleanup(curl_handle_); 806 curl_handle_ = nullptr; 807 } 808 if (curl_multi_handle_) { 809 CHECK_EQ(curl_multi_cleanup(curl_multi_handle_), CURLM_OK); 810 curl_multi_handle_ = nullptr; 811 } 812 transfer_in_progress_ = false; 813 transfer_paused_ = false; 814 restart_transfer_on_unpause_ = false; 815 } 816 817 void LibcurlHttpFetcher::GetHttpResponseCode() { 818 long http_response_code = 0; // NOLINT(runtime/int) - curl needs long. 819 if (base::StartsWith(url_, "file://", base::CompareCase::INSENSITIVE_ASCII)) { 820 // Fake out a valid response code for file:// URLs. 821 http_response_code_ = 299; 822 } else if (curl_easy_getinfo(curl_handle_, 823 CURLINFO_RESPONSE_CODE, 824 &http_response_code) == CURLE_OK) { 825 http_response_code_ = static_cast<int>(http_response_code); 826 } else { 827 LOG(ERROR) << "Unable to get http response code from curl_easy_getinfo"; 828 } 829 } 830 831 CURLcode LibcurlHttpFetcher::GetCurlCode() { 832 CURLcode curl_code = CURLE_OK; 833 while (true) { 834 // Repeated calls to |curl_multi_info_read| will return a new struct each 835 // time, until a NULL is returned as a signal that there is no more to get 836 // at this point. 837 int msgs_in_queue; 838 CURLMsg* curl_msg = 839 curl_multi_info_read(curl_multi_handle_, &msgs_in_queue); 840 if (curl_msg == nullptr) 841 break; 842 // When |curl_msg| is |CURLMSG_DONE|, a transfer of an easy handle is done, 843 // and then data contains the return code for this transfer. 844 if (curl_msg->msg == CURLMSG_DONE) { 845 // Make sure |curl_multi_handle_| has one and only one easy handle 846 // |curl_handle_|. 847 CHECK_EQ(curl_handle_, curl_msg->easy_handle); 848 // Transfer return code reference: 849 // https://curl.haxx.se/libcurl/c/libcurl-errors.html 850 curl_code = curl_msg->data.result; 851 } 852 } 853 854 // Gets connection error if exists. 855 long connect_error = 0; // NOLINT(runtime/int) - curl needs long. 856 CURLcode res = 857 curl_easy_getinfo(curl_handle_, CURLINFO_OS_ERRNO, &connect_error); 858 if (res == CURLE_OK && connect_error) { 859 LOG(ERROR) << "Connect error code from the OS: " << connect_error; 860 } 861 862 return curl_code; 863 } 864 865 void UnresolvedHostStateMachine::UpdateState(bool failed_to_resolve_host) { 866 switch (state_) { 867 case State::kInit: 868 if (failed_to_resolve_host) { 869 state_ = State::kRetry; 870 } 871 break; 872 case State::kRetry: 873 if (failed_to_resolve_host) { 874 state_ = State::kNotRetry; 875 } else { 876 state_ = State::kRetriedSuccess; 877 } 878 break; 879 case State::kNotRetry: 880 break; 881 case State::kRetriedSuccess: 882 break; 883 default: 884 NOTREACHED(); 885 break; 886 } 887 } 888 889 } // namespace chromeos_update_engine 890