1 /* 2 * Copyright (C) 2023 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.adservices.service.common.httpclient; 18 19 import static android.adservices.exceptions.RetryableAdServicesNetworkException.DEFAULT_RETRY_AFTER_VALUE; 20 21 import static com.android.adservices.service.stats.AdsRelevanceStatusUtils.ENCODING_FETCH_STATUS_NETWORK_FAILURE; 22 import static com.android.adservices.service.stats.AdsRelevanceStatusUtils.ENCODING_FETCH_STATUS_SUCCESS; 23 import static com.android.adservices.service.stats.AdsRelevanceStatusUtils.ENCODING_FETCH_STATUS_TIMEOUT; 24 import static com.android.adservices.service.stats.AdsRelevanceStatusUtils.ENCODING_FETCH_STATUS_UNSET; 25 26 import android.adservices.exceptions.AdServicesNetworkException; 27 import android.adservices.exceptions.RetryableAdServicesNetworkException; 28 import android.annotation.NonNull; 29 import android.annotation.Nullable; 30 import android.annotation.SuppressLint; 31 import android.net.Uri; 32 33 import com.android.adservices.LogUtil; 34 import com.android.adservices.service.common.ValidatorUtil; 35 import com.android.adservices.service.common.WebAddresses; 36 import com.android.adservices.service.common.cache.CacheProviderFactory; 37 import com.android.adservices.service.common.cache.DBCacheEntry; 38 import com.android.adservices.service.common.cache.HttpCache; 39 import com.android.adservices.service.devapi.DevContext; 40 import com.android.adservices.service.profiling.Tracing; 41 import com.android.adservices.service.stats.FetchProcessLogger; 42 import com.android.adservices.service.stats.FetchProcessLoggerNoLoggingImpl; 43 import com.android.internal.annotations.VisibleForTesting; 44 45 import com.google.common.base.Charsets; 46 import com.google.common.base.Preconditions; 47 import com.google.common.collect.ImmutableMap; 48 import com.google.common.collect.ImmutableSet; 49 import com.google.common.io.BaseEncoding; 50 import com.google.common.io.ByteStreams; 51 import com.google.common.util.concurrent.ClosingFuture; 52 import com.google.common.util.concurrent.ListenableFuture; 53 import com.google.common.util.concurrent.ListeningExecutorService; 54 import com.google.common.util.concurrent.MoreExecutors; 55 56 import java.io.BufferedInputStream; 57 import java.io.BufferedOutputStream; 58 import java.io.ByteArrayOutputStream; 59 import java.io.Closeable; 60 import java.io.IOException; 61 import java.io.InputStream; 62 import java.io.OutputStream; 63 import java.io.OutputStreamWriter; 64 import java.net.HttpURLConnection; 65 import java.net.MalformedURLException; 66 import java.net.SocketTimeoutException; 67 import java.net.URL; 68 import java.net.URLConnection; 69 import java.nio.charset.StandardCharsets; 70 import java.security.SecureRandom; 71 import java.security.cert.X509Certificate; 72 import java.time.Duration; 73 import java.util.ArrayList; 74 import java.util.HashMap; 75 import java.util.List; 76 import java.util.Map; 77 import java.util.Objects; 78 import java.util.concurrent.ExecutorService; 79 80 import javax.net.ssl.HttpsURLConnection; 81 import javax.net.ssl.SSLContext; 82 import javax.net.ssl.SSLSocketFactory; 83 import javax.net.ssl.TrustManager; 84 import javax.net.ssl.X509TrustManager; 85 86 /** 87 * This is an HTTPS client to be used by the PP API services. The primary uses of this client 88 * include fetching payloads from ad tech-provided URIs and reporting on generated reporting URLs 89 * through GET or POST calls. 90 */ 91 public class AdServicesHttpsClient { 92 93 public static final long DEFAULT_MAX_BYTES = 1048576; 94 private static final int DEFAULT_TIMEOUT_MS = 5000; 95 // Setting default max content size to 1024 * 1024 which is ~ 1MB 96 private static final String CONTENT_SIZE_ERROR = "Content size exceeds limit!"; 97 private static final String RETRY_AFTER_HEADER_FIELD = "Retry-After"; 98 private final int mConnectTimeoutMs; 99 private final int mReadTimeoutMs; 100 private final long mMaxBytes; 101 private final ListeningExecutorService mExecutorService; 102 private final UriConverter mUriConverter; 103 private final HttpCache mCache; 104 105 /** 106 * Create an HTTPS client with the input {@link ExecutorService} and initial connect and read 107 * timeouts (in milliseconds). Using this constructor does not provide any caching. 108 * 109 * @param executorService an {@link ExecutorService} that allows connection and fetching to be 110 * executed outside the main calling thread 111 * @param connectTimeoutMs the timeout, in milliseconds, for opening an initial link with to a 112 * target resource using this client. If set to 0, this timeout is interpreted as infinite 113 * (see {@link URLConnection#setConnectTimeout(int)}). 114 * @param readTimeoutMs the timeout, in milliseconds, for reading a response from a target 115 * address using this client. If set to 0, this timeout is interpreted as infinite (see 116 * {@link URLConnection#setReadTimeout(int)}). 117 * @param maxBytes The maximum size of an HTTPS response in bytes. 118 */ AdServicesHttpsClient( ExecutorService executorService, int connectTimeoutMs, int readTimeoutMs, long maxBytes)119 public AdServicesHttpsClient( 120 ExecutorService executorService, 121 int connectTimeoutMs, 122 int readTimeoutMs, 123 long maxBytes) { 124 this( 125 executorService, 126 connectTimeoutMs, 127 readTimeoutMs, 128 maxBytes, 129 new UriConverter(), 130 CacheProviderFactory.createNoOpCache()); 131 } 132 133 /** 134 * Create an HTTPS client with the input {@link ExecutorService} and default initial connect and 135 * read timeouts. This will also contain the default size of an HTTPS response, 1 MB. 136 * 137 * @param executorService an {@link ExecutorService} that allows connection and fetching to be 138 * executed outside the main calling thread 139 * @param cache A {@link HttpCache} that caches requests and response based on the use case 140 */ AdServicesHttpsClient( @onNull ExecutorService executorService, @NonNull HttpCache cache)141 public AdServicesHttpsClient( 142 @NonNull ExecutorService executorService, @NonNull HttpCache cache) { 143 this( 144 executorService, 145 DEFAULT_TIMEOUT_MS, 146 DEFAULT_TIMEOUT_MS, 147 DEFAULT_MAX_BYTES, 148 new UriConverter(), 149 cache); 150 } 151 152 @VisibleForTesting AdServicesHttpsClient( ExecutorService executorService, int connectTimeoutMs, int readTimeoutMs, long maxBytes, UriConverter uriConverter, @NonNull HttpCache cache)153 AdServicesHttpsClient( 154 ExecutorService executorService, 155 int connectTimeoutMs, 156 int readTimeoutMs, 157 long maxBytes, 158 UriConverter uriConverter, 159 @NonNull HttpCache cache) { 160 mConnectTimeoutMs = connectTimeoutMs; 161 mReadTimeoutMs = readTimeoutMs; 162 mExecutorService = MoreExecutors.listeningDecorator(executorService); 163 mMaxBytes = maxBytes; 164 mUriConverter = uriConverter; 165 mCache = cache; 166 } 167 168 /** Opens the Url Connection */ 169 @NonNull openUrl(@onNull URL url)170 private URLConnection openUrl(@NonNull URL url) throws IOException { 171 Objects.requireNonNull(url); 172 return url.openConnection(); 173 } 174 175 @NonNull setupConnection(@onNull URL url, @NonNull DevContext devContext)176 private HttpsURLConnection setupConnection(@NonNull URL url, @NonNull DevContext devContext) 177 throws IOException { 178 Objects.requireNonNull(url); 179 Objects.requireNonNull(devContext); 180 // We validated that the URL is https in toUrl 181 HttpsURLConnection urlConnection = (HttpsURLConnection) openUrl(url); 182 urlConnection.setConnectTimeout(mConnectTimeoutMs); 183 urlConnection.setReadTimeout(mReadTimeoutMs); 184 // Setting true explicitly to follow redirects 185 Uri uri = Uri.parse(url.toString()); 186 if (WebAddresses.isLocalhost(uri) && devContext.getDevOptionsEnabled()) { 187 LogUtil.v("Using unsafe HTTPS for url %s", url.toString()); 188 urlConnection.setSSLSocketFactory(getUnsafeSslSocketFactory()); 189 } else if (WebAddresses.isLocalhost(uri)) { 190 LogUtil.v( 191 String.format( 192 "Using normal HTTPS without unsafe SSL socket factory for a localhost" 193 + " address, DevOptionsEnabled: %s, CallingPackageName: %s", 194 devContext.getDevOptionsEnabled(), 195 devContext.getCallingAppPackageName())); 196 } 197 urlConnection.setInstanceFollowRedirects(true); 198 return urlConnection; 199 } 200 201 @NonNull setupPostConnectionWithPlainText( @onNull URL url, @NonNull DevContext devContext)202 private HttpsURLConnection setupPostConnectionWithPlainText( 203 @NonNull URL url, @NonNull DevContext devContext) throws IOException { 204 Objects.requireNonNull(url); 205 Objects.requireNonNull(devContext); 206 HttpsURLConnection urlConnection = setupConnection(url, devContext); 207 urlConnection.setRequestMethod("POST"); 208 urlConnection.setRequestProperty("Content-Type", "text/plain"); 209 urlConnection.setDoOutput(true); 210 return urlConnection; 211 } 212 213 @SuppressLint({"TrustAllX509TrustManager", "CustomX509TrustManager"}) getUnsafeSslSocketFactory()214 private static SSLSocketFactory getUnsafeSslSocketFactory() { 215 try { 216 TrustManager[] bypassTrustManagers = 217 new TrustManager[] { 218 new X509TrustManager() { 219 public X509Certificate[] getAcceptedIssuers() { 220 return new X509Certificate[0]; 221 } 222 223 public void checkClientTrusted( 224 X509Certificate[] chain, String authType) {} 225 226 public void checkServerTrusted( 227 X509Certificate[] chain, String authType) {} 228 } 229 }; 230 SSLContext sslContext = SSLContext.getInstance("TLS"); 231 sslContext.init(null, bypassTrustManagers, new SecureRandom()); 232 return sslContext.getSocketFactory(); 233 } catch (Exception e) { 234 LogUtil.e(e, "getUnsafeSslSocketFactory caught exception"); 235 return null; 236 } 237 } 238 239 /** 240 * Performs a GET request on the given URI in order to fetch a payload. 241 * 242 * @param uri a {@link Uri} pointing to a target server, converted to a URL for fetching 243 * @return a string containing the fetched payload 244 */ 245 @NonNull fetchPayload( @onNull Uri uri, @NonNull DevContext devContext)246 public ListenableFuture<AdServicesHttpClientResponse> fetchPayload( 247 @NonNull Uri uri, @NonNull DevContext devContext) { 248 LogUtil.v("Fetching payload from uri: " + uri); 249 return fetchPayload( 250 AdServicesHttpClientRequest.builder() 251 .setUri(uri) 252 .setDevContext(devContext) 253 .build()); 254 } 255 256 /** 257 * Performs a GET request on the given URI in order to fetch a payload. with FetchProcessLogger 258 * logging. 259 * 260 * @param uri a {@link Uri} pointing to a target server, converted to a URL for fetching 261 * @return a string containing the fetched payload 262 */ 263 @NonNull fetchPayloadWithLogging( @onNull Uri uri, @NonNull DevContext devContext, @NonNull FetchProcessLogger fetchProcessLogger)264 public ListenableFuture<AdServicesHttpClientResponse> fetchPayloadWithLogging( 265 @NonNull Uri uri, 266 @NonNull DevContext devContext, 267 @NonNull FetchProcessLogger fetchProcessLogger) { 268 return fetchPayloadWithLogging( 269 AdServicesHttpClientRequest.builder().setUri(uri).setDevContext(devContext).build(), 270 fetchProcessLogger); 271 } 272 273 /** 274 * Performs a GET request on the given URI in order to fetch a payload. 275 * 276 * @param uri a {@link Uri} pointing to a target server, converted to a URL for fetching 277 * @param headers keys of the headers we want in the response 278 * @return a string containing the fetched payload 279 */ 280 @NonNull fetchPayload( @onNull Uri uri, @NonNull ImmutableSet<String> headers, @NonNull DevContext devContext)281 public ListenableFuture<AdServicesHttpClientResponse> fetchPayload( 282 @NonNull Uri uri, 283 @NonNull ImmutableSet<String> headers, 284 @NonNull DevContext devContext) { 285 LogUtil.v("Fetching payload from uri: " + uri + " with headers: " + headers.toString()); 286 return fetchPayload( 287 AdServicesHttpClientRequest.builder() 288 .setUri(uri) 289 .setResponseHeaderKeys(headers) 290 .setDevContext(devContext) 291 .build()); 292 } 293 294 /** 295 * Performs a GET request on the given URI in order to fetch a payload. 296 * 297 * @param request of type {@link AdServicesHttpClientRequest} 298 * @return a string containing the fetched payload 299 */ 300 @NonNull fetchPayload( @onNull AdServicesHttpClientRequest request)301 public ListenableFuture<AdServicesHttpClientResponse> fetchPayload( 302 @NonNull AdServicesHttpClientRequest request) { 303 return fetchPayloadWithLogging(request, new FetchProcessLoggerNoLoggingImpl()); 304 } 305 306 /** 307 * Performs a GET request on the given URI in order to fetch a payload with EncodingFetchStats 308 * logging. 309 * 310 * @param request of type {@link AdServicesHttpClientRequest} 311 * @param fetchProcessLogger of {@link FetchProcessLogger} 312 * @return a string containing the fetched payload 313 */ 314 @NonNull fetchPayloadWithLogging( @onNull AdServicesHttpClientRequest request, @NonNull FetchProcessLogger fetchProcessLogger)315 public ListenableFuture<AdServicesHttpClientResponse> fetchPayloadWithLogging( 316 @NonNull AdServicesHttpClientRequest request, 317 @NonNull FetchProcessLogger fetchProcessLogger) { 318 Objects.requireNonNull(request.getUri()); 319 320 StringBuilder logBuilder = 321 new StringBuilder( 322 "Fetching payload for request: uri: " 323 + request.getUri() 324 + " use cache: " 325 + request.getUseCache() 326 + " dev context: " 327 + request.getDevContext().getDevOptionsEnabled()); 328 if (request.getRequestProperties() != null) { 329 logBuilder 330 .append(" request properties: ") 331 .append(request.getRequestProperties().toString()); 332 } 333 if (request.getResponseHeaderKeys() != null) { 334 logBuilder 335 .append(" response headers keys to be read in response: ") 336 .append(request.getResponseHeaderKeys().toString()); 337 } 338 339 LogUtil.v(logBuilder.toString()); 340 return ClosingFuture.from( 341 mExecutorService.submit(() -> mUriConverter.toUrl(request.getUri()))) 342 .transformAsync( 343 (closer, url) -> 344 ClosingFuture.from( 345 mExecutorService.submit( 346 () -> 347 doFetchPayload( 348 url, 349 closer, 350 request, 351 fetchProcessLogger))), 352 mExecutorService) 353 .finishToFuture(); 354 } 355 doFetchPayload( @onNull URL url, @NonNull ClosingFuture.DeferredCloser closer, AdServicesHttpClientRequest request, FetchProcessLogger fetchProcessLogger)356 private AdServicesHttpClientResponse doFetchPayload( 357 @NonNull URL url, 358 @NonNull ClosingFuture.DeferredCloser closer, 359 AdServicesHttpClientRequest request, 360 FetchProcessLogger fetchProcessLogger) 361 throws IOException, AdServicesNetworkException { 362 int jsFetchStatusCode = ENCODING_FETCH_STATUS_UNSET; 363 int traceCookie = Tracing.beginAsyncSection(Tracing.FETCH_PAYLOAD); 364 LogUtil.v("Downloading payload from: \"%s\"", url.toString()); 365 if (request.getUseCache()) { 366 AdServicesHttpClientResponse cachedResponse = getResultsFromCache(url); 367 if (cachedResponse != null) { 368 jsFetchStatusCode = ENCODING_FETCH_STATUS_SUCCESS; 369 fetchProcessLogger.logEncodingJsFetchStats(jsFetchStatusCode); 370 return cachedResponse; 371 } 372 LogUtil.v("Cache miss for url: %s", url.toString()); 373 } 374 int httpTraceCookie = Tracing.beginAsyncSection(Tracing.HTTP_REQUEST); 375 HttpsURLConnection urlConnection; 376 try { 377 urlConnection = setupConnection(url, request.getDevContext()); 378 } catch (IOException e) { 379 LogUtil.d(e, "Failed to open URL"); 380 jsFetchStatusCode = ENCODING_FETCH_STATUS_NETWORK_FAILURE; 381 fetchProcessLogger.logEncodingJsFetchStats(jsFetchStatusCode); 382 throw new IllegalArgumentException("Failed to open URL!"); 383 } 384 385 InputStream inputStream = null; 386 try { 387 // TODO(b/237342352): Both connect and read timeouts are kludged in this method and if 388 // necessary need to be separated 389 for (Map.Entry<String, String> entry : request.getRequestProperties().entrySet()) { 390 urlConnection.setRequestProperty(entry.getKey(), entry.getValue()); 391 } 392 closer.eventuallyClose(new CloseableConnectionWrapper(urlConnection), mExecutorService); 393 Map<String, List<String>> requestPropertiesMap = urlConnection.getRequestProperties(); 394 fetchProcessLogger.startDownloadScriptTimestamp(); 395 fetchProcessLogger.startNetworkCallTimestamp(); 396 int responseCode = urlConnection.getResponseCode(); 397 fetchProcessLogger.logServerAuctionKeyFetchCalledStatsFromNetwork(responseCode); 398 fetchProcessLogger.endDownloadScriptTimestamp(responseCode); 399 LogUtil.v("Received %s response status code.", responseCode); 400 if (isSuccessfulResponse(responseCode)) { 401 inputStream = new BufferedInputStream(urlConnection.getInputStream()); 402 String responseBody = 403 fromInputStream(inputStream, urlConnection.getContentLengthLong()); 404 Map<String, List<String>> responseHeadersMap = 405 pickRequiredHeaderFields( 406 urlConnection.getHeaderFields(), request.getResponseHeaderKeys()); 407 if (request.getUseCache()) { 408 LogUtil.v("Putting data in cache for url: %s", url); 409 mCache.put(url, responseBody, requestPropertiesMap, responseHeadersMap); 410 } 411 AdServicesHttpClientResponse response = 412 AdServicesHttpClientResponse.builder() 413 .setResponseBody(responseBody) 414 .setResponseHeaders( 415 ImmutableMap.<String, List<String>>builder() 416 .putAll(responseHeadersMap.entrySet()) 417 .build()) 418 .build(); 419 jsFetchStatusCode = ENCODING_FETCH_STATUS_SUCCESS; 420 return response; 421 } else { 422 jsFetchStatusCode = ENCODING_FETCH_STATUS_NETWORK_FAILURE; 423 throwError(urlConnection, responseCode); 424 return null; 425 } 426 } catch (SocketTimeoutException e) { 427 jsFetchStatusCode = ENCODING_FETCH_STATUS_TIMEOUT; 428 throw new IOException("Connection timed out while reading response!", e); 429 } finally { 430 fetchProcessLogger.logEncodingJsFetchStats(jsFetchStatusCode); 431 maybeDisconnect(urlConnection); 432 maybeClose(inputStream); 433 Tracing.endAsyncSection(Tracing.HTTP_REQUEST, httpTraceCookie); 434 Tracing.endAsyncSection(Tracing.FETCH_PAYLOAD, traceCookie); 435 } 436 } 437 pickRequiredHeaderFields( Map<String, List<String>> allHeaderFields, ImmutableSet<String> requiredHeaderKeys)438 private Map<String, List<String>> pickRequiredHeaderFields( 439 Map<String, List<String>> allHeaderFields, ImmutableSet<String> requiredHeaderKeys) { 440 HashMap<String, List<String>> result = new HashMap<>(); 441 for (String headerKey : requiredHeaderKeys) { 442 if (allHeaderFields.containsKey(headerKey)) { 443 List<String> headerValues = new ArrayList<>(allHeaderFields.get(headerKey)); 444 LogUtil.v( 445 String.format( 446 "Found header: %s in response headers with value as %s", 447 headerKey, String.join(", ", headerValues))); 448 result.put(headerKey, headerValues); 449 } 450 } 451 LogUtil.v("requiredHeaderFields: " + result); 452 return result; 453 } 454 getResultsFromCache(URL url)455 private AdServicesHttpClientResponse getResultsFromCache(URL url) { 456 DBCacheEntry cachedEntry = mCache.get(url); 457 if (cachedEntry != null) { 458 LogUtil.v("Cache hit for url: %s", url.toString()); 459 return AdServicesHttpClientResponse.builder() 460 .setResponseBody(cachedEntry.getResponseBody()) 461 .setResponseHeaders( 462 ImmutableMap.<String, List<String>>builder() 463 .putAll(cachedEntry.getResponseHeaders().entrySet()) 464 .build()) 465 .build(); 466 } 467 return null; 468 } 469 470 /** 471 * Performs a GET request on a Uri without reading the response. 472 * 473 * @param uri The URI to perform the GET request on. 474 */ getAndReadNothing( @onNull Uri uri, @NonNull DevContext devContext)475 public ListenableFuture<Void> getAndReadNothing( 476 @NonNull Uri uri, @NonNull DevContext devContext) { 477 Objects.requireNonNull(uri); 478 479 return ClosingFuture.from(mExecutorService.submit(() -> mUriConverter.toUrl(uri))) 480 .transformAsync( 481 (closer, url) -> 482 ClosingFuture.from( 483 mExecutorService.submit( 484 () -> 485 doGetAndReadNothing( 486 url, closer, devContext))), 487 mExecutorService) 488 .finishToFuture(); 489 } 490 doGetAndReadNothing( @onNull URL url, @NonNull ClosingFuture.DeferredCloser closer, @NonNull DevContext devContext)491 private Void doGetAndReadNothing( 492 @NonNull URL url, 493 @NonNull ClosingFuture.DeferredCloser closer, 494 @NonNull DevContext devContext) 495 throws IOException, AdServicesNetworkException { 496 LogUtil.v( 497 "doGetAndReadNothing to: \"%s\", dev context: %s", 498 url.toString(), devContext.getDevOptionsEnabled()); 499 HttpsURLConnection urlConnection; 500 501 try { 502 urlConnection = setupConnection(url, devContext); 503 } catch (IOException e) { 504 LogUtil.d(e, "Failed to open URL"); 505 throw new IllegalArgumentException("Failed to open URL!"); 506 } 507 508 try { 509 // TODO(b/237342352): Both connect and read timeouts are kludged in this method and if 510 // necessary need to be separated 511 closer.eventuallyClose(new CloseableConnectionWrapper(urlConnection), mExecutorService); 512 int responseCode = urlConnection.getResponseCode(); 513 if (isSuccessfulResponse(responseCode)) { 514 LogUtil.d("GET request succeeded for URL: " + url); 515 } else { 516 LogUtil.d("GET request failed for URL: " + url); 517 throwError(urlConnection, responseCode); 518 } 519 return null; 520 } catch (SocketTimeoutException e) { 521 throw new IOException("Connection timed out while reading response!", e); 522 } finally { 523 maybeDisconnect(urlConnection); 524 } 525 } 526 527 /** 528 * Performs a POST request on a Uri and attaches {@code String} to the request 529 * 530 * @param uri to do the POST request on 531 * @param requestBody Attached to the POST request. 532 */ postPlainText( @onNull Uri uri, @NonNull String requestBody, @NonNull DevContext devContext)533 public ListenableFuture<Void> postPlainText( 534 @NonNull Uri uri, @NonNull String requestBody, @NonNull DevContext devContext) { 535 Objects.requireNonNull(uri); 536 Objects.requireNonNull(requestBody); 537 538 return ClosingFuture.from(mExecutorService.submit(() -> mUriConverter.toUrl(uri))) 539 .transformAsync( 540 (closer, url) -> 541 ClosingFuture.from( 542 mExecutorService.submit( 543 () -> 544 doPostPlainText( 545 url, 546 requestBody, 547 closer, 548 devContext))), 549 mExecutorService) 550 .finishToFuture(); 551 } 552 doPostPlainText( URL url, String data, ClosingFuture.DeferredCloser closer, DevContext devContext)553 private Void doPostPlainText( 554 URL url, String data, ClosingFuture.DeferredCloser closer, DevContext devContext) 555 throws IOException, AdServicesNetworkException { 556 LogUtil.v("Reporting to: \"%s\"", url.toString()); 557 HttpsURLConnection urlConnection; 558 559 try { 560 urlConnection = setupPostConnectionWithPlainText(url, devContext); 561 562 } catch (IOException e) { 563 LogUtil.d(e, "Failed to open URL"); 564 throw new IllegalArgumentException("Failed to open URL!"); 565 } 566 567 try { 568 // TODO(b/237342352): Both connect and read timeouts are kludged in this method and if 569 // necessary need to be separated 570 closer.eventuallyClose(new CloseableConnectionWrapper(urlConnection), mExecutorService); 571 572 OutputStream os = urlConnection.getOutputStream(); 573 OutputStreamWriter osw = new OutputStreamWriter(os, StandardCharsets.UTF_8); 574 osw.write(data); 575 osw.flush(); 576 osw.close(); 577 578 int responseCode = urlConnection.getResponseCode(); 579 if (isSuccessfulResponse(responseCode)) { 580 LogUtil.d("POST request succeeded for URL: " + url); 581 } else { 582 LogUtil.d("POST request failed for URL: " + url); 583 throwError(urlConnection, responseCode); 584 } 585 return null; 586 } catch (SocketTimeoutException e) { 587 throw new IOException("Connection timed out while reading response!", e); 588 } finally { 589 maybeDisconnect(urlConnection); 590 } 591 } 592 593 /** 594 * Performs an HTTP request according to the request object and returns the response in byte 595 * array. 596 */ performRequestGetResponseInBase64String( @onNull AdServicesHttpClientRequest request)597 public ListenableFuture<AdServicesHttpClientResponse> performRequestGetResponseInBase64String( 598 @NonNull AdServicesHttpClientRequest request) { 599 return performRequestGetResponseInBase64StringWithLogging( 600 request, new FetchProcessLoggerNoLoggingImpl()); 601 } 602 603 /** 604 * Performs an HTTP request according to the request object and returns the response in byte 605 * array. 606 */ 607 public ListenableFuture<AdServicesHttpClientResponse> performRequestGetResponseInBase64StringWithLogging( @onNull AdServicesHttpClientRequest request, @NonNull FetchProcessLogger fetchProcessLogger)608 performRequestGetResponseInBase64StringWithLogging( 609 @NonNull AdServicesHttpClientRequest request, 610 @NonNull FetchProcessLogger fetchProcessLogger) { 611 Objects.requireNonNull(request.getUri()); 612 return ClosingFuture.from( 613 mExecutorService.submit(() -> mUriConverter.toUrl(request.getUri()))) 614 .transformAsync( 615 (closer, url) -> 616 ClosingFuture.from( 617 mExecutorService.submit( 618 () -> 619 doPerformRequestAndGetResponse( 620 url, 621 closer, 622 request, 623 ResponseBodyType 624 .BASE64_ENCODED_STRING, 625 fetchProcessLogger))), 626 mExecutorService) 627 .finishToFuture(); 628 } 629 630 /** 631 * Performs an HTTP request according to the request object and returns the response in plain 632 * String 633 */ performRequestGetResponseInPlainString( @onNull AdServicesHttpClientRequest request)634 public ListenableFuture<AdServicesHttpClientResponse> performRequestGetResponseInPlainString( 635 @NonNull AdServicesHttpClientRequest request) { 636 Objects.requireNonNull(request.getUri()); 637 LogUtil.d("Making request expecting a response in plain string"); 638 FetchProcessLogger fetchProcessLogger = new FetchProcessLoggerNoLoggingImpl(); 639 return ClosingFuture.from( 640 mExecutorService.submit(() -> mUriConverter.toUrl(request.getUri()))) 641 .transformAsync( 642 (closer, url) -> 643 ClosingFuture.from( 644 mExecutorService.submit( 645 () -> 646 doPerformRequestAndGetResponse( 647 url, 648 closer, 649 request, 650 ResponseBodyType.PLAIN_TEXT_STRING, 651 fetchProcessLogger))), 652 mExecutorService) 653 .finishToFuture(); 654 } 655 doPerformRequestAndGetResponse( @onNull URL url, @NonNull ClosingFuture.DeferredCloser closer, AdServicesHttpClientRequest request, ResponseBodyType responseType, @NonNull FetchProcessLogger fetchProcessLogger)656 private AdServicesHttpClientResponse doPerformRequestAndGetResponse( 657 @NonNull URL url, 658 @NonNull ClosingFuture.DeferredCloser closer, 659 AdServicesHttpClientRequest request, 660 ResponseBodyType responseType, 661 @NonNull FetchProcessLogger fetchProcessLogger) 662 throws IOException, AdServicesNetworkException { 663 HttpsURLConnection urlConnection; 664 try { 665 urlConnection = setupConnection(url, request.getDevContext()); 666 urlConnection.setRequestMethod(request.getHttpMethodType().name()); 667 } catch (IOException e) { 668 LogUtil.e(e, "Failed to open URL"); 669 throw new IllegalArgumentException("Failed to open URL!"); 670 } 671 672 InputStream inputStream = null; 673 try { 674 // TODO(b/237342352): Both connect and read timeouts are kludged in this method and if 675 // necessary need to be separated 676 for (Map.Entry<String, String> entry : request.getRequestProperties().entrySet()) { 677 urlConnection.setRequestProperty(entry.getKey(), entry.getValue()); 678 } 679 680 if (request.getHttpMethodType() == AdServicesHttpUtil.HttpMethodType.POST 681 && request.getBodyInBytes() != null 682 && request.getBodyInBytes().length > 0) { 683 urlConnection.setDoOutput(true); 684 try (BufferedOutputStream out = 685 new BufferedOutputStream(urlConnection.getOutputStream())) { 686 out.write(request.getBodyInBytes()); 687 } 688 } 689 690 closer.eventuallyClose(new CloseableConnectionWrapper(urlConnection), mExecutorService); 691 fetchProcessLogger.startNetworkCallTimestamp(); 692 int responseCode = urlConnection.getResponseCode(); 693 fetchProcessLogger.logServerAuctionKeyFetchCalledStatsFromNetwork(responseCode); 694 LogUtil.v("Received %s response status code.", responseCode); 695 696 if (isSuccessfulResponse(responseCode)) { 697 LogUtil.d(" request succeeded for URL: " + url); 698 Map<String, List<String>> responseHeadersMap = 699 pickRequiredHeaderFields( 700 urlConnection.getHeaderFields(), request.getResponseHeaderKeys()); 701 inputStream = new BufferedInputStream(urlConnection.getInputStream()); 702 String responseBody; 703 if (responseType == ResponseBodyType.BASE64_ENCODED_STRING) { 704 responseBody = 705 BaseEncoding.base64() 706 .encode( 707 getByteArray( 708 inputStream, 709 urlConnection.getContentLengthLong())); 710 } else { 711 responseBody = 712 fromInputStream(inputStream, urlConnection.getContentLengthLong()); 713 } 714 return AdServicesHttpClientResponse.builder() 715 .setResponseBody(responseBody) 716 .setResponseHeaders( 717 ImmutableMap.<String, List<String>>builder() 718 .putAll(responseHeadersMap.entrySet()) 719 .build()) 720 .build(); 721 } else { 722 LogUtil.d(" request failed for URL: " + url); 723 throwError(urlConnection, responseCode); 724 return null; 725 } 726 } catch (SocketTimeoutException e) { 727 throw new IOException("Connection timed out while reading response!", e); 728 } finally { 729 maybeDisconnect(urlConnection); 730 maybeClose(inputStream); 731 } 732 } 733 getByteArray(@ullable InputStream in, long contentLength)734 private byte[] getByteArray(@Nullable InputStream in, long contentLength) throws IOException { 735 if (contentLength == 0) { 736 return new byte[0]; 737 } 738 try { 739 byte[] buffer = new byte[1024]; 740 ByteArrayOutputStream out = new ByteArrayOutputStream(); 741 int bytesRead; 742 while ((bytesRead = in.read(buffer)) != -1) { 743 out.write(buffer, 0, bytesRead); 744 } 745 return out.toByteArray(); 746 } finally { 747 in.close(); 748 } 749 } 750 throwError(final HttpsURLConnection urlConnection, int responseCode)751 private void throwError(final HttpsURLConnection urlConnection, int responseCode) 752 throws AdServicesNetworkException { 753 LogUtil.v("Error occurred while executing HTTP request."); 754 755 // Default values for AdServiceNetworkException fields. 756 @AdServicesNetworkException.ErrorCode 757 int errorCode = AdServicesNetworkException.ERROR_OTHER; 758 Duration retryAfterDuration = RetryableAdServicesNetworkException.UNSET_RETRY_AFTER_VALUE; 759 760 // Assign a relevant error code to the HTTP response code. 761 switch (responseCode / 100) { 762 case 3: 763 errorCode = AdServicesNetworkException.ERROR_REDIRECTION; 764 break; 765 case 4: 766 // If an HTTP 429 response code was received, extract the retry-after duration 767 if (responseCode == 429) { 768 errorCode = AdServicesNetworkException.ERROR_TOO_MANY_REQUESTS; 769 String headerValue = urlConnection.getHeaderField(RETRY_AFTER_HEADER_FIELD); 770 if (headerValue != null) { 771 // TODO(b/282017541): Add a maximum allowed retry-after duration. 772 retryAfterDuration = Duration.ofMillis(Long.parseLong(headerValue)); 773 } else { 774 retryAfterDuration = DEFAULT_RETRY_AFTER_VALUE; 775 } 776 } else { 777 errorCode = AdServicesNetworkException.ERROR_CLIENT; 778 } 779 break; 780 case 5: 781 errorCode = AdServicesNetworkException.ERROR_SERVER; 782 } 783 LogUtil.v("Received %s error status code.", responseCode); 784 785 // Throw the appropriate exception. 786 AdServicesNetworkException exception; 787 if (retryAfterDuration.compareTo( 788 RetryableAdServicesNetworkException.UNSET_RETRY_AFTER_VALUE) 789 <= 0) { 790 exception = new AdServicesNetworkException(errorCode); 791 } else { 792 exception = new RetryableAdServicesNetworkException(errorCode, retryAfterDuration); 793 } 794 LogUtil.e("Throwing %s.", exception.toString()); 795 throw exception; 796 } 797 maybeDisconnect(@ullable URLConnection urlConnection)798 private static void maybeDisconnect(@Nullable URLConnection urlConnection) { 799 if (urlConnection == null) { 800 return; 801 } 802 803 if (urlConnection instanceof HttpURLConnection) { 804 HttpURLConnection httpUrlConnection = (HttpURLConnection) urlConnection; 805 httpUrlConnection.disconnect(); 806 } else { 807 LogUtil.d("Not closing URLConnection of type %s", urlConnection.getClass()); 808 } 809 } 810 maybeClose(@ullable InputStream inputStream)811 private static void maybeClose(@Nullable InputStream inputStream) throws IOException { 812 if (inputStream == null) { 813 return; 814 } else { 815 inputStream.close(); 816 } 817 } 818 819 /** 820 * @return the connection timeout, in milliseconds, when opening an initial link to a target 821 * address with this client 822 */ getConnectTimeoutMs()823 public int getConnectTimeoutMs() { 824 return mConnectTimeoutMs; 825 } 826 827 /** 828 * @return the read timeout, in milliseconds, when reading the response from a target address 829 * with this client 830 */ getReadTimeoutMs()831 public int getReadTimeoutMs() { 832 return mReadTimeoutMs; 833 } 834 835 /** 836 * @return true if responseCode matches 2.*, i.e. 200, 204, 206 837 */ isSuccessfulResponse(int responseCode)838 public static boolean isSuccessfulResponse(int responseCode) { 839 return (responseCode / 100) == 2; 840 } 841 842 /** 843 * Reads a {@link InputStream} and returns a {@code String}. To enforce content size limits, we 844 * employ the following strategy: 1. If {@link URLConnection} cannot determine the content size, 845 * we invoke {@code manualStreamToString(InputStream)} where we manually apply the content 846 * restriction. 2. Otherwise, we invoke {@code streamToString(InputStream, long)}. 847 * 848 * @throws IOException if content size limit of is exceeded 849 */ 850 @NonNull fromInputStream(@onNull InputStream in, long size)851 private String fromInputStream(@NonNull InputStream in, long size) throws IOException { 852 Objects.requireNonNull(in); 853 if (size == 0) { 854 return ""; 855 } else if (size < 0) { 856 return manualStreamToString(in); 857 } else { 858 return streamToString(in, size); 859 } 860 } 861 862 @NonNull streamToString(@onNull InputStream in, long size)863 private String streamToString(@NonNull InputStream in, long size) throws IOException { 864 Objects.requireNonNull(in); 865 if (size > mMaxBytes) { 866 throw new IOException(CONTENT_SIZE_ERROR); 867 } 868 return new String(ByteStreams.toByteArray(in), Charsets.UTF_8); 869 } 870 871 @NonNull manualStreamToString(@onNull InputStream in)872 private String manualStreamToString(@NonNull InputStream in) throws IOException { 873 Objects.requireNonNull(in); 874 ByteArrayOutputStream into = new ByteArrayOutputStream(); 875 byte[] buf = new byte[1024]; 876 long total = 0; 877 for (int n; 0 < (n = in.read(buf)); ) { 878 total += n; 879 if (total <= mMaxBytes) { 880 into.write(buf, 0, n); 881 } else { 882 into.close(); 883 throw new IOException(CONTENT_SIZE_ERROR); 884 } 885 } 886 into.close(); 887 return into.toString("UTF-8"); 888 } 889 890 private static class CloseableConnectionWrapper implements Closeable { 891 @Nullable final HttpsURLConnection mURLConnection; 892 CloseableConnectionWrapper(HttpsURLConnection urlConnection)893 private CloseableConnectionWrapper(HttpsURLConnection urlConnection) { 894 mURLConnection = urlConnection; 895 } 896 897 @Override close()898 public void close() throws IOException { 899 maybeClose(mURLConnection.getInputStream()); 900 maybeClose(mURLConnection.getErrorStream()); 901 maybeDisconnect(mURLConnection); 902 } 903 } 904 905 /** A light-weight class to convert Uri to URL */ 906 public static final class UriConverter { 907 908 @NonNull toUrl(@onNull Uri uri)909 URL toUrl(@NonNull Uri uri) { 910 Objects.requireNonNull(uri); 911 Preconditions.checkArgument( 912 ValidatorUtil.HTTPS_SCHEME.equalsIgnoreCase(uri.getScheme()), 913 "URI \"%s\" must use HTTPS", 914 uri.toString()); 915 916 URL url; 917 try { 918 url = new URL(uri.toString()); 919 } catch (MalformedURLException e) { 920 LogUtil.d(e, "Uri is malformed! "); 921 throw new IllegalArgumentException("Uri is malformed!"); 922 } 923 return url; 924 } 925 } 926 927 /** 928 * @return the cache associated with this instance of client 929 */ getAssociatedCache()930 public HttpCache getAssociatedCache() { 931 return mCache; 932 } 933 934 enum ResponseBodyType { 935 BASE64_ENCODED_STRING, 936 PLAIN_TEXT_STRING 937 } 938 } 939