1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.adservices.service.measurement.registration; 17 18 import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_MEASUREMENT_REGISTRATIONS; 19 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.net.Uri; 23 24 import com.android.adservices.LoggerFactory; 25 import com.android.adservices.service.Flags; 26 import com.android.adservices.service.FlagsFactory; 27 import com.android.adservices.service.common.WebAddresses; 28 import com.android.adservices.service.measurement.FilterMap; 29 import com.android.adservices.service.measurement.Source; 30 import com.android.adservices.service.measurement.util.UnsignedLong; 31 import com.android.adservices.service.stats.AdServicesLogger; 32 import com.android.adservices.service.stats.MeasurementRegistrationResponseStats; 33 34 import org.json.JSONArray; 35 import org.json.JSONException; 36 import org.json.JSONObject; 37 38 import java.nio.charset.StandardCharsets; 39 import java.util.ArrayList; 40 import java.util.HashMap; 41 import java.util.Iterator; 42 import java.util.List; 43 import java.util.Map; 44 import java.util.Optional; 45 import java.util.regex.Pattern; 46 47 /** 48 * Common handling for Response Based Registration 49 * 50 * @hide 51 */ 52 public class FetcherUtil { 53 static final Pattern HEX_PATTERN = Pattern.compile("\\p{XDigit}+"); 54 55 /** 56 * Determine all redirects. 57 * 58 * <p>Generates a map of: (redirectType, List<Uri>) 59 */ parseRedirects( @onNull Map<String, List<String>> headers)60 static Map<AsyncRegistration.RedirectType, List<Uri>> parseRedirects( 61 @NonNull Map<String, List<String>> headers) { 62 Map<AsyncRegistration.RedirectType, List<Uri>> uriMap = new HashMap<>(); 63 uriMap.put(AsyncRegistration.RedirectType.LOCATION, parseLocationRedirects(headers)); 64 uriMap.put(AsyncRegistration.RedirectType.LIST, parseListRedirects(headers)); 65 return uriMap; 66 } 67 68 /** 69 * Check HTTP response codes that indicate a redirect. 70 */ isRedirect(int responseCode)71 static boolean isRedirect(int responseCode) { 72 return (responseCode / 100) == 3; 73 } 74 75 /** 76 * Check HTTP response code for success. 77 */ isSuccess(int responseCode)78 static boolean isSuccess(int responseCode) { 79 return (responseCode / 100) == 2; 80 } 81 82 /** Validates both string type and unsigned long parsing */ extractUnsignedLong(JSONObject obj, String key)83 public static Optional<UnsignedLong> extractUnsignedLong(JSONObject obj, String key) { 84 try { 85 Object maybeValue = obj.get(key); 86 if (!(maybeValue instanceof String)) { 87 return Optional.empty(); 88 } 89 return Optional.of(new UnsignedLong((String) maybeValue)); 90 } catch (JSONException | NumberFormatException e) { 91 LoggerFactory.getMeasurementLogger() 92 .e(e, "extractUnsignedLong: caught exception. Key: %s", key); 93 return Optional.empty(); 94 } 95 } 96 97 /** Validates both string type and long parsing */ extractLongString(JSONObject obj, String key)98 public static Optional<Long> extractLongString(JSONObject obj, String key) { 99 try { 100 Object maybeValue = obj.get(key); 101 if (!(maybeValue instanceof String)) { 102 return Optional.empty(); 103 } 104 return Optional.of(Long.parseLong((String) maybeValue)); 105 } catch (JSONException | NumberFormatException e) { 106 LoggerFactory.getMeasurementLogger() 107 .e(e, "extractLongString: caught exception. Key: %s", key); 108 return Optional.empty(); 109 } 110 } 111 112 /** Validates an integral number */ is64BitInteger(Object obj)113 public static boolean is64BitInteger(Object obj) { 114 return (obj instanceof Integer) || (obj instanceof Long); 115 } 116 117 /** Validates both number type and long parsing */ extractLong(JSONObject obj, String key)118 public static Optional<Long> extractLong(JSONObject obj, String key) { 119 try { 120 Object maybeValue = obj.get(key); 121 if (!is64BitInteger(maybeValue)) { 122 return Optional.empty(); 123 } 124 return Optional.of(Long.parseLong(String.valueOf(maybeValue))); 125 } catch (JSONException | NumberFormatException e) { 126 LoggerFactory.getMeasurementLogger() 127 .e(e, "extractLong: caught exception. Key: %s", key); 128 return Optional.empty(); 129 } 130 } 131 extractLookbackWindow(JSONObject obj)132 private static Optional<Long> extractLookbackWindow(JSONObject obj) { 133 try { 134 long lookbackWindow = Long.parseLong(obj.optString(FilterMap.LOOKBACK_WINDOW)); 135 if (lookbackWindow <= 0) { 136 LoggerFactory.getMeasurementLogger() 137 .e( 138 "extractLookbackWindow: non positive lookback window found: %d", 139 lookbackWindow); 140 return Optional.empty(); 141 } 142 return Optional.of(lookbackWindow); 143 } catch (NumberFormatException e) { 144 LoggerFactory.getMeasurementLogger() 145 .e( 146 e, 147 "extractLookbackWindow: caught exception. Key: %s", 148 FilterMap.LOOKBACK_WINDOW); 149 return Optional.empty(); 150 } 151 } 152 153 /** Extract string from an obj with max length. */ extractString(Object obj, int maxLength)154 public static Optional<String> extractString(Object obj, int maxLength) { 155 if (!(obj instanceof String)) { 156 LoggerFactory.getMeasurementLogger().e("obj should be a string."); 157 return Optional.empty(); 158 } 159 String stringValue = (String) obj; 160 if (stringValue.length() > maxLength) { 161 LoggerFactory.getMeasurementLogger() 162 .e("Length of string value should be non-empty and smaller than " + maxLength); 163 return Optional.empty(); 164 } 165 return Optional.of(stringValue); 166 } 167 168 /** Extract list of strings from an obj with max array size and max string length. */ extractStringArray( JSONObject json, String key, int maxArraySize, int maxStringLength)169 public static Optional<List<String>> extractStringArray( 170 JSONObject json, String key, int maxArraySize, int maxStringLength) 171 throws JSONException { 172 JSONArray jsonArray = json.getJSONArray(key); 173 if (jsonArray.length() > maxArraySize) { 174 LoggerFactory.getMeasurementLogger() 175 .e("Json array size should not be greater " + "than " + maxArraySize); 176 return Optional.empty(); 177 } 178 List<String> strings = new ArrayList<>(); 179 for (int i = 0; i < jsonArray.length(); ++i) { 180 Optional<String> string = FetcherUtil.extractString(jsonArray.get(i), maxStringLength); 181 if (string.isEmpty()) { 182 return Optional.empty(); 183 } 184 strings.add(string.get()); 185 } 186 return Optional.of(strings); 187 } 188 189 /** 190 * Validate aggregate key ID. 191 */ isValidAggregateKeyId(String id)192 static boolean isValidAggregateKeyId(String id) { 193 return id != null 194 && !id.isEmpty() 195 && id.getBytes(StandardCharsets.UTF_8).length 196 <= FlagsFactory.getFlags() 197 .getMeasurementMaxBytesPerAttributionAggregateKeyId(); 198 } 199 200 /** Validate aggregate deduplication key. */ isValidAggregateDeduplicationKey(String deduplicationKey)201 static boolean isValidAggregateDeduplicationKey(String deduplicationKey) { 202 if (deduplicationKey == null || deduplicationKey.isEmpty()) { 203 return false; 204 } 205 try { 206 Long.parseUnsignedLong(deduplicationKey); 207 } catch (NumberFormatException exception) { 208 return false; 209 } 210 return true; 211 } 212 213 /** 214 * Validate aggregate key-piece. 215 */ isValidAggregateKeyPiece(String keyPiece, Flags flags)216 static boolean isValidAggregateKeyPiece(String keyPiece, Flags flags) { 217 if (keyPiece == null || keyPiece.isEmpty()) { 218 return false; 219 } 220 int length = keyPiece.getBytes(StandardCharsets.UTF_8).length; 221 if (!(keyPiece.startsWith("0x") || keyPiece.startsWith("0X"))) { 222 return false; 223 } 224 // Key-piece is restricted to a maximum of 128 bits and the hex strings therefore have 225 // at most 32 digits. 226 if (length < 3 || length > 34) { 227 return false; 228 } 229 if (!HEX_PATTERN.matcher(keyPiece.substring(2)).matches()) { 230 return false; 231 } 232 return true; 233 } 234 235 /** Validate attribution filters JSONArray. */ areValidAttributionFilters( @onNull JSONArray filterSet, Flags flags, boolean canIncludeLookbackWindow, boolean shouldCheckFilterSize)236 static boolean areValidAttributionFilters( 237 @NonNull JSONArray filterSet, 238 Flags flags, 239 boolean canIncludeLookbackWindow, 240 boolean shouldCheckFilterSize) throws JSONException { 241 if (filterSet.length() 242 > FlagsFactory.getFlags().getMeasurementMaxFilterMapsPerFilterSet()) { 243 return false; 244 } 245 for (int i = 0; i < filterSet.length(); i++) { 246 if (!areValidAttributionFilters( 247 filterSet.optJSONObject(i), 248 flags, 249 canIncludeLookbackWindow, 250 shouldCheckFilterSize)) { 251 return false; 252 } 253 } 254 return true; 255 } 256 257 /** 258 * Parses header error debug report opt-in info from "Attribution-Reporting-Info" header. The 259 * header is a structured header and only supports dictionary format. Check HTTP [RFC8941] 260 * Section3.2 for details. 261 * 262 * <p>Examples of this type of header: 263 * 264 * <ul> 265 * <li>"Attribution-Reporting-Info":“report-header-errors=?0" 266 * <li>"Attribution-Reporting-Info": “report-header-errors,chrome-param=value" 267 * <li>"Attribution-Reporting-Info": "report-header-errors=?1;chrome-param=value, 268 * report-header-errors=?0" 269 * </ul> 270 * 271 * <p>The header may contain information that is only used in Chrome. Android will ignore it and 272 * be less strict in parsing in the current version. When "report-header-errors" value can't be 273 * extracted, Android will skip sending the debug report instead of dropping the whole 274 * registration. 275 */ isHeaderErrorDebugReportEnabled( @ullable List<String> attributionInfoHeaders, Flags flags)276 public static boolean isHeaderErrorDebugReportEnabled( 277 @Nullable List<String> attributionInfoHeaders, Flags flags) { 278 if (attributionInfoHeaders == null || attributionInfoHeaders.size() == 0) { 279 return false; 280 } 281 if (!flags.getMeasurementEnableDebugReport() 282 || !flags.getMeasurementEnableHeaderErrorDebugReport()) { 283 LoggerFactory.getMeasurementLogger().d("Debug report is disabled for header errors."); 284 return false; 285 } 286 287 // When there are multiple headers or the same key appears multiple times, find the last 288 // appearance and get the value. 289 for (int i = attributionInfoHeaders.size() - 1; i >= 0; i--) { 290 String[] parsed = attributionInfoHeaders.get(i).split("[,;]+"); 291 for (int j = parsed.length - 1; j >= 0; j--) { 292 String parsedStr = parsed[j].trim(); 293 if (parsedStr.equals("report-header-errors") 294 || parsedStr.equals("report-header-errors=?1")) { 295 return true; 296 } else if (parsedStr.equals("report-header-errors=?0")) { 297 return false; 298 } 299 } 300 } 301 // Skip sending the debug report when the key is not found. 302 return false; 303 } 304 305 /** Validate attribution filters JSONObject. */ areValidAttributionFilters( JSONObject filtersObj, Flags flags, boolean canIncludeLookbackWindow, boolean shouldCheckFilterSize)306 static boolean areValidAttributionFilters( 307 JSONObject filtersObj, 308 Flags flags, 309 boolean canIncludeLookbackWindow, 310 boolean shouldCheckFilterSize) throws JSONException { 311 if (filtersObj == null 312 || filtersObj.length() 313 > FlagsFactory.getFlags().getMeasurementMaxAttributionFilters()) { 314 return false; 315 } 316 Iterator<String> keys = filtersObj.keys(); 317 while (keys.hasNext()) { 318 String key = keys.next(); 319 if (shouldCheckFilterSize 320 && key.getBytes(StandardCharsets.UTF_8).length 321 > FlagsFactory.getFlags() 322 .getMeasurementMaxBytesPerAttributionFilterString()) { 323 return false; 324 } 325 // Process known reserved keys that start with underscore first, then invalidate on 326 // catch-all. 327 if (flags.getMeasurementEnableLookbackWindowFilter() 328 && FilterMap.LOOKBACK_WINDOW.equals(key)) { 329 if (!canIncludeLookbackWindow || extractLookbackWindow(filtersObj).isEmpty()) { 330 return false; 331 } 332 continue; 333 } 334 // Invalidate catch-all reserved prefix. 335 if (key.startsWith(FilterMap.RESERVED_PREFIX)) { 336 return false; 337 } 338 JSONArray values = filtersObj.optJSONArray(key); 339 if (values == null) { 340 return false; 341 } 342 if (shouldCheckFilterSize 343 && values.length() 344 > FlagsFactory.getFlags() 345 .getMeasurementMaxValuesPerAttributionFilter()) { 346 return false; 347 } 348 for (int i = 0; i < values.length(); i++) { 349 Object value = values.get(i); 350 if (!(value instanceof String)) { 351 return false; 352 } 353 if (shouldCheckFilterSize 354 && ((String) value).getBytes(StandardCharsets.UTF_8).length 355 > FlagsFactory.getFlags() 356 .getMeasurementMaxBytesPerAttributionFilterString()) { 357 return false; 358 } 359 } 360 } 361 return true; 362 } 363 getSourceRegistrantToLog(AsyncRegistration asyncRegistration)364 static String getSourceRegistrantToLog(AsyncRegistration asyncRegistration) { 365 if (asyncRegistration.isSourceRequest()) { 366 return asyncRegistration.getRegistrant().toString(); 367 } 368 369 return ""; 370 } 371 emitHeaderMetrics( long headerSizeLimitBytes, AdServicesLogger logger, AsyncRegistration asyncRegistration, AsyncFetchStatus asyncFetchStatus)372 static void emitHeaderMetrics( 373 long headerSizeLimitBytes, 374 AdServicesLogger logger, 375 AsyncRegistration asyncRegistration, 376 AsyncFetchStatus asyncFetchStatus) { 377 long headerSize = asyncFetchStatus.getResponseSize(); 378 String adTechDomain = null; 379 380 if (headerSize > headerSizeLimitBytes) { 381 adTechDomain = 382 WebAddresses.topPrivateDomainAndScheme(asyncRegistration.getRegistrationUri()) 383 .map(Uri::toString) 384 .orElse(null); 385 } 386 387 logger.logMeasurementRegistrationsResponseSize( 388 new MeasurementRegistrationResponseStats.Builder( 389 AD_SERVICES_MEASUREMENT_REGISTRATIONS, 390 getRegistrationType(asyncRegistration), 391 headerSize, 392 getSourceType(asyncRegistration), 393 getSurfaceType(asyncRegistration), 394 getStatus(asyncFetchStatus), 395 getFailureType(asyncFetchStatus), 396 asyncFetchStatus.getRegistrationDelay(), 397 getSourceRegistrantToLog(asyncRegistration), 398 asyncFetchStatus.getRetryCount(), 399 asyncFetchStatus.isRedirectOnly(), 400 asyncFetchStatus.isPARequest()) 401 .setAdTechDomain(adTechDomain) 402 .build()); 403 } 404 parseListRedirects(Map<String, List<String>> headers)405 private static List<Uri> parseListRedirects(Map<String, List<String>> headers) { 406 List<Uri> redirects = new ArrayList<>(); 407 List<String> field = headers.get(AsyncRedirects.REDIRECT_LIST_HEADER_KEY); 408 int maxRedirects = FlagsFactory.getFlags().getMeasurementMaxRegistrationRedirects(); 409 if (field != null) { 410 for (int i = 0; i < Math.min(field.size(), maxRedirects); i++) { 411 redirects.add(Uri.parse(field.get(i))); 412 } 413 } 414 return redirects; 415 } 416 parseLocationRedirects(Map<String, List<String>> headers)417 private static List<Uri> parseLocationRedirects(Map<String, List<String>> headers) { 418 List<Uri> redirects = new ArrayList<>(); 419 List<String> field = headers.get(AsyncRedirects.REDIRECT_LOCATION_HEADER_KEY); 420 if (field != null && !field.isEmpty()) { 421 redirects.add(Uri.parse(field.get(0))); 422 if (field.size() > 1) { 423 LoggerFactory.getMeasurementLogger() 424 .e("Expected one Location redirect only, others ignored!"); 425 } 426 } 427 return redirects; 428 } 429 calculateHeadersCharactersLength(Map<String, List<String>> headers)430 public static long calculateHeadersCharactersLength(Map<String, List<String>> headers) { 431 long size = 0; 432 for (String headerKey : headers.keySet()) { 433 if (headerKey != null) { 434 size = size + headerKey.length(); 435 List<String> headerValues = headers.get(headerKey); 436 if (headerValues != null) { 437 for (String headerValue : headerValues) { 438 size = size + headerValue.length(); 439 } 440 } 441 } 442 } 443 444 return size; 445 } 446 getRegistrationType(AsyncRegistration asyncRegistration)447 private static int getRegistrationType(AsyncRegistration asyncRegistration) { 448 if (asyncRegistration.isSourceRequest()) { 449 return RegistrationEnumsValues.TYPE_SOURCE; 450 } else if (asyncRegistration.isTriggerRequest()) { 451 return RegistrationEnumsValues.TYPE_TRIGGER; 452 } else { 453 return RegistrationEnumsValues.TYPE_UNKNOWN; 454 } 455 } 456 getSourceType(AsyncRegistration asyncRegistration)457 private static int getSourceType(AsyncRegistration asyncRegistration) { 458 if (asyncRegistration.getSourceType() == Source.SourceType.EVENT) { 459 return RegistrationEnumsValues.SOURCE_TYPE_EVENT; 460 } else if (asyncRegistration.getSourceType() == Source.SourceType.NAVIGATION) { 461 return RegistrationEnumsValues.SOURCE_TYPE_NAVIGATION; 462 } else { 463 return RegistrationEnumsValues.SOURCE_TYPE_UNKNOWN; 464 } 465 } 466 getSurfaceType(AsyncRegistration asyncRegistration)467 private static int getSurfaceType(AsyncRegistration asyncRegistration) { 468 if (asyncRegistration.isAppRequest()) { 469 return RegistrationEnumsValues.SURFACE_TYPE_APP; 470 } else if (asyncRegistration.isWebRequest()) { 471 return RegistrationEnumsValues.SURFACE_TYPE_WEB; 472 } else { 473 return RegistrationEnumsValues.SURFACE_TYPE_UNKNOWN; 474 } 475 } 476 getStatus(AsyncFetchStatus asyncFetchStatus)477 private static int getStatus(AsyncFetchStatus asyncFetchStatus) { 478 if (asyncFetchStatus.getEntityStatus() == AsyncFetchStatus.EntityStatus.SUCCESS 479 || (asyncFetchStatus.getResponseStatus() == AsyncFetchStatus.ResponseStatus.SUCCESS 480 && (asyncFetchStatus.getEntityStatus() 481 == AsyncFetchStatus.EntityStatus.UNKNOWN 482 || asyncFetchStatus.getEntityStatus() 483 == AsyncFetchStatus.EntityStatus.HEADER_MISSING))) { 484 // successful source/trigger fetching/parsing and successful redirects (with no header) 485 return RegistrationEnumsValues.STATUS_SUCCESS; 486 } else if (asyncFetchStatus.getEntityStatus() == AsyncFetchStatus.EntityStatus.UNKNOWN 487 && asyncFetchStatus.getResponseStatus() 488 == AsyncFetchStatus.ResponseStatus.UNKNOWN) { 489 return RegistrationEnumsValues.STATUS_UNKNOWN; 490 } else { 491 return RegistrationEnumsValues.STATUS_FAILURE; 492 } 493 } 494 getFailureType(AsyncFetchStatus asyncFetchStatus)495 private static int getFailureType(AsyncFetchStatus asyncFetchStatus) { 496 if (asyncFetchStatus.getResponseStatus() == AsyncFetchStatus.ResponseStatus.NETWORK_ERROR) { 497 return RegistrationEnumsValues.FAILURE_TYPE_NETWORK; 498 } else if (asyncFetchStatus.getResponseStatus() 499 == AsyncFetchStatus.ResponseStatus.INVALID_URL) { 500 return RegistrationEnumsValues.FAILURE_TYPE_INVALID_URL; 501 } else if (asyncFetchStatus.getResponseStatus() 502 == AsyncFetchStatus.ResponseStatus.SERVER_UNAVAILABLE) { 503 return RegistrationEnumsValues.FAILURE_TYPE_SERVER_UNAVAILABLE; 504 } else if (asyncFetchStatus.getResponseStatus() 505 == AsyncFetchStatus.ResponseStatus.HEADER_SIZE_LIMIT_EXCEEDED) { 506 return RegistrationEnumsValues.FAILURE_TYPE_HEADER_SIZE_LIMIT_EXCEEDED; 507 } else if (asyncFetchStatus.getEntityStatus() 508 == AsyncFetchStatus.EntityStatus.INVALID_ENROLLMENT) { 509 return RegistrationEnumsValues.FAILURE_TYPE_ENROLLMENT; 510 } else if (asyncFetchStatus.getEntityStatus() 511 == AsyncFetchStatus.EntityStatus.VALIDATION_ERROR 512 || asyncFetchStatus.getEntityStatus() == AsyncFetchStatus.EntityStatus.PARSING_ERROR 513 || asyncFetchStatus.getEntityStatus() 514 == AsyncFetchStatus.EntityStatus.HEADER_ERROR) { 515 return RegistrationEnumsValues.FAILURE_TYPE_PARSING; 516 } else if (asyncFetchStatus.getEntityStatus() 517 == AsyncFetchStatus.EntityStatus.STORAGE_ERROR) { 518 return RegistrationEnumsValues.FAILURE_TYPE_STORAGE; 519 } else if (asyncFetchStatus.isRedirectError()) { 520 return RegistrationEnumsValues.FAILURE_TYPE_REDIRECT; 521 } else { 522 return RegistrationEnumsValues.FAILURE_TYPE_UNKNOWN; 523 } 524 } 525 526 /** AdservicesMeasurementRegistrations atom enum values. */ 527 public interface RegistrationEnumsValues { 528 int TYPE_UNKNOWN = 0; 529 int TYPE_SOURCE = 1; 530 int TYPE_TRIGGER = 2; 531 int SOURCE_TYPE_UNKNOWN = 0; 532 int SOURCE_TYPE_EVENT = 1; 533 int SOURCE_TYPE_NAVIGATION = 2; 534 int SURFACE_TYPE_UNKNOWN = 0; 535 int SURFACE_TYPE_WEB = 1; 536 int SURFACE_TYPE_APP = 2; 537 int STATUS_UNKNOWN = 0; 538 int STATUS_SUCCESS = 1; 539 int STATUS_FAILURE = 2; 540 int FAILURE_TYPE_UNKNOWN = 0; 541 int FAILURE_TYPE_PARSING = 1; 542 int FAILURE_TYPE_NETWORK = 2; 543 int FAILURE_TYPE_ENROLLMENT = 3; 544 int FAILURE_TYPE_REDIRECT = 4; 545 int FAILURE_TYPE_STORAGE = 5; 546 int FAILURE_TYPE_HEADER_SIZE_LIMIT_EXCEEDED = 7; 547 int FAILURE_TYPE_SERVER_UNAVAILABLE = 8; 548 int FAILURE_TYPE_INVALID_URL = 9; 549 } 550 } 551