1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package libcore; 18 19 import java.io.IOException; 20 import java.io.PrintStream; 21 import java.nio.file.Path; 22 import java.util.ArrayList; 23 import java.util.Arrays; 24 import java.util.Collections; 25 import java.util.Comparator; 26 import java.util.LinkedHashMap; 27 import java.util.List; 28 import java.util.Locale; 29 import java.util.Map; 30 import java.util.Objects; 31 import java.util.regex.Matcher; 32 import java.util.regex.Pattern; 33 34 /** 35 * Helps compare openjdk_java_files contents against upstream file contents. 36 * 37 * Outputs a tab-separated table comparing each openjdk_java_files entry 38 * against OpenJDK upstreams. This can help verify updates to later upstreams 39 * or focus attention towards files that may have been missed in a previous 40 * update (http://b/36461944) or are otherwise surprising (http://b/36429512). 41 * 42 * - Identifies each file as identical to, different from or missing from 43 * each upstream; diffs are not produced. 44 * - Optionally, copies all openjdk_java_files from the default upstream 45 * (eg. OpenJDK8u121-b13) to a new directory, for easy directory comparison 46 * using e.g. kdiff3, which allows inspecting detailed diffs. 47 * - The ANDROID_BUILD_TOP environment variable must be set to point to the 48 * AOSP root directory (parent of libcore). 49 * 50 * To check out upstreams OpenJDK 7u40, 8u60, 8u121-b13, 8u222-b01 and 9+181, run: 51 * 52 * mkdir ~/openjdk 53 * cd ~/openjdk 54 * export OPENJDK_HOME=$PWD 55 * hg clone http://hg.openjdk.java.net/jdk7u/jdk7u40/ 7u40 56 * (cd !$ ; sh get_source.sh) 57 * hg clone http://hg.openjdk.java.net/jdk8u/jdk8u 8u121-b13 58 * (cd !$ ; hg update -r jdk8u121-b13 && sh get_source.sh && sh common/bin/hgforest.sh update -r jdk8u121-b13) 59 * hg clone http://hg.openjdk.java.net/jdk8u/jdk8u60/ 8u60 60 * (cd !$ ; sh get_source.sh) 61 * hg clone http://hg.openjdk.java.net/jdk8u/jdk8u 8u222-b01 62 * (cd !$ ; hg update -r jdk8u222-b01 && sh get_source.sh && sh common/bin/hgforest.sh update -r jdk8u222-b01) 63 * hg clone http://hg.openjdk.java.net/jdk9/jdk9/ 9+181 64 * (cd !$ ; hg update -r jdk-9+181 && sh get_source.sh && sh common/bin/hgforest.sh update -r jdk-9+181) 65 * 66 * To get the 9b113+ upstream, follow the instructions from the commit 67 * message of AOSP libcore commit 29957558cf0db700bfaae360a80c42dc3871d0e5 68 * at https://android-review.googlesource.com/c/304056/ 69 * 70 * To get OpenJDK head: hg clone http://hg.openjdk.java.net/jdk/jdk/ head 71 */ 72 public class CompareUpstreams { 73 74 /** 75 * Whether to compare against snapshots based on (a) the output of {@link CopyUpstreamFiles}, 76 * as opposed to (b) directly against checked-out upstream source {@link Repository}s. 77 * 78 * Because the snapshots are currently kept on x20 which is slow to access, (b) run much 79 * faster (a few seconds vs. 30 minutes), but it requires the checked-out and compiled 80 * upstream repositories to exist which is not the case for everyone / not easily achievable 81 * (OpenJDK 8 requires an old C++ compiler to build). 82 */ 83 public static final boolean COMPARE_AGAINST_UPSTREAM_SNAPSHOT = true; 84 85 private final StandardRepositories standardRepositories; 86 CompareUpstreams(StandardRepositories standardRepositories)87 public CompareUpstreams(StandardRepositories standardRepositories) { 88 this.standardRepositories = Objects.requireNonNull(standardRepositories); 89 } 90 androidChangedComments(List<String> lines)91 private static Map<String, Integer> androidChangedComments(List<String> lines) { 92 List<String> problems = new ArrayList<>(); 93 Map<String, Integer> result = new LinkedHashMap<>(); 94 Pattern pattern = Pattern.compile( 95 "// (BEGIN |END |)Android-((?:changed|added|removed|note)(?:: )?.*)$"); 96 for (String line : lines) { 97 Matcher matcher = pattern.matcher(line); 98 if (matcher.find()) { 99 String type = matcher.group(1); 100 if (type.equals("END")) { 101 continue; 102 } 103 String match = matcher.group(2); 104 if (match.isEmpty()) { 105 match = "[empty comment]"; 106 } 107 Integer oldCount = result.get(match); 108 if (oldCount == null) { 109 oldCount = 0; 110 } 111 result.put(match, oldCount + 1); 112 } else if (line.contains("Android-")) { 113 problems.add(line); 114 } 115 } 116 if (!problems.isEmpty()) { 117 throw new IllegalArgumentException(problems.toString()); 118 } 119 return result; 120 } 121 androidChangedCommentsSummary(List<String> lines)122 private static String androidChangedCommentsSummary(List<String> lines) { 123 Map<String, Integer> map = androidChangedComments(lines); 124 List<String> comments = new ArrayList<>(map.keySet()); 125 Collections.sort(comments, Comparator.comparing(map::get).reversed()); 126 List<String> result = new ArrayList<>(); 127 for (String comment : comments) { 128 int count = map.get(comment); 129 if (count == 1) { 130 result.add(comment); 131 } else { 132 result.add(comment + " (x" + count + ")"); 133 } 134 } 135 return escapeTsv(String.join("\n", result)); 136 } 137 escapeTsv(String value)138 private static String escapeTsv(String value) { 139 if (value.contains("\t")) { 140 throw new IllegalArgumentException(value); // tsv doesn't support escaping tabs 141 } 142 return "\"" + value.replace("\"", "\"\"") + "\""; 143 } 144 printTsv(PrintStream out, List<String> values)145 private static void printTsv(PrintStream out, List<String> values) { 146 out.println(String.join("\t", values)); 147 } 148 149 /** 150 * Prints tab-separated values comparing ojluni files vs. each 151 * upstream, for each of the rel_paths, suitable for human 152 * analysis in a spreadsheet. 153 * This includes whether the corresponding upstream file is 154 * missing, identical, or by how many lines it differs, and 155 * a guess as to the correct upstream based on minimal line 156 * difference (ties broken in favor of upstreams that occur 157 * earlier in the list). 158 */ run(PrintStream out, List<Path> relPaths)159 private void run(PrintStream out, List<Path> relPaths) throws IOException { 160 // upstreams are in decreasing order of preference 161 List<String> headers = new ArrayList<>(); 162 headers.addAll(Arrays.asList( 163 "rel_path", "expected_upstream", "guessed_upstream", "changes", "vs. expected")); 164 for (Repository upstream : standardRepositories.historicUpstreams()) { 165 headers.add(upstream.name()); 166 } 167 headers.add("diff"); 168 headers.add("guessed_upstream_path"); 169 printTsv(out, headers); 170 171 Path snapshotRoot = COMPARE_AGAINST_UPSTREAM_SNAPSHOT 172 ? Util.pathFromEnvOrThrow("OJLUNI_UPSTREAMS") 173 : null; 174 175 for (Path relPath : relPaths) { 176 Repository expectedUpstream = standardRepositories.referenceUpstream(relPath); 177 out.print(relPath + "\t"); 178 Path ojluniFile = standardRepositories.ojluni().absolutePath(relPath); 179 List<String> linesB = Util.readLines(ojluniFile); 180 int bestDistance = Integer.MAX_VALUE; 181 Repository guessedUpstream = null; 182 List<Repository> upstreams = new ArrayList<>(); 183 upstreams.add(expectedUpstream); 184 upstreams.addAll(standardRepositories.historicUpstreams()); 185 List<String> comparisons = new ArrayList<>(upstreams.size()); 186 for (Repository upstream : upstreams) { 187 final String comparison; 188 final Path upstreamFile; 189 if (COMPARE_AGAINST_UPSTREAM_SNAPSHOT) { 190 Path maybePath = snapshotRoot 191 .resolve(upstream.name()) 192 .resolve(relPath); 193 upstreamFile = maybePath.toFile().exists() ? maybePath : null; 194 } else { 195 upstreamFile = upstream.absolutePath(relPath); 196 } 197 if (upstreamFile == null) { 198 comparison = "missing"; 199 } else { 200 List<String> linesA = Util.readLines(upstreamFile); 201 int distance = Util.editDistance(linesA, linesB); 202 if (distance == 0) { 203 comparison = "identical"; 204 } else { 205 double percentDifferent = 100.0 * distance / Math 206 .max(linesA.size(), linesB.size()); 207 comparison = String 208 .format(Locale.US, "%.1f%% different (%d lines)", percentDifferent, 209 distance); 210 } 211 if (distance < bestDistance) { 212 bestDistance = distance; 213 guessedUpstream = upstream; 214 } 215 } 216 comparisons.add(comparison); 217 } 218 String changedCommentsSummary = androidChangedCommentsSummary(linesB); 219 220 String diffCommand = ""; 221 String guessed_upstream_path = guessedUpstream != null ? 222 "" + guessedUpstream.pathFromRepository(relPath) : ""; 223 if (!comparisons.get(0).equals("identical")) { 224 Path expectedUpstreamPath = expectedUpstream.pathFromRepository(relPath); 225 if (expectedUpstreamPath != null) { 226 diffCommand = "${ANDROID_BUILD_TOP}/libcore/tools/upstream/upstream-diff " 227 + "-r ojluni," + expectedUpstream.name() + " " + relPath; 228 } else { 229 diffCommand = "FILE MISSING"; 230 } 231 } 232 List<String> values = new ArrayList<>(); 233 values.add(expectedUpstream.name()); 234 values.add(guessedUpstream == null ? "" : guessedUpstream.name()); 235 values.add(changedCommentsSummary); 236 values.addAll(comparisons); 237 values.add(diffCommand); 238 values.add(guessed_upstream_path); 239 printTsv(out, values); 240 } 241 } 242 run()243 public void run() throws IOException { 244 List<Path> relPaths = standardRepositories.ojluni().loadRelPathsFromBlueprint(); 245 run(System.out, relPaths); 246 } 247 main(String[] args)248 public static void main(String[] args) throws IOException { 249 StandardRepositories standardRepositories = StandardRepositories.fromEnv(); 250 CompareUpstreams action = new CompareUpstreams(standardRepositories); 251 action.run(); 252 } 253 } 254