1#!/usr/bin/env ruby 2# coding: binary 3# 4# Copyright 2015 Google Inc. All rights reserved 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17 18require 'fileutils' 19 20# suppress GNU make jobserver magic when calling "make" 21ENV.delete('MAKEFLAGS') 22ENV.delete('MAKELEVEL') 23 24while true 25 if ARGV[0] == '-s' 26 test_serialization = true 27 ARGV.shift 28 elsif ARGV[0] == '-c' 29 ckati = true 30 ARGV.shift 31 ENV['KATI_VARIANT'] = 'c' 32 elsif ARGV[0] == '-n' 33 via_ninja = true 34 ARGV.shift 35 ENV['NINJA_STATUS'] = 'NINJACMD: ' 36 elsif ARGV[0] == '-a' 37 gen_all_targets = true 38 ARGV.shift 39 elsif ARGV[0] == '-v' 40 show_failing = true 41 ARGV.shift 42 else 43 break 44 end 45end 46 47def get_output_filenames 48 files = Dir.glob('*') 49 files.delete('Makefile') 50 files.delete('build.ninja') 51 files.delete('env.sh') 52 files.delete('ninja.sh') 53 files.delete('gmon.out') 54 files.delete('submake') 55 files.reject!{|f|f =~ /\.json$/} 56 files.reject!{|f|f =~ /^kati\.*/} 57 files 58end 59 60def cleanup 61 (get_output_filenames + Dir.glob('.*')).each do |fname| 62 next if fname == '.' || fname == '..' 63 FileUtils.rm_rf fname 64 end 65end 66 67def move_circular_dep(l) 68 # We don't care when circular dependency detection happens. 69 circ = '' 70 while l.sub!(/Circular .* dropped\.\n/, '') do 71 circ += $& 72 end 73 circ + l 74end 75 76expected_failures = [] 77unexpected_passes = [] 78failures = [] 79passes = [] 80 81if !ARGV.empty? 82 test_files = ARGV.map do |test| 83 "testcase/#{File.basename(test)}" 84 end 85else 86 test_files = Dir.glob('testcase/*.mk').sort 87 test_files += Dir.glob('testcase/*.sh').sort 88end 89 90def run_in_testdir(test_filename) 91 c = File.read(test_filename) 92 name = File.basename(test_filename) 93 dir = "out/#{name}" 94 95 FileUtils.mkdir_p(dir) 96 Dir.glob("#{dir}/*").each do |fname| 97 FileUtils.rm_rf(fname) 98 end 99 100 Dir.chdir(dir) do 101 yield name 102 end 103end 104 105def normalize_ninja_log(log, mk) 106 log.gsub!(/^NINJACMD: .*\n/, '') 107 log.gsub!(/^ninja: no work to do\.\n/, '') 108 log.gsub!(/^ninja: error: (.*, needed by .*),.*/, 109 '*** No rule to make target \\1.') 110 log.gsub!(/^ninja: warning: multiple rules generate (.*)\. builds involving this target will not be correct.*$/, 111 'ninja: warning: multiple rules generate \\1.') 112 113 if mk =~ /err_error_in_recipe.mk/ 114 # This test expects ninja fails. Strip ninja specific error logs. 115 ninja_failed_subst = '' 116 elsif mk =~ /\/fail_/ 117 # Recipes in these tests fail. 118 ninja_failed_subst = "*** [test] Error 1\n" 119 end 120 if ninja_failed_subst 121 log.gsub!(/^FAILED: (.*\n\/bin\/bash)?.*\n/, ninja_failed_subst) 122 log.gsub!(/^ninja: .*\n/, '') 123 end 124 log 125end 126 127def normalize_quotes(log) 128 log.gsub!(/[`'"]/, '"') 129 # For recent GNU find, which uses Unicode characters. 130 log.gsub!(/(\xe2\x80\x98|\xe2\x80\x99)/, '"') 131 log 132end 133 134def normalize_make_log(expected, mk, via_ninja) 135 expected = normalize_quotes(expected) 136 expected.gsub!(/^make(?:\[\d+\])?: (Entering|Leaving) directory.*\n/, '') 137 expected.gsub!(/^make(?:\[\d+\])?: /, '') 138 expected = move_circular_dep(expected) 139 140 # Normalizations for old/new GNU make. 141 expected.gsub!(' recipe for target ', ' commands for target ') 142 expected.gsub!(' recipe commences ', ' commands commence ') 143 expected.gsub!('missing rule before recipe.', 'missing rule before commands.') 144 expected.gsub!(' (did you mean TAB instead of 8 spaces?)', '') 145 expected.gsub!('Extraneous text after', 'extraneous text after') 146 # Not sure if this is useful. 147 expected.gsub!(/\s+Stop\.$/, '') 148 # GNU make 4.0 has this output. 149 expected.gsub!(/Makefile:\d+: commands for target ".*?" failed\n/, '') 150 # We treat some warnings as errors. 151 expected.gsub!(/^\/bin\/(ba)?sh: line 0: /, '') 152 # We print out some ninja warnings in some tests to match what we expect 153 # ninja to produce. Remove them if we're not testing ninja. 154 if !via_ninja 155 expected.gsub!(/^ninja: warning: .*\n/, '') 156 end 157 # Normalization for "include foo" with C++ kati. 158 expected.gsub!(/(: )(\S+): (No such file or directory)\n\*\*\* No rule to make target "\2"./, '\1\2: \3') 159 160 expected 161end 162 163def normalize_kati_log(output) 164 output = normalize_quotes(output) 165 output = move_circular_dep(output) 166 167 # kati specific log messages. 168 output.gsub!(/^\*kati\*.*\n/, '') 169 output.gsub!(/^c?kati: /, '') 170 output.gsub!(/\/bin\/sh: ([^:]*): command not found/, 171 "\\1: Command not found") 172 output.gsub!(/.*: warning for parse error in an unevaluated line: .*\n/, '') 173 output.gsub!(/^([^ ]+: )?FindEmulator: /, '') 174 output.gsub!(/^\/bin\/sh: line 0: /, '') 175 output.gsub!(/ (\.\/+)+kati\.\S+/, '') # kati log files in find_command.mk 176 output.gsub!(/ (\.\/+)+test\S+.json/, '') # json files in find_command.mk 177 # Normalization for "include foo" with Go kati. 178 output.gsub!(/(: )open (\S+): n(o such file or directory)\nNOTE:.*/, 179 "\\1\\2: N\\3") 180 # Bionic libc has different error messages than glibc 181 output.gsub!(/Too many symbolic links encountered/, 'Too many levels of symbolic links') 182 output 183end 184 185bash_var = ' SHELL=/bin/bash' 186 187run_make_test = proc do |mk| 188 c = File.read(mk) 189 expected_failure = false 190 if c =~ /\A# TODO(?:\(([-a-z|]+)\))?/ 191 if $1 192 todos = $1.split('|') 193 if todos.include?('go') && !ckati 194 expected_failure = true 195 end 196 if todos.include?('c') && ckati 197 expected_failure = true 198 end 199 if todos.include?('go-ninja') && !ckati && via_ninja 200 expected_failure = true 201 end 202 if todos.include?('c-ninja') && ckati && via_ninja 203 expected_failure = true 204 end 205 if todos.include?('c-exec') && ckati && !via_ninja 206 expected_failure = true 207 end 208 if todos.include?('ninja') && via_ninja 209 expected_failure = true 210 end 211 else 212 expected_failure = true 213 end 214 end 215 216 run_in_testdir(mk) do |name| 217 File.open("Makefile", 'w') do |ofile| 218 ofile.print(c) 219 end 220 File.symlink('../../testcase/submake', 'submake') 221 222 expected = '' 223 output = '' 224 225 testcases = c.scan(/^test\d*/).sort.uniq 226 if testcases.empty? 227 testcases = [''] 228 end 229 230 is_silent_test = mk =~ /\/submake_/ 231 232 cleanup 233 testcases.each do |tc| 234 cmd = 'make' 235 if via_ninja || is_silent_test 236 cmd += ' -s' 237 end 238 cmd += bash_var 239 cmd += " #{tc} 2>&1" 240 res = IO.popen(cmd, 'r:binary', &:read) 241 res = normalize_make_log(res, mk, via_ninja) 242 expected += "=== #{tc} ===\n" + res 243 expected_files = get_output_filenames 244 expected += "\n=== FILES ===\n#{expected_files * "\n"}\n" 245 end 246 247 cleanup 248 testcases.each do |tc| 249 json = "#{tc.empty? ? 'test' : tc}" 250 cmd = "../../kati -save_json=#{json}.json -log_dir=. --use_find_emulator" 251 if ckati 252 cmd = "../../ckati --use_find_emulator" 253 end 254 if via_ninja 255 cmd += ' --ninja' 256 end 257 if gen_all_targets 258 if !ckati || !via_ninja 259 raise "-a should be used with -c -n" 260 end 261 cmd += ' --gen_all_targets' 262 end 263 if is_silent_test 264 cmd += ' -s' 265 end 266 cmd += bash_var 267 if !gen_all_targets || mk =~ /makecmdgoals/ 268 cmd += " #{tc}" 269 end 270 cmd += " 2>&1" 271 res = IO.popen(cmd, 'r:binary', &:read) 272 if via_ninja && File.exist?('build.ninja') && File.exists?('ninja.sh') 273 cmd = './ninja.sh -j1 -v' 274 if gen_all_targets 275 cmd += " #{tc}" 276 end 277 cmd += ' 2>&1' 278 log = IO.popen(cmd, 'r:binary', &:read) 279 res += normalize_ninja_log(log, mk) 280 end 281 res = normalize_kati_log(res) 282 output += "=== #{tc} ===\n" + res 283 output_files = get_output_filenames 284 output += "\n=== FILES ===\n#{output_files * "\n"}\n" 285 end 286 287 File.open('out.make', 'w'){|ofile|ofile.print(expected)} 288 File.open('out.kati', 'w'){|ofile|ofile.print(output)} 289 290 if expected =~ /FAIL/ 291 puts %Q(#{name} has a string "FAIL" in its expectation) 292 exit 1 293 end 294 295 if expected != output 296 if expected_failure 297 puts "#{name}: FAIL (expected)" 298 expected_failures << name 299 else 300 puts "#{name}: FAIL" 301 failures << name 302 end 303 if !expected_failure || show_failing 304 puts `diff -u out.make out.kati` 305 end 306 else 307 if expected_failure 308 puts "#{name}: PASS (unexpected)" 309 unexpected_passes << name 310 else 311 puts "#{name}: PASS" 312 passes << name 313 end 314 end 315 316 if name !~ /^err_/ && test_serialization && !expected_failure 317 testcases.each do |tc| 318 json = "#{tc.empty? ? 'test' : tc}" 319 cmd = "../../kati -save_json=#{json}_2.json -load_json=#{json}.json -n -log_dir=. #{tc} 2>&1" 320 res = IO.popen(cmd, 'r:binary', &:read) 321 if !File.exist?("#{json}.json") || !File.exist?("#{json}_2.json") 322 puts "#{name}##{json}: Serialize failure (not exist)" 323 puts res 324 else 325 json1 = File.read("#{json}.json") 326 json2 = File.read("#{json}_2.json") 327 if json1 != json2 328 puts "#{name}##{json}: Serialize failure" 329 puts res 330 end 331 end 332 end 333 end 334 end 335end 336 337run_shell_test = proc do |sh| 338 is_ninja_test = sh =~ /\/ninja_/ 339 if is_ninja_test && (!ckati || !via_ninja) 340 next 341 end 342 343 run_in_testdir(sh) do |name| 344 cleanup 345 cmd = "sh ../../#{sh} make" 346 if is_ninja_test 347 cmd += ' -s' 348 end 349 cmd += bash_var 350 expected = IO.popen(cmd, 'r:binary', &:read) 351 cleanup 352 353 if is_ninja_test 354 if ckati 355 cmd = "sh ../../#{sh} ../../ckati --ninja --regen" 356 else 357 next 358 end 359 else 360 if ckati 361 cmd = "sh ../../#{sh} ../../ckati" 362 else 363 cmd = "sh ../../#{sh} ../../kati --use_cache -log_dir=." 364 end 365 end 366 cmd += bash_var 367 368 output = IO.popen(cmd, 'r:binary', &:read) 369 370 expected = normalize_make_log(expected, sh, is_ninja_test) 371 output = normalize_kati_log(output) 372 if is_ninja_test 373 output = normalize_ninja_log(output, sh) 374 end 375 File.open('out.make', 'w'){|ofile|ofile.print(expected)} 376 File.open('out.kati', 'w'){|ofile|ofile.print(output)} 377 378 if expected != output 379 puts "#{name}: FAIL" 380 puts `diff -u out.make out.kati` 381 failures << name 382 else 383 puts "#{name}: PASS" 384 passes << name 385 end 386 end 387end 388 389test_files.each do |test| 390 if /\.mk$/ =~ test 391 run_make_test.call(test) 392 elsif /\.sh$/ =~ test 393 run_shell_test.call(test) 394 else 395 raise "Unknown test type: #{test}" 396 end 397end 398 399puts 400 401if !expected_failures.empty? 402 puts "=== Expected failures ===" 403 expected_failures.each do |n| 404 puts n 405 end 406end 407 408if !unexpected_passes.empty? 409 puts "=== Unexpected passes ===" 410 unexpected_passes.each do |n| 411 puts n 412 end 413end 414 415if !failures.empty? 416 puts "=== Failures ===" 417 failures.each do |n| 418 puts n 419 end 420end 421 422puts 423 424if !unexpected_passes.empty? || !failures.empty? 425 puts "FAIL! (#{failures.size + unexpected_passes.size} fails #{passes.size} passes)" 426 exit 1 427else 428 puts 'PASS!' 429end 430