1#!/bin/bash -u 2# Copyright 2017 Google Inc. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16################################################################################ 17 18# A minimal number of runs to test fuzz target with a non-empty input. 19MIN_NUMBER_OF_RUNS=4 20 21# The "example" target has 73 with ASan, 65 with UBSan, and 6648 with MSan. 22# Real world targets have greater values (arduinojson: 407, zlib: 664). 23# Mercurial's bdiff_fuzzer has 116 PCs when built with ASan. 24THRESHOLD_FOR_NUMBER_OF_EDGES=100 25 26# A fuzz target is supposed to have at least two functions, such as 27# LLVMFuzzerTestOneInput and an API that is being called from there. 28THRESHOLD_FOR_NUMBER_OF_FUNCTIONS=2 29 30# Threshold values for different sanitizers used by instrumentation checks. 31ASAN_CALLS_THRESHOLD_FOR_ASAN_BUILD=1000 32ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD=0 33 34# The value below can definitely be higher (like 500-1000), but avoid being too 35# agressive here while still evaluating the DFT-based fuzzing approach. 36DFSAN_CALLS_THRESHOLD_FOR_DFSAN_BUILD=100 37DFSAN_CALLS_THRESHOLD_FOR_NON_DFSAN_BUILD=0 38 39MSAN_CALLS_THRESHOLD_FOR_MSAN_BUILD=1000 40# Some engines (e.g. honggfuzz) may make a very small number of calls to msan 41# for memory poisoning. 42MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD=2 43 44# Usually, a non UBSan build (e.g. ASan) has 165 calls to UBSan runtime. The 45# majority of targets built with UBSan have 200+ UBSan calls, but there are 46# some very small targets that may have < 200 UBSan calls even in a UBSan build. 47# Use the threshold value of 169 (slightly > 165) for UBSan build. 48UBSAN_CALLS_THRESHOLD_FOR_UBSAN_BUILD=169 49 50# It would be risky to use the threshold value close to 165 for non UBSan build, 51# as UBSan runtime may change any time and thus we could have different number 52# of calls to UBSan runtime even in ASan build. With that, we use the threshold 53# value of 200 that would detect unnecessary UBSan instrumentation in the vast 54# majority of targets, except of a handful very small ones, which would not be 55# a big concern either way as the overhead for them would not be significant. 56UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD=200 57 58# ASan builds on i386 generally have about 250 UBSan runtime calls. 59if [[ $ARCHITECTURE == 'i386' ]] 60then 61 UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD=280 62fi 63 64 65# Verify that the given fuzz target is correctly built to run with a particular 66# engine. 67function check_engine { 68 local FUZZER=$1 69 local FUZZER_NAME=$(basename $FUZZER) 70 local FUZZER_OUTPUT="/tmp/$FUZZER_NAME.output" 71 local CHECK_FAILED=0 72 73 if [[ "$FUZZING_ENGINE" == libfuzzer ]]; then 74 # Store fuzz target's output into a temp file to be used for further checks. 75 $FUZZER -seed=1337 -runs=$MIN_NUMBER_OF_RUNS &>$FUZZER_OUTPUT 76 CHECK_FAILED=$(egrep "ERROR: no interesting inputs were found. Is the code instrumented" -c $FUZZER_OUTPUT) 77 if (( $CHECK_FAILED > 0 )); then 78 echo "BAD BUILD: $FUZZER does not seem to have coverage instrumentation." 79 cat $FUZZER_OUTPUT 80 # Bail out as the further check does not make any sense, there are 0 PCs. 81 return 1 82 fi 83 84 local NUMBER_OF_EDGES=$(grep -Po "INFO: Loaded [[:digit:]]+ module.*\(.*(counters|guards)\):[[:space:]]+\K[[:digit:]]+" $FUZZER_OUTPUT) 85 86 # If a fuzz target fails to start, grep won't find anything, so bail out early to let check_startup_crash deal with it. 87 [[ -z "$NUMBER_OF_EDGES" ]] && return 88 89 if (( $NUMBER_OF_EDGES < $THRESHOLD_FOR_NUMBER_OF_EDGES )); then 90 echo "BAD BUILD: $FUZZER seems to have only partial coverage instrumentation." 91 fi 92 elif [[ "$FUZZING_ENGINE" == afl ]]; then 93 # TODO(https://github.com/google/oss-fuzz/issues/2470): Dont use 94 # AFL_DRIVER_DONT_DEFER by default, support .options files in 95 # bad_build_check instead. 96 AFL_DRIVER_DONT_DEFER=1 AFL_NO_UI=1 SKIP_SEED_CORPUS=1 timeout --preserve-status -s INT 20s run_fuzzer $FUZZER_NAME &>$FUZZER_OUTPUT 97 CHECK_PASSED=$(egrep "All set and ready to roll" -c $FUZZER_OUTPUT) 98 if (( $CHECK_PASSED == 0 )); then 99 echo "BAD BUILD: fuzzing $FUZZER with afl-fuzz failed." 100 cat $FUZZER_OUTPUT 101 return 1 102 fi 103 elif [[ "$FUZZING_ENGINE" == honggfuzz ]]; then 104 SKIP_SEED_CORPUS=1 timeout --preserve-status -s INT 20s run_fuzzer $FUZZER_NAME &>$FUZZER_OUTPUT 105 CHECK_PASSED=$(egrep "^Sz:[0-9]+ Tm:[0-9]+" -c $FUZZER_OUTPUT) 106 if (( $CHECK_PASSED == 0 )); then 107 echo "BAD BUILD: fuzzing $FUZZER with honggfuzz failed." 108 cat $FUZZER_OUTPUT 109 return 1 110 fi 111 elif [[ "$FUZZING_ENGINE" == dataflow ]]; then 112 $FUZZER &> $FUZZER_OUTPUT 113 local NUMBER_OF_FUNCTIONS=$(grep -Po "INFO:\s+\K[[:digit:]]+(?=\s+instrumented function.*)" $FUZZER_OUTPUT) 114 [[ -z "$NUMBER_OF_FUNCTIONS" ]] && NUMBER_OF_FUNCTIONS=0 115 if (( $NUMBER_OF_FUNCTIONS < $THRESHOLD_FOR_NUMBER_OF_FUNCTIONS )); then 116 echo "BAD BUILD: $FUZZER does not seem to be properly built in 'dataflow' config." 117 cat $FUZZER_OUTPUT 118 return 1 119 fi 120 return 0 121 fi 122 123 # TODO: add checks for other fuzzing engines if possible. 124 return 0 125} 126 127# Verify that the given fuzz target has been built properly and works. 128function check_startup_crash { 129 local FUZZER=$1 130 local FUZZER_NAME=$(basename $FUZZER) 131 local FUZZER_OUTPUT="/tmp/$FUZZER_NAME.output" 132 local CHECK_PASSED=0 133 134 if [[ "$FUZZING_ENGINE" = libfuzzer ]]; then 135 # Skip seed corpus as there is another explicit check that uses seed corpora. 136 SKIP_SEED_CORPUS=1 run_fuzzer $FUZZER_NAME -seed=1337 -runs=$MIN_NUMBER_OF_RUNS &>$FUZZER_OUTPUT 137 CHECK_PASSED=$(egrep "Done $MIN_NUMBER_OF_RUNS runs" -c $FUZZER_OUTPUT) 138 elif [[ "$FUZZING_ENGINE" = afl ]]; then 139 # TODO(https://github.com/google/oss-fuzz/issues/2470): Dont use 140 # AFL_DRIVER_DONT_DEFER by default, support .options files in 141 # bad_build_check instead. 142 AFL_DRIVER_DONT_DEFER=1 AFL_NO_UI=1 SKIP_SEED_CORPUS=1 timeout --preserve-status -s INT 20s run_fuzzer $FUZZER_NAME &>$FUZZER_OUTPUT 143 if [ $(egrep "target binary (crashed|terminated)" -c $FUZZER_OUTPUT) -eq 0 ]; then 144 CHECK_PASSED=1 145 fi 146 elif [[ "$FUZZING_ENGINE" = dataflow ]]; then 147 # TODO(https://github.com/google/oss-fuzz/issues/1632): add check for 148 # binaries compiled with dataflow engine when the interface becomes stable. 149 CHECK_PASSED=1 150 else 151 # TODO: add checks for another fuzzing engines if possible. 152 CHECK_PASSED=1 153 fi 154 155 if [ "$CHECK_PASSED" -eq "0" ]; then 156 echo "BAD BUILD: $FUZZER seems to have either startup crash or exit:" 157 cat $FUZZER_OUTPUT 158 return 1 159 fi 160 161 return 0 162} 163 164# Mixed sanitizers check for ASan build. 165function check_asan_build { 166 local FUZZER=$1 167 local ASAN_CALLS=$2 168 local DFSAN_CALLS=$3 169 local MSAN_CALLS=$4 170 local UBSAN_CALLS=$5 171 172 # Perform all the checks for more detailed error message. 173 if (( $ASAN_CALLS < $ASAN_CALLS_THRESHOLD_FOR_ASAN_BUILD )); then 174 echo "BAD BUILD: $FUZZER does not seem to be compiled with ASan." 175 return 1 176 fi 177 178 if (( $DFSAN_CALLS > $DFSAN_CALLS_THRESHOLD_FOR_NON_DFSAN_BUILD )); then 179 echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with DFSan." 180 return 1 181 fi 182 183 if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then 184 echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with MSan." 185 return 1 186 fi 187 188 if (( $UBSAN_CALLS > $UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD )); then 189 echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with UBSan." 190 return 1 191 fi 192 193 return 0 194} 195 196# Mixed sanitizers check for DFSan build. 197function check_dfsan_build { 198 local FUZZER=$1 199 local ASAN_CALLS=$2 200 local DFSAN_CALLS=$3 201 local MSAN_CALLS=$4 202 local UBSAN_CALLS=$5 203 204 # Perform all the checks for more detailed error message. 205 if (( $ASAN_CALLS > $ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD )); then 206 echo "BAD BUILD: DFSan build of $FUZZER seems to be compiled with ASan." 207 return 1 208 fi 209 210 if (( $DFSAN_CALLS < $DFSAN_CALLS_THRESHOLD_FOR_DFSAN_BUILD )); then 211 echo "BAD BUILD: $FUZZER does not seem to be compiled with DFSan." 212 return 1 213 fi 214 215 if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then 216 echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with MSan." 217 return 1 218 fi 219 220 if (( $UBSAN_CALLS > $UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD )); then 221 echo "BAD BUILD: ASan build of $FUZZER seems to be compiled with UBSan." 222 return 1 223 fi 224 225 return 0 226} 227 228 229# Mixed sanitizers check for MSan build. 230function check_msan_build { 231 local FUZZER=$1 232 local ASAN_CALLS=$2 233 local DFSAN_CALLS=$3 234 local MSAN_CALLS=$4 235 local UBSAN_CALLS=$5 236 237 # Perform all the checks for more detailed error message. 238 if (( $ASAN_CALLS > $ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD )); then 239 echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with ASan." 240 return 1 241 fi 242 243 if (( $DFSAN_CALLS > $DFSAN_CALLS_THRESHOLD_FOR_NON_DFSAN_BUILD )); then 244 echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with DFSan." 245 return 1 246 fi 247 248 if (( $MSAN_CALLS < $MSAN_CALLS_THRESHOLD_FOR_MSAN_BUILD )); then 249 echo "BAD BUILD: $FUZZER does not seem to be compiled with MSan." 250 return 1 251 fi 252 253 if (( $UBSAN_CALLS > $UBSAN_CALLS_THRESHOLD_FOR_NON_UBSAN_BUILD )); then 254 echo "BAD BUILD: MSan build of $FUZZER seems to be compiled with UBSan." 255 return 1 256 fi 257 258 return 0 259} 260 261# Mixed sanitizers check for UBSan build. 262function check_ubsan_build { 263 local FUZZER=$1 264 local ASAN_CALLS=$2 265 local DFSAN_CALLS=$3 266 local MSAN_CALLS=$4 267 local UBSAN_CALLS=$5 268 269 if [[ "$FUZZING_ENGINE" != libfuzzer ]]; then 270 # Ignore UBSan checks for fuzzing engines other than libFuzzer because: 271 # A) we (probably) are not going to use those with UBSan 272 # B) such builds show indistinguishable number of calls to UBSan 273 return 0 274 fi 275 276 # Perform all the checks for more detailed error message. 277 if (( $ASAN_CALLS > $ASAN_CALLS_THRESHOLD_FOR_NON_ASAN_BUILD )); then 278 echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with ASan." 279 return 1 280 fi 281 282 if (( $DFSAN_CALLS > $DFSAN_CALLS_THRESHOLD_FOR_NON_DFSAN_BUILD )); then 283 echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with DFSan." 284 return 1 285 fi 286 287 if (( $MSAN_CALLS > $MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD )); then 288 echo "BAD BUILD: UBSan build of $FUZZER seems to be compiled with MSan." 289 return 1 290 fi 291 292 if (( $UBSAN_CALLS < $UBSAN_CALLS_THRESHOLD_FOR_UBSAN_BUILD )); then 293 echo "BAD BUILD: $FUZZER does not seem to be compiled with UBSan." 294 return 1 295 fi 296} 297 298# Verify that the given fuzz target is compiled with correct sanitizer. 299function check_mixed_sanitizers { 300 local FUZZER=$1 301 local result=0 302 local CALL_INSN= 303 304 if [ "${FUZZING_LANGUAGE:-}" = "jvm" ]; then 305 # Sanitizer runtime is linked into the Jazzer driver, so this check does not 306 # apply. 307 return 0 308 fi 309 310 if [ "${FUZZING_LANGUAGE:-}" = "python" ]; then 311 # Sanitizer runtime is loaded via LD_PRELOAD, so this check does not apply. 312 return 0 313 fi 314 315 if [[ $ARCHITECTURE == 'i386' ]] 316 then 317 CALL_INSN="call\s+[0-9a-f]+\s+<" 318 else 319 case $(uname -m) in 320 x86_64) 321 CALL_INSN="callq?\s+[0-9a-f]+\s+<" 322 ;; 323 aarch64) 324 CALL_INSN="bl\s+[0-9a-f]+\s+<" 325 ;; 326 *) 327 echo "Error: unsupported machine hardware $(uname -m)" 328 exit 1 329 ;; 330 esac 331 fi 332 local ASAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__asan" -c) 333 local DFSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__dfsan" -c) 334 local MSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__msan" -c) 335 local UBSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__ubsan" -c) 336 337 338 if [[ "$SANITIZER" = address ]]; then 339 check_asan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS 340 result=$? 341 elif [[ "$SANITIZER" = dataflow ]]; then 342 check_dfsan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS 343 result=$? 344 elif [[ "$SANITIZER" = memory ]]; then 345 check_msan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS 346 result=$? 347 elif [[ "$SANITIZER" = undefined ]]; then 348 check_ubsan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS 349 result=$? 350 elif [[ "$SANITIZER" = thread ]]; then 351 # TODO(metzman): Implement this. 352 result=0 353 fi 354 355 return $result 356} 357 358# Verify that the given fuzz target doesn't crash on the seed corpus. 359function check_seed_corpus { 360 local FUZZER=$1 361 local FUZZER_NAME="$(basename $FUZZER)" 362 local FUZZER_OUTPUT="/tmp/$FUZZER_NAME.output" 363 364 if [[ "$FUZZING_ENGINE" != libfuzzer ]]; then 365 return 0 366 fi 367 368 # Set up common fuzzing arguments, otherwise "run_fuzzer" errors out. 369 if [ -z "$FUZZER_ARGS" ]; then 370 export FUZZER_ARGS="-rss_limit_mb=2560 -timeout=25" 371 fi 372 373 bash -c "run_fuzzer $FUZZER_NAME -runs=0" &> $FUZZER_OUTPUT 374 375 # Don't output anything if fuzz target hasn't crashed. 376 if [ $? -ne 0 ]; then 377 echo "BAD BUILD: $FUZZER has a crashing input in its seed corpus:" 378 cat $FUZZER_OUTPUT 379 return 1 380 fi 381 382 return 0 383} 384 385function check_architecture { 386 local FUZZER=$1 387 local FUZZER_NAME=$(basename $FUZZER) 388 389 if [ "${FUZZING_LANGUAGE:-}" = "jvm" ]; then 390 # The native dependencies of a JVM project are not packaged, but loaded 391 # dynamically at runtime and thus cannot be checked here. 392 return 0; 393 fi 394 395 if [ "${FUZZING_LANGUAGE:-}" = "python" ]; then 396 FUZZER=${FUZZER}.pkg 397 fi 398 399 FILE_OUTPUT=$(file $FUZZER) 400 if [[ $ARCHITECTURE == "x86_64" ]] 401 then 402 echo $FILE_OUTPUT | grep "x86-64" > /dev/null 403 elif [[ $ARCHITECTURE == "i386" ]] 404 then 405 echo $FILE_OUTPUT | grep "80386" > /dev/null 406 else 407 echo "UNSUPPORTED ARCHITECTURE" 408 return 1 409 fi 410 result=$? 411 if [[ $result != 0 ]] 412 then 413 echo "BAD BUILD $FUZZER is not built for architecture: $ARCHITECTURE" 414 echo "file command output: $FILE_OUTPUT" 415 echo "check_mixed_sanitizers test will fail." 416 fi 417 return $result 418} 419 420function main { 421 local FUZZER=$1 422 local checks_failed=0 423 local result=0 424 425 export RUN_FUZZER_MODE="batch" 426 check_engine $FUZZER 427 result=$? 428 checks_failed=$(( $checks_failed + $result )) 429 430 check_architecture $FUZZER 431 result=$? 432 checks_failed=$(( $checks_failed + $result )) 433 434 check_mixed_sanitizers $FUZZER 435 result=$? 436 checks_failed=$(( $checks_failed + $result )) 437 438 check_startup_crash $FUZZER 439 result=$? 440 checks_failed=$(( $checks_failed + $result )) 441 442 # TODO: re-enable after introducing bug auto-filing for bad builds. 443 # check_seed_corpus $FUZZER 444 return $checks_failed 445} 446 447 448if [ $# -ne 1 ]; then 449 echo "Usage: $0 <fuzz_target_binary>" 450 exit 1 451fi 452 453# Fuzz target path. 454FUZZER=$1 455 456main $FUZZER 457exit $? 458