1#!/bin/bash
2# //===--------------------------- testit ---------------------------------===//
3# //
4# //                     The LLVM Compiler Infrastructure
5# //
6# // This file is distributed under the University of Illinois Open Source
7# // License. See LICENSE.TXT for details.
8# //
9# //===--------------------------------------------------------------------===//
10
11currentpath=`pwd`
12origpath=$currentpath
13currentdir=`basename $currentpath`
14while [ $currentdir != "test" ]; do
15	if [ $currentdir = "/" ]
16	then
17		echo "current directory must be in or under \"test\"."
18		exit 1
19	fi
20	cd ..
21	currentpath=`pwd`
22	currentdir=`basename $currentpath`
23done
24
25cd ..
26LIBCXX_ROOT=`pwd`/../../llvm-libc++/libcxx
27cd $origpath
28
29VERBOSE=1
30
31run () {
32  if [ "$VERBOSE" -gt 1 ]; then
33    echo "COMMAND: $@"
34  fi
35  case $VERBOSE in
36    0|1)
37      # Hide command output and errors.
38      "$@" >/dev/null 2>&1
39      ;;
40    2)
41      # Only hide command output
42      "$@" >/dev/null
43      ;;
44    *)
45      # Show command output and errors.
46      "$@"
47      ;;
48  esac
49}
50
51run2 () {
52  if [ "$VERBOSE" -gt 2 ]; then
53    echo "COMMAND: $@"
54  fi
55  case $VERBOSE in
56    0|1)
57      # Hide command output and errors.
58      "$@" >/dev/null 2>&1
59      ;;
60    2)
61      # Only hide command output
62      "$@" >/dev/null
63      ;;
64    *)
65      # Show command output and errors.
66      "$@"
67      ;;
68  esac
69}
70
71# The list of valid target abis supported by this script.
72VALID_ABIS="armeabi armeabi-v7a armeabi-v7a-hard x86 mips"
73
74DO_HELP=
75DO_STATIC=
76TARGET_ABI=
77TARGET_ARCH=
78if [ -n "$SHARD_TEST_TOOLCHAIN" ]; then
79  TARGET_PATH=/data/local/tmp/libcxx-${SHARD_TEST_TOOLCHAIN}-$(dirname $PWD | sum | cut -d' ' -f1)-$(basename $PWD)
80else
81  TARGET_PATH=/data/local/tmp/libcxx
82fi
83CXX=
84WITH_COMPILER_RT=
85OPTIMIZE=
86for OPT; do
87  case $OPT in
88    --help|-?)
89      DO_HELP=true
90      ;;
91    --abi=*)
92      TARGET_ABI=${OPT##--abi=}
93      ;;
94    --static)
95      DO_STATIC=true
96      ;;
97    --shared)
98      DO_STATIC=
99      ;;
100    --cxx=*)
101      CXX=${OPT##--cxx=}
102      ;;
103    --verbose)
104      VERBOSE=$(( $VERBOSE + 1 ))
105      ;;
106    --with-compiler-rt)
107      WITH_COMPILER_RT=yes
108      ;;
109    -O*)
110      OPTIMIZE=$OPT
111      ;;
112    -*)
113      echo "Unknown option: $OPT. See --help."
114      exit 1
115      ;;
116    *)
117      echo "This script doesn't take parameters. See --help."
118      exit 1
119      ;;
120  esac
121done
122
123if [ "$DO_HELP" ]; then
124  echo \
125"Usage: $(basename $0) [options]
126
127This script is used to run the libc++ test suite for Android.
128You will need the following things:
129
130  - The prebuild libc++ libraries in your NDK install.
131  - A prebuilt Android toolchain in your path.
132  - The 'adb' tool in your path.
133  - An Android device connected to ADB.
134
135The toolchain and device must match your target ABI. For example, if
136you use --abi=armeabi-v7a, your device must run ARMv7-A Android binaries,
137and arm-linux-androideabi-g++ will be used to compile all tests, unless
138you use --cxx=<command> to override it.
139
140Valid options:
141  --help|-?        Display this message.
142  --abi=<name>     Specify target ABI. Use --abi=list for list.
143  --static         Link against static libc++ library.
144  --cxx=<program>  Override C++ compiler/linker.
145  --verbose        Increase verbosity.
146"
147  exit 0
148fi
149
150# Check target ABI.
151if [ "$TARGET_ABI" = "list" ]; then
152  echo "List of valid target ABIs:"
153  for ABI in $VALID_ABIS; do
154    printf " %s" $ABI
155  done
156  printf "\n"
157  exit 0
158fi
159
160if [ -z "$TARGET_ABI" ]; then
161  echo "ERROR: Please specify a target ABI (--abi=<name>)."
162  exit 1
163fi
164
165FOUND_ABI=
166for ABI in $VALID_ABIS; do
167  if [ "$ABI" = "$TARGET_ABI" ]; then
168    FOUND_ABI=true
169    break
170  fi
171done
172
173if [ -z "$FOUND_ABI" ]; then
174  echo "ERROR: Invalid abi '$TARGET_ABI'. Must be one of: $VALID_ABIS"
175  exit 1
176fi
177
178LIBCXX_LIBS=$(cd $LIBCXX_ROOT/.. && pwd)/libs/$TARGET_ABI
179for LIB in libc++_static.a libc++_shared.so; do
180  if [ ! -f "$LIBCXX_LIBS/$LIB" ]; then
181    echo "ERROR: Missing prebuilt library: $LIBCXX_LIBS/$LIB"
182    echo "Please run: build/tools/build-cxx-stl.sh --stl=libc++"
183    exit 1
184  fi
185done
186
187LIBCOMPILER_RT_LIBS=$(cd "$LIBCXX_ROOT"/../../../android/compiler-rt && pwd)/libs/$TARGET_ABI
188if [ "$WITH_COMPILER_RT" = "yes" ]; then
189  for LIB in libcompiler_rt_static.a libcompiler_rt_shared.so; do
190    if [ ! -f "$LIBCOMPILER_RT_LIBS/$LIB" ]; then
191      echo "ERROR: Missing prebuilt library: $LIBCOMPILER_RT_LIBS/$LIB"
192      echo "Please run: build/tools/build-compiler-rt.sh --ndk-dir=$NDK \
193--src-dir=/tmp/ndk-$USER/src/llvm-3.4/compiler-rt --llvm-version=3.4"
194      exit 1
195    fi
196  done
197fi
198
199# Check or detect C++ toolchain.
200TOOLCHAIN_CFLAGS=
201TOOLCHAIN_LDFLAGS=
202THUMB_MODE="-mthumb"
203LIBM="-lm"
204if [ -z "$TOOLCHAIN_PREFIX" ]; then
205  # Compute
206  case $TARGET_ABI in
207    armeabi)
208      TOOLCHAIN_PREFIX=arm-linux-androideabi
209      TOOLCHAIN_CFLAGS="$THUMB_MODE"
210      TOOLCHAIN_LDFLAGS="$THUMB_MODE"
211      ;;
212    armeabi-v7a)
213      TOOLCHAIN_PREFIX=arm-linux-androideabi
214      TOOLCHAIN_CFLAGS="-march=armv7-a $THUMB_MODE -mfpu=vfpv3-d16"
215      TOOLCHAIN_LDFLAGS="-march=armv7-a $THUMB_MODE -Wl,--fix-cortex-a8"
216      ;;
217    armeabi-v7a-hard)
218      TOOLCHAIN_PREFIX=arm-linux-androideabi
219      TOOLCHAIN_CFLAGS="-march=armv7-a $THUMB_MODE -mfpu=vfpv3-d16 -mhard-float -D_NDK_MATH_NO_SOFTFP=1"
220      TOOLCHAIN_LDFLAGS="-march=armv7-a $THUMB_MODE -Wl,--fix-cortex-a8 -Wl,--no-warn-mismatch"
221      LIBM="-lm_hard"
222      ;;
223    x86)
224      TOOLCHAIN_PREFIX=i686-linux-android
225      ;;
226    mips)
227      TOOLCHAIN_PREFIX=mipsel-linux-android
228      ;;
229    *)
230      echo "ERROR: Unknown ABI '$ABI'"
231      exit 1
232      ;;
233  esac
234  if [ -z "$CXX" ]; then
235      CXX=$TOOLCHAIN_PREFIX-g++
236  fi
237fi
238
239REAL_CXX=$(which "$CXX" 2>/dev/null)
240if [ -z "$REAL_CXX" ]; then
241  echo "ERROR: Missing C++ compiler: $CXX"
242  exit 1
243fi
244CC=$CXX
245
246function version_ge {
247    input_string=$1
248    compare_string=$2
249    input_major=$(echo $input_string | cut -d\. -f 1)
250    input_minor=$(echo $input_string | cut -d\. -f 2)
251    compare_major=$(echo $compare_string | cut -d\. -f 1)
252    compare_minor=$(echo $compare_string | cut -d\. -f 2)
253    true=0
254    false=1
255    if [ "$input_major" -gt "$compare_major" ]; then return $true; fi
256    if [ "$input_major" -lt "$compare_major" ]; then return $false; fi
257    if [ "$input_minor" -ge "$compare_minor" ]; then return $true; fi
258    return $false
259}
260
261if [ -z "$OPTIONS" ]
262then
263  OPTIONS="-std=c++11 -g -funwind-tables $OPTIMIZE"
264  # Note that some tests use assert() to check condition but -O2 defines assert() to nothing,
265  # unless we specify -UNDEBUG to bring assert() back.
266  # But currently adding -UNDEBUG crashes both clang3.4 and clang3.3 for test
267  # like libcxx/test/atomics/atomics.types.generic/address.pass.cpp.  Define -UNDEBUG
268  # only when we are not using clang.  ToDo
269  if [ "$CXX" = "${CXX%%*clang++*}" ] ; then
270    OPTIONS="$OPTIONS -UNDEBUG"
271  fi
272  if [ -n "$LLVM_VERSION" ]; then
273    if version_ge "$LLVM_VERSION" "3.4"; then
274      OPTIONS="${OPTIONS} -mllvm -arm-enable-ehabi-descriptors -mllvm -arm-enable-ehabi"
275    fi
276  fi
277fi
278OPTIONS="$OPTIONS $TOOLCHAIN_CFLAGS $TOOLCHAIN_LDFLAGS"
279OPTIONS="$OPTIONS -I$LIBCXX_ROOT/test/support"
280# llvm-libc++/libcxx/test/lit.cfg line #278 defineds the following for testing only on Linux
281OPTIONS="$OPTIONS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS"
282
283if [ -z "$ADB" ]
284then
285  ADB=adb
286fi
287
288# Run a shell command through ADB, return its status.
289# Variable ERR contains output if $RET is non-zero
290adb_shell () {
291  # We need a temporary file to store the output of our command
292  local CMD_OUT RET
293  ERR=
294  CMD_OUT=$(mktemp /tmp/testit_android-cmdout-XXXXXX)
295  # Run the command, while storing the standard output to CMD_OUT
296  # and appending the exit code as the last line.
297  if [ "$VERBOSE" -gt 2 ]; then
298    echo "COMMAND: $ADB shell $@"
299  fi
300  $ADB shell "$@ ; echo \$?" | sed -e 's![[:cntrl:]]!!g' > $CMD_OUT 2>&1
301  # Get last line in log, which contains the exit code from the command
302  RET=$(sed -e '$!d' $CMD_OUT)
303  # Get output, which corresponds to everything except the last line
304  OUT=$(sed -e '$d' $CMD_OUT)
305  if [ "$RET" != "0" ]; then
306    ERR=$OUT
307  fi
308  if [ "$VERBOSE" -gt 2 ]; then
309    printf "%s" "$OUT"
310  fi
311  rm -f $CMD_OUT
312  return $RET
313}
314
315# Push a given file through ADB.
316# $1: File path
317adb_push () {
318  local FILE=$1
319  local FILE_BASENAME=$(basename "$FILE")
320  run2 $ADB push $FILE $TARGET_PATH/$FILE_BASENAME 2>/dev/null
321}
322
323# Run a given executable through ADB.
324# $1: Executable path
325adb_run () {
326  local EXECUTABLE=$1
327  local EXECUTABLE_BASENAME=$(basename "$EXECUTABLE")
328  run2 $ADB push $EXECUTABLE $TARGET_PATH/$EXECUTABLE_BASENAME 2>/dev/null
329  if [ "$?" != 0 ]; then
330    return 1;
331  fi
332  # Retry up to 10 times if fail is due to "Text file busy"
333  for i in 1 2 3 4 5 6 7 8 9 10; do
334    adb_shell "cd $TARGET_PATH; LD_LIBRARY_PATH=$TARGET_PATH LIBUNWIND_PRINT_UNWINDING=1 ./$EXECUTABLE_BASENAME"
335    if [ "$?" = "0" ]; then
336      return 0
337    fi
338    if ! $(echo $ERR | grep -iq "Text file busy"); then
339      if [ "$i" != "1" ]; then
340        # Dump error message to help diagnostics
341        echo "ERR=$ERR"
342      fi
343      break;
344    fi
345    echo "Text file busy.  Re-try $i"
346    sleep 1
347    run2 $ADB push $EXECUTABLE $TARGET_PATH/$EXECUTABLE_BASENAME 2>/dev/null
348    sleep 2  # try again
349  done
350  return 1
351}
352
353adb_shell "rm -rf $TARGET_PATH"
354adb_shell "mkdir -p $TARGET_PATH"
355
356if [ "$DO_STATIC" ]; then
357  # Statically link to ensure the executable can be run easily through ADB
358  if [ "$WITH_COMPILER_RT" = "yes" ]; then
359    LIBS="-nodefaultlibs -lc++_static -latomic -ldl $LIBM -lc -lcompiler_rt_static"
360  else
361    LIBS="-nodefaultlibs -latomic -ldl $LIBM -lc"
362  fi
363else
364  run2 $ADB push $LIBCXX_LIBS/libc++_shared.so $TARGET_PATH 2>/dev/null
365  if [ $? != 0 ]; then
366    echo "ERROR: Can't push shared libc++ to target device!"
367    exit 1
368  fi
369  if [ "$WITH_COMPILER_RT" = "yes" ]; then
370    run2 $ADB push $LIBCOMPILER_RT_LIBS/libcompiler_rt_shared.so $TARGET_PATH 2>/dev/null
371    if [ $? != 0 ]; then
372      echo "ERROR: Can't push shared libcompiler_rt to target device!"
373      exit 1
374    fi
375    LIBS="-nodefaultlibs -lc++_shared -latomic -ldl $LIBM -lc -lcompiler_rt_shared"
376  else
377    LIBS="-nodefaultlibs -lc++_shared -latomic -ldl $LIBM -lc"
378  fi
379fi
380
381case $TRIPLE in
382  *-*-mingw* | *-*-cygwin* | *-*-win*)
383    TEST_EXE=test.exe
384    ;;
385  *)
386    TEST_EXE=a.out
387    ;;
388esac
389
390TEST_EXE=/tmp/testit_android-$USER-$$-$TEST_EXE
391
392FAIL=0
393PASS=0
394UNIMPLEMENTED=0
395IMPLEMENTED_FAIL=0
396IMPLEMENTED_PASS=0
397
398# Run tests in current directory, recursively
399#
400# Note that file path containing EQ are symlink to the existing tests whose path contain '=',
401# to workaround an issue in ndk-build which doesn't handle LOCAL_SRC_FILES with '='.
402# See tests/device/test-libc++-static-full/jni/Android.mk  We need to filter out path containing
403# EQ such that we don't run same tests twice
404#
405# An alternative is to do "find . -type f", but this doesn't work in NDK windows package
406# where zip turns symlink into physical file it points to.
407#
408# We also sort the test to make the test report comparable to previous test
409#
410
411afunc() {
412	fail=0
413	pass=0
414	if (ls ${TEST_PREFIX}*fail.cpp > /dev/null 2>&1)
415	then
416		for FILE in $(ls ${TEST_PREFIX}*fail.cpp | tr ' ' '\n' | grep -v EQ | sort); do
417			if run $CC $OPTIONS $HEADER_INCLUDE $SOURCE_LIB $FILE $LIBS -o $TEST_EXE > /dev/null 2>&1
418			then
419				rm $TEST_EXE
420				echo "$FILE should not compile"
421				fail=$(($fail+1))
422			else
423				pass=$(($pass+1))
424			fi
425		done
426	fi
427
428	if (ls ${TEST_PREFIX}*.cpp > /dev/null 2>&1)
429	then
430		if (ls *.dat > /dev/null 2>&1)
431		then
432			adb_shell "rm -f $TARGET_PATH/*.dat"
433			for FILE in $(ls *.dat | tr ' ' '\n' | grep -v EQ | sort); do
434	                      if [ "$VERBOSE" -gt 1 ]; then
435	                          echo "Pushing data: " $FILE
436			      fi
437			      adb_push $FILE
438			      if [ $? != 0 ]; then
439				  echo "Failed to push file $FILE"
440	                      fi
441			done
442		fi
443		for FILE in $(ls ${TEST_PREFIX}*.cpp | tr ' ' '\n' | grep -v EQ | sort); do
444                      if [ "$VERBOSE" -gt 1 ]; then
445                          echo "Running test: " $FILE
446                      fi
447                        COMMAND="( cd $(pwd) && $CC $OPTIONS $HEADER_INCLUDE $SOURCE_LIB $FILE $LIBS )"
448			if run $CC $OPTIONS $HEADER_INCLUDE $SOURCE_LIB $FILE $LIBS -o $TEST_EXE
449			then
450				if adb_run $TEST_EXE
451				then
452					rm $TEST_EXE
453					pass=$(($pass+1))
454				else
455					echo "`pwd`/$FILE failed at run time"
456					echo "Compile line was: $COMMAND # run-time"
457					fail=$(($fail+1))
458					rm $TEST_EXE
459				fi
460			else
461				echo "`pwd`/$FILE failed to compile"
462				echo "Compile line was: $COMMAND # compile-time"
463				fail=$(($fail+1))
464			fi
465		done
466	fi
467
468	if [ $fail -gt 0 ]
469	then
470		echo "failed $fail tests in `pwd`"
471		IMPLEMENTED_FAIL=$(($IMPLEMENTED_FAIL+1))
472	fi
473	if [ $pass -gt 0 ]
474	then
475		echo "passed $pass tests in `pwd`"
476		if [ $fail -eq 0 ]
477		then
478			IMPLEMENTED_PASS=$((IMPLEMENTED_PASS+1))
479		fi
480	fi
481	if [ $fail -eq 0 -a $pass -eq 0 ]
482	then
483		echo "not implemented:  `pwd`"
484		UNIMPLEMENTED=$(($UNIMPLEMENTED+1))
485	fi
486
487	FAIL=$(($FAIL+$fail))
488	PASS=$(($PASS+$pass))
489
490	for FILE in $(ls | tr ' ' '\n' | grep -v EQ | sort)
491	do
492		if [ -d "$FILE" ];
493		then
494			cd $FILE
495			afunc
496			cd ..
497		fi
498	done
499}
500
501afunc
502
503echo "****************************************************"
504echo "Results for `pwd`:"
505echo "using `$CC --version`"
506echo "with $OPTIONS $HEADER_INCLUDE $SOURCE_LIB"
507echo "----------------------------------------------------"
508echo "sections without tests   : $UNIMPLEMENTED"
509echo "sections with failures   : $IMPLEMENTED_FAIL"
510echo "sections without failures: $IMPLEMENTED_PASS"
511echo "                       +   ----"
512echo "total number of sections : $(($UNIMPLEMENTED+$IMPLEMENTED_FAIL+$IMPLEMENTED_PASS))"
513echo "----------------------------------------------------"
514echo "number of tests failed   : $FAIL"
515echo "number of tests passed   : $PASS"
516echo "                       +   ----"
517echo "total number of tests    : $(($FAIL+$PASS))"
518echo "****************************************************"
519
520exit $FAIL
521