1#!/bin/bash
2#
3# Requires pip install autobahntestsuite
4#
5# you should run this from ./build, after building with
6# cmake .. -DLWS_WITH_MINIMAL_EXAMPLES=1
7#
8# It will use the minimal echo client and server to run
9# autobahn ws tests as both client and server.
10
11echo
12echo "----------------------------------------------"
13echo "-------   tests: autobahn as client"
14echo
15
16set -u
17
18PARALLEL=1
19N=1
20OS=`uname`
21
22CLIE=bin/lws-minimal-ws-client-echo
23SERV=bin/lws-minimal-ws-server-echo
24
25RESULT=0
26
27which wstest 2>/dev/null
28if [ $? -ne 0 ]; then
29	echo "wstest is not installed"
30	exit 8
31fi
32
33killall wstest 2>/dev/null
34
35#
36# 2.10 / 2.11:      There is no requirement to handle multiple PING / PONG
37#                   in flight in RFC6455.  lws doesn't waste memory on it
38#                   since it is useless.
39#
40# 12.3.1 / 12.3.2
41# 12.4.* / 12.5.*:  Autobahn has been broken for these tests since Aug 2017
42#                   https://github.com/crossbario/autobahn-testsuite/issues/71
43
44
45cat << EOF >fuzzingserver.json
46{
47   "url": "ws://127.0.0.1:9001",
48   "outdir": "./reports/clients",
49   "cases": ["*"],
50   "exclude-cases": [ "2.10", "2.11", "12.3.1", "12.3.2", "12.4.*", "12.5.*"],
51   "exclude-agent-cases": {}
52}
53EOF
54
55PYTHONHASHSEED=0 wstest -m fuzzingserver &
56Q=$!
57sleep 2s
58ps -p $Q > /dev/null
59if [ $? -ne 0 ] ; then
60	echo "Problem with autobahn wstest install"
61	exit 9
62fi
63
64# 1) lws-as-client tests first
65
66ok=1
67while [ $ok -eq 1 ] ; do
68		$CLIE -s 127.0.0.1 -p 9001 -u "/runCase?case=$N&agent=libwebsockets" -d3
69		if [ $? -ne 0 ]; then
70			ok=0
71		fi
72	N=$(( $N + 1 ))
73done
74
75# generate the report in ./reports
76#
77$CLIE -s 127.0.0.1 -p 9001 -u "/updateReports?agent=libwebsockets" -o -d3
78sleep 2s
79killall wstest
80sleep 1s
81
82# this squashes the results into single lines like
83#
84#  "9.8.4": { "behavior": "OK", "behaviorClose": "OK", "duration": 1312, "remoteCloseCode": 1000, "reportfile": "libwebsockets_case_9_8_4.json"
85
86cat reports/clients/index.json | tr '\n' '!' | sed "s|\},\!|\n|g" | tr '!' ' ' | tr -s ' ' > /tmp/ji
87
88echo -n "AUTOBAHN SERVER / LWS CLIENT: Total tests: " `cat /tmp/ji | wc -l` " : "
89R="`cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | wc -l`"
90if [ "$R" == "0" ] ; then
91	echo "All pass"
92else
93	RESULT=1
94	echo -n "$R FAIL : "
95	cat /tmp/ji | grep -v '"behavior": "OK"' | grep -v '"behavior": "NON-STRICT"' | grep -v '"behavior": "INFORMATIONAL"' | cut -d\" -f2 | tr '\n' ','
96	echo
97fi
98
99echo $RESULT
100exit $RESULT
101
102