Mercurial > dropbear

comparison libtommath/testme.sh @ 1739:13d834efc376 fuzz

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
merge from main
author Matt Johnston <matt@ucc.asn.au>
date Thu, 15 Oct 2020 19:55:15 +0800
parents 1051e4eea25a
children
comparison
equal deleted inserted replaced
1562:768ebf737aa0 1739:13d834efc376
2 # 2 #
3 # return values of this script are: 3 # return values of this script are:
4 # 0 success 4 # 0 success
5 # 128 a test failed 5 # 128 a test failed
6 # >0 the number of timed-out tests 6 # >0 the number of timed-out tests
7 # 255 parsing of parameters failed
7 8
8 set -e 9 set -e
9 10
10 if [ -f /proc/cpuinfo ] 11 if [ -f /proc/cpuinfo ]
11 then 12 then
19 20
20 _help() 21 _help()
21 { 22 {
22 echo "Usage options for $(basename $0) [--with-cc=arg [other options]]" 23 echo "Usage options for $(basename $0) [--with-cc=arg [other options]]"
23 echo 24 echo
24 echo "Executing this script without any parameter will only run the default configuration" 25 echo "Executing this script without any parameter will only run the default"
25 echo "that has automatically been determined for the architecture you're running." 26 echo "configuration that has automatically been determined for the"
27 echo "architecture you're running."
26 echo 28 echo
27 echo " --with-cc=* The compiler(s) to use for the tests" 29 echo " --with-cc=* The compiler(s) to use for the tests"
28 echo " This is an option that will be iterated." 30 echo " This is an option that will be iterated."
29 echo 31 echo
30 echo "To be able to specify options a compiler has to be given." 32 echo " --test-vs-mtest=* Run test vs. mtest for '*' operations."
31 echo "All options will be tested with all MP_xBIT configurations." 33 echo " Only the first of each options will be"
32 echo 34 echo " taken into account."
33 echo " --with-{m64,m32,mx32} The architecture(s) to build and test for," 35 echo
34 echo " e.g. --with-mx32." 36 echo "To be able to specify options a compiler has to be given with"
35 echo " This is an option that will be iterated, multiple selections are possible." 37 echo "the option --with-cc=compilername"
36 echo " The mx32 architecture is not supported by clang and will not be executed." 38 echo "All other options will be tested with all MP_xBIT configurations."
39 echo
40 echo " --with-{m64,m32,mx32} The architecture(s) to build and test"
41 echo " for, e.g. --with-mx32."
42 echo " This is an option that will be iterated,"
43 echo " multiple selections are possible."
44 echo " The mx32 architecture is not supported"
45 echo " by clang and will not be executed."
37 echo 46 echo
38 echo " --cflags=* Give an option to the compiler," 47 echo " --cflags=* Give an option to the compiler,"
39 echo " e.g. --cflags=-g" 48 echo " e.g. --cflags=-g"
40 echo " This is an option that will always be passed as parameter to CC." 49 echo " This is an option that will always be"
50 echo " passed as parameter to CC."
41 echo 51 echo
42 echo " --make-option=* Give an option to make," 52 echo " --make-option=* Give an option to make,"
43 echo " e.g. --make-option=\"-f makefile.shared\"" 53 echo " e.g. --make-option=\"-f makefile.shared\""
44 echo " This is an option that will always be passed as parameter to make." 54 echo " This is an option that will always be"
55 echo " passed as parameter to make."
45 echo 56 echo
46 echo " --with-low-mp Also build&run tests with -DMP_{8,16,32}BIT." 57 echo " --with-low-mp Also build&run tests with -DMP_{8,16,32}BIT."
47 echo 58 echo
59 echo " --mtest-real-rand Use real random data when running mtest."
60 echo
61 echo " --with-valgrind"
62 echo " --with-valgrind=* Run in valgrind (slow!)."
63 echo
64 echo " --with-travis-valgrind Run with valgrind on Travis on specific branches."
65 echo
66 echo " --valgrind-options Additional Valgrind options"
67 echo " Some of the options like e.g.:"
68 echo " --track-origins=yes add a lot of extra"
69 echo " runtime and may trigger the 30 minutes"
70 echo " timeout."
71 echo
48 echo "Godmode:" 72 echo "Godmode:"
49 echo 73 echo
50 echo " --all Choose all architectures and gcc and clang as compilers" 74 echo " --all Choose all architectures and gcc and clang"
51 echo 75 echo " as compilers but does not run valgrind."
76 echo
77 echo " --format Runs the various source-code formatters"
78 echo " and generators and checks if the sources"
79 echo " are clean."
80 echo
81 echo " -h"
52 echo " --help This message" 82 echo " --help This message"
83 echo
84 echo " -v"
85 echo " --version Prints the version. It is just the number"
86 echo " of git commits to this file, no deeper"
87 echo " meaning attached"
53 exit 0 88 exit 0
54 } 89 }
55 90
56 _die() 91 _die()
57 { 92 {
59 if [ "$2" != "124" ] 94 if [ "$2" != "124" ]
60 then 95 then
61 exit 128 96 exit 128
62 else 97 else
63 echo "assuming timeout while running test - continue" 98 echo "assuming timeout while running test - continue"
99 local _tail=""
100 which tail >/dev/null && _tail="tail -n 1 test_${suffix}.log" && \
101 echo "last line of test_"${suffix}".log was:" && $_tail && echo ""
64 ret=$(( $ret + 1 )) 102 ret=$(( $ret + 1 ))
65 fi 103 fi
66 } 104 }
67 105
68 _runtest() 106 _make()
69 { 107 {
70 echo -ne " Compile $1 $2" 108 echo -ne " Compile $1 $2"
71 make clean > /dev/null
72 suffix=$(echo ${1}${2} | tr ' ' '_') 109 suffix=$(echo ${1}${2} | tr ' ' '_')
73 CC="$1" CFLAGS="$2 $TEST_CFLAGS" make -j$MAKE_JOBS test_standalone $MAKE_OPTIONS > /dev/null 2>gcc_errors_${suffix}.txt 110 CC="$1" CFLAGS="$2 $TEST_CFLAGS" make -j$MAKE_JOBS $3 $MAKE_OPTIONS > /dev/null 2>gcc_errors_${suffix}.log
74 errcnt=$(wc -l < gcc_errors_${suffix}.txt) 111 errcnt=$(wc -l < gcc_errors_${suffix}.log)
75 if [[ ${errcnt} -gt 1 ]]; then 112 if [[ ${errcnt} -gt 1 ]]; then
76 echo " failed" 113 echo " failed"
77 cat gcc_errors_${suffix}.txt 114 cat gcc_errors_${suffix}.log
78 exit 128 115 exit 128
79 fi 116 fi
80 echo -e "\rRun test $1 $2" 117 }
118
119
120 _runtest()
121 {
122 make clean > /dev/null
81 local _timeout="" 123 local _timeout=""
82 which timeout >/dev/null && _timeout="timeout --foreground 90" 124 which timeout >/dev/null && _timeout="timeout --foreground 90"
83 $_timeout ./test > test_${suffix}.txt || _die "running tests" $? 125 if [[ "$MAKE_OPTIONS" =~ "tune" ]]
84 } 126 then
127 # "make tune" will run "tune_it.sh" automatically, hence "autotune", but it cannot
128 # get switched off without some effort, so we just let it run twice for testing purposes
129 echo -e "\rRun autotune $1 $2"
130 _make "$1" "$2" ""
131 $_timeout $TUNE_CMD > test_${suffix}.log || _die "running autotune" $?
132 else
133 _make "$1" "$2" "test"
134 echo -e "\rRun test $1 $2"
135 $_timeout ./test > test_${suffix}.log || _die "running tests" $?
136 fi
137 }
138
139 # This is not much more of a C&P of _runtest with a different timeout
140 # and the additional valgrind call.
141 # TODO: merge
142 _runvalgrind()
143 {
144 make clean > /dev/null
145 local _timeout=""
146 # 30 minutes? Yes. Had it at 20 minutes and the Valgrind run needed over 25 minutes.
147 # A bit too close for comfort.
148 which timeout >/dev/null && _timeout="timeout --foreground 1800"
149 echo "MAKE_OPTIONS = \"$MAKE_OPTIONS\""
150 if [[ "$MAKE_OPTIONS" =~ "tune" ]]
151 then
152 echo "autotune branch"
153 _make "$1" "$2" ""
154 # The shell used for /bin/sh is DASH 0.5.7-4ubuntu1 on the author's machine which fails valgrind, so
155 # we just run on instance of etc/tune with the same options as in etc/tune_it.sh
156 echo -e "\rRun etc/tune $1 $2 once inside valgrind"
157 $_timeout $VALGRIND_BIN $VALGRIND_OPTS $TUNE_CMD > test_${suffix}.log || _die "running etc/tune" $?
158 else
159 _make "$1" "$2" "test"
160 echo -e "\rRun test $1 $2 inside valgrind"
161 $_timeout $VALGRIND_BIN $VALGRIND_OPTS ./test > test_${suffix}.log || _die "running tests" $?
162 fi
163 }
164
85 165
86 _banner() 166 _banner()
87 { 167 {
88 echo "uname="$(uname -a) 168 echo "uname="$(uname -a)
89 [[ "$#" != "0" ]] && (echo $1=$($1 -dumpversion)) || true 169 [[ "$#" != "0" ]] && (echo $1=$($1 -dumpversion)) || true
103 183
104 ARCHFLAGS="" 184 ARCHFLAGS=""
105 COMPILERS="" 185 COMPILERS=""
106 CFLAGS="" 186 CFLAGS=""
107 WITH_LOW_MP="" 187 WITH_LOW_MP=""
188 TEST_VS_MTEST=""
189 MTEST_RAND=""
190 # timed with an AMD A8-6600K
191 # 25 minutes
192 #VALGRIND_OPTS=" --track-origins=yes --leak-check=full --show-leak-kinds=all --error-exitcode=1 "
193 # 9 minutes (14 minutes with --test-vs-mtest=333333 --mtest-real-rand)
194 VALGRIND_OPTS=" --leak-check=full --show-leak-kinds=all --error-exitcode=1 "
195 #VALGRIND_OPTS=""
196 VALGRIND_BIN=""
197 CHECK_FORMAT=""
198 TUNE_CMD="./etc/tune -t -r 10 -L 3"
199
200 alive_pid=0
201
202 function kill_alive() {
203 disown $alive_pid || true
204 kill $alive_pid 2>/dev/null
205 }
206
207 function start_alive_printing() {
208 [ "$alive_pid" == "0" ] || return 0;
209 for i in `seq 1 10` ; do sleep 300 && echo "Tests still in Progress..."; done &
210 alive_pid=$!
211 trap kill_alive EXIT
212 }
108 213
109 while [ $# -gt 0 ]; 214 while [ $# -gt 0 ];
110 do 215 do
111 case $1 in 216 case $1 in
112 "--with-m64" | "--with-m32" | "--with-mx32") 217 "--with-m64" | "--with-m32" | "--with-mx32")
116 COMPILERS="$COMPILERS ${1#*=}" 221 COMPILERS="$COMPILERS ${1#*=}"
117 ;; 222 ;;
118 --cflags=*) 223 --cflags=*)
119 CFLAGS="$CFLAGS ${1#*=}" 224 CFLAGS="$CFLAGS ${1#*=}"
120 ;; 225 ;;
226 --valgrind-options=*)
227 VALGRIND_OPTS="$VALGRIND_OPTS ${1#*=}"
228 ;;
229 --with-valgrind*)
230 if [[ ${1#*d} != "" ]]
231 then
232 VALGRIND_BIN="${1#*=}"
233 else
234 VALGRIND_BIN="valgrind"
235 fi
236 start_alive_printing
237 ;;
238 --with-travis-valgrind*)
239 if [[ ("$TRAVIS_BRANCH" == "develop" && "$TRAVIS_PULL_REQUEST" == "false") || "$TRAVIS_BRANCH" == *"valgrind"* || "$TRAVIS_COMMIT_MESSAGE" == *"valgrind"* ]]
240 then
241 if [[ ${1#*d} != "" ]]
242 then
243 VALGRIND_BIN="${1#*=}"
244 else
245 VALGRIND_BIN="valgrind"
246 fi
247 start_alive_printing
248 fi
249 ;;
121 --make-option=*) 250 --make-option=*)
122 MAKE_OPTIONS="$MAKE_OPTIONS ${1#*=}" 251 MAKE_OPTIONS="$MAKE_OPTIONS ${1#*=}"
123 ;; 252 ;;
124 --with-low-mp) 253 --with-low-mp)
125 WITH_LOW_MP="1" 254 WITH_LOW_MP="1"
255 ;;
256 --test-vs-mtest=*)
257 TEST_VS_MTEST="${1#*=}"
258 if ! [ "$TEST_VS_MTEST" -eq "$TEST_VS_MTEST" ] 2> /dev/null
259 then
260 echo "--test-vs-mtest Parameter has to be int"
261 exit 255
262 fi
263 start_alive_printing
264 ;;
265 --mtest-real-rand)
266 MTEST_RAND="-DLTM_MTEST_REAL_RAND"
267 ;;
268 --format)
269 CHECK_FORMAT="1"
126 ;; 270 ;;
127 --all) 271 --all)
128 COMPILERS="gcc clang" 272 COMPILERS="gcc clang"
129 ARCHFLAGS="-m64 -m32 -mx32" 273 ARCHFLAGS="-m64 -m32 -mx32"
130 ;; 274 ;;
131 --help | -h) 275 --help | -h)
132 _help 276 _help
133 ;; 277 ;;
278 --version | -v)
279 echo $(git rev-list HEAD --count -- testme.sh) || echo "Unknown. Please run in original libtommath git repository."
280 exit 0
281 ;;
134 *) 282 *)
135 echo "Ignoring option ${1}" 283 echo "Ignoring option ${1}"
136 ;; 284 ;;
137 esac 285 esac
138 shift 286 shift
139 done 287 done
140 288
141 # default to gcc if no compiler is defined but some other options 289 function _check_git() {
290 git update-index --refresh >/dev/null || true
291 git diff-index --quiet HEAD -- . || ( echo "FAILURE: $*" && exit 1 )
292 }
293
294 if [[ "$CHECK_FORMAT" == "1" ]]
295 then
296 make astyle
297 _check_git "make astyle"
298 perl helper.pl --update-files
299 _check_git "helper.pl --update-files"
300 perl helper.pl --check-all
301 _check_git "helper.pl --check-all"
302 exit $?
303 fi
304
305 [[ "$VALGRIND_BIN" == "" ]] && VALGRIND_OPTS=""
306
307 # default to CC environment variable if no compiler is defined but some other options
142 if [[ "$COMPILERS" == "" ]] && [[ "$ARCHFLAGS$MAKE_OPTIONS$CFLAGS" != "" ]] 308 if [[ "$COMPILERS" == "" ]] && [[ "$ARCHFLAGS$MAKE_OPTIONS$CFLAGS" != "" ]]
143 then 309 then
144 COMPILERS="gcc" 310 COMPILERS="$CC"
145 # default to gcc and run only default config if no option is given 311 # default to CC environment variable and run only default config if no option is given
146 elif [[ "$COMPILERS" == "" ]] 312 elif [[ "$COMPILERS" == "" ]]
147 then 313 then
148 _banner gcc 314 _banner "$CC"
149 _runtest "gcc" "" 315 if [[ "$VALGRIND_BIN" != "" ]]
316 then
317 _runvalgrind "$CC" ""
318 else
319 _runtest "$CC" ""
320 fi
150 _exit 321 _exit
151 fi 322 fi
323
152 324
153 archflags=( $ARCHFLAGS ) 325 archflags=( $ARCHFLAGS )
154 compilers=( $COMPILERS ) 326 compilers=( $COMPILERS )
155 327
156 # choosing a compiler without specifying an architecture will use the default architecture 328 # choosing a compiler without specifying an architecture will use the default architecture
158 then 330 then
159 archflags[0]=" " 331 archflags[0]=" "
160 fi 332 fi
161 333
162 _banner 334 _banner
335
336 if [[ "$TEST_VS_MTEST" != "" ]]
337 then
338 make clean > /dev/null
339 _make "${compilers[0]} ${archflags[0]}" "$CFLAGS" "mtest_opponent"
340 echo
341 _make "gcc" "$MTEST_RAND" "mtest"
342 echo
343 echo "Run test vs. mtest for $TEST_VS_MTEST iterations"
344 _timeout=""
345 which timeout >/dev/null && _timeout="timeout --foreground 1800"
346 $_timeout ./mtest/mtest $TEST_VS_MTEST | $VALGRIND_BIN $VALGRIND_OPTS ./mtest_opponent > valgrind_test.log 2> test_vs_mtest_err.log
347 retval=$?
348 head -n 5 valgrind_test.log
349 tail -n 2 valgrind_test.log
350 exit $retval
351 fi
163 352
164 for i in "${compilers[@]}" 353 for i in "${compilers[@]}"
165 do 354 do
166 if [ -z "$(which $i)" ] 355 if [ -z "$(which $i)" ]
167 then 356 then
183 if [[ $(expr "$i" : "clang") -ne 0 && "$a" == "-mx32" ]] 372 if [[ $(expr "$i" : "clang") -ne 0 && "$a" == "-mx32" ]]
184 then 373 then
185 echo "clang -mx32 tests skipped" 374 echo "clang -mx32 tests skipped"
186 continue 375 continue
187 fi 376 fi
188 377 if [[ "$VALGRIND_BIN" != "" ]]
189 _runtest "$i $a" "$CFLAGS" 378 then
190 [ "$WITH_LOW_MP" != "1" ] && continue 379 _runvalgrind "$i $a" "$CFLAGS"
191 _runtest "$i $a" "-DMP_8BIT $CFLAGS" 380 [ "$WITH_LOW_MP" != "1" ] && continue
192 _runtest "$i $a" "-DMP_16BIT $CFLAGS" 381 _runvalgrind "$i $a" "-DMP_8BIT $CFLAGS"
193 _runtest "$i $a" "-DMP_32BIT $CFLAGS" 382 _runvalgrind "$i $a" "-DMP_16BIT $CFLAGS"
383 _runvalgrind "$i $a" "-DMP_32BIT $CFLAGS"
384 else
385 _runtest "$i $a" "$CFLAGS"
386 [ "$WITH_LOW_MP" != "1" ] && continue
387 _runtest "$i $a" "-DMP_8BIT $CFLAGS"
388 _runtest "$i $a" "-DMP_16BIT $CFLAGS"
389 _runtest "$i $a" "-DMP_32BIT $CFLAGS"
390 fi
194 done 391 done
195 done 392 done
196 393
197 _exit 394 _exit