cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

torture.sh (16414B)


      1#!/bin/bash
      2# SPDX-License-Identifier: GPL-2.0+
      3#
      4# Run a series of torture tests, intended for overnight or
      5# longer timeframes, and also for large systems.
      6#
      7# Usage: torture.sh [ options ]
      8#
      9# Copyright (C) 2020 Facebook, Inc.
     10#
     11# Authors: Paul E. McKenney <paulmck@kernel.org>
     12
     13scriptname=$0
     14args="$*"
     15
     16RCUTORTURE="`pwd`/tools/testing/selftests/rcutorture"; export RCUTORTURE
     17PATH=${RCUTORTURE}/bin:$PATH; export PATH
     18. functions.sh
     19
     20TORTURE_ALLOTED_CPUS="`identify_qemu_vcpus`"
     21MAKE_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS*2))
     22HALF_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS/2))
     23if test "$HALF_ALLOTED_CPUS" -lt 1
     24then
     25	HALF_ALLOTED_CPUS=1
     26fi
     27VERBOSE_BATCH_CPUS=$((TORTURE_ALLOTED_CPUS/16))
     28if test "$VERBOSE_BATCH_CPUS" -lt 2
     29then
     30	VERBOSE_BATCH_CPUS=0
     31fi
     32
     33# Configurations/scenarios.
     34configs_rcutorture=
     35configs_locktorture=
     36configs_scftorture=
     37kcsan_kmake_args=
     38
     39# Default compression, duration, and apportionment.
     40compress_concurrency="`identify_qemu_vcpus`"
     41duration_base=10
     42duration_rcutorture_frac=7
     43duration_locktorture_frac=1
     44duration_scftorture_frac=2
     45
     46# "yes" or "no" parameters
     47do_allmodconfig=yes
     48do_rcutorture=yes
     49do_locktorture=yes
     50do_scftorture=yes
     51do_rcuscale=yes
     52do_refscale=yes
     53do_kvfree=yes
     54do_kasan=yes
     55do_kcsan=no
     56do_clocksourcewd=yes
     57do_rt=yes
     58
     59# doyesno - Helper function for yes/no arguments
     60function doyesno () {
     61	if test "$1" = "$2"
     62	then
     63		echo yes
     64	else
     65		echo no
     66	fi
     67}
     68
     69usage () {
     70	echo "Usage: $scriptname optional arguments:"
     71	echo "       --compress-concurrency concurrency"
     72	echo "       --configs-rcutorture \"config-file list w/ repeat factor (3*TINY01)\""
     73	echo "       --configs-locktorture \"config-file list w/ repeat factor (10*LOCK01)\""
     74	echo "       --configs-scftorture \"config-file list w/ repeat factor (2*CFLIST)\""
     75	echo "       --do-all"
     76	echo "       --do-allmodconfig / --do-no-allmodconfig"
     77	echo "       --do-clocksourcewd / --do-no-clocksourcewd"
     78	echo "       --do-kasan / --do-no-kasan"
     79	echo "       --do-kcsan / --do-no-kcsan"
     80	echo "       --do-kvfree / --do-no-kvfree"
     81	echo "       --do-locktorture / --do-no-locktorture"
     82	echo "       --do-none"
     83	echo "       --do-rcuscale / --do-no-rcuscale"
     84	echo "       --do-rcutorture / --do-no-rcutorture"
     85	echo "       --do-refscale / --do-no-refscale"
     86	echo "       --do-rt / --do-no-rt"
     87	echo "       --do-scftorture / --do-no-scftorture"
     88	echo "       --duration [ <minutes> | <hours>h | <days>d ]"
     89	echo "       --kcsan-kmake-arg kernel-make-arguments"
     90	exit 1
     91}
     92
     93while test $# -gt 0
     94do
     95	case "$1" in
     96	--compress-concurrency)
     97		checkarg --compress-concurrency "(concurrency level)" $# "$2" '^[0-9][0-9]*$' '^error'
     98		compress_concurrency=$2
     99		shift
    100		;;
    101	--config-rcutorture|--configs-rcutorture)
    102		checkarg --configs-rcutorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
    103		configs_rcutorture="$configs_rcutorture $2"
    104		shift
    105		;;
    106	--config-locktorture|--configs-locktorture)
    107		checkarg --configs-locktorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
    108		configs_locktorture="$configs_locktorture $2"
    109		shift
    110		;;
    111	--config-scftorture|--configs-scftorture)
    112		checkarg --configs-scftorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
    113		configs_scftorture="$configs_scftorture $2"
    114		shift
    115		;;
    116	--do-all|--doall)
    117		do_allmodconfig=yes
    118		do_rcutorture=yes
    119		do_locktorture=yes
    120		do_scftorture=yes
    121		do_rcuscale=yes
    122		do_refscale=yes
    123		do_rt=yes
    124		do_kvfree=yes
    125		do_kasan=yes
    126		do_kcsan=yes
    127		do_clocksourcewd=yes
    128		;;
    129	--do-allmodconfig|--do-no-allmodconfig)
    130		do_allmodconfig=`doyesno "$1" --do-allmodconfig`
    131		;;
    132	--do-clocksourcewd|--do-no-clocksourcewd)
    133		do_clocksourcewd=`doyesno "$1" --do-clocksourcewd`
    134		;;
    135	--do-kasan|--do-no-kasan)
    136		do_kasan=`doyesno "$1" --do-kasan`
    137		;;
    138	--do-kcsan|--do-no-kcsan)
    139		do_kcsan=`doyesno "$1" --do-kcsan`
    140		;;
    141	--do-kvfree|--do-no-kvfree)
    142		do_kvfree=`doyesno "$1" --do-kvfree`
    143		;;
    144	--do-locktorture|--do-no-locktorture)
    145		do_locktorture=`doyesno "$1" --do-locktorture`
    146		;;
    147	--do-none|--donone)
    148		do_allmodconfig=no
    149		do_rcutorture=no
    150		do_locktorture=no
    151		do_scftorture=no
    152		do_rcuscale=no
    153		do_refscale=no
    154		do_rt=no
    155		do_kvfree=no
    156		do_kasan=no
    157		do_kcsan=no
    158		do_clocksourcewd=no
    159		;;
    160	--do-rcuscale|--do-no-rcuscale)
    161		do_rcuscale=`doyesno "$1" --do-rcuscale`
    162		;;
    163	--do-rcutorture|--do-no-rcutorture)
    164		do_rcutorture=`doyesno "$1" --do-rcutorture`
    165		;;
    166	--do-refscale|--do-no-refscale)
    167		do_refscale=`doyesno "$1" --do-refscale`
    168		;;
    169	--do-rt|--do-no-rt)
    170		do_rt=`doyesno "$1" --do-rt`
    171		;;
    172	--do-scftorture|--do-no-scftorture)
    173		do_scftorture=`doyesno "$1" --do-scftorture`
    174		;;
    175	--duration)
    176		checkarg --duration "(minutes)" $# "$2" '^[0-9][0-9]*\(m\|h\|d\|\)$' '^error'
    177		mult=1
    178		if echo "$2" | grep -q 'm$'
    179		then
    180			mult=1
    181		elif echo "$2" | grep -q 'h$'
    182		then
    183			mult=60
    184		elif echo "$2" | grep -q 'd$'
    185		then
    186			mult=1440
    187		fi
    188		ts=`echo $2 | sed -e 's/[smhd]$//'`
    189		duration_base=$(($ts*mult))
    190		shift
    191		;;
    192	--kcsan-kmake-arg|--kcsan-kmake-args)
    193		checkarg --kcsan-kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$'
    194		kcsan_kmake_args="`echo "$kcsan_kmake_args $2" | sed -e 's/^ *//' -e 's/ *$//'`"
    195		shift
    196		;;
    197	*)
    198		echo Unknown argument $1
    199		usage
    200		;;
    201	esac
    202	shift
    203done
    204
    205ds="`date +%Y.%m.%d-%H.%M.%S`-torture"
    206startdate="`date`"
    207starttime="`get_starttime`"
    208
    209T=/tmp/torture.sh.$$
    210trap 'rm -rf $T' 0 2
    211mkdir $T
    212
    213echo " --- " $scriptname $args | tee -a $T/log
    214echo " --- Results directory: " $ds | tee -a $T/log
    215
    216# Calculate rcutorture defaults and apportion time
    217if test -z "$configs_rcutorture"
    218then
    219	configs_rcutorture=CFLIST
    220fi
    221duration_rcutorture=$((duration_base*duration_rcutorture_frac/10))
    222if test "$duration_rcutorture" -eq 0
    223then
    224	echo " --- Zero time for rcutorture, disabling" | tee -a $T/log
    225	do_rcutorture=no
    226fi
    227
    228# Calculate locktorture defaults and apportion time
    229if test -z "$configs_locktorture"
    230then
    231	configs_locktorture=CFLIST
    232fi
    233duration_locktorture=$((duration_base*duration_locktorture_frac/10))
    234if test "$duration_locktorture" -eq 0
    235then
    236	echo " --- Zero time for locktorture, disabling" | tee -a $T/log
    237	do_locktorture=no
    238fi
    239
    240# Calculate scftorture defaults and apportion time
    241if test -z "$configs_scftorture"
    242then
    243	configs_scftorture=CFLIST
    244fi
    245duration_scftorture=$((duration_base*duration_scftorture_frac/10))
    246if test "$duration_scftorture" -eq 0
    247then
    248	echo " --- Zero time for scftorture, disabling" | tee -a $T/log
    249	do_scftorture=no
    250fi
    251
    252touch $T/failures
    253touch $T/successes
    254
    255# torture_one - Does a single kvm.sh run.
    256#
    257# Usage:
    258#	torture_bootargs="[ kernel boot arguments ]"
    259#	torture_one flavor [ kvm.sh arguments ]
    260#
    261# Note that "flavor" is an arbitrary string.  Supply --torture if needed.
    262# Note that quoting is problematic.  So on the command line, pass multiple
    263# values with multiple kvm.sh argument instances.
    264function torture_one {
    265	local cur_bootargs=
    266	local boottag=
    267
    268	echo " --- $curflavor:" Start `date` | tee -a $T/log
    269	if test -n "$torture_bootargs"
    270	then
    271		boottag="--bootargs"
    272		cur_bootargs="$torture_bootargs"
    273	fi
    274	"$@" $boottag "$cur_bootargs" --datestamp "$ds/results-$curflavor" > $T/$curflavor.out 2>&1
    275	retcode=$?
    276	resdir="`grep '^Results directory: ' $T/$curflavor.out | tail -1 | sed -e 's/^Results directory: //'`"
    277	if test -z "$resdir"
    278	then
    279		cat $T/$curflavor.out | tee -a $T/log
    280		echo retcode=$retcode | tee -a $T/log
    281	fi
    282	if test "$retcode" == 0
    283	then
    284		echo "$curflavor($retcode)" $resdir >> $T/successes
    285	else
    286		echo "$curflavor($retcode)" $resdir >> $T/failures
    287	fi
    288}
    289
    290# torture_set - Does a set of tortures with and without KASAN and KCSAN.
    291#
    292# Usage:
    293#	torture_bootargs="[ kernel boot arguments ]"
    294#	torture_set flavor [ kvm.sh arguments ]
    295#
    296# Note that "flavor" is an arbitrary string that does not affect kvm.sh
    297# in any way.  So also supply --torture if you need something other than
    298# the default.
    299function torture_set {
    300	local cur_kcsan_kmake_args=
    301	local kcsan_kmake_tag=
    302	local flavor=$1
    303	shift
    304	curflavor=$flavor
    305	torture_one "$@"
    306	if test "$do_kasan" = "yes"
    307	then
    308		curflavor=${flavor}-kasan
    309		torture_one "$@" --kasan
    310	fi
    311	if test "$do_kcsan" = "yes"
    312	then
    313		curflavor=${flavor}-kcsan
    314		if test -n "$kcsan_kmake_args"
    315		then
    316			kcsan_kmake_tag="--kmake-args"
    317			cur_kcsan_kmake_args="$kcsan_kmake_args"
    318		fi
    319		torture_one "$@" --kconfig "CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y" $kcsan_kmake_tag $cur_kcsan_kmake_args --kcsan
    320	fi
    321}
    322
    323# make allmodconfig
    324if test "$do_allmodconfig" = "yes"
    325then
    326	echo " --- allmodconfig:" Start `date` | tee -a $T/log
    327	amcdir="tools/testing/selftests/rcutorture/res/$ds/allmodconfig"
    328	mkdir -p "$amcdir"
    329	echo " --- make clean" > "$amcdir/Make.out" 2>&1
    330	make -j$MAKE_ALLOTED_CPUS clean >> "$amcdir/Make.out" 2>&1
    331	echo " --- make allmodconfig" >> "$amcdir/Make.out" 2>&1
    332	cp .config $amcdir
    333	make -j$MAKE_ALLOTED_CPUS allmodconfig >> "$amcdir/Make.out" 2>&1
    334	echo " --- make " >> "$amcdir/Make.out" 2>&1
    335	make -j$MAKE_ALLOTED_CPUS >> "$amcdir/Make.out" 2>&1
    336	retcode="$?"
    337	echo $retcode > "$amcdir/Make.exitcode"
    338	if test "$retcode" == 0
    339	then
    340		echo "allmodconfig($retcode)" $amcdir >> $T/successes
    341	else
    342		echo "allmodconfig($retcode)" $amcdir >> $T/failures
    343	fi
    344fi
    345
    346# --torture rcu
    347if test "$do_rcutorture" = "yes"
    348then
    349	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000"
    350	torture_set "rcutorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "$configs_rcutorture" --trust-make
    351fi
    352
    353if test "$do_locktorture" = "yes"
    354then
    355	torture_bootargs="torture.disable_onoff_at_boot"
    356	torture_set "locktorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture lock --allcpus --duration "$duration_locktorture" --configs "$configs_locktorture" --trust-make
    357fi
    358
    359if test "$do_scftorture" = "yes"
    360then
    361	torture_bootargs="scftorture.nthreads=$HALF_ALLOTED_CPUS torture.disable_onoff_at_boot csdlock_debug=1"
    362	torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 2G --trust-make
    363fi
    364
    365if test "$do_rt" = "yes"
    366then
    367	# With all post-boot grace periods forced to normal.
    368	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 rcupdate.rcu_normal=1"
    369	torture_set "rcurttorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "TREE03" --trust-make
    370
    371	# With all post-boot grace periods forced to expedited.
    372	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 rcupdate.rcu_expedited=1"
    373	torture_set "rcurttorture-exp" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "TREE03" --trust-make
    374fi
    375
    376if test "$do_refscale" = yes
    377then
    378	primlist="`grep '\.name[ 	]*=' kernel/rcu/refscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
    379else
    380	primlist=
    381fi
    382for prim in $primlist
    383do
    384	torture_bootargs="refscale.scale_type="$prim" refscale.nreaders=$HALF_ALLOTED_CPUS refscale.loops=10000 refscale.holdoff=20 torture.disable_onoff_at_boot"
    385	torture_set "refscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture refscale --allcpus --duration 5 --kconfig "CONFIG_TASKS_TRACE_RCU=y CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --bootargs "verbose_batched=$VERBOSE_BATCH_CPUS torture.verbose_sleep_frequency=8 torture.verbose_sleep_duration=$VERBOSE_BATCH_CPUS" --trust-make
    386done
    387
    388if test "$do_rcuscale" = yes
    389then
    390	primlist="`grep '\.name[ 	]*=' kernel/rcu/rcuscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
    391else
    392	primlist=
    393fi
    394for prim in $primlist
    395do
    396	torture_bootargs="rcuscale.scale_type="$prim" rcuscale.nwriters=$HALF_ALLOTED_CPUS rcuscale.holdoff=20 torture.disable_onoff_at_boot"
    397	torture_set "rcuscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 5 --kconfig "CONFIG_TASKS_TRACE_RCU=y CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make
    398done
    399
    400if test "$do_kvfree" = "yes"
    401then
    402	torture_bootargs="rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot"
    403	torture_set "rcuscale-kvfree" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 10 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 2G --trust-make
    404fi
    405
    406if test "$do_clocksourcewd" = "yes"
    407then
    408	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000"
    409	torture_set "clocksourcewd-1" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make
    410
    411	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 clocksource.max_cswd_read_retries=1"
    412	torture_set "clocksourcewd-2" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make
    413
    414	# In case our work is already done...
    415	if test "$do_rcutorture" != "yes"
    416	then
    417		torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000"
    418		torture_set "clocksourcewd-3" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --trust-make
    419	fi
    420fi
    421
    422echo " --- " $scriptname $args
    423echo " --- " Done `date` | tee -a $T/log
    424ret=0
    425nsuccesses=0
    426echo SUCCESSES: | tee -a $T/log
    427if test -s "$T/successes"
    428then
    429	cat "$T/successes" | tee -a $T/log
    430	nsuccesses="`wc -l "$T/successes" | awk '{ print $1 }'`"
    431fi
    432nfailures=0
    433echo FAILURES: | tee -a $T/log
    434if test -s "$T/failures"
    435then
    436	awk < "$T/failures" -v sq="'" '{ print "echo " sq $0 sq; print "sed -e " sq "1,/^ --- .* Test summary:$/d" sq " " $2 "/log | grep Summary: | sed -e " sq "s/^[^S]*/  /" sq; }' | sh | tee -a $T/log | tee "$T/failuresum"
    437	nfailures="`wc -l "$T/failures" | awk '{ print $1 }'`"
    438	grep "^  Summary: " "$T/failuresum" |
    439		grep -v '^  Summary: Bugs: [0-9]* (all bugs kcsan)$' > "$T/nonkcsan"
    440	if test -s "$T/nonkcsan"
    441	then
    442		nonkcsanbug="yes"
    443	fi
    444	ret=2
    445fi
    446if test "$do_kcsan" = "yes"
    447then
    448	TORTURE_KCONFIG_KCSAN_ARG=1 tools/testing/selftests/rcutorture/bin/kcsan-collapse.sh tools/testing/selftests/rcutorture/res/$ds > tools/testing/selftests/rcutorture/res/$ds/kcsan.sum
    449fi
    450echo Started at $startdate, ended at `date`, duration `get_starttime_duration $starttime`. | tee -a $T/log
    451echo Summary: Successes: $nsuccesses Failures: $nfailures. | tee -a $T/log
    452if test -z "$nonkcsanbug" && test -s "$T/failuresum"
    453then
    454	echo "  All bugs were KCSAN failures."
    455fi
    456tdir="`cat $T/successes $T/failures | head -1 | awk '{ print $NF }' | sed -e 's,/[^/]\+/*$,,'`"
    457if test -n "$tdir" && test $compress_concurrency -gt 0
    458then
    459	# KASAN vmlinux files can approach 1GB in size, so compress them.
    460	echo Looking for K[AC]SAN files to compress: `date` > "$tdir/log-xz" 2>&1
    461	find "$tdir" -type d -name '*-k[ac]san' -print > $T/xz-todo
    462	ncompresses=0
    463	batchno=1
    464	if test -s $T/xz-todo
    465	then
    466		for i in `cat $T/xz-todo`
    467		do
    468			find $i -name 'vmlinux*' -print
    469		done | wc -l | awk '{ print $1 }' > $T/xz-todo-count
    470		n2compress="`cat $T/xz-todo-count`"
    471		echo Size before compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log
    472		for i in `cat $T/xz-todo`
    473		do
    474			echo Compressing vmlinux files in ${i}: `date` >> "$tdir/log-xz" 2>&1
    475			for j in $i/*/vmlinux
    476			do
    477				xz "$j" >> "$tdir/log-xz" 2>&1 &
    478				ncompresses=$((ncompresses+1))
    479				if test $ncompresses -ge $compress_concurrency
    480				then
    481					echo Waiting for batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log
    482					wait
    483					ncompresses=0
    484					batchno=$((batchno+1))
    485				fi
    486			done
    487		done
    488		if test $ncompresses -gt 0
    489		then
    490			echo Waiting for final batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log
    491		fi
    492		wait
    493		echo Size after compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log
    494		echo Total duration `get_starttime_duration $starttime`. | tee -a $T/log
    495	else
    496		echo No compression needed: `date` >> "$tdir/log-xz" 2>&1
    497	fi
    498fi
    499if test -n "$tdir"
    500then
    501	cp $T/log "$tdir"
    502fi
    503exit $ret