#!/bin/bash # # Test scripts found in the wild for both correctness and performance. # # Usage: # ./runtime.sh set -o nounset set -o pipefail set -o errexit source test/common.sh source benchmarks/common.sh # csv-concat readonly BASE_DIR=_tmp/osh-runtime readonly TAR_DIR=$PWD/_deps/osh-runtime # Make it absolute # # Dependencies # readonly -a TAR_SUBDIRS=( ocaml-4.06.0 tcc-0.9.26 yash-2.46 ) # NOTE: Same list in oilshell.org/blob/run.sh. tarballs() { cat < $files_out_dir/STDOUT.txt ;; cpython) # NOTE: $task_arg unused. # This configure script defeats --runtime-mem-dump, I think because of: # exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"} # autoconf supports running configure from a different directory. pushd $files_out_dir >/dev/null "${TIME_PREFIX[@]}" -- \ "$sh_path" $extra_args $PY27_DIR/configure \ > $files_out_dir/STDOUT.txt popd >/dev/null ;; configure) local conf_dir=$task_arg pushd $conf_dir >/dev/null touch __TIMESTAMP "${TIME_PREFIX[@]}" -- "$sh_path" $extra_args ./configure \ > $files_out_dir/STDOUT.txt find . -type f -newer __TIMESTAMP \ | xargs -I {} -- cp --verbose {} $files_out_dir popd >/dev/null ;; *) die "Invalid task type $task_type" ;; esac } # TODO: # - Add Python's configure -- same or different? # - Unify abuild -h -- time it # - --runtime-dump-mem and rename to --parser-dump-mem # # benchmark_name,shell,out_dir # abuild-h # X-configure # config.status? # # Yeah need to come up with a name. Not just conf-dirs. # $dir-configure # Do I add host/host_id? Or just host_label and rely on provenance? # Turn this into write-tasks? # And then run-tasks? run-all? # Yeah it should be # osh-parser.sh write-tasks # osh-runtime.sh write-tasks # virtual-memory.sh write-tasks # # And then auto.sh run-tasks? Then you can have consistent logging? # For each configure file. print-tasks() { local provenance=$1 # Add 1 field for each of 5 fields. cat $provenance | while read \ job_id host_name host_hash sh_path shell_hash; do case $sh_path in mksh|zsh|bin/osh) log "--- Skipping $sh_path" continue ;; esac # Need $PWD/$sh_path because we must change dirs to configure. case $sh_path in */osh) sh_path=$PWD/$sh_path ;; esac local prefix="$job_id $host_name $host_hash $sh_path $shell_hash" # NOTE: 'abuild-help' is a dummy label. echo "$prefix" abuild abuild-help echo "$prefix" cpython cpython-configure for dir in "${TAR_SUBDIRS[@]}"; do echo "$prefix" configure $TAR_DIR/$dir done done } readonly HEADER='status,elapsed_secs,host_name,host_hash,shell_name,shell_hash,task_type,task_arg' readonly NUM_COLUMNS=7 # 5 from provenence, then task_type / task_arg measure() { local provenance=$1 local raw_dir=${2:-$BASE_DIR/raw} local pattern=${3:-} #local base_dir=${2:-../benchmark-data/osh-parser} # Job ID is everything up to the first dot in the filename. local name=$(basename $provenance) local prefix=${name%.provenance.txt} # strip suffix local times_out="$raw_dir/$prefix.times.csv" mkdir -p $BASE_DIR/{raw,stage1} $raw_dir # Write Header of the CSV file that is appended to. echo $HEADER > $times_out local tasks=$BASE_DIR/tasks.txt print-tasks $provenance > $tasks # An empty pattern matches every line. time egrep "$pattern" $tasks | xargs -n $NUM_COLUMNS -- $0 runtime-task $raw_dir || die "*** Some tasks failed. ***" cp -v $provenance $raw_dir } stage1() { local raw_dir=${1:-$BASE_DIR/raw} local out_dir=$BASE_DIR/stage1 mkdir -p $out_dir local times_csv=$out_dir/times.csv # Globs are in lexicographical order, which works for our dates. local -a a=($raw_dir/flanders.*.times.csv) local -a b=($raw_dir/lisa.*.times.csv) csv-concat ${a[-1]} ${b[-1]} > $times_csv local vm_csv=$out_dir/virtual-memory.csv local -a c=($raw_dir/flanders.*.virtual-memory) local -a d=($raw_dir/lisa.*.virtual-memory) benchmarks/virtual_memory.py osh-runtime ${c[-1]} ${d[-1]} > $vm_csv #local raw_dir=${1:-../benchmark-data/osh-parser} } print-report() { local in_dir=$1 local base_url='../../web' cat < OSH Runtime Performance

OSH Runtime Performance

Elapsed Time by Shell (milliseconds)

Some benchmarks call many external tools, while some exercise the shell interpreter itself. Parse time is included.

EOF csv2html $in_dir/times.csv cat <Memory Used to Run

Running under osh-ovm. Memory usage is measured in MB (powers of 10), not MiB (powers of 2).

EOF csv2html $in_dir/virtual-memory.csv cat <Shell and Host Details EOF csv2html $in_dir/shells.csv csv2html $in_dir/hosts.csv cat < EOF } # # Non-configure scripts # abuild-h() { local out_dir=_tmp/osh-runtime mkdir -p $out_dir # TODO: Should test the correctness too. local out=$out_dir/abuild-h-times.csv echo 'status,elapsed_secs,sh_path' > $out for sh_path in bash dash mksh zsh $OSH_OVM; do benchmarks/time.py --output $out --field "$sh_path" -- \ $sh_path benchmarks/testdata/abuild -h done } # # Misc # # Same problem as tcc qemu-old() { local out_dir=$PWD/_tmp/qemu-old mkdir -p $out_dir configure-and-copy ~/src/qemu-1.6.0 $OSH_OVM $out_dir } # This doesn't work for ash either, because it uses the busybox pattern. It # says "exe: applet not found". I guess yash doesn't configure under ash! self-exe() { set +o errexit dash <