#!/bin/bash # # Usage: # ./wild-runner.sh set -o nounset set -o pipefail set -o errexit source test/common.sh proc process-file { var proj = $1 var abs_path = $2 var rel_path = $3 var raw_base = "_tmp/wild/raw/$proj/$rel_path" var www_base = "_tmp/wild/www/$proj/$rel_path" mkdir -p $[dirname $raw_base] mkdir -p $[dirname $www_base] log "--- Processing $proj - $rel_path" # Count the number of lines. This creates a tiny file, but we're doing # everything involving $abs_path at once so it's in the FS cache. wc $abs_path > $(raw_base)__wc.txt # Make a literal copy with .txt extension, so we can browse it cp $abs_path $(www_base).txt # Parse the file. var task_file = "$(raw_base)__parse.task.txt" var stderr_file = "$(raw_base)__parse.stderr.txt" var out_file = "$(www_base)__ast.html" run-task-with-status $task_file \ bin/osh --ast-format abbrev-html -n $abs_path \ > $out_file !2 > $stderr_file # Convert the file. set task_file = "$(raw_base)__osh2oil.task.txt" set stderr_file = "$(raw_base)__osh2oil.stderr.txt" set out_file = "$(www_base)__oil.txt" run-task-with-status $task_file \ $OSH -n --fix $abs_path \ > $out_file !2 > $stderr_file } const NUM_TASKS = '200' const MANIFEST = '_tmp/wild/MANIFEST.txt' proc parse-in-parallel { var failed = '''' xargs -n 3 -P $JOBS -- $0 process-file || set failed = '1' # Limit the output depth tree -L 3 _tmp/wild } # Takes 3m 47s on 7 cores for 513K lines. # So that's like 230 seconds or so. It should really take 1 second! proc parse-and-report { var manifest_regex = $(1:-) # egrep regex for manifest line time do { test/wild.sh write-manifest if test -n $manifest_regex { egrep -- $manifest_regex $MANIFEST | parse-in-parallel } else { cat $MANIFEST | parse-in-parallel } make-report } } proc wild-report { env PYTHONPATH=~/hg/json-template/python test/wild_report.py @Argv } proc _link { ln -s -f -v @Argv } proc make-report { cat $MANIFEST | wild-report summarize-dirs # This has to go inside the www dir because of the way that relative links # are calculated. _link \ $PWD/web/osh-to-oil.{html,js} \ _tmp/wild/www _link $PWD/web _tmp } proc test-wild-report { egrep -- '^oil|^perf-tools' $MANIFEST | wild-report summarize-dirs } if test $[basename $0] = 'wild-runner.sh' { @Argv }