#!/bin/bash # # Usage: # ./wild-runner.sh set -o nounset set -o pipefail set -o errexit source test/common.sh proc process-file { local proj=$1 local abs_path=$2 local rel_path=$3 local raw_base=_tmp/wild/raw/$proj/$rel_path local www_base=_tmp/wild/www/$proj/$rel_path mkdir -p $[dirname $raw_base] mkdir -p $[dirname $www_base] log "--- Processing $proj - $rel_path" # Count the number of lines. This creates a tiny file, but we're doing # everything involving $abs_path at once so it's in the FS cache. wc $abs_path > $(raw_base)__wc.txt # Make a literal copy with .txt extension, so we can browse it cp $abs_path $(www_base).txt # Parse the file. local task_file=$(raw_base)__parse.task.txt local stderr_file=$(raw_base)__parse.stderr.txt local out_file=$(www_base)__ast.html run-task-with-status $task_file \ bin/osh --ast-format abbrev-html -n $abs_path \ > $out_file !2 > $stderr_file # Convert the file. setglobal task_file = "$(raw_base)__osh2oil.task.txt" setglobal stderr_file = "$(raw_base)__osh2oil.stderr.txt" setglobal out_file = "$(www_base)__oil.txt" run-task-with-status $task_file \ $OSH -n --fix $abs_path \ > $out_file !2 > $stderr_file } readonly NUM_TASKS=200 readonly MANIFEST=_tmp/wild/MANIFEST.txt proc parse-in-parallel { local failed='' xargs -n 3 -P $JOBS -- $0 process-file || setglobal failed = '1' # Limit the output depth tree -L 3 _tmp/wild } # Takes 3m 47s on 7 cores for 513K lines. # So that's like 230 seconds or so. It should really take 1 second! proc parse-and-report { local manifest_regex=$(1:-) # egrep regex for manifest line time do { test/wild.sh write-manifest if test -n $manifest_regex { egrep -- $manifest_regex $MANIFEST | parse-in-parallel } else { cat $MANIFEST | parse-in-parallel } make-report } } proc wild-report { env PYTHONPATH=~/hg/json-template/python test/wild_report.py @Argv } proc _link { ln -s -f -v @Argv } proc make-report { cat $MANIFEST | wild-report summarize-dirs # This has to go inside the www dir because of the way that relative links # are calculated. _link \ $PWD/web/osh-to-oil.{html,js} \ _tmp/wild/www _link $PWD/web _tmp } proc test-wild-report { egrep -- '^oil|^perf-tools' $MANIFEST | wild-report summarize-dirs } if test $[basename $0] = 'wild-runner.sh' { @Argv }