(CommandList children: [ (C {(set)} {(-o)} {(nounset)}) (C {(set)} {(-o)} {(pipefail)}) (C {(set)} {(-o)} {(errexit)}) (Assignment keyword: Assign_Readonly pairs: [(assign_pair lhs:(LhsName name:BASE_DIR) op:Equal rhs:{(_tmp/osh-parser)} spids:[46])] spids: [44] ) (Assignment keyword: Assign_Readonly pairs: [ (assign_pair lhs: (LhsName name:SORTED) op: Equal rhs: {($ VSub_Name '$BASE_DIR') (/tmp/sorted.txt)} spids: [51] ) ] spids: [49] ) (FuncDef name: import-files body: (BraceGroup children: [ (Pipeline children: [ (C {(grep)} {(-v)} {(SQ <'^#'>)} {(benchmarks/osh-parser-originals.txt)}) (C {(xargs)} {(--verbose)} {(-I)} {(Lit_LBrace '{') (Lit_RBrace '}')} {(--)} {(cp)} {(Lit_LBrace '{') (Lit_RBrace '}')} {(benchmarks/testdata)} ) ] negated: False ) ] spids: [60] ) spids: [56 59] ) (FuncDef name: sh-one body: (BraceGroup children: [ (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:append_out) op: Equal rhs: {($ VSub_Number '$1')} spids: [128] ) ] spids: [126] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:sh_path) op: Equal rhs: {($ VSub_Number '$2')} spids: [134] ) ] spids: [132] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:host_name) op: Equal rhs: {($ VSub_Number '$3')} spids: [140] ) ] spids: [138] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:host_hash) op: Equal rhs: {($ VSub_Number '$4')} spids: [146] ) ] spids: [144] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:shell_hash) op: Equal rhs: {($ VSub_Number '$5')} spids: [152] ) ] spids: [150] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:path) op: Equal rhs: {($ VSub_Number '$6')} spids: [158] ) ] spids: [156] ) (C {(echo)} {(DQ ('--- ') ($ VSub_Name '$sh_path') (' ') ($ VSub_Name '$path') (' ---'))}) (Assignment keyword: Assign_Local pairs: [(assign_pair lhs:(LhsName name:shell_name) op:Equal spids:[176])] spids: [174] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:shell_name) op: Equal rhs: { (CommandSubPart command_list: (CommandList children: [(C {(basename)} {($ VSub_Name '$sh_path')})] ) left_token: <Left_CommandSub '$('> spids: [180 184] ) } spids: [179] ) ] spids: [179] ) (Assignment keyword: Assign_None pairs: [(assign_pair lhs:(LhsName name:extra_args) op:Equal rhs:{(SQ )} spids:[196])] spids: [196] ) (If arms: [ (if_arm cond: [ (Sentence child: (C {(test)} {(DQ ($ VSub_Name '$shell_name'))} {(Lit_Other '=')} {(SQ <osh>)}) terminator: <Op_Semi ';'> ) ] action: [ (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:extra_args) op: Equal rhs: {(SQ <'--ast-format none'>)} spids: [219] ) ] spids: [219] ) ] spids: [-1 216] ) ] spids: [-1 225] ) (AndOr children: [ (C {(benchmarks/time.py)} {(--output)} {($ VSub_Name '$append_out')} {(--field)} {(DQ ($ VSub_Name '$host_name'))} {(--field)} {(DQ ($ VSub_Name '$host_hash'))} {(--field)} {(DQ ($ VSub_Name '$shell_name'))} {(--field)} {(DQ ($ VSub_Name '$shell_hash'))} {(--field)} {(DQ ($ VSub_Name '$path'))} {(--)} {(DQ ($ VSub_Name '$sh_path'))} {(-n)} {($ VSub_Name '$extra_args')} {(DQ ($ VSub_Name '$path'))} ) (C {(echo)} {(FAILED)}) ] op_id: Op_DPipe ) ] spids: [123] ) spids: [119 122] ) (FuncDef name: import-files body: (BraceGroup children: [ (Pipeline children: [ (C {(grep)} {(-v)} {(SQ <'^#'>)} {(benchmarks/osh-parser-originals.txt)}) (C {(xargs)} {(--verbose)} {(-I)} {(Lit_LBrace '{') (Lit_RBrace '}')} {(--)} {(cp)} {(Lit_LBrace '{') (Lit_RBrace '}')} {(benchmarks/testdata)} ) ] negated: False ) ] spids: [310] ) spids: [306 309] ) (FuncDef name: write-sorted-manifest body: (BraceGroup children: [ (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:files) op: Equal rhs: { (BracedVarSub token: <VSub_Number 1> suffix_op: (StringUnary op_id: VTest_ColonHyphen arg_word: {(benchmarks) (Lit_Slash /) (osh-parser-files.txt)} ) spids: [357 363] ) } spids: [356] ) ] spids: [354] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:counts) op: Equal rhs: {($ VSub_Name '$BASE_DIR') (/raw/line-counts.txt)} spids: [368] ) ] spids: [366] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:csv) op: Equal rhs: {($ VSub_Number '$2')} spids: [375] ) ] spids: [373] ) (Pipeline children: [ (C {(grep)} {(-v)} {(SQ <'^#'>)} {($ VSub_Name '$files')}) (C {(xargs)} {(wc)} {(-l)}) (SimpleCommand words: [{(sort)} {(-n)}] redirects: [ (Redir op_id: Redir_Great fd: -1 arg_word: {($ VSub_Name '$counts')} spids: [408] ) ] ) ] negated: False ) (Pipeline children: [ (C {(cat)} {($ VSub_Name '$counts')}) (SimpleCommand words: [{(awk)} {(SQ <'$2 != "total" { print $2 }'>)}] redirects: [ (Redir op_id: Redir_Great fd: -1 arg_word: {($ VSub_Name '$SORTED')} spids: [431] ) ] ) ] negated: False ) (Pipeline children: [ (C {(cat)} {($ VSub_Name '$counts')}) (SimpleCommand words: [ {(awk)} { (SQ <'\n'> <' BEGIN { print "num_lines,path" }\n'> <' $2 != "total" { print $1 "," $2 }'> ) } ] redirects: [ (Redir op_id: Redir_Great fd: -1 arg_word: {($ VSub_Name '$csv')} spids: [457] ) ] ) ] negated: False ) (C {(cat)} {($ VSub_Name '$SORTED')}) (C {(echo)} {(---)}) (C {(cat)} {($ VSub_Name '$csv')}) ] spids: [351] ) spids: [347 350] ) (FuncDef name: run body: (BraceGroup children: [ (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:preview) op: Equal rhs: { (BracedVarSub token: <VSub_Number 1> suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{}) spids: [494 497] ) } spids: [493] ) ] spids: [491] ) (Assignment keyword: Assign_Local pairs: [(assign_pair lhs:(LhsName name:host) op:Equal spids:[502])] spids: [500] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:host) op: Equal rhs: { (CommandSubPart command_list: (CommandList children:[(C {(hostname)})]) left_token: <Left_CommandSub '$('> spids: [506 508] ) } spids: [505] ) ] spids: [505] ) (Assignment keyword: Assign_Local pairs: [(assign_pair lhs:(LhsName name:job_id) op:Equal spids:[514])] spids: [512] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:job_id) op: Equal rhs: { (DQ ($ VSub_Name '$host') (.) (CommandSubPart command_list: (CommandList children: [ (C {(date)} {(Lit_Other '+') (Lit_Other '%') (Y-) (Lit_Other '%') (m-) (Lit_Other '%') (d__) (Lit_Other '%') (H-) (Lit_Other '%') (M-) (Lit_Other '%') (S) } ) ] ) left_token: <Left_CommandSub '$('> spids: [521 537] ) ) } spids: [517] ) ] spids: [517] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:out_dir) op: Equal rhs: {(SQ <../benchmark-data/osh-parser/>)} spids: [544] ) ] spids: [542] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:out) op: Equal rhs: {(DQ ($ VSub_Name '$out_dir') (/) ($ VSub_Name '$job_id') (.times.csv))} spids: [552] ) ] spids: [550] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:lines_out) op: Equal rhs: {(DQ ($ VSub_Name '$out_dir') (/) ($ VSub_Name '$job_id') (.lines.csv))} spids: [563] ) ] spids: [561] ) (C {(mkdir)} {(-p)} { (CommandSubPart command_list: (CommandList children:[(C {(dirname)} {($ VSub_Name '$out')})]) left_token: <Left_CommandSub '$('> spids: [579 583] ) } (BracedWordTree parts: [ ($ VSub_Name '$BASE_DIR') (/) (BracedAltPart words:[{(tmp)}{(raw)}{(stage1)}{(www)}]) ] ) ) (C {(write-sorted-manifest)} {(SQ )} {($ VSub_Name '$lines_out')}) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:sorted) op: Equal rhs: {($ VSub_Name '$SORTED')} spids: [611] ) ] spids: [609] ) (SimpleCommand words: [ {(echo)} {(SQ <'status,elapsed_secs,host_name,host_hash,shell_name,shell_hash,path'>)} ] redirects: [(Redir op_id:Redir_Great fd:-1 arg_word:{($ VSub_Name '$out')} spids:[626])] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:tmp_dir) op: Equal rhs: {(_tmp/host-id/) ($ VSub_Name '$host')} spids: [634] ) ] spids: [632] ) (C {(benchmarks/id.sh)} {(dump-host-id)} {($ VSub_Name '$tmp_dir')}) (Assignment keyword: Assign_Local pairs: [(assign_pair lhs:(LhsName name:host_hash) op:Equal spids:[649])] spids: [647] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:host_hash) op: Equal rhs: { (CommandSubPart command_list: (CommandList children: [ (C {(benchmarks/id.sh)} {(publish-host-id)} {($ VSub_Name '$tmp_dir')}) ] ) left_token: <Left_CommandSub '$('> spids: [653 659] ) } spids: [652] ) ] spids: [652] ) (C {(echo)} {($ VSub_Name '$host')} {($ VSub_Name '$host_hash')}) (Assignment keyword: Assign_Local pairs: [(assign_pair lhs:(LhsName name:shell_hash) op:Equal spids:[672])] spids: [670] ) (ForEach iter_name: sh_path iter_words: [{(bash)} {(dash)} {(mksh)} {(zsh)} {(bin/osh)} {(_bin/osh)}] do_arg_iter: False body: (DoGroup children: [ (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:name) op: Equal rhs: { (CommandSubPart command_list: (CommandList children: [(C {(basename)} {($ VSub_Name '$sh_path')})] ) left_token: <Left_CommandSub '$('> spids: [709 713] ) } spids: [708] ) ] spids: [706] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:tmp_dir) op: Equal rhs: {(_tmp/shell-id/) ($ VSub_Name '$name')} spids: [717] ) ] spids: [717] ) (C {(benchmarks/id.sh)} {(dump-shell-id)} {($ VSub_Name '$sh_path')} {($ VSub_Name '$tmp_dir')} ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:shell_hash) op: Equal rhs: { (CommandSubPart command_list: (CommandList children: [ (C {(benchmarks/id.sh)} {(publish-shell-id)} {($ VSub_Name '$tmp_dir')} ) ] ) left_token: <Left_CommandSub '$('> spids: [733 739] ) } spids: [732] ) ] spids: [732] ) (C {(echo)} {(DQ ($ VSub_Name '$sh_path') (' ID: ') ($ VSub_Name '$shell_hash'))}) (If arms: [ (if_arm cond: [ (Sentence child: (Pipeline children: [(C {(test)} {(-n)} {(DQ ($ VSub_Name '$preview'))})] negated: True ) terminator: <Op_Semi ';'> ) ] action: [ (AndOr children: [ (Pipeline children: [ (C {(cat)} {($ VSub_Name '$sorted')}) (C {(xargs)} {(-n)} {(1)} {(--)} {($ VSub_Number '$0')} {(sh-one)} {($ VSub_Name '$out')} {($ VSub_Name '$sh_path')} {($ VSub_Name '$host')} {($ VSub_Name '$host_hash')} {($ VSub_Name '$shell_hash')} ) ] negated: False ) (C {(true)}) ] op_id: Op_DPipe ) ] spids: [-1 775] ) ] spids: [-1 817] ) ] spids: [699 820] ) spids: [685 697] ) (C {(cat)} {($ VSub_Name '$out')}) (C {(echo)} {(DQ ('Wrote ') ($ VSub_Name '$out'))}) ] spids: [488] ) spids: [484 487] ) (FuncDef name: summarize body: (BraceGroup children: [ (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:out) op: Equal rhs: {(_tmp/osh-parser/stage1)} spids: [851] ) ] spids: [849] ) (C {(mkdir)} {(-p)} {($ VSub_Name '$out')}) (Assignment keyword: Assign_Local flags: ["'-a'"] pairs: [ (assign_pair lhs: (LhsName name:m1) op: Equal rhs: { (ArrayLiteralPart words: [ {(../benchmark-data/osh-parser/flanders.) (Lit_Other '*') (.times.csv)} ] ) } spids: [871] ) ] spids: [867] ) (Assignment keyword: Assign_Local flags: ["'-a'"] pairs: [ (assign_pair lhs: (LhsName name:m2) op: Equal rhs: { (ArrayLiteralPart words: [{(../benchmark-data/osh-parser/lisa.) (Lit_Other '*') (.times.csv)}] ) } spids: [883] ) ] spids: [879] ) (Assignment keyword: Assign_Local flags: ["'-a'"] pairs: [ (assign_pair lhs: (LhsName name:latest) op: Equal rhs: { (ArrayLiteralPart words: [ { (BracedVarSub token: <VSub_Name m1> bracket_op: (ArrayIndex expr: (ArithUnary op_id: Node_UnaryMinus child: (ArithWord w:{(Lit_Digits 1)}) ) ) spids: [902 908] ) } { (BracedVarSub token: <VSub_Name m2> bracket_op: (ArrayIndex expr: (ArithUnary op_id: Node_UnaryMinus child: (ArithWord w:{(Lit_Digits 1)}) ) ) spids: [910 916] ) } ] ) } spids: [900] ) ] spids: [896] ) (C {(benchmarks/osh-parser.R)} {($ VSub_Name '$out')} { (DQ (BracedVarSub token: <VSub_Name latest> bracket_op: (WholeArray op_id:Lit_At) spids: [926 931] ) ) } ) (C {(tree)} {($ VSub_Name '$BASE_DIR')}) ] spids: [846] ) spids: [842 845] ) (FuncDef name: _print-report body: (BraceGroup children: [ (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:base_url) op: Equal rhs: {(SQ <../../web/table>)} spids: [968] ) ] spids: [966] ) (SimpleCommand words: [{(cat)}] redirects: [ (HereDoc op_id: Redir_DLess fd: -1 body: { (DQ ('<!DOCTYPE html>\n') ('<html>\n') (' <head>\n') (' <title>OSH Parser Performance</title>\n') (' <script type=') (Right_DoubleQuote '"') (text/javascript) (Right_DoubleQuote '"') (' src=') (Right_DoubleQuote '"') ($ VSub_Name '$base_url') (/table-sort.js) (Right_DoubleQuote '"') ('></script>\n') (' <link rel=') (Right_DoubleQuote '"') (stylesheet) (Right_DoubleQuote '"') (' type=') (Right_DoubleQuote '"') (text/css) (Right_DoubleQuote '"') (' href=') (Right_DoubleQuote '"') ($ VSub_Name '$base_url') (/table-sort.css) (Right_DoubleQuote '"') (' />\n') ('\n') (' <style>\n') (' body {\n') (' margin: 0 auto;\n') (' width: 60em;\n') (' }\n') (' code {\n') (' color: green;\n') (' }\n') (' table {\n') (' margin-left: 3em;\n') (' font-family: sans-serif;\n') (' }\n') (' td {\n') (' padding: 8px; /* override default of 5px */\n') (' }\n') (' h3, h4 {\n') (' color: darkgreen;\n') (' }\n') ('\n') (' /* these two tables are side by side */\n') (' #shells, #hosts, #raw_times {\n') (' display: inline-block;\n') (' vertical-align: top;\n') (' }\n') (' #home-link {\n') (' text-align: right;\n') (' }\n') ('\n') (' /* columns */\n') (' #osh-ovm, #osh-cpython {\n') (' background-color: oldlace;\n') (' }\n') (' /* rows */\n') (' .osh-row {\n') (' background-color: oldlace;\n') (' }\n') ('\n') (' </style>\n') (' </head>\n') (' <body>\n') (' <p id=') (Right_DoubleQuote '"') (home-link) (Right_DoubleQuote '"') ('>\n') (' <a href=') (Right_DoubleQuote '"') (/) (Right_DoubleQuote '"') ('>oilshell.org</a>\n') (' </p>\n') (' <h2>OSH Parser Performance</h2>\n') ('\n') (' <p>We run <code>') (EscapedLiteralPart token:<Lit_EscapedChar '\\$'>) ('sh -n ') (EscapedLiteralPart token:<Lit_EscapedChar '\\$'>) ('file</code> for various files under various\n') (' shells. This means that shell startup time is included in the\n') (' elapsed time measurements, but long files are chosen to minimize its\n') (' effect.</p>\n') ('\n') (' <h3>Summary</h3>\n') ('\n') ) } do_expansion: True here_end: EOF was_filled: True spids: [977] ) ] ) (C {(web/table/csv2html.py)} {($ VSub_Name '$BASE_DIR') (/stage1/summary.csv)}) (SimpleCommand words: [{(cat)}] redirects: [ (HereDoc op_id: Redir_DLess fd: -1 body: {(DQ ('\n') (' <h3>Shell and Host Details</h3>\n'))} do_expansion: True here_end: EOF was_filled: True spids: [1082] ) ] ) (C {(web/table/csv2html.py)} {($ VSub_Name '$BASE_DIR') (/stage1/shells.csv)}) (C {(web/table/csv2html.py)} {($ VSub_Name '$BASE_DIR') (/stage1/hosts.csv)}) (SimpleCommand words: [{(cat)}] redirects: [ (HereDoc op_id: Redir_DLess fd: -1 body: {(DQ (' <h3>Raw Timing Data</h3>\n'))} do_expansion: True here_end: EOF was_filled: True spids: [1102] ) ] ) (C {(web/table/csv2html.py)} {($ VSub_Name '$BASE_DIR') (/stage1/raw_times.csv)}) (SimpleCommand words: [{(cat)}] redirects: [ (HereDoc op_id: Redir_DLess fd: -1 body: { (DQ ('\n') (' <h3>Per-File Breakdown</h3>\n') ('\n') (' <h4>Elasped Time in milliseconds</h4>\n') ) } do_expansion: True here_end: EOF was_filled: True spids: [1114] ) ] ) (C {(web/table/csv2html.py)} {($ VSub_Name '$BASE_DIR') (/stage1/elapsed.csv)}) (SimpleCommand words: [{(cat)}] redirects: [ (HereDoc op_id: Redir_DLess fd: -1 body: {(DQ ('\n') (' <h4>Parsing Rate in lines/millisecond</h4>\n'))} do_expansion: True here_end: EOF was_filled: True spids: [1130] ) ] ) (C {(web/table/csv2html.py)} {($ VSub_Name '$BASE_DIR') (/stage1/rate.csv)}) (SimpleCommand words: [{(cat)}] redirects: [ (HereDoc op_id: Redir_DLess fd: -1 body: {(DQ (' </body>\n') ('</html>\n'))} do_expansion: True here_end: EOF was_filled: True spids: [1144] ) ] ) ] spids: [963] ) spids: [959 962] ) (FuncDef name: report body: (BraceGroup children: [ (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:out) op: Equal rhs: {($ VSub_Name '$BASE_DIR') (/index.html)} spids: [1161] ) ] spids: [1159] ) (C {(mkdir)} {(-p)} { (CommandSubPart command_list: (CommandList children:[(C {(dirname)} {($ VSub_Name '$out')})]) left_token: <Left_CommandSub '$('> spids: [1170 1174] ) } ) (SimpleCommand words: [{(_print-report)}] redirects: [ (Redir op_id: Redir_Great fd: -1 arg_word: {($ VSub_Name '$out')} spids: [1179] ) ] ) (C {(echo)} {(DQ ('Wrote ') ($ VSub_Name '$out'))}) ] spids: [1156] ) spids: [1152 1155] ) (FuncDef name: _banner body: (BraceGroup children: [(C {(echo)} {(-----)}) (C {(echo)} {(DQ ($ VSub_At '$@'))}) (C {(echo)} {(-----)})] spids: [1198] ) spids: [1194 1197] ) (FuncDef name: auto body: (BraceGroup children: [ (C {(test/spec.sh)} {(install-shells)}) (C {(sudo)} {(apt)} {(install)} {(python-dev)}) (C {(build/dev.sh)} {(all)}) (C {(_banner)} {(SQ <'OSH dev build'>)}) (C {(bin/osh)} {(-c)} {(SQ <'echo OSH dev build'>)}) (C {(build/prepare.sh)} {(configure)}) (C {(build/prepare.sh)} {(build-python)}) (C {(make)} {(_bin/oil.ovm)}) (C {(scripts/run.sh)} {(make-bin-links)}) (C {(_banner)} {(SQ <'OSH production build'>)}) (C {(_bin/osh)} {(-c)} {(SQ <'echo OSH production build'>)}) (C {(run)}) ] spids: [1233] ) spids: [1229 1232] ) (FuncDef name: time-test body: (BraceGroup children: [ (C {(benchmarks/time.py)} {(--field)} {(bash)} {(--field)} {(foo.txt)} {(--output)} {(_tmp/bench.csv)} {(sleep)} {(0.123)} ) (C {(cat)} {(_tmp/bench.csv)}) ] spids: [1340] ) spids: [1336 1339] ) (C {(DQ ($ VSub_At '$@'))}) ] )