(List (If (DBracket {B1 UNARY_STRING_z {[DQ [VarSub 1]]}}) (List (Com {[LIT_CHARS echo]} {[DQ [LIT_CHARS "Usage "][VarSub 0][LIT_CHARS " 'environment' [role|IP] [verbose]"]]}) (Com {[LIT_CHARS exit]}) ) ) (If (DBracket {B1 UNARY_STRING_z {[ComSub (Com {[LIT_CHARS which]} {[LIT_CHARS fping]})]}}) (List (Com {[LIT_CHARS echo]} {[DQ [LIT_CHARS "This tool uses fping. You should be able to install fpring with "][ComSub (Com {[LIT_CHARS sudo]} {[LIT_CHARS apt-get]} {[LIT_CHARS install]} {[LIT_CHARS fping]})]]}) (Com {[LIT_CHARS exit]}) ) ) (Com {[LIT_CHARS echo]} {[DQ [VarSub 0][LIT_CHARS " : Checking which hosts are online..."]]}) (= scope= flags=0 words=[] bindings=[('UPHOSTS', {[ComSub (Com {[LIT_CHARS ./cluster-whatsup.sh]} {[VarSub 2]})]})]) (= scope= flags=0 words=[] bindings=[('ENVIRONMENT', {[DQ [VarSub 1]]})]) (= scope= flags=0 words=[] bindings=[('HOSTWANTED', {[DQ [VarSub 2]]})]) (= scope= flags=0 words=[] bindings=[('VERBOSE', {[DQ [VarSub 3]]})]) (FunctionDef vtrace [] (If (DBracket {B! {B1 UNARY_STRING_z {[DQ [VarSub VERBOSE]]}}}) (For STR [{[DQ [VarSub @]]}]) (Com {[LIT_CHARS echo]} {[LIT_CHARS -e]} {[VarSub STR]}) ) ) ) (= scope= flags=0 words=[{[LIT_CHARS -A]}, {[LIT_CHARS HOSTNAMES]}] bindings=[]) (If (DBracket {B1 UNARY_FILE_f {[LIT_CHARS cluster.txt]}}) (List (While (Com {[LIT_CHARS read]} {[LIT_CHARS HOSTNAME]} {[LIT_CHARS MACADDR]} {[LIT_CHARS IPADDR]} {[LIT_CHARS ILOIPADDR]} {[LIT_CHARS DOMAIN]} {[LIT_CHARS ROLE]}) (List (If (DBracket {B2 BINARY_STRING_EQUAL {[VarSub HOSTNAME]} {[DQ [LIT_CHARS end]]}}) (Com {[LIT_CHARS continue]}) ) (If (DBracket {B2 BINARY_STRING_EQUAL {[DQ [VarSub ROLE]]} {[DQ [LIT_CHARS bootstrap]]}}) (Com {[LIT_CHARS continue]}) ) (= scope= flags=0 words=[] bindings=[('THISUP', {[DQ [LIT_CHARS false]]})]) (For UPHOST [{[VarSub UPHOSTS]}]) (If (DBracket {B2 BINARY_STRING_EQUAL {[DQ [VarSub IPADDR]]} {[DQ [VarSub UPHOST]]}}) (List (= scope= flags=0 words=[] bindings=[('THISUP', {[DQ [LIT_CHARS true]]})]) (= scope= flags=0 words=[] bindings=[('UP', {[ArithSub {A2 AS_OP_PLUS {A Atom NODE_ARITH_WORD {[VarSub UP]}} {A Atom NODE_ARITH_WORD {[AS_NUM_LITERAL 1]}}}]})]) ) ) ) (If (DBracket {B? LOGICAL_BINARY_OR {B1 UNARY_STRING_z {[DQ [VarSub HOSTWANTED]]}} {B? LOGICAL_BINARY_OR {B2 BINARY_STRING_EQUAL {[DQ [VarSub HOSTWANTED]]} {[LIT_CHARS all]}} {B? LOGICAL_BINARY_OR {B2 BINARY_STRING_EQUAL {[DQ [VarSub HOSTWANTED]]} {[DQ [VarSub ROLE]]}} {B? LOGICAL_BINARY_OR {B2 BINARY_STRING_EQUAL {[DQ [VarSub HOSTWANTED]]} {[DQ [VarSub IPADDR]]}} {B2 BINARY_STRING_EQUAL {[DQ [VarSub HOSTWANTED]]} {[DQ [VarSub HOSTNAME]]}}}}}}) (List (If (DBracket {B2 BINARY_STRING_EQUAL {[DQ [VarSub THISUP]]} {[DQ [LIT_CHARS false]]}}) (List (Com {[LIT_CHARS echo]} {[DQ [VarSub HOSTNAME][LIT_CHARS " is down"]]}) (Com {[LIT_CHARS continue]}) ) (ElseTrue) (Com {[LIT_CHARS vtrace]} {[DQ [VarSub HOSTNAME][LIT_CHARS " is up"]]}) ) (= scope= flags=0 words=[] bindings=[('HOSTS', {[DQ [VarSub HOSTS][LIT_CHARS " "][VarSub IPADDR]]})]) (= scope= flags=0 words=[] bindings=[('IDX', {[ComSub (Pipeline (Com {[LIT_CHARS echo]} {[VarSub IPADDR]}) (Com {[LIT_CHARS tr]} {[SQ ]} {[SQ ]}) )]})]) (Com {[LIT_CHARS HOSTNAMES] [LIT_OTHER "["] [DQ [VarSub IDX]] [LIT_OTHER "]"] [LIT_OTHER "="] [DQ [VarSub HOSTNAME]]}) ) ) ) ) (Com {[LIT_CHARS vtrace]} {[DQ [LIT_CHARS "HOSTS = "][VarSub HOSTS]]}) (Com {[LIT_CHARS echo]}) (For HOST [{[VarSub HOSTS]}]) (List (Com {[LIT_CHARS echo]} {[DQ [LIT_CHARS "Checking name resolution"]]}) (Com {[LIT_CHARS ./nodessh.sh]} {[VarSub ENVIRONMENT]} {[VarSub HOST]} {[DQ [LIT_CHARS "grep -m1 server /etc/ntp.conf | cut -f2 -d' ' > /tmp/clusterjunk.txt "]]}) (Com {[LIT_CHARS ./nodessh.sh]} {[VarSub ENVIRONMENT]} {[VarSub HOST]} {[DQ [LIT_CHARS "cat /tmp/clusterjunk.txt | xargs -n1 host"]]}) (Com {[LIT_CHARS echo]} {[DQ [LIT_CHARS "checking NTP server"]]}) (Com {[LIT_CHARS ./nodessh.sh]} {[VarSub ENVIRONMENT]} {[VarSub HOST]} {[DQ [LIT_CHARS "cat /tmp/clusterjunk.txt | xargs -n1 ping -c 1"]]}) (= scope= flags=0 words=[] bindings=[('IDX', {[ComSub (Pipeline (Com {[LIT_CHARS echo]} {[VarSub HOST]}) (Com {[LIT_CHARS tr]} {[SQ ]} {[SQ ]}) )]})]) (= scope= flags=0 words=[] bindings=[('NAME', {[VarSub HOSTNAMES bracket_op=(Index {A Atom NODE_ARITH_WORD {[DQ [VarSub IDX]]}})]})]) (Com {[LIT_CHARS vtrace]} {[DQ [LIT_CHARS "Checking "][VarSub NAME][LIT_CHARS " ("][VarSub HOST][LIT_CHARS ")..."]]}) (= scope= flags=0 words=[] bindings=[('ROOTSIZE', {[ComSub (Com {[LIT_CHARS ./nodessh.sh]} {[VarSub ENVIRONMENT]} {[VarSub HOST]} {[DQ [LIT_CHARS "df -k / | grep -v Filesystem"]]})]})]) (= scope= flags=0 words=[] bindings=[('ROOTSIZE', {[ComSub (Pipeline (Com {[LIT_CHARS echo]} {[VarSub ROOTSIZE]}) (Com {[LIT_CHARS awk]} {[SQ ]}) )]})]) (= scope= flags=0 words=[] bindings=[('ROOTGIGS', {[ArithSub {A2 AS_OP_SLASH {A Atom NODE_ARITH_WORD {[VarSub ROOTSIZE]}} {A2 AS_OP_STAR {A Atom NODE_ARITH_WORD {[AS_NUM_LITERAL 1024]}} {A Atom NODE_ARITH_WORD {[AS_NUM_LITERAL 1024]}}}}]})]) (If (DBracket {B2 BINARY_INT_EQ {[VarSub ROOTSIZE]} {[LIT_CHARS 0]}}) (List (Com {[LIT_CHARS echo]} {[DQ [LIT_CHARS "Root fileystem size = "][VarSub ROOTSIZE][LIT_CHARS " ("][VarSub ROOTGIGS][LIT_CHARS " GB) !!WARNING!!"]]}) (Com {[LIT_CHARS echo]} {[DQ [LIT_CHARS "Machine may still be installing the operating system ... skipping"]]}) (Com {[LIT_CHARS continue]}) ) (DBracket {B2 BINARY_INT_LT {[VarSub ROOTSIZE]} {[LIT_CHARS 100] [LIT_OTHER "*"] [LIT_CHARS 1024] [LIT_OTHER "*"] [LIT_CHARS 1024]}}) (Com {[LIT_CHARS echo]} {[DQ [LIT_CHARS "Root fileystem size = "][VarSub ROOTSIZE][LIT_CHARS " ("][VarSub ROOTGIGS][LIT_CHARS " GB) !!WARNING!!"]]}) (ElseTrue) (Com {[LIT_CHARS vtrace]} {[DQ [LIT_CHARS "Root fileystem size = "][VarSub ROOTSIZE][LIT_CHARS " ("][VarSub ROOTGIGS][LIT_CHARS " GB) "]]}) ) (If (DBracket {B1 UNARY_STRING_z {[ComSub (Com {[LIT_CHARS ./nodessh.sh]} {[VarSub ENVIRONMENT]} {[VarSub HOST]} {[DQ [LIT_CHARS "ip route show table mgmt | grep default"]]})]}}) (List (Com {[LIT_CHARS echo]} {[DQ [VarSub HOST][LIT_CHARS " no mgmt default route !!WARNING!!"]]}) (= scope= flags=0 words=[] bindings=[('BADHOSTS', {[DQ [VarSub BADHOSTS][LIT_CHARS " "][VarSub HOST]]})]) ) (ElseTrue) (List (Com {[LIT_CHARS vtrace]} {[DQ [VarSub HOST][LIT_CHARS " has a default mgmt route"]]}) (= scope= flags=0 words=[] bindings=[('MG', {[ArithSub {A2 AS_OP_PLUS {A Atom NODE_ARITH_WORD {[VarSub MG]}} {A Atom NODE_ARITH_WORD {[AS_NUM_LITERAL 1]}}}]})]) ) ) (If (DBracket {B1 UNARY_STRING_z {[ComSub (Com {[LIT_CHARS ./nodessh.sh]} {[VarSub ENVIRONMENT]} {[VarSub HOST]} {[DQ [LIT_CHARS "ip route show table storage | grep default"]]})]}}) (List (Com {[LIT_CHARS echo]} {[DQ [VarSub HOST][LIT_CHARS " has no storage default route !!WARNING!!"]]}) (= scope= flags=0 words=[] bindings=[('BADHOSTS', {[DQ [VarSub BADHOSTS][LIT_CHARS " "][VarSub HOST]]})]) ) (ElseTrue) (List (Com {[LIT_CHARS vtrace]} {[DQ [VarSub HOST][LIT_CHARS " has a default storage route"]]}) (= scope= flags=0 words=[] bindings=[('SG', {[ArithSub {A2 AS_OP_PLUS {A Atom NODE_ARITH_WORD {[VarSub SG]}} {A Atom NODE_ARITH_WORD {[AS_NUM_LITERAL 1]}}}]})]) ) ) (= scope= flags=0 words=[] bindings=[('CHEF', {[ComSub (Com {[LIT_CHARS ./nodessh.sh]} {[VarSub ENVIRONMENT]} {[VarSub HOST]} {[DQ [LIT_CHARS "which chef-client"]]})]})]) (If (DBracket {B1 UNARY_STRING_z {[DQ [VarSub CHEF]]}}) (List (Com {[LIT_CHARS echo]} {[DQ [VarSub HOST][LIT_CHARS " doesn't seem to have chef installed so probably hasn't been assigned a role"]]}) (Com {[LIT_CHARS echo]}) (Com {[LIT_CHARS continue]}) ) ) (= scope= flags=0 words=[] bindings=[('STAT', {[ComSub (Com {[LIT_CHARS ./nodessh.sh]} {[VarSub ENVIRONMENT]} {[VarSub HOST]} {[DQ [LIT_CHARS "ceph -s | grep HEALTH"]]} {[LIT_CHARS sudo]})]})]) (= scope= flags=0 words=[] bindings=[('STAT', {[ComSub (Pipeline (Com {[LIT_CHARS echo]} {[VarSub STAT]}) (Com {[LIT_CHARS cut]} {[LIT_CHARS -f2]} {[LIT_CHARS -d] [LIT_OTHER ":"]}) )]})]) (If (DBracket {B2 BINARY_STRING_TILDE_EQUAL {[DQ [VarSub STAT]]} {[DQ [LIT_CHARS HEALTH_OK]]}}) (Com {[LIT_CHARS vtrace]} {[DQ [VarSub HOST][LIT_CHARS " ceph : healthy"]]}) (ElseTrue) (Com {[LIT_CHARS printf]} {[DQ [VarSub HOST][LIT_CHARS " %20s %s"][\ LIT_ESCAPED_CHAR "\\n"]]} {[LIT_CHARS ceph]} {[DQ [VarSub STAT]]}) ) (= scope= flags=0 words=[] bindings=[('FLUENTD', {[ComSub (Com {[LIT_CHARS ./nodessh.sh]} {[VarSub ENVIRONMENT]} {[VarSub HOST]} {[DQ [LIT_CHARS "ps w -C ruby -C td-agent --no-heading | grep -v chef-client"]]} {[LIT_CHARS sudo]})]})]) (= scope= flags=0 words=[] bindings=[('STAT', {[ComSub (Com {[LIT_CHARS ./nodessh.sh]} {[VarSub ENVIRONMENT]} {[VarSub HOST]} {[DQ [LIT_CHARS "ps w -C ruby -C td-agent --no-heading | grep -v chef-client | wc -l"]]} {[LIT_CHARS sudo]})]})]) (= scope= flags=0 words=[] bindings=[('STAT', {[ComSub (Pipeline (Com {[LIT_CHARS echo]} {[VarSub STAT]}) (Com {[LIT_CHARS cut]} {[LIT_CHARS -f2]} {[LIT_CHARS -d] [LIT_OTHER ":"]}) )]})]) (If (DBracket {B2 BINARY_STRING_TILDE_EQUAL {[DQ [VarSub STAT]]} {[LIT_CHARS 2]}}) (If (DBracket {B! {B1 UNARY_STRING_z {[DQ [VarSub VERBOSE]]}}}) (Com {[LIT_CHARS printf]} {[DQ [VarSub HOST][LIT_CHARS " %20s %s"][\ LIT_ESCAPED_CHAR "\\n"]]} {[DQ [LIT_CHARS fluentd]]} {[DQ [LIT_CHARS normal]]}) ) (ElseTrue) (Com {[LIT_CHARS printf]} {[DQ [VarSub HOST][LIT_CHARS " %20s %s"][\ LIT_ESCAPED_CHAR "\\n"]]} {[LIT_CHARS fluentd]} {[DQ [VarSub FLUENTD]]}) ) (For SERVICE [{[LIT_CHARS keystone]}, {[LIT_CHARS glance-api]}, {[LIT_CHARS glance-registry]}, {[LIT_CHARS cinder-scheduler]}, {[LIT_CHARS cinder-volume]}, {[LIT_CHARS cinder-api]}, {[LIT_CHARS nova-api]}, {[LIT_CHARS nova-novncproxy]}, {[LIT_CHARS nova-scheduler]}, {[LIT_CHARS nova-consoleauth]}, {[LIT_CHARS nova-cert]}, {[LIT_CHARS nova-conductor]}, {[LIT_CHARS nova-compute]}, {[LIT_CHARS nova-network]}, {[LIT_CHARS haproxy]}]) (List (= scope= flags=0 words=[] bindings=[('STAT', {[ComSub (Com {[LIT_CHARS ./nodessh.sh]} {[VarSub ENVIRONMENT]} {[VarSub HOST]} {[DQ [LIT_CHARS "service "][VarSub SERVICE][LIT_CHARS " status | grep running"]]} {[LIT_CHARS sudo]})]})]) (If (DBracket {B! {B2 BINARY_STRING_TILDE_EQUAL {[DQ [VarSub STAT]]} {[DQ [LIT_CHARS unrecognized]]}}}) (List (= scope= flags=0 words=[] bindings=[('STAT', {[ComSub (Pipeline (Com {[LIT_CHARS echo]} {[VarSub STAT]}) (Com {[LIT_CHARS cut]} {[LIT_CHARS -f2]} {[LIT_CHARS -d] [DQ [LIT_CHARS ":"]]}) )]})]) (If (DBracket {B! {B2 BINARY_STRING_TILDE_EQUAL {[DQ [VarSub STAT]]} {[DQ [LIT_CHARS start/running]]}}}) (List (Com {[LIT_CHARS printf]} {[DQ [VarSub HOST][LIT_CHARS " %20s %s"][\ LIT_ESCAPED_CHAR "\\n"]]} {[DQ [VarSub SERVICE]]} {[DQ [VarSub STAT]]}) (= scope= flags=0 words=[] bindings=[('BADHOSTS', {[DQ [VarSub BADHOSTS][LIT_CHARS " "][VarSub HOST]]})]) ) (ElseTrue) (If (DBracket {B! {B1 UNARY_STRING_z {[DQ [VarSub VERBOSE]]}}}) (Com {[LIT_CHARS printf]} {[DQ [VarSub HOST][LIT_CHARS " %20s %s"][\ LIT_ESCAPED_CHAR "\\n"]]} {[DQ [VarSub SERVICE]]} {[DQ [VarSub STAT]]}) ) ) ) ) ) ) (Com {[LIT_CHARS echo]}) ) ) ) (ElseTrue) (Com {[LIT_CHARS echo]} {[DQ [LIT_CHARS "Warning 'cluster.txt' not found"]]}) ) (Com {[LIT_CHARS echo]} {[DQ [VarSub ENVIRONMENT][LIT_CHARS " cluster summary: "][VarSub UP][LIT_CHARS " hosts up. "][VarSub MG][LIT_CHARS " hosts with default mgmt route. "][VarSub SG][LIT_CHARS " hosts with default storage route"]]}) (= scope= flags=0 words=[] bindings=[('BADHOSTS', {[ComSub (Pipeline (Com {[LIT_CHARS echo]} {[VarSub BADHOSTS]}) (Com {[LIT_CHARS uniq]}) (Com {[LIT_CHARS sort]}) )]})]) (If (DBracket {B! {B1 UNARY_STRING_z {[DQ [VarSub BADHOSTS]]}}}) (Com {[LIT_CHARS echo]} {[DQ [LIT_CHARS "Bad hosts "][VarSub BADHOSTS][LIT_CHARS " - definite issues on these"]]}) ) )