#!/bin/sh
# posixcube.sh
#   posixcube.sh is a POSIX compliant shell script server automation framework.
#   Use consistent APIs for common tasks and package functionality and file
#   templates in cubes (like recipes/playbooks from other frameworks).
#
# Authors:
#   Kevin Grigorenko (kevin@myplaceonline.com)
#
# Version History:
#   0.1
#     * Version 0.1
#
# Development guidelines:
#   1. See references [1, 2, 7].
#   2. Indent with two spaces.
#   3. Use lower-case variables unless an envar may be used by other scripts [4].
#      All scripts are `source`d together, so exporting is usually unnecessary.
#   4. Try to keep lines less than 120 characters.
#   5. Use [ for tests instead of [[ and use = instead of ==.
#   6. Use a separate [ invocation for each single test, combine them with && and ||.
#   7. Don't use `set -e`. Handle failures consciously (see Philosophy section).
#
# References:
#   1. http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html
#   2. https://www.gnu.org/software/autoconf/manual/autoconf.html#Portable-Shell
#   3. printf: http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap05.html
#   4. "The name space of environment variable names containing lowercase letters is reserved for applications."
#      http://pubs.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap08.html
#   5. test: http://pubs.opengroup.org/onlinepubs/9699919799/utilities/test.html
#   6. expr: http://pubs.opengroup.org/onlinepubs/9699919799/utilities/expr.html
#   7. https://wiki.ubuntu.com/DashAsBinSh

proc p666_show_usage {
  if test $Argc -ne 0 {
    p666_printf_error ${@}
  }

  # When updating usage, also update README.md.
  cat <<< '''
usage: posixcube.sh -h HOST... [OPTION]... COMMAND...

  posixcube.sh is a POSIX compliant shell script server automation framework.
  Use consistent APIs for common tasks and package functionality and file
  templates in cubes (like recipes/playbooks from other frameworks).

  -?        Help.
  -h HOST   Target host. Option may be specified multiple times. If a host has
            a wildcard ('*'), then HOST is interpeted as a regular expression,
            with '*' replaced with '.*' and any matching hosts in the following
            files are added to the HOST list: /etc/ssh_config,
            /etc/ssh/ssh_config, ~/.ssh/config, /etc/ssh_known_hosts,
            /etc/ssh/ssh_known_hosts, ~/.ssh/known_hosts, and /etc/hosts.
  -c CUBE   Execute a cube. Option may be specified multiple times. If COMMANDS
            are also specified, cubes are run first.
  -u USER   SSH user. Defaults to ${USER}.
  -e ENVAR  Shell script with environment variable assignments which is
            uploaded and sourced on each HOST. Option may be specified
            multiple times. Files ending with .enc will be decrypted
            temporarily. If not specified, defaults to envars*sh envars*sh.enc
  -p PWD    Password for decrypting .enc ENVAR files.
  -w PWDF   File that contains the password for decrypting .enc ENVAR files.
            Defaults to ~/.posixcube.pwd
  -r ROLE   Role name. Option may be specified multiple times.
  -o P=V    Set the specified parameter P with the value V. Do not put double
            quotes around V. If V contains *, try to find matching hosts per
            the -h algorithm. Option may be specified multiple times.
  -i CUBE   Upload a CUBE but do not execute it. This is needed when one CUBE
            includes this CUBE using cube_include.
  -v        Show version information.
  -d        Print debugging information.
  -q        Quiet; minimize output.
  -b        If using bash, install programmable tab completion for SSH hosts.
  -s        Skip remote host initialization (making ~/posixcubes, uploading
            posixcube.sh, etc.
  -k        Keep the cube_exec.sh generated script.
  -z SPEC   Use the SPEC set of options from the ./cubespecs.ini file
  -a        Asynchronously execute remote CUBEs/COMMANDs. Works on Bash only.
  -y        If a HOST returns a non-zero code, continue processing other HOSTs.
  COMMAND   Remote command to run on each HOST. Option may be specified
            multiple times. If no HOSTs are specified, available sub-commands:
              edit: Decrypt, edit, and re-encrypt ENVAR file with $EDITOR.
              show: Decrypt and print ENVAR file.
              source: Source all ENVAR files. Must be run with
                      POSIXCUBE_SOURCED (see Public Variables section below).

Description:

  posixcube.sh is used to execute CUBEs and/or COMMANDs on one or more HOSTs.
  
  A CUBE is a shell script or directory containing shell scripts. The CUBE
  is rsync'ed to each HOST. If CUBE is a shell script, it's executed. If
  CUBE is a directory, a shell script of the same name in that directory
  is executed. In both cases, the directory is changed to the directory
  containing the script before execution so that you may reference files
  such as templates using relative paths.
  
  An ENVAR script is encouraged to use environment variable names of the form
  cubevar_${uniquecontext}_envar="value". If a CUBE directory contains the
  file `envars.sh`, it's sourced before anything else (including `-e ENVARs`).
  
  Both CUBEs and COMMANDs may execute any of the functions defined in the
  "Public APIs" in the posixcube.sh script. Short descriptions of the functions
  are in the APIs section below. See the source comments above each function
  for details.
  
Examples (assuming posixcube.sh is on ${PATH}, or executed absolutely):

  posixcube.sh -h socrates uptime
  
    Run the `uptime` command on host `socrates`. This is not very different
    from ssh ${USER}@socrates uptime, except that COMMANDs (`uptime`) have
    access to the cube_* public functions.
  
  posixcube.sh -h socrates -c test.sh
  
    Run the `test.sh` script (CUBE) on host `socrates`. The script has
    access to the cube_* public functions.
  
  posixcube.sh -h socrates -c test
  
    Upload the entire `test` directory (CUBE) to the host `socrates` and
    then execute the `test.sh` script within that directory (the name
    of the script is expected to be the same as the name of the CUBE). This
    allows for easily packaging other scripts and resources needed by
    `test.sh`.
  
  posixcube.sh -u root -h socrates -h seneca uptime
  
    Run the `uptime` command on hosts `socrates` and `seneca`
    as the user `root`.
  
  posixcube.sh -h web*.test.com uptime
  
    Run the `uptime` command on all hosts matching the regular expression
    web.*.test.com in the SSH configuration files.
  
  sudo ${PATH_TO}/posixcube.sh -b && . /etc/bash_completion.d/posixcube_completion.sh
  
    For Bash users, install a programmable completion script to support tab
    auto-completion of hosts from SSH configuration files.

  posixcube.sh -e production.sh.enc show
  
    Decrypt and show the contents of production.sh
  
  posixcube.sh -e production.sh.enc edit
  
    Decrypt, edit, and re-encrypt the contents of production.sh with $EDITOR

Philosophy:

  Fail hard and fast. In principle, a well written script would check ${?}
  after each command and either gracefully handle it, or report an error.
  Few people write scripts this well, so we enforce this check (using
  `cube_check_return` within all APIs) and we encourage you to do the same
  in your scripts with `touch /etc/fstab || cube_check_return`. All cube_*
  APIs are guaranteed to do their own checks, so you don't have to do this
  for those calls; however, note that if you're executing a cube_* API in a
  sub-shell, although any failures will be reported by cube_check_return,
  the script will continue unless you also check the return of the sub-shell.
  For example: $(cube_readlink /etc/localtime) || cube_check_return
  With this strategy, unfortunately piping becomes more difficult. There are
  non-standard mechanisms like pipefail and PIPESTATUS, but the standardized
  approach is to run each command separately and check the status. For example:
  cube_app_result1="$(command1 || cube_check_return)" || cube_check_return
  cube_app_result2="$(printf '%s' "${cube_app_result1}" | command2 || cube_check_return)" || cube_check_return
  
  We do not use `set -e` because some functions may handle all errors
  internally (with `cube_check_return`) and use a positive return code as a
  "benign" result (e.g. `cube_set_file_contents`).

Frequently Asked Questions:

  * Why is there a long delay between "Preparing hosts" and the first remote
    execution?
  
    You can see details of what's happening with the `-d` flag. By default,
    the script first loops through every host and ensures that ~/posixcubes/
    exists, then it transfers itself to the remote host. These two actions
    may be skipped with the `-s` parameter if you've already run the script
    at least once and your version of this script hasn't been updated. Next,
    the script loops through every host and transfers any CUBEs and a script
    containing the CUBEs and COMMANDs to run (`cube_exec.sh`). Finally,
    you'll see the "Executing on HOST..." line and the real execution starts.

Cube Development:

  Shell scripts don't have scoping, so to reduce the chances of function name
  conflicts, name functions cube_${cubename}_${function} and name variables
  cubevar_${cubename}_${var}.

Public APIs:
  
  * cube_echo
      Print ${@} to stdout prefixed with ([$(date)] [$(hostname)]) and
      suffixed with a newline.
      Example: cube_echo "Hello World"

  * cube_printf
      Print $1 to stdout prefixed with ([$(date)] [$(hostname)]) and
      suffixed with a newline (with optional printf arguments in $@).
      Example: cube_printf "Hello World from PID %5s" $$

  * cube_error_echo
      Same as cube_echo except output to stderr and include a red "Error: "
      message prefix.
      Example: cube_error "Goodbye World"

  * cube_error_printf
      Same as cube_printf except output to stderr and include a red "Error: "
      message prefix.
      Example: cube_error "Goodbye World from PID %5s" $$

  * cube_throw
      Same as cube_error_echo but also print a stack of functions and processes
      (if available) and then call `exit 1`.
      Example: cube_throw "Expected some_file."

  * cube_check_return
      Check if $? is non-zero and call cube_throw if so.
      Example: some_command || cube_check_return

  * cube_include
      Include the ${1} cube
      Example: cube_include core_cube

  * cube_check_numargs
      Call cube_throw if there are less than $1 arguments in $@
      Example: cube_check_numargs 2 "${@}"

  * cube_service
      Run the $1 action on the $2 service.
      Example: cube_service start crond

  * cube_package
      Pass $@ to the package manager. Implicitly passes the the parameter
      to say yes to questions.
      Example: cube_package install python

  * cube_append_str
      Echo $1 with $2 appended after a space if $1 was not blank.
      Example: cubevar_app_str=$(cube_append_str "${cubevar_app_str}" "Test")

  * cube_command_exists
      Check if $1 command or function exists in the current context.
      Example: cube_command_exists systemctl

  * cube_dir_exists
      Check if $1 exists as a directory.
      Example: cube_dir_exists /etc/cron.d/

  * cube_file_exists
      Check if $1 exists as a file with read access.
      Example: cube_file_exists /etc/cron.d/0hourly

  * cube_operating_system
      Detect operating system and return one of the POSIXCUBE_OS_* values.
      Example: [ $(cube_operating_system) -eq ${POSIXCUBE_OS_LINUX} ] && ...

  * cube_operating_system_has_flavor
      Check if the operating system flavor includes the flavor specified in $1
      by one of the POSIXCUBE_OS_FLAVOR_* values.
      Example: cube_operating_system_has_flavor ${POSIXCUBE_OS_FLAVOR_FEDORA} && ...

  * cube_shell
      Detect running shell and return one of the CUBE_SHELL_* values.
      Example: [ $(cube_shell) -eq ${POSIXCUBE_SHELL_BASH} ] && ...

  * cube_current_script_name
      echo the basename of the currently executing script.
      Example: script_name=$(cube_current_script_name)

  * cube_current_script_abs_path
      echo the absolute path the currently executing script.
      Example: script_name=$(cube_current_script_abs_path)

  * cube_file_size
      echo the size of a file $1 in bytes
      Example: cube_file_size some_file

  * cube_set_file_contents
      Copy the contents of $2 on top of $1 if $1 doesn't exist or the contents
      are different than $2. If $2 ends with ".template", first evaluate all
      ${VARIABLE} expressions (except for \${VARIABLE}).
      Example: cube_set_file_contents "/etc/npt.conf" "templates/ntp.conf"

  * cube_set_file_contents_string
      Set the contents of $1 to the string $@. Create file if it doesn't exist.
      Example: cube_set_file_contents_string ~/.info "Hello World"

  * cube_expand_parameters
      echo stdin to stdout with all ${VAR}'s evaluated (except for \${VAR})
      Example: cube_expand_parameters < template > output

  * cube_readlink
      Echo the absolute path of $1 without any symbolic links.
      Example: cube_readlink /etc/localtime

  * cube_random_number
      Echo a random number between 1 and $1
      Example: cube_random_number 10

  * cube_tmpdir
      Echo a temporary directory
      Example: cube_tmpdir

  * cube_total_memory
      Echo total system memory in bytes
      Example: cube_total_memory

  * cube_ensure_directory
      Ensure directory $1 exists
      Example: cube_ensure_directory ~/.ssh/

  * cube_ensure_file
      Ensure file $1 exists
      Example: cube_ensure_file ~/.ssh/authorized_keys

  * cube_pushd
      Equivalent to `pushd` with $1
      Example: cube_pushd ~/.ssh/

  * cube_popd
      Equivalent to `popd`
      Example: cube_popd

  * cube_has_role
      Return true if the role $1 is set.
      Example: cube_has_role "database_backup"

  * cube_file_contains
      Check if the file $1 contains $2
      Example: cube_file_contains /etc/fstab nfsmount

  * cube_stdin_contains
      Check if stdin contains $1
      Example: echo "Hello World" | cube_stdin_contains "Hello"

  * cube_interface_ipv4_address
      Echo the IPv4 address of interface $1
      Example: cube_interface_ipv4_address eth0

  * cube_interface_ipv6_address
      Echo the IPv6 address of interface $1
      Example: cube_interface_ipv6_address eth0

  * cube_prompt
      Prompt the question $1 followed by " (y/N)" and prompt for an answer.
      A blank string answer is equivalent to No. Return true if yes, false otherwise.
      Example: cube_prompt "Are you sure?"

  * cube_hostname
      Echo full hostname.
      Example: cube_hostname

  * cube_user_exists
      Check if the $1 user exists
      Example: cube_user_exists nginx

  * cube_create_user
      Create the user $1
      Example: cube_create_user nginx

  * cube_group_exists
      Check if the $1 group exists
      Example: cube_group_exists nginx

  * cube_create_group
      Create the group $1
      Example: cube_create_group nginx

  * cube_group_contains_user
      Check if the $1 group contains the user $2
      Example: cube_group_contains_user nginx nginx

  * cube_add_group_user
      Add the user $2 to group $1
      Example: cube_add_group_user nginx nginx

Public Variables:

  * POSIXCUBE_APIS_ONLY
      Set this to any value to only source the public APIs in posixcube.sh.
      Example: POSIXCUBE_APIS_ONLY=true . posixcube.sh && cube_echo $(cube_random_number 10)
  
  * POSIXCUBE_SOURCED
      Set this to any value to only run a sub-COMMAND, most commonly `source`,
      to source in all ENVAR files, but skip actual execution of posixcube.
      Example: POSIXCUBE_SOURCED=true . posixcube.sh source; POSIXCUBE_SOURCED= ; cube_echo Test

Source: https://github.com/myplaceonline/posixcube
'''

  if test $Argc -ne 0 {
    p666_printf_error ${@}
  }

  if test ${POSIXCUBE_SOURCED} = "" {
    exit 1
  }
}

###############
# Public APIs #
###############

setvar POSIXCUBE_VERSION = '0.1'
setvar POSIXCUBE_DEBUG = '0'
setvar POSIXCUBE_COLOR_RESET = ""\x1B[0m""
setvar POSIXCUBE_COLOR_RED = ""\x1B[31m""
setvar POSIXCUBE_COLOR_GREEN = ""\x1B[32m""
setvar POSIXCUBE_COLOR_YELLOW = ""\x1B[33m""
setvar POSIXCUBE_COLOR_BLUE = ""\x1B[34m""
setvar POSIXCUBE_COLOR_PURPLE = ""\x1B[35m""
setvar POSIXCUBE_COLOR_CYAN = ""\x1B[36m""
setvar POSIXCUBE_COLOR_WHITE = ""\x1B[37m""

setvar POSIXCUBE_NEWLINE = ""
""
setvar POSIXCUBE_CUBE_NAME = """"
setvar POSIXCUBE_CUBE_NAME_WITH_PREFIX = """"

setvar POSIXCUBE_OS_UNKNOWN = '-1'
setvar POSIXCUBE_OS_LINUX = '1'
setvar POSIXCUBE_OS_MAC_OSX = '2'
setvar POSIXCUBE_OS_WINDOWS = '3'

setvar POSIXCUBE_OS_FLAVOR_UNKNOWN = '-1'
setvar POSIXCUBE_OS_FLAVOR_FEDORA = '1'
setvar POSIXCUBE_OS_FLAVOR_DEBIAN = '2'
setvar POSIXCUBE_OS_FLAVOR_UBUNTU = '3'

setvar POSIXCUBE_SHELL_UNKNOWN = '-1'
setvar POSIXCUBE_SHELL_BASH = '1'

setvar POSIXCUBE_NODE_HOSTNAME = "$(hostname)"

# Description:
#   Print ${@} to stdout prefixed with ([$(date)]  [$(hostname)]) and suffixed with
#   a newline.
# Example call:
#   cube_echo "Hello World"
# Example output:
#   [Sun Dec 18 09:40:22 PST 2016] [socrates] Hello World
# Arguments: ${@} passed to echo
proc cube_echo {
  if test ${POSIXCUBE_CUBE_NAME_WITH_PREFIX} = "" {
    printf "[$(date)] [${POSIXCUBE_COLOR_GREEN}${POSIXCUBE_NODE_HOSTNAME}${POSIXCUBE_COLOR_RESET}] "
  } else {
    printf "[$(date)] [${POSIXCUBE_COLOR_GREEN}${POSIXCUBE_NODE_HOSTNAME}${POSIXCUBE_COLOR_RESET}${POSIXCUBE_COLOR_CYAN}${POSIXCUBE_CUBE_NAME_WITH_PREFIX}$(cube_line_number ":")${POSIXCUBE_COLOR_RESET}] "
  }
  echo ${@}
}

# Description:
#   Print $1 to stdout prefixed with ([$(date)]  [$(hostname)]) and suffixed with
#   a newline.
# Example call:
#   cube_printf "Hello World from PID %5s" $$
# Example output:
#   [Sun Dec 18 09:40:22 PST 2016] [socrates] Hello World from PID   123
# Arguments:
#   Required:
#     $1: String to print (printf-compatible)
#   Optional: 
#     $2: printf arguments 
proc cube_printf {
  setvar cube_printf_str = "$1"; shift
  if test ${POSIXCUBE_CUBE_NAME_WITH_PREFIX} = "" {
    printf "[$(date)] [${POSIXCUBE_COLOR_GREEN}${POSIXCUBE_NODE_HOSTNAME}${POSIXCUBE_COLOR_RESET}] ${cube_printf_str}\n" ${@}
  } else {
    printf "[$(date)] [${POSIXCUBE_COLOR_GREEN}${POSIXCUBE_NODE_HOSTNAME}${POSIXCUBE_COLOR_RESET}${POSIXCUBE_COLOR_CYAN}${POSIXCUBE_CUBE_NAME_WITH_PREFIX}$(cube_line_number ":")${POSIXCUBE_COLOR_RESET}] ${cube_printf_str}\n" ${@}
  }
}

# Description:
#   Print $1 to stderr prefixed with ([$(date)]  [$(hostname)] Error: ) and
#   suffixed with a newline.
# Example call:
#   cube_error_echo "Goodbye World"
# Example output:
#   [Sun Dec 18 09:40:22 PST 2016] [socrates] Goodbye World
# Arguments: ${@} passed to echo
proc cube_error_echo {
  if test ${POSIXCUBE_CUBE_NAME_WITH_PREFIX} = "" {
    printf "[$(date)] [${POSIXCUBE_COLOR_RED}${POSIXCUBE_NODE_HOSTNAME}${POSIXCUBE_COLOR_RESET}] ${POSIXCUBE_COLOR_RED}Error${POSIXCUBE_COLOR_RESET}: " 1>&2
  } else {
    printf "[$(date)] [${POSIXCUBE_COLOR_RED}${POSIXCUBE_NODE_HOSTNAME}${POSIXCUBE_CUBE_NAME_WITH_PREFIX}$(cube_line_number ":")${POSIXCUBE_COLOR_RESET}] ${POSIXCUBE_COLOR_RED}Error${POSIXCUBE_COLOR_RESET}: " 1>&2
  }
  echo ${@} 1>&2
}

# Description:
#   Print $1 to stderr prefixed with ([$(date)]  [$(hostname)] Error: ) and
#   suffixed with a newline.
# Example call:
#   cube_error_printf "Goodbye World from PID %5s" $$
# Example output:
#   [Sun Dec 18 09:40:22 PST 2016] [socrates] Goodbye World from PID   123
# Arguments:
#   Required:
#     $1: String to print (printf-compatible)
#   Optional: 
#     $2: printf arguments 
proc cube_error_printf {
  setvar cube_error_printf_str = "$1"; shift
  if test ${POSIXCUBE_CUBE_NAME_WITH_PREFIX} = "" {
    printf "[$(date)] [${POSIXCUBE_COLOR_RED}${POSIXCUBE_NODE_HOSTNAME}${POSIXCUBE_COLOR_RESET}] ${POSIXCUBE_COLOR_RED}Error${POSIXCUBE_COLOR_RESET}: ${cube_error_printf_str}\n" ${@} 1>&2
  } else {
    printf "[$(date)] [${POSIXCUBE_COLOR_RED}${POSIXCUBE_NODE_HOSTNAME}${POSIXCUBE_CUBE_NAME_WITH_PREFIX}$(cube_line_number ":")${POSIXCUBE_COLOR_RESET}] ${POSIXCUBE_COLOR_RED}Error${POSIXCUBE_COLOR_RESET}: ${cube_error_printf_str}\n" ${@} 1>&2
  }
}

# Description:
#   Print $1 and a stack of functions and processes (if available) with
#   cube_error_echo and then call `exit 1`.
# Example call:
#   cube_throw "Expected some_file to exist."
# Arguments: ${@} passed to cube_error_echo
proc cube_throw {
  cube_error_echo ${@}
  
  setvar cube_throw_pid = ""$$
  
  if cube_command_exists caller || test -r /proc/${cube_throw_pid}/cmdline {
    cube_error_echo Stack:
  }
  
  if cube_command_exists caller  {
    setvar x = '0'
    while true {
      setvar cube_error_caller = $(caller $x)
      setvar cube_error_caller_result = ${?}
      if test ${cube_error_caller_result} -eq 0 {
        setvar cube_error_caller_result_lineno = $(echo "${cube_error_caller}" | awk '{ print $1 }')
        setvar cube_error_caller_result_subroutine = $(echo "${cube_error_caller}" | awk '{ print $2 }')
        setvar cube_error_caller_result_sourcefile = $(echo "${cube_error_caller}" | awk '{ for(i=3;i<=NF;i++){ printf "%s ", $i }; printf "\n" }' | sed 's/ $//')
        cube_error_printf "  [func] %4s ${cube_error_caller_result_subroutine} ${cube_error_caller_result_sourcefile}" ${cube_error_caller_result_lineno}
      } else {
        break
      }
      setvar x = $((${x}+1))
    }
  }
  
  # http://stackoverflow.com/a/1438241/5657303
  if test -r /proc/${cube_throw_pid}/cmdline {
    while true
    {
      setvar cube_throw_cmdline = $(cat /proc/${cube_throw_pid}/cmdline)
      setvar cube_throw_ppid = $(grep PPid /proc/${cube_throw_pid}/status | awk '{ print $2; }')
      cube_error_printf "  [pid] %5s ${cube_throw_cmdline}" ${cube_throw_pid}
      if test ${cube_throw_pid} = "1" { # init
        break
      }
      setvar cube_throw_pid = ${cube_throw_ppid}
    }
  }
  
  exit 1
}

# Description:
#   Skipping any call stack frames in posixcube.sh, return the line number of the calling stack frame
# Example call:
#   cube_line_number
# Arguments:
#   Optional:
#     $1: If a result is returned, prepend with $1
proc cube_line_number {
  if cube_command_exists caller  {
    setvar x = '0'
    while true {
      setvar cube_api_caller_output = $(caller $x)
      setvar cube_api_caller_result = ${?}
      if test ${cube_api_caller_result} -eq 0 {
        setvar cube_api_caller_result_lineno = $(echo "${cube_api_caller_output}" | awk '{ print $1 }')
        setvar cube_api_caller_result_subroutine = $(echo "${cube_api_caller_output}" | awk '{ print $2 }')
        setvar cube_api_caller_result_sourcefile = $(echo "${cube_api_caller_output}" | awk '{ for(i=3;i<=NF;i++){ printf "%s ", $i }; printf "\n" }' | sed 's/ $//')
        setvar cube_api_caller_result_sourcefile_basename = $(basename "${cube_api_caller_result_sourcefile}")
        if test ${cube_api_caller_result_sourcefile_basename} != "posixcube.sh" {
          if test ${1} != "" {
            printf "%s" ${1}
          }
          printf "%s" ${cube_api_caller_result_lineno}
          break
        }
      } else {
        break
      }
      setvar x = $((${x}+1))
    }
  }
}

# Description:
#   Check if $? is non-zero and call cube_throw if so.
# Example call:
#   some_command || cube_check_return
# Arguments: None
proc cube_check_return {
  setvar cube_check_return_val = ${?}
  if test ${cube_check_return_val} -ne 0 {
    setvar cube_check_return_info = """"
    if test ${*} != "" {
      setvar cube_check_return_info = "" (${*})""
    }
    cube_throw "Previous command failed with code ${cube_check_return_val}${cube_check_return_info}"
  }
  return ${cube_check_return_val}
}

# Description:
#   Echo $1 with $2 appended after a space if $1 was not blank.
# Example call:
#   cubevar_app_str=$(cube_append_str "${cubevar_app_str}" "Test")
# Arguments:
#   Required:
#     $1: Original string
#     $2: Strings to append
#   Optional:
#     $3: Delimeter
proc cube_append_str {
  cube_check_numargs 1 ${@}
  
  if test ${1} = "" {
    echo ${2}
  } else {
    if test ${3} = "" {
      echo "${1} ${2}"
    } else {
      echo "${1}${3}${2}"
    }
  }
}

# Description:
#   Check if $1 command or function exists in the current context.
# Example call:
#   cube_command_exists systemctl
# Arguments:
#   Required:
#     $1: Command or function name.
proc cube_command_exists {
  cube_check_numargs 1 ${@}
  command -v ${1} >/dev/null 2>&1
}

# Description:
#   Check if $1 exists as a directory.
# Example call:
#   cube_dir_exists /etc/cron.d/
# Arguments:
#   Required:
#     $1: Directory name.
proc cube_dir_exists {
  cube_check_numargs 1 ${@}
  test -d ${1}
}

# Description:
#   Check if $1 exists as a file with read access.
# Example call:
#   cube_file_exists /etc/cron.d/0hourly
# Arguments:
#   Required:
#     $1: File name.
proc cube_file_exists {
  cube_check_numargs 1 ${@}
  test -r ${1}
}

# Description:
#   Detect operating system and return one of the POSIXCUBE_OS_* values.
# Example call:
#   if [ $(cube_operating_system) -eq ${POSIXCUBE_OS_LINUX} ]; then ...
# Arguments: None
proc cube_operating_system {
  # http://stackoverflow.com/a/27776822/5657303
  case{
    Linux {
      echo ${POSIXCUBE_OS_LINUX}
      }
    Darwin {
      echo ${POSIXCUBE_OS_MAC_OSX}
      }
    CYGWIN*|MINGW32*|MSYS* {
      echo ${POSIXCUBE_OS_WINDOWS}
      }
    * {
      echo ${POSIXCUBE_OS_UNKNOWN}
      }
  }
}

# Description:
#   Check if the operating system flavor includes the flavor specified in $1 by one of the POSIXCUBE_OS_FLAVOR_* values.
# Example call:
#   if cube_operating_system_has_flavor ${POSIXCUBE_OS_FLAVOR_FEDORA} ; then ...
# Arguments:
#   Required:
#     $1: One of the POSIXCUBE_OS_FLAVOR_* values.
proc cube_operating_system_has_flavor {
  cube_check_numargs 1 ${@}
  case{
    ${POSIXCUBE_OS_FLAVOR_FEDORA} {
      if cube_file_exists "/etc/fedora-release" {
        return 0
      }
      }
    ${POSIXCUBE_OS_FLAVOR_UBUNTU} {
      if cube_file_exists "/etc/lsb-release" {
        return 0
      }
      }
    ${POSIXCUBE_OS_FLAVOR_DEBIAN} {
      if cube_file_contains /etc/os-release "NAME=\"Debian" || cube_file_exists "/etc/lsb-release" {
        return 0
      }
      }
    * {
      cube_throw "Unknown flavor ${1}"
      }
  }
  return 1
}

# Description:
#   Detect current shell and return one of the CUBE_SHELL_* values.
# Example call:
#   if [ $(cube_shell) -eq ${POSIXCUBE_SHELL_BASH} ]; then ...
# Arguments: None
proc cube_shell {
  # http://stackoverflow.com/questions/3327013/how-to-determine-the-current-shell-im-working-on
  if test $BASH != "" {
    echo ${POSIXCUBE_SHELL_BASH}
  } else {
    echo ${POSIXCUBE_SHELL_UNKNOWN}
  }
}

# Description:
#   Throw an error if there are fewer than $1 arguments.
# Example call:
#   cube_check_numargs 2 "${@}"
# Arguments:
#   Required:
#     $1: String to print (printf-compatible)
#     $@: Arguments to check
proc cube_check_numargs {
  setvar cube_check_numargs_expected = "$1"; shift
  test ${#} -lt ${cube_check_numargs_expected} && cube_throw "Expected ${cube_check_numargs_expected} arguments, received ${#}."
  return 0
}

# Description:
#   Run the $1 action on the $2 service.
# Example call:
#   cube_service start crond
# Arguments:
#   Required:
#     $1: Action name supported by $2 (e.g. start, stop, restart, enable, etc.)
#     $2: Service name.
proc cube_service {
  cube_check_numargs 1 ${@}
  if cube_command_exists systemctl  {
    if test ${1} = "daemon-reload" {
      systemctl $1 || cube_check_return
    } else {
      systemctl $1 $2 || cube_check_return
    }
  } elif cube_command_exists service  {
    if test ${1} != "daemon-reload" {
      service $2 $1 || cube_check_return
    }
  } else {
    cube_throw "Could not find service program"
  }
  if test ${2} != "" {
    case{
      stop {
        setvar cube_service_verb = ""stopped""
        }
      enable|disable {
        setvar cube_service_verb = ""${1}d""
        }
      * {
        setvar cube_service_verb = ""${1}ed""
        }
    }
    cube_echo "$(echo ${cube_service_verb} | cut -c1 | tr [a-z] [A-Z])$(echo ${cube_service_verb} | cut -c2-) $2 service"
  } else {
    cube_echo "Executed $1"
  }
}

# Description:
#   Return 0 if service $1 exists; otherwise, 1
# Example call:
#   cube_service_exists kdump
# Arguments:
#   Required:
#     $1: Service name.
proc cube_service_exists {
  cube_check_numargs 1 ${@}
  if cube_command_exists systemctl  {
    setvar cube_service_exists_output = "$(systemctl status ${1} 2>&1)"
    echo ${cube_service_exists_output} | grep -l loaded >/dev/null 2>&1
    return $?
  } else {
    cube_throw "Not implemented"
  }
}

# Description:
#   Pass $@ to the package manager. Implicitly passes the the parameter
#   to say yes to questions.
# Example call:
#   cube_package install python
# Arguments:
#   Required:
#     $@: Arguments to the package manager.
proc cube_package {
  cube_check_numargs 1 ${@}
  
  if cube_command_exists dnf  {
    cube_echo "Executing dnf -y ${@}"
    dnf -y ${@} || cube_check_return
  } elif cube_command_exists yum  {
    cube_echo "Executing yum -y ${@}"
    yum -y ${@} || cube_check_return
  } elif cube_command_exists apt-get  {
    cube_echo "Executing apt-get -y ${@}'"
    # -o Dpkg::Options::="--force-confnew"
    DEBIAN_FRONTEND=noninteractive' apt-get -y ${@} || cube_check_return
  } else {
    cube_throw "cube_package has not implemented your operating system yet"
  }
}

# Description:
#   echo the basename of the currently executing script.
# Example call:
#   script_name=$(cube_current_script_name)
# Arguments: None
proc cube_current_script_name {
  basename $0
}

# Description:
#   echo the absolute path the currently executing script.
# Example call:
#   script_abspath=$(cube_current_script_abs_path)
# Arguments: None
proc cube_current_script_abs_path {
  setvar cube_current_script_abs_path_dirname = $( cd "$(dirname "$0")" ; pwd -P )
  echo "${cube_current_script_abs_path_dirname}/$(cube_current_script_name)"
}

# Description:
#   echo the size of a file $1 in bytes
# Example call:
#   cube_file_size some_file
# Arguments:
#   Required:
#     $1: File
proc cube_file_size {
  cube_check_numargs 1 ${@}
  if cube_file_exists ${1}  {
    wc -c < "${1}"
  } else {
    cube_throw "Could not find or read file ${1}"
  }
}

# Description:
#   echo stdin to stdout with all ${VAR}'s evaluated (except for \${VAR})
# Example call:
#   cube_expand_parameters < template > output
# Arguments: None
proc cube_expand_parameters {
  # http://stackoverflow.com/a/40167919/5657303
  
  setvar cube_expand_parameters_is_bash = '0'
  if test $(cube_shell) -eq ${POSIXCUBE_SHELL_BASH} {
    # No win from using regex in parameter expansion because we can't use backreferences to make sure we don't
    # unescape
    setvar cube_expand_parameters_is_bash = '0'
    #cube_expand_parameters_is_bash=1
  }'
  
  # the `||` clause ensures that the last line is read even if it doesn't end with \n
  while IFS=''' read -r cube_expand_parameters_line || test -n ${cube_expand_parameters_line} {
    # Escape any characters that might trip up eval
    setvar cube_expand_parameters_line_escaped = $(printf %s "${cube_expand_parameters_line}" | tr '`([$\\"' '\1\2\3\4\5\6')
    
    # Then re-enable un-escaped ${ references
    if test $cube_expand_parameters_is_bash -eq 1 {
      setvar cube_expand_parameters_line_escaped = ${cube_expand_parameters_line_escaped//$'\4'{/\${}
    } else {
      setvar cube_expand_parameters_line_escaped = $(printf %s "${cube_expand_parameters_line_escaped}" | sed 's/\([^\x05]\)\x04{/\1${/g' | sed 's/^\x04{/${/g')
    }
    
    setvar cube_expand_parameters_output = $(eval "printf '%s\n' \"${cube_expand_parameters_line_escaped}\"") || cube_check_return ${cube_expand_parameters_line_escaped}
    
    echo ${cube_expand_parameters_output} | tr '\1\2\3\4\5\6' '`([$\\"'
  }
}

# Description:
#   Read stdin into ${cube_read_heredoc_result}
# Example call:
#   cube_read_heredoc <<'HEREDOC'; cubevar_app_str="${cube_read_heredoc_result}"
#     `([$\{\
#   HEREDOC
# Arguments: None
proc cube_read_heredoc {
  setvar cube_read_heredoc_first = '1'
  setvar cube_read_heredoc_result = ""'""
  while IFS='\n'' read -r cube_read_heredoc_line {
    if test ${cube_read_heredoc_first} -eq 1 {
      setvar cube_read_heredoc_result = "${cube_read_heredoc_line}"
      setvar cube_read_heredoc_first = '0'
    } else {
      setvar cube_read_heredoc_result = ""${cube_read_heredoc_result}${POSIXCUBE_NEWLINE}${cube_read_heredoc_line}""
    }
  }
}

# Description:
#   Copy the contents of $2 on top of $1 if $1 doesn't exist or the contents
#   are different than $2. If $2 ends with ".template" then first process
#   the file with `cube_expand_parameters`.
# Example call:
#   cube_set_file_contents "/etc/npt.conf" "templates/ntp.conf"
# Arguments:
#   Required:
#     $1: Target file
#     $2: Source file
# Returns: success/true if the file was updated
proc cube_set_file_contents {
  cube_check_numargs 2 ${@}
  setvar cube_set_file_contents_target_file = "$1"; shift
  setvar cube_set_file_contents_input_file = "$1"; shift

  setvar cube_set_file_contents_debug = "${cube_set_file_contents_input_file}"
  
  setvar cube_set_file_contents_needs_replace = '0'
  setvar cube_set_file_contents_input_file_needs_remove = '0'
  
  if ! cube_file_exists ${cube_set_file_contents_input_file}  {
    cube_throw "Could not find or read input ${cube_set_file_contents_input_file}"
  }
  
  cube_ensure_directory $(dirname "${cube_set_file_contents_target_file}")
  
  setvar cube_set_file_contents_input_file_is_template = $(expr "${cube_set_file_contents_input_file}" : '.*\.template$')
  if test ${cube_set_file_contents_input_file_is_template} -ne 0 {
    
    setvar cube_set_file_contents_input_file_original = "${cube_set_file_contents_input_file}"
    setvar cube_set_file_contents_input_file = ""${cube_set_file_contents_input_file}.tmp""

    # Parameter expansion can introduce very large delays with large files, so point that out
    #if [ ${POSIXCUBE_DEBUG} -eq 1 ]; then
      cube_echo "Expanding parameters of ${cube_set_file_contents_input_file_original}"
    #fi
    
    # awk, perl, sed, envsubst, etc. can do this easily but would require exported envars
    # perl -pe 's/([^\\]|^)\$\{([a-zA-Z_][a-zA-Z_0-9]*)\}/$1.$ENV{$2}/eg' < "${cube_set_file_contents_input_file_original}" > "${cube_set_file_contents_input_file}" || cube_check_return
    # http://stackoverflow.com/questions/415677/how-to-replace-placeholders-in-a-text-file
    # http://stackoverflow.com/questions/2914220/bash-templating-how-to-build-configuration-files-from-templates-with-bash
    cube_expand_parameters < "${cube_set_file_contents_input_file_original}" > "${cube_set_file_contents_input_file}" || cube_check_return
    
    #if [ ${POSIXCUBE_DEBUG} -eq 1 ]; then
      cube_echo "Expansion complete"
    #fi
    
    setvar cube_set_file_contents_input_file_needs_remove = '1'
  }
  
  if cube_file_exists ${cube_set_file_contents_target_file}  {
  
    # If the file sizes are different, then replace the file (http://stackoverflow.com/a/5920355/5657303)
    setvar cube_set_file_contents_target_file_size = $(cube_file_size "${cube_set_file_contents_target_file}")
    setvar cube_set_file_contents_input_file_size = $(cube_file_size "${cube_set_file_contents_input_file}")
    
    if test ${POSIXCUBE_DEBUG} -eq 1 {
      cube_echo "Target file ${cube_set_file_contents_target_file} exists. Target size: ${cube_set_file_contents_target_file_size}, source size: ${cube_set_file_contents_input_file_size}"
    }
    
    if test ${cube_set_file_contents_target_file_size} -eq ${cube_set_file_contents_input_file_size} {

      # Sizes are equal, so do a quick cksum
      setvar cube_set_file_contents_target_file_cksum = $(cksum "${cube_set_file_contents_target_file}" | awk '{print $1}')
      setvar cube_set_file_contents_input_file_cksum = $(cksum "${cube_set_file_contents_input_file}" | awk '{print $1}')

      if test ${POSIXCUBE_DEBUG} -eq 1 {
        cube_echo "Target cksum: ${cube_set_file_contents_target_file_cksum}, source cksum: ${cube_set_file_contents_input_file_cksum}"
      }
      
      if test ${cube_set_file_contents_target_file_cksum} != ${cube_set_file_contents_input_file_cksum} {
        setvar cube_set_file_contents_needs_replace = '1'
      }
    } else {
      setvar cube_set_file_contents_needs_replace = '1'
    }
  } else {
    setvar cube_set_file_contents_needs_replace = '1'
  }

  if test ${cube_set_file_contents_needs_replace} -eq 1  {
    cube_echo "Updating file contents of ${cube_set_file_contents_target_file} with ${cube_set_file_contents_debug}"
    cp ${cube_set_file_contents_input_file} ${cube_set_file_contents_target_file} || cube_check_return
    if test ${cube_set_file_contents_input_file_needs_remove} -eq 1  {
      rm -f ${cube_set_file_contents_input_file} || cube_check_return
    }
    return 0
  } else {
    if test ${cube_set_file_contents_input_file_needs_remove} -eq 1  {
      rm -f ${cube_set_file_contents_input_file} || cube_check_return
    }
    return 1
  }
}

# Description:
#   Echo a random number between 1 and $1
# Example call:
#   cube_random_number 10
# Arguments:
#   Required:
#     $1: Maximum value
proc cube_random_number {
  cube_check_numargs 1 ${@}
  echo "" | awk "{ srand(); print int(${1} * rand()) + 1; }"
}

# Description:
#   Echo a temporary directory
# Example call:
#   cube_tmpdir
# Arguments: None
proc cube_tmpdir {
  echo "/tmp/"
}

# Description:
#   Set the contents of $1 to the string $@. Create file if it doesn't exist.
# Example call:
#   cube_set_file_contents_string ~/.info "Hello World"
# Arguments:
#   Required:
#     $1: Target file
#     $@: String contents
# Returns: success/true if the file was updated
proc cube_set_file_contents_string {
  cube_check_numargs 2 ${@}
  setvar cube_set_file_contents_target_file = "$1"; shift
  
  setvar cube_set_file_contents_tmp = ""$(cube_tmpdir)/tmpcontents_$(cube_random_number 10000).template""
  echo ${@} > "${cube_set_file_contents_tmp}"
  cube_set_file_contents ${cube_set_file_contents_target_file} ${cube_set_file_contents_tmp} "from string"
  setvar cube_set_file_contents_result = ""$?
  rm ${cube_set_file_contents_tmp}
  return ${cube_set_file_contents_result}
}

# Description:
#   Echo the absolute path of $1 without any symbolic links.
# Example call:
#   cube_readlink /etc/localtime
# Arguments:
#   Required:
#     $1: File
proc cube_readlink {
  cube_check_numargs 1 ${@}

  # http://stackoverflow.com/a/697552/5657303
  # Don't bother trying to short-circuit with readlink because of issues on
  # Mac. We could special case that, but meh.
  #if cube_command_exists readlink ; then
  #  readlink -f $1
  #else
    setvar cube_readlink_target = "$1"
    setvar cube_readlink_path = $(cd -P -- "$(dirname -- "${cube_readlink_target}")" && pwd -P) && setvar cube_readlink_path = "${cube_readlink_path}/$(basename -- "${cube_readlink_target}")"
    
    while test -h ${cube_readlink_path} {
      setvar cube_readlink_dir = $(dirname -- "${cube_readlink_path}")
      setvar cube_readlink_sym = $(readlink "${cube_readlink_path}")
      setvar cube_readlink_path = "$(cd "${cube_readlink_dir}" && cd "$(dirname -- "${cube_readlink_sym}")" && pwd)/$(basename -- "${cube_readlink_sym}")"
    }
    
    echo ${cube_readlink_path}
  #fi
}

# Description:
#   Echo total system memory in bytes
# Example call:
#   cube_total_memory
# Arguments:
#   Optional:
#     #1: [kb|mb|gb| to return in that number
proc cube_total_memory {
  case{
    kb|KB {
      setvar cube_total_memory_divisor = '1024'
      }
    mb|MB {
      setvar cube_total_memory_divisor = '1048576'
      }
    gb|GB {
      setvar cube_total_memory_divisor = '1073741824'
      }
    * {
      setvar cube_total_memory_divisor = '1'
      }
  }
  echo $((($(grep "^MemTotal:" /proc/meminfo | awk '{print $2}')*1024)/${cube_total_memory_divisor}))
}

# Description:
#   Ensure directory $1 exists
# Example call:
#   cube_ensure_directory ~/.ssh/
# Arguments:
#   Requires;
#     $1: Directory name
#   Optional:
#     $2: Permissions (passed to chmod)
#     $3: Owner (passed to chown)
#     $4: Group (passed to chgrp)
proc cube_ensure_directory {
  cube_check_numargs 1 ${@}
  setvar cube_ensure_directory_result = '1'
  
  if ! cube_dir_exists ${1} {
    mkdir -p ${1} || cube_check_return
    setvar cube_ensure_directory_result = '0'
    cube_echo "Created directory ${1}"
  }
  if test ${2} != "" {
    chmod ${2} ${1} || cube_check_return
  }
  if test ${3} != "" {
    chown ${3} ${1} || cube_check_return
  }
  if test ${4} != "" {
    chgrp ${4} ${1} || cube_check_return
  }
  
  return ${cube_ensure_directory_result}
}

# Description:
#   Ensure file $1 exists
# Example call:
#   cube_ensure_file ~/.ssh/authorized_keys
# Arguments:
#   Requires;
#     $1: File name
#   Optional:
#     $2: Permissions (passed to chmod)
#     $3: Owner (passed to chown)
#     $4: Group (passed to chgrp)
proc cube_ensure_file {
  cube_check_numargs 1 ${@}
  setvar cube_ensure_file_result = '1'
  
  if ! cube_file_exists ${1} {
  
    cube_ensure_directory $(dirname "${1}") $2 $3 $4
  
    touch ${1} || cube_check_return
    setvar cube_ensure_file_result = '0'
    cube_echo "Created file ${1}"
  }
  if test ${2} != "" {
    chmod ${2} ${1} || cube_check_return
  }
  if test ${3} != "" {
    chown ${3} ${1} || cube_check_return
  }
  if test ${4} != "" {
    chgrp ${4} ${1} || cube_check_return
  }

  return ${cube_ensure_file_result}
}

# Description:
#   Equivalent to `pushd` with $1
# Example call:
#   cube_pushd ~/.ssh/
# Arguments:
#   Requires;
#     $1: Directory
proc cube_pushd {
  cube_check_numargs 1 ${@}
  
  if cube_command_exists pushd  {
    pushd ${@} || cube_check_return
  } else {
    cube_throw "TODO: Not implemented"
  }
}

# Description:
#   Equivalent to `popd`
# Example call:
#   cube_popd
# Arguments: None
proc cube_popd {
  if cube_command_exists popd  {
    popd ${@} || cube_check_return
  } else {
    cube_throw "TODO: Not implemented"
  }
}

# Description:
#   Return true if the role $1 is set.
# Example call:
#   cube_has_role "database_backup"
# Arguments:
#   Requires;
#     $1: Role name
proc cube_has_role {
  cube_check_numargs 1 ${@}
  
  for cube_has_role_name in ${cubevar_api_roles} {
    if test ${cube_has_role_name} = ${1} {
      return 0
    }
  }
  return 1
}

# Description:
#   Check if the file $1 contains $2
# Example call:
#   cube_file_contains /etc/fstab nfsmount
# Arguments:
#   Required:
#     $1: File name.
#     $2: Case sensitive search string
proc cube_file_contains {
  cube_check_numargs 2 ${@}
  
  # "Normally the exit status is 0 if a line is selected, 1 if no lines were selected, and 2 if an error occurred."
  setvar cube_file_contains_grep_output = "$(grep -l "${2}" "${1}")"
  setvar cube_file_contains_grep_results = ""$?
  
  if test ${cube_file_contains_grep_results} -eq 2 {
    cube_throw ${cube_file_contains_grep_output}
  } else {
    return ${cube_file_contains_grep_results}
  }
}

# Description:
#   Check if stdin contains $1
# Example call:
#   echo "Hello World" | cube_stdin_contains "Hello"
# Arguments:
#   Required:
#     $1: Search
proc cube_stdin_contains {
  cube_check_numargs 1 ${@}
  setvar cube_stdin_contains_output = $(grep -l "${1}" -)
  setvar cube_stdin_contains_result = ""$?
  if test ${cube_stdin_contains_result} -eq 2 {
    cube_throw ${cube_stdin_contains_output}
  } else {
    return ${cube_stdin_contains_result}
  }
}

# Description:
#   Echo the IPv4 address of interface $1
# Example call:
#   cube_interface_ipv4_address eth0
# Arguments:
#   Required:
#     $1: Interface name
proc cube_interface_ipv4_address {
  cube_check_numargs 1 ${@}
  ip -4 -o address show dev ${1} | head -1 | awk '{print $4}' | sed 's/\/.*$//g' || cube_check_return
}

# Description:
#   Echo the IPv6 address of interface $1
# Example call:
#   cube_interface_ipv6_address eth0
# Arguments:
#   Required:
#     $1: Interface name
proc cube_interface_ipv6_address {
  cube_check_numargs 1 ${@}
  ip -6 -o address show dev ${1} | head -1 | awk '{print $4}' | sed 's/\/.*$//g' || cube_check_return
}

# Description:
#   Prompt the question $1 followed by " (y/N)" and prompt for an answer.
#   A blank string answer is equivalent to No. Return true if yes, false otherwise.
# Example call:
#   cube_prompt "Are you sure?"
# Arguments:
#   Required:
#     $1: Prompt test
proc cube_prompt {
  cube_check_numargs 1 ${@}
  while true {
    printf "${1} (y/N)? "
    read cube_prompt_response || cube_check_return
    case{
      [Yy]* {
        return 0
        }
      ""|[Nn]* {
        return 1
        }
      * {
        echo "Please answer yes or no."
        }
    }
  }
}

# Description:
#   Echo full hostname.
# Example call:
#   cube_hostname
# Arguments:
#   Optional:
#     $1: Pass true to return a hostname devoid of any domain.
proc cube_hostname {
  if test ${1} = "" {
    uname -n || cube_check_return
  } else {
    uname -n | sed 's/\..*$//g' || cube_check_return
  }
}

# Description:
#   Check if the $1 user exists
# Example call:
#   cube_user_exists nginx
# Arguments:
#   Required:
#     $1: User name
proc cube_user_exists {
  cube_check_numargs 1 ${@}
  id -u ${1} >/dev/null 2>&1
  return $?
}

# Description:
#   Create the user $1
# Example call:
#   cube_create_user nginx
# Arguments:
#   Required:
#     $1: User name
#   Optional:
#     $2: Shell
proc cube_create_user {
  cube_check_numargs 1 ${@}
  
  useradd ${1} || cube_check_return
  
  if test ${2} != "" {
    usermod -s ${2} ${1} || cube_check_return
  }
}

# Description:
#   Check if the $1 group exists
# Example call:
#   cube_group_exists nginx
# Arguments:
#   Required:
#     $1: Group name
proc cube_group_exists {
  cube_check_numargs 1 ${@}
  cube_file_contains /etc/group "^${1}:"
}

# Description:
#   Create the group $1
# Example call:
#   cube_create_group nginx
# Arguments:
#   Required:
#     $1: Group name
proc cube_create_group {
  cube_check_numargs 1 ${@}
  
  groupadd ${1} || cube_check_return
}

# Description:
#   Check if the $1 group contains the user $2
# Example call:
#   cube_group_contains_user nginx nginx
# Arguments:
#   Required:
#     $1: Group name
#     $2: User name
proc cube_group_contains_user {
  cube_check_numargs 2 ${@}
  
  for cube_group_contains_user in $(groups "${2}" | sed "s/${2} : //g") {
    if test ${cube_group_contains_user} = ${1} {
      return 0
    }
  }
  return 1
}

# Description:
#   Add the user $2 to group $1
# Example call:
#   cube_add_group_user nginx nginx
# Arguments:
#   Required:
#     $1: Group name
#     $2: User name
#   Optional:
#     $3: true if group is primary
proc cube_add_group_user {
  cube_check_numargs 2 ${@}
  
  if test ${3} = "" {
    usermod -a -G ${1} ${2}
  } else {
    usermod -g ${1} ${2}
  }
}

# Description:
#   Include the ${1} cube
# Example call:
#   cube_include core_cube
# Arguments:
#   Required:
#     $1: CUBE name
proc cube_include {
  cube_check_numargs 1 ${@}
  
  setvar cube_include_name = "${1%%/}"
  if test -d "../${cube_include_name}" {
    setvar cube_include_name_base = $(basename "${cube_include_name}" .sh)
    if test -r "../${cube_include_name}/${cube_include_name_base}.sh" {
      cube_echo "Including ${cube_include_name} cube..."
      source "../${cube_include_name}/${cube_include_name_base}.sh"
    } else {
      cube_throw "Cannot read ${cube_include_name}/${cube_include_name_base}.sh"
    }
  } elif test -r "../${cube_include_name}" {
    cube_echo "Including ${cube_include_name} cube..."
    source "../${cube_include_name}"
  } elif test -r "${cube_include_name}.sh" {
    cube_echo "Including ${cube_include_name} cube..."
    source "../${cube_include_name}.sh"
  } else {
    cube_throw "Cube ${cube_include_name} not found (did you upload it with -i ${cube_include_name} ?)."
  }
}

# Append space-delimited service names to this variable to restart services after all CUBEs and COMMANDs
setvar cubevar_api_post_restart = """"

setvar cubevar_api_roles = """"

###################
# End Public APIs #
###################

################################
# Core internal implementation #
################################

# If we're being sourced on the remote machine, then we don't want to run any of the below
if test ${POSIXCUBE_APIS_ONLY} = "" {
  setvar p666_debug = '0'
  setvar p666_quiet = '0'
  setvar p666_skip_init = '0'
  setvar p666_keep_exec = '0'
  setvar p666_skip_host_errors = '0'
  setvar p666_hosts = """"
  setvar p666_cubes = """"
  setvar p666_include_cubes = """"
  setvar p666_envar_scripts = """"
  setvar p666_envar_scripts_password = """"
  setvar p666_user = "${USER}"
  setvar p666_cubedir = ""~/posixcubes/""
  setvar p666_roles = """"
  setvar p666_options = """"
  setvar p666_specfile = ""./cubespecs.ini""
  setvar p666_parallel = '0'
  setvar p666_async_cubes = '0'
  setvar p666_default_envars = ""envars*sh envars*sh.enc""
  
  if test $(cube_shell) -eq ${POSIXCUBE_SHELL_BASH} {
    setvar p666_parallel = '64'
  }

  proc p666_show_version {
    p666_printf "posixcube.sh version ${POSIXCUBE_VERSION}\n"
  }

  proc p666_printf {
    setvar p666_printf_str = "$1"
    shift
    printf "[$(date)] ${p666_printf_str}" ${@}
  }

  proc p666_printf_error {
    setvar p666_printf_str = "$1"
    shift
    printf "\n[$(date)] ${POSIXCUBE_COLOR_RED}Error${POSIXCUBE_COLOR_RESET}: ${p666_printf_str}\n\n" ${@} 1>&2
  }
  
  proc p666_exit {
    for p666_envar_script in ${p666_envar_scripts} {
      setvar p666_envar_script_enc_matches = $(expr ${p666_envar_script} : '.*\.dec$')
      if test ${p666_envar_script_enc_matches} -ne 0 {
        # If multiple posixcube.sh executions run at the same time, then one of them might
        # delete the decryption file, but that's okay
        rm ${p666_envar_script} 2>/dev/null
      }
    }

    test ${p666_keep_exec} -eq 0 && rm -f ${p666_script} 2>/dev/null
    
    exit ${1}
  }

  proc p666_install {
    setvar p666_func_result = '0'
    if test -d "/etc/bash_completion.d/" {
      setvar p666_autocomplete_file = "/etc/bash_completion.d/posixcube_completion.sh"
      
      # Autocomplete Hostnames for SSH etc.
      # by Jean-Sebastien Morisset (http://surniaulula.com/)
      
      cat <<< ''' | tee ${p666_autocomplete_file} > /dev/null
_posixcube_complete() {
  COMPREPLY=()
  cur="${COMP_WORDS[COMP_CWORD]}"
  prev="${COMP_WORDS[COMP_CWORD-1]}"
  case "${prev}" in
    \-h)
      p666_host_list=$({ 
        for c in /etc/ssh_config /etc/ssh/ssh_config ~/.ssh/config
        do [ -r $c ] && sed -n -e 's/^Host[[:space:]]//p' -e 's/^[[:space:]]*HostName[[:space:]]//p' $c
        done
        for k in /etc/ssh_known_hosts /etc/ssh/ssh_known_hosts ~/.ssh/known_hosts
        do [ -r $k ] && egrep -v '^[#\[]' $k|cut -f 1 -d ' '|sed -e 's/[,:].*//g'
        done
        sed -n -e 's/^[0-9][0-9\.]*//p' /etc/hosts; }|tr ' ' '\n'|grep -v '*')
      COMPREPLY=( $(compgen -W "${p666_host_list}" -- $cur))
      ;;
    \-z)
      p666_complete_specs="$(sed 's/=.*//g' cubespecs.ini)"
      COMPREPLY=( $(compgen -W "${p666_complete_specs}" -- $cur))
      ;;
    *)
      ;;
  esac
  return 0
}
complete -o default -F _posixcube_complete posixcube.sh
'''
tee ${p666_autocomplete_file} > /dev/null
_posixcube_complete() {
  COMPREPLY=()
  cur="${COMP_WORDS[COMP_CWORD]}"
  prev="${COMP_WORDS[COMP_CWORD-1]}"
  case "${prev}" in
    \-h)
      p666_host_list=$({ 
        for c in /etc/ssh_config /etc/ssh/ssh_config ~/.ssh/config
        do [ -r $c ] && sed -n -e 's/^Host[[:space:]]//p' -e 's/^[[:space:]]*HostName[[:space:]]//p' $c
        done
        for k in /etc/ssh_known_hosts /etc/ssh/ssh_known_hosts ~/.ssh/known_hosts
        do [ -r $k ] && egrep -v '^[#\[]' $k|cut -f 1 -d ' '|sed -e 's/[,:].*//g'
        done
        sed -n -e 's/^[0-9][0-9\.]*//p' /etc/hosts; }|tr ' ' '\n'|grep -v '*')
      COMPREPLY=( $(compgen -W "${p666_host_list}" -- $cur))
      ;;
    \-z)
      p666_complete_specs="$(sed 's/=.*//g' cubespecs.ini)"
      COMPREPLY=( $(compgen -W "${p666_complete_specs}" -- $cur))
      ;;
    *)
      ;;
  esac
  return 0
}
complete -o default -F _posixcube_complete posixcube.sh
HEREDOC

      setvar p666_func_result = ""$?
      if test ${p666_func_result} -eq 0 {
        chmod +x ${p666_autocomplete_file}
        setvar p666_func_result = ""$?
        if test ${p666_func_result} -eq 0 {
          source ${p666_autocomplete_file}
          setvar p666_func_result = ""$?
          if test ${p666_func_result} -eq 0 {
            p666_printf "Installed Bash programmable completion script into ${p666_autocomplete_file}\n"
          } else {
            p666_printf "Could not execute ${p666_autocomplete_file}\n"
          }
        } else {
          p666_printf "Could not chmod +x ${p666_autocomplete_file}\n"
        }
      } else {
        p666_printf "Could not create ${p666_autocomplete_file}\n"
        p666_printf "You may need to try with sudo. For example:\n"
        p666_printf "  sudo $(cube_current_script_abs_path) -b && . ${p666_autocomplete_file}\n"
        p666_printf "You only need to source the command the first time. Subsequent shells will automatically source it.\n"
      }
    } else {
      p666_printf "No directory /etc/bash_completion.d/ found, skipping Bash programmable completion installation.\n"
    }
    exit ${p666_func_result}
  }

  setvar p666_all_hosts = """"

  proc p666_process_hostname {
    setvar p666_hostname_wildcard = $(expr "${1}" : '.*\*.*')
    if test ${p666_hostname_wildcard} -ne 0 {
    
      # Use or create cache of hosts
      if test ${p666_all_hosts} = "" {
        setvar p666_all_hosts = $({ 
          for c in /etc/ssh_config /etc/ssh/ssh_config ~/.ssh/config
          do [ -r $c ] && sed -n -e 's/^Host[[:space:]]//p' -e 's/^[[:space:]]*HostName[[:space:]]//p' $c
          done
          for k in /etc/ssh_known_hosts /etc/ssh/ssh_known_hosts ~/.ssh/known_hosts
          do [ -r $k ] && egrep -v '^[#\[]' $k|cut -f 1 -d ' '|sed -e 's/[,:].*//g'
          done
          sed -n -e 's/^[0-9][0-9\.]*//p' /etc/hosts; }|tr '\n' ' '|grep -v '*')
      }
      
      setvar p666_process_hostname_search = $(printf "${1}" | sed 's/\*/\.\*/g')
      
      setvar p666_process_hostname_list = """"
      for p666_all_host in ${p666_all_hosts} {
        setvar p666_all_host_match = $(expr ${p666_all_host} : ${p666_process_hostname_search})
        if test ${p666_all_host_match} -ne 0 {
          setvar p666_process_hostname_list = $(cube_append_str "${p666_process_hostname_list}" "${p666_all_host}")
        }
      }
      echo ${p666_process_hostname_list}
    } else {
      echo ${1}
    }
  }
  
  proc p666_process_options {
    # getopts processing based on http://stackoverflow.com/a/14203146/5657303
    setvar OPTIND = '1' # Reset in case getopts has been used previously in the shell.
    
    while getopts "?vdqbskyah:u:c:e:p:w:r:o:z:i:" p666_opt ${@} {
      case (p666_opt) {
      \? {
        p666_show_usage
        }
      v {
        p666_show_version
        exit 1
        }
      d {
        setvar p666_debug = '1'
        }
      q {
        setvar p666_quiet = '1'
        }
      s {
        setvar p666_skip_init = '1'
        }
      k {
        setvar p666_keep_exec = '1'
        }
      y {
        setvar p666_skip_host_errors = '1'
        }
      a {
        setvar p666_async_cubes = '1'
        }
      b {
        p666_install
        }
      h {
        setvar p666_processed_hostname = $(p666_process_hostname "${OPTARG}")
        if test ${p666_processed_hostname} != "" {
          setvar p666_hosts = $(cube_append_str "${p666_hosts}" "${p666_processed_hostname}")
        } else {
          p666_printf_error "No known hosts match ${OPTARG} from ${p666_all_hosts}"
          exit 1
        }
        }
      c {
        setvar p666_cubes = $(cube_append_str "${p666_cubes}" "${OPTARG}")
        }
      i {
        setvar p666_include_cubes = $(cube_append_str "${p666_include_cubes}" "${OPTARG}")
        }
      e {
        if test ! -r ${OPTARG} {
          p666_printf_error "Could not find ${OPTARG} ENVAR script."
          exit 1
        }
        setvar p666_envar_scripts = $(cube_append_str "${p666_envar_scripts}" "${OPTARG}")
        }
      u {
        setvar p666_user = "${OPTARG}"
        }
      p {
        setvar p666_envar_scripts_password = "${OPTARG}"
        }
      w {
        setvar p666_envar_scripts_password = "$(cat ${OPTARG})" || cube_check_return
        }
      r {
        setvar p666_roles = $(cube_append_str "${p666_roles}" "${OPTARG}")
        }
      o {
        # Break up into name and value
        setvar p666_option_name = $(echo "${OPTARG}" | sed 's/=.*//')
        setvar p666_option_value = $(echo "${OPTARG}" | sed "s/^${p666_option_name}=//")
        setvar p666_option_value = $(p666_process_hostname "${p666_option_value}")
        setvar p666_options = $(cube_append_str "${p666_options}" "${p666_option_name}=\"${p666_option_value}\"" "${POSIXCUBE_NEWLINE}")
        }
      z {
        if test -r ${p666_specfile} {
          setvar p666_foundspec = '0'
          
          setvar p666_foundspec_names = """"
          
          while read p666_specfile_line {
            setvar p666_specfile_line_name = $(echo "${p666_specfile_line}" | sed 's/=.*//')
            setvar p666_foundspec_names = $(cube_append_str "${p666_foundspec_names}" "${p666_specfile_line_name}")
            if test ${p666_specfile_line_name} = ${OPTARG} {
              setvar p666_foundspec = '1'
              setvar p666_specfile_line_value = $(echo "${p666_specfile_line}" | sed "s/^${p666_specfile_line_name}=//")
              p666_process_options ${p666_specfile_line_value}
              break
            }
          } < "${p666_specfile}"
          
          if test ${p666_foundspec} -eq 0 {
            p666_printf_error "Could not find ${OPTARG} in ${p666_specfile} file with specs ${p666_foundspec_names}"
            exit 1
          }
        } else {
          p666_printf_error "Could not find ${p666_specfile} file"
          exit 1
        }
        }
      }
    }
  }

  p666_process_options ${@}
  
  # If no password specified, check for the ~/.posixcube.pwd file
  if test ${p666_envar_scripts_password} = "" && test -r ~/.posixcube.pwd {
    setvar p666_envar_scripts_password = "$(cat ~/.posixcube.pwd)" || cube_check_return
  }

  shift $((${OPTIND}-1))

  test $1 = "--" && shift
  
  if test ${p666_envar_scripts} = "" {
    setvar p666_envar_scripts = "$(ls -1 ${p666_default_envars} 2>/dev/null | paste -sd ' ' -)"
  }

  if test ${p666_envar_scripts} != "" {
    test ${p666_debug} -eq 1 && p666_printf "Using ENVAR files: ${p666_envar_scripts}\n"
  }
  
  setvar p666_commands = "${@}"

  if test ${p666_hosts} = "" {
    # If there are no hosts, check COMMANDs for sub-commands
    if test ${p666_commands} != "" {
      case{
        edit|show|source {
          if test ${p666_envar_scripts} != "" {
            for p666_envar_script in ${p666_envar_scripts} {
              setvar p666_envar_scripts_enc = $(expr ${p666_envar_script} : '.*enc$')
              if test ${p666_envar_scripts_enc} -ne 0 {
                if cube_command_exists gpg  {
                  setvar p666_envar_script_new = $(echo "${p666_envar_script}" | sed 's/enc$/dec/g')
                  
                  if test ${p666_envar_scripts_password} = "" {
                    p666_printf "Enter the password for ${p666_envar_script}:\n"
                    gpg --output ${p666_envar_script_new} --yes --decrypt ${p666_envar_script} || cube_check_return
                  } else {
                    test ${p666_debug} -eq 1 && p666_printf "Decrypting ${p666_envar_script} ...\n"
                    setvar p666_gpg_output = "$(echo "${p666_envar_scripts_password}" | gpg --passphrase-fd 0 --batch --yes --output "${p666_envar_script_new}" --decrypt "${p666_envar_script}" 2>&1)" || cube_check_return ${p666_gpg_output}
                  }
                  
                  case{
                    show {
                      p666_printf "Contents of ${p666_envar_script}:\n"
                      cat ${p666_envar_script_new}
                      }
                    source {
                      chmod u+x ${p666_envar_script_new}
                      source "$(cube_readlink "${p666_envar_script_new}")"
                      test ${p666_debug} -eq 1 && p666_printf "Sourced ${p666_envar_script}...\n"
                      }
                    edit {
                      ${EDITOR:-vi} ${p666_envar_script_new} || cube_check_return
                      
                      if test ${p666_envar_scripts_password} = "" {
                        p666_printf "Enter the password to re-encrypt ${p666_envar_script}:\n"
                        gpg --yes --s2k-mode 3 --s2k-count 65536 --force-mdc --cipher-algo AES256 --s2k-digest-algo SHA512 -o ${p666_envar_script} --symmetric ${p666_envar_script_new} || cube_check_return
                      } else {
                        test ${p666_debug} -eq 1 && p666_printf "Re-encrypting ${p666_envar_script} ...\n"
                        
                        setvar p666_gpg_output = "$(echo "${p666_envar_scripts_password}" | gpg --batch --passphrase-fd 0 --yes --no-use-agent --s2k-mode 3 --s2k-count 65536 --force-mdc --cipher-algo AES256 --s2k-digest-algo SHA512 -o "${p666_envar_script}" --symmetric "${p666_envar_script_new}" 2>&1)" || cube_check_return ${p666_gpg_output}
                      }
                      }
                    * {
                      p666_show_usage "Not implemented"
                      }
                  }
                  
                  rm -f ${p666_envar_script_new} || cube_check_return
                } else {
                  p666_show_usage "gpg program not found on the PATH"
                }
              } else {
                case{
                  show {
                    cat ${p666_envar_script} | grep -v "^#!/bin/sh$"
                    }
                  source {
                    source "$(cube_readlink "${p666_envar_script}")"
                    test ${p666_debug} -eq 1 && p666_printf "Sourced ${p666_envar_script}...\n"
                    }
                  * {
                    p666_show_usage "Not implemented"
                    }
                }
              }
            }
            
            case{
              source {
                true
                }
              * {
                exit 0
                }
            }
          } else {
            p666_show_usage "Sub-COMMAND without -e ENVAR file and ${p666_default_envars} not found."
          }
          case{
            source {
              true
              }
            * {
              exit 0
              }
          }
          }
        * {
          p666_show_usage "Unknown sub-COMMAND ${1}"
          }
      }
    } else {
      p666_show_usage "No hosts specified with -h and no sub-COMMAND specified."
    }
  }
  
  if test ${POSIXCUBE_SOURCED} = "" {
    if test ${p666_commands} = "" && test ${p666_cubes} = "" {
      p666_show_usage "No COMMANDs or CUBEs specified."
    }

    test ${p666_debug} -eq 1 && p666_show_version
    
    proc p666_handle_remote_response {
      setvar p666_handle_remote_response_context = "${1}"
      if test ${p666_handle_remote_response_context} = "" {
        setvar p666_handle_remote_response_context = ""Last command""
      }
      setvar p666_host_output_color = ${POSIXCUBE_COLOR_GREEN}
      setvar p666_host_output = """"
      if test ${p666_host_output_result} -ne 0 {
        setvar p666_host_output_color = ${POSIXCUBE_COLOR_RED}
        setvar p666_host_output = ""${p666_handle_remote_response_context} failed with return code ${p666_host_output_result}""
      } else {
        test ${p666_debug} -eq 1 && setvar p666_host_output = ""Commands succeeded.""
      }
      test ${p666_host_output} != "" && p666_printf "[${p666_host_output_color}${p666_host}${POSIXCUBE_COLOR_RESET}] %s\n" ${p666_host_output}
      if test ${p666_host_output_result} -ne 0 {
        p666_exit ${p666_host_output_result}
      }
    }

    proc p666_remote_ssh {
      setvar p666_remote_ssh_commands = "$1"
      test ${p666_debug} -eq 1 && p666_printf "[${POSIXCUBE_COLOR_GREEN}${p666_host}${POSIXCUBE_COLOR_RESET}] Executing ssh ${p666_user}@${p666_host} \"${p666_remote_ssh_commands}\" ...\n"
      
      if test ${p666_parallel} -gt 0 && test ${p666_async} -eq 1 {
        ssh -o ConnectTimeout=10 ${p666_user}@${p666_host} ${p666_remote_ssh_commands} 2>&1 &
        setvar p666_wait_pids = $(cube_append_str "${p666_wait_pids}" "$!")
      } else {
        ssh -o ConnectTimeout=10 ${p666_user}@${p666_host} ${p666_remote_ssh_commands} 2>&1
        setvar p666_host_output_result = ""$?
        
        test ${p666_debug} -eq 1 && p666_printf "Finished executing on ${p666_host}\n"
        
        p666_handle_remote_response "Remote commands through SSH"
        return ${p666_host_output_result}
      }
    }

    proc p666_remote_transfer {
      setvar p666_remote_transfer_source = "$1"
      setvar p666_remote_transfer_dest = "$2"
      test ${p666_debug} -eq 1 && p666_printf "[${POSIXCUBE_COLOR_GREEN}${p666_host}${POSIXCUBE_COLOR_RESET}] Executing rsync ${p666_remote_transfer_source} to ${p666_user}@${p666_host}:${p666_remote_transfer_dest} ...\n"
      
      # Don't use -a so that ownership is picked up from the specified user
      if test ${p666_parallel} -gt 0 && test ${p666_async} -eq 1 {
        test ${p666_debug} -eq 1 && p666_printf "Rsyncing in background: ${p666_remote_transfer_source} ${p666_user}@${p666_host}:${p666_remote_transfer_dest}\n"
        rsync -rlpt ${p666_remote_transfer_source} "${p666_user}@${p666_host}:${p666_remote_transfer_dest}" &
        setvar p666_wait_pids = $(cube_append_str "${p666_wait_pids}" "$!")
      } else {
        test ${p666_debug} -eq 1 && p666_printf "Rsyncing in foreground: ${p666_remote_transfer_source} ${p666_user}@${p666_host}:${p666_remote_transfer_dest}\n"
        rsync -rlpt ${p666_remote_transfer_source} "${p666_user}@${p666_host}:${p666_remote_transfer_dest}"
        setvar p666_host_output_result = ""$?
        p666_handle_remote_response "rsync"
      }
    }

    setvar p666_cubedir = ${p666_cubedir%/}
    
    setvar p666_script_name = "$(cube_current_script_name)"
    setvar p666_script_path = "$(cube_current_script_abs_path)"
    
    setvar p666_remote_script = ""${p666_cubedir}/${p666_script_name}""

    # Create a script that we'll execute on the remote end
    setvar p666_script_contents = ""cube_initial_directory=\${PWD}""
    setvar p666_script_envar_contents = """"
    
    setvar p666_envar_scripts_final = """"

    for p666_envar_script in ${p666_envar_scripts} {
    
      setvar p666_envar_script_remove = '0'
    
      setvar p666_envar_script_enc_matches = $(expr ${p666_envar_script} : '.*\.enc$')
      
      if test ${p666_envar_script_enc_matches} -ne 0 {
        if cube_command_exists gpg  {
          test ${p666_debug} -eq 1 && p666_printf "Decrypting ${p666_envar_script}"
          
          setvar p666_envar_script_new = $(echo "${p666_envar_script}" | sed 's/enc$/dec/g')
          
          if test ${p666_envar_scripts_password} = "" {
            p666_printf "Enter the password for ${p666_envar_script}:\n"
            gpg --output ${p666_envar_script_new} --yes --decrypt ${p666_envar_script} || cube_check_return
          } else {
            test ${p666_debug} -eq 1 && p666_printf "Decrypting ${p666_envar_script} ...\n"
            setvar p666_gpg_output = "$(echo "${p666_envar_scripts_password}" | gpg --passphrase-fd 0 --batch --yes --output "${p666_envar_script_new}" --decrypt "${p666_envar_script}" 2>&1)" || cube_check_return ${p666_gpg_output}
          }
          
          setvar p666_envar_script = "${p666_envar_script_new}"
          setvar p666_envar_script_remove = '1'
        } else {
          p666_printf_error "gpg program not found on the PATH"
          p666_exit 1
        }
      }
      
      setvar p666_envar_scripts_final = $(cube_append_str "${p666_envar_scripts_final}" "${p666_envar_script}")
      
      chmod u+x ${p666_envar_script}
      
      setvar p666_script_contents = ""${p666_script_contents}
cd ${p666_cubedir}/ || cube_check_return
. ${p666_cubedir}/$(basename ${p666_envar_script}) || cube_check_return""

      if test ${p666_envar_script_remove} -eq 1 {
        setvar p666_script_contents = ""${p666_script_contents}
rm -f ${p666_cubedir}/$(basename ${p666_envar_script}) || cube_check_return""
      }
    }
    
    setvar p666_envar_scripts = "${p666_envar_scripts_final}"
    
    for p666_cube in ${p666_cubes} {
      if test -d ${p666_cube} {
        setvar p666_cube_name = $(basename "${p666_cube}")
        if test -r "${p666_cube}/${p666_cube_name}.sh" {
          chmod u+x ${p666_cube}/*.sh
          setvar p666_cube = ${p666_cube%/}
          setvar p666_script_contents = ""${p666_script_contents}
cd ${p666_cubedir}/${p666_cube}/ || cube_check_return
cube_echo \"Started cube: ${p666_cube_name}\"
POSIXCUBE_CUBE_NAME=\"${p666_cube_name}\" POSIXCUBE_CUBE_NAME_WITH_PREFIX=\" ${p666_cube_name}.sh\" . ${p666_cubedir}/${p666_cube}/${p666_cube_name}.sh || cube_check_return \"Last command in cube\"
cube_echo \"Finished cube: ${p666_cube_name}\"
""
          if test -r "${p666_cube}/envars.sh" {
            setvar p666_script_envar_contents = ""${p666_script_envar_contents}
. ${p666_cubedir}/${p666_cube}/envars.sh || cube_check_return \"Failed loading cube envars\"
""
          }
        } else {
          p666_printf_error "Could not find ${p666_cube_name}.sh in cube ${p666_cube} directory."
          p666_exit 1
        }
      } elif test -r ${p666_cube} {
        setvar p666_cube_name = $(basename "${p666_cube}")
        chmod u+x ${p666_cube}
        setvar p666_script_contents = ""${p666_script_contents}
cd ${p666_cubedir}/ || cube_check_return
cube_echo \"Started cube: ${p666_cube_name}\"
POSIXCUBE_CUBE_NAME=\"${p666_cube_name}\" POSIXCUBE_CUBE_NAME_WITH_PREFIX=\" ${p666_cube_name}\" . ${p666_cubedir}/${p666_cube_name} || cube_check_return \"Last command in cube\"
cube_echo \"Finished cube: ${p666_cube_name}\"
""
      } elif test -r "${p666_cube}.sh" {
        setvar p666_cube_name = $(basename "${p666_cube}.sh")
        chmod u+x "${p666_cube}.sh"
        setvar p666_script_contents = ""${p666_script_contents}
cd ${p666_cubedir}/ || cube_check_return
cube_echo \"Started cube: ${p666_cube_name}\"
POSIXCUBE_CUBE_NAME=\"${p666_cube_name}\" POSIXCUBE_CUBE_NAME_WITH_PREFIX=\" ${p666_cube_name}\" . ${p666_cubedir}/${p666_cube_name} || cube_check_return \"Last command in cube\"
cube_echo \"Finished cube: ${p666_cube_name}\"
""
      } else {
        p666_printf_error "Cube ${p666_cube} could not be found as a directory or script, or you don't have read permissions."
        p666_exit 1
      }
      setvar p666_script_contents = ""${p666_script_contents}${POSIXCUBE_NEWLINE}cd \${cube_initial_directory}""
    }
    
    for p666_cube in ${p666_include_cubes} {
      if test -d ${p666_cube} {
        setvar p666_cube_name = $(basename "${p666_cube}")
        if test -r "${p666_cube}/${p666_cube_name}.sh" {
          chmod u+x ${p666_cube}/*.sh
        }
      } elif test -r ${p666_cube} {
        chmod u+x ${p666_cube}
      } elif test -r "${p666_cube}.sh" {
        chmod u+x "${p666_cube}.sh"
      } else {
        p666_printf_error "Cube ${p666_cube} could not be found as a directory or script, or you don't have read permissions."
        p666_exit 1
      }
    }
    
    if test ${p666_commands} != "" {
      setvar p666_script_contents = ""${p666_script_contents}
  ${p666_commands}""
    }
    
    setvar p666_script = ""./cube_exec.sh""
    
    cat <<< """ > "${p666_script}"
#!/bin/sh
POSIXCUBE_APIS_ONLY=1
. ${p666_remote_script}
if [ '$'? -ne 0 ] ; then
  echo "Could not source ${p666_remote_script} script" 1>&2
  exit 1
fi

POSIXCUBE_DEBUG=${p666_debug}
cubevar_api_roles="${p666_roles}"
${p666_script_envar_contents}
${p666_options}
${p666_script_contents}

if [ "'$'{cubevar_api_post_restart}" != "" ]; then
  for p666_post in '$'{cubevar_api_post_restart}; do
    cube_service restart "'$'{p666_post}"
  done
fi
"""
> "${p666_script}"
#!/bin/sh
POSIXCUBE_APIS_ONLY=1
. ${p666_remote_script}
if [ \$? -ne 0 ] ; then
  echo "Could not source ${p666_remote_script} script" 1>&2
  exit 1
fi

POSIXCUBE_DEBUG=${p666_debug}
cubevar_api_roles="${p666_roles}"
${p666_script_envar_contents}
${p666_options}
${p666_script_contents}

if [ "\${cubevar_api_post_restart}" != "" ]; then
  for p666_post in \${cubevar_api_post_restart}; do
    cube_service restart "\${p666_post}"
  done
fi
HEREDOC

    chmod +x ${p666_script}
    
    setvar p666_upload = ""${p666_script} ""

    if test ${p666_cubes} != "" {
      for p666_cube in ${p666_cubes} {
        if test -d ${p666_cube} {
          setvar p666_cube_name = $(basename "${p666_cube}")
          if test -r "${p666_cube}/${p666_cube_name}.sh" {
            setvar p666_cube = ${p666_cube%/}
            setvar p666_upload = ""${p666_upload} ${p666_cube}""
          }
        } elif test -r ${p666_cube} {
          setvar p666_cube_name = $(basename "${p666_cube}")
          setvar p666_upload = ""${p666_upload} ${p666_cube}""
        } elif test -r "${p666_cube}.sh" {
          setvar p666_cube_name = $(basename "${p666_cube}.sh")
          setvar p666_upload = ""${p666_upload} ${p666_cube}.sh""
        }
      }
    }

    if test ${p666_include_cubes} != "" {
      for p666_cube in ${p666_include_cubes} {
        if test -d ${p666_cube} {
          setvar p666_cube_name = $(basename "${p666_cube}")
          if test -r "${p666_cube}/${p666_cube_name}.sh" {
            setvar p666_cube = ${p666_cube%/}
            setvar p666_upload = ""${p666_upload} ${p666_cube}""
          }
        } elif test -r ${p666_cube} {
          setvar p666_cube_name = $(basename "${p666_cube}")
          setvar p666_upload = ""${p666_upload} ${p666_cube}""
        } elif test -r "${p666_cube}.sh" {
          setvar p666_cube_name = $(basename "${p666_cube}.sh")
          setvar p666_upload = ""${p666_upload} ${p666_cube}.sh""
        } else {
          p666_printf_error "Could not find ${p666_cube}"
          p666_exit 1
        }
      }
    }

    test ${p666_quiet} -eq 0 && p666_printf "Transferring files to hosts: ${p666_hosts} ...\n"
    
    setvar p666_async = '1'
    
    if test ${p666_skip_init} -eq 0 {
      setvar p666_wait_pids = """"
      for p666_host in ${p666_hosts} {
        # Debian doesn't have rsync installed by default
        #p666_remote_ssh "[ ! -d \"${p666_cubedir}\" ] && mkdir -p ${p666_cubedir}"
        p666_remote_ssh "[ ! -d \"${p666_cubedir}\" ] && mkdir -p ${p666_cubedir}; RC=\$?; command -v rsync >/dev/null 2>&1 || (command -v apt-get >/dev/null 2>&1 && apt-get -y install rsync); exit \${RC};"
      }
      if test ${p666_wait_pids} != "" {
        test ${p666_debug} -eq 1 && p666_printf "Waiting on initialization PIDs: ${p666_wait_pids} ...\n"
        wait ${p666_wait_pids}
        setvar p666_host_output_result = ""$?
        p666_handle_remote_response "Remote commands through SSH"
      }
    }
    
    setvar p666_wait_pids = """"
    for p666_host in ${p666_hosts} {
      if test ${p666_skip_init} -eq 0 {
        p666_remote_transfer "${p666_upload} ${p666_script_path} ${p666_envar_scripts}" "${p666_cubedir}/"
      } else {
        p666_remote_transfer "${p666_upload} ${p666_envar_scripts}" "${p666_cubedir}/"
      }
    }
    
    if test ${p666_wait_pids} != "" {
      test ${p666_debug} -eq 1 && p666_printf "Waiting on transfer PIDs: ${p666_wait_pids} ...\n"
      wait ${p666_wait_pids}
      setvar p666_host_output_result = ""$?
      p666_handle_remote_response "rsync"
    }

    setvar p666_wait_pids = """"
    setvar p666_async = ${p666_async_cubes}
    
    for p666_host in ${p666_hosts} {
      test ${p666_quiet} -eq 0 && p666_printf "[${POSIXCUBE_COLOR_GREEN}${p666_host}${POSIXCUBE_COLOR_RESET}] Executing on ${p666_host} ...\n"
      p666_remote_ssh ". ${p666_cubedir}/${p666_script}"
      setvar p666_remote_ssh_result = ""$?
      if test ${p666_async} -eq 0 {
        test ${p666_skip_host_errors} -eq 0 && test ${p666_remote_ssh_result} -ne 0 && p666_exit ${p666_remote_ssh_result}
      }
    }

    if test ${p666_wait_pids} != "" {
      test ${p666_debug} -eq 1 && p666_printf "Waiting on cube execution PIDs: ${p666_wait_pids} ...\n"
      wait ${p666_wait_pids}
      setvar p666_host_output_result = ""$?
      p666_handle_remote_response "Cube execution"
    }

    p666_exit 0
  }
}