#!/bin/bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Common utilities, variables and checks for all build scripts.
set -o errexit
set -o nounset
set -o pipefail

setglobal USER_ID = $[id -u]
setglobal GROUP_ID = $[id -g]

setglobal DOCKER_OPTS = $(DOCKER_OPTS:-"")
setglobal DOCKER = ''(docker ${DOCKER_OPTS})
setglobal DOCKER_HOST = $(DOCKER_HOST:-"")
setglobal DOCKER_MACHINE_NAME = $(DOCKER_MACHINE_NAME:-"kube-dev")
readonly DOCKER_MACHINE_DRIVER=$(DOCKER_MACHINE_DRIVER:-"virtualbox --virtualbox-cpu-count -1")

# This will canonicalize the path
setglobal KUBE_ROOT = $[cd $[dirname $(BASH_SOURCE)]/.. && pwd -P]

source "$(KUBE_ROOT)/hack/lib/init.sh"

# Constants
readonly KUBE_BUILD_IMAGE_REPO=kube-build
readonly KUBE_BUILD_IMAGE_CROSS_TAG="$[cat $(KUBE_ROOT)/build/build-image/cross/VERSION]"

# This version number is used to cause everyone to rebuild their data containers
# and build image.  This is especially useful for automated build systems like
# Jenkins.
#
# Increment/change this number if you change the build image (anything under
# build/build-image) or change the set of volumes in the data container.
readonly KUBE_BUILD_IMAGE_VERSION_BASE="$[cat $(KUBE_ROOT)/build/build-image/VERSION]"
readonly KUBE_BUILD_IMAGE_VERSION="$(KUBE_BUILD_IMAGE_VERSION_BASE)-$(KUBE_BUILD_IMAGE_CROSS_TAG)"

# Here we map the output directories across both the local and remote _output
# directories:
#
# *_OUTPUT_ROOT    - the base of all output in that environment.
# *_OUTPUT_SUBPATH - location where golang stuff is built/cached.  Also
#                    persisted across docker runs with a volume mount.
# *_OUTPUT_BINPATH - location where final binaries are placed.  If the remote
#                    is really remote, this is the stuff that has to be copied
#                    back.
# OUT_DIR can come in from the Makefile, so honor it.
readonly LOCAL_OUTPUT_ROOT="$(KUBE_ROOT)/$(OUT_DIR:-_output)"
readonly LOCAL_OUTPUT_SUBPATH="$(LOCAL_OUTPUT_ROOT)/dockerized"
readonly LOCAL_OUTPUT_BINPATH="$(LOCAL_OUTPUT_SUBPATH)/bin"
readonly LOCAL_OUTPUT_GOPATH="$(LOCAL_OUTPUT_SUBPATH)/go"
readonly LOCAL_OUTPUT_IMAGE_STAGING="$(LOCAL_OUTPUT_ROOT)/images"

# This is a symlink to binaries for "this platform" (e.g. build tools).
readonly THIS_PLATFORM_BIN="$(LOCAL_OUTPUT_ROOT)/bin"

readonly REMOTE_ROOT="/go/src/$(KUBE_GO_PACKAGE)"
readonly REMOTE_OUTPUT_ROOT="$(REMOTE_ROOT)/_output"
readonly REMOTE_OUTPUT_SUBPATH="$(REMOTE_OUTPUT_ROOT)/dockerized"
readonly REMOTE_OUTPUT_BINPATH="$(REMOTE_OUTPUT_SUBPATH)/bin"
readonly REMOTE_OUTPUT_GOPATH="$(REMOTE_OUTPUT_SUBPATH)/go"

# This is the port on the workstation host to expose RSYNC on.  Set this if you
# are doing something fancy with ssh tunneling.
readonly KUBE_RSYNC_PORT="$(KUBE_RSYNC_PORT:-)"

# This is the port that rsync is running on *inside* the container. This may be
# mapped to KUBE_RSYNC_PORT via docker networking.
readonly KUBE_CONTAINER_RSYNC_PORT=8730

# Get the set of master binaries that run in Docker (on Linux)
# Entry format is "<name-of-binary>,<base-image>".
# Binaries are placed in /usr/local/bin inside the image.
#
# $1 - server architecture
proc kube::build::get_docker_wrapped_binaries {
  setglobal debian_iptables_version = 'v8'
  ### If you change any of these lists, please also update DOCKERIZED_BINARIES
  ### in build/BUILD.
  match $1 {
    with "amd64"
        local targets=(
          cloud-controller-manager,busybox
          kube-apiserver,busybox
          kube-controller-manager,busybox
          kube-scheduler,busybox
          kube-aggregator,busybox
          kube-proxy,gcr.io/google-containers/debian-iptables-amd64:${debian_iptables_version}
        )
    with "arm"
        local targets=(
          cloud-controller-manager,arm32v7/busybox
          kube-apiserver,arm32v7/busybox
          kube-controller-manager,arm32v7/busybox
          kube-scheduler,arm32v7/busybox
          kube-aggregator,arm32v7/busybox
          kube-proxy,gcr.io/google-containers/debian-iptables-arm:${debian_iptables_version}
        )
    with "arm64"
        local targets=(
          cloud-controller-manager,arm64v8/busybox
          kube-apiserver,arm64v8/busybox
          kube-controller-manager,arm64v8/busybox
          kube-scheduler,arm64v8/busybox
          kube-aggregator,arm64v8/busybox
          kube-proxy,gcr.io/google-containers/debian-iptables-arm64:${debian_iptables_version}
        )
    with "ppc64le"
        local targets=(
          cloud-controller-manager,ppc64le/busybox
          kube-apiserver,ppc64le/busybox
          kube-controller-manager,ppc64le/busybox
          kube-scheduler,ppc64le/busybox
          kube-aggregator,ppc64le/busybox
          kube-proxy,gcr.io/google-containers/debian-iptables-ppc64le:${debian_iptables_version}
        )
    with "s390x"
        local targets=(
          cloud-controller-manager,s390x/busybox
          kube-apiserver,s390x/busybox
          kube-controller-manager,s390x/busybox
          kube-scheduler,s390x/busybox
          kube-aggregator,s390x/busybox
          kube-proxy,gcr.io/google-containers/debian-iptables-s390x:${debian_iptables_version}
        )
  }

  echo $(targets[@])
}

# ---------------------------------------------------------------------------
# Basic setup functions

# Verify that the right utilities and such are installed for building Kube. Set
# up some dynamic constants.
# Args:
#   $1 - boolean of whether to require functioning docker (default true)
#
# Vars set:
#   KUBE_ROOT_HASH
#   KUBE_BUILD_IMAGE_TAG_BASE
#   KUBE_BUILD_IMAGE_TAG
#   KUBE_BUILD_IMAGE
#   KUBE_BUILD_CONTAINER_NAME_BASE
#   KUBE_BUILD_CONTAINER_NAME
#   KUBE_DATA_CONTAINER_NAME_BASE
#   KUBE_DATA_CONTAINER_NAME
#   KUBE_RSYNC_CONTAINER_NAME_BASE
#   KUBE_RSYNC_CONTAINER_NAME
#   DOCKER_MOUNT_ARGS
#   LOCAL_OUTPUT_BUILD_CONTEXT
proc kube::build::verify_prereqs {
  local -r require_docker=$(1:-true)
  kube::log::status "Verifying Prerequisites...."
  kube::build::ensure_tar || return 1
  kube::build::ensure_rsync || return 1
  if $(require_docker) {
    kube::build::ensure_docker_in_path || return 1
    if kube::build::is_osx {
        kube::build::docker_available_on_osx || return 1
    }
    kube::util::ensure_docker_daemon_connectivity || return 1

    if sh-expr ' ${KUBE_VERBOSE} > 6 ' {
      kube::log::status "Docker Version:"
      $(DOCKER[@]) version | kube::log::info_from_stdin
    }
  }

  setglobal KUBE_GIT_BRANCH = $[git symbolic-ref --short -q HEAD !2 >/dev/null || true]
  setglobal KUBE_ROOT_HASH = $[kube::build::short_hash "$(HOSTNAME:-):$(KUBE_ROOT):$(KUBE_GIT_BRANCH)]
  setglobal KUBE_BUILD_IMAGE_TAG_BASE = ""build-$(KUBE_ROOT_HASH)""
  setglobal KUBE_BUILD_IMAGE_TAG = ""$(KUBE_BUILD_IMAGE_TAG_BASE)-$(KUBE_BUILD_IMAGE_VERSION)""
  setglobal KUBE_BUILD_IMAGE = ""$(KUBE_BUILD_IMAGE_REPO):$(KUBE_BUILD_IMAGE_TAG)""
  setglobal KUBE_BUILD_CONTAINER_NAME_BASE = ""kube-build-$(KUBE_ROOT_HASH)""
  setglobal KUBE_BUILD_CONTAINER_NAME = ""$(KUBE_BUILD_CONTAINER_NAME_BASE)-$(KUBE_BUILD_IMAGE_VERSION)""
  setglobal KUBE_RSYNC_CONTAINER_NAME_BASE = ""kube-rsync-$(KUBE_ROOT_HASH)""
  setglobal KUBE_RSYNC_CONTAINER_NAME = ""$(KUBE_RSYNC_CONTAINER_NAME_BASE)-$(KUBE_BUILD_IMAGE_VERSION)""
  setglobal KUBE_DATA_CONTAINER_NAME_BASE = ""kube-build-data-$(KUBE_ROOT_HASH)""
  setglobal KUBE_DATA_CONTAINER_NAME = ""$(KUBE_DATA_CONTAINER_NAME_BASE)-$(KUBE_BUILD_IMAGE_VERSION)""
  setglobal DOCKER_MOUNT_ARGS = ''(--volumes-from "${KUBE_DATA_CONTAINER_NAME}")
  setglobal LOCAL_OUTPUT_BUILD_CONTEXT = ""$(LOCAL_OUTPUT_IMAGE_STAGING)/$(KUBE_BUILD_IMAGE)""

  kube::version::get_version_vars
  kube::version::save_version_vars "$(KUBE_ROOT)/.dockerized-kube-version-defs"
}

# ---------------------------------------------------------------------------
# Utility functions

proc kube::build::docker_available_on_osx {
  if [[ -z "${DOCKER_HOST}" ]] {
    if [[ -S "/var/run/docker.sock" ]] {
      kube::log::status "Using Docker for MacOS"
      return 0
    }

    kube::log::status "No docker host is set. Checking options for setting one..."
    if [[ -z "$(which docker-machine)" ]] {
      kube::log::status "It looks like you're running Mac OS X, yet neither Docker for Mac nor docker-machine can be found."
      kube::log::status "See: https://docs.docker.com/engine/installation/mac/ for installation instructions."
      return 1
    } elif [[ -n "$(which docker-machine)" ]] {
      kube::build::prepare_docker_machine
    }
  }
}

proc kube::build::prepare_docker_machine {
  kube::log::status "docker-machine was found."

  local available_memory_bytes=$[sysctl -n hw.memsize !2 >/dev/null]

  local bytes_in_mb=1048576

  # Give virtualbox 1/2 the system memory. Its necessary to divide by 2, instead
  # of multiple by .5, because bash can only multiply by ints.
  local memory_divisor=2

  local virtualbox_memory_mb=$shExpr(' ${available_memory_bytes} / (${bytes_in_mb} * ${memory_divisor}) ')

  docker-machine inspect $(DOCKER_MACHINE_NAME) &> /dev/null || do {
    kube::log::status "Creating a machine to build Kubernetes"
    docker-machine create --driver $(DOCKER_MACHINE_DRIVER) \
      --virtualbox-memory $(virtualbox_memory_mb) \
      --engine-env HTTP_PROXY="$(KUBERNETES_HTTP_PROXY:-)" \
      --engine-env HTTPS_PROXY="$(KUBERNETES_HTTPS_PROXY:-)" \
      --engine-env NO_PROXY="$(KUBERNETES_NO_PROXY:-127.0.0.1)" \
      $(DOCKER_MACHINE_NAME) > /dev/null || do {
      kube::log::error "Something went wrong creating a machine."
      kube::log::error "Try the following: "
      kube::log::error "docker-machine create -d $(DOCKER_MACHINE_DRIVER) --virtualbox-memory $(virtualbox_memory_mb) $(DOCKER_MACHINE_NAME)"
      return 1
    }
  }
  docker-machine start $(DOCKER_MACHINE_NAME) &> /dev/null
  # it takes `docker-machine env` a few seconds to work if the machine was just started
  local docker_machine_out
  while ! setglobal docker_machine_out = $[docker-machine env $(DOCKER_MACHINE_NAME) !2 > !1] {
    if [[ ${docker_machine_out} =~ "Error checking TLS connection" ]] {
      echo $(docker_machine_out)
      docker-machine regenerate-certs $(DOCKER_MACHINE_NAME)
    } else {
      sleep 1
    }
  }
  eval $[docker-machine env $(DOCKER_MACHINE_NAME)]
  kube::log::status "A Docker host using docker-machine named '$(DOCKER_MACHINE_NAME)' is ready to go!"
  return 0
}

proc kube::build::is_osx {
  [[ "$(uname)" == "Darwin" ]]
}

proc kube::build::is_gnu_sed {
  [[ $(sed --version 2>&1) == *GNU* ]]
}

proc kube::build::ensure_rsync {
  if [[ -z "$(which rsync)" ]] {
    kube::log::error "Can't find 'rsync' in PATH, please fix and retry."
    return 1
  }
}

proc kube::build::update_dockerfile {
  if kube::build::is_gnu_sed {
    setglobal sed_opts = ''(-i)
  } else {
    setglobal sed_opts = ''(-i '')
  }
  sed $(sed_opts[@]) "s/KUBE_BUILD_IMAGE_CROSS_TAG/$(KUBE_BUILD_IMAGE_CROSS_TAG)/" "$(LOCAL_OUTPUT_BUILD_CONTEXT)/Dockerfile"
}

proc kube::build::set_proxy {
  if [[ -n "${KUBERNETES_HTTPS_PROXY:-}" ]] {
    echo "ENV https_proxy $KUBERNETES_HTTPS_PROXY" >> "$(LOCAL_OUTPUT_BUILD_CONTEXT)/Dockerfile"
  }
  if [[ -n "${KUBERNETES_HTTP_PROXY:-}" ]] {
    echo "ENV http_proxy $KUBERNETES_HTTP_PROXY" >> "$(LOCAL_OUTPUT_BUILD_CONTEXT)/Dockerfile"
  }
  if [[ -n "${KUBERNETES_NO_PROXY:-}" ]] {
    echo "ENV no_proxy $KUBERNETES_NO_PROXY" >> "$(LOCAL_OUTPUT_BUILD_CONTEXT)/Dockerfile"
  }
}

proc kube::build::ensure_docker_in_path {
  if [[ -z "$(which docker)" ]] {
    kube::log::error "Can't find 'docker' in PATH, please fix and retry."
    kube::log::error "See https://docs.docker.com/installation/#installation for installation instructions."
    return 1
  }
}

proc kube::build::ensure_tar {
  if [[ -n "${TAR:-}" ]] {
    return
  }

  # Find gnu tar if it is available, bomb out if not.
  setglobal TAR = 'tar'
  if which gtar &>/dev/null {
      setglobal TAR = 'gtar'
  } else {
      if which gnutar &>/dev/null {
	  setglobal TAR = 'gnutar'
      }
  }
  if ! $(TAR) --version | grep -q GNU {
    echo "  !!! Cannot find GNU tar. Build on Linux or install GNU tar"
    echo "      on Mac OS X (brew install gnu-tar)."
    return 1
  }
}

proc kube::build::has_docker {
  which docker &> /dev/null
}

proc kube::build::has_ip {
  ip -Version | grep 'iproute2' &> /dev/null
}

# Detect if a specific image exists
#
# $1 - image repo name
# #2 - image tag
proc kube::build::docker_image_exists {
  [[ -n $1 && -n $2 ]] || do {
    kube::log::error "Internal error. Image not specified in docker_image_exists."
    exit 2
  }

  [[ $("${DOCKER[@]}" images -q "${1}:${2}") ]]
}

# Delete all images that match a tag prefix except for the "current" version
#
# $1: The image repo/name
# $2: The tag base. We consider any image that matches $2*
# $3: The current image not to delete if provided
proc kube::build::docker_delete_old_images {
  # In Docker 1.12, we can replace this with
  #    docker images "$1" --format "{{.Tag}}"
  for tag in [$[$(DOCKER[@]) images $(1) | tail -n +2 | awk '{print $2}]]  {
    if [[ "${tag}" != "${2}"* ]]  {
      env V=3 kube::log::status "Keeping image $(1):$(tag)"
      continue
    }

    if [[ -z "${3:-}" || "${tag}" != "${3}" ]]  {
      env V=2 kube::log::status "Deleting image $(1):$(tag)"
      $(DOCKER[@]) rmi "$(1):$(tag)" >/dev/null
    } else {
      env V=3 kube::log::status "Keeping image $(1):$(tag)"
    }
  }
}

# Stop and delete all containers that match a pattern
#
# $1: The base container prefix
# $2: The current container to keep, if provided
proc kube::build::docker_delete_old_containers {
  # In Docker 1.12 we can replace this line with
  #   docker ps -a --format="{{.Names}}"
  for container in [$[$(DOCKER[@]) ps -a | tail -n +2 | awk '{print $NF}]]  {
    if [[ "${container}" != "${1}"* ]]  {
      env V=3 kube::log::status "Keeping container $(container)"
      continue
    }
    if [[ -z "${2:-}" || "${container}" != "${2}" ]]  {
      env V=2 kube::log::status "Deleting container $(container)"
      kube::build::destroy_container $(container)
    } else {
      env V=3 kube::log::status "Keeping container $(container)"
    }
  }
}

# Takes $1 and computes a short has for it. Useful for unique tag generation
proc kube::build::short_hash {
  [[ $# -eq 1 ]] || do {
    kube::log::error "Internal error.  No data based to short_hash."
    exit 2
  }

  local short_hash
  if which md5 >/dev/null !2 > !1 {
    setglobal short_hash = $[md5 -q -s $1]
  } else {
    setglobal short_hash = $[echo -n $1 | md5sum]
  }
  echo $(short_hash:0:10)
}

# Pedantically kill, wait-on and remove a container. The -f -v options
# to rm don't actually seem to get the job done, so force kill the
# container, wait to ensure it's stopped, then try the remove. This is
# a workaround for bug https://github.com/docker/docker/issues/3968.
proc kube::build::destroy_container {
  $(DOCKER[@]) kill $1 >/dev/null !2 > !1 || true
  if [[ $("${DOCKER[@]}" version --format '{{.Server.Version}}') = 17.06.0* ]] {
    # Workaround https://github.com/moby/moby/issues/33948.
    # TODO: remove when 17.06.0 is not relevant anymore
    env DOCKER_API_VERSION=v1.29 $(DOCKER[@]) wait $1 >/dev/null !2 > !1 || true
  } else {
    $(DOCKER[@]) wait $1 >/dev/null !2 > !1 || true
  }
  $(DOCKER[@]) rm -f -v $1 >/dev/null !2 > !1 || true
}

# ---------------------------------------------------------------------------
# Building


proc kube::build::clean {
  if kube::build::has_docker  {
    kube::build::docker_delete_old_containers $(KUBE_BUILD_CONTAINER_NAME_BASE)
    kube::build::docker_delete_old_containers $(KUBE_RSYNC_CONTAINER_NAME_BASE)
    kube::build::docker_delete_old_containers $(KUBE_DATA_CONTAINER_NAME_BASE)
    kube::build::docker_delete_old_images $(KUBE_BUILD_IMAGE_REPO) $(KUBE_BUILD_IMAGE_TAG_BASE)

    env V=2 kube::log::status "Cleaning all untagged docker images"
    $(DOCKER[@]) rmi $[$(DOCKER[@]) images -q --filter 'dangling=true] !2 > /dev/null || true
  }

  if [[ -d "${LOCAL_OUTPUT_ROOT}" ]] {
    kube::log::status "Removing _output directory"
    rm -rf $(LOCAL_OUTPUT_ROOT)
  }
}

# Set up the context directory for the kube-build image and build it.
proc kube::build::build_image {
  mkdir -p $(LOCAL_OUTPUT_BUILD_CONTEXT)
  # Make sure the context directory owned by the right user for syncing sources to container.
  chown -R $(USER_ID):$(GROUP_ID) $(LOCAL_OUTPUT_BUILD_CONTEXT)

  cp /etc/localtime "$(LOCAL_OUTPUT_BUILD_CONTEXT)/"

  cp build/build-image/Dockerfile "$(LOCAL_OUTPUT_BUILD_CONTEXT)/Dockerfile"
  cp build/build-image/rsyncd.sh "$(LOCAL_OUTPUT_BUILD_CONTEXT)/"
  dd if=/dev/urandom bs=512 count=1 !2 >/dev/null | env LC_ALL=C tr -dc 'A-Za-z0-9' | dd bs=32 count=1 !2 >/dev/null > "$(LOCAL_OUTPUT_BUILD_CONTEXT)/rsyncd.password"
  chmod go= "$(LOCAL_OUTPUT_BUILD_CONTEXT)/rsyncd.password"

  kube::build::update_dockerfile
  kube::build::set_proxy
  kube::build::docker_build $(KUBE_BUILD_IMAGE) $(LOCAL_OUTPUT_BUILD_CONTEXT) 'false'

  # Clean up old versions of everything
  kube::build::docker_delete_old_containers $(KUBE_BUILD_CONTAINER_NAME_BASE) $(KUBE_BUILD_CONTAINER_NAME)
  kube::build::docker_delete_old_containers $(KUBE_RSYNC_CONTAINER_NAME_BASE) $(KUBE_RSYNC_CONTAINER_NAME)
  kube::build::docker_delete_old_containers $(KUBE_DATA_CONTAINER_NAME_BASE) $(KUBE_DATA_CONTAINER_NAME)
  kube::build::docker_delete_old_images $(KUBE_BUILD_IMAGE_REPO) $(KUBE_BUILD_IMAGE_TAG_BASE) $(KUBE_BUILD_IMAGE_TAG)

  kube::build::ensure_data_container
  kube::build::sync_to_container
}

# Build a docker image from a Dockerfile.
# $1 is the name of the image to build
# $2 is the location of the "context" directory, with the Dockerfile at the root.
# $3 is the value to set the --pull flag for docker build; true by default
proc kube::build::docker_build {
  local -r image=$1
  local -r context_dir=$2
  local -r pull="$(3:-true)"
  local -ra build_cmd=("${DOCKER[@]}" build -t "${image}" "--pull=${pull}" "${context_dir}")

  kube::log::status "Building Docker image $(image)"
  local docker_output
  setglobal docker_output = $[$(build_cmd[@]) !2 > !1] || do {
    cat << """ >&2
+++ Docker build command failed for $(image)

$(docker_output)

To retry manually, run:

$(build_cmd[*])

"""
> !2
+++ Docker build command failed for ${image}

${docker_output}

To retry manually, run:

${build_cmd[*]}

EOF
    return 1
  }
}

proc kube::build::ensure_data_container {
  # If the data container exists AND exited successfully, we can use it.
  # Otherwise nuke it and start over.
  local ret=0
  local code=$[docker inspect \
      -f '{{.State.ExitCode}}' \
      $(KUBE_DATA_CONTAINER_NAME) !2 >/dev/null || setglobal ret = $Status]
  if [[ "${ret}" == 0 && "${code}" != 0 ]] {
    kube::build::destroy_container $(KUBE_DATA_CONTAINER_NAME)
    setglobal ret = '1'
  }
  if [[ "${ret}" != 0 ]] {
    kube::log::status "Creating data container $(KUBE_DATA_CONTAINER_NAME)"
    # We have to ensure the directory exists, or else the docker run will
    # create it as root.
    mkdir -p $(LOCAL_OUTPUT_GOPATH)
    # We want this to run as root to be able to chown, so non-root users can
    # later use the result as a data container.  This run both creates the data
    # container and chowns the GOPATH.
    #
    # The data container creates volumes for all of the directories that store
    # intermediates for the Go build. This enables incremental builds across
    # Docker sessions. The *_cgo paths are re-compiled versions of the go std
    # libraries for true static building.
    local -ra docker_cmd=(
      "${DOCKER[@]}" run
      --volume "${REMOTE_ROOT}"   # white-out the whole output dir
      --volume /usr/local/go/pkg/linux_386_cgo
      --volume /usr/local/go/pkg/linux_amd64_cgo
      --volume /usr/local/go/pkg/linux_arm_cgo
      --volume /usr/local/go/pkg/linux_arm64_cgo
      --volume /usr/local/go/pkg/linux_ppc64le_cgo
      --volume /usr/local/go/pkg/darwin_amd64_cgo
      --volume /usr/local/go/pkg/darwin_386_cgo
      --volume /usr/local/go/pkg/windows_amd64_cgo
      --volume /usr/local/go/pkg/windows_386_cgo
      --name "${KUBE_DATA_CONTAINER_NAME}"
      --hostname "${HOSTNAME}"
      "${KUBE_BUILD_IMAGE}"
      chown -R ${USER_ID}:${GROUP_ID}
        "${REMOTE_ROOT}"
        /usr/local/go/pkg/
    )
    $(docker_cmd[@])
  }
}

# Run a command in the kube-build image.  This assumes that the image has
# already been built.
proc kube::build::run_build_command {
  kube::log::status "Running build command..."
  kube::build::run_build_command_ex $(KUBE_BUILD_CONTAINER_NAME) -- @Argv
}

# Run a command in the kube-build image.  This assumes that the image has
# already been built.
#
# Arguments are in the form of
#  <container name> <extra docker args> -- <command>
proc kube::build::run_build_command_ex {
  [[ $# != 0 ]] || do { echo "Invalid input - please specify a container name." > !2; return 4; }
  local container_name="$(1)"
  shift

  local -a docker_run_opts=(
    "--name=${container_name}"
    "--user=$(id -u):$(id -g)"
    "--hostname=${HOSTNAME}"
    "${DOCKER_MOUNT_ARGS[@]}"
  )

  local detach=false

  [[ $# != 0 ]] || do { echo "Invalid input - please specify docker arguments followed by --." > !2; return 4; }
  # Everything before "--" is an arg to docker
  while not test -z $(1-) {
    if [[ "$1" == "--" ]] {
      shift
      break
    }
    setglobal docker_run_opts = ''("$1")
    if [[ "$1" == "-d" || "$1" == "--detach" ]]  {
      setglobal detach = 'true'
    }
    shift
  }

  # Everything after "--" is the command to run
  [[ $# != 0 ]] || do { echo "Invalid input - please specify a command to run." > !2; return 4; }
  local -a cmd=()
  while not test -z $(1-) {
    setglobal cmd = ''("$1")
    shift
  }

  setglobal docker_run_opts = ''(
    --env "KUBE_FASTBUILD=${KUBE_FASTBUILD:-false}"
    --env "KUBE_BUILDER_OS=${OSTYPE:-notdetected}"
    --env "KUBE_VERBOSE=${KUBE_VERBOSE}"
    --env "GOFLAGS=${GOFLAGS:-}"
    --env "GOLDFLAGS=${GOLDFLAGS:-}"
    --env "GOGCFLAGS=${GOGCFLAGS:-}"
  )

  # If we have stdin we can run interactive.  This allows things like 'shell.sh'
  # to work.  However, if we run this way and don't have stdin, then it ends up
  # running in a daemon-ish mode.  So if we don't have a stdin, we explicitly
  # attach stderr/stdout but don't bother asking for a tty.
  if [[ -t 0 ]] {
    setglobal docker_run_opts = ''(--interactive --tty)
  } elif [[ "${detach}" == false ]] {
    setglobal docker_run_opts = ''(--attach=stdout --attach=stderr)
  }

  local -ra docker_cmd=(
    "${DOCKER[@]}" run "${docker_run_opts[@]}" "${KUBE_BUILD_IMAGE}")

  # Clean up container from any previous run
  kube::build::destroy_container $(container_name)
  $(docker_cmd[@]) $(cmd[@])
  if [[ "${detach}" == false ]] {
    kube::build::destroy_container $(container_name)
  }
}

proc kube::build::rsync_probe::build::rsync_probe {
  # Wait unil rsync is up and running.
  local tries=20
  while sh-expr ' ${tries} > 0 ' {
    if rsync "rsync://k8s@$(1):$(2)/" \
         --password-file="$(LOCAL_OUTPUT_BUILD_CONTEXT)/rsyncd.password" \
         &> /dev/null  {
      return 0
    }
    setglobal tries = $shExpr(' ${tries} - 1')
    sleep 0.1
  }

  return 1
}

# Start up the rsync container in the background. This should be explicitly
# stopped with kube::build::stop_rsyncd_container.
#
# This will set the global var KUBE_RSYNC_ADDR to the effective port that the
# rsync daemon can be reached out.
proc kube::build::start_rsyncd_container {
  setglobal IPTOOL = 'ifconfig'
  if kube::build::has_ip  {
    setglobal IPTOOL = '"ip address'"
  }
  kube::build::stop_rsyncd_container
  env V=3 kube::log::status "Starting rsyncd container"
  kube::build::run_build_command_ex \
    $(KUBE_RSYNC_CONTAINER_NAME) -p 127.0.0.1:$(KUBE_RSYNC_PORT):$(KUBE_CONTAINER_RSYNC_PORT) -d \
    -e ALLOW_HOST="$[$(IPTOOL) | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1]" \
    -- /rsyncd.sh >/dev/null

  local mapped_port
  if ! setglobal mapped_port = $[$(DOCKER[@]) port $(KUBE_RSYNC_CONTAINER_NAME) $(KUBE_CONTAINER_RSYNC_PORT) !2 > /dev/null | cut -d: -f 2]  {
    kube::log::error "Could not get effective rsync port"
    return 1
  }

  local container_ip
  setglobal container_ip = $[$(DOCKER[@]) inspect --format '{{ .NetworkSettings.IPAddress }}' $(KUBE_RSYNC_CONTAINER_NAME)]

  # Sometimes we can reach rsync through localhost and a NAT'd port.  Other
  # times (when we are running in another docker container on the Jenkins
  # machines) we have to talk directly to the container IP.  There is no one
  # strategy that works in all cases so we test to figure out which situation we
  # are in.
  if kube::build::rsync_probe 127.0.0.1 $(mapped_port) {
    setglobal KUBE_RSYNC_ADDR = ""127.0.0.1:$(mapped_port)""
    return 0
  } elif kube::build::rsync_probe $(container_ip) $(KUBE_CONTAINER_RSYNC_PORT) {
    setglobal KUBE_RSYNC_ADDR = ""$(container_ip):$(KUBE_CONTAINER_RSYNC_PORT)""
    return 0
  }

  kube::log::error "Could not connect to rsync container. See build/README.md for setting up remote Docker engine."
  return 1
}

proc kube::build::stop_rsyncd_container {
  env V=3 kube::log::status "Stopping any currently running rsyncd container"
  unset KUBE_RSYNC_ADDR
  kube::build::destroy_container $(KUBE_RSYNC_CONTAINER_NAME)
}

proc kube::build::rsync::build::rsync {
  local -a rsync_opts=(
    --archive
    --password-file="${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password"
  )
  if sh-expr ' ${KUBE_VERBOSE} >= 6 ' {
    setglobal rsync_opts = ''("-iv")
  }
  if sh-expr ' ${KUBE_RSYNC_COMPRESS} > 0 ' {
     setglobal rsync_opts = ''("--compress-level=${KUBE_RSYNC_COMPRESS}")
  }
  env V=3 kube::log::status "Running rsync"
  rsync $(rsync_opts[@]) @Argv
}

# This will launch rsyncd in a container and then sync the source tree to the
# container over the local network.
proc kube::build::sync_to_container {
  kube::log::status "Syncing sources to container"

  kube::build::start_rsyncd_container

  # rsync filters are a bit confusing.  Here we are syncing everything except
  # output only directories and things that are not necessary like the git
  # directory and generated files. The '- /' filter prevents rsync
  # from trying to set the uid/gid/perms on the root of the sync tree.
  # As an exception, we need to sync generated files in staging/, because
  # they will not be re-generated by 'make'. Note that the 'H' filtered files
  # are hidden from rsync so they will be deleted in the target container if
  # they exist. This will allow them to be re-created in the container if
  # necessary.
  kube::build::rsync \
    --delete \
    --filter='H /.git/' \
    --filter='- /.make/' \
    --filter='- /_tmp/' \
    --filter='- /_output/' \
    --filter='- /' \
    --filter='H zz_generated.*' \
    --filter='H generated.proto' \
    "$(KUBE_ROOT)/" "rsync://k8s@$(KUBE_RSYNC_ADDR)/k8s/"

  kube::build::stop_rsyncd_container
}

# Copy all build results back out.
proc kube::build::copy_output {
  kube::log::status "Syncing out of container"

  kube::build::start_rsyncd_container

  local rsync_extra=""
  if sh-expr ' ${KUBE_VERBOSE} >= 6 ' {
    setglobal rsync_extra = '"-iv'"
  }

  # The filter syntax for rsync is a little obscure. It filters on files and
  # directories.  If you don't go in to a directory you won't find any files
  # there.  Rules are evaluated in order.  The last two rules are a little
  # magic. '+ */' says to go in to every directory and '- /**' says to ignore
  # any file or directory that isn't already specifically allowed.
  #
  # We are looking to copy out all of the built binaries along with various
  # generated files.
  kube::build::rsync \
    --prune-empty-dirs \
    --filter='- /_temp/' \
    --filter='+ /vendor/' \
    --filter='+ /Godeps/' \
    --filter='+ /staging/***/Godeps/**' \
    --filter='+ /_output/dockerized/bin/**' \
    --filter='+ zz_generated.*' \
    --filter='+ generated.proto' \
    --filter='+ *.pb.go' \
    --filter='+ types.go' \
    --filter='+ */' \
    --filter='- /**' \
    "rsync://k8s@$(KUBE_RSYNC_ADDR)/k8s/" $(KUBE_ROOT)

  kube::build::stop_rsyncd_container
}