#!/bin/bash

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# !!!EXPERIMENTAL !!! Upgrade script for GCE. Expect this to get
# rewritten in Go in relatively short order, but it allows us to start
# testing the concepts.

set -o errexit
set -o nounset
set -o pipefail

if [[ "${KUBERNETES_PROVIDER:-gce}" != "gce" ]] {
  echo "!!! $(1) only works on GCE" > !2
  exit 1
}

setglobal KUBE_ROOT = "$[dirname $(BASH_SOURCE)]/../.."
source "$(KUBE_ROOT)/hack/lib/util.sh"
source "$(KUBE_ROOT)/cluster/kube-util.sh"

proc usage {
  echo "!!! EXPERIMENTAL !!!"
  echo ""
  echo "$(0) [-M | -N | -P] [-o] (-l | <version number or publication>)"
  echo "  Upgrades master and nodes by default"
  echo "  -M:  Upgrade master only"
  echo "  -N:  Upgrade nodes only"
  echo "  -P:  Node upgrade prerequisites only (create a new instance template)"
  echo "  -c:  Upgrade NODE_UPGRADE_PARALLELISM nodes in parallel (default=1) within a single instance group. The MIGs themselves are dealt serially."
  echo "  -o:  Use os distro sepcified in KUBE_NODE_OS_DISTRIBUTION for new nodes. Options include 'debian' or 'gci'"
  echo "  -l:  Use local(dev) binaries. This is only supported for master upgrades."
  echo ""
  echo '  Version number or publication is either a proper version number'
  echo '  (e.g. "v1.0.6", "v1.2.0-alpha.1.881+376438b69c7612") or a version'
  echo '  publication of the form <bucket>/<version> (e.g. "release/stable",'
  echo '  "ci/latest-1").  Some common ones are:'
  echo '    - "release/stable"'
  echo '    - "release/latest"'
  echo '    - "ci/latest"'
  echo '  See the docs on getting builds for more information about version publication.'
  echo ""
  echo "(... Fetching current release versions ...)"
  echo ""

  # NOTE: IF YOU CHANGE THE FOLLOWING LIST, ALSO UPDATE test/e2e/cluster_upgrade.go
  local release_stable
  local release_latest
  local ci_latest

  setglobal release_stable = $[gsutil cat gs://kubernetes-release/release/stable.txt]
  setglobal release_latest = $[gsutil cat gs://kubernetes-release/release/latest.txt]
  setglobal ci_latest = $[gsutil cat gs://kubernetes-release-dev/ci/latest.txt]

  echo "Right now, versions are as follows:"
  echo "  release/stable: $(0) $(release_stable)"
  echo "  release/latest: $(0) $(release_latest)"
  echo "  ci/latest:      $(0) $(ci_latest)"
}

proc print-node-version-info {
  echo "== $1 Node OS and Kubelet Versions =="
  "$(KUBE_ROOT)/cluster/kubectl.sh" get nodes -o=jsonpath='{range .items[*]}name: "{.metadata.name}", osImage: "{.status.nodeInfo.osImage}", kubeletVersion: "{.status.nodeInfo.kubeletVersion}"{"\n"}{end}'
}

proc upgrade-master {
  local num_masters
  setglobal num_masters = $[get-master-replicas-count]
  if [[ "${num_masters}" -gt 1 ]] {
    echo "Upgrade of master not supported if more than one master replica present. The current number of master replicas: $(num_masters)"
    exit 1
  }

  echo "== Upgrading master to '$(SERVER_BINARY_TAR_URL)'. Do not interrupt, deleting master instance. =="

  # Tries to figure out KUBE_USER/KUBE_PASSWORD by first looking under
  # kubeconfig:username, and then under kubeconfig:username-basic-auth.
  # TODO: KUBE_USER is used in generating ABAC policy which the
  # apiserver may not have enabled. If it's enabled, we must have a user
  # to generate a valid ABAC policy. If the username changes, should
  # the script fail? Should we generate a default username and password
  # if the section is missing in kubeconfig? Handle this better in 1.5.
  get-kubeconfig-basicauth
  get-kubeconfig-bearertoken

  detect-master
  parse-master-env
  upgrade-master-env

  backfile-kubeletauth-certs

  # Delete the master instance. Note that the master-pd is created
  # with auto-delete=no, so it should not be deleted.
  gcloud compute instances delete \
    --project $(PROJECT) \
    --quiet \
    --zone $(ZONE) \
    $(MASTER_NAME)

  create-master-instance "$(MASTER_NAME)-ip"
  wait-for-master
}

proc upgrade-master-env {
  echo "== Upgrading master environment variables. =="
  # Generate the node problem detector token if it isn't present on the original
  # master.
 if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" && "${NODE_PROBLEM_DETECTOR_TOKEN:-}" == "" ]] {
    setglobal NODE_PROBLEM_DETECTOR_TOKEN = $[dd if=/dev/urandom bs=128 count=1 !2 >/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 !2 >/dev/null]
  }
}

# TODO(mikedanese): delete when we don't support < 1.6
proc backfile-kubeletauth-certs {
  if [[ ! -z "${KUBEAPISERVER_CERT_BASE64:-}" && ! -z "${KUBEAPISERVER_CERT_BASE64:-}" ]] {
    return 0
  }

  mkdir -p "$(KUBE_TEMP)/pki"
  echo $(CA_KEY_BASE64) | base64 -d > "$(KUBE_TEMP)/pki/ca.key"
  echo $(CA_CERT_BASE64) | base64 -d > "$(KUBE_TEMP)/pki/ca.crt"
  shell {cd "$(KUBE_TEMP)/pki"
    kube::util::ensure-cfssl "$(KUBE_TEMP)/cfssl"
    cat << """ > ca-config.json
{
  "signing": {
    "client": {
      "expiry": "43800h",
      "usages": [
        "signing",
        "key encipherment",
        "client auth"
      ]
    }
  }
}
"""
> ca-config.json
{
  "signing": {
    "client": {
      "expiry": "43800h",
      "usages": [
        "signing",
        "key encipherment",
        "client auth"
      ]
    }
  }
}
EOF
    # the name kube-apiserver is bound to the node proxy
    # subpaths required for the apiserver to hit proxy
    # endpoints on the kubelet's handler.
    cat << """ \
      | "${CFSSL_BIN}" gencert \
        -ca=ca.crt \
        -ca-key=ca.key \
        -config=ca-config.json \
        -profile=client \
        - \
      | "${CFSSLJSON_BIN}" -bare kube-apiserver
{
  "CN": "kube-apiserver"
}
"""
$(CFSSL_BIN) gencert \
        -ca=ca.crt \
        -ca-key=ca.key \
        -config=ca-config.json \
        -profile=client \
        - \
      | $(CFSSLJSON_BIN) -bare kube-apiserver
{
  "CN": "kube-apiserver"
}
EOF
  }
  setglobal KUBEAPISERVER_CERT_BASE64 = $[cat "$(KUBE_TEMP)/pki/kube-apiserver.pem" | base64 | tr -d '\r\n]
  setglobal KUBEAPISERVER_KEY_BASE64 = $[cat "$(KUBE_TEMP)/pki/kube-apiserver-key.pem" | base64 | tr -d '\r\n]
}

proc wait-for-master {
  echo "== Waiting for new master to respond to API requests =="

  local curl_auth_arg
  if [[ -n ${KUBE_BEARER_TOKEN:-} ]] {
    setglobal curl_auth_arg = ''(-H "Authorization: Bearer ${KUBE_BEARER_TOKEN}")
  } elif [[ -n ${KUBE_PASSWORD:-} ]] {
    setglobal curl_auth_arg = ''(--user "${KUBE_USER}:${KUBE_PASSWORD}")
  } else {
    echo "can't get auth credentials for the current master"
    exit 1
  }

  while not curl --insecure $(curl_auth_arg[@]) --max-time 5 \
    --fail --output /dev/null --silent "https://$(KUBE_MASTER_IP)/healthz {
    printf "."
    sleep 2
  }

  echo "== Done =="
}

# Perform common upgrade setup tasks
#
# Assumed vars
#   KUBE_VERSION
proc prepare-upgrade {
  kube::util::ensure-temp-dir
  detect-project
  detect-subnetworks
  detect-node-names # sets INSTANCE_GROUPS
  write-cluster-name
  tars_from_version
}

# Reads kube-env metadata from first node in NODE_NAMES.
#
# Assumed vars:
#   NODE_NAMES
#   PROJECT
#   ZONE
proc get-node-env {
  # TODO(zmerlynn): Make this more reliable with retries.
  gcloud compute --project $(PROJECT) ssh --zone $(ZONE) $(NODE_NAMES[0]) --command \
    "curl --fail --silent -H 'Metadata-Flavor: Google' \
      'http://metadata/computeMetadata/v1/instance/attributes/kube-env'" !2 >/dev/null
}

# Read os distro information from /os/release on node.
# $1: The name of node
#
# Assumed vars:
#   PROJECT
#   ZONE
proc get-node-os {
  gcloud compute ssh $1 \
    --project $(PROJECT) \
    --zone $(ZONE) \
    --command \
    "cat /etc/os-release | grep \"^ID=.*\" | cut -c 4-"
}

# Assumed vars:
#   KUBE_VERSION
#   NODE_SCOPES
#   NODE_INSTANCE_PREFIX
#   PROJECT
#   ZONE
#
# Vars set:
#   KUBELET_TOKEN
#   KUBE_PROXY_TOKEN
#   NODE_PROBLEM_DETECTOR_TOKEN
#   CA_CERT_BASE64
#   EXTRA_DOCKER_OPTS
#   KUBELET_CERT_BASE64
#   KUBELET_KEY_BASE64
proc upgrade-nodes {
  prepare-node-upgrade
  do-node-upgrade
}

proc setup-base-image {
  if [[ "${env_os_distro}" == "false" ]] {
    echo "== Ensuring that new Node base OS image matched the existing Node base OS image"
    setglobal NODE_OS_DISTRIBUTION = $[get-node-os $(NODE_NAMES[0])]

    if [[ "${NODE_OS_DISTRIBUTION}" == "cos" ]] {
        setglobal NODE_OS_DISTRIBUTION = '"gci'"
    }
    
    source "$(KUBE_ROOT)/cluster/gce/$(NODE_OS_DISTRIBUTION)/node-helper.sh"
    # Reset the node image based on current os distro
    set-node-image
  }
}

# prepare-node-upgrade creates a new instance template suitable for upgrading
# to KUBE_VERSION and echos a single line with the name of the new template.
#
# Assumed vars:
#   KUBE_VERSION
#   NODE_SCOPES
#   NODE_INSTANCE_PREFIX
#   PROJECT
#   ZONE
#
# Vars set:
#   SANITIZED_VERSION
#   INSTANCE_GROUPS
#   KUBELET_TOKEN
#   KUBE_PROXY_TOKEN
#   NODE_PROBLEM_DETECTOR_TOKEN
#   CA_CERT_BASE64
#   EXTRA_DOCKER_OPTS
#   KUBELET_CERT_BASE64
#   KUBELET_KEY_BASE64
proc prepare-node-upgrade {
  echo "== Preparing node upgrade (to $(KUBE_VERSION)). ==" > !2
  setup-base-image

  setglobal SANITIZED_VERSION = $[echo $(KUBE_VERSION) | sed 's/[\.\+]/-/g]

  # TODO(zmerlynn): Refactor setting scope flags.
  local scope_flags=
  if test -n $(NODE_SCOPES) {
    setglobal scope_flags = ""--scopes $(NODE_SCOPES)""
  } else {
    setglobal scope_flags = '"--no-scopes'"
  }

  # Get required node env vars from exiting template.
  local node_env=$[get-node-env]
  setglobal KUBELET_TOKEN = $[get-env-val $(node_env) "KUBELET_TOKEN]
  setglobal KUBE_PROXY_TOKEN = $[get-env-val $(node_env) "KUBE_PROXY_TOKEN]
  setglobal NODE_PROBLEM_DETECTOR_TOKEN = $[get-env-val $(node_env) "NODE_PROBLEM_DETECTOR_TOKEN]
  setglobal CA_CERT_BASE64 = $[get-env-val $(node_env) "CA_CERT]
  setglobal EXTRA_DOCKER_OPTS = $[get-env-val $(node_env) "EXTRA_DOCKER_OPTS]
  setglobal KUBELET_CERT_BASE64 = $[get-env-val $(node_env) "KUBELET_CERT]
  setglobal KUBELET_KEY_BASE64 = $[get-env-val $(node_env) "KUBELET_KEY]

  upgrade-node-env

  # TODO(zmerlynn): How do we ensure kube-env is written in a ${version}-
  #                 compatible way?
  write-node-env

  # TODO(zmerlynn): Get configure-vm script from ${version}. (Must plumb this
  #                 through all create-node-instance-template implementations).
  local template_name=$[get-template-name-from-version $(SANITIZED_VERSION)]
  create-node-instance-template $(template_name)
  # The following is echo'd so that callers can get the template name.
  echo "Instance template name: $(template_name)"
  echo "== Finished preparing node upgrade (to $(KUBE_VERSION)). ==" > !2
}

proc upgrade-node-env {
  echo "== Upgrading node environment variables. =="
  # Get the node problem detector token from master if it isn't present on
  # the original node.
  if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" && "${NODE_PROBLEM_DETECTOR_TOKEN:-}" == "" ]] {
    detect-master
    local master_env=$[get-master-env]
    setglobal NODE_PROBLEM_DETECTOR_TOKEN = $[get-env-val $(master_env) "NODE_PROBLEM_DETECTOR_TOKEN]
  }
}

# Upgrades a single node.
# $1: The name of the node
#
# Note: This is called multiple times from do-node-upgrade() in parallel, so should be thread-safe.
proc do-single-node-upgrade {
  local -r instance="$1"
  setglobal instance_id = $[gcloud compute instances describe $(instance) \
    --format='get(id)' \
    --project="$(PROJECT)" \
    --zone="$(ZONE)" !2 > !1] && setglobal describe_rc = $Status || setglobal describe_rc = $Status
  if [[ "${describe_rc}" != 0 ]] {
    echo "== FAILED to describe $(instance) =="
    echo $(instance_id)
    return ${describe_rc}
  }

  # Drain node
  echo "== Draining $(instance). == " > !2
  "$(KUBE_ROOT)/cluster/kubectl.sh" drain --delete-local-data --force --ignore-daemonsets $(instance) \
    && setglobal drain_rc = $Status || setglobal drain_rc = $Status
  if [[ "${drain_rc}" != 0 ]] {
    echo "== FAILED to drain $(instance) =="
    return ${drain_rc}
  }

  # Recreate instance
  echo "== Recreating instance $(instance). ==" > !2
  setglobal recreate = $[gcloud compute instance-groups managed recreate-instances $(group) \
    --project="$(PROJECT)" \
    --zone="$(ZONE)" \
    --instances="$(instance)" !2 > !1] && setglobal recreate_rc = $Status || setglobal recreate_rc = $Status
  if [[ "${recreate_rc}" != 0 ]] {
    echo "== FAILED to recreate $(instance) =="
    echo $(recreate)
    return ${recreate_rc}
  }

  # Wait for instance to be recreated
  echo "== Waiting for instance $(instance) to be recreated. ==" > !2
  while true {
    setglobal new_instance_id = $[gcloud compute instances describe $(instance) \
      --format='get(id)' \
      --project="$(PROJECT)" \
      --zone="$(ZONE)" !2 > !1] && setglobal describe_rc = $Status || setglobal describe_rc = $Status
    if [[ "${describe_rc}" != 0 ]] {
      echo "== FAILED to describe $(instance) =="
      echo $(new_instance_id)
      echo "  (Will retry.)"
    } elif [[ "${new_instance_id}" == "${instance_id}" ]] {
      echo -n .
    } else {
      echo "Instance $(instance) recreated."
      break
    }
    sleep 1
  }

  # Wait for k8s node object to reflect new instance id
  echo "== Waiting for new node to be added to k8s.  ==" > !2
  while true {
    setglobal external_id = $["$(KUBE_ROOT)/cluster/kubectl.sh" get node $(instance) --output=jsonpath='{.spec.externalID}' !2 > !1] && setglobal kubectl_rc = $Status || setglobal kubectl_rc = $Status
    if [[ "${kubectl_rc}" != 0 ]] {
      echo "== FAILED to get node $(instance) =="
      echo $(external_id)
      echo "  (Will retry.)"
    } elif [[ "${external_id}" == "${new_instance_id}" ]] {
      echo "Node $(instance) recreated."
      break
    } elif [[ "${external_id}" == "${instance_id}" ]] {
      echo -n .
    } else {
      echo "Unexpected external_id '$(external_id)' matches neither old ('$(instance_id)') nor new ('$(new_instance_id)')."
      echo "  (Will retry.)"
    }
    sleep 1
  }

  # Wait for the node to not have SchedulingDisabled=True and also to have
  # Ready=True.
  echo "== Waiting for $(instance) to become ready. ==" > !2
  while true {
    setglobal cordoned = $["$(KUBE_ROOT)/cluster/kubectl.sh" get node $(instance) --output='jsonpath={.status.conditions[?(@.type == "SchedulingDisabled")].status}]
    setglobal ready = $["$(KUBE_ROOT)/cluster/kubectl.sh" get node $(instance) --output='jsonpath={.status.conditions[?(@.type == "Ready")].status}]
    if [[ "${cordoned}" == 'True' ]] {
      echo "Node $(instance) is still not ready: SchedulingDisabled=$(ready)"
    } elif [[ "${ready}" != 'True' ]] {
      echo "Node $(instance) is still not ready: Ready=$(ready)"
    } else {
      echo "Node $(instance) Ready=$(ready)"
      break
    }
    sleep 1
  }
}

# Prereqs:
# - prepare-node-upgrade should have been called successfully
proc do-node-upgrade {
  echo "== Upgrading nodes to $(KUBE_VERSION) with max parallelism of $(node_upgrade_parallelism). ==" > !2
  # Do the actual upgrade.
  # NOTE(zmerlynn): If you are changing this gcloud command, update
  #                 test/e2e/cluster_upgrade.go to match this EXACTLY.
  local template_name=$[get-template-name-from-version $(SANITIZED_VERSION)]
  local old_templates=()
  local updates=()
  for group in [$(INSTANCE_GROUPS[@])] {
    setglobal old_templates = ''($(gcloud compute instance-groups managed list \
        --project="${PROJECT}" \
        --filter="name ~ '${group}' AND zone:(${ZONE})" \
        --format='value(instanceTemplate)' || true))
    setglobal set_instance_template_out = $[gcloud compute instance-groups managed set-instance-template $(group) \
      --template="$(template_name)" \
      --project="$(PROJECT)" \
      --zone="$(ZONE)" !2 > !1] && setglobal set_instance_template_rc = $Status || setglobal set_instance_template_rc = $Status
    if [[ "${set_instance_template_rc}" != 0 ]] {
      echo "== FAILED to set-instance-template for $(group) to $(template_name) =="
      echo $(set_instance_template_out)
      return ${set_instance_template_rc}
    }
    setglobal instances = ''()
    setglobal instances = ''($(gcloud compute instance-groups managed list-instances "${group}" \
        --format='value(instance)' \
        --project="${PROJECT}" \
        --zone="${ZONE}" 2>&1)) && setglobal list_instances_rc = $Status || setglobal list_instances_rc = $Status
    if [[ "${list_instances_rc}" != 0 ]] {
      echo "== FAILED to list instances in group $(group) =="
      echo $(instances)
      return ${list_instances_rc}
    }

    setglobal process_count_left = $(node_upgrade_parallelism)
    setglobal pids = ''()
    setglobal ret_code_sum = '0'  # Should stay 0 in the loop iff all parallel node upgrades succeed.
    for instance in [$(instances[@])] {
      do-single-node-upgrade $(instance) & setglobal pids = ''("$!")

      # We don't want to run more than ${node_upgrade_parallelism} upgrades at a time,
      # so wait once we hit that many nodes. This isn't ideal, since one might take much
      # longer than the others, but it should help.
      setglobal process_count_left = $shExpr('process_count_left - 1')
      if [[ process_count_left -eq 0 || "${instance}" == "${instances[-1]}" ]] {
        # Wait for each of the parallel node upgrades to finish.
        for pid in [$(pids[@])] {
          wait $pid
          setglobal ret_code_sum = $shExpr(' ret_code_sum + $? ')
        }
        # Return even if at least one of the node upgrades failed.
        if [[ ${ret_code_sum} != 0 ]] {
          echo "== Some of the $(node_upgrade_parallelism) parallel node upgrades failed. =="
          return ${ret_code_sum}
        }
        setglobal process_count_left = $(node_upgrade_parallelism)
      }
    }
  }

  # Remove the old templates.
  echo "== Deleting old templates in $(PROJECT). ==" > !2
  for tmpl in [$(old_templates[@])] {
    gcloud compute instance-templates delete \
        --quiet \
        --project="$(PROJECT)" \
        $(tmpl) || true
  }

  echo "== Finished upgrading nodes to $(KUBE_VERSION). ==" > !2
}

setglobal master_upgrade = 'true'
setglobal node_upgrade = 'true'
setglobal node_prereqs = 'false'
setglobal local_binaries = 'false'
setglobal env_os_distro = 'false'
setglobal node_upgrade_parallelism = '1'

while getopts ":MNPlcho" opt {
  match $(opt) {
    with M
      setglobal node_upgrade = 'false'
      
    with N
      setglobal master_upgrade = 'false'
      
    with P
      setglobal node_prereqs = 'true'
      
    with l
      setglobal local_binaries = 'true'
      
    with c
      setglobal node_upgrade_parallelism = $(NODE_UPGRADE_PARALLELISM:-1)
      
    with o
      setglobal env_os_distro = 'true'
      
    with h
      usage
      exit 0
      
    with \?
      echo "Invalid option: -$OPTARG" > !2
      usage
      exit 1
      
  }
}
shift $shExpr('OPTIND-1')

if [[ $# -gt 1 ]] {
  echo "Error: Only one parameter (<version number or publication>) may be passed after the set of flags!" > !2
  usage
  exit 1
}

if [[ $# -lt 1 ]] && [[ "${local_binaries}" == "false" ]] {
  usage
  exit 1
}

if [[ "${master_upgrade}" == "false" ]] && [[ "${node_upgrade}" == "false" ]] {
  echo "Can't specify both -M and -N" > !2
  exit 1
}

# prompt if etcd storage media type isn't set unless using etcd2 when doing master upgrade
if [[ -z "${STORAGE_MEDIA_TYPE:-}" ]] && [[ "${STORAGE_BACKEND:-}" != "etcd2" ]] && [[ "${master_upgrade}" == "true" ]] {
  echo "The default etcd storage media type in 1.6 has changed from application/json to application/vnd.kubernetes.protobuf."
  echo "Documentation about the change can be found at https://kubernetes.io/docs/admin/etcd_upgrade."
  echo ""
  echo "ETCD2 DOES NOT SUPPORT PROTOBUF: If you wish to have to ability to downgrade to etcd2 later application/json must be used."
  echo ""
  echo "It's HIGHLY recommended that etcd be backed up before this step!!"
  echo ""
  echo "To enable using json, before running this script set:"
  echo "export STORAGE_MEDIA_TYPE=application/json"
  echo ""
  if test -t 0 && test -t 1 {
    read -p "Would you like to continue with the new default, and lose the ability to downgrade to etcd2? [y/N] " confirm
    if [[ "${confirm}" != "y" ]] {
      exit 1
    }
  } else {
    echo "To enable using protobuf, before running this script set:"
    echo "export STORAGE_MEDIA_TYPE=application/vnd.kubernetes.protobuf"
    echo ""
    echo "STORAGE_MEDIA_TYPE must be specified when run non-interactively." > !2
    exit 1
  }
}

print-node-version-info "Pre-Upgrade"

if [[ "${local_binaries}" == "false" ]] {
  set_binary_version $(1)
}

prepare-upgrade

if [[ "${node_prereqs}" == "true" ]] {
  prepare-node-upgrade
  exit 0
}

if [[ "${master_upgrade}" == "true" ]] {
  upgrade-master
}

if [[ "${node_upgrade}" == "true" ]] {
  if [[ "${local_binaries}" == "true" ]] {
    echo "Upgrading nodes to local binaries is not yet supported." > !2
    exit 1
  } else {
    upgrade-nodes
  }
}

echo "== Validating cluster post-upgrade =="
"$(KUBE_ROOT)/cluster/validate-cluster.sh"

print-node-version-info "Post-Upgrade"