#!/bin/bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Script to test cluster/update-storage-objects.sh works as expected.

set -o errexit
set -o nounset
set -o pipefail

global KUBE_ROOT := "$[dirname $(BASH_SOURCE)]/.."
source "$(KUBE_ROOT)/hack/lib/init.sh"

# The api version in which objects are currently stored in etcd.
global KUBE_OLD_API_VERSION := $(KUBE_OLD_API_VERSION:-"v1")
# The api version in which our etcd objects should be converted to.
# The new api version
global KUBE_NEW_API_VERSION := $(KUBE_NEW_API_VERSION:-"v1")

global KUBE_OLD_STORAGE_VERSIONS := $(KUBE_OLD_STORAGE_VERSIONS:-"")
global KUBE_NEW_STORAGE_VERSIONS := $(KUBE_NEW_STORAGE_VERSIONS:-"")

global KUBE_STORAGE_MEDIA_TYPE_JSON := '"application/json'"
global KUBE_STORAGE_MEDIA_TYPE_PROTOBUF := '"application/vnd.kubernetes.protobuf'"

global ETCD_HOST := $(ETCD_HOST:-127.0.0.1)
global ETCD_PORT := $(ETCD_PORT:-2379)
global ETCD_PREFIX := $(ETCD_PREFIX:-randomPrefix)
global API_PORT := $(API_PORT:-8080)
global API_HOST := $(API_HOST:-127.0.0.1)
global KUBE_API_VERSIONS := ''""
global RUNTIME_CONFIG := ''""

global ETCDCTL := $[which etcdctl]
global KUBECTL := ""$(KUBE_OUTPUT_HOSTBIN)/kubectl""
global UPDATE_ETCD_OBJECTS_SCRIPT := ""$(KUBE_ROOT)/cluster/update-storage-objects.sh""

proc startApiServer {
  var storage_versions = $(1:-"")
  var storage_media_type = $(2:-"")
  kube::log::status "Starting kube-apiserver with KUBE_API_VERSIONS: $(KUBE_API_VERSIONS)"
  kube::log::status "                        and storage-media-type: $(storage_media_type)"
  kube::log::status "                            and runtime-config: $(RUNTIME_CONFIG)"
  kube::log::status "                 and storage-version overrides: $(storage_versions)"

  env KUBE_API_VERSIONS=$(KUBE_API_VERSIONS) \
    "$(KUBE_OUTPUT_HOSTBIN)/kube-apiserver" \
    --insecure-bind-address="$(API_HOST)" \
    --bind-address="$(API_HOST)" \
    --insecure-port="$(API_PORT)" \
    --storage-backend="etcd3" \
    --etcd-servers="http://$(ETCD_HOST):$(ETCD_PORT)" \
    --etcd-prefix="/$(ETCD_PREFIX)" \
    --runtime-config="$(RUNTIME_CONFIG)" \
    --cert-dir="$(TMPDIR:-/tmp/)" \
    --service-cluster-ip-range="10.0.0.0/24" \
    --storage-versions="$(storage_versions)" \
    --storage-media-type=$(storage_media_type) !1 > !2 &
  global APISERVER_PID := $BgPid

  # url, prefix, wait, times
  kube::util::wait_for_url "http://$(API_HOST):$(API_PORT)/healthz" "apiserver: " 1 120
}

proc killApiServer {
  kube::log::status "Killing api server"
  if [[ -n ${APISERVER_PID-} ]] {
    kill $(APISERVER_PID) !1 > !2 !2 >/dev/null
    wait $(APISERVER_PID) || true
    kube::log::status "api server exited"
  }
  unset APISERVER_PID
}

proc cleanup {
  killApiServer

  kube::etcd::cleanup

  kube::log::status "Clean up complete"
}

trap cleanup EXIT SIGINT

make -C $(KUBE_ROOT) WHAT=cmd/kube-apiserver
make -C $(KUBE_ROOT) WHAT=cluster/images/etcd/attachlease

kube::etcd::start
echo $(ETCD_VERSION) > "$(ETCD_DIR)/version.txt"

### BEGIN TEST DEFINITION CUSTOMIZATION ###

# source_file,resource,namespace,name,old_version,new_version
global tests := '(
'examples/persistent-volume-provisioning/rbd/rbd-storage-class.yaml,storageclasses,,slow,v1beta1,v1
)

global KUBE_OLD_API_VERSION := '"networking.k8s.io/v1,storage.k8s.io/v1beta1,extensions/v1beta1'"
global KUBE_NEW_API_VERSION := '"networking.k8s.io/v1,storage.k8s.io/v1,extensions/v1beta1'"
global KUBE_OLD_STORAGE_VERSIONS := '"storage.k8s.io/v1beta1'"
global KUBE_NEW_STORAGE_VERSIONS := '"storage.k8s.io/v1'"

### END TEST DEFINITION CUSTOMIZATION ###

#######################################################
# Step 1: Start a server which supports both the old and new api versions,
# but KUBE_OLD_API_VERSION is the latest (storage) version.
# Additionally use KUBE_STORAGE_MEDIA_TYPE_JSON for storage encoding.
#######################################################
global KUBE_API_VERSIONS := ""v1,$(KUBE_OLD_API_VERSION),$(KUBE_NEW_API_VERSION)""
global RUNTIME_CONFIG := ""api/all=false,api/v1=true,$(KUBE_OLD_API_VERSION)=true,$(KUBE_NEW_API_VERSION)=true""
startApiServer $(KUBE_OLD_STORAGE_VERSIONS) $(KUBE_STORAGE_MEDIA_TYPE_JSON)


# Create object(s)
for test in [$(tests[@])] {
  env IFS=',' read -ra test_data <<<$test
  global source_file := $(test_data[0])

  kube::log::status "Creating $(source_file)"
  $(KUBECTL) create -f $(source_file)

  # Verify that the storage version is the old version
  global resource := $(test_data[1])
  global namespace := $(test_data[2])
  global name := $(test_data[3])
  global old_storage_version := $(test_data[4])

  if test -n $(namespace) {
    global namespace := ""$(namespace)/""
  }
  kube::log::status "Verifying $(resource)/$(namespace)$(name) has storage version $(old_storage_version) in etcd"
  env ETCDCTL_API=3 $(ETCDCTL) --endpoints="http://$(ETCD_HOST):$(ETCD_PORT)" get "/$(ETCD_PREFIX)/$(resource)/$(namespace)$(name)" | grep $(old_storage_version)
}

killApiServer


#######################################################
# Step 2: Start a server which supports both the old and new api versions,
# but KUBE_NEW_API_VERSION is the latest (storage) version.
# Still use KUBE_STORAGE_MEDIA_TYPE_JSON for storage encoding.
#######################################################

global KUBE_API_VERSIONS := ""v1,$(KUBE_NEW_API_VERSION),$(KUBE_OLD_API_VERSION)""
global RUNTIME_CONFIG := ""api/all=false,api/v1=true,$(KUBE_OLD_API_VERSION)=true,$(KUBE_NEW_API_VERSION)=true""
startApiServer $(KUBE_NEW_STORAGE_VERSIONS) $(KUBE_STORAGE_MEDIA_TYPE_JSON)

# Update etcd objects, so that will now be stored in the new api version.
kube::log::status "Updating storage versions in etcd"
$(UPDATE_ETCD_OBJECTS_SCRIPT)

# Verify that the storage version was changed in etcd
for test in [$(tests[@])] {
  env IFS=',' read -ra test_data <<<$test
  global resource := $(test_data[1])
  global namespace := $(test_data[2])
  global name := $(test_data[3])
  global new_storage_version := $(test_data[5])

  if test -n $(namespace) {
    global namespace := ""$(namespace)/""
  }
  kube::log::status "Verifying $(resource)/$(namespace)$(name) has updated storage version $(new_storage_version) in etcd"
  env ETCDCTL_API=3 $(ETCDCTL) --endpoints="http://$(ETCD_HOST):$(ETCD_PORT)" get "/$(ETCD_PREFIX)/$(resource)/$(namespace)$(name)" | grep $(new_storage_version)
}

killApiServer


#######################################################
# Step 3 : Start a server which supports only the new api version.
# However, change storage encoding to KUBE_STORAGE_MEDIA_TYPE_PROTOBUF.
#######################################################

global KUBE_API_VERSIONS := ""v1,$(KUBE_NEW_API_VERSION)""
global RUNTIME_CONFIG := ""api/all=false,api/v1=true,$(KUBE_NEW_API_VERSION)=true""

# This seems to reduce flakiness.
sleep 1
startApiServer $(KUBE_NEW_STORAGE_VERSIONS) $(KUBE_STORAGE_MEDIA_TYPE_PROTOBUF)

for test in [$(tests[@])] {
  env IFS=',' read -ra test_data <<<$test
  global resource := $(test_data[1])
  global namespace := $(test_data[2])
  global name := $(test_data[3])
  global namespace_flag := ''""

  # Verify that the server is able to read the object.
  if test -n $(namespace) {
    global namespace_flag := ""--namespace=$(namespace)""
    global namespace := ""$(namespace)/""
  }
  kube::log::status "Verifying we can retrieve $(resource)/$(namespace)$(name) via kubectl"
  # We have to remove the cached discovery information about the old version; otherwise,
  # the 'kubectl get' will use that and fail to find the resource.
  rm -rf $(HOME)/.kube/cache/discovery/localhost_8080/$(KUBE_OLD_STORAGE_VERSIONS)
  $(KUBECTL) get $(namespace_flag) $(resource)/$(name)
}

killApiServer
(CommandList
  children: [
    (C {(set)} {(-o)} {(errexit)})
    (C {(set)} {(-o)} {(nounset)})
    (C {(set)} {(-o)} {(pipefail)})
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:KUBE_ROOT)
          op: Equal
          rhs: 
            {
              (CommandSubPart
                command_list: 
                  (CommandList
                    children: [(C {(dirname)} {(DQ (${ VSub_Name BASH_SOURCE))})]
                  )
                left_token: <Left_CommandSub "$(">
                spids: [68 76]
              ) (/..)
            }
          spids: [67]
        )
      ]
      spids: [67]
    )
    (C {(source)} {(DQ (${ VSub_Name KUBE_ROOT) (/hack/lib/init.sh))})
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:KUBE_OLD_API_VERSION)
          op: Equal
          rhs: 
            {
              (BracedVarSub
                token: <VSub_Name KUBE_OLD_API_VERSION>
                suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(DQ (v1))})
                spids: [93 99]
              )
            }
          spids: [92]
        )
      ]
      spids: [92]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:KUBE_NEW_API_VERSION)
          op: Equal
          rhs: 
            {
              (BracedVarSub
                token: <VSub_Name KUBE_NEW_API_VERSION>
                suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(DQ (v1))})
                spids: [108 114]
              )
            }
          spids: [107]
        )
      ]
      spids: [107]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:KUBE_OLD_STORAGE_VERSIONS)
          op: Equal
          rhs: 
            {
              (BracedVarSub
                token: <VSub_Name KUBE_OLD_STORAGE_VERSIONS>
                suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(DQ )})
                spids: [118 123]
              )
            }
          spids: [117]
        )
      ]
      spids: [117]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:KUBE_NEW_STORAGE_VERSIONS)
          op: Equal
          rhs: 
            {
              (BracedVarSub
                token: <VSub_Name KUBE_NEW_STORAGE_VERSIONS>
                suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(DQ )})
                spids: [126 131]
              )
            }
          spids: [125]
        )
      ]
      spids: [125]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:KUBE_STORAGE_MEDIA_TYPE_JSON)
          op: Equal
          rhs: {(DQ (application/json))}
          spids: [134]
        )
      ]
      spids: [134]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:KUBE_STORAGE_MEDIA_TYPE_PROTOBUF)
          op: Equal
          rhs: {(DQ (application/vnd.kubernetes.protobuf))}
          spids: [139]
        )
      ]
      spids: [139]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:ETCD_HOST)
          op: Equal
          rhs: 
            {
              (BracedVarSub
                token: <VSub_Name ETCD_HOST>
                suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(127.0.0.1)})
                spids: [146 150]
              )
            }
          spids: [145]
        )
      ]
      spids: [145]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:ETCD_PORT)
          op: Equal
          rhs: 
            {
              (BracedVarSub
                token: <VSub_Name ETCD_PORT>
                suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(2379)})
                spids: [153 157]
              )
            }
          spids: [152]
        )
      ]
      spids: [152]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:ETCD_PREFIX)
          op: Equal
          rhs: 
            {
              (BracedVarSub
                token: <VSub_Name ETCD_PREFIX>
                suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(randomPrefix)})
                spids: [160 164]
              )
            }
          spids: [159]
        )
      ]
      spids: [159]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:API_PORT)
          op: Equal
          rhs: 
            {
              (BracedVarSub
                token: <VSub_Name API_PORT>
                suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(8080)})
                spids: [167 171]
              )
            }
          spids: [166]
        )
      ]
      spids: [166]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:API_HOST)
          op: Equal
          rhs: 
            {
              (BracedVarSub
                token: <VSub_Name API_HOST>
                suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(127.0.0.1)})
                spids: [174 178]
              )
            }
          spids: [173]
        )
      ]
      spids: [173]
    )
    (Assignment
      keyword: Assign_None
      pairs: [(assign_pair lhs:(LhsName name:KUBE_API_VERSIONS) op:Equal rhs:{(DQ )} spids:[180])]
      spids: [180]
    )
    (Assignment
      keyword: Assign_None
      pairs: [(assign_pair lhs:(LhsName name:RUNTIME_CONFIG) op:Equal rhs:{(DQ )} spids:[184])]
      spids: [184]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:ETCDCTL)
          op: Equal
          rhs: 
            {
              (CommandSubPart
                command_list: (CommandList children:[(C {(which)} {(etcdctl)})])
                left_token: <Left_CommandSub "$(">
                spids: [190 194]
              )
            }
          spids: [189]
        )
      ]
      spids: [189]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:KUBECTL)
          op: Equal
          rhs: {(DQ (${ VSub_Name KUBE_OUTPUT_HOSTBIN) (/kubectl))}
          spids: [196]
        )
      ]
      spids: [196]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:UPDATE_ETCD_OBJECTS_SCRIPT)
          op: Equal
          rhs: {(DQ (${ VSub_Name KUBE_ROOT) (/cluster/update-storage-objects.sh))}
          spids: [204]
        )
      ]
      spids: [204]
    )
    (FuncDef
      name: startApiServer
      body: 
        (BraceGroup
          children: [
            (Assignment
              keyword: Assign_Local
              pairs: [
                (assign_pair
                  lhs: (LhsName name:storage_versions)
                  op: Equal
                  rhs: 
                    {
                      (BracedVarSub
                        token: <VSub_Number 1>
                        suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(DQ )})
                        spids: [225 230]
                      )
                    }
                  spids: [224]
                )
              ]
              spids: [222]
            )
            (Assignment
              keyword: Assign_Local
              pairs: [
                (assign_pair
                  lhs: (LhsName name:storage_media_type)
                  op: Equal
                  rhs: 
                    {
                      (BracedVarSub
                        token: <VSub_Number 2>
                        suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(DQ )})
                        spids: [236 241]
                      )
                    }
                  spids: [235]
                )
              ]
              spids: [233]
            )
            (C {(kube) (Lit_Other ":") (Lit_Other ":") (log) (Lit_Other ":") (Lit_Other ":") (status)} 
              {
                (DQ ("Starting kube-apiserver with KUBE_API_VERSIONS: ") 
                  (${ VSub_Name KUBE_API_VERSIONS)
                )
              }
            )
            (C {(kube) (Lit_Other ":") (Lit_Other ":") (log) (Lit_Other ":") (Lit_Other ":") (status)} 
              {
                (DQ ("                        and storage-media-type: ") 
                  (${ VSub_Name storage_media_type)
                )
              }
            )
            (C {(kube) (Lit_Other ":") (Lit_Other ":") (log) (Lit_Other ":") (Lit_Other ":") (status)} 
              {
                (DQ ("                            and runtime-config: ") (${ VSub_Name RUNTIME_CONFIG))
              }
            )
            (C {(kube) (Lit_Other ":") (Lit_Other ":") (log) (Lit_Other ":") (Lit_Other ":") (status)} 
              {
                (DQ ("                 and storage-version overrides: ") 
                  (${ VSub_Name storage_versions)
                )
              }
            )
            (Sentence
              child: 
                (SimpleCommand
                  words: [
                    {(DQ (${ VSub_Name KUBE_OUTPUT_HOSTBIN) (/kube-apiserver))}
                    {(--insecure-bind-address) (Lit_Other "=") (DQ (${ VSub_Name API_HOST))}
                    {(--bind-address) (Lit_Other "=") (DQ (${ VSub_Name API_HOST))}
                    {(--insecure-port) (Lit_Other "=") (DQ (${ VSub_Name API_PORT))}
                    {(--storage-backend) (Lit_Other "=") (DQ (etcd3))}
                    {(--etcd-servers) (Lit_Other "=") 
                      (DQ ("http://") (${ VSub_Name ETCD_HOST) (":") (${ VSub_Name ETCD_PORT))
                    }
                    {(--etcd-prefix) (Lit_Other "=") (DQ (/) (${ VSub_Name ETCD_PREFIX))}
                    {(--runtime-config) (Lit_Other "=") (DQ (${ VSub_Name RUNTIME_CONFIG))}
                    {(--cert-dir) (Lit_Other "=") 
                      (DQ 
                        (BracedVarSub
                          token: <VSub_Name TMPDIR>
                          suffix_op: 
                            (StringUnary
                              op_id: VTest_ColonHyphen
                              arg_word: {(Lit_Slash /) (tmp) (Lit_Slash /)}
                            )
                          spids: [404 410]
                        )
                      )
                    }
                    {(--service-cluster-ip-range) (Lit_Other "=") (DQ (10.0.0.0/24))}
                    {(--storage-versions) (Lit_Other "=") (DQ (${ VSub_Name storage_versions))}
                    {(--storage-media-type) (Lit_Other "=") (${ VSub_Name storage_media_type)}
                  ]
                  redirects: [(Redir op_id:Redir_GreatAnd fd:1 arg_word:{(2)} spids:[439])]
                  more_env: [
                    (env_pair
                      name: KUBE_API_VERSIONS
                      val: {(DQ (${ VSub_Name KUBE_API_VERSIONS))}
                      spids: [309]
                    )
                  ]
                )
              terminator: <Op_Amp "&">
            )
            (Assignment
              keyword: Assign_None
              pairs: [
                (assign_pair
                  lhs: (LhsName name:APISERVER_PID)
                  op: Equal
                  rhs: {($ VSub_Bang "$!")}
                  spids: [445]
                )
              ]
              spids: [445]
            )
            (C 
              {(kube) (Lit_Other ":") (Lit_Other ":") (util) (Lit_Other ":") (Lit_Other ":") 
                (wait_for_url)
              } {(DQ ("http://") (${ VSub_Name API_HOST) (":") (${ VSub_Name API_PORT) (/healthz))} 
              {(DQ ("apiserver: "))} {(1)} {(120)}
            )
          ]
          spids: [219]
        )
      spids: [213 218]
    )
    (FuncDef
      name: killApiServer
      body: 
        (BraceGroup
          children: [
            (C {(kube) (Lit_Other ":") (Lit_Other ":") (log) (Lit_Other ":") (Lit_Other ":") (status)} 
              {(DQ ("Killing api server"))}
            )
            (If
              arms: [
                (if_arm
                  cond: [
                    (Sentence
                      child: 
                        (DBracket
                          expr: 
                            (BoolUnary
                              op_id: BoolUnary_n
                              child: 
                                {
                                  (BracedVarSub
                                    token: <VSub_Name APISERVER_PID>
                                    suffix_op: (StringUnary op_id:VTest_Hyphen arg_word:{})
                                    spids: [513 516]
                                  )
                                }
                            )
                        )
                      terminator: <Op_Semi ";">
                    )
                  ]
                  action: [
                    (SimpleCommand
                      words: [{(kill)} {(${ VSub_Name APISERVER_PID)}]
                      redirects: [
                        (Redir
                          op_id: Redir_GreatAnd
                          fd: 1
                          arg_word: {(2)}
                          spids: [530]
                        )
                        (Redir
                          op_id: Redir_Great
                          fd: 2
                          arg_word: {(/dev/null)}
                          spids: [533]
                        )
                      ]
                    )
                    (AndOr
                      children: [(C {(wait)} {(${ VSub_Name APISERVER_PID)}) (C {(true)})]
                      op_id: Op_DPipe
                    )
                    (C 
                      {(kube) (Lit_Other ":") (Lit_Other ":") (log) (Lit_Other ":") (Lit_Other ":") 
                        (status)
                      } {(DQ ("api server exited"))}
                    )
                  ]
                  spids: [-1 521]
                )
              ]
              spids: [-1 561]
            )
            (C {(unset)} {(APISERVER_PID)})
          ]
          spids: [491]
        )
      spids: [485 490]
    )
    (FuncDef
      name: cleanup
      body: 
        (BraceGroup
          children: [
            (C {(killApiServer)})
            (C 
              {(kube) (Lit_Other ":") (Lit_Other ":") (etcd) (Lit_Other ":") (Lit_Other ":") (cleanup)}
            )
            (C {(kube) (Lit_Other ":") (Lit_Other ":") (log) (Lit_Other ":") (Lit_Other ":") (status)} 
              {(DQ ("Clean up complete"))}
            )
          ]
          spids: [577]
        )
      spids: [571 576]
    )
    (C {(trap)} {(cleanup)} {(EXIT)} {(SIGINT)})
    (C {(make)} {(-C)} {(DQ (${ VSub_Name KUBE_ROOT))} {(Lit_VarLike "WHAT=") (cmd/kube-apiserver)})
    (C {(make)} {(-C)} {(DQ (${ VSub_Name KUBE_ROOT))} 
      {(Lit_VarLike "WHAT=") (cluster/images/etcd/attachlease)}
    )
    (C {(kube) (Lit_Other ":") (Lit_Other ":") (etcd) (Lit_Other ":") (Lit_Other ":") (start)})
    (SimpleCommand
      words: [{(echo)} {(DQ (${ VSub_Name ETCD_VERSION))}]
      redirects: [
        (Redir
          op_id: Redir_Great
          fd: -1
          arg_word: {(DQ (${ VSub_Name ETCD_DIR) (/version.txt))}
          spids: [661]
        )
      ]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:tests)
          op: Equal
          rhs: 
            {
              (ArrayLiteralPart
                words: [
                  {(examples/persistent-volume-provisioning/rbd/rbd-storage-class.yaml) 
                    (Lit_Comma ",") (storageclasses) (Lit_Comma ",") (Lit_Comma ",") (slow) (Lit_Comma ",") (v1beta1) (Lit_Comma ",") (v1)
                  }
                ]
              )
            }
          spids: [678]
        )
      ]
      spids: [678]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:KUBE_OLD_API_VERSION)
          op: Equal
          rhs: {(DQ ("networking.k8s.io/v1,storage.k8s.io/v1beta1,extensions/v1beta1"))}
          spids: [695]
        )
      ]
      spids: [695]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:KUBE_NEW_API_VERSION)
          op: Equal
          rhs: {(DQ ("networking.k8s.io/v1,storage.k8s.io/v1,extensions/v1beta1"))}
          spids: [700]
        )
      ]
      spids: [700]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:KUBE_OLD_STORAGE_VERSIONS)
          op: Equal
          rhs: {(DQ (storage.k8s.io/v1beta1))}
          spids: [705]
        )
      ]
      spids: [705]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:KUBE_NEW_STORAGE_VERSIONS)
          op: Equal
          rhs: {(DQ (storage.k8s.io/v1))}
          spids: [710]
        )
      ]
      spids: [710]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:KUBE_API_VERSIONS)
          op: Equal
          rhs: 
            {
              (DQ ("v1,") (${ VSub_Name KUBE_OLD_API_VERSION) (",") (${ VSub_Name KUBE_NEW_API_VERSION))
            }
          spids: [735]
        )
      ]
      spids: [735]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:RUNTIME_CONFIG)
          op: Equal
          rhs: 
            {
              (DQ ("api/all=false,api/v1=true,") (${ VSub_Name KUBE_OLD_API_VERSION) ("=true,") 
                (${ VSub_Name KUBE_NEW_API_VERSION) ("=true")
              )
            }
          spids: [747]
        )
      ]
      spids: [747]
    )
    (C {(startApiServer)} {(${ VSub_Name KUBE_OLD_STORAGE_VERSIONS)} 
      {(${ VSub_Name KUBE_STORAGE_MEDIA_TYPE_JSON)}
    )
    (ForEach
      iter_name: test
      iter_words: [
        {(BracedVarSub token:<VSub_Name tests> bracket_op:(WholeArray op_id:Lit_At) spids:[781786])}
      ]
      do_arg_iter: False
      body: 
        (DoGroup
          children: [
            (SimpleCommand
              words: [{(read)} {(-ra)} {(test_data)}]
              redirects: [
                (Redir
                  op_id: Redir_TLess
                  fd: -1
                  arg_word: {(DQ ($ VSub_Name "$test"))}
                  spids: [803]
                )
              ]
              more_env: [(env_pair name:IFS val:{(SQ <",">)} spids:[792])]
            )
            (Assignment
              keyword: Assign_None
              pairs: [
                (assign_pair
                  lhs: (LhsName name:source_file)
                  op: Equal
                  rhs: 
                    {
                      (BracedVarSub
                        token: <VSub_Name test_data>
                        bracket_op: (ArrayIndex expr:(ArithWord w:{(Lit_Digits 0)}))
                        spids: [810 815]
                      )
                    }
                  spids: [809]
                )
              ]
              spids: [809]
            )
            (C {(kube) (Lit_Other ":") (Lit_Other ":") (log) (Lit_Other ":") (Lit_Other ":") (status)} 
              {(DQ ("Creating ") (${ VSub_Name source_file))}
            )
            (C {(${ VSub_Name KUBECTL)} {(create)} {(-f)} {(DQ (${ VSub_Name source_file))})
            (Assignment
              keyword: Assign_None
              pairs: [
                (assign_pair
                  lhs: (LhsName name:resource)
                  op: Equal
                  rhs: 
                    {
                      (BracedVarSub
                        token: <VSub_Name test_data>
                        bracket_op: (ArrayIndex expr:(ArithWord w:{(Lit_Digits 1)}))
                        spids: [856 861]
                      )
                    }
                  spids: [855]
                )
              ]
              spids: [855]
            )
            (Assignment
              keyword: Assign_None
              pairs: [
                (assign_pair
                  lhs: (LhsName name:namespace)
                  op: Equal
                  rhs: 
                    {
                      (BracedVarSub
                        token: <VSub_Name test_data>
                        bracket_op: (ArrayIndex expr:(ArithWord w:{(Lit_Digits 2)}))
                        spids: [865 870]
                      )
                    }
                  spids: [864]
                )
              ]
              spids: [864]
            )
            (Assignment
              keyword: Assign_None
              pairs: [
                (assign_pair
                  lhs: (LhsName name:name)
                  op: Equal
                  rhs: 
                    {
                      (BracedVarSub
                        token: <VSub_Name test_data>
                        bracket_op: (ArrayIndex expr:(ArithWord w:{(Lit_Digits 3)}))
                        spids: [874 879]
                      )
                    }
                  spids: [873]
                )
              ]
              spids: [873]
            )
            (Assignment
              keyword: Assign_None
              pairs: [
                (assign_pair
                  lhs: (LhsName name:old_storage_version)
                  op: Equal
                  rhs: 
                    {
                      (BracedVarSub
                        token: <VSub_Name test_data>
                        bracket_op: (ArrayIndex expr:(ArithWord w:{(Lit_Digits 4)}))
                        spids: [883 888]
                      )
                    }
                  spids: [882]
                )
              ]
              spids: [882]
            )
            (If
              arms: [
                (if_arm
                  cond: [
                    (Sentence
                      child: 
                        (C {(Lit_Other "[")} {(-n)} {(DQ (${ VSub_Name namespace))} {(Lit_Other "]")})
                      terminator: <Op_Semi ";">
                    )
                  ]
                  action: [
                    (Assignment
                      keyword: Assign_None
                      pairs: [
                        (assign_pair
                          lhs: (LhsName name:namespace)
                          op: Equal
                          rhs: {(DQ (${ VSub_Name namespace) (/))}
                          spids: [910]
                        )
                      ]
                      spids: [910]
                    )
                  ]
                  spids: [-1 907]
                )
              ]
              spids: [-1 919]
            )
            (C {(kube) (Lit_Other ":") (Lit_Other ":") (log) (Lit_Other ":") (Lit_Other ":") (status)} 
              {
                (DQ ("Verifying ") (${ VSub_Name resource) (/) (${ VSub_Name namespace) 
                  (${ VSub_Name name) (" has storage version ") (${ VSub_Name old_storage_version) (" in etcd")
                )
              }
            )
            (Pipeline
              children: [
                (SimpleCommand
                  words: [
                    {(${ VSub_Name ETCDCTL)}
                    {(--endpoints) (Lit_Other "=") 
                      (DQ ("http://") (${ VSub_Name ETCD_HOST) (":") (${ VSub_Name ETCD_PORT))
                    }
                    {(get)}
                    {
                      (DQ (/) (${ VSub_Name ETCD_PREFIX) (/) (${ VSub_Name resource) (/) 
                        (${ VSub_Name namespace) (${ VSub_Name name)
                      )
                    }
                  ]
                  more_env: [(env_pair name:ETCDCTL_API val:{(3)} spids:[950])]
                )
                (C {(grep)} {(${ VSub_Name old_storage_version)})
              ]
              negated: False
            )
          ]
          spids: [789 998]
        )
      spids: [780 787]
    )
    (C {(killApiServer)})
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:KUBE_API_VERSIONS)
          op: Equal
          rhs: 
            {
              (DQ ("v1,") (${ VSub_Name KUBE_NEW_API_VERSION) (",") (${ VSub_Name KUBE_OLD_API_VERSION))
            }
          spids: [1021]
        )
      ]
      spids: [1021]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:RUNTIME_CONFIG)
          op: Equal
          rhs: 
            {
              (DQ ("api/all=false,api/v1=true,") (${ VSub_Name KUBE_OLD_API_VERSION) ("=true,") 
                (${ VSub_Name KUBE_NEW_API_VERSION) ("=true")
              )
            }
          spids: [1033]
        )
      ]
      spids: [1033]
    )
    (C {(startApiServer)} {(${ VSub_Name KUBE_NEW_STORAGE_VERSIONS)} 
      {(${ VSub_Name KUBE_STORAGE_MEDIA_TYPE_JSON)}
    )
    (C {(kube) (Lit_Other ":") (Lit_Other ":") (log) (Lit_Other ":") (Lit_Other ":") (status)} 
      {(DQ ("Updating storage versions in etcd"))}
    )
    (C {(${ VSub_Name UPDATE_ETCD_OBJECTS_SCRIPT)})
    (ForEach
      iter_name: test
      iter_words: [
        {
          (BracedVarSub
            token: <VSub_Name tests>
            bracket_op: (WholeArray op_id:Lit_At)
            spids: [1086 1091]
          )
        }
      ]
      do_arg_iter: False
      body: 
        (DoGroup
          children: [
            (SimpleCommand
              words: [{(read)} {(-ra)} {(test_data)}]
              redirects: [
                (Redir
                  op_id: Redir_TLess
                  fd: -1
                  arg_word: {(DQ ($ VSub_Name "$test"))}
                  spids: [1108]
                )
              ]
              more_env: [(env_pair name:IFS val:{(SQ <",">)} spids:[1097])]
            )
            (Assignment
              keyword: Assign_None
              pairs: [
                (assign_pair
                  lhs: (LhsName name:resource)
                  op: Equal
                  rhs: 
                    {
                      (BracedVarSub
                        token: <VSub_Name test_data>
                        bracket_op: (ArrayIndex expr:(ArithWord w:{(Lit_Digits 1)}))
                        spids: [1115 1120]
                      )
                    }
                  spids: [1114]
                )
              ]
              spids: [1114]
            )
            (Assignment
              keyword: Assign_None
              pairs: [
                (assign_pair
                  lhs: (LhsName name:namespace)
                  op: Equal
                  rhs: 
                    {
                      (BracedVarSub
                        token: <VSub_Name test_data>
                        bracket_op: (ArrayIndex expr:(ArithWord w:{(Lit_Digits 2)}))
                        spids: [1124 1129]
                      )
                    }
                  spids: [1123]
                )
              ]
              spids: [1123]
            )
            (Assignment
              keyword: Assign_None
              pairs: [
                (assign_pair
                  lhs: (LhsName name:name)
                  op: Equal
                  rhs: 
                    {
                      (BracedVarSub
                        token: <VSub_Name test_data>
                        bracket_op: (ArrayIndex expr:(ArithWord w:{(Lit_Digits 3)}))
                        spids: [1133 1138]
                      )
                    }
                  spids: [1132]
                )
              ]
              spids: [1132]
            )
            (Assignment
              keyword: Assign_None
              pairs: [
                (assign_pair
                  lhs: (LhsName name:new_storage_version)
                  op: Equal
                  rhs: 
                    {
                      (BracedVarSub
                        token: <VSub_Name test_data>
                        bracket_op: (ArrayIndex expr:(ArithWord w:{(Lit_Digits 5)}))
                        spids: [1142 1147]
                      )
                    }
                  spids: [1141]
                )
              ]
              spids: [1141]
            )
            (If
              arms: [
                (if_arm
                  cond: [
                    (Sentence
                      child: 
                        (C {(Lit_Other "[")} {(-n)} {(DQ (${ VSub_Name namespace))} {(Lit_Other "]")})
                      terminator: <Op_Semi ";">
                    )
                  ]
                  action: [
                    (Assignment
                      keyword: Assign_None
                      pairs: [
                        (assign_pair
                          lhs: (LhsName name:namespace)
                          op: Equal
                          rhs: {(DQ (${ VSub_Name namespace) (/))}
                          spids: [1169]
                        )
                      ]
                      spids: [1169]
                    )
                  ]
                  spids: [-1 1166]
                )
              ]
              spids: [-1 1178]
            )
            (C {(kube) (Lit_Other ":") (Lit_Other ":") (log) (Lit_Other ":") (Lit_Other ":") (status)} 
              {
                (DQ ("Verifying ") (${ VSub_Name resource) (/) (${ VSub_Name namespace) 
                  (${ VSub_Name name) (" has updated storage version ") (${ VSub_Name new_storage_version) (" in etcd")
                )
              }
            )
            (Pipeline
              children: [
                (SimpleCommand
                  words: [
                    {(${ VSub_Name ETCDCTL)}
                    {(--endpoints) (Lit_Other "=") 
                      (DQ ("http://") (${ VSub_Name ETCD_HOST) (":") (${ VSub_Name ETCD_PORT))
                    }
                    {(get)}
                    {
                      (DQ (/) (${ VSub_Name ETCD_PREFIX) (/) (${ VSub_Name resource) (/) 
                        (${ VSub_Name namespace) (${ VSub_Name name)
                      )
                    }
                  ]
                  more_env: [(env_pair name:ETCDCTL_API val:{(3)} spids:[1209])]
                )
                (C {(grep)} {(${ VSub_Name new_storage_version)})
              ]
              negated: False
            )
          ]
          spids: [1094 1257]
        )
      spids: [1085 1092]
    )
    (C {(killApiServer)})
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:KUBE_API_VERSIONS)
          op: Equal
          rhs: {(DQ ("v1,") (${ VSub_Name KUBE_NEW_API_VERSION))}
          spids: [1277]
        )
      ]
      spids: [1277]
    )
    (Assignment
      keyword: Assign_None
      pairs: [
        (assign_pair
          lhs: (LhsName name:RUNTIME_CONFIG)
          op: Equal
          rhs: {(DQ ("api/all=false,api/v1=true,") (${ VSub_Name KUBE_NEW_API_VERSION) ("=true"))}
          spids: [1285]
        )
      ]
      spids: [1285]
    )
    (C {(sleep)} {(1)})
    (C {(startApiServer)} {(${ VSub_Name KUBE_NEW_STORAGE_VERSIONS)} 
      {(${ VSub_Name KUBE_STORAGE_MEDIA_TYPE_PROTOBUF)}
    )
    (ForEach
      iter_name: test
      iter_words: [
        {
          (BracedVarSub
            token: <VSub_Name tests>
            bracket_op: (WholeArray op_id:Lit_At)
            spids: [1319 1324]
          )
        }
      ]
      do_arg_iter: False
      body: 
        (DoGroup
          children: [
            (SimpleCommand
              words: [{(read)} {(-ra)} {(test_data)}]
              redirects: [
                (Redir
                  op_id: Redir_TLess
                  fd: -1
                  arg_word: {(DQ ($ VSub_Name "$test"))}
                  spids: [1341]
                )
              ]
              more_env: [(env_pair name:IFS val:{(SQ <",">)} spids:[1330])]
            )
            (Assignment
              keyword: Assign_None
              pairs: [
                (assign_pair
                  lhs: (LhsName name:resource)
                  op: Equal
                  rhs: 
                    {
                      (BracedVarSub
                        token: <VSub_Name test_data>
                        bracket_op: (ArrayIndex expr:(ArithWord w:{(Lit_Digits 1)}))
                        spids: [1348 1353]
                      )
                    }
                  spids: [1347]
                )
              ]
              spids: [1347]
            )
            (Assignment
              keyword: Assign_None
              pairs: [
                (assign_pair
                  lhs: (LhsName name:namespace)
                  op: Equal
                  rhs: 
                    {
                      (BracedVarSub
                        token: <VSub_Name test_data>
                        bracket_op: (ArrayIndex expr:(ArithWord w:{(Lit_Digits 2)}))
                        spids: [1357 1362]
                      )
                    }
                  spids: [1356]
                )
              ]
              spids: [1356]
            )
            (Assignment
              keyword: Assign_None
              pairs: [
                (assign_pair
                  lhs: (LhsName name:name)
                  op: Equal
                  rhs: 
                    {
                      (BracedVarSub
                        token: <VSub_Name test_data>
                        bracket_op: (ArrayIndex expr:(ArithWord w:{(Lit_Digits 3)}))
                        spids: [1366 1371]
                      )
                    }
                  spids: [1365]
                )
              ]
              spids: [1365]
            )
            (Assignment
              keyword: Assign_None
              pairs: [
                (assign_pair
                  lhs: (LhsName name:namespace_flag)
                  op: Equal
                  rhs: {(DQ )}
                  spids: [1374]
                )
              ]
              spids: [1374]
            )
            (If
              arms: [
                (if_arm
                  cond: [
                    (Sentence
                      child: 
                        (C {(Lit_Other "[")} {(-n)} {(DQ (${ VSub_Name namespace))} {(Lit_Other "]")})
                      terminator: <Op_Semi ";">
                    )
                  ]
                  action: [
                    (Assignment
                      keyword: Assign_None
                      pairs: [
                        (assign_pair
                          lhs: (LhsName name:namespace_flag)
                          op: Equal
                          rhs: {(DQ ("--namespace=") (${ VSub_Name namespace))}
                          spids: [1402]
                        )
                      ]
                      spids: [1402]
                    )
                    (Assignment
                      keyword: Assign_None
                      pairs: [
                        (assign_pair
                          lhs: (LhsName name:namespace)
                          op: Equal
                          rhs: {(DQ (${ VSub_Name namespace) (/))}
                          spids: [1411]
                        )
                      ]
                      spids: [1411]
                    )
                  ]
                  spids: [-1 1399]
                )
              ]
              spids: [-1 1420]
            )
            (C {(kube) (Lit_Other ":") (Lit_Other ":") (log) (Lit_Other ":") (Lit_Other ":") (status)} 
              {
                (DQ ("Verifying we can retrieve ") (${ VSub_Name resource) (/) (${ VSub_Name namespace) 
                  (${ VSub_Name name) (" via kubectl")
                )
              }
            )
            (C {(rm)} {(-rf)} 
              {(${ VSub_Name HOME) (/.kube/cache/discovery/localhost_8080/) 
                (${ VSub_Name KUBE_OLD_STORAGE_VERSIONS)
              }
            )
            (C {(${ VSub_Name KUBECTL)} {(get)} {(${ VSub_Name namespace_flag)} 
              {(${ VSub_Name resource) (/) (${ VSub_Name name)}
            )
          ]
          spids: [1327 1486]
        )
      spids: [1318 1325]
    )
    (C {(killApiServer)})
  ]
)