#!/bin/bash # Copyright 2016 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # A library of helper functions and constant for the Container Linux distro. source "$(KUBE_ROOT)/cluster/gce/container-linux/helper.sh" # create-master-instance creates the master instance. If called with # an argument, the argument is used as the name to a reserved IP # address for the master. (In the case of upgrade/repair, we re-use # the same IP.) # # It requires a whole slew of assumed variables, partially due to to # the call to write-master-env. Listing them would be rather # futile. Instead, we list the required calls to ensure any additional # # variables are set: # ensure-temp-dir # detect-project # get-bearer-token proc create-master-instance { var address = ''"" [[ -n ${1:-} ]] && address := $(1) write-master-env create-master-instance-internal $(MASTER_NAME) $(address) } proc replicate-master-instance { var existing_master_zone = $(1) var existing_master_name = $(2) var existing_master_replicas = $(3) var kube_env = $[get-metadata $(existing_master_zone) $(existing_master_name) kube-env] # Substitute INITIAL_ETCD_CLUSTER to enable etcd clustering. kube_env := $[echo $(kube_env) | grep -v "INITIAL_ETCD_CLUSTER] kube_env := $[echo -e "$(kube_env)\nINITIAL_ETCD_CLUSTER: '$(existing_master_replicas),$(REPLICA_NAME)'] global ETCD_CA_KEY := $[echo $(kube_env) | grep "ETCD_CA_KEY" | sed "s/^.*: '//" | sed "s/'$//] global ETCD_CA_CERT := $[echo $(kube_env) | grep "ETCD_CA_CERT" | sed "s/^.*: '//" | sed "s/'$//] create-etcd-certs $(REPLICA_NAME) $(ETCD_CA_CERT) $(ETCD_CA_KEY) kube_env := $[echo $(kube_env) | grep -v "ETCD_PEER_KEY] kube_env := $[echo -e "$(kube_env)\nETCD_PEER_KEY: '$(ETCD_PEER_KEY_BASE64)'] kube_env := $[echo $(kube_env) | grep -v "ETCD_PEER_CERT] kube_env := $[echo -e "$(kube_env)\nETCD_PEER_CERT: '$(ETCD_PEER_CERT_BASE64)'] echo $(kube_env) > $(KUBE_TEMP)/master-kube-env.yaml get-metadata $(existing_master_zone) $(existing_master_name) cluster-name > "$(KUBE_TEMP)/cluster-name.txt" create-master-instance-internal $(REPLICA_NAME) } proc create-master-instance-internal { var gcloud = '"gcloud'" var retries = '5' if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]] { gcloud := '"gcloud beta'" } var -r master_name = $(1) var -r address = $(2:-) var preemptible_master = ''"" if [[ "${PREEMPTIBLE_MASTER:-}" == "true" ]] { preemptible_master := '"--preemptible --maintenance-policy TERMINATE'" } var network = $[make-gcloud-network-argument \ $(NETWORK_PROJECT) $(REGION) $(NETWORK) $(SUBNETWORK:-) \ $(address:-) $(ENABLE_IP_ALIASES:-) $(IP_ALIAS_SIZE:-)] var metadata = ""kube-env=$(KUBE_TEMP)/master-kube-env.yaml"" metadata := ""$(metadata),user-data=$(KUBE_ROOT)/cluster/gce/container-linux/master.yaml"" metadata := ""$(metadata),configure-sh=$(KUBE_ROOT)/cluster/gce/container-linux/configure.sh"" metadata := ""$(metadata),cluster-name=$(KUBE_TEMP)/cluster-name.txt"" var disk = ""name=$(master_name)-pd"" disk := ""$(disk),device-name=master-pd"" disk := ""$(disk),mode=rw"" disk := ""$(disk),boot=no"" disk := ""$(disk),auto-delete=no"" for attempt in [$[seq 1 $(retries)]] { if global result := $[$(gcloud) compute instances create $(master_name) \ --project $(PROJECT) \ --zone $(ZONE) \ --machine-type $(MASTER_SIZE) \ --image-project="$(MASTER_IMAGE_PROJECT)" \ --image $(MASTER_IMAGE) \ --tags $(MASTER_TAG) \ --scopes "storage-ro,compute-rw,monitoring,logging-write" \ --metadata-from-file $(metadata) \ --disk $(disk) \ --boot-disk-size $(MASTER_ROOT_DISK_SIZE) \ $(preemptible_master) \ $(network) !2 > !1] { echo $(result) > !2 return 0 } else { echo $(result) > !2 if [[ ! "${result}" =~ "try again later" ]] { echo "Failed to create master instance due to non-retryable error" > !2 return 1 } sleep 10 } } echo "Failed to create master instance despite $(retries) attempts" > !2 return 1 } proc get-metadata { var zone = $(1) var name = $(2) var key = $(3) var metadata_url = ""http://metadata.google.internal/computeMetadata/v1/instance/attributes/$(key)"" gcloud compute ssh $(name) \ --project $(PROJECT) \ --zone $(zone) \ --command "curl '$(metadata_url)' -H 'Metadata-Flavor: Google'" !2 >/dev/null } (CommandList children: [ (C {(source)} {(DQ (${ VSub_Name KUBE_ROOT) (/cluster/gce/container-linux/helper.sh))}) (FuncDef name: create-master-instance body: (BraceGroup children: [ (Assignment keyword: Assign_Local pairs: [(assign_pair lhs:(LhsName name:address) op:Equal rhs:{(DQ )} spids:[105])] spids: [103] ) (AndOr children: [ (DBracket expr: (BoolUnary op_id: BoolUnary_n child: { (BracedVarSub token: suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{}) spids: [114 117] ) } ) ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:address) op: Equal rhs: {(DQ (${ VSub_Number 1))} spids: [123] ) ] spids: [123] ) ] op_id: Op_DAmp ) (C {(write-master-env)}) (C {(create-master-instance-internal)} {(DQ (${ VSub_Name MASTER_NAME))} {(DQ (${ VSub_Name address))} ) ] spids: [100] ) spids: [96 99] ) (FuncDef name: replicate-master-instance body: (BraceGroup children: [ (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:existing_master_zone) op: Equal rhs: {(DQ (${ VSub_Number 1))} spids: [163] ) ] spids: [161] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:existing_master_name) op: Equal rhs: {(DQ (${ VSub_Number 2))} spids: [173] ) ] spids: [171] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:existing_master_replicas) op: Equal rhs: {(DQ (${ VSub_Number 3))} spids: [183] ) ] spids: [181] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:kube_env) op: Equal rhs: { (DQ (CommandSubPart command_list: (CommandList children: [ (C {(get-metadata)} {(DQ (${ VSub_Name existing_master_zone))} {(DQ (${ VSub_Name existing_master_name))} {(kube-env)} ) ] ) left_token: spids: [196 212] ) ) } spids: [194] ) ] spids: [192] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:kube_env) op: Equal rhs: { (DQ (CommandSubPart command_list: (CommandList children: [ (Pipeline children: [ (C {(echo)} {(DQ (${ VSub_Name kube_env))}) (C {(grep)} {(-v)} {(DQ (INITIAL_ETCD_CLUSTER))}) ] negated: False ) ] ) left_token: spids: [222 240] ) ) } spids: [220] ) ] spids: [220] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:kube_env) op: Equal rhs: { (DQ (CommandSubPart command_list: (CommandList children: [ (C {(echo)} {(-e)} { (DQ (${ VSub_Name kube_env) (EscapedLiteralPart token: ) ("INITIAL_ETCD_CLUSTER: '") (${ VSub_Name existing_master_replicas) (",") (${ VSub_Name REPLICA_NAME) ("'") ) } ) ] ) left_token: spids: [246 266] ) ) } spids: [244] ) ] spids: [244] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:ETCD_CA_KEY) op: Equal rhs: { (DQ (CommandSubPart command_list: (CommandList children: [ (Pipeline children: [ (C {(echo)} {(DQ (${ VSub_Name kube_env))}) (C {(grep)} {(DQ (ETCD_CA_KEY))}) (C {(sed)} {(DQ ("s/^.*: '//"))}) (C {(sed)} {(DQ ("s/'") (Lit_Other "$") (//))}) ] negated: False ) ] ) left_token: spids: [272 306] ) ) } spids: [270] ) ] spids: [270] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:ETCD_CA_CERT) op: Equal rhs: { (DQ (CommandSubPart command_list: (CommandList children: [ (Pipeline children: [ (C {(echo)} {(DQ (${ VSub_Name kube_env))}) (C {(grep)} {(DQ (ETCD_CA_CERT))}) (C {(sed)} {(DQ ("s/^.*: '//"))}) (C {(sed)} {(DQ ("s/'") (Lit_Other "$") (//))}) ] negated: False ) ] ) left_token: spids: [312 346] ) ) } spids: [310] ) ] spids: [310] ) (C {(create-etcd-certs)} {(DQ (${ VSub_Name REPLICA_NAME))} {(DQ (${ VSub_Name ETCD_CA_CERT))} {(DQ (${ VSub_Name ETCD_CA_KEY))} ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:kube_env) op: Equal rhs: { (DQ (CommandSubPart command_list: (CommandList children: [ (Pipeline children: [ (C {(echo)} {(DQ (${ VSub_Name kube_env))}) (C {(grep)} {(-v)} {(DQ (ETCD_PEER_KEY))}) ] negated: False ) ] ) left_token: spids: [375 393] ) ) } spids: [373] ) ] spids: [373] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:kube_env) op: Equal rhs: { (DQ (CommandSubPart command_list: (CommandList children: [ (C {(echo)} {(-e)} { (DQ (${ VSub_Name kube_env) (EscapedLiteralPart token: ) ("ETCD_PEER_KEY: '") (${ VSub_Name ETCD_PEER_KEY_BASE64) ("'") ) } ) ] ) left_token: spids: [399 415] ) ) } spids: [397] ) ] spids: [397] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:kube_env) op: Equal rhs: { (DQ (CommandSubPart command_list: (CommandList children: [ (Pipeline children: [ (C {(echo)} {(DQ (${ VSub_Name kube_env))}) (C {(grep)} {(-v)} {(DQ (ETCD_PEER_CERT))}) ] negated: False ) ] ) left_token: spids: [421 439] ) ) } spids: [419] ) ] spids: [419] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:kube_env) op: Equal rhs: { (DQ (CommandSubPart command_list: (CommandList children: [ (C {(echo)} {(-e)} { (DQ (${ VSub_Name kube_env) (EscapedLiteralPart token: ) ("ETCD_PEER_CERT: '") (${ VSub_Name ETCD_PEER_CERT_BASE64) ("'") ) } ) ] ) left_token: spids: [445 461] ) ) } spids: [443] ) ] spids: [443] ) (SimpleCommand words: [{(echo)} {(DQ (${ VSub_Name kube_env))}] redirects: [ (Redir op_id: Redir_Great fd: -1 arg_word: {(${ VSub_Name KUBE_TEMP) (/master-kube-env.yaml)} spids: [474] ) ] ) (SimpleCommand words: [ {(get-metadata)} {(DQ (${ VSub_Name existing_master_zone))} {(DQ (${ VSub_Name existing_master_name))} {(cluster-name)} ] redirects: [ (Redir op_id: Redir_Great fd: -1 arg_word: {(DQ (${ VSub_Name KUBE_TEMP) (/cluster-name.txt))} spids: [498] ) ] ) (C {(create-master-instance-internal)} {(DQ (${ VSub_Name REPLICA_NAME))}) ] spids: [158] ) spids: [152 157] ) (FuncDef name: create-master-instance-internal body: (BraceGroup children: [ (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:gcloud) op: Equal rhs: {(DQ (gcloud))} spids: [532] ) ] spids: [530] ) (Assignment keyword: Assign_Local pairs: [(assign_pair lhs:(LhsName name:retries) op:Equal rhs:{(5)} spids:[540])] spids: [538] ) (If arms: [ (if_arm cond: [ (Sentence child: (DBracket expr: (BoolBinary op_id: BoolBinary_GlobDEqual left: { (DQ (BracedVarSub token: suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(SQ )}) spids: [549 552] ) ) } right: {(SQ )} ) ) terminator: ) ] action: [ (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:gcloud) op: Equal rhs: {(DQ ("gcloud beta"))} spids: [567] ) ] spids: [567] ) ] spids: [-1 564] ) ] spids: [-1 573] ) (Assignment keyword: Assign_Local flags: ["'-r'"] pairs: [ (assign_pair lhs: (LhsName name:master_name) op: Equal rhs: {(DQ (${ VSub_Number 1))} spids: [581] ) ] spids: [577] ) (Assignment keyword: Assign_Local flags: ["'-r'"] pairs: [ (assign_pair lhs: (LhsName name:address) op: Equal rhs: { (DQ (BracedVarSub token: suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(SQ )}) spids: [595 598] ) ) } spids: [593] ) ] spids: [589] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:preemptible_master) op: Equal rhs: {(DQ )} spids: [605] ) ] spids: [603] ) (If arms: [ (if_arm cond: [ (Sentence child: (DBracket expr: (BoolBinary op_id: BoolBinary_GlobDEqual left: { (DQ (BracedVarSub token: suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(SQ )}) spids: [615 618] ) ) } right: {(DQ (true))} ) ) terminator: ) ] action: [ (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:preemptible_master) op: Equal rhs: {(DQ ("--preemptible --maintenance-policy TERMINATE"))} spids: [633] ) ] spids: [633] ) ] spids: [-1 630] ) ] spids: [-1 639] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:network) op: Equal rhs: { (CommandSubPart command_list: (CommandList children: [ (C {(make-gcloud-network-argument)} {(DQ (${ VSub_Name NETWORK_PROJECT))} {(DQ (${ VSub_Name REGION))} {(DQ (${ VSub_Name NETWORK))} { (DQ (BracedVarSub token: suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(SQ )}) spids: [670 673] ) ) } { (DQ (BracedVarSub token: suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(SQ )}) spids: [679 682] ) ) } { (DQ (BracedVarSub token: suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(SQ )}) spids: [686 689] ) ) } { (DQ (BracedVarSub token: suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(SQ )}) spids: [693 696] ) ) } ) ] ) left_token: spids: [646 698] ) } spids: [645] ) ] spids: [643] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:metadata) op: Equal rhs: {(DQ ("kube-env=") (${ VSub_Name KUBE_TEMP) (/master-kube-env.yaml))} spids: [704] ) ] spids: [702] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:metadata) op: Equal rhs: { (DQ (${ VSub_Name metadata) (",user-data=") (${ VSub_Name KUBE_ROOT) (/cluster/gce/container-linux/master.yaml) ) } spids: [714] ) ] spids: [714] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:metadata) op: Equal rhs: { (DQ (${ VSub_Name metadata) (",configure-sh=") (${ VSub_Name KUBE_ROOT) (/cluster/gce/container-linux/configure.sh) ) } spids: [727] ) ] spids: [727] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:metadata) op: Equal rhs: { (DQ (${ VSub_Name metadata) (",cluster-name=") (${ VSub_Name KUBE_TEMP) (/cluster-name.txt) ) } spids: [740] ) ] spids: [740] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:disk) op: Equal rhs: {(DQ ("name=") (${ VSub_Name master_name) (-pd))} spids: [756] ) ] spids: [754] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:disk) op: Equal rhs: {(DQ (${ VSub_Name disk) (",device-name=master-pd"))} spids: [766] ) ] spids: [766] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:disk) op: Equal rhs: {(DQ (${ VSub_Name disk) (",mode=rw"))} spids: [775] ) ] spids: [775] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:disk) op: Equal rhs: {(DQ (${ VSub_Name disk) (",boot=no"))} spids: [784] ) ] spids: [784] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:disk) op: Equal rhs: {(DQ (${ VSub_Name disk) (",auto-delete=no"))} spids: [793] ) ] spids: [793] ) (ForEach iter_name: attempt iter_words: [ { (CommandSubPart command_list: (CommandList children:[(C {(seq)} {(1)} {(${ VSub_Name retries)})]) left_token: spids: [809 817] ) } ] do_arg_iter: False body: (DoGroup children: [ (If arms: [ (if_arm cond: [ (Sentence child: (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:result) op: Equal rhs: { (CommandSubPart command_list: (CommandList children: [ (SimpleCommand words: [ {(${ VSub_Name gcloud)} {(compute)} {(instances)} {(create)} {(DQ (${ VSub_Name master_name))} {(--project)} {(DQ (${ VSub_Name PROJECT))} {(--zone)} {(DQ (${ VSub_Name ZONE))} {(--machine-type)} {(DQ (${ VSub_Name MASTER_SIZE))} {(--image-project) (Lit_Other "=") (DQ (${ VSub_Name MASTER_IMAGE_PROJECT)) } {(--image)} {(DQ (${ VSub_Name MASTER_IMAGE))} {(--tags)} {(DQ (${ VSub_Name MASTER_TAG))} {(--scopes)} { (DQ ( "storage-ro,compute-rw,monitoring,logging-write" ) ) } {(--metadata-from-file)} {(DQ (${ VSub_Name metadata))} {(--disk)} {(DQ (${ VSub_Name disk))} {(--boot-disk-size)} {(DQ (${ VSub_Name MASTER_ROOT_DISK_SIZE))} {(${ VSub_Name preemptible_master)} {(${ VSub_Name network)} ] redirects: [ (Redir op_id: Redir_GreatAnd fd: 2 arg_word: {(1)} spids: [953] ) ] ) ] ) left_token: spids: [826 955] ) } spids: [825] ) ] spids: [825] ) terminator: ) ] action: [ (SimpleCommand words: [{(echo)} {(DQ (${ VSub_Name result))}] redirects: [ (Redir op_id: Redir_GreatAnd fd: -1 arg_word: {(2)} spids: [969] ) ] ) (ControlFlow token: arg_word: {(0)} ) ] spids: [-1 958] ) ] else_action: [ (SimpleCommand words: [{(echo)} {(DQ (${ VSub_Name result))}] redirects: [(Redir op_id:Redir_GreatAnd fd:-1 arg_word:{(2)} spids:[989])] ) (If arms: [ (if_arm cond: [ (Sentence child: (DBracket expr: (LogicalNot child: (BoolBinary op_id: BoolBinary_EqualTilde left: {(DQ (${ VSub_Name result))} right: {(DQ ("try again later"))} ) ) ) terminator: ) ] action: [ (SimpleCommand words: [ {(echo)} { (DQ ( "Failed to create master instance due to non-retryable error" ) ) } ] redirects: [ (Redir op_id: Redir_GreatAnd fd: -1 arg_word: {(2)} spids: [1023] ) ] ) (ControlFlow token: arg_word: {(1)} ) ] spids: [-1 1014] ) ] spids: [-1 1032] ) (C {(sleep)} {(10)}) ] spids: [978 1040] ) ] spids: [820 1043] ) spids: [808 818] ) (SimpleCommand words: [ {(echo)} { (DQ ("Failed to create master instance despite ") (${ VSub_Name retries) (" attempts") ) } ] redirects: [(Redir op_id:Redir_GreatAnd fd:-1 arg_word:{(2)} spids:[1057])] ) (ControlFlow token: arg_word:{(1)}) ] spids: [527] ) spids: [521 526] ) (FuncDef name: get-metadata body: (BraceGroup children: [ (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:zone) op: Equal rhs: {(DQ (${ VSub_Number 1))} spids: [1079] ) ] spids: [1077] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:name) op: Equal rhs: {(DQ (${ VSub_Number 2))} spids: [1089] ) ] spids: [1087] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:key) op: Equal rhs: {(DQ (${ VSub_Number 3))} spids: [1099] ) ] spids: [1097] ) (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:metadata_url) op: Equal rhs: { (DQ ("http://metadata.google.internal/computeMetadata/v1/instance/attributes/") (${ VSub_Name key) ) } spids: [1110] ) ] spids: [1108] ) (SimpleCommand words: [ {(gcloud)} {(compute)} {(ssh)} {(DQ (${ VSub_Name name))} {(--project)} {(DQ (${ VSub_Name PROJECT))} {(--zone)} {(DQ (${ VSub_Name zone))} {(--command)} {(DQ ("curl '") (${ VSub_Name metadata_url) ("' -H 'Metadata-Flavor: Google'"))} ] redirects: [(Redir op_id:Redir_Great fd:2 arg_word:{(/dev/null)} spids:[1164])] ) ] spids: [1074] ) spids: [1068 1073] ) ] )