#!/bin/bash # Copyright 2014 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # LIMITATIONS # 1. Exit code is probably not always correct. # 2. There are no unittests. # 3. Will not work if the total length of paths to addons is greater than # bash can handle. Probably it is not a problem: ARG_MAX=2097152 on GCE. # cosmetic improvements to be done # 1. Improve the log function; add timestamp, file name, etc. # 2. Logging doesn't work from files that print things out. # 3. Kubectl prints the output to stderr (the output should be captured and then # logged) # The business logic for whether a given object should be created # was already enforced by salt, and /etc/kubernetes/addons is the # managed result is of that. Start everything below that directory. global KUBECTL := $(KUBECTL_BIN:-/usr/local/bin/kubectl) global KUBECTL_OPTS := $(KUBECTL_OPTS:-) global ADDON_CHECK_INTERVAL_SEC := $(TEST_ADDON_CHECK_INTERVAL_SEC:-60) global ADDON_PATH := $(ADDON_PATH:-/etc/kubernetes/addons) global SYSTEM_NAMESPACE := 'kube-system' # Addons could use this label with two modes: # - ADDON_MANAGER_LABEL=Reconcile # - ADDON_MANAGER_LABEL=EnsureExists global ADDON_MANAGER_LABEL := '"addonmanager.kubernetes.io/mode'" # This label is deprecated (only for Addon Manager). In future release # addon-manager may not respect it anymore. Addons with # CLUSTER_SERVICE_LABEL=true and without ADDON_MANAGER_LABEL=EnsureExists # will be reconciled for now. global CLUSTER_SERVICE_LABEL := '"kubernetes.io/cluster-service'" # Remember that you can't log from functions that print some output (because # logs are also printed on stdout). # $1 level # $2 message proc log { # manage log levels manually here # add the timestamp if you find it useful matchstr $1 { DB3 { # echo "$1: $2" } DB2 { # echo "$1: $2" } DBG { # echo "$1: $2" } INFO { echo "$1: $2" } WRN { echo "$1: $2" } ERR { echo "$1: $2" } * { echo "INVALID_LOG_LEVEL $1: $2" } } } # $1 filename of addon to start. # $2 count of tries to start the addon. # $3 delay in seconds between two consecutive tries # $4 namespace proc start_addon { var -r addon_filename = $1; var -r tries = $2; var -r delay = $3; var -r namespace = $4 create_resource_from_string $[cat $(addon_filename)] $(tries) $(delay) $(addon_filename) $(namespace) } # $1 string with json or yaml. # $2 count of tries to start the addon. # $3 delay in seconds between two consecutive tries # $4 name of this object to use when logging about it. # $5 namespace for this object proc create_resource_from_string { var -r config_string = $1; var tries = $2; var -r delay = $3; var -r config_name = $4; var -r namespace = $5; while [ ${tries} -gt 0 ] { echo $(config_string) | $(KUBECTL) $(KUBECTL_OPTS) --namespace="$(namespace)" apply -f - && \ log INFO "== Successfully started $(config_name) in namespace $(namespace) at $[date -Is]" && \ return 0; let tries=tries-1; log WRN "== Failed to start $(config_name) in namespace $(namespace) at $[date -Is]. $(tries) tries remaining. ==" sleep $(delay); } return 1; } proc reconcile_addons { # TODO: Remove the first command in future release. # Adding this for backward compatibility. Old addons have CLUSTER_SERVICE_LABEL=true and don't have # ADDON_MANAGER_LABEL=EnsureExists will still be reconciled. # Filter out `configured` message to not noisily log. # `created`, `pruned` and errors will be logged. log INFO "== Reconciling with deprecated label ==" $(KUBECTL) $(KUBECTL_OPTS) apply --namespace=$(SYSTEM_NAMESPACE) -f $(ADDON_PATH) \ -l $(CLUSTER_SERVICE_LABEL)=true,$(ADDON_MANAGER_LABEL)!=EnsureExists \ --prune=true --recursive | grep -v configured log INFO "== Reconciling with addon-manager label ==" $(KUBECTL) $(KUBECTL_OPTS) apply --namespace=$(SYSTEM_NAMESPACE) -f $(ADDON_PATH) \ -l $(CLUSTER_SERVICE_LABEL)!=true,$(ADDON_MANAGER_LABEL)=Reconcile \ --prune=true --recursive | grep -v configured log INFO "== Kubernetes addon reconcile completed at $[date -Is] ==" } proc ensure_addons { # Create objects already exist should fail. # Filter out `AlreadyExists` message to not noisily log. $(KUBECTL) $(KUBECTL_OPTS) create --namespace=$(SYSTEM_NAMESPACE) -f $(ADDON_PATH) \ -l $(ADDON_MANAGER_LABEL)=EnsureExists --recursive !2 > !1 | grep -v AlreadyExists log INFO "== Kubernetes addon ensure completed at $[date -Is] ==" } # The business logic for whether a given object should be created # was already enforced by salt, and /etc/kubernetes/addons is the # managed result is of that. Start everything below that directory. log INFO "== Kubernetes addon manager started at $[date -Is] with ADDON_CHECK_INTERVAL_SEC=$(ADDON_CHECK_INTERVAL_SEC) ==" # Create the namespace that will be used to host the cluster-level add-ons. start_addon /opt/namespace.yaml 100 10 "" & # Wait for the default service account to be created in the kube-system namespace. global token_found := ''"" while [ -z "${token_found}" ] { sleep .5 global token_found := $[$(KUBECTL) $(KUBECTL_OPTS) get --namespace="$(SYSTEM_NAMESPACE)" serviceaccount default -o go-template="{{with index .secrets 0}}{{.name}}{{end}}] if [[ $? -ne 0 ]] { global token_found := ''""; log WRN "== Error getting default service account, retry in 0.5 second ==" } } log INFO "== Default service account in the $(SYSTEM_NAMESPACE) namespace has token $(token_found) ==" # Create admission_control objects if defined before any other addon services. If the limits # are defined in a namespace other than default, we should still create the limits for the # default namespace. for obj in [$[find /etc/kubernetes/admission-controls '(' -name '*'.yaml -o -name '*'.json ')']] { start_addon $(obj) 100 10 default & log INFO "++ obj $(obj) is created ++" } # Start the apply loop. # Check if the configuration has changed recently - in case the user # created/updated/deleted the files on the master. log INFO "== Entering periodical apply loop at $[date -Is] ==" while true { global start_sec := $[date +"%s] ensure_addons reconcile_addons global end_sec := $[date +"%s] global len_sec := $(${end_sec}-${start_sec}) # subtract the time passed from the sleep time if [[ ${len_sec} -lt ${ADDON_CHECK_INTERVAL_SEC} ]] { global sleep_time := $(${ADDON_CHECK_INTERVAL_SEC}-${len_sec}) sleep $(sleep_time) } } (CommandList children: [ (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:KUBECTL) op: Equal rhs: { (BracedVarSub token: suffix_op: (StringUnary op_id: VTest_ColonHyphen arg_word: {(Lit_Slash /) (usr) (Lit_Slash /) (local) (Lit_Slash /) (bin) (Lit_Slash /) (kubectl) } ) spids: [86 97] ) } spids: [85] ) ] spids: [85] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:KUBECTL_OPTS) op: Equal rhs: { (BracedVarSub token: suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{}) spids: [100 103] ) } spids: [99] ) ] spids: [99] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:ADDON_CHECK_INTERVAL_SEC) op: Equal rhs: { (BracedVarSub token: suffix_op: (StringUnary op_id:VTest_ColonHyphen arg_word:{(60)}) spids: [107 111] ) } spids: [106] ) ] spids: [106] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:ADDON_PATH) op: Equal rhs: { (BracedVarSub token: suffix_op: (StringUnary op_id: VTest_ColonHyphen arg_word: {(Lit_Slash /) (etc) (Lit_Slash /) (kubernetes) (Lit_Slash /) (addons)} ) spids: [114 123] ) } spids: [113] ) ] spids: [113] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:SYSTEM_NAMESPACE) op: Equal rhs: {(kube-system)} spids: [126] ) ] spids: [126] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:ADDON_MANAGER_LABEL) op: Equal rhs: {(DQ (addonmanager.kubernetes.io/mode))} spids: [139] ) ] spids: [139] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:CLUSTER_SERVICE_LABEL) op: Equal rhs: {(DQ (kubernetes.io/cluster-service))} spids: [156] ) ] spids: [156] ) (FuncDef name: log body: (BraceGroup children: [ (Case to_match: {($ VSub_Number "$1")} arms: [ (case_arm pat_list:[{(DB3)}] spids:[199201207-1]) (case_arm pat_list:[{(DB2)}] spids:[210212218-1]) (case_arm pat_list:[{(DBG)}] spids:[221223229-1]) (case_arm pat_list: [{(INFO)}] action: [(C {(echo)} {(DQ ($ VSub_Number "$1") (": ") ($ VSub_Number "$2"))})] spids: [232 234 246 -1] ) (case_arm pat_list: [{(WRN)}] action: [(C {(echo)} {(DQ ($ VSub_Number "$1") (": ") ($ VSub_Number "$2"))})] spids: [249 251 263 -1] ) (case_arm pat_list: [{(ERR)}] action: [(C {(echo)} {(DQ ($ VSub_Number "$1") (": ") ($ VSub_Number "$2"))})] spids: [266 268 280 -1] ) (case_arm pat_list: [{(Lit_Other "*")}] action: [ (C {(echo)} {(DQ ("INVALID_LOG_LEVEL ") ($ VSub_Number "$1") (": ") ($ VSub_Number "$2"))} ) ] spids: [283 285 298 -1] ) ] spids: [192 196 301] ) ] spids: [180] ) spids: [174 179] ) (FuncDef name: start_addon body: (BraceGroup children: [ (Sentence child: (Assignment keyword: Assign_Local flags: ["'-r'"] pairs: [ (assign_pair lhs: (LhsName name:addon_filename) op: Equal rhs: {($ VSub_Number "$1")} spids: [331] ) ] spids: [327] ) terminator: ) (Sentence child: (Assignment keyword: Assign_Local flags: ["'-r'"] pairs: [ (assign_pair lhs: (LhsName name:tries) op: Equal rhs: {($ VSub_Number "$2")} spids: [340] ) ] spids: [336] ) terminator: ) (Sentence child: (Assignment keyword: Assign_Local flags: ["'-r'"] pairs: [ (assign_pair lhs: (LhsName name:delay) op: Equal rhs: {($ VSub_Number "$3")} spids: [349] ) ] spids: [345] ) terminator: ) (Assignment keyword: Assign_Local flags: ["'-r'"] pairs: [ (assign_pair lhs: (LhsName name:namespace) op: Equal rhs: {($ VSub_Number "$4")} spids: [358] ) ] spids: [354] ) (C {(create_resource_from_string)} { (DQ (CommandSubPart command_list: (CommandList children: [(C {(cat)} {(${ VSub_Name addon_filename)})] ) left_token: spids: [366 372] ) ) } {(DQ (${ VSub_Name tries))} {(DQ (${ VSub_Name delay))} {(DQ (${ VSub_Name addon_filename))} {(DQ (${ VSub_Name namespace))} ) ] spids: [324] ) spids: [318 323] ) (FuncDef name: create_resource_from_string body: (BraceGroup children: [ (Sentence child: (Assignment keyword: Assign_Local flags: ["'-r'"] pairs: [ (assign_pair lhs: (LhsName name:config_string) op: Equal rhs: {($ VSub_Number "$1")} spids: [430] ) ] spids: [426] ) terminator: ) (Sentence child: (Assignment keyword: Assign_Local pairs: [ (assign_pair lhs: (LhsName name:tries) op: Equal rhs: {($ VSub_Number "$2")} spids: [437] ) ] spids: [435] ) terminator: ) (Sentence child: (Assignment keyword: Assign_Local flags: ["'-r'"] pairs: [ (assign_pair lhs: (LhsName name:delay) op: Equal rhs: {($ VSub_Number "$3")} spids: [446] ) ] spids: [442] ) terminator: ) (Sentence child: (Assignment keyword: Assign_Local flags: ["'-r'"] pairs: [ (assign_pair lhs: (LhsName name:config_name) op: Equal rhs: {($ VSub_Number "$4")} spids: [455] ) ] spids: [451] ) terminator: ) (Sentence child: (Assignment keyword: Assign_Local flags: ["'-r'"] pairs: [ (assign_pair lhs: (LhsName name:namespace) op: Equal rhs: {($ VSub_Number "$5")} spids: [464] ) ] spids: [460] ) terminator: ) (While cond: [ (Sentence child: (C {(Lit_Other "[")} {(${ VSub_Name tries)} {(-gt)} {(0)} {(Lit_Other "]")}) terminator: ) ] body: (DoGroup children: [ (Sentence child: (AndOr children: [ (Pipeline children: [ (C {(echo)} {(DQ (${ VSub_Name config_string))}) (C {(${ VSub_Name KUBECTL)} {(${ VSub_Name KUBECTL_OPTS)} {(--namespace) (Lit_Other "=") (DQ (${ VSub_Name namespace))} {(apply)} {(-f)} {(-)} ) ] negated: False ) (AndOr children: [ (C {(log)} {(INFO)} { (DQ ("== Successfully started ") (${ VSub_Name config_name) (" in namespace ") (${ VSub_Name namespace) (" at ") (CommandSubPart command_list: (CommandList children:[(C {(date)} {(-Is)})]) left_token: spids: [537 541] ) ) } ) (ControlFlow token: arg_word: {(0)} ) ] op_id: Op_DAmp ) ] op_id: Op_DAmp ) terminator: ) (Sentence child: (C {(let)} {(Lit_VarLike "tries=") (tries-1)}) terminator: ) (C {(log)} {(WRN)} { (DQ ("== Failed to start ") (${ VSub_Name config_name) (" in namespace ") (${ VSub_Name namespace) (" at ") (CommandSubPart command_list: (CommandList children:[(C {(date)} {(-Is)})]) left_token: spids: [575 579] ) (". ") (${ VSub_Name tries) (" tries remaining. ==") ) } ) (Sentence child: (C {(sleep)} {(${ VSub_Name delay)}) terminator: ) ] spids: [484 596] ) ) (Sentence child: (ControlFlow token: arg_word:{(1)}) terminator: ) ] spids: [423] ) spids: [417 422] ) (FuncDef name: reconcile_addons body: (BraceGroup children: [ (C {(log)} {(INFO)} {(DQ ("== Reconciling with deprecated label =="))}) (Pipeline children: [ (C {(${ VSub_Name KUBECTL)} {(${ VSub_Name KUBECTL_OPTS)} {(apply)} {(--namespace) (Lit_Other "=") (${ VSub_Name SYSTEM_NAMESPACE)} {(-f)} {(${ VSub_Name ADDON_PATH)} {(-l)} {(${ VSub_Name CLUSTER_SERVICE_LABEL) (Lit_Other "=") (true) (Lit_Comma ",") (${ VSub_Name ADDON_MANAGER_LABEL) (KW_Bang "!") (Lit_Other "=") (EnsureExists) } {(--prune) (Lit_Other "=") (true)} {(--recursive)} ) (C {(grep)} {(-v)} {(configured)}) ] negated: False ) (C {(log)} {(INFO)} {(DQ ("== Reconciling with addon-manager label =="))}) (Pipeline children: [ (C {(${ VSub_Name KUBECTL)} {(${ VSub_Name KUBECTL_OPTS)} {(apply)} {(--namespace) (Lit_Other "=") (${ VSub_Name SYSTEM_NAMESPACE)} {(-f)} {(${ VSub_Name ADDON_PATH)} {(-l)} {(${ VSub_Name CLUSTER_SERVICE_LABEL) (KW_Bang "!") (Lit_Other "=") (true) (Lit_Comma ",") (${ VSub_Name ADDON_MANAGER_LABEL) (Lit_Other "=") (Reconcile) } {(--prune) (Lit_Other "=") (true)} {(--recursive)} ) (C {(grep)} {(-v)} {(configured)}) ] negated: False ) (C {(log)} {(INFO)} { (DQ ("== Kubernetes addon reconcile completed at ") (CommandSubPart command_list: (CommandList children:[(C {(date)} {(-Is)})]) left_token: spids: [774 778] ) (" ==") ) } ) ] spids: [613] ) spids: [607 612] ) (FuncDef name: ensure_addons body: (BraceGroup children: [ (Pipeline children: [ (SimpleCommand words: [ {(${ VSub_Name KUBECTL)} {(${ VSub_Name KUBECTL_OPTS)} {(create)} {(--namespace) (Lit_Other "=") (${ VSub_Name SYSTEM_NAMESPACE)} {(-f)} {(${ VSub_Name ADDON_PATH)} {(-l)} {(${ VSub_Name ADDON_MANAGER_LABEL) (Lit_Other "=") (EnsureExists)} {(--recursive)} ] redirects: [(Redir op_id:Redir_GreatAnd fd:2 arg_word:{(1)} spids:[836])] ) (C {(grep)} {(-v)} {(AlreadyExists)}) ] negated: False ) (C {(log)} {(INFO)} { (DQ ("== Kubernetes addon ensure completed at ") (CommandSubPart command_list: (CommandList children:[(C {(date)} {(-Is)})]) left_token: spids: [855 859] ) (" ==") ) } ) ] spids: [791] ) spids: [785 790] ) (C {(log)} {(INFO)} { (DQ ("== Kubernetes addon manager started at ") (CommandSubPart command_list: (CommandList children:[(C {(date)} {(-Is)})]) left_token: spids: [881 885] ) (" with ADDON_CHECK_INTERVAL_SEC=") (${ VSub_Name ADDON_CHECK_INTERVAL_SEC) (" ==") ) } ) (Sentence child: (C {(start_addon)} {(/opt/namespace.yaml)} {(100)} {(10)} {(DQ )}) terminator: ) (Assignment keyword: Assign_None pairs: [(assign_pair lhs:(LhsName name:token_found) op:Equal rhs:{(DQ )} spids:[914])] spids: [914] ) (While cond: [ (Sentence child: (C {(Lit_Other "[")} {(-z)} {(DQ (${ VSub_Name token_found))} {(Lit_Other "]")}) terminator: ) ] body: (DoGroup children: [ (C {(sleep)} {(.5)}) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:token_found) op: Equal rhs: { (CommandSubPart command_list: (CommandList children: [ (C {(${ VSub_Name KUBECTL)} {(${ VSub_Name KUBECTL_OPTS)} {(get)} {(--namespace) (Lit_Other "=") (DQ (${ VSub_Name SYSTEM_NAMESPACE))} {(serviceaccount)} {(default)} {(-o)} {(go-template) (Lit_Other "=") (DQ ("{{with index .secrets 0}}{{.name}}{{end}}")) } ) ] ) left_token: spids: [942 972] ) } spids: [941] ) ] spids: [941] ) (If arms: [ (if_arm cond: [ (Sentence child: (DBracket expr: (BoolBinary op_id:BoolBinary_ne left:{($ VSub_QMark "$?")} right:{(0)}) ) terminator: ) ] action: [ (Sentence child: (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:token_found) op: Equal rhs: {(DQ )} spids: [991] ) ] spids: [991] ) terminator: ) (C {(log)} {(WRN)} {(DQ ("== Error getting default service account, retry in 0.5 second =="))} ) ] spids: [-1 988] ) ] spids: [-1 1006] ) ] spids: [933 1008] ) ) (C {(log)} {(INFO)} { (DQ ("== Default service account in the ") (${ VSub_Name SYSTEM_NAMESPACE) (" namespace has token ") (${ VSub_Name token_found) (" ==") ) } ) (ForEach iter_name: obj iter_words: [ { (CommandSubPart command_list: (CommandList children: [ (C {(find)} {(/etc/kubernetes/admission-controls)} {(EscapedLiteralPart token:)} {(-name)} {(EscapedLiteralPart token:) (.yaml)} {(-o)} {(-name)} {(EscapedLiteralPart token:) (.json)} {(EscapedLiteralPart token:)} ) ] ) left_token: spids: [1043 1063] ) } ] do_arg_iter: False body: (DoGroup children: [ (Sentence child: (C {(start_addon)} {(DQ (${ VSub_Name obj))} {(100)} {(10)} {(default)}) terminator: ) (C {(log)} {(INFO)} {(DQ ("++ obj ") (${ VSub_Name obj) (" is created ++"))}) ] spids: [1066 1098] ) spids: [1042 1064] ) (C {(log)} {(INFO)} { (DQ ("== Entering periodical apply loop at ") (CommandSubPart command_list: (CommandList children:[(C {(date)} {(-Is)})]) left_token: spids: [1116 1120] ) (" ==") ) } ) (While cond: [(Sentence child:(C {(true)}) terminator:)] body: (DoGroup children: [ (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:start_sec) op: Equal rhs: { (CommandSubPart command_list: (CommandList children: [(C {(date)} {(Lit_Other "+") (DQ ("%s"))})] ) left_token: spids: [1133 1140] ) } spids: [1132] ) ] spids: [1132] ) (C {(ensure_addons)}) (C {(reconcile_addons)}) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:end_sec) op: Equal rhs: { (CommandSubPart command_list: (CommandList children: [(C {(date)} {(Lit_Other "+") (DQ ("%s"))})] ) left_token: spids: [1150 1157] ) } spids: [1149] ) ] spids: [1149] ) (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:len_sec) op: Equal rhs: { (ArithSubPart anode: (ArithBinary op_id: Arith_Minus left: (ArithWord w:{(${ VSub_Name end_sec)}) right: (ArithWord w:{(${ VSub_Name start_sec)}) ) spids: [1161 1170] ) } spids: [1160] ) ] spids: [1160] ) (If arms: [ (if_arm cond: [ (Sentence child: (DBracket expr: (BoolBinary op_id: BoolBinary_lt left: {(${ VSub_Name len_sec)} right: {(${ VSub_Name ADDON_CHECK_INTERVAL_SEC)} ) ) terminator: ) ] action: [ (Assignment keyword: Assign_None pairs: [ (assign_pair lhs: (LhsName name:sleep_time) op: Equal rhs: { (ArithSubPart anode: (ArithBinary op_id: Arith_Minus left: (ArithWord w:{(${ VSub_Name ADDON_CHECK_INTERVAL_SEC)}) right: (ArithWord w:{(${ VSub_Name len_sec)}) ) spids: [1198 1207] ) } spids: [1197] ) ] spids: [1197] ) (C {(sleep)} {(${ VSub_Name sleep_time)}) ] spids: [-1 1194] ) ] spids: [-1 1217] ) ] spids: [1129 1219] ) ) ] )