From a93639650f469bbfa6e7996b3944afe12a5f4da1 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 16 Sep 2016 09:59:13 +0400 Subject: [PATCH 01/34] Allow calico to configure pool if tree exists, but no pools defined --- roles/network_plugin/calico/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml index 69f91949a..d2cbd9685 100644 --- a/roles/network_plugin/calico/tasks/main.yml +++ b/roles/network_plugin/calico/tasks/main.yml @@ -91,7 +91,7 @@ environment: NO_DEFAULT_POOLS: true run_once: true - when: calico_conf.status == 404 + when: calico_conf.status == 404 or "nodes" not in calico_conf.content - name: Calico | Get calico configuration from etcd uri: From 34a27b012775f51fec12201a1d911242b507f45b Mon Sep 17 00:00:00 2001 From: Anthony Haussmann Date: Wed, 28 Sep 2016 16:15:18 +0200 Subject: [PATCH 02/34] Move kube_version var to defaults Move the variable kube_version to defaults to have the possibility to overwrite it via group_vars inventory if needed. --- roles/download/defaults/main.yml | 2 +- roles/download/tasks/main.yml | 2 -- roles/download/vars/kube_versions.yml | 1 - roles/uploads/defaults/main.yml | 2 +- roles/uploads/tasks/main.yml | 2 -- roles/uploads/vars/kube_versions.yml | 1 - 6 files changed, 2 insertions(+), 8 deletions(-) delete mode 100644 roles/download/vars/kube_versions.yml delete mode 100644 roles/uploads/vars/kube_versions.yml diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 51f0b02fd..f1f276c62 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -5,7 +5,7 @@ local_release_dir: /tmp download_run_once: False # Versions -include_vars: kube_versions.yml +kube_version: v1.3.0 etcd_version: v3.0.6 #TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index 9b9caf126..9e97770cb 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -1,6 +1,4 @@ --- -- include_vars: kube_versions.yml - - name: downloading... debug: msg: "{{ download.url }}" diff --git a/roles/download/vars/kube_versions.yml b/roles/download/vars/kube_versions.yml deleted file mode 100644 index 1ea3eb24f..000000000 --- a/roles/download/vars/kube_versions.yml +++ /dev/null @@ -1 +0,0 @@ -kube_version: v1.3.0 diff --git a/roles/uploads/defaults/main.yml b/roles/uploads/defaults/main.yml index ad6865da2..30c71d76b 100644 --- a/roles/uploads/defaults/main.yml +++ b/roles/uploads/defaults/main.yml @@ -2,7 +2,7 @@ local_release_dir: /tmp # Versions -include_vars: kube_versions.yml +kube_version: v1.3.0 etcd_version: v3.0.6 calico_version: v0.20.0 diff --git a/roles/uploads/tasks/main.yml b/roles/uploads/tasks/main.yml index 68fcd4320..2d6000599 100644 --- a/roles/uploads/tasks/main.yml +++ b/roles/uploads/tasks/main.yml @@ -1,6 +1,4 @@ --- -- include_vars: "kube_versions.yml" - - name: Create dest directories file: path={{local_release_dir}}/{{item.dest|dirname}} state=directory recurse=yes with_items: '{{downloads}}' diff --git a/roles/uploads/vars/kube_versions.yml b/roles/uploads/vars/kube_versions.yml deleted file mode 100644 index 1ea3eb24f..000000000 --- a/roles/uploads/vars/kube_versions.yml +++ /dev/null @@ -1 +0,0 @@ -kube_version: v1.3.0 From d9641771ed48d70ec75b9050932fcd43630a800d Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Mon, 26 Sep 2016 20:14:19 +0300 Subject: [PATCH 03/34] add kube-masters to SSL certificate --- roles/kubernetes/secrets/templates/openssl.conf.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/kubernetes/secrets/templates/openssl.conf.j2 b/roles/kubernetes/secrets/templates/openssl.conf.j2 index fa00163a3..5eab64979 100644 --- a/roles/kubernetes/secrets/templates/openssl.conf.j2 +++ b/roles/kubernetes/secrets/templates/openssl.conf.j2 @@ -11,7 +11,11 @@ DNS.1 = kubernetes DNS.2 = kubernetes.default DNS.3 = kubernetes.default.svc DNS.4 = kubernetes.default.svc.{{ dns_domain }} +{% for host in groups['kube-master'] %} +DNS.{{ 4 + loop.index }} = {{ host }} +{% endfor %} {% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} +{% set idx = groups['kube-master'] | length | int + 4 %} DNS.5 = {{ apiserver_loadbalancer_domain_name }} {% endif %} {% for host in groups['kube-master'] %} From 84052ff0b6f00b1610a7d5540b7c5f5613abb376 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Wed, 28 Sep 2016 14:05:08 +0300 Subject: [PATCH 04/34] use nginx proxy on non-master nodes to proxy apiserver traffic Also adds all masters by hostname and localhost/127.0.0.1 to apiserver SSL certificate. Includes documentation update on how localhost loadbalancer works. --- docs/figures/loadbalancer_localhost.png | Bin 0 -> 58266 bytes docs/ha-mode.md | 56 +++++++----------- inventory/group_vars/all.yml | 5 +- roles/kubernetes/node/defaults/main.yml | 3 + roles/kubernetes/node/tasks/main.yml | 3 + roles/kubernetes/node/tasks/nginx-proxy.yml | 9 +++ .../manifests/nginx-proxy.manifest.j2 | 20 +++++++ roles/kubernetes/node/templates/nginx.conf.j2 | 26 ++++++++ roles/kubernetes/preinstall/defaults/main.yml | 2 + .../kubernetes/preinstall/tasks/set_facts.yml | 8 +-- roles/kubernetes/secrets/files/make-ssl.sh | 7 ++- roles/kubernetes/secrets/tasks/gen_certs.yml | 27 +++++++++ .../secrets/templates/openssl.conf.j2 | 10 ++-- 13 files changed, 129 insertions(+), 47 deletions(-) create mode 100644 docs/figures/loadbalancer_localhost.png create mode 100644 roles/kubernetes/node/tasks/nginx-proxy.yml create mode 100644 roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 create mode 100644 roles/kubernetes/node/templates/nginx.conf.j2 diff --git a/docs/figures/loadbalancer_localhost.png b/docs/figures/loadbalancer_localhost.png new file mode 100644 index 0000000000000000000000000000000000000000..0732d5489a919007e65f58f15ac0e181b612e7ee GIT binary patch literal 58266 zcmeFZWmr{R)HVu;l1hn63cBeM36W0On^d|(x*Mb$1f{#XJEc(&5D<{=?(UF0a~sd| zeDArwpXblH-d`fR)|zX~F-P3@JyyVLDbc&=gy;we2zOynVHpI3Tj2-@Na?6|z;7~L z7BLVI?uo`JZTcq$oKqfSvk9ty5WO|towX)==V9d|btJEnhueC*42lY#D)H5dJu^UwFGZekg zVg>$JNJr`K-EqeFz62&`Gh!VG7kw(>7Q0UWswfhrrhXxyRJyZq;v5j0vwbG7U7)vh zSi*XMarFe^cP)d;;oc$AFm&U8z{7o@SJW}z_v|tTnXrUUDtUOa=90Ph@UkcR?6RnX zZ^d5v=i>t6j)P6ef%gF)b;7V!?N^?$)a%GhpWycokRN_v=(H#w8LXyiBpS9-3i#$L zr|cuuAZ%rgcYGO^Cw)kDOm;B;sL9TV#+4SZu7|VMJbkAVFWa<9`4XxL|aY0 zSfrLn!$H?hZsl2T5B`jYfy2>4i0q{Beey|vmFSFtF=iK!{1eK8XTgUho<{1qc8tR0 z&&bT%s{>BUgapk3)B*=Gi>*du(3BWH=bN5Um=1q@wQtlxyy|c#(tJ4n(9>Wq%dM;Q z8>QKnwea`00C}km_7nRBan|%8XlDq|$~?VsHud#X?sE-M{FVr7e`1|SMc&r5S*Fu| z)7KvZwGA(d_mnem?<+z5JXz5@$uq~U0isN4uNsP5EKEUa1mt0-28`T^ygYot z`&Za;S@=HT;&Eaf!H*jAOiZ|Q1jiZ2v`n53n3;`i-KFQ)b{59U7HV)CDC%cu_BFAwYn z4lfQi#pS-N_1UfXv;V& z_Y3W27&Pib40Df^t#4RU|J^Fi0~x~%J$;H7<-|z^gkj;VagjOJc4xebe$E;pv*x9% zK?d6|X7t$Wn#+3_2jp~V_QC}hbIHl%bIQKTkblX+-z>dXgP>Pi5+wdQ5Ha=pS<3*1 zXO)hg#8!0ndRmdJcF$g$HxcUMEqC5gL9HJsg0Pp-n>jL10&OrH85OHH^3mc}XwOfQ zH>aOtrS@6-*4xrtlC)>T+MUVZ=Qc%`3G9x*q^0r|lamvNlb^CV3F z+KFTT^_5n6eKUui)rCiL49GQcDZFzTZ>Vd=3%QLnE=h#^^ zxhGeb(U12nFP>8zbAA_J?){*>#6fu`efWI2?DCO7_3G&R%q0<4DqX$lg2Ptq5E-&cyW-5$ca6Kj9(U4{u6OO(hJQ zVx(V;LOI@xRv1Klc(!Gg!@x5a@5l)=rz7dN;Xf-uvtr(&eZqHMMDR@O!LtvKuy4JcYfe6I;yoeg zjmz6vPaYh!8MU+OvCOX9D5^Jh+F1XzQAF{8L8bB=6x#yYLdf5nm?&A-{qcncg5+9^CXGG zly*0Lg@iJ7>v3NS->gcd;Tr`*PNs6rN(A2&CGh zkwHv8?HV}}AiKDYS=7wm;W}T-*j`O>w`V)vnV!7f4{;w07iUy{)$;*kTm?T|(`Dbc zsOfy;WHE$Kg_&8CeT$a%vw@nz8=L%rA!D3y7x!|O2C?JYvzuE-_|d>+kHHfNaIoJC zqz++SWW{J0Q>D0At9N!hc5euJ#GIZ?#qz4kE@m!_wSGN2A(2B-5XyM%fk?UR(z>9j z${-lND}Bi;b>1c8`ex&l(#V+_3wXB&6QWNh6U~7DnQ-bD)|1*O!WOn8I~nFl-NgJ+L^s?t}?z z!PKaF)x9S(gge(`kp3VMWdH7)>{$BGpg!j+_oVrc4ccUXsb&<9?U~Z~ORe z8V(=jPhl`M{NS|P7$U>cOlVOfq@~^pIAmae z zZ_?~jRRA6pXT8Iu*^7?aXVcc*3@596H{(~1d0ElrSbUeI!Ls4ihEwN|&6_5^wlm@7^zsMvN?Ixh?tMI`W8F;6 zri8l;s@1lqbgopa#m6_6fs;~EZ+fdd@4h*Q)U?M0S!hO{7v-`lREpdHE#rT4vD@67r;)`64(!S2TUa%xu*rprBSk@n6xWk7 zi{`89FT8908UA?Z^Zpo3vy(078P!ZX4!x%rvRn2<{@q-=4U6Z8$D^7Sm1|WaD&3?` zYvh|_WjgA~HQREC%W6ah-`C@*O-F4+S>1@2p}L0!w|@oldk-RB_+r% zTZ?nPppe+Uo_;PKz$l_G?=$k2hK9U$Bfo%CI|r@qtN)X>lkLe*QeUr5n{TRSFIQgl zMuN!I$3A5#v^s7)SzUzJge!WW+l2uM=>~qZO6h!%c$=nF9uxA!mIZ+35L`;JaOZ>kl#QyeV?$%Q|QQG8>cY&!t6!y*oTx$>xfCEjEs@QqeB z*`tUCD?)MaJ9n)|#eFu@JZoq_hlZa^UMY@A`${ZSGTdw3<1jQVghIqQP_P~?2|HH`3AJ-$>qVjY zTcr8N_E))gx{q~7q`0=1qCZAEO(aH!M!vBA9?N|E9_o_le*JUDLM7O7+HSP=G```s zMxjq|n0Z9148WLDPp;H1y}x?%}+%5gPmGkO>1 z0^XPm@O??Uu?sFy-95CNMCY?Gm1r|kCH8w>Ii%EoD|&yiqn^F< z0~3>8=u%Mls2Hq8xw{xd(4TT0W;!H!uDxG4Mjh5G2c-(SX_+db4ZC0Q#5x{yQf=~n zSPYZ!vWwhR)8_m6&cORl`+PbIlL|>%=7k4oB|c@)?IOR|12S)!(PXBa#kILIo&^q4 zS&f|?iRkF$=hd{=rbISqhl2_$R}5G+e~bO2xiM0faJsZdQl?KPx1M^PZJxjG#!#L) zU&erbg4s&)9-Yz6LZ#8*lrxVN*4Loxn@*~fA;*XA7n8FSc+%YcbHY;H@$A#KgKc$W zQ^^lCUHZ45V#w5(IyxYQ$jY$eOZhBT%8CYj%SeSVXIvGl#MiZ;(?Qs$mSB;i3>>Gj zGaaAn$9`lgcYAodVe*d7R)hH7c3wT^!zwko@Ql7PEv@Vc4kK1mc*FPs^tO&D>BL-` z=ounRrTYG+()N1P3(Y)B?uYe^!J!iAu5X%b+Sb8LE4=VDZF6z6eQ2fS&u)X^?2{& z#TzrK2ZFij&;YPhbj%$@6|b)QOCmG46peR0MIxmU8O10VnB<=MlKGkqyD{^GF-pvH zqn|yMmIm6I;JmO@2rLtJJjoa@$-p zmpf#lLOvGV49jK+_Xnsr>Sic{j?K!@j1sHcw3Q5ngmgr@_gQh|xgx!mYZp>ch;XCD z(9J8l8S86KjmBjPnU=#w9K>PO=e)1qKZQgXY65R5E0ZyH>6lN~%Tjq#n2EqKuU8=8 zI?pegFI0d;Sa%aS*h@Xyz8=A0>T;f5#yl*iwm2m%7DUJEkAR#Xfgo7Ud2zQOA(sEl zU!64i);*5T%B;pi)Sn{dU=b9O;rOZ9X_4RQj<02S@vA7z5+f|oV2I6Zhvu3w{6U3Tm!Eo$0hMy^X+O)muvgsjTs*ccW9 zvDWMwy?l}bL7vOKR(7KSNW`O|gYVC7T3lU&?zUO-ql;W9dumO?*G+g zETX`KXj3(X_x(V14W=t8CpB(?aBX<>PL~eR%&0Rs?~|q4ts6hGjl^}Xa&{%>I!9%; z{F++tsuPJ0W&V1-x~leh+>=DMN-se>zmSdB2xVGvwS6^=IY~k4L^;w6r8GINlAUaw z&U$&w9jl;Xgcr;elP{E_wUOE=84vi4NO-~RRDzlUPC?>DJ<>Po*YBkda8R-iSWg@Y+1nT@$REW2 zpdIUakMne@Bk&^;{UBYtlz5=`azfYR#EO?TD-x>iSI3R#3Oc!aQr{?T!VVP9TOkwH`L`gGQPmI1?kIZ81xfbeGiDcL|}RvGUQuN;~hMbbRM zHLu&=7nG9%&8w~=&eu!*WL^aa?Ko^kRZPJQSvx3JH9n^YG}y|c#p(|WUb)AhXu`0x zZNCTa!Yu=vi2-(D(+_l=iByYdP>d$DfuDQb7zDxR^{(D}1eC9h%jwHfuWY}N(rrg@ z9S_NFRzXEEkPY&Kh+Y)WSdnu+5frEQmXm;PP$Q8XkVwsEP)Tv`f2}K*;=8i_WFQSA zeeUS#mC1h}-9AmvSLN-a#DH~{{{7CuM-?^fMtRzwMUpeLq;pLJWV5|Cp&Yr=_di2t zmdNrhwKQW+x^5$tjbRTTjFRQPeixD?Y!vG}A71$^$1j*LEQym*fM)7b+c_CLI35w8 zAXZpvCCN&?Y-6ljdvY4)#cD+FCp%HiIusH(;56_1ur37M+dv&yJ^OvF7^lHOvhz+J z-IL&4Sb;0Z(kE9m?8S?gXrd3NL5a0hT)(4IhLz0@;$|=pv zQJn$WnjK;>$Nc+9ng;nWALQ=(7QyG#Ow^(RX0{TZURTg1D1w-lVQpdbYpE38M#ipu zZa&=710Q3oRLmw@f zV^n$YFh>IIV9rJSw&Re0Ex`7O)98Jdv5 z@RN+&p6?;lDEEk!LM-5Bj+@A!sRl*RZfx(~<<+jcMId8-1;g?5(5cD|)`@+zvHQTR zXU*)(apkQGSyRi7d6{=J^JN{Rn@uI>qz#9?Tst)(bM-kWXmwcYnDVn1(u9J~dt*Pe zw&ySrnrQwmgB)8Sw}8E)fW76%%^pn;A&W`QAzj>u3JdbXoXsl>D_t*6c5V(_jA-Q2 z9K<7xMO=oz99lH}%DjN?@F^KPwrQ*_=9VsRgN zen-|7{k~-NU8VTB305;Bkf_K9Pt(#2>)-NNFMX>!#PNQ<5E#!oq4+&oW;W&RBGSjGHCs;ezKM{mWsox;ja}c84QQFp;Vn~1 zUn$2eU?eFdJWvlBT(myE*1GxG=RTEEIVHNNhw8+vU z(641_*8S$HS10>Gck+R%sWzvJYIhr<*nw^*2HBfT7E%fo-GnE-><~AX2~}F@yuOTy z%;jwKuJmdXtn}#cF7HgeJb^&jZ0<0J8OI#1Vxi^0o;yUQb6%~`sZCE0i|`J`x4aw` zvKofRpO7OomVKii$|oCc<6uY!Mu{eiZkIG}m3L2BOEb&yb<)u=tHcH}KPcfa`_NS? z$LDW$kLklOM)2a$Il*rsD6 z6?1hD3_8qJlZ!qHp)0RyuPh$T#~`UlyNG(C`PhC;d!)hf5W2`INluU>Z7V6qp0%Tw z)g&wVg$;C|r!z2KU`|fBIWtm9aY4a|gOo)X#OCFxFZc&X3$+3zqJ9dDy?WBzfnf|Gn{VEB&u2{_%wWHO2p$;(ucW688V^!zS$$h(S-h9>9z3 z{Uj6Edzvl}x}mQWWc91$jT(qPobDOtU9;&ryNYBUXZ?O*Q(6HsW^$T- z4*M-7Dl!b7;CU)6C&IKGTt`VEKTIo0BJOvV8VS()zex;!Cg&viC-@6+qgv?jy^%8S zLa3cjcecM$I72VJ&Ec$Eh8LK=3D(oD~rD=Ffo`*(v!cmUV@JHi{XYkmAE@< zly6Lip~5s}mc0K%(S2q@u0&P{rn$mE&YDTFY(dlNBJ=zvD5C+bO^X`J1Iz_G((|D* z)K1J`YY9*|{Iy0dArfxnz$0r78bDKMvpXHF<(D-pU>-jR-}tdJU0)wmejq3&QBGjL zuk96D7*%p_+vnelp%ETlj$*=HTzNHg5tNcGo>7#0@AaI~U;!N(jZU@H;(?#@rHZP^ zIGfaP!kaR`q5P+!@LWL76k2K+h&u{weQrL*VGDhAM-oO(BdxpIo0Rlkj#a_TM~_K& zvC%*D`?qiHn172HG6YXG>`>A-{L&&0QL%}fcC=2ATRf^VP%+^Rn1`^g<+b>ot=>X3 z)Xwkrcl|(R^|wlqV@51`9~nDT@D-A0gmxc4TtJJ2!654WS3Kpq3heMKg0Nfv{9iy@ zy+QlNLW7g1SByLtAjb6bG6)^!J3;bH%LAh@W4UlE;K(~9^k`9-@!xeJq6$-xtwu-J z(jtWP#$g6SKC0aN*-iB$H59N70QRmQs7i}kE4)s{3BNAJF)%hFYz_cyY&aHAVz~dD;P*oOHXnMExVV@ytJjRoi+)zF5CNYR8rqEqFD-)b z6JI@{*5#!bzp+iJOkXn3v@(N_6)UikC~41=GD$STx8=gk6xDn2Y&7P zDjL#0U4~$!3C!OPPMzY>5O%=t=k0gL#yqNN(!74$Zd6?~(eL2RXw;m~5}=XIr|TUY z^3``?`)HB4;Rjz03Cc98_%Lw@QrHw%ygj~Vo}hINeXIgG!DG|{g?`=xPu)HIYHv^&$fq}bB-{N z7a~jE9_9z<=HH0&cJHA5hM(W6 zSi*+#*ig)S67seRjZ|Pw&De&97PnmfT@9A!+u*WN#REV9+K2*nKWl|Fu>W%a8n4g8 zKJU+_tSy9$lT)AmZiRjBroEzD=KgIu>FsJJ{L(H;(nq; z@Mz}MQ|ZjFnHm(7uzt|VX6)0(5s@q7;t-JiAoG;rL(uv+1qI>k#fc)fFmqT-h-)0b zV6c(rmq6MRCelRhG;9!Cc?FFY7QO@SL2&EuiG_%L7KPqHbhN$n4@P_w&u$j~fwTmH zo4Ds;kvMjE|GTK~L80^k?=i9eD^r0+dW3{V8g5P1R!lz3t0?fk|CPD_=GehiidGVA zdiP9$DFgt>oH-Cdl$BvDp&MXX*Y#XUX-SBfagPz>@~P^wG>{v;{p&HRimQJ*yz7=r@^J{9~P zK)5jRk$`0jKrK~vBg#3j3u5x|%(J|PK(w=K1UV}_W|BW%;u(p0*83EKO(mUddN-gg z6?3_NHIooKyrkhE@OX2y&+Tj;rG72_gXsr`>e6TiRd(3yQ%C|bSJaA!$1WECy?8Oa zzpes}g)as;lQV_)w_>z}xPk@i`LHbz;#^P04Nmqu9#2a1T@4wc01XlKajy_GJ2)2I zTI};F3RzB1j`H7Kh!ptjK#Bt#mtJr)mRYAzI)!(n!a!OUn^a~I*_s@3YhNKh11CHS zb^j%Dd@y_wgUZmJzgO! zL9V^ew~E|vu1B)OV*+jO3YI^K&4c(OyO?jiG-a3k>s@4s@YT$^w5=SW#U^}s0_;|W zQ!N@o$`14zPD$x=oMw~NRkb#&6OlBsrU_=%u*N~`@VuwlUWvGSjb*EqKUp#UddTx% zYdq{Edg**(x*9g*Jg@-Y2S8&h!FlNiCP_Y-CsK?`rKxYpU4Y)EhYUN!V>gILV_og` zU*iCCf%iPb=dyU-N1Rc~@KDgYsA|O80GjM_(8+E&$FJ&oJWO&6t+RfouKOOx9D>Po zU6t$g&($|$U%Nhpe)MG+5P3^{OdZ29`i**a)3(C#a=+tgmG}2!w6mG9`M;ZhfQ- z@8dgip)r~*NpvyijWuB55^MrD#Un(?zd?nJ5^>hi$D)bMdUJ2SRdN^h0q>h{sOe~N z5}YY!H~c{+8$0v)=}U_l?)tqJjBPIr#$2T#3Lo8b&5F{R!5GyGO@PYVtJ;t2Z93An z{yfr{S{|NL1b}0+iXL{GcDJK`(UE!w3piP12Ba)@hzYi+MKU!X+Yh!u2xhlh z5RTFGO!l%KqzWq`bg|g_Y0OsS`KJUXLj&9XKJ$=5+qo(_41@evsyyE)_;B(C96yVv zqQMf4De!$R#&f{+gDYPqAJ;DdHB~ic)oW<7{e+SIMqW;x)!xyd^dtah6Uzd(3jA^4 z6yIGE&LPLLlDdujHlIw3rX#eaiaWDJ&b+G91%CqowLds}Pu5+Q$JDk*R|KJM=j%Du z)Ao~WPQBv(_ei#Y9;l=U+#v7tq2P17uTsz8xgO|G-f#f|e}k4@%}<3IQgvjzT{T(_ zxFwE~G>s@$Mh)j}w$uD(-wv4?%vqOpyl)0~n_ir_R3|I^je@W5fo%~U_}NufJ5kc( z8!AV&HU(wSX}r0Fi>9Ovn)vBN&T zU9Syl8iWw;#GFLyK#U0+;Dp1s`_QE2J}eKGj{;4@cz`0PSivT|%d(Lt1+Bp0SV1Mc z?BEB#g}Yx{1^Z*gWfkbg8P;~z=1e2kDs)AsVI-hsj<^$a(vu>~t_uNFY7hlS>L|av zMEbYxU|JgZUjvOABl>2~i&;}qVk;lKMD(GG{#Qh`4HG)fXY>B`He{pf4(jzM6DCFC zF^nIE<>^z94QHENw}BYzJI;C+d)sNdhH^nJLpMN>m3K%?;m7v1^6w?dS4WNK7v=lv z&Rrb}5YC*gMDUrDa)(=WTB#S|fnz?(Ni}hU0mP~qzs0IZzb_lM#*(WFH61BbZnU}e z!EOl@-|NvV*-6;pMtIqm4ZmV#)m0@OCr}vJ>Z>7wct=)(?6QgjTkWQkPO+D-3UvXD z=Tv#SszqZ}f3=vYdBcADNr?MpE6HZ)uiz7Va0qf>d-CYXVDTDQd4HVMZTJp@nBf+3REWQ3lG&#eI@o}wL1hROtXN@@O%z}q)Y zCtK^i&#C02e;?&2Grel#?x4^HP;Z+7&$kCmagFy1e%T?ggp~ot2@v3sT&3z28&Ub_ zPWaI9ftR<~ax~-H4wM4Wd=i1;r6B(y(zF8<8|v-R9v%GTsXWNp@F)>wqJ3iA0S{P^@1h`0dHEZW{1Fe>mYbf&Zsbd$#JhHJB6P zEYAs4ksI8w1L`m`Y=8c-0zf@M6@A(Vfx2v(j>h%(MrO^HZ;il9qwVU+iHbg66W}xr zu2;vSJC=?l3Xi2!&8ni{lwsZ}GCVfPh04MmJdr&`zYVY!9}$b%nfjHRfmF=F;JO+} z<96zIc`1Lg@lp;LHuYxlZk4v0o@|Zxb~4*-$T=jl(Ww7<>}0-aOfm?><$B@?eC|zn z5h`8z8N88<4SRw>Z*^4&=+u#X`GP>=2O*)~Pdqy6w%|Tx@aeXaf)QX<#PCY4&z+7p zmcQx%=6jkh;$mZWu&|mRi|%B`*&>14vD}&Dg}Iu7WKWR#ucNnoN9&{Ck|0zJ;s)$@ zvXWpi0lEa!j8{Q$E#)MDUG!P>zgu>=!P`K03IEXiQ4ir{e)-wt+&o zB9uc6YTTQu_a0d&Z@K?*fy239GrS;=?l#cq?C|M@%q|Q&7vgC?U02%} z&Y3N>3tPwjeP)wJKZ39sDXo)mm~a+l z81p2NKHR|~ZSdJ)gP_WVBQ#q?rEPb=1ku1qo!z94jg7L0_`{dRELtiQ93( z|NEXGINnFIbcf+Eio`%ZpV0+Shha7)M)bzK-VV& z2Prp~IV83}1f*hNsU*%@W%fYfI`kA?roxZ6?5?kj;6Z8Yy$S$w)ZH?&d&U6r|1*cz zHTTeVr|Ulj5wP5Wz*{n03b+|7WzfAp-{5Fn%Ii~~Sxa$h>Z!!^sDKpZ3zB(Bm8yAd zTk^XFGdwDo8_4WAPz+yOXZ-OJ$w|3ceho=Ul3# z(uaVDv;PLR~9$-(N)-s#tkoLL9NR` ztGH7k%mlzNsMgZAupO>c6#B*XAeP%xa7YFl8V|S2x)Ti+(v#0;1>ZULmJ@B8x+)nH zs>mRCI}aa|=h1po6g!pZ6~_xN2Eb}qs2MAeA;iugNJm%D;&d`rs7GDodW>%)7R&t%@cGb4Q~_7 z>+;}dbRqK}V;Je`Nkl)?5uL=q5W?SA^yq07wyRMxQvCqCtkHY6NFq)-> z=5DiRF+hkGQ&HDJ;h9JRRJZn^UaYM_h7Sp_Y(~mTSz*1pINn^Xj&cDBY3C7RR?XWuJbuca=<)RI;uzb;PzR2%xUD)9w98?S-NmWF_t<&v<8y}4N7FhXJZdCx3HKz@{)nnYZNqsZc;V>>1EpCr(^6Eoi?Z9Es3MI3C$^J)*5#1XV&|ZnY1eC#q2l{6s+w%4mik@cQ$!;i7WyB;qLtHO62i*hxf^&Llam#GQcC+#$vLbBPnn zQD=td>I$@XpJb(ZhS8)+UK|5)KK&hGg9kT2&Rq>UlffiS;s&f3I-K!pGgTs1!%%gX z{WdA$8Gz6i7lImT5TH4a9cC=yr8k%pv2%|xl%FBxapKsWtzAtv*##n~w}RzLlYzta z6UGJOz;o6VTA^|pU<}1>rmejp5A=b%No(@8a8XS?qgMK2^-6EjX0KIrjhapn+mwtZ zaFI3wS9p864PvX*yG@sgIsSM1JfK6^29maXyd|ryRYKG=+&z58rsc2_Z6tX3SM0t5 zEpOgqhAX9Y(Ix3ecW#H5Zn*{%7NLrNwW*nH` zC{WY#ov%GTojh32NtpnW{jDxksw?%5u?7IgF-q@gIJW2&HJ(fW_(YDX7+^Gu4dGv` zRmLG-2FvxjUpYf(_{NF3A2C!`lvU#wC<@9yIf>N?R;I6W)Db)Y5Kigr^~tAT?}p02 z*rs_uBF_5b3%csM6Zlk<0XNFWMU96J-x%w+%6kv#HR7qhJ*BOO18*1C7rToOy6pgA z`&qF@m^SkJ2b^EwNM$Eb4Hd?4(;x9;4yBTY>puB-hDefXSy1o9Ed z6TltYT;6vzxAHAiry?6^w-Ar-zW@Nhg1z{k~M z6A7rF;*8IvLZL>0;CX900zS@mZGu$2tALhGSmu8F&j$XUt83=~E z=D${QhXWT60@#jiQwNru%PnZSw|_Fgtvb#30JbUA}j z^fH~=KxNv4t`m5yRneMMA_5K$=6ZxCLFYbYoo3VfvV(oX@aZ6yh%CfMSN3&|6nrn; zkQdN90WCA;h(i&|KCX**N+=bmb?(o1`Gw&Wvzh6B_<7PmQ#WwZX-5dY%L>5L97K!) z@=op-gNEj~D;j&yTK1~9^BP}Kjk!FOUcusZobX!Frt!;atT9Z`_S07Fvs0Bz?DbwuOxiwB&G7b~mYoU5Gt@OIddV}V4f+L;2mu-2X z^SB@F3_WIz!bT+41D;-ot(ib=zJXShOmF;HxXd+FdG@Jt_pPvMIIoq9?DZ zRbE`Fb=NNFWR%7u_yj6;pJq;q1IEe&j_n8_oTyU0PJIPs!s;kCjuNs^QgCy1I)d#) zjdP)XEYj87uP$7mk zN1?9`DV64xnzebc<5!5dKE03^jbT(z{1M}zxhNx$cVyXm>csQ;+uH+tM&-)Jc+&&; zX)=BrSy=>H5htX=$Ja7wYNW2Z_1YcC}r0z!Dt za}O0z@*+d!N-23h{*W<36KteiPxJR_0!#`|&5u?unY=U)si4~l=#vWchO*46a<5)D zO%zcKI>n~F?{a#Vj@sQc8e^;~>$*dy7T?v>FtiD;@*U?N=8%BJ)b+2SQNK}L#6&Fq z1zmc+1>MUqA1~z9Qq&->q9fp>SbHI;dIl-Ht_{AKJ}C`ODl0U77T682vL{EM$b3)z z9<K=>5odabI=)Di8tVXg>|R7mh`=C?xNJ350}3v zXk+|U5Z^T`nr%i{_MK&HQ+9L+3&Uail_l|fu-H|x;j5**=n z@ltXl&oxyIc9mIH1rS5;XndAI{lfzN!k@|9k%3l3n{(kw_ZRzeu|`546@~HJ_T*67 zL7`vOLEMO51-MtsRm;GLkEaarZ*DnN0%_tLH}NTrG-+oFg5W@kyv|EkU-6%8!}e&u zi;5wTikdHWxE!=fN(`R&+yiqcu$b~ciqn47Uju6NZTwK<=9FG4h(M|gvWu^H)>fwp z&##_TF`0K8pj<~rhFJ$-v{xkF=?`3mlbL%#ylEnH+sOIz5T~I1(6P3(Z)L~anR=1d+p9e={Cp(dX!k^Me;Znb--O} zAN%UVUoF5v*I?UPpQUI zJCIG571@n^j!sQWf%I3WOV_ut(L_idXL&;o!Mh`7^=BQAkm1lh9K@PZ_OWO@jyAH$ z?A6`^Q@-%A)g7g~Nj*?6{t37>ChV6QUEoKrn``v#wh)wZ>S7rc-VBceHImsxr4gE{ zh!ys?JDyTWoXQ=+BxayL?)DQ<0&waOGU9B2j^yXmFnlV|j$vB=+te}o>dK}AfkQLs zZ5?X%K-4w>%`QI!_7Cz2z;%DR`s(4>(?^X|F+aG1Z8l>0rvZf z=!JP=(FvkkuwYK8d8t2BQt;FAGlm$)61F;B@tIA_l;=@uEfDKvDf}rD8^#pd5=lF0J=l z4Gn01KEAkhm>&u6Z%J1F3lpiC6jg139=95>DdS+4-LRmfsV;+o+!0U&PAzVe}H_H^us}D7R~hq6nl@{BlIXum=y5-9D*1~M%Y@HSps(Z z&s|@F-uHP?w>|@iw}JNO005at74TO35&ZdSo~zoRdnX zTScz~0EY;kny71B`G&>k$=1D)r>*`Vg!$t-0I=L`>h!-}gkIDeH1vD~43W>m zk&1&F(0p}E@f^#2y2-}{RSa6!P?#tjqUWwoVsCp2Ao`A?2PNp~LN@wl3@2`bpOkUV zt7wA_uhvg|f6)!=)wGjYtFNH?w^;bX2gmBZM;+MnjTu!!VCsH@g#6 zS*0t$M5%(xwa8?&I2)jYK-z;H=K|=D?9)tJI2#_`8zW_spCJcNCqAL{ScZuK+KDDz z72ySgWOL3s(i!Gmg;Gc^rv(+W@m(7-Z-x`P7&m(O5Zg8M&c}XX+rI#%rB~1m%sL|T znlIbFLvi8o(S9&sJXt+qOt(4P+-%l-b%MVRkZi};Ry-jF#JJ~rfpDE%H9$D(Yy`aT z_vV~(G9Be+1V2xrae{w-13uqd4rh>eOj|&qG62|D)ra4*(sUGUT|Nc}z;z*T0L1pf z2>(oEd94E8?Vq1)n*gvR&J_%toP0rfJO=-U2OQ3{0|32w(@%X>y(oFpLCF_($0mI{ zU~1e9bVemy&2ks-6yb@ZsLKsJS}iV96BRHt+IKwtc5c1)OKPtCefAbe&UZU@rpbWs&C)KppCOWY60|2d5dEeutK5ha+9e)n& z_rK|Cb(l~Eg>e==kCPhZ8LBpfu5W2^`$L}!JrEHL-2Yk@ZAV|Y* zSpF_y9w&T?PcayIk)@AY2a9{pqr1Qc3M4vBQ`X%tf599MOW)x%2|SJAHGofNG;aYE z!=}Z~pTxctTr^0+#{xtOvXwLN&re)wBE`w20HB=2{FfAH0UPy6&ZhRlP@e2ded>F{ z#R0i|56qF3u zuHC=kQ~b};1Ne+!F8<1Rp4@XSTuN=@Dbrv*0RTN?Ytr;z&?~a&Hx~z>YHyH!MXM)OxXTTce#`D7r_TgY)M}?GJ=erZSCDa zC@XN2PaH(Gd&htzPZ@_$j*d3osEiRS;c^dkOI;nf@v{z|C*-$PrRSD<-Z&G@&r~Rez<}>6duG z8s)j5H-MyX&N}!egs^yY!FOfa-fGHSnF0RRzJCIq)=&)>^hBWTiw_=rENQ;3Gs@cl zqw{K?KFNDY{$cj5G+Ij$hMJTdQI7LnpE=172>BAQ=#K&HMQPyKvHI&^?r>aJr@KD7 z?~IC~tD1mFq#9t{Tp@5`*=V=2lzV?sSWun;9TszhZjD55F_g`0^cTC){#kr%Lk)C|N96a*){0e^%W^Hh>$ zN&oRa)#z!$dmkA5BJU85sTP|KaPs zx`7chZ)32 zN}+!!s{Wvqj^FZDIyhCV&LObBkxf~Nw3+qD2d=!dcVEv;K}G<@N()isp$_nw3R<^R z=Cq%HAx%jEk{DIWCc_Y(0Eh)k9eT|Sqr9hQdKDO*km>pKfYo|$kSjI>HdKWD_ruAX zS=;qF!=+qbbLAZ>4U@7U5YX#A_HRSE(2$vqSja3pqOBn36w;LT3w=)Y91$Hz5c}?l z4zz!Z4k!a6ZWglT8}eo47wqW`eE#SS);Jk}ed|7#Kbt*l8c^c3(3f+GnNIflV1^Rm z@~G?c!^f$y{=Uq?oV|sCiImZ6{@nIOP4E5VKU~^jtx{Njv^nK*z2CHbT3W#`d(EdB z8fw>h^W@w>DDfWqz9A?mfuX|!by{=!QV1EmFJkerHG7BCe5P&ZsXH5A>;;Oo;?mV3 z2YGa1D)&PIQf7lkX1A)?bH;f?XLEm<_%rSg&{j{KWc=NYP z5dMVlL4OU4MA9Uj`Vz0DA*LHru~ff;Z2R%4P$KAJ+>FMk9J3|7qwJtr*epKA%H@*x z)e(z;220}XFX^f;M3sW2eh`13J$>DcCnzL)(3Dgl?-*gkVe%8n@lO)^v+S8LRZ<(g zW%MK2Ru+S#?VM87PFlUfUipE4`@PO1ek-FJV__JTXkNC9naucO84nSbUC%5~=c9$3 za$-_(%UOqwLj?4{r7XOg=vTz;FlYf}aN_?hjp$&@tlKBO;nqsjLw7p@&Co*4S$nlK zT*~-Y$Gx>$_{i|A40@Yr>T|D7|sJy!43}T2jLn zk=wJ_*==`3KZfnrE1ev?Dj~!FcIr9n))T4sDxcmN$7*k~>D0Q2Jl?|En+e9f53ne^ zSz~TtMf^#k@Q{stuoQDJ(|k7w!&Z7O9`T}3;!sU#5lK4~&kj8>M5+@j)Nq@pjD)%1 z`BF8r@qx)aIATzmy%wgLZhdkapiI@1=W%8d1m75R)Xgs2Jtj5=3B=dhC%dl@AD;0m zcX*B8Q){c)a4gOrq$EfQ`6Z)J?Hwu}05)TkTxI2U+F6U6jRkgB%AU`&vG;=ps1-&& z^wpC2TfYS3GlYfd`m*G2J{5`bA1kP2r6E*Xfc9<0)T4lLfYElKgf2nG;d`wFM6a(w z6T`zs*n7U`X=%eE_tIE(5s!j}jE&f$N-qLfzK&QN+1$)YFzi5zn$73=n+S$L+-2C7 zhUj7roNx_VFjkDutGMx%M$P@;IQ@)6u@mA9w!n;r#z8uaYolJbLucKFsEU^!Fb*;s zAf-vLv6Eeyii8ARCPKw0!P}1{KDSWC7Dl+kh|FFXlXThdk#x3JhJT*tB>D*i&&C|u zx0pe@FKjXuk1=s)uFk|LET=e>K4eQYc7MoL7WPgF|I>h$dUXt|LIX!S6}yTnAz}_Q z1Vx?LQgNl@e9wI$0Q4N>$CZGyOOg0CeZ%>7r0gvG{hrfRXTyh#fHBFPWeY_8f&Z)J zVdlS(GkN3<;`4>fAurFws(S}WDh-x!l=+he^eJ-G`*%YcZ$tVZmWjS;Acd;|u#qF^ zcGZ4c_ z83VGb`gE9m2C)dw4)l+-UPfcS%c1`$=r~G4v^$hH5%>lu!3hBkQ)p0>f!dD!<2@kl z1>3{KlQZuh^tz1+sUvz{Ty>wA_+x$o zC*G(dBg&l(+SnS9>FIgxoRLf=z#hHzc-AUb-m18ej=3Nw9BH>NYEvLjPAJ@`sl!Wk z_?o~*s3yKf$SRy2w~U(dPusA5llNcv4@O6s#t#87s(o_FMt6%nJ^(Q*;>d!?R@$g? z!MVZ#5XIM{*w#f|zY68DvDo_6hNYqEJZ+;J-n2YV62$o?$^#XIwh*4kGBrr|#xB?p z_#DxHuJ~kEw7-$^$k$3$;-x;OUr~kLsUU&hgBLY>vajU_Vqrujutxu9k=aO|ns|@4 zdai1Z*EmgH*lk`0ti(s&2mOvqAMB_bnJ?a~+r2qvbU!eXwD5uFz$FEdkMEGw9rAVJ zPwH;e%F+oCs9N|5goW2r*Qx720~7VsFJ6aMLp`SlDK)w58t+`pDcw!RpWhV5A?d8Z zI@Im~7qO4xu?H>{XdCTkHYmOd0sp%*}WI9lxT{swXyc+NAWY8D;|`6 zvt~~h2xWl!0q*n4)Gi6?$%kj9(L=P@?R)rB#!v%)ZJ|=HLl>`Ua36-&2FYV6@h4W4 z2i7bIWwFh%#9EmdkwRl_vxC=$pC7p}flTggx=tfp;|?Fa5@`bKE~T=aYEl(8CukDG9Lbzg~9VN`CGd zfK;}r8LNi(QDVkH2}}~;X3~8wXEstv@<40ixF2!5JBfQ>2_hpK=4Ko_3?H32Mm(>g z_^K7W8dk2b%s>4B277Otuw!#FAyU}iLavI#gSxu#uu+aZSQ0{;t>Gs>W;3oWEKBgl z9%`{5Z)RH|L!{C;YQ?=tGe6c(BNox4@Y+8_wUc6_i2D7y5oeorez_nWq9;_xPRjkYm}AQ<(#01sUmG?a%2p1uxVgEi2f9EW7KxP1lCt)-AUvyxW+=N1%y)g^EMXbNC6=Hmg5@?5hSFtFFfoTID-WQTM;~78*)` z+V|-w1>c1U$B9&tLxCP?b*|CCo zFVH?2^?p6tsCMts=NHFo0G-#L<84LP5ZM^MaMF8b7M<0)H8sLxN9DDr4VUsK*ci_{ ze{{R#5QmeZA*jG6DxgNX`s+n}a4=(72TFT&(4G?emtNrObGkqLPy=oMx*+SV5;@X! z^->N8)wD;#$TCrd@ zx1xb2KM$X~orv5T8kIGmHXH+y>!i))<6oY$?w}^9^$YZqBmGc!s&mS}Kl{pCJ)o&3 zn*oxI?rx;ZJd~%-c%0v|?TWW@Xi>WVJAaUxj3OcnQI8_+fIB7}6kHU1$6KTiNGmsU z$M=_pK4FPG`T_Q0oW#KXJw@;W6>-^_ewSL=S;SwDp$^ML9`*}fTypF(nkuBh8m!y` zmsHXxCC#!R!&f`mp$S8}&k-JKPv#KguVV--iphZBqeU0^dXw}Xo@I`cgDeyypnULW zESMG_Cv#B%1B4B>|1>TBdsk6W4~_ z)L^JYcnC`IwGO*v8$=%n%%V?XheiGqLd7iGry+!1GU_h)(hrur(!WZF#+LTFYx~Kw zQ6dh04QtAyG)K+Xnh67^jjZdwYdVo6nvD#%E^8Y-zsDnmNmxe*@w5evO<>9+-P1=3 zPx;BWnuUmQo>OY1{GsGa3-4o-4|8`fXJGvB4I}$O6}Y)`0n$$Y(ImuQ4-pe1^3L-N zsC9{?lluer^y6f~9Y!G-l_bAvV#W0Of}FxJpJR^J!>Y#kvLUiN%QK?~@1*XD9~w8t zKq|Ibm|ZFA_1+AI2bB8;BxMFoj?Ec$8dkq{1KpJmLpx&e8n*v5wPx7Q(?E}r9wlf&Y;12c<6|$hPB8F_hY1b^aYO2X{OfaQk{sT1pbeJA`&IFkfj{u?6_JkugJt`Gb-5w}vZD zikfKek5iK{X)yOSA|mf=f#iDExz+FeS3;^rycUI5>cyd&nPgCmr-R%%7XL{vyZ0UL zrpx3fsA62JdmrU7el8?CUd&K1=RiI(ktMe103}1=UFiYM_}@QONuPu+5~D(Fsi7iB zqBwdL%yh>IX^qXHs}g#rh5JYjR<;{VDwiR^`NYW5gz3a+v;9DQqt3T^5GmyDSAFjz zFj_vsTQHJ3T>9v%;Zk3Y@(s%`o|kN{ax+CIl%w^z17R3~FROr{@T>lmXTPdi8VDOH zOG(Gpfu7-GZ@$`j=wR*Mk1kB7yoUJw!eGdhTsP8mS^E^gEvle}BCV2g^eV0Ou? zvn_BakmMdjf(HoB@p8bZb|1cunmI^WSA&bP9`Ur@yLMVIoJksL_+J>UUYHz9vUyDq z=IP+J0y^e1V2)Rtrqg+Yt#6iYDCX24w(hc?vifBXNQc24lp2=4yw?XiPVVZ?-{&I1 z5Bsw#GNh1Q0Ur{qnI??qhjFQ&J-P!ri^;yrw-q1hDzAZLS~;Nn*OWPym5F)u{>1~d zya=cH^o_j2(7YYz-l)$U$AOWzG}6lnDe zp-dXT`B=kMd|>b&A#4a|>);_ov;1n`{rh_-wftBxjN^2g27b|Lg#B!Sx!}j4NKXEH zz#>@^;Ra5#qrBL$LU(zZ@BZ$^ECcx?1dWr;Yy%L7|56j&B%bhnAL$K6z~0~i(CqgX zexis`dwiqZ+Q zeO(gepN$Th8;a#n_g;5+1ikb(FJS@b5-G6_?RKLXP1u}_B$Ok4t29$A0nAWd7HyE7 zLi0McYTavhe#VJ>a58&2AYVf;l3PZEbr1p1_t#u876k>#f-A3XAr-biSG5x6&yDib z(jCG0G`^NaK@iS}y*;_70+n_Bu=@G8p(iLZxyV0pWU_H?lg;BDl=l8!|J%;sD2MC( z{qQl!4=yk`nuQD@G8LZ1fBf*M{#eRHv#X3|>lSh5OyMgR#m6T=3CwbzgA-8#w|}h5 zwE9AVKF=NERvvfl)AUm{4eak;|19}2&{6I<#aPV1!t>^FmX`42KlX49wRAse#DS`w zaEF%i>K%^PC@u!PH`2v@v&nIp12DVxB^bu-83IZ>B=JhF!MOcpoj!NnmD@9Q(aW1B zk0K*z`A6}Q;SLyQnc(%<1;JoZtK;&i8&c&BnmblEJXxyV{?P)6$Xm2BqdNDQxZ0&- z4zZkkOUHS{8G?VxOfp5l?G7o81RFfPi%dHNC?+I;=8m>f%J8#Hu#7H2Tfikc5Ig%ck{k8BdVxdi>{%KBw!~}5)jDkCiJ60k{@P`YBUPg+7kqiO#lAMR_J>P?{>29i} zW3*$Q(uJQOW?*y`k>S5qVvOTZp@BAk`z&nL7K4niBGb|UGEZUL<)xyI5srP9xC3&9 ziGAP)Q9a-YntpH*v8_E3^gk^g>?lXvUP0`ejarpp zc9Toy$Gu@H>{|=2nRv^O*b8k;9RH6|gdNogs34ZXiSexNRE}zc`z+7Ut)`euQ51l~ zhcAH9M8sjTb=16?cpGuM??l~i+ca3%Y*q8==3pVmkux9C2Of)RdQlO}UWP%Rr6_8i z!7QdTui+G$G0lTj=7KV9pErrd-0i;0^LvwlDkq27IAEoE zsMf_S^}Jiw3iPnz3lsb)*~!1ktEC9Ei#)%oIW55B1yjQUuV9Rmk7VCYXb-Q6oYSln zdPw(rzu~Ww3pQ@i%}_u)UjO!-aT(A_4Tvjhx9Njg`7%1e;^UQ%i)GLuSmJ3r4Kz`2 zw05NWvB{aPf*-&k555VnLKH%KLaOgnJ!?Y{*2a(`NDORL*WXN^s6KORDT`fha=lO@ zzE3o#^f{}6f3pB5%FBQpPf^);c6)HkCA$xf^FIzKmch-?B$WZoQfDk=0Z z8}WP-+Ua>eus=H|j&pbTlty7rNl+iB!8QoE(k*@Z1n88qwOT-Z{bU-0TnypwCqwFO zEBrOJ5}48S2CgQ#Wj|LV0g8bJr5K1z-yT0_t%9;(lko~{^j?3S;%E^vV-fK?%GC4C0EDlTR5#T1BN0PViLLsM>z^t6qW< zal2+{R}*Yp3fHjCL4G|6ykipnRaA-D+wflQ694z98j}P9_ryS#?F6F~gOeEUTGOnD zFtM)|ZYxsM-5CA;HuX&=O#VkV=BNB4=kWpyPc+tUO@U$PS>omgKMz8XykX&!UmLFz zN5O0$mmXH|YvUlC1qFUe|4EOov&iI0bN(|Jv7nI?CeEeeC|4V?jL);Uu(4QKtTViw zA#8A2Ai0pQ0&|GR|Ix4(9*L5ZVdPFIla_3U01>kRyA4tjXU=l|SM%!Y4&eZSO^Tn~U!{ys5&hzI}FBef* znuB5i$5Ii_c_2PA#J4_*kb4YQKS_T=$x2CkB37s3p>H3E&%|LfH)yWrbdl7|ZmmXI zs+mR1pT3vKhPpAYz`cr);n0K`Jbcq{eQvN)N}Vz!k7S!=&X2^gJi+Kj0>R!&{Q?TR z1XF>VCdlA~N;ehZb!ItMrZRJ;qdANlr_-nvb)H>X|A4yT$E_dD@|auh^QjKqlQdbJ z55sEY9sC}9z!!$G-wHyxar!(A%@S1&(JT|?55@U-gx6qVVv1q!<-|+Q_C3^*cx+%a zmh$1HO#>ndBbxp4hT#Kef-Axxej2}669sjc*V$-`Z|yfk2R|$y6+QI#Fz>@RV9B)j zgj^C}1k#|1qkLFKJdw1J73a6{5!dt-UbB@jM`sD3({7bD=%%y^KXdRrxb7y$BBRME zaDHR94?LKDW20YOiyv!deikCU@qz{jNTsl~bF*0z!Q}9GZ+8E;pu08mHz<&|G2#*4s0YcvV~pv5(bYO|F*6{f~p1$LDh9N zk4aUb(H5jlSy(ReT3#ufvph;Xf-)vzkWwc+G@A6Xfx|*CHXozywYbkIW$ys@;Fa1r zX7Wx7{Eacl@=M@f-|94T1aVmnu=vQ(e_zop>>Wyst+s&40y(dZPS5N7473iD`Z~;s zt0-Xd4{5sL-#*x$*}$|f`bSxt!i@t*q6Y7KGVS5*-|?$L$s8mkRY-A5k_1rdLXq`c zA0Np}y!}M_uHQ}5nnm%F%7rzJHn5^E`4gvqmR%|!QH*CSK06d3L4yJ46Lemkcx{5D z3g<$hOn;JPD8if_hLL7DSj?17-PFE4OBqP&k1C-cXu0j*qHtMjKgY2E*4+crg)*Af zw&@iFmY;I4aBw@ViUZ>S4gX0^0BXd7RfQ)9B!7PQK*Z@3v0Prco?!xE37JIrn$Oh7 zvQC)X&v`gzRv-Y7yrM&?U?KUGkQXyumEqD|7)?P30FYmjvfW=i@xuM0!sMY;#gt=z z{4S99R7o&o8sLN|FxoG+^AB6z3=Ww`s^k@eLyWV`d2s|*^~xGLxh#*Ak`k5%RdQPg zwUk;zErTpUP2;U_L%?CE>l$EB|LkW6;*|6&2x|O^C=-7B;UF1532|fqPm7tBYy#5u zdqFM6Oh;z}2cEe0jYd&M8Qx5hJ)=IDcaB3xU1W!6*C?qhjfNKcj3XvLzZ4o-wTPZ7 z?89J#`(NXxUpYG{C~6I38yBP^BYv}8Nmi+jTSIeXoFwo9zA6hU=LZ`*8^afjV?_?SJZB`$?%aTQ%_o zrdHqoqKr;iap+yOdd1M}F75Dzf{>PMq6o5G%X^j^WHY3H-&gQ5g)cHlZ>*O!3xcN# zsGlzc$PZ=8^)ZnavmQv0&DNFAl74qMl${F8uV;3uz5o)TIv#j?rPp7We;Rv zbX6tq#YZh%Yq2)sVB``T`AB*zXiId=1J9R1(P{tk5QzfG7Bbc>Po>Y1{P=Y!MqA7C z#~o39MRg+Z);s@}fWV4>SB2}pjJFS8Rd)LBx$bhLP(oV1jWBy;5Uh9q8Ybw3ht4AW9lPt$@L@ zVHBC^1TG!FQk`J1{|Zc`elcVqBRO@sU4GQ^08!2#2M^fJDCKF06SW9gEFy>UroNSD z)tS()tGPd?z4O?u)=!^IV$k0<>Bcs&R257EbC|}<0<3(2a%p5&gq8# zWXXIWj-qE{qKJ;CqtanT9drx5Cpmvxw){su|42R#65NOXnOGbyX%e#OKC$!e&S~Sj z~;x2SrQTFi@#eB-mOlgy}uNP*4)MXyeW^S;P!N%4W<_OhW{w8cObG7RKrf9{zZ zqRix}ASV?P<-@h0le&tJbVi&;$}fM-yRh4y06j(IdZc%>FXGAb&p;16ELiI#ZWP7G zNAB-|3wjiWVBp+gs;ORLYbLfiSV{+UXS#C2mr;NJwG;zMSDHvaXyD&QJz4@cDv+(T zzKA@Y_%gRIHx2Gvn=8I(M~3t7y(o@>0YoH?zWGg@C8usEs$h1kOe`rv9o1@Nx& zFj2dK$wuIYO`q&1||uxNFn4FBrL30+FFL zf?{w5Oi0R+QUJ7g8;A67l7MCfj4Li7mKlG7a`0X95!(lmD#|7SU_>u-|o z>W9;je+z?u-uo*(|N0*I=b%`>&xKtAIAaA&mwSl&o4`#$KORN$Hssd01poa;{}o5j z_sM96?u8A~lR$b7aM#g!htZ0hboUOf8rXLO;>Y({f`FD^EHIc2_m{X5P`7$@#BtR; zCleb%1dzyOSdsLC9de#*(o6>d5gW`CI_^c!-#{2~uFWD%uYu`knR5xg3G?P0qmYnV4R-Eyt z*-#!aEGVsxQ*jR|Wr*dtQOqbg^tUHNKXzYlWq5FGW_%F&Ao)DtiRq?SnMcN>d)03o zTs~~2J=!!Bp5kP@{{5lne2;XE^CT%haW*%saN)ZTbGS=|PH6`7j_j*jYiqzpDh(Fu z_R37r0daN93X`Dbv#lt5kf&w>o zc%Pnhvo%N!QiJ<`k8J$fEm_h~0pelVRq5+}KWC*~xNZuH4u@{_DL99{btj!ouanyx`a zj=$0xk$g2)HANr>dIGKMfq6H&S%NU02^Y$VaB-+2We-g|$Y?t9FFoi|nt_2ctBE?d z3h-WNlV-ApUw`)Et4N$LU@|`v3oXe}0)G}fd^29C|7WJ@s}g+F=c#N&;<|qatqaLIoU0H0TBGE!Wd+$?!je+Ki3{_tm&E2ZLo; z%1ViH!l@W=|5OH9n-x@T6<{jHW!hACFd($u8Tm!AwdMbq<1!P7dM|UbDJKYQv3;!n zzySBr>CJ%BB<`F|cQXn8CGeEF%WX>kx5$D5#gl}ZOH)?2#E>oQAZ@}agNinoL7#!n zRVwBKVc^NRPq8c@55ud%$M- zoBOaHWL}4?aZ-mAQ#}E4FI~vWAqH`MkkT;G@)=`bIwQp+7Dlg?ozEE`2)2X+$9|lK z76(B|HP^$n&zq*PKOSi2D7gck>-_fleU8V6Zhp^sjRO6LSssU9^6ODbp&R;~jvpktCsY&*W8XAwOy1MoGG8F7!_s{R- z{Gdh227bhA&kmkG->S0gdKRAbPI~wP?wYK9SIK8Tp{pe$6GdwuUT*y>`TZd8O? zOvN8FR_(B#3_k&aKkwTI{QrEXVkYnaScw4d1u%qY=P|f1Lx@~>m7{TACQGK${AozH zz=3z%#r2izceFU0n(}3_}pvkRRpqYc5ii*G;Z$+>w61haGqj??O%s!-~jcgb;+TDw2Ll+y~Z- zX-2vukU-KwRehYjM-A$}t}VrG<=iYSaaV`X>;q|NTHIrZp~ic~T2K^&D9Yn_jdXL> z(z&fV;$k3a%kY^r5Qr0JCqX)CZOi`~;^FUoD89j6I5_|qE9?ikN1t4OqFrii;5qJK zE34=8h1fw0!4B0D&AT|cmx+8oKCArQGxcoYM;M)J>Q|>883EPK3dDYbQam>jJ+7D& z)_(e5A{dh^9X=9K5jX{eIj5k*P#jF_x4|&*bwRY1^`Z0iJarTD2N*DCS$mMz(LQ6=SA8<`l|qikB!AB*gjmw|197#Wvmp( z$qX9uxOc60#JcRe+(0mx-7g-pr{?(cMsD#3!)O5-4p0ubNzJKkZ-TIK76??$C)WaF zeX#h!*$75q#NFSYn1g{-G=%ui0_T=Yj8(+4jU0)k5J*PCwddX4A&kZ5X!hh9L_8W2 zRT?a5uAA}!h>fBnB1wO36oI3+r8*zDT{Y7YuME74m2)*5LA#;iWH7`Ko_nKb>=tI+ z;K?;+-SfB;mepkLOjj5VmI<09cqeZk2nK+!>f42!#@fDbty{qW4L@Ry=%O6J#}0|+ z;rf|BHl%yM)0=`5OeBj<8g)sjIhi1^3D&c{P7|?6aRKKV?Luuu!)kl0;T7ALvycP$ ze3<5Na)(qi89uT@z*9m2xu-J)c<|vF2Q`G3IzA*cq%YIYx->^j4cX9vF;oM-=* z*bJr5X(Y-5w2KVJ-9;pAJ6V%%~ih#3RMP)H<&{mEu8D=^;;pE?V8R%p`}h6v+Mo59E4N$LiWjqj&61f3_m)Hr&?v<9OKt@3fdh0)>|hy{t4_ zvyYkti~DcnD;~_61w*7n*er_BBs7+%^jHB4x2hW~PJdq)v_)?a2$Xk&mr;&!y?d?? z3@OIJm8U7@;;=@sST<#PH^tA45N3W}Rg1<6#KtO8dMpoUEIIfR!0@aS0+JOBGR>7h zN=&9vQazX@_n_Xo4I^#p`<9TFs&xYmE6aLc{v^&;AZf)GlqXEj(G z61cQp2rUV`dI^&;Z0Z@}xv${7CU3B&9F@I8v@7B{k3r<%HIuP7)4)u6;VR>0X52F# zvPaaiE;o3u*w-Z4OSXfNSSQ&3@*+6JHlrP2*SNgOxTNo=%tN4@tb%1%vzw@UNb>E0 zT4UfM^u9 zVD~z)%p{isv=j;anQKG}b`u{{2Ujpr7LE9cF z#3x7gJyaK$*Ce1#rI>Kq%vqP+tEeD72Z8{WM~}}Bcy-mvpWoM`NBaulvx|-EHDT(L zlSj9>e7&Y>)*SZS0kH%9xM7++vANWwF7xJUD(_&C-jSLZsEtIf|7bD>z|PJT{bL6#m7GRvj3_fUg>J#T47HixOIdR^V$@k;4u!Efd zWwQYaUaAd0o{a#AMb{6dxT{ni7Uay64jr&6hhaW;8Fz>u2wX}AZmQ=8@J_Nw0S6*r zfwEVr!s;6hu%XBL6zJU;;LNUH4UxnBEOeaI#5t_L&{>F?Yl}r?@_`Nx&G|@FhQKP; zo_sFP&khQOx#bxMvJ_Q~NTvjnc>tfJQyWb1fd|r+693s-s`&ow>kvHn!4z)VX%%`z zbfB7Sw<}<~&so-6b-Ttw84|hC)C3NHy$`>X!}aYm!gdVvjqh+|o<0WKFO#FEO;3Sf z;oxIL*Ur_}ezb}Eg``^OW!RfRCgsi(2Brm(tpBH@OE|z z1aYZx%YGY_mskNjqn91P4_l3slkP=U(3`i8*X+MR(1GYU z;Bpk?;@_8J;;Xc5u$Y{|IR>~t`de^N?)Q<$fsMfX3yn-dMoMN9N0g+Ws=@f66A}m; zT+e&0`Q4Tz3rgbS6;a~9kYbt*AIdrb8)ls(k^A$0s*@f5z`kXQ0Fpa5?gUd7^jAw@ zW?}Ax4bUX%=0ZaHfT*j8F^jlw^A=n12-oIXw9)fVpEQehm_WG;VKqtsMwoD;A2%|2&##Fqm?S zt){&N1NQq{P1$x>Ms3*staVMA#Bpp z*rpK8KxDiulsz=0EBFq~N6JLXz&QI!rYb;zbFd3^_%)DP3qg=@^LifX1!wys5ELX% z*!@EypJmR(^A?zuhsqcQ)6Hk=ZQPyLz|9y9Gc8si48EKQZ^~%3Xc&eje#xW@&S zU?4mhqFlQQLG4gh zBeOF|j=g!g?|Y0tn6JchKOKf))@zsE8h|!wK&B&*7ml!Dkb=%X8!08IRDKFBRe$){ zsp4ZIhdKkew@P7#mBE;R|XrW-ZRr5}SAO~$EXA%Bq$UI~zqrl{IX=Da@E)DU18~crZk(!#r z=fF56q-INfT$s8KzlPElrlz%vbXB2eb=$lVN_{NIk_~iAjp@A0U@%G1vw95xZ|xuv z{T)8Yq@G)SqPWm7OP=ZeI`&ed0V#b3)PjuI*@7y;#e#It(*7rA?J;L^@nNSZIYOxod}F) z_0E}p&UrWL3}|i)ntUIsqzducX6ABmX=dL_JMYHQHIGoaaL+97{sUy5);8tbkp59O z@NuIA)I-t^8G&C7Mw|R9+n?r3+XFX)f|(8sc&{{{mFT@y48T7@U2+T6mHYGihYRcy zZQyc9b=732ug1w}w8}2lV)-Q|8$ux+Gw$n!njQ!vgCAg zHJCtc2dB!LwI=XB7w|bQv*5l@d;(7|7m)X^L9em{99(D8eWqf708#q+V|NxNqdcfN zO+*IsOTZuMV5RbF#l@qO@Y)ZU4et|2AONE8j#}LR0;K{9sJ6ctk~}Er4sjQdJ!7>l zr(tI&0QXCO)*B?yonR=(0-)4KTU`=t7L#F2mEzI;9EduaS+XPx&^G)u`{vq?s!}2! zFFezM_GA+Q^DE#hmVKrxP<|uy*O^aB~JhhVrpd9? zxmyz84AeB$AIDwxQxvXR+ecRNN#$SB0lU3aehrylwUz^7Hu!QGM-|wt9gBYe3=SPl zvA~}zLXZ$1JWr1NCEFQrHb?lvyT{J59J)m`-8)}NFV0ytP-_U zK#n!$*%Y|@spV!+YTtFv7^u?Qz+1)=;QZyF8kaYGDPn7|i%9}@fg8j?-tUAvSFY@j zYm`N~KCR>R$$VSKHkIB7{0e(fk zJ1XU$vqlgMG83iG?^;zq(Q_m~nhYizN703`)Z3Fz5Vk8#8Bx$BIQN6+EBZIZs-^BW| zeE7qwR1T=qV_;85?b2(*2_tc>fW2-vYep9_28=l6bsN&xfjP4N_ByEDLM<)|d{M=i&`iG<<$K!Tp@INM8)qGu%ope)6!=lL3L zih}!#yb!nj6XS^>+Ej$zflIgS&4fdA{B7{R*x&>n33vnnt`9_Sm96pb#G!`&utjj^ zpZ6t#k9ZLx$%esiCZ_!H0_hSs?ICU%Bmoi+rNlG#zwaYsMwu+0@4`K>L8-*^-o}&f z3WGyr?vQrEIw?KBdHA_0Bg}4{knkUHe4leZ8ltYB|F;tle zjSC<9#H&uUGJV6VdFwdG!I?F)$Ojq$LYFl5%iS2HKBeDtODQh_B6y0qm_Gx5RWa~6 zNYM^-qSeUK*h}}>j&|tFrj4uM+V*+gI&_{YrzHUfrr%0i;fu5IWC!_wa z$+cgqb|EMLx_rb!EUop1ozL*-)M&L%>))~N@s>=q4U8kXg`1f zoEA>k=>NsOlD@!?o$G&XI8#u76AMqv?OwzE=`aavmMs%Tiewbu-2k{FDY6Z@yksN* z(f)#<_{TR3S(SWh38;tWALdx$J^|Eq{b4gu;N)l*Xs$td+iI}}qrHkijO4&3z5H=u z3;l|13vm65yR5=W0JJRE&buLU{{M>B6ROP%GHWoC9fway+6JEg@7)G$OMrS5w&8rA z04;z{I9DjM$5*4~J_^4l$6i}Mzl&3Ve?sBq9c%}}QlY}Kods^)yz}x2&|z!fiIj4$ zPHz7v&&2^nNWI?b+pkK}WzY?S9wRti*=ZhpZtBx|X;~v_L~{yEroOSBMYc{2Dt3sz{{yV+oh19=Zr94MM==6n!lBD) z335Sq;$E*v=|^J4m-PfUaDF(LQ{Y2nh~3@UREFh#E~;NHT4H5wM1Xi>i@bC=JzR*C4q~m;v3=mk0IhB zpiSDr7ik9S$vA+lZ9rjIfpk5JJ!wQ}5L$NnNFIeYvqsXcBH_H7r$V6l37NhFYPolC zj>js8D>S1zVN?hObp-pjF31p80Df~VxLNMdo_4+_R-dnNLh@=|9OosLt}etkOV+wX zr$`s#5#yt;0QuBb6HYr_L~r-4c+&p`?9EOD+UUZ}J}E(DWNrcm%jK`yAvdAT-aPJ% zferk;cT#EjEH0IyS!u{=DVe7A+CjW*4Djq|jntQrNHY)B`1 zrYl*?ejuOO-x`Du&)k1}_gfgREn6O|=BbleXz4C!(R=WlH}9+DF0g`~*9RBZ`N0#OOlPrJ z8Y*GuUxF+2-Z7WymZVTT2V8765R`Oo0)%zeT6RL4L)*v>3+rRA3I zW}gS<^po!u6QEhV{IOtuAX7#yTK>3~FIf|wOkc3t(|=TSEo{?Y#j%js&lV^NY+6n1 z1q4t#%)qP*Nr19kcB%YhCc!#ClPH4-kY;vlNBfGwb-&``uOUW zB_tQDu*+2WCeoz}A&opKmLUE145a^X5dsg~XUCA^)qaRMWmHovmP8aIDiJ~r7<%bgV$n{a z%=Ax>)k6194%*WweI71NV-}gF1&_{otybdJr?Oehw5Z+qYac?TIL;UBLcB8|M|4ig`c&|#0g zKJG2rz|pYo!zW{Q{r=63>rz(e2{G?e8FCFWB0#Kh=@e==ZSq@paYAoD4xsA!Y>p%f zCBSmNxL9rE#z~5l7=~BK1GZ&x>l{4Vgsw3n)!3h#3vc@;ymBETs`mlvs zD=i>5RRDeG>viSI8DYhv!P+!f;87DHQguR1mz6{P!%33gI*CUUi!=bZbqbRzBYHXE zBBe7S53lWRUw-9q4ISCG4fWH@#*&QWfh910IB_`Ewx0<9EG?FsJ`WsX2`ndQUiP@i zZ;Vs!4z;Mt?Nq@jLb?>mcd2nt%lWV%$$+KO3%vKm@b-UQsH~NIUSr$)?3Qw)(iLzB z1l|!wof2Uqy=>G8#wSFA0`|i7YmHr*CsRpURM3TOi(q(3feF?GM)w^03rBgEv&YXX zV-weZ?wIwzgvY)8LlyUg2gQ=)m#4w~SAOzGCa)BRXx_6c;fn(DUxw%~J*qSDbaePz zFtWtL&ToSoD2{!d0$6e?o?C~M!Ow&L!iDjpUd#n=ysMz4#K{DKn;Y&%6u7~!yHjB? zYaVJ@jP_u}`F)Kf^B#?)9neoqu`Mw4d};%&F^UVdoAbRF;ZSox*&J=-49U+5z*1IG z^nEL^PRLT27Wjy2ttZIhMk#fOP7Gi-1$jw$8h8kh~;9=3++RHXp}Kt1={HSJp`gY!ug zO~MAO+<8UkwJrU0q=+WMFn1@LXVq{vTQ6HCtdfhD8+s3Dz)@*`ya0Ivg#b+e=4VaQ zk)APJx@`sY^yew?+89S*hi0j9z2%!b2QAc_y_pO4mqK%Fj;X9vrZA~$z1(@3g6(G% z%xhM@Q^Hv4yu!qPQaez2UUo?r@sLM4 zP0&!N6`TGxqGSXEKMIHYi*||x@r%erPZRViu6_gcULy4{4jWN-J4o`w(mTxD)#39T z4JKdQUvJUND%2JreYD~tZa%@mHh5p}zySPwbgJ*xQ{C~<>{8jqwaN%|2{Rl=3 zX4$Qwq|1!JSl+~04d3*jQkBA615Ty438(GeJbo0%XJV**>}CSANRy<}pq7jdw}ala zyp%nLA!40G|0tH8v<{F5dwGfZ z+NgGwKxKc#_Y}@Ll!pfq4Gl=88hQ#|YJHFZYw(Y?_YGb|v@Ax(~J$0|lQt zkY57TNA*i7!`s|=$}*b$SLeC2`P7#>vfmt(>HEkU^y9CR<~l`BNeQCOI;hl8o^1vF zGKAko_>O1h^Yv(xM<7|U6GtO1uOdDLAGoj!?4*&yTVgTU)6d|Ib44k16G$gmDL z^dLT;>Xts5NSDuu)!h|AAxak9v#gTc+`%1IR>hx{kKhTDM;dGBRbTbv>0 z)16I7!V7pd4(wtg<}lfH;Zm|P{N;=(gKe$~tfdK`$eF${3`H)&CBa- z6+%Hc2Ew7TZc1Gg>4XWPZ<13H5>vNKcqm8|C-+Oha85D^(Mh(Hw=MwalE=|d1~HPT zWQZFGuz@_2@YT78KTaCL06i;ovtpno2!HegC{A=3Da}lC@cwHpbBmG+C=|l{Z@;=| z>&dRCRo?#1mQ+z9yzl@-xQTuaHAuE_xRm!bFGqn9KViPkRC~>wMQK|p6d&% z1H%o*>@hHh%VW9;w44)5-Pgsz&LUg!@iOXz?aej1fzuZULZc5hLR9d2_25C*7Z*~3 z?^06xeHKa&L@vscpT6LmCb_3p3DP5;d6lXMF`xQ0?=*tX zz|O6a61~}AZI#iCjf|ofj0oZTn-Tq_HEMRI*OHx zpgy}GxnV>otNLxeTp3*$G!QsvxHwee@Wa~b#rtzHtAr%;i5%Pv!^H*`rmX6`H}?N} zxzktixk+hx4K2O<>_o{W@blOT&S47K1P$I!l{*d2Pg=c+ z+k0-+jb)VX(_)%dz(+D#_!q&`@$!BaR(;VAf+Y7&N{|t89OsnXNcbNuz$F=^#kMV` z%`>m|0363HQY*fBhE&n7i%hkOG1$qsn$ZRGA8O`e zyW)8gZOhZ^a{iR~j1gia@D-(|j|>ZSOSt2C^p5VPQmQo z?U7!oMRSRyj$1bm{(+y}%$W;fo)j~H`N*z0ml;+oNq(E&&js=$L_g9s0*jFpRsgVA zERRYbdj;p3Za;xXGj*nwcouuPx4f}!6?wdU{( zz8-3V6N*7wZ_%~p)O#=UTapy?$#bNF@!3`NY9aed+>$ptc&+?SQWDp@nypT2Mb!5=!!x$e|OC`I`fMix~en{H8iB2GEK7x+YxFa!H~wsb4vz^*20@7*EP`4!`zbsrRZVb@j_HSzH2vod05Hh zc%zgSxho29E}V5Tmdd7y zLSscScMRww52^E>EZGAqdtakkkZybgua?66+W*(ycSS{&t$zw4sYQ~AAV@|sN>q@X zii~853MffHq68686v-GlC_zAh`3~Cm-v7+>eVWJVwYndA zQB~)hefHkp{-v+6%lgA#hl_&+Ym81)G{WToR6h;UD)L2#%*#I0AYms~6(kE(-2KeU z;sa|H?#32Wroo?J2Src!WLYABJt9sVID=_ZF8tZLW&837@J!a;qg$}&1;V}J|QDEu7kpaolvT>7Y+9r3Cn zL{TeynegNlD>2Ah;H>yomjqvc6{bpT?fKt+q00(W;eyX&ATGC?uP!(-*pSOJ$&&dP z%I>UcPir>YH{xwI6k*~=*ifz_yptJdCkP*tV&Mf2SI+3ODMco_qQaYhy#C`sU?S>( zLIhbHUP=#^=&6AAmJMM26N_KrkjKTtOe<({>z=!EpYUsDTon9VA!whwvt$I}r;-r; ztQ$y(2_EJ@FPyBjGq#ozrcBUYrnWjSQb}oo+D~!v(A0k$xG#T(H3Veo(tH z=edhQcXO&>2h$xb-vadh)Cq}4254wG6Kx4NguctQNnXr9Ig z^pvv8Bm3DVJ|g1ngTo_rypRs!Tti?|5E=_bFl3GF)1hEKEEq;|6;NA4ZG1DW?9_O7 zDN%?43ThZs_~A3n-}0@=ByEG`v>g!(fj#l1z}xZX^hT(Kp?)|cbigs^rXkrT=G2}p zlJYZ)d46hFNzv{Mwxv6~(;9bfR)vqto&5J?mBcf5ZHB3Izo)%JLC<#yR#cB7qwKjC ze}L}rkcggJ2NA53Bv`wqp^mm1UGY?ZH`7s`6@7|@g`9itr?`@+(e&3;kMi4}=Ok-G zq{Mon6xrOpTHra+NIlJ;HqGZf!a+(%b>UPmJifXqP%1A2f_MWh7_OPZWMT84TkC)DR`s@;zJe=hD zoHK(i^Z27u@=W@1Oj8|tDgp=B#q})dyIXg6+=sI>_JbND zabqa#IGPh~kD$u5-NxgEvghkzgzreGsQavj_0k_OQ~WF2(HkK0<2<)jef_n-D2sHh zV0gPQt+xe=5I?W@-NsUSp?z;VWJ_+vQ*o%ZzX_)Wk&e{twQo?9o&&-u7Q*XTScyzi zE*dT<+wdhYgS4ZQoc7XtmO|FQrSc4#vJc4izselMLXIDGwcaZ^Tz@*s)B8R z2-hsO8Xq06cQitTY?s(*+avRErAW>`5-z2fSuf;TJ`mY%%p=#T1SPtFTcU!75=x?t zG8rT_7ZYX2S}!#%6!)|t8I(0wvqD63_drUgJ3s{Me(?K2T3Gkt*gwF)pjB$uRth2j z4%Q-*e|l21X*r=oAS`qF{`J^BvoS^uZ6*9vL%uaVjI%k;pmjiizQjT=$$m|aA42=Y zc;ldoCrvHH&2FaWhhgJS!cWS-A0OUTLoKjanP(2W$CA+auk{wE1GMO&Xz`D?A|R@u zCqI86d6HQ1L0}ZWrc$EG4aCfR{WT$Uqr4MA;`(5D&Jji%gSasf#IWdt2K-n=XFUz< z6ocx%$=B}gb!ynn^bn@ zgWlYf(%DGy1|c73v={rso+VWA)7q$2Xk0LEbK|pd$(b%`oESm6EF5JB1RLKI2IoKf zHACPiN|E(et9O##TanG+?JmnC2}3nY6M=}JCm<7DR~5%UH-URQu1DgO(oywOyoP+q zvsw?-9VDUwxkwqc1gXKs<6&?k<-^KInbv0O)i!5lL+^M`m1-7hTK6cxB@ios!_-$*U_Tl^_WY^2Q10+TlMh-b%V9%5=F(d9a5cVrB z(H*Y!kI4#I3C!f-8BnQQYS#$Pe~|tOUr+zyf_q>gVZYO3E#gO?@oy#D1qBki_tC7v z3q{XTa$hJ_DcX5{#Bwhe;c@x^`W$I=&3ifn(#At@0P31iE2!8Tu6Zc2!8S}EX+TcC zh~XzmfG3kG^(Ba)oO?6-a0^yjp7tefjD$tV>8!fHA0@hE^{DvO?9pv~*|(pACN#~e zMd9h|mS$oNe7(F>0zoMzuFJcR+l3TspF&2`C7J=DYOAem zQD!dm%qr@Q1@6yLMI-{Nb}ttH>7LNU2tdK6rX^0oHU&Y`yi>o!Xl#huyFig z&#h;n5*R5<5-egEIW-1(qm8LKJ9`lF-Dgra)0$6P?wdn`v zD=>V*7xth~9WB9V*k3qj=)T z@d<0%T}{I>e_W>VCFy>E>w~0bN9($`S%gu2sB#$DI1m}NgKEi%c?i5wh-C|81&RaN z%4OPDgp(h!L6f}0SXnw~4~VWt4!+-nUL3wkZ4*WR$Gd!6Z~NfQrxV{BeQ>7(=#6!a zE+O^nA94VvgY?1Zx{3wSw@QM8$nHsl2lkh(@ZpUGs38GlQ-tJn?+OeN!P7IZ3J@X) zvu6L_wiP&fJO!RD^&mW3KB#qu#B5=>YLaF*fb1B44g0kpU(=L>2G^g%5ilOKaH$)t z^yG*clvAk8**?;%=9xCGA!&>49Nbt*vIt4}*~>%eSLkh2UeurHV)KD8Jw=WF!)Ivq zpoADNi2|Ns+m&PzJX)%$16o>%E(OlfOZy_)zEhWwI-k=!e^z^VgB9@EGHe`uxRq^| z)pfxikD((ia0SHzmhfF9>ZSxB@Ce>BV&k2qhWCUYzX{?=zMX_`z5$!->|aKj-!4Fw zPn_=bGG<&;E^9JV zayzCZYrT%VTJ-8Hc)VW@7YwISF`ua*qKKF=FcBfPWW^hihEeU)9zAMV=zi(piy?uq zSX_Iv>b9+?j#_XCcOLC`iCfar@Ovt~E=`b*VvgZ5Gh*|k#pB)7+Mp7XYaJ)>p7X>U zj5+(~e{q5MMb5yeoAf$(C71fZesdiJ&Dsd%R1(I~CtXmA8@tSkyuQ2VcOeEM)x}=K z(gyfQ&3lZ`!WgN~L7XmAa=nzfQh{u1G=T_FVmIF2eT1;g#k<3WKvhee*WTG;#>H@% z0mOCEt5bnI-H zMp(5u1U|koSS=Ve+hj>iNIn_u#M2lbNcs-Zy&ommfNqw=YNQ0YnzH?J91Uzr2ZBTV zLqP{gM*~K$_aiz}%;ddiq1C5%@91tBx477i;u^KNs)BC~wlYIc1LOP6)V|tAW}u#S zZ1|P{&=+!+Kvdoa#|xjEXN@F*I{aQ?) z^iKEu*@fz=;QV{mc}!;T1|&fCPqK@vkWKrwG|A=ki{RkCuj@3FYic1%9<#5*Jf(yW z2Pm379DljV#sl3R#V1trN-mPM+Nva_z=&djf{px?6EKQHn8B1fZN#h4`y!8J4S!47 z6lD>01z2WHOKPeWl9nQQ)Be-Yu$B7zgx@OmFu&sNdL-jECAvW;KNdxjKRr@o{>%&Z z$y0QoeWk-7$P@@_IxqJ~l;r1YYdtw*stXCy@=u^1b^}F3ZPDctw23YImyh|p!thB^ z^3!;7DQ>iSNAXu!v+;G`s#cYjvD%aJFYyg7$^efQHvKxgeg)|HT$)c_pO z-vTBAQ^1m3_V=O0C;*aB`NL)CQ*yH?{L$nKPx zbnFee;h03^UO|}l3>K3w<1#h(79XmPi`VQw{N9()23v6mi84(LIQGQIU%AS-Q2S}S zf1al;=|HBg8$!S#o5cH=FFC`fB1EWIf~KQn$SxLzGg;pK_HYbB58Qx7Ox-xE#fM#v zqjq{%7*DyD5HmoYJ$7^`FInZQ|X6_+l3n~Y#gnYsG2RwfGL(=EAGsGih)*XIZ1 zWuykuG(F_2me2e^n)!SmY-(w>KX@*K1ArHizXs)6sr2y8e=x)$McElV8Vn?g43+cD zJ>Bblp|D!QDg;pkeGu%6u$d7T2l%s1JdRb#Js{+yt*JCdV&-Q(Y3h6{2q${284={- zl3oEv-E#hJ#(;m^f~9U*3H(V`jrq?rKi0)cehjQYSVB9)8I05UWgu(H?MnEG_v^~g`?DNwh$H%h7FyIQTHvypKluEyslK;#TJV43mvwAbB z(;M6d$G;RdfOwxXa3EB^&#R}O`49yPrquc74sKO!EL(`(#Pz)`%ZKV_pJM6HKMZnGjSnD|GnL(ZePW)+dCU>*E7*IXB5(BO zW<+V_cN1>JSIOJcp=cyt1xGIHIKm5vi6Nn|{ydP0OH77)<=GuxTgEI45-T47XRuBs;VjNeR=K zfEmKNp?|1E@7-fDvc6bdmXrbiqd(M&142X5!Oz?5R6Uq!R|HAYB(XI)KP`ZwF zyn{YTn)=;$@BBl2y+oOro24Yk!{cX(^L4U4)5Wz72INFgx!dXTUUQx7FRp;dIw95J z`bm0yR8QDd{wAIs2xCefG@&W+jT-1Gi*2$hv-SX~(QHJO^kng>XiVIxF41C3KEw^u zTxshd`l7-48`*j7aedU>k%5@lE+X7Ym$gw78BWJTGdPm-4Kf zBlVMineI{MHTHAPrxg~w@w=*VAQaFudY7EJGblE0M&;`@@U-$CPB6uz=>8~a&uiy( zzPBN;9Md>=iNsc@f-~xdK~;Q5BJvKxLx&G|Hs2bp5p># zN|;KAXQ>aqgJ$r;E8TDl(>!IeI5ic_10)9((LdJaCVglaZ^RcX`N=u>qA`MM_mvn2 z%mNh$8#D8qD=#bd>Ug8`oyks&XD+2v(xq@TNg2J%a2^E#Ysm(P3L@nAc?dV}Isw0^ z8OYW3u0+o4^R85l81UbA4V-{_4Jq;swdx0Pz0CO{R&zwZ!(_E=bK5NW!xA9^gC)L(m%gQk;v#eYMcqW-2|r7bF+CJUb#nz${s z7Z-8ay3YK>su*8^880?=N;Ef<=)iBJTOsfW#|jg1Q)wajTGJZ?OhZ!(d;OD2-uv|4 znrMHcBS#p-RTbs*RWVOWPwU5~^tsp4x2a+@5h5KxVRUwk2H%l{kA=M1L3v85$K1pG zn9L0Zr-Yk)t88OE>+3-;;(W(wEh67bu?Npl-w;C^oIfg+eD|k6KYLs>w~vvKEBak% z85Ft5%cI3IRo}CcADmBU^Rd=8S``lasBmzVSh19_>K=V*7sF2D=NdjC14H2Iq|TG9 zC;tQI^b|>_$hGYG>6=6FHY+n^fd82VmqnVM#hPl}E?D1V`KaG1Cj)th`~t;eJiID@ zuX7yio1M2{ZOR8X(`Ob_^z%v_ zcuwtAz&OMjKajEyOx9sZF@|fA7s|P&_0iDk2Qc}8KH5EM3l62UVYFF{Eg#MaM%3GF zyfl*k4gnOiX^jXMSTo@A#l!CnP(2z1)hM=?7hH2MDBAz-oWcy+kzKQD2sQYoR-KWL znt!THb1gMmpKWrXBTM;OeXZi+SaspAUJo&ls0;JvT5wz3_1dt?Y8IMUS_y%=)_JDa ziNQ6T6f3K1MaC-yXU)eKgOk32%HCGtpv!1__P6hq)>Dz=oEs>JCc z$=`3NRa8egTstcTQE3^+NKaPxPwP9q5ATVxSkfy!K}9q4oIZp3U0@ORJpZeXRMmeODh5ij$O>CdHg1h{RPhy}-%6Vp z2?9j(OE1ZyvvG-KW3t4W=FTQHo}jd(Z;kdn`Z(_SGB{=dMXs^Wve$4WZiLW~9j=%g z6xNAQf~i8xv64HDl5X&))*e_#(95_eYkz%AdzfFYQ){j3-qsdmWr9|=mP*1;8jW!u!4L;@oK`<^WhUoi{w zu+{C9d`V`m-TM0*7l};bCE(ESgIK!3$3u);`zf(f&NS>^e2H1rK#lHC0>}QzKwp z-FnM?Fn9Fo;y{Ttdffp?Fm;i0-H<`i^tPXKdb>JAFg?^5vM z<9;ng889_Q`;mCN{(?!h!gw85KXyl6DD~oU=EcxKnQUoofcR*?o=kXQ{4tA`KO1kA*4CbK-FEP?x`t>zMY^-;Xgazy0!mimnH*QR* zPeeLfQlN1&iv>d!_q_ZE%;j19P z5;wIFzR_V*8JcgLaS;KrphaQwa(R=GD!dKf@6g-C{NjOCXX%3TK#huv31nxHad9H0;)|Ffl;V5n_8Pv=}s{a?4^# zMrdQ@^zR1}5znOqgj5zQAqz>fI4CC1PWisW;!gzLCEAttM&F*8-I(;mI>6 zGd@&WMlU{g`vKx801WTu#3*c82z!qt;(*AA7A#__HA1c;+WF3ogLCD`Yf7=rgxAGi zB1z=o$LqsMM~8*98I2PhmEcDv)J+RauzfesAXW3z4_?z786UlTEKm!t71cUKhs~_- z`za!ImTU}TL%p(mri8`0>?>xQ>Syn-c#rpQZIZLtGpSc$YyebSyftUJ@n;SlZcImk zHGBcJ4mYdIKeIi_FAoF-ZlgmqH$HE#TV-9oOu+9+#u6G+ClXkEJjj@0N`d=Se2XV} za6aZ2UDpIbu0mX`SZchzmcmRhZn!jrYr0OIbc9$XP{BjHS@xVTa5=wp31b zwACT65?V131b*K0GGo*HwjUrK;t|uBl@fENe7=h5SfBQi$60tKq@-&z#Jh8E#KJo5 zc63#7SMK+RD_+Vyqqt29;Jn`DPiG}5&@0~(sQFsJz>6fZ2tnU6TzWSw9rs&dz81hD zf%p%=oA1cyOc&?3mcmHg8P!Mv&s7`(xGsoYz{i)vzsq&uv@Uo;Qgp5MpG#9*GG8Bb z;5pxSe8=x{Tken4g$m$mI0G7oS<&k$jNz2i5q`|<3cq5&x`*3ff{G%lUGo2i`XXfF zE2WzReaGi7iTYO?T~Q5^6(KJ6F1LK~{8HTFn>rVO$gKx1)@lMSA8eLw%L*X`rt^?t zKc_YgKyXBIM85GM$!Epu$vnjgbf~z`d%6Zz_KZ(N;#YIXe}IJfY5pe0*?m)Ojaq!? zG{`c1w{S|b*PZ{Jfg!XEIy#VDiR z;sLjRhfeI4v7&SJf8%4fqmF<+#PyQAt|B0Q%t(+R*mmC|iIK2mN7@nm9+ZFJsPC!- zf{ddPsevvz?*YBT4uA~ysmO%rKfNP|g!3fjl<=sG0N~l6{;Y*O2zf*d)AKrY*BS|0 zRQe7A%>Oc6L6Bk}xJnFYQ^QnBYo0l7N{Q}XJ8B4}_C8xHEC+X2U^9U{1;nYTDNqcu!qfyTm`jtJ zVHvUzk9QY zIH^o^rQ)kET;v9O0IYW=488Vh>yp1W8iTr&mYc$Zmi zZzePaKOfsCU!4(86~4c@@D6Iiw%E3^)wkBG*GRV3$F~$lJXlW zmvv|@LX?^C`~=~+CvJHQ$v20vO?v-if##!3Y62c6JZTXz8aIe;`$4!UhG4sMD|(R< zO|nhTGT%&+P5zLl&ttP1b{Om_Gy+*(6&-LYrZ4#c?VJmpK}*=>7=&g~AY}WF|5+!1 z=xxx1sZovRG5r8bXug}fOjy|hDfg7|CUuPCg{8yLC;6q>nZ~ccG$OISNH8M2A7PZ0 zfTAaY6!a=$Lb(d42NDh8v&MMs%9H{YukhM^bz(60&w$FUOm|;1SA91m1vzA-!)`XO z-}AwRR-TH*{-8KaKAdMj1ci1sfHnlS1rR7OE5P#sj>yJPxIVrrdYmSZK1iG*)F^M0 zkoCL6?E=KV1D<`itB#u!L0%q8x=LC5C%}y$c-F&o5yXl7@Ndi2*k-VlO+$9og<)_i z8Q{S49v-H~-(-cPF8h3XNZXlsRqYkQwvLqza)gHq8k2u-jIDix*_jw~tilFsCAF_ypxD}+|BLsuE zTs&`&hqXvP^zknM1KfaoCK6WCx_+?fL;Hb;xrAe?D5S9K`;dR%2s2gMf}niMj;&|w zO^sNfV&DbRD7hK>vcSA!K(Y&K9^U_V8qAnRpsk6xpWuxKHg>(Yyx6is9g5S^ed{z* zsnrc2{8L*+5JDj|xRf{E1WC>1*%F~Ib(S0CDyJ9SL?Fb^T1Pj>psnkyiTM>!9MoT> z8(7Xq97_;@%zcH0?(u%9Esy9m?NBNXs*`AhmZf~mW3o9`5T;_j31%q-&wt@LgE%~A zBW4jnhoi6I6XP1IsF|#fk1ojQjvgIDa9w1Xy$s9jx#&3BN_i0VD?=3OzTgu#rh`~x=0w}wLo8v9 z3{Sci9Pn~*L5}~rp#Mz(eDVa{iw_9VC+p%9JGV z9E2TOgZEXkLBI<%+1(U~M?7Tle=j2Gs;wRC)xv(k2BDBr@bMz7A?W}bsm8J@uj@GdJNb6S%ZT8S@26$E_IOMWI($5o| z5Ey9uNDj}dRTlh66fk`CGYp&YpKS3pydcp3Agx4@Fkbk*W_4yId0nR2jhssGVa>jb z`t6SzJT=}>REE}&R;DFTgn(Xdn76{fAJiu#_$nX!GYLlsJmiqrjGI>!4+_1LkeM-Q zLW6ctW1fqWKJVxUC&wC`JaR?EbC?tfqK&^!-Ct^a=VJUjjba}nI5(mX^w`-k&U*pN z~)Q%kM)xqP)CqNyeq#@vu)jy&iLq$Oo93MkUL9rs+V$}5OjD6Xo@-+Fyje*y0Ff+*@p3@DHcaQ zef&yxgat$NI~T!6A;#oin=@KN9@tzB#on$B2>N*fou+~k)SRywskGmt4Z2PefA-E# z#S>-Y>HNrNDY)=C&e=ppH&YlcoZ7w}!x#QC`P@#6+>@2}I?Kpi1|ESwdn6E3uiy9W zdTetJX4>oP!6)XFs;=9$$LsIjySFmC!hP`9(i0W*g~KmIJ`&_l(?0ihUw0-tIy$n8ovy)|7ol}oMuus8e0S zs;2gIf4|<3l=CB2lpCQQR4+?%89v;4ON#^y{+N|cx_J+!!9}c5_F?>Yaf2~cg zmVXw>>M46MMW8Bv^6z1j<;N$98Xg{Y37TO2(0=CKVD8rfe49 zKVO%oa#fF#-O)kQljYx!^6OSeNXW`a3qkwHts$vr!&_q$hVNGj8zjlmGj+m=5^}t> zw%jS%X*==roK@GKR5CwJ3m?Ex{dqfxmz0^q=lGJ|J5PL>H9R}Nfvt7Q9HhZtCAn(~ zx7uSfe#;~^d~;@ICfdCtE$Vv%+eGPgu<@D={JAd@z; zwpQ);sLSMB1z+M*?PcLQ_nj@~rzG(@DN_kOx9D>Jo)7ThHGlk(4z`>Nc)H%dq<_#y zZRE*)cWLbsXY(=DeS35GNX3c6742}U-P2{x+w1Un(iIyh:8080 - server master2 :8080 - balance leastconn -``` - -And the corresponding example global vars config: -``` -loadbalancer_apiserver_localhost: true -``` - -This var overrides an external LB configuration, if any. Note that for this -example, the `kubernetes-apiserver-http` endpoint has backends receiving -unencrypted traffic, which may be a security issue when interconnecting -different nodes, or may be not, if those belong to the isolated management -network without external access. +Specifying an external LB overrides any internal localhost LB configuration. +Note that for this example, the `kubernetes-apiserver-http` endpoint +has backends receiving unencrypted traffic, which may be a security issue +when interconnecting different nodes, or maybe not, if those belong to the +isolated management network without external access. In order to achieve HA for HAProxy instances, those must be running on the each node in the `k8s-cluster` group as well, but require no VIP, thus @@ -109,8 +97,8 @@ Access endpoints are evaluated automagically, as the following: | Endpoint type | kube-master | non-master | |------------------------------|---------------|---------------------| -| Local LB (overrides ext) | http://lc:p | http://lc:p | -| External LB, no internal | https://lb:lp | https://lb:lp | +| Local LB | http://lc:p | http://lc:sp | +| External LB, no internal | http://lc:p | https://lb:lp | | No ext/int LB (default) | http://lc:p | https://m[0].aip:sp | Where: diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index add9fdd2a..ec715b960 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -64,8 +64,9 @@ ndots: 5 # This may be the case if clients support and loadbalance multiple etcd servers natively. etcd_multiaccess: false -# Assume there are no internal loadbalancers for apiservers exist -loadbalancer_apiserver_localhost: false +# Assume there are no internal loadbalancers for apiservers exist and listen on +# kube_apiserver_port (default 443) +loadbalancer_apiserver_localhost: true # Choose network plugin (calico, weave or flannel) kube_network_plugin: flannel diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 94da756be..79b1faef0 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -11,3 +11,6 @@ kube_proxy_mode: iptables # kube_api_runtime_config: # - extensions/v1beta1/daemonsets=true # - extensions/v1beta1/deployments=true + +nginx_image_repo: nginx +nginx_image_tag: 1.11.4-alpine diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 9c14e7a4c..a8cb6ce5a 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -1,6 +1,9 @@ --- - include: install.yml +- include: nginx-proxy.yml + when: is_kube_master == false and loadbalancer_apiserver_localhost|default(false) + - name: Write Calico cni config template: src: "cni-calico.conf.j2" diff --git a/roles/kubernetes/node/tasks/nginx-proxy.yml b/roles/kubernetes/node/tasks/nginx-proxy.yml new file mode 100644 index 000000000..056c55a93 --- /dev/null +++ b/roles/kubernetes/node/tasks/nginx-proxy.yml @@ -0,0 +1,9 @@ +--- +- name: nginx-proxy | Write static pod + template: src=manifests/nginx-proxy.manifest.j2 dest=/etc/kubernetes/manifests/nginx-proxy.yml + +- name: nginx-proxy | Make nginx directory + file: path=/etc/nginx state=directory mode=0700 owner=root + +- name: nginx-proxy | Write nginx-proxy configuration + template: src=nginx.conf.j2 dest="/etc/nginx/nginx.conf" owner=root mode=0755 backup=yes diff --git a/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 new file mode 100644 index 000000000..50e054268 --- /dev/null +++ b/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx-proxy + namespace: kube-system +spec: + hostNetwork: true + containers: + - name: nginx-proxy + image: {{ nginx_image_repo }}:{{ nginx_image_tag }} + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/nginx + name: etc-nginx + readOnly: true + volumes: + - name: etc-nginx + hostPath: + path: /etc/nginx diff --git a/roles/kubernetes/node/templates/nginx.conf.j2 b/roles/kubernetes/node/templates/nginx.conf.j2 new file mode 100644 index 000000000..edcee08a9 --- /dev/null +++ b/roles/kubernetes/node/templates/nginx.conf.j2 @@ -0,0 +1,26 @@ +error_log stderr notice; + +worker_processes auto; +events { + multi_accept on; + use epoll; + worker_connections 1024; +} + +stream { + upstream kube_apiserver { + least_conn; + {% for host in groups['kube-master'] -%} + server {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }}; + {% endfor %} + } + + server { + listen {{ kube_apiserver_port }}; + proxy_pass kube_apiserver; + proxy_timeout 3s; + proxy_connect_timeout 1s; + + } + +} diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml index 343ba5707..5d1b2cd2e 100644 --- a/roles/kubernetes/preinstall/defaults/main.yml +++ b/roles/kubernetes/preinstall/defaults/main.yml @@ -21,6 +21,8 @@ kube_log_dir: "/var/log/kubernetes" # pods on startup kube_manifest_dir: "{{ kube_config_dir }}/manifests" +# change to 0.0.0.0 to enable insecure access from anywhere (not recommended) +kube_apiserver_insecure_bind_address: 127.0.0.1 common_required_pkgs: - python-httplib2 diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml index e3f4757a7..19f08df78 100644 --- a/roles/kubernetes/preinstall/tasks/set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_facts.yml @@ -5,12 +5,12 @@ - set_fact: is_kube_master="{{ inventory_hostname in groups['kube-master'] }}" - set_fact: first_kube_master="{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}" - set_fact: - kube_apiserver_insecure_bind_address: |- - {% if loadbalancer_apiserver_localhost %}{{ kube_apiserver_address }}{% else %}127.0.0.1{% endif %} + loadbalancer_apiserver_localhost: false + when: loadbalancer_apiserver is defined - set_fact: kube_apiserver_endpoint: |- - {% if loadbalancer_apiserver_localhost -%} - http://127.0.0.1:{{ kube_apiserver_insecure_port }} + {% if not is_kube_master and loadbalancer_apiserver_localhost -%} + https://localhost:{{ kube_apiserver_port }} {%- elif is_kube_master and loadbalancer_apiserver is not defined -%} http://127.0.0.1:{{ kube_apiserver_insecure_port }} {%- else -%} diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh index a2f698541..f90fb7e8b 100755 --- a/roles/kubernetes/secrets/files/make-ssl.sh +++ b/roles/kubernetes/secrets/files/make-ssl.sh @@ -26,8 +26,8 @@ Usage : $(basename $0) -f [-d ] -h | --help : Show this message -f | --config : Openssl configuration file -d | --ssldir : Directory where the certificates will be installed - - ex : + + ex : $(basename $0) -f openssl.conf -d /srv/ssl EOF } @@ -37,7 +37,7 @@ while (($#)); do case "$1" in -h | --help) usage; exit 0;; -f | --config) CONFIG=${2}; shift 2;; - -d | --ssldir) SSLDIR="${2}"; shift 2;; + -d | --ssldir) SSLDIR="${2}"; shift 2;; *) usage echo "ERROR : Unknown option" @@ -68,6 +68,7 @@ openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN openssl genrsa -out apiserver-key.pem 2048 > /dev/null 2>&1 openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config ${CONFIG} > /dev/null 2>&1 openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1 +cat ca.pem >> apiserver.pem # Nodes and Admin for i in node admin; do diff --git a/roles/kubernetes/secrets/tasks/gen_certs.yml b/roles/kubernetes/secrets/tasks/gen_certs.yml index 7178bce0c..6057c0676 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs.yml @@ -65,3 +65,30 @@ shell: chmod 0600 {{ kube_cert_dir}}/*key.pem when: inventory_hostname in groups['kube-master'] changed_when: false + +- name: Gen_certs | target ca-certificates directory + set_fact: + ca_cert_dir: |- + {% if ansible_os_family == "Debian" -%} + /usr/local/share/ca-certificates + {%- elif ansible_os_family == "RedHat" -%} + /etc/pki/ca-trust/source/anchors + {%- elif ansible_os_family == "CoreOS" -%} + /etc/ssl/certs + {%- endif %} + +- name: Gen_certs | add CA to trusted CA dir + copy: + src: "{{ kube_cert_dir }}/ca.pem" + dest: "{{ ca_cert_dir }}/kube-ca.crt" + remote_src: true + register: kube_ca_cert + +- name: Gen_certs | update ca-certificates (Debian/Ubuntu/CoreOS) + command: update-ca-certificates + when: kube_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS"] + +- name: Gen_certs | update ca-certificatesa (RedHat) + command: update-ca-trust extract + when: kube_ca_cert.changed and ansible_os_family == "RedHat" + diff --git a/roles/kubernetes/secrets/templates/openssl.conf.j2 b/roles/kubernetes/secrets/templates/openssl.conf.j2 index 5eab64979..ac94b6800 100644 --- a/roles/kubernetes/secrets/templates/openssl.conf.j2 +++ b/roles/kubernetes/secrets/templates/openssl.conf.j2 @@ -11,16 +11,18 @@ DNS.1 = kubernetes DNS.2 = kubernetes.default DNS.3 = kubernetes.default.svc DNS.4 = kubernetes.default.svc.{{ dns_domain }} +DNS.5 = localhost {% for host in groups['kube-master'] %} -DNS.{{ 4 + loop.index }} = {{ host }} +DNS.{{ 5 + loop.index }} = {{ host }} {% endfor %} {% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} -{% set idx = groups['kube-master'] | length | int + 4 %} -DNS.5 = {{ apiserver_loadbalancer_domain_name }} +{% set idx = groups['kube-master'] | length | int + 5 %} +DNS.{{ idx | string }} = {{ apiserver_loadbalancer_domain_name }} {% endif %} {% for host in groups['kube-master'] %} IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} {% endfor %} {% set idx = groups['kube-master'] | length | int * 2 + 1 %} -IP.{{ idx | string }} = {{ kube_apiserver_ip }} +IP.{{ idx }} = {{ kube_apiserver_ip }} +IP.{{ idx + 1 }} = 127.0.0.1 From a6344f7561cdf6bf387a25cd6259eb0881347812 Mon Sep 17 00:00:00 2001 From: Sergey Vasilenko Date: Thu, 6 Oct 2016 13:06:19 +0300 Subject: [PATCH 05/34] Changes in Kubernetes and Calico-CNI plugin config files required for usage of Calico CNI plugin version 1.4.2 --- roles/kubernetes/node/templates/cni-calico.conf.j2 | 4 +++- roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/kubernetes/node/templates/cni-calico.conf.j2 b/roles/kubernetes/node/templates/cni-calico.conf.j2 index c48b084a5..c992d984d 100644 --- a/roles/kubernetes/node/templates/cni-calico.conf.j2 +++ b/roles/kubernetes/node/templates/cni-calico.conf.j2 @@ -1,9 +1,11 @@ { "name": "calico-k8s-network", "type": "calico", - "etcd_authority": "{{ etcd_authority }}", "log_level": "info", "ipam": { "type": "calico-ipam" + }, + "kubernetes": { + "kubeconfig": "{{ kube_config_dir }}/node-kubeconfig.yaml" } } diff --git a/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 b/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 index d21b8eef3..43a41eab6 100644 --- a/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 +++ b/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 @@ -4,6 +4,7 @@ clusters: - name: local cluster: certificate-authority: {{ kube_cert_dir }}/ca.pem + server: http://127.0.0.1:8080 users: - name: kubelet user: From dea4210da1b8d4bb6d4ee61a52e38b04be50e280 Mon Sep 17 00:00:00 2001 From: Sergey Vasilenko Date: Thu, 6 Oct 2016 18:42:33 +0300 Subject: [PATCH 06/34] Bump Calico-CNI plugin binaries versions and correct checksums --- roles/download/defaults/main.yml | 6 +++--- roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 | 2 +- roles/uploads/defaults/main.yml | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index f1f276c62..473951892 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -11,7 +11,7 @@ etcd_version: v3.0.6 #TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download calico_version: v0.20.0 -calico_cni_version: v1.3.1 +calico_cni_version: v1.4.2 weave_version: v1.6.1 flannel_version: 0.5.5 flannel_server_helper_version: 0.1 @@ -23,8 +23,8 @@ calico_cni_ipam_download_url: "https://storage.googleapis.com/kargo/{{calico_cni weave_download_url: "https://storage.googleapis.com/kargo/{{weave_version}}_weave" # Checksums -calico_cni_checksum: "ac05cb9254b5aaa5822cf10325983431bd25489147f2edf9dec7e43d99c43e77" -calico_cni_ipam_checksum: "3df6951a30749c279229e7e318e74ac4e41263996125be65257db7cd25097273" +calico_cni_checksum: "9cab29764681e9d80da826e4b2cd10841cc01a749e0018867d96dd76a4691548" +calico_cni_ipam_checksum: "09d076b15b791956efee91646e47fdfdcf382db16082cef4f542a9fff7bae172" weave_checksum: "9bf9d6e5a839e7bcbb28cc00c7acae9d09284faa3e7a3720ca9c2b9e93c68580" etcd_checksum: "385afd518f93e3005510b7aaa04d38ee4a39f06f5152cd33bb86d4f0c94c7485" diff --git a/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 b/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 index 43a41eab6..e1593303d 100644 --- a/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 +++ b/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 @@ -4,7 +4,7 @@ clusters: - name: local cluster: certificate-authority: {{ kube_cert_dir }}/ca.pem - server: http://127.0.0.1:8080 + server: {{ kube_apiserver_endpoint }} users: - name: kubelet user: diff --git a/roles/uploads/defaults/main.yml b/roles/uploads/defaults/main.yml index 30c71d76b..77ae446c4 100644 --- a/roles/uploads/defaults/main.yml +++ b/roles/uploads/defaults/main.yml @@ -6,7 +6,7 @@ kube_version: v1.3.0 etcd_version: v3.0.6 calico_version: v0.20.0 -calico_cni_version: v1.3.1 +calico_cni_version: v1.4.2 weave_version: v1.6.1 # Download URL's @@ -16,8 +16,8 @@ calico_cni_ipam_download_url: "https://github.com/projectcalico/calico-cni/relea weave_download_url: "https://github.com/weaveworks/weave/releases/download/{{weave_version}}/weave" # Checksums -calico_cni_checksum: "ac05cb9254b5aaa5822cf10325983431bd25489147f2edf9dec7e43d99c43e77" -calico_cni_ipam_checksum: "3df6951a30749c279229e7e318e74ac4e41263996125be65257db7cd25097273" +calico_cni_checksum: "9cab29764681e9d80da826e4b2cd10841cc01a749e0018867d96dd76a4691548" +calico_cni_ipam_checksum: "09d076b15b791956efee91646e47fdfdcf382db16082cef4f542a9fff7bae172" weave_checksum: "9bf9d6e5a839e7bcbb28cc00c7acae9d09284faa3e7a3720ca9c2b9e93c68580" etcd_checksum: "385afd518f93e3005510b7aaa04d38ee4a39f06f5152cd33bb86d4f0c94c7485" From 3919d666c19669d9244088c3e64db7f8a4650edd Mon Sep 17 00:00:00 2001 From: Artem Roma Date: Mon, 10 Oct 2016 17:09:50 +0300 Subject: [PATCH 07/34] Add possibility to enable network policy via Calico network controller The requirements for network policy feature are described here [1]. In order to enable it, appropriate configuration must be provided to the CNI plug in and Calico policy controller must be set up. Beside that corresponding extensions needed to be enabled in k8s API. Now to turn on the feature user can define `enable_network_policy` customization variable for Ansible. [1] http://kubernetes.io/docs/user-guide/networkpolicies/ --- .../tasks/calico-policy-controller.yml | 10 +++++ roles/kubernetes-apps/ansible/tasks/main.yaml | 4 ++ .../templates/calico-policy-controller.yml.j2 | 40 +++++++++++++++++++ .../manifests/kube-apiserver.manifest.j2 | 3 ++ .../node/templates/cni-calico.conf.j2 | 5 +++ 5 files changed, 62 insertions(+) create mode 100644 roles/kubernetes-apps/ansible/tasks/calico-policy-controller.yml create mode 100644 roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 diff --git a/roles/kubernetes-apps/ansible/tasks/calico-policy-controller.yml b/roles/kubernetes-apps/ansible/tasks/calico-policy-controller.yml new file mode 100644 index 000000000..f4ac65aeb --- /dev/null +++ b/roles/kubernetes-apps/ansible/tasks/calico-policy-controller.yml @@ -0,0 +1,10 @@ +- name: Write calico-policy-controller yaml + template: src=calico-policy-controller.yml.j2 dest=/etc/kubernetes/calico-policy-controller.yml + when: inventory_hostname == groups['kube-master'][0] + + +- name: Start of Calico policy controller + kube: + kubectl: "{{bin_dir}}/kubectl" + filename: /etc/kubernetes/calico-policy-controller.yml + when: inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/ansible/tasks/main.yaml b/roles/kubernetes-apps/ansible/tasks/main.yaml index aadd9c1a5..f31eb442b 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yaml +++ b/roles/kubernetes-apps/ansible/tasks/main.yaml @@ -17,3 +17,7 @@ state: "{{item.changed | ternary('latest','present') }}" with_items: "{{ manifests.results }}" when: inventory_hostname == groups['kube-master'][0] + + +- include: tasks/calico-policy-controller.yml + when: enable_network_policy is defined and enable_network_policy == True diff --git a/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 b/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 new file mode 100644 index 000000000..7c0a21cfa --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 @@ -0,0 +1,40 @@ +apiVersion: extensions/v1beta1 +kind: ReplicaSet +metadata: + name: calico-policy-controller + namespace: kube-system + labels: + k8s-app: calico-policy + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + matchLabels: + kubernetes.io/cluster-service: "true" + k8s-app: calico-policy + template: + metadata: + name: calico-policy-controller + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + k8s-app: calico-policy + spec: + hostNetwork: true + containers: + - name: calico-policy-controller + image: calico/kube-policy-controller:latest + env: + - name: ETCD_ENDPOINTS + value: "{{ etcd_endpoint }}" + # Location of the Kubernetes API - this shouldn't need to be + # changed so long as it is used in conjunction with + # CONFIGURE_ETC_HOSTS="true". + - name: K8S_API + value: "https://kubernetes.default:443" + # Configure /etc/hosts within the container to resolve + # the kubernetes.default Service to the correct clusterIP + # using the environment provided by the kubelet. + # This removes the need for KubeDNS to resolve the Service. + - name: CONFIGURE_ETC_HOSTS + value: "true" diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 56023b997..ddd6f2085 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -30,6 +30,9 @@ spec: {% for conf in kube_api_runtime_config %} - --runtime-config={{ conf }} {% endfor %} +{% endif %} +{% if enable_network_policy is defined and enable_network_policy == True %} + - --runtime-config=extensions/v1beta1/networkpolicies=true {% endif %} - --v={{ kube_log_level | default('2') }} - --allow-privileged=true diff --git a/roles/kubernetes/node/templates/cni-calico.conf.j2 b/roles/kubernetes/node/templates/cni-calico.conf.j2 index c992d984d..4615cdabd 100644 --- a/roles/kubernetes/node/templates/cni-calico.conf.j2 +++ b/roles/kubernetes/node/templates/cni-calico.conf.j2 @@ -5,6 +5,11 @@ "ipam": { "type": "calico-ipam" }, +{% if enable_network_policy is defined and enable_network_policy == True %} + "policy": { + "type": "k8s" + }, +{% endif %} "kubernetes": { "kubeconfig": "{{ kube_config_dir }}/node-kubeconfig.yaml" } From 056f4b6c00c2f15b4cb91d792ce745d33b136d4f Mon Sep 17 00:00:00 2001 From: Smana Date: Sat, 8 Oct 2016 19:19:25 +0200 Subject: [PATCH 08/34] upgrade to kubernetes version 1.4.0 test to change the machine type Revert "test to change the machine type" This reverts commit 7a91f1b5405a39bee6cb91940b09a0b0f9d3aee1. use google dns server when no upstream dns are defined comment upstream_dns_servers update documentation remove deprecated kubelet flags Revert "remove deprecated kubelet flags" This reverts commit 21e3b893c896d0291c36a07d0414f4cb88b8d8ac. --- README.md | 4 ++-- inventory/group_vars/all.yml | 6 +++--- roles/dnsmasq/templates/01-kube-dns.conf.j2 | 2 ++ roles/download/defaults/main.yml | 2 +- roles/network_plugin/calico/defaults/main.yml | 2 +- roles/network_plugin/calico/tasks/main.yml | 15 ++------------- roles/network_plugin/weave/tasks/main.yml | 12 ------------ roles/uploads/defaults/main.yml | 2 +- 8 files changed, 12 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 48da75f7f..1fe1f5145 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ If you have questions, you can [invite yourself](https://slack.kubespray.io/) to To deploy the cluster you can use : -[**kargo-cli**](https://github.com/kubespray/kargo-cli)
+[**kargo-cli**](https://github.com/kubespray/kargo-cli) (deprecated, a newer [go](https://github.com/Smana/kargo-cli/tree/kargogo) version soon)
**Ansible** usual commands
**vagrant** by simply running `vagrant up` (for tests purposes)
@@ -41,7 +41,7 @@ Supported Linux distributions Versions -------------- -[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.3.0
+[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.4.0
[etcd](https://github.com/coreos/etcd/releases) v3.0.1
[calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.20.0
[flanneld](https://github.com/coreos/flannel/releases) v0.5.5
diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index ec715b960..cbf2e63a2 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -109,9 +109,9 @@ kube_apiserver_insecure_port: 8080 # (http) # Do not install additional dnsmasq skip_dnsmasq: false # Upstream dns servers used by dnsmasq -upstream_dns_servers: - - 8.8.8.8 - - 8.8.4.4 +#upstream_dns_servers: +# - 8.8.8.8 +# - 8.8.4.4 # # # Use dns server : https://github.com/ansibl8s/k8s-skydns/blob/master/skydns-README.md dns_setup: true diff --git a/roles/dnsmasq/templates/01-kube-dns.conf.j2 b/roles/dnsmasq/templates/01-kube-dns.conf.j2 index cad7f8ea3..aed68a6fe 100644 --- a/roles/dnsmasq/templates/01-kube-dns.conf.j2 +++ b/roles/dnsmasq/templates/01-kube-dns.conf.j2 @@ -13,6 +13,8 @@ server=/{{ dns_domain }}/{{ skydns_server }} {% for srv in upstream_dns_servers %} server={{ srv }} {% endfor %} +{% elif cloud_provider == "gce" %} +server=169.254.169.254 {% else %} server=8.8.8.8 server=8.8.4.4 diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 473951892..1f2158a25 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -5,7 +5,7 @@ local_release_dir: /tmp download_run_once: False # Versions -kube_version: v1.3.0 +kube_version: v1.4.0 etcd_version: v3.0.6 #TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml index 45b04c8a9..aec7a5e15 100644 --- a/roles/network_plugin/calico/defaults/main.yml +++ b/roles/network_plugin/calico/defaults/main.yml @@ -7,4 +7,4 @@ ipip: false # Set to true if you want your calico cni binaries to overwrite the # ones from hyperkube while leaving other cni plugins intact. -overwrite_hyperkube_cni: false +overwrite_hyperkube_cni: true diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml index 45f6c352b..46f729883 100644 --- a/roles/network_plugin/calico/tasks/main.yml +++ b/roles/network_plugin/calico/tasks/main.yml @@ -22,16 +22,6 @@ changed_when: false notify: restart calico-node -- name: Calico | Do not use hyperkube cni if kube_version under v1.3.4 - set_fact: - use_hyperkube_cni: false - when: kube_version | version_compare('v1.3.4','<') - -- name: Calico | Use hyperkube cni if kube_version above v1.3.4 - set_fact: - use_hyperkube_cni: true - when: kube_version | version_compare('v1.3.4','>=') - - name: Calico | Copy cni plugins from hyperkube command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -a /opt/cni/bin/ /cnibindir/" register: cni_task_result @@ -39,17 +29,16 @@ retries: 4 delay: "{{ retry_stagger | random + 3 }}" changed_when: false - when: "{{ use_hyperkube_cni|bool }}" - name: Calico | Install calico cni bin command: rsync -pi "{{ local_release_dir }}/calico/bin/calico" "/opt/cni/bin/calico" changed_when: false - when: "{{ not use_hyperkube_cni|bool or overwrite_hyperkube_cni|bool }}" + when: "{{ overwrite_hyperkube_cni|bool }}" - name: Calico | Install calico-ipam cni bin command: rsync -pi "{{ local_release_dir }}/calico/bin/calico-ipam" "/opt/cni/bin/calico-ipam" changed_when: false - when: "{{ not use_hyperkube_cni|bool or overwrite_hyperkube_cni|bool }}" + when: "{{ overwrite_hyperkube_cni|bool }}" - name: Calico | wait for etcd uri: url=http://localhost:2379/health diff --git a/roles/network_plugin/weave/tasks/main.yml b/roles/network_plugin/weave/tasks/main.yml index 25a9837db..59cc1bf37 100644 --- a/roles/network_plugin/weave/tasks/main.yml +++ b/roles/network_plugin/weave/tasks/main.yml @@ -9,17 +9,6 @@ notify: - restart docker -- name: Weave | Determine hyperkube cni to use depending of the version of kube - set_fact: - use_hyperkube_cni: > - {%- if kube_version | version_compare('v1.3.4','>=') -%} - true - {%- elif kube_version | version_compare('v1.3.4','<') -%} - false - {%- else -%} - {{ ErrorCannotRecognizeVersion }} - {%- endif -%} - - name: Weave | Copy cni plugins from hyperkube command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/" register: cni_task_result @@ -27,7 +16,6 @@ retries: 4 delay: "{{ retry_stagger | random + 3 }}" changed_when: false - when: "{{ use_hyperkube_cni|bool }}" - name: Weave | Install weave command: rsync -piu "{{ local_release_dir }}/weave/bin/weave" "{{ bin_dir }}/weave" diff --git a/roles/uploads/defaults/main.yml b/roles/uploads/defaults/main.yml index 77ae446c4..661dd053a 100644 --- a/roles/uploads/defaults/main.yml +++ b/roles/uploads/defaults/main.yml @@ -2,7 +2,7 @@ local_release_dir: /tmp # Versions -kube_version: v1.3.0 +kube_version: v1.4.0 etcd_version: v3.0.6 calico_version: v0.20.0 From 71347322d6c99e61805d9f7f4d23d7b6633c4166 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Wed, 12 Oct 2016 19:12:02 +0300 Subject: [PATCH 09/34] Add cluster-cidr to kube-proxy config This option enables masquerading for traffic directed at pods that comes frmom outside the cluster. --- roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 index f0c4bc211..7abffe053 100644 --- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 @@ -17,6 +17,7 @@ spec: - --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml {% endif %} - --bind-address={{ ip | default(ansible_default_ipv4.address) }} + - --cluster-cidr={{ kube_pods_subnet }} - --proxy-mode={{ kube_proxy_mode }} {% if kube_proxy_masquerade_all and kube_proxy_mode == "iptables" %} - --masquerade-all From 21273926cedd26616a708ac6f92695914a0cdc80 Mon Sep 17 00:00:00 2001 From: Smana Date: Wed, 12 Oct 2016 21:54:54 +0200 Subject: [PATCH 10/34] upgrade flannel version --- README.md | 2 +- roles/download/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1fe1f5145..fa51d189b 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Versions [kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.4.0
[etcd](https://github.com/coreos/etcd/releases) v3.0.1
[calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.20.0
-[flanneld](https://github.com/coreos/flannel/releases) v0.5.5
+[flanneld](https://github.com/coreos/flannel/releases) v0.6.2
[weave](http://weave.works/) v1.6.1
[docker](https://www.docker.com/) v1.10.3
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 1f2158a25..1ef7b45df 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -13,7 +13,7 @@ etcd_version: v3.0.6 calico_version: v0.20.0 calico_cni_version: v1.4.2 weave_version: v1.6.1 -flannel_version: 0.5.5 +flannel_version: v0.6.2 flannel_server_helper_version: 0.1 # Download URL's From dd022f2dbc5d26062af62f560f99c19404835dc6 Mon Sep 17 00:00:00 2001 From: Smana Date: Wed, 12 Oct 2016 22:18:43 +0200 Subject: [PATCH 11/34] upgrade calico version v0.22.0 --- README.md | 2 +- roles/download/defaults/main.yml | 2 +- roles/uploads/defaults/main.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fa51d189b..d3ed06b4a 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,8 @@ Versions [kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.4.0
[etcd](https://github.com/coreos/etcd/releases) v3.0.1
-[calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.20.0
[flanneld](https://github.com/coreos/flannel/releases) v0.6.2
+[calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.22.0
[weave](http://weave.works/) v1.6.1
[docker](https://www.docker.com/) v1.10.3
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 1ef7b45df..3feef7554 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -10,7 +10,7 @@ kube_version: v1.4.0 etcd_version: v3.0.6 #TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download -calico_version: v0.20.0 +calico_version: v0.22.0 calico_cni_version: v1.4.2 weave_version: v1.6.1 flannel_version: v0.6.2 diff --git a/roles/uploads/defaults/main.yml b/roles/uploads/defaults/main.yml index 661dd053a..fb07e4c2e 100644 --- a/roles/uploads/defaults/main.yml +++ b/roles/uploads/defaults/main.yml @@ -5,7 +5,7 @@ local_release_dir: /tmp kube_version: v1.4.0 etcd_version: v3.0.6 -calico_version: v0.20.0 +calico_version: v0.22.0 calico_cni_version: v1.4.2 weave_version: v1.6.1 From c402feffbd9d8038f69ea9d56c823db51d18e5ca Mon Sep 17 00:00:00 2001 From: Chad Swenson Date: Fri, 14 Oct 2016 16:46:44 -0500 Subject: [PATCH 12/34] Parameterize several dependency endpoints so that they can be overridden with internal mirrors. Signed-off-by: Chad Swenson --- roles/dnsmasq/defaults/main.yml | 7 +++++++ roles/dnsmasq/templates/dnsmasq-ds.yml | 2 +- roles/download/defaults/main.yml | 7 +++++++ roles/kubernetes-apps/ansible/defaults/main.yml | 12 ++++++++++++ .../kubernetes-apps/ansible/templates/kubedns-rc.yml | 6 +++--- roles/kubernetes/node/meta/main.yml | 2 ++ roles/kubernetes/node/templates/kubelet.j2 | 6 +++--- roles/kubernetes/preinstall/defaults/main.yml | 2 ++ roles/kubernetes/preinstall/tasks/main.yml | 2 +- 9 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 roles/kubernetes-apps/ansible/defaults/main.yml diff --git a/roles/dnsmasq/defaults/main.yml b/roles/dnsmasq/defaults/main.yml index 48b52c121..7a1e77023 100644 --- a/roles/dnsmasq/defaults/main.yml +++ b/roles/dnsmasq/defaults/main.yml @@ -10,3 +10,10 @@ # Max of 2 is allowed here (a 1 is reserved for the dns_server) #nameservers: # - 127.0.0.1 + +# Versions +dnsmasq_version: 2.72 + +# Images +dnsmasq_image_repo: "andyshinn/dnsmasq" +dnsmasq_image_tag: "{{ dnsmasq_version }}" \ No newline at end of file diff --git a/roles/dnsmasq/templates/dnsmasq-ds.yml b/roles/dnsmasq/templates/dnsmasq-ds.yml index f1f622bbd..49223124e 100644 --- a/roles/dnsmasq/templates/dnsmasq-ds.yml +++ b/roles/dnsmasq/templates/dnsmasq-ds.yml @@ -14,7 +14,7 @@ spec: spec: containers: - name: dnsmasq - image: andyshinn/dnsmasq:2.72 + image: "{{ dnsmasq_image_repo }}:{{ dnsmasq_image_tag }}" command: - dnsmasq args: diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 1ef7b45df..56b21fe06 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -15,6 +15,7 @@ calico_cni_version: v1.4.2 weave_version: v1.6.1 flannel_version: v0.6.2 flannel_server_helper_version: 0.1 +pod_infra_version: 3.0 # Download URL's etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd" @@ -43,6 +44,8 @@ calico_node_image_repo: "calico/node" calico_node_image_tag: "{{ calico_version }}" hyperkube_image_repo: "quay.io/coreos/hyperkube" hyperkube_image_tag: "{{ kube_version }}_coreos.0" +pod_infra_image_repo: "gcr.io/google_containers/pause-amd64" +pod_infra_image_tag: "{{ pod_infra_version }}" downloads: calico_cni_plugin: @@ -108,6 +111,10 @@ downloads: repo: "{{ calico_node_image_repo }}" tag: "{{ calico_node_image_tag }}" enabled: "{{ kube_network_plugin == 'calico' }}" + pod_infra: + container: true + repo: "{{ pod_infra_image_repo }}" + tag: "{{ pod_infra_image_tag }}" download: container: "{{ file.container|default('false') }}" diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml new file mode 100644 index 000000000..b1086aa0d --- /dev/null +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -0,0 +1,12 @@ +# Versions +kubedns_version: 1.7 +kubednsmasq_version: 1.3 +exechealthz_version: 1.1 + +# Images +kubedns_image_repo: "gcr.io/google_containers/kubedns-amd64" +kubedns_image_tag: "{{ kubedns_version }}" +kubednsmasq_image_repo: "gcr.io/google_containers/kube-dnsmasq-amd64" +kubednsmasq_image_tag: "{{ kubednsmasq_version }}" +exechealthz_image_repo: "gcr.io/google_containers/exechealthz-amd64" +exechealthz_image_tag: "{{ exechealthz_version }}" \ No newline at end of file diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-rc.yml b/roles/kubernetes-apps/ansible/templates/kubedns-rc.yml index 3d193d1dc..ed38d671d 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-rc.yml +++ b/roles/kubernetes-apps/ansible/templates/kubedns-rc.yml @@ -21,7 +21,7 @@ spec: spec: containers: - name: kubedns - image: gcr.io/google_containers/kubedns-amd64:1.7 + image: "{{ kubedns_image_repo }}:{{ kubedns_image_tag }}" resources: # TODO: Set memory limits when we've profiled the container for large # clusters, then set request = limit to keep this container in @@ -63,7 +63,7 @@ spec: name: dns-tcp-local protocol: TCP - name: dnsmasq - image: gcr.io/google_containers/kube-dnsmasq-amd64:1.3 + image: "{{ kubednsmasq_image_repo }}:{{ kubednsmasq_image_tag }}" args: - --log-facility=- - --cache-size=1000 @@ -77,7 +77,7 @@ spec: name: dns-tcp protocol: TCP - name: healthz - image: gcr.io/google_containers/exechealthz-amd64:1.1 + image: "{{ exechealthz_image_repo }}:{{ exechealthz_image_tag }}" resources: # keep request = limit to keep this container in guaranteed class limits: diff --git a/roles/kubernetes/node/meta/main.yml b/roles/kubernetes/node/meta/main.yml index b9cbbd9ff..9c52b2d80 100644 --- a/roles/kubernetes/node/meta/main.yml +++ b/roles/kubernetes/node/meta/main.yml @@ -2,4 +2,6 @@ dependencies: - role: download file: "{{ downloads.hyperkube }}" + - role: download + file: "{{ downloads.pod_infra }}" - role: kubernetes/secrets diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index f55feefa9..53f2915d9 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -20,11 +20,11 @@ KUBELET_REGISTER_NODE="--register-node=false" {% endif %} # location of the api-server {% if dns_setup|bool and skip_dnsmasq|bool %} -KUBELET_ARGS="--cluster_dns={{ skydns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }}" +KUBELET_ARGS="--cluster_dns={{ skydns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}" {% elif dns_setup|bool %} -KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }}" +KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}" {% else %} -KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}" +KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}" {% endif %} {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "weave"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d" diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml index 5d1b2cd2e..61cad7467 100644 --- a/roles/kubernetes/preinstall/defaults/main.yml +++ b/roles/kubernetes/preinstall/defaults/main.yml @@ -21,6 +21,8 @@ kube_log_dir: "/var/log/kubernetes" # pods on startup kube_manifest_dir: "{{ kube_config_dir }}/manifests" +epel_rpm_download_url: "https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm" + # change to 0.0.0.0 to enable insecure access from anywhere (not recommended) kube_apiserver_insecure_bind_address: 127.0.0.1 diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index 8c2aecec5..49e69a907 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -91,7 +91,7 @@ changed_when: False - name: Install epel-release on RedHat/CentOS - shell: rpm -qa | grep epel-release || rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + shell: rpm -qa | grep epel-release || rpm -ivh {{ epel_rpm_download_url }} when: ansible_distribution in ["CentOS","RedHat"] and ansible_distribution_major_version >= 7 changed_when: False From 40de468413d5b733bfeb73f089ac904714438b16 Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Mon, 17 Oct 2016 15:42:30 +0200 Subject: [PATCH 13/34] Update ha docs Fix mismatch in code and docs, see https://github.com/kubespray/kargo/pull/528 Signed-off-by: Bogdan Dobrelya --- docs/ha-mode.md | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/docs/ha-mode.md b/docs/ha-mode.md index 851f50693..792c18a19 100644 --- a/docs/ha-mode.md +++ b/docs/ha-mode.md @@ -49,9 +49,11 @@ type. The following diagram shows how traffic to the apiserver is directed. ![Image](figures/loadbalancer_localhost.png?raw=true) -..note:: Kubernetes master nodes still use insecure localhost access because + Note: Kubernetes master nodes still use insecure localhost access because there are bugs in Kubernetes <1.5.0 in using TLS auth on master role - services. + services. This makes backends receiving unencrypted traffic and may be a + security issue when interconnecting different nodes, or maybe not, if those + belong to the isolated management network without external access. A user may opt to use an external loadbalancer (LB) instead. An external LB provides access for external clients, while the internal LB accepts client @@ -81,24 +83,19 @@ loadbalancer_apiserver: This domain name, or default "lb-apiserver.kubernetes.local", will be inserted into the `/etc/hosts` file of all servers in the `k8s-cluster` group. Note that the HAProxy service should as well be HA and requires a VIP management, which -is out of scope of this doc. +is out of scope of this doc. Specifying an external LB overrides any internal +localhost LB configuration. -Specifying an external LB overrides any internal localhost LB configuration. -Note that for this example, the `kubernetes-apiserver-http` endpoint -has backends receiving unencrypted traffic, which may be a security issue -when interconnecting different nodes, or maybe not, if those belong to the -isolated management network without external access. - -In order to achieve HA for HAProxy instances, those must be running on the -each node in the `k8s-cluster` group as well, but require no VIP, thus -no VIP management. + Note: In order to achieve HA for HAProxy instances, those must be running on + the each node in the `k8s-cluster` group as well, but require no VIP, thus + no VIP management. Access endpoints are evaluated automagically, as the following: | Endpoint type | kube-master | non-master | |------------------------------|---------------|---------------------| -| Local LB | http://lc:p | http://lc:sp | -| External LB, no internal | http://lc:p | https://lb:lp | +| Local LB | http://lc:p | https://lc:sp | +| External LB, no internal | https://lb:lp | https://lb:lp | | No ext/int LB (default) | http://lc:p | https://m[0].aip:sp | Where: From 91a101c8551dbb90e8bde2add77a33dbd4d68707 Mon Sep 17 00:00:00 2001 From: Smana Date: Tue, 18 Oct 2016 12:52:35 +0200 Subject: [PATCH 14/34] upgrade to k8s v1.4.3 --- README.md | 2 +- roles/download/defaults/main.yml | 2 +- roles/uploads/defaults/main.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d3ed06b4a..f050bb462 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Supported Linux distributions Versions -------------- -[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.4.0
+[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.4.3
[etcd](https://github.com/coreos/etcd/releases) v3.0.1
[flanneld](https://github.com/coreos/flannel/releases) v0.6.2
[calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.22.0
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 3feef7554..ffba5488a 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -5,7 +5,7 @@ local_release_dir: /tmp download_run_once: False # Versions -kube_version: v1.4.0 +kube_version: v1.4.3 etcd_version: v3.0.6 #TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults diff --git a/roles/uploads/defaults/main.yml b/roles/uploads/defaults/main.yml index fb07e4c2e..0774d324c 100644 --- a/roles/uploads/defaults/main.yml +++ b/roles/uploads/defaults/main.yml @@ -2,7 +2,7 @@ local_release_dir: /tmp # Versions -kube_version: v1.4.0 +kube_version: v1.4.3 etcd_version: v3.0.6 calico_version: v0.22.0 From 7d6fc1d6802885b23a4156c1dd1083c75d077e54 Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Tue, 18 Oct 2016 13:44:45 +0200 Subject: [PATCH 15/34] Update roadmap for the kubeadm LCM track Signed-off-by: Bogdan Dobrelya --- docs/roadmap.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 298750493..4396a9b07 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -26,13 +26,14 @@ Kargo's roadmap - single test with the Ansible version n-1 per day - Test idempotency on on single OS but for all network plugins/container engines - single test on AWS per day -- test different achitectures : +- test different achitectures : - 3 instances, 3 are members of the etcd cluster, 2 of them acting as master and node, 1 as node - 5 instances, 3 are etcd and nodes, 2 are masters only - 7 instances, 3 etcd only, 2 masters, 2 nodes - test scale up cluster: +1 etcd, +1 master, +1 node ### Lifecycle +- Adopt the kubeadm tool by delegating CM tasks it is capable to accomplish well [#553](https://github.com/kubespray/kargo/issues/553) - Drain worker node when upgrading k8s components in a worker node. [#154](https://github.com/kubespray/kargo/issues/154) - Drain worker node when shutting down/deleting an instance @@ -56,7 +57,7 @@ While waiting for the issue [kubernetes/kubernetes#18174](https://github.com/kub ### Kargo API - Perform all actions through an **API** - Store inventories / configurations of mulltiple clusters -- make sure that state of cluster is completely saved in no more than one config file beyond hosts inventory +- make sure that state of cluster is completely saved in no more than one config file beyond hosts inventory ### Addons (with kpm) Include optionals deployments to init the cluster: @@ -65,7 +66,7 @@ Include optionals deployments to init the cluster: - **Prometheus** ##### Others - + ##### Dashboards: - kubernetes-dashboard - Fabric8 From 6113a3f3500feba47cd19f178a8d537a6bd75d78 Mon Sep 17 00:00:00 2001 From: Smana Date: Tue, 18 Oct 2016 13:51:36 +0200 Subject: [PATCH 16/34] update roadmap, kubeadm adoption --- docs/roadmap.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/roadmap.md b/docs/roadmap.md index 298750493..95419f00d 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,6 +1,10 @@ Kargo's roadmap ================= +### Kubeadm +- Propose kubeadm as an option in order to setup the kubernetes cluster. +That would probably improve deployment speed and certs management [#553](https://github.com/kubespray/kargo/issues/553) + ### Self deployment (pull-mode) [#320](https://github.com/kubespray/kargo/issues/320) - the playbook would install and configure docker/rkt and the etcd cluster - the following data would be inserted into etcd: certs,tokens,users,inventory,group_vars. From e6902d8eccb0a8790d18ba8edf3c634dc30a36d6 Mon Sep 17 00:00:00 2001 From: Chad Swenson Date: Thu, 20 Oct 2016 14:56:52 -0500 Subject: [PATCH 17/34] Use absolute path for etcdctl Small fix. The shell module won't automatically resolve the path to the etcdctl binary, so i prefixed with {{ bin_dir }}/ --- roles/etcd/tasks/configure.yml | 4 ++-- roles/etcd/tasks/set_cluster_health.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/etcd/tasks/configure.yml b/roles/etcd/tasks/configure.yml index 56b01da3f..514a79d73 100644 --- a/roles/etcd/tasks/configure.yml +++ b/roles/etcd/tasks/configure.yml @@ -1,6 +1,6 @@ --- - name: Configure | Check if member is in cluster - shell: "etcdctl --no-sync --peers={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}" + shell: "{{ bin_dir }}/etcdctl --no-sync --peers={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}" register: etcd_member_in_cluster ignore_errors: true changed_when: false @@ -8,7 +8,7 @@ - name: Configure | Add member to the cluster if it is not there when: is_etcd_master and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0 - shell: "etcdctl --peers={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}" + shell: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}" - name: Configure | Copy etcd.service systemd file template: diff --git a/roles/etcd/tasks/set_cluster_health.yml b/roles/etcd/tasks/set_cluster_health.yml index be0d938dd..1a27e4dcf 100644 --- a/roles/etcd/tasks/set_cluster_health.yml +++ b/roles/etcd/tasks/set_cluster_health.yml @@ -1,6 +1,6 @@ --- - name: Configure | Check if cluster is healthy - shell: "etcdctl --peers={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'" + shell: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'" register: etcd_cluster_is_healthy ignore_errors: true changed_when: false From 4b7347f1cdd6730b79fe7113661e43db5a8bbb2a Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 21 Oct 2016 12:59:50 +0300 Subject: [PATCH 18/34] fix dnsmasq template cloud_provider lookup --- roles/dnsmasq/templates/01-kube-dns.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/dnsmasq/templates/01-kube-dns.conf.j2 b/roles/dnsmasq/templates/01-kube-dns.conf.j2 index aed68a6fe..4d73eebdb 100644 --- a/roles/dnsmasq/templates/01-kube-dns.conf.j2 +++ b/roles/dnsmasq/templates/01-kube-dns.conf.j2 @@ -13,7 +13,7 @@ server=/{{ dns_domain }}/{{ skydns_server }} {% for srv in upstream_dns_servers %} server={{ srv }} {% endfor %} -{% elif cloud_provider == "gce" %} +{% elif cloud_provider is defined and cloud_provider == "gce" %} server=169.254.169.254 {% else %} server=8.8.8.8 From 65d2a3b0e55604d2caaf5a38f49a0b8d742775b0 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 21 Oct 2016 14:39:58 +0300 Subject: [PATCH 19/34] Use only native cachable hostvars for etcd set_facts --- roles/etcd/templates/etcd-proxy.j2 | 2 +- roles/etcd/templates/etcd.j2 | 2 +- roles/kubernetes/preinstall/tasks/set_facts.yml | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/roles/etcd/templates/etcd-proxy.j2 b/roles/etcd/templates/etcd-proxy.j2 index 90d6f6470..0a1492a37 100644 --- a/roles/etcd/templates/etcd-proxy.j2 +++ b/roles/etcd/templates/etcd-proxy.j2 @@ -2,4 +2,4 @@ ETCD_DATA_DIR=/var/lib/etcd-proxy ETCD_PROXY=on ETCD_LISTEN_CLIENT_URLS={{ etcd_access_endpoint }} ETCD_NAME={{ etcd_proxy_member_name | default("etcd-proxy") }} -ETCD_INITIAL_CLUSTER={% for host in groups['etcd'] %}etcd{{ loop.index|string }}={{ hostvars[host]['etcd_peer_url'] }}{% if not loop.last %},{% endif %}{% endfor %} +ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }} diff --git a/roles/etcd/templates/etcd.j2 b/roles/etcd/templates/etcd.j2 index 1f7385939..b82116612 100644 --- a/roles/etcd/templates/etcd.j2 +++ b/roles/etcd/templates/etcd.j2 @@ -13,4 +13,4 @@ ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd ETCD_LISTEN_PEER_URLS=http://{{ etcd_address }}:2380 ETCD_NAME={{ etcd_member_name }} ETCD_PROXY=off -ETCD_INITIAL_CLUSTER={% for host in groups['etcd'] %}etcd{{ loop.index|string }}={{ hostvars[host]['etcd_peer_url'] }}{% if not loop.last %},{% endif %}{% endfor %} +ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }} diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml index 19f08df78..37bd6a33c 100644 --- a/roles/kubernetes/preinstall/tasks/set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_facts.yml @@ -29,8 +29,7 @@ - set_fact: etcd_endpoint="http://{{ etcd_authority }}" - set_fact: etcd_access_addresses: |- - {% for item in groups['etcd'] -%} - http://{{ hostvars[item].etcd_access_address }}:2379{% if not loop.last %},{% endif %} + http://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2379{% if not loop.last %},{% endif %} {%- endfor %} - set_fact: etcd_access_endpoint="{% if etcd_multiaccess %}{{ etcd_access_addresses }}{% else %}{{ etcd_endpoint }}{% endif %}" - set_fact: @@ -38,6 +37,11 @@ {% for host in groups['etcd'] %} {% if inventory_hostname == host %}{{"etcd"+loop.index|string }}{% endif %} {% endfor %} +- set_fact: + etcd_peer_addresses: |- + {% for item in groups['etcd'] -%} + http://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %} + {%- endfor %} - set_fact: etcd_proxy_member_name: |- {% for host in groups['k8s-cluster'] %} From 0e9d1e09e3c9533ad14e4e98e5f48713af66ff38 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 21 Oct 2016 14:43:41 +0300 Subject: [PATCH 20/34] Sync master tokens only with those in play_hosts --- roles/kubernetes/secrets/tasks/check-tokens.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes/secrets/tasks/check-tokens.yml b/roles/kubernetes/secrets/tasks/check-tokens.yml index 1ecaa7006..14cfbb124 100644 --- a/roles/kubernetes/secrets/tasks/check-tokens.yml +++ b/roles/kubernetes/secrets/tasks/check-tokens.yml @@ -27,7 +27,7 @@ sync_tokens: true when: >- {%- set tokens = {'sync': False} -%} - {%- for server in groups['kube-master'] + {%- for server in groups['kube-master'] | intersect(play_hosts) if (not hostvars[server].known_tokens.stat.exists) or (hostvars[server].known_tokens.stat.checksum != known_tokens_master.stat.checksum|default('')) -%} {%- set _ = tokens.update({'sync': True}) -%} From 11f1f71b3b9aaa0e6794b66c7da5f27deb94307d Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 21 Oct 2016 14:59:27 +0300 Subject: [PATCH 21/34] dynamically calculate etcd peer names --- roles/kubernetes/preinstall/tasks/set_facts.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml index 37bd6a33c..2dd947dda 100644 --- a/roles/kubernetes/preinstall/tasks/set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_facts.yml @@ -29,6 +29,7 @@ - set_fact: etcd_endpoint="http://{{ etcd_authority }}" - set_fact: etcd_access_addresses: |- + {% for item in groups['etcd'] -%} http://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2379{% if not loop.last %},{% endif %} {%- endfor %} - set_fact: etcd_access_endpoint="{% if etcd_multiaccess %}{{ etcd_access_addresses }}{% else %}{{ etcd_endpoint }}{% endif %}" @@ -40,7 +41,7 @@ - set_fact: etcd_peer_addresses: |- {% for item in groups['etcd'] -%} - http://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %} + {{ "etcd"+loop.index|string }}=http://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %} {%- endfor %} - set_fact: etcd_proxy_member_name: |- From c59c3a1bcfeee654449f75b5675106b886cb63fd Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Mon, 24 Oct 2016 15:11:52 +0200 Subject: [PATCH 22/34] Fix idempotency/recurrence of download and preinstall * Don't push containers if not changed * Do preinstall role only once and redistribute defaults to corresponding roles Signed-off-by: Bogdan Dobrelya --- cluster.yml | 7 ++++--- roles/download/tasks/main.yml | 17 ++++++++++++++--- roles/kubernetes/master/defaults/main.yml | 18 ++++++++++++++++++ roles/kubernetes/node/defaults/main.yml | 18 ++++++++++++++++++ roles/kubernetes/preinstall/defaults/main.yml | 3 --- roles/kubernetes/secrets/defaults/main.yml | 13 +++++++++++++ 6 files changed, 67 insertions(+), 9 deletions(-) diff --git a/cluster.yml b/cluster.yml index 208382354..295bb668a 100644 --- a/cluster.yml +++ b/cluster.yml @@ -10,21 +10,22 @@ - hosts: all gather_facts: true -- hosts: etcd:!k8s-cluster +- hosts: all roles: - { role: kubernetes/preinstall, tags: preinstall } + +- hosts: etcd:!k8s-cluster + roles: - { role: etcd, tags: etcd } - hosts: k8s-cluster roles: - - { role: kubernetes/preinstall, tags: preinstall } - { role: etcd, tags: etcd } - { role: kubernetes/node, tags: node } - { role: network_plugin, tags: network } - hosts: kube-master roles: - - { role: kubernetes/preinstall, tags: preinstall } - { role: kubernetes/master, tags: master } - hosts: k8s-cluster diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index b1b307e88..e715f380d 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -61,11 +61,22 @@ - set_fact: fname: "{{local_release_dir}}/containers/{{download.repo|regex_replace('/|\0|:', '_')}}:{{download.tag|regex_replace('/|\0|:', '_')}}.tar" +- name: "Set default value for 'container_changed' to false" + set_fact: + container_changed: false + +- name: "Update the 'container_changed' fact" + set_fact: + container_changed: "{{ not 'up to date' in pull_task_result.stdout }}" + when: "{{ download.enabled|bool and download.container|bool }}" + delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else inventory_hostname }}" + run_once: "{{ download_run_once|bool }}" + - name: Download | save container images shell: docker save "{{ download.repo }}:{{ download.tag }}" > "{{ fname }}" delegate_to: "{{groups['kube-master'][0]}}" run_once: true - when: ansible_os_family != "CoreOS" and download_run_once|bool and download.enabled|bool and download.container|bool + when: ansible_os_family != "CoreOS" and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool - name: Download | get container images synchronize: @@ -76,8 +87,8 @@ until: get_task|success retries: 4 delay: "{{ retry_stagger | random + 3 }}" - when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] and download_run_once|bool and download.enabled|bool and download.container|bool + when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool - name: Download | load container images shell: docker load < "{{ fname }}" - when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] and download_run_once|bool and download.enabled|bool and download.container|bool + when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index d0be14d64..ee32ccf57 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -10,3 +10,21 @@ kube_users_dir: "{{ kube_config_dir }}/users" # An experimental dev/test only dynamic volumes provisioner, # for PetSets. Works for kube>=v1.3 only. kube_hostpath_dynamic_provisioner: "false" + +# This is where you can drop yaml/json files and the kubelet will run those +# pods on startup +kube_manifest_dir: "{{ kube_config_dir }}/manifests" + +# This directory is where all the additional config stuff goes +# the kubernetes normally puts in /srv/kubernets. +# This puts them in a sane location. +# Editting this value will almost surely break something. Don't +# change it. Things like the systemd scripts are hard coded to +# look in here. Don't do it. +kube_config_dir: /etc/kubernetes + +# change to 0.0.0.0 to enable insecure access from anywhere (not recommended) +kube_apiserver_insecure_bind_address: 127.0.0.1 + +# Logging directory (sysvinit systems) +kube_log_dir: "/var/log/kubernetes" diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index ed85a3a53..8c4ce38a5 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -1,6 +1,13 @@ # This is where all the cert scripts and certs will be located kube_cert_dir: "{{ kube_config_dir }}/ssl" +# change to 0.0.0.0 to enable insecure access from anywhere (not recommended) +kube_apiserver_insecure_bind_address: 127.0.0.1 + +# This is where you can drop yaml/json files and the kubelet will run those +# pods on startup +kube_manifest_dir: "{{ kube_config_dir }}/manifests" + dns_domain: "{{ cluster_name }}" # resolv.conf to base dns config @@ -15,5 +22,16 @@ kube_proxy_masquerade_all: true # - extensions/v1beta1/daemonsets=true # - extensions/v1beta1/deployments=true +# Logging directory (sysvinit systems) +kube_log_dir: "/var/log/kubernetes" + +# This directory is where all the additional config stuff goes +# the kubernetes normally puts in /srv/kubernets. +# This puts them in a sane location. +# Editting this value will almost surely break something. Don't +# change it. Things like the systemd scripts are hard coded to +# look in here. Don't do it. +kube_config_dir: /etc/kubernetes + nginx_image_repo: nginx nginx_image_tag: 1.11.4-alpine diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml index 61cad7467..3eae9757d 100644 --- a/roles/kubernetes/preinstall/defaults/main.yml +++ b/roles/kubernetes/preinstall/defaults/main.yml @@ -23,9 +23,6 @@ kube_manifest_dir: "{{ kube_config_dir }}/manifests" epel_rpm_download_url: "https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm" -# change to 0.0.0.0 to enable insecure access from anywhere (not recommended) -kube_apiserver_insecure_bind_address: 127.0.0.1 - common_required_pkgs: - python-httplib2 - openssl diff --git a/roles/kubernetes/secrets/defaults/main.yml b/roles/kubernetes/secrets/defaults/main.yml index a5b88d7ac..c6011a9bf 100644 --- a/roles/kubernetes/secrets/defaults/main.yml +++ b/roles/kubernetes/secrets/defaults/main.yml @@ -6,3 +6,16 @@ kube_token_dir: "{{ kube_config_dir }}/tokens" # This is where to save basic auth file kube_users_dir: "{{ kube_config_dir }}/users" + +# This directory is where all the additional config stuff goes +# the kubernetes normally puts in /srv/kubernets. +# This puts them in a sane location. +# Editting this value will almost surely break something. Don't +# change it. Things like the systemd scripts are hard coded to +# look in here. Don't do it. +kube_config_dir: /etc/kubernetes + +# This directory is where all the additional scripts go +# that Kubernetes normally puts in /srv/kubernetes. +# This puts them in a sane location +kube_script_dir: "{{ bin_dir }}/kubernetes-scripts" From 93f7a2689637cf74eb04c15c1e695c3374ec1553 Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Tue, 25 Oct 2016 12:23:49 +0200 Subject: [PATCH 23/34] Enable smart facts cache for CI jobs Signed-off-by: Bogdan Dobrelya --- .travis.yml | 4 ++-- ansible.cfg | 5 ++++- tests/ansible.cfg | 5 ++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 998a0aa31..13604765e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,7 +110,7 @@ before_script: - cp tests/ansible.cfg . # - "echo $HOME/.local/bin/ansible-playbook -i inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}' $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml" ## Configure ansible deployment logs to be collected as an artifact. Enable when GCS configured, see https://docs.travis-ci.com/user/deployment/gcs -# - $HOME/.local/bin/ansible-playbook -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scritps/configure-logs.yaml +# - $HOME/.local/bin/ansible-playbook -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scripts/configure-logs.yaml script: - > @@ -134,7 +134,7 @@ script: ## Ping the between 2 pod - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL ## Collect env info, enable it once GCS configured, see https://docs.travis-ci.com/user/deployment/gcs -# - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scritps/collect-info.yaml +# - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scripts/collect-info.yaml after_script: - > diff --git a/ansible.cfg b/ansible.cfg index 2be6f4d02..f0e4ef652 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,4 +1,7 @@ [ssh_connection] pipelining=True -[defaults] +[defaults] host_key_checking=False +gathering = smart +fact_caching = jsonfile +fact_caching_connection = /tmp diff --git a/tests/ansible.cfg b/tests/ansible.cfg index 2be6f4d02..f0e4ef652 100644 --- a/tests/ansible.cfg +++ b/tests/ansible.cfg @@ -1,4 +1,7 @@ [ssh_connection] pipelining=True -[defaults] +[defaults] host_key_checking=False +gathering = smart +fact_caching = jsonfile +fact_caching_connection = /tmp From c7b00caeaac992e8dbb623994515dd9affe1b1b3 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 21 Oct 2016 14:21:46 +0300 Subject: [PATCH 24/34] Use tar+register instead of copy/slurp for distributing tokens and certs Related bug: https://github.com/ansible/ansible/issues/15405 Uses tar and register because synchronize module cannot sudo on the remote side correctly and copy is too slow. This patch dramatically cuts down the number of tasks to process for cert synchronization. --- roles/kubernetes/secrets/tasks/gen_certs.yml | 33 +++++++++---------- roles/kubernetes/secrets/tasks/gen_tokens.yml | 17 ++++------ 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/roles/kubernetes/secrets/tasks/gen_certs.yml b/roles/kubernetes/secrets/tasks/gen_certs.yml index 6057c0676..bec1d9f16 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs.yml @@ -27,31 +27,30 @@ master_certs: ['ca-key.pem', 'admin.pem', 'admin-key.pem', 'apiserver-key.pem', 'apiserver.pem'] node_certs: ['ca.pem', 'node.pem', 'node-key.pem'] -- name: Gen_certs | Get the certs from first master - slurp: - src: "{{ kube_cert_dir }}/{{ item }}" +- name: Gen_certs | Gather master certs + shell: "tar cfz - -C {{ kube_cert_dir }} {{ master_certs|join(' ') }} {{ node_certs|join(' ') }} | base64 --wrap=0" + register: master_cert_data delegate_to: "{{groups['kube-master'][0]}}" - register: slurp_certs - with_items: '{{ master_certs + node_certs }}' - when: sync_certs|default(false) run_once: true - notify: set secret_changed + when: sync_certs|default(false) + +- name: Gen_certs | Gather node certs + shell: "tar cfz - -C {{ kube_cert_dir }} {{ node_certs|join(' ') }} | base64 --wrap=0" + register: node_cert_data + delegate_to: "{{groups['kube-master'][0]}}" + run_once: true + when: sync_certs|default(false) - name: Gen_certs | Copy certs on masters - copy: - content: "{{ item.content|b64decode }}" - dest: "{{ item.source }}" - with_items: '{{slurp_certs.results}}' + shell: "echo '{{master_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ kube_cert_dir }}" + changed_when: false when: inventory_hostname in groups['kube-master'] and sync_certs|default(false) and inventory_hostname != groups['kube-master'][0] - name: Gen_certs | Copy certs on nodes - copy: - content: "{{ item.content|b64decode }}" - dest: "{{ item.source }}" - with_items: '{{slurp_certs.results}}' - when: item.item in node_certs and - inventory_hostname in groups['kube-node'] and sync_certs|default(false) and + shell: "echo '{{node_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ kube_cert_dir }}" + changed_when: false + when: inventory_hostname in groups['kube-node'] and sync_certs|default(false) and inventory_hostname != groups['kube-master'][0] - name: Gen_certs | check certificate permissions diff --git a/roles/kubernetes/secrets/tasks/gen_tokens.yml b/roles/kubernetes/secrets/tasks/gen_tokens.yml index 796657f65..dbe35811b 100644 --- a/roles/kubernetes/secrets/tasks/gen_tokens.yml +++ b/roles/kubernetes/secrets/tasks/gen_tokens.yml @@ -43,20 +43,15 @@ delegate_to: "{{groups['kube-master'][0]}}" when: sync_tokens|default(false) -- name: Gen_tokens | Get the tokens from first master - slurp: - src: "{{ item }}" - register: slurp_tokens - with_items: '{{tokens_list.stdout_lines}}' - run_once: true +- name: Gen_tokens | Gather tokens + shell: "tar cfz - {{ tokens_list.stdout_lines | join(' ') }} | base64 --wrap=0" + register: tokens_data delegate_to: "{{groups['kube-master'][0]}}" + run_once: true when: sync_tokens|default(false) - notify: set secret_changed - name: Gen_tokens | Copy tokens on masters - copy: - content: "{{ item.content|b64decode }}" - dest: "{{ item.source }}" - with_items: '{{slurp_tokens.results}}' + shell: "echo '{{ tokens_data.stdout|quote }}' | base64 -d | tar xz -C /" + changed_when: false when: inventory_hostname in groups['kube-master'] and sync_tokens|default(false) and inventory_hostname != groups['kube-master'][0] From 2778ac61a432b016cc19f0671dae82f2e2d08fee Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Wed, 26 Oct 2016 17:56:15 +0300 Subject: [PATCH 25/34] Add new var skip_dnsmasq_k8s If skip_dnsmasq is set, it will still not set up dnsmasq k8s pod. This enables independent setup of resolvconf section before kubelet is up. --- roles/dnsmasq/defaults/main.yml | 8 +++++++- roles/dnsmasq/tasks/main.yml | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/roles/dnsmasq/defaults/main.yml b/roles/dnsmasq/defaults/main.yml index 7a1e77023..89ab02ab8 100644 --- a/roles/dnsmasq/defaults/main.yml +++ b/roles/dnsmasq/defaults/main.yml @@ -16,4 +16,10 @@ dnsmasq_version: 2.72 # Images dnsmasq_image_repo: "andyshinn/dnsmasq" -dnsmasq_image_tag: "{{ dnsmasq_version }}" \ No newline at end of file +dnsmasq_image_tag: "{{ dnsmasq_version }}" + +# Skip dnsmasq setup +skip_dnsmasq: false + +# Skip setting up dnsmasq daemonset +skip_dnsmasq_k8s: "{{ skip_dnsmasq }}" diff --git a/roles/dnsmasq/tasks/main.yml b/roles/dnsmasq/tasks/main.yml index 46c1604f6..6b271a1e2 100644 --- a/roles/dnsmasq/tasks/main.yml +++ b/roles/dnsmasq/tasks/main.yml @@ -1,5 +1,5 @@ --- - include: dnsmasq.yml - when: "{{ not skip_dnsmasq|bool }}" + when: "{{ not skip_dnsmasq_k8s|bool }}" - include: resolvconf.yml From 03e162b342072b8229dc61ce569dcdd9e87848ae Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 28 Oct 2016 11:16:11 +0400 Subject: [PATCH 26/34] Update OWNERS --- OWNERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/OWNERS b/OWNERS index 583a0314b..6ecbee5c9 100644 --- a/OWNERS +++ b/OWNERS @@ -4,3 +4,6 @@ owners: - Smana - ant31 + - bogdando + - mattymo + - rsmitty From 50f77cca1d011e75c5b330cca3114ce54c13f502 Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Fri, 28 Oct 2016 14:56:48 +0200 Subject: [PATCH 27/34] Add CI test layouts * Drop Wily from test matrix * Replace the Wily cases dropped with extra cases to test separate roles deployment Signed-off-by: Bogdan Dobrelya --- .travis.yml | 34 ++++++++++++++++++++-------- tests/cloud_playbooks/create-gce.yml | 2 +- tests/cloud_playbooks/delete-gce.yml | 2 +- tests/templates/inventory-gce.j2 | 11 +++++++++ 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 13604765e..863374c26 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,80 +11,92 @@ env: CONTAINER_ENGINE=docker PRIVATE_KEY=$GCE_PRIVATE_KEY ANSIBLE_KEEP_REMOTE_FILES=1 + CLUSTER_MODE=default matrix: # Debian Jessie - >- KUBE_NETWORK_PLUGIN=flannel CLOUD_IMAGE=debian-8-kubespray CLOUD_REGION=europe-west1-b + CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=calico CLOUD_IMAGE=debian-8-kubespray CLOUD_REGION=us-central1-c + CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=debian-8-kubespray CLOUD_REGION=us-east1-d + CLUSTER_MODE=default # Centos 7 - >- KUBE_NETWORK_PLUGIN=flannel CLOUD_IMAGE=centos-7-sudo CLOUD_REGION=asia-east1-c - + CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=calico CLOUD_IMAGE=centos-7-sudo CLOUD_REGION=europe-west1-b - + CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=centos-7-sudo CLOUD_REGION=us-central1-c + CLUSTER_MODE=default # Redhat 7 - >- KUBE_NETWORK_PLUGIN=flannel CLOUD_IMAGE=rhel-7-sudo CLOUD_REGION=us-east1-d - + CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=calico CLOUD_IMAGE=rhel-7-sudo CLOUD_REGION=asia-east1-c - + CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=rhel-7-sudo CLOUD_REGION=europe-west1-b + CLUSTER_MODE=default # Ubuntu 16.04 - >- KUBE_NETWORK_PLUGIN=flannel CLOUD_IMAGE=ubuntu-1604-xenial CLOUD_REGION=us-central1-c + CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=calico CLOUD_IMAGE=ubuntu-1604-xenial CLOUD_REGION=us-east1-d + CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=ubuntu-1604-xenial CLOUD_REGION=asia-east1-c + CLUSTER_MODE=default - # Ubuntu 15.10 + # Extra cases for separated roles - >- KUBE_NETWORK_PLUGIN=flannel - CLOUD_IMAGE=ubuntu-1510-wily + CLOUD_IMAGE=rhel-7-sudo CLOUD_REGION=europe-west1-b + CLUSTER_MODE=separate - >- KUBE_NETWORK_PLUGIN=calico - CLOUD_IMAGE=ubuntu-1510-wily + CLOUD_IMAGE=ubuntu-1604-xenial CLOUD_REGION=us-central1-a + CLUSTER_MODE=separate - >- KUBE_NETWORK_PLUGIN=weave - CLOUD_IMAGE=ubuntu-1510-wily + CLOUD_IMAGE=debian-8-kubespray CLOUD_REGION=us-east1-d + CLUSTER_MODE=separate before_install: @@ -92,7 +104,8 @@ before_install: - pip install --user boto -U - pip install --user ansible - pip install --user netaddr - - pip install --user apache-libcloud + # W/A https://github.com/ansible/ansible-modules-core/issues/5196#issuecomment-253766186 + - pip install --user apache-libcloud==0.20.1 cache: - directories: @@ -114,7 +127,8 @@ before_script: script: - > - $HOME/.local/bin/ansible-playbook tests/cloud_playbooks/create-gce.yml -i tests/local_inventory/hosts -c local $LOG_LEVEL + $HOME/.local/bin/ansible-playbook tests/cloud_playbooks/create-gce.yml -i tests/local_inventory/hosts.cfg -c local $LOG_LEVEL + -e mode=${CLUSTER_MODE} -e test_id=${TEST_ID} -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} -e gce_project_id=${GCE_PROJECT_ID} diff --git a/tests/cloud_playbooks/create-gce.yml b/tests/cloud_playbooks/create-gce.yml index 840cf2e7c..b2c3e3020 100644 --- a/tests/cloud_playbooks/create-gce.yml +++ b/tests/cloud_playbooks/create-gce.yml @@ -1,6 +1,6 @@ --- - hosts: localhost - sudo: False + become: false gather_facts: no vars: cloud_machine_type: g1-small diff --git a/tests/cloud_playbooks/delete-gce.yml b/tests/cloud_playbooks/delete-gce.yml index d42c6cc91..54902fb6f 100644 --- a/tests/cloud_playbooks/delete-gce.yml +++ b/tests/cloud_playbooks/delete-gce.yml @@ -1,6 +1,6 @@ --- - hosts: localhost - sudo: False + become: false gather_facts: no vars: cloud_machine_type: f1-micro diff --git a/tests/templates/inventory-gce.j2 b/tests/templates/inventory-gce.j2 index 72ad469de..418910771 100644 --- a/tests/templates/inventory-gce.j2 +++ b/tests/templates/inventory-gce.j2 @@ -2,6 +2,16 @@ node1 ansible_ssh_host={{gce.instance_data[0].public_ip}} node2 ansible_ssh_host={{gce.instance_data[1].public_ip}} node3 ansible_ssh_host={{gce.instance_data[2].public_ip}} +{% if mode is defined and mode == "separate" %} +[kube-master] +node1 + +[kube-node] +node2 + +[etcd] +node3 +{% else %} [kube-master] node1 node2 @@ -14,6 +24,7 @@ node3 [etcd] node1 node2 +{% endif %} [k8s-cluster:children] kube-node From 9d7142f476e50b34f0f7425adb8da8faf3430fe9 Mon Sep 17 00:00:00 2001 From: Alexander Kanevskiy Date: Fri, 28 Oct 2016 23:26:25 +0300 Subject: [PATCH 28/34] Vagrantfile: use Ubuntu 16.04 LTS Use recent supported version of Ubuntu for local development setup with Vagrant. --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 44f80db8c..4aa9b9180 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -16,7 +16,7 @@ $vm_cpus = 1 $shared_folders = {} $forwarded_ports = {} $subnet = "172.17.8" -$box = "bento/ubuntu-14.04" +$box = "bento/ubuntu-16.04" host_vars = {} From 9f976e568df027b350fd85b26af045984fedba28 Mon Sep 17 00:00:00 2001 From: Alexander Kanevskiy Date: Sat, 29 Oct 2016 00:32:56 +0300 Subject: [PATCH 29/34] Vagrantfile: setup proxy inside virtual machines In corporate networks, it is good to pre-configure proxy variables. --- Vagrantfile | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Vagrantfile b/Vagrantfile index 44f80db8c..0fca78bd6 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -38,6 +38,13 @@ if ! File.exist?(File.join(File.dirname($inventory), "hosts")) end end +if Vagrant.has_plugin?("vagrant-proxyconf") + $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost" + (1..$num_instances).each do |i| + $no_proxy += ",#{$subnet}.#{i+100}" + end +end + Vagrant.configure("2") do |config| # always use Vagrants insecure key config.ssh.insert_key = false @@ -52,6 +59,12 @@ Vagrant.configure("2") do |config| config.vm.define vm_name = "%s-%02d" % [$instance_name_prefix, i] do |config| config.vm.hostname = vm_name + if Vagrant.has_plugin?("vagrant-proxyconf") + config.proxy.http = ENV['HTTP_PROXY'] || ENV['http_proxy'] || "" + config.proxy.https = ENV['HTTPS_PROXY'] || ENV['https_proxy'] || "" + config.proxy.no_proxy = $no_proxy + end + if $expose_docker_tcp config.vm.network "forwarded_port", guest: 2375, host: ($expose_docker_tcp + i - 1), auto_correct: true end From 2ca6819cdf9473dd9ddf6ba7efa2e73a37d8d787 Mon Sep 17 00:00:00 2001 From: Jan Jungnickel Date: Mon, 31 Oct 2016 10:15:10 +0100 Subject: [PATCH 30/34] Reload docker.socket after installing flannel on coreos Workaround for #569 --- roles/docker/handlers/main.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/roles/docker/handlers/main.yml b/roles/docker/handlers/main.yml index 6f54f33d5..5a9037688 100644 --- a/roles/docker/handlers/main.yml +++ b/roles/docker/handlers/main.yml @@ -4,6 +4,7 @@ notify: - Docker | reload systemd - Docker | reload docker + - Docker | reload docker.socket - Docker | pause while Docker restarts - Docker | wait for docker @@ -16,6 +17,12 @@ name: docker state: restarted +- name: Docker | reload docker.socket + service: + name: docker.socket + state: restarted + when: ansible_os_family == 'CoreOS' + - name: Docker | pause while Docker restarts pause: seconds=10 prompt="Waiting for docker restart" From f9355ea14d08c0f2b2842a7da2da74a11b2b5139 Mon Sep 17 00:00:00 2001 From: Jan Jungnickel Date: Tue, 1 Nov 2016 13:08:21 +0100 Subject: [PATCH 31/34] Swap order in which we reload docker/socket --- roles/docker/handlers/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/docker/handlers/main.yml b/roles/docker/handlers/main.yml index 5a9037688..04d761796 100644 --- a/roles/docker/handlers/main.yml +++ b/roles/docker/handlers/main.yml @@ -3,8 +3,8 @@ command: /bin/true notify: - Docker | reload systemd - - Docker | reload docker - Docker | reload docker.socket + - Docker | reload docker - Docker | pause while Docker restarts - Docker | wait for docker From 3b2554217b1bf73126bfc2251e70e4679664ff48 Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Thu, 3 Nov 2016 13:57:33 +0100 Subject: [PATCH 32/34] Upload logs to GCS after failure Delete configure logs script as not needed Rework collect info script defaults Signed-off-by: Bogdan Dobrelya --- .travis.yml | 14 +++++--- scripts/collect-info.yaml | 23 ++++++------- scripts/configure-logs.yaml | 39 ----------------------- tests/cloud_playbooks/upload-logs-gcs.yml | 28 ++++++++++++++++ 4 files changed, 48 insertions(+), 56 deletions(-) delete mode 100644 scripts/configure-logs.yaml create mode 100644 tests/cloud_playbooks/upload-logs-gcs.yml diff --git a/.travis.yml b/.travis.yml index 863374c26..42c32ae5b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,8 @@ env: TEST_ID=$TRAVIS_JOB_NUMBER CONTAINER_ENGINE=docker PRIVATE_KEY=$GCE_PRIVATE_KEY + GS_ACCESS_KEY_ID=$GS_KEY + GS_SECRET_ACCESS_KEY=$GS_SECRET ANSIBLE_KEEP_REMOTE_FILES=1 CLUSTER_MODE=default matrix: @@ -122,8 +124,6 @@ before_script: - $HOME/.local/bin/ansible-playbook --version - cp tests/ansible.cfg . # - "echo $HOME/.local/bin/ansible-playbook -i inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}' $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml" - ## Configure ansible deployment logs to be collected as an artifact. Enable when GCS configured, see https://docs.travis-ci.com/user/deployment/gcs -# - $HOME/.local/bin/ansible-playbook -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scripts/configure-logs.yaml script: - > @@ -147,8 +147,14 @@ script: - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/020_check-create-pod.yml $LOG_LEVEL ## Ping the between 2 pod - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL - ## Collect env info, enable it once GCS configured, see https://docs.travis-ci.com/user/deployment/gcs -# - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scripts/collect-info.yaml + +after_failure: + - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scripts/collect-info.yaml >/dev/null + - > + $HOME/.local/bin/ansible-playbook tests/cloud_playbooks/upload-logs-gcs.yml -i "localhost," -c local + -e test_id=${TEST_ID} + -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} + >/dev/null after_script: - > diff --git a/scripts/collect-info.yaml b/scripts/collect-info.yaml index 67d4c8b35..877b5bf36 100644 --- a/scripts/collect-info.yaml +++ b/scripts/collect-info.yaml @@ -6,16 +6,10 @@ vars: debug: false commands: - - name: git_info - cmd: find . -type d -name .git -execdir sh -c 'gen-gitinfos.sh global|head -12' \; - name: timedate_info cmd: timedatectl status - - name: space_info - cmd: df -h - name: kernel_info cmd: uname -r - - name: distro_info - cmd: cat /etc/issue.net - name: docker_info cmd: docker info - name: ip_info @@ -24,23 +18,26 @@ cmd: ip ro - name: proc_info cmd: ps auxf | grep -v ]$ - - name: systemctl_info - cmd: systemctl status - name: systemctl_failed_info cmd: systemctl --state=failed --no-pager - name: k8s_info cmd: kubectl get all --all-namespaces -o wide - name: errors_info cmd: journalctl -p err --utc --no-pager + - name: etcd_info + cmd: etcdctl --debug cluster-health logs: - - /var/log/ansible.log - - /var/log/ansible/ansible.log - /var/log/syslog - /var/log/daemon.log - /var/log/kern.log - - inventory/inventory.ini - - cluster.yml + - /var/log/dpkg.log + - /var/log/apt/history.log + - /var/log/yum.log + - /var/log/calico/bird/current + - /var/log/calico/bird6/current + - /var/log/calico/felix/current + - /var/log/calico/confd/current tasks: - name: Storing commands output @@ -50,7 +47,7 @@ with_items: "{{commands}}" - debug: var=item - with_items: output.results + with_items: "{{output.results}}" when: debug - name: Fetch results diff --git a/scripts/configure-logs.yaml b/scripts/configure-logs.yaml deleted file mode 100644 index d093e9279..000000000 --- a/scripts/configure-logs.yaml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- hosts: localhost - become: true - gather_facts: no - - vars: - log_path: /var/log/ansible/ - conf_file: /etc/ansible/ansible.cfg - human_readable_plugin: false - callback_plugin_path: /usr/share/ansible/plugins/callback - - tasks: - - name: LOGS | ensure log path - file: path="{{log_path}}" state=directory owner={{ansible_ssh_user}} - - - name: LOGS | ensure plugin path - file: path="{{callback_plugin_path}}" state=directory owner={{ansible_ssh_user}} - when: human_readable_plugin - - - name: LOGS | get plugin - git: repo=https://gist.github.com/cd706de198c85a8255f6.git dest=/tmp/cd706de198c85a8255f6 - when: human_readable_plugin - - - name: LOGS | install plugin - copy: src=/tmp/cd706de198c85a8255f6/human_log.py dest="{{callback_plugin_path}}" - when: human_readable_plugin - - - name: LOGS | config - lineinfile: - line: "log_path={{log_path}}/ansible.log" - regexp: "^#log_path|^log_path" - dest: "{{conf_file}}" - - - name: LOGS | callback plugin - lineinfile: - line: "callback_plugins={{callback_plugin_path}}" - regexp: "^#callback_plugins|^callback_plugins" - dest: "{{conf_file}}" - when: human_readable_plugin diff --git a/tests/cloud_playbooks/upload-logs-gcs.yml b/tests/cloud_playbooks/upload-logs-gcs.yml new file mode 100644 index 000000000..7a7a022c9 --- /dev/null +++ b/tests/cloud_playbooks/upload-logs-gcs.yml @@ -0,0 +1,28 @@ +--- +- hosts: localhost + become: false + gather_facts: no + + vars: + expire: 72000 + + tasks: + - name: replace_test_id + set_fact: + test_name: "{{ test_id | regex_replace('\\.', '-') }}" + + - name: Create a bucket + gc_storage: + bucket: "{{ test_name }}" + mode: create + expiration: "{{ expire }}" + permission: private + + - name: Upload collected diagnostic info + gc_storage: + bucket: "{{ test_name }}" + mode: put + permission: private + expiration: "{{ expire }}" + object: "build-{{ test_name }}-{{ kube_network_plugin }}-logs.tar.gz" + src: logs.tar.gz From 8f20d90f8814cb9be0624321101419bd8fa291b6 Mon Sep 17 00:00:00 2001 From: Spencer Smith Date: Fri, 4 Nov 2016 12:54:35 -0400 Subject: [PATCH 33/34] update admission controllers for > 1.4 --- .../master/templates/manifests/kube-apiserver.manifest.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index ddd6f2085..4100e8a34 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -16,7 +16,7 @@ spec: - --etcd-quorum-read=true - --insecure-bind-address={{ kube_apiserver_insecure_bind_address }} - --apiserver-count={{ kube_apiserver_count }} - - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,ServiceAccount,ResourceQuota + - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota - --service-cluster-ip-range={{ kube_service_addresses }} - --client-ca-file={{ kube_cert_dir }}/ca.pem - --basic-auth-file={{ kube_users_dir }}/known_users.csv From 39b8336f3fff9b43d1bd35c52085cd8196f342cd Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Fri, 4 Nov 2016 17:53:34 +0100 Subject: [PATCH 34/34] Fix upload logs Signed-off-by: Bogdan Dobrelya --- .travis.yml | 3 ++- tests/cloud_playbooks/upload-logs-gcs.yml | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 42c32ae5b..ac922fd80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -154,7 +154,8 @@ after_failure: $HOME/.local/bin/ansible-playbook tests/cloud_playbooks/upload-logs-gcs.yml -i "localhost," -c local -e test_id=${TEST_ID} -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} - >/dev/null + -e gs_key=${GS_ACCESS_KEY_ID} + -e gs_skey=${GS_SECRET_ACCESS_KEY} after_script: - > diff --git a/tests/cloud_playbooks/upload-logs-gcs.yml b/tests/cloud_playbooks/upload-logs-gcs.yml index 7a7a022c9..ba31ab56f 100644 --- a/tests/cloud_playbooks/upload-logs-gcs.yml +++ b/tests/cloud_playbooks/upload-logs-gcs.yml @@ -17,6 +17,8 @@ mode: create expiration: "{{ expire }}" permission: private + gs_access_key: gs_key + gs_secret_key: gs_skey - name: Upload collected diagnostic info gc_storage: @@ -26,3 +28,16 @@ expiration: "{{ expire }}" object: "build-{{ test_name }}-{{ kube_network_plugin }}-logs.tar.gz" src: logs.tar.gz + gs_access_key: gs_key + gs_secret_key: gs_skey + + - name: Get a link + gc_storage: + bucket: "{{ test_name }}" + object: "build-{{ test_name }}-{{ kube_network_plugin }}-logs.tar.gz" + mode: get_url + register: url + gs_access_key: gs_key + gs_secret_key: gs_skey + + - debug: msg="Download URL {{get_url}}"