Hello community,
here is the log from the commit of package kubernetes-salt for openSUSE:Factory checked in at 2018-04-27 16:10:18
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/kubernetes-salt (Old)
and /work/SRC/openSUSE:Factory/.kubernetes-salt.new (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "kubernetes-salt"
Fri Apr 27 16:10:18 2018 rev:15 rq:601903 version:3.0.0+git_r742_8508870
Changes:
--------
--- /work/SRC/openSUSE:Factory/kubernetes-salt/kubernetes-salt.changes 2018-04-24 15:34:13.207854421 +0200
+++ /work/SRC/openSUSE:Factory/.kubernetes-salt.new/kubernetes-salt.changes 2018-04-27 16:10:20.752038484 +0200
@@ -1,0 +2,84 @@
+Thu Apr 26 09:56:06 UTC 2018 - containers-bugowner@suse.de
+
+- Commit c3b81a6 by Flavio Castelli fcastelli@suse.com
+ Ensure swap is disabled before kubelet is started
+
+ We have to ensure the swap state is executed before the kubelet service is
+ started, otherwise kubelt won't run and this will lead to issues like the
+ ones causing bsc#1090337
+
+ Signed-off-by: Flavio Castelli
+
+
+-------------------------------------------------------------------
+Wed Apr 25 12:10:02 UTC 2018 - containers-bugowner@suse.de
+
+- Commit 24bea3d by Nirmoy Das ndas@suse.de
+ cni: add cilium as alternate to flannel plugin
+
+
+-------------------------------------------------------------------
+Tue Apr 24 15:58:27 UTC 2018 - containers-bugowner@suse.de
+
+- Commit 1fd2a98 by Alvaro Saurin alvaro.saurin@gmail.com
+ Remove leftover file
+
+ feature#node_removal
+
+
+-------------------------------------------------------------------
+Tue Apr 24 09:21:14 UTC 2018 - containers-bugowner@suse.de
+
+- Commit e1b9c75 by Kiall Mac Innes kiall@macinnes.ie
+ Update tiller tag to 2.8.2
+
+ This matches the tag used in the updated image via SR#162727.
+
+
+-------------------------------------------------------------------
+Tue Apr 24 08:42:18 UTC 2018 - containers-bugowner@suse.de
+
+- Commit 3e70e4f by Alvaro Saurin alvaro.saurin@gmail.com
+ Use get_with_expr()
+
+ feature#node_removal
+
+ Commit b4d09dd by Alvaro Saurin alvaro.saurin@gmail.com
+ Convert integers in the pillar to real integers. Unit tests for the
+ get_pillar() function.
+
+ See https://trello.com/c/O7daOErL
+
+ feature#node_removal
+
+ Commit 0d65d79 by Alvaro Saurin alvaro.saurin@gmail.com
+ Fix: do not include the current node in the list
+ of endpoints when adding a new member. Unit tests for the etcd modoule.
+
+ See https://trello.com/c/O7daOErL
+
+ feature#node_removal
+
+ Commit 399f7ea by Alvaro Saurin alvaro.saurin@gmail.com
+ Try to resist unresponsive nodes when removing a node.
+ * the replacement will not be chosen from
+ the unresponsive nodes
+ * affected nodes will exclude them too. Possibility to skip any action on
+ the target (with the `skip` pillar), so we can remove unresponsive targets
+ while still looking for replacements.
+
+ See https://trello.com/c/O7daOErL
+
+ feature#node_removal
+
+
+-------------------------------------------------------------------
+Tue Apr 24 07:58:31 UTC 2018 - containers-bugowner@suse.de
+
+- Commit f80f752 by Alvaro Saurin alvaro.saurin@gmail.com
+ Don't to remove some things that are not so important.
+
+ feature#node_removal
+
+
+-------------------------------------------------------------------
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ kubernetes-salt.spec ++++++
--- /var/tmp/diff_new_pack.UxiAiM/_old 2018-04-27 16:10:21.396014867 +0200
+++ /var/tmp/diff_new_pack.UxiAiM/_new 2018-04-27 16:10:21.400014720 +0200
@@ -32,7 +32,7 @@
Name: kubernetes-salt
%define gitrepo salt
-Version: 3.0.0+git_r727_68280fa
+Version: 3.0.0+git_r742_8508870
Release: 0
BuildArch: noarch
Summary: Production-Grade Container Scheduling and Management
++++++ master.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/pillar/certificates.sls new/salt-master/pillar/certificates.sls
--- old/salt-master/pillar/certificates.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/pillar/certificates.sls 2018-04-26 11:56:10.000000000 +0200
@@ -50,3 +50,6 @@
kube_proxy_key: '/etc/pki/kube-proxy.key'
kube_proxy_crt: '/etc/pki/kube-proxy.crt'
+
+ cilium_key: '/etc/pki/cilium.key'
+ cilium_crt: '/etc/pki/cilium.crt'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/_modules/caasp_etcd.py new/salt-master/salt/_modules/caasp_etcd.py
--- old/salt-master/salt/_modules/caasp_etcd.py 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/_modules/caasp_etcd.py 2018-04-26 11:56:10.000000000 +0200
@@ -173,6 +173,7 @@
error('no etcd members available!!')
raise NoEtcdServersException()
+ etcd_members_lst.sort()
return sep.join(etcd_members_lst)
@@ -221,7 +222,7 @@
return member_line.split(':')[0]
except Exception as e:
- error("cannot get member ID: %s", e)
- error("output: %s", members_output)
+ error('cannot get member ID for "%s": %s', e, this_nodename)
+ error('output: %s', members_output)
return ''
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/_modules/caasp_nodes.py new/salt-master/salt/_modules/caasp_nodes.py
--- old/salt-master/salt/_modules/caasp_nodes.py 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/_modules/caasp_nodes.py 2018-04-26 11:56:10.000000000 +0200
@@ -100,9 +100,10 @@
}
-# filter out empty/None and sort a list
+# for a list `lst`, filter out empty/None, remove duplicates and sort it
def _sanitize_list(lst):
res = [x for x in lst if x]
+ res = list(set(res))
res.sort()
return res
@@ -240,6 +241,9 @@
elif replacement_provided and replacement in forbidden:
abort('%s cannot be replaced by %s: the replacement has a "ca" or "admin" role',
target, replacement)
+ elif replacement_provided and replacement in excluded:
+ abort('%s cannot be replaced by %s: the replacement is in the list of nodes excluded',
+ target, replacement)
masters = get_from_args_or_with_expr(
'masters', kwargs, 'G@roles:kube-master')
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/_modules/caasp_pillar.py new/salt-master/salt/_modules/caasp_pillar.py
--- old/salt-master/salt/_modules/caasp_pillar.py 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/_modules/caasp_pillar.py 2018-04-26 11:56:10.000000000 +0200
@@ -20,9 +20,16 @@
res = __salt__['pillar.get'](name, None)
if res is None:
res = default
+
if isinstance(res, basestring):
+ try:
+ return int(res)
+ except ValueError:
+ pass
+
if res.lower() in ["true", "yes", "on"]:
return True
elif res.lower() in ["false", "no", "off"]:
return False
+
return res
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/_modules/tests/test_caasp_etcd.py new/salt-master/salt/_modules/tests/test_caasp_etcd.py
--- old/salt-master/salt/_modules/tests/test_caasp_etcd.py 1970-01-01 01:00:00.000000000 +0100
+++ new/salt-master/salt/_modules/tests/test_caasp_etcd.py 2018-04-26 11:56:10.000000000 +0200
@@ -0,0 +1,55 @@
+from __future__ import absolute_import
+
+import unittest
+
+import caasp_etcd
+from caasp_etcd import ETCD_CLIENT_PORT, get_endpoints
+from caasp_log import ExecutionAborted
+
+try:
+ from mock import patch, MagicMock
+except ImportError:
+ _mocking_lib_available = False
+else:
+ _mocking_lib_available = True
+
+
+caasp_etcd.__salt__ = {}
+
+
+class TestGetEndpoints(unittest.TestCase):
+ '''
+ Some basic tests for get_from_args_or_with_expr()
+ '''
+
+ def test_get_endpoints(self):
+ nodes = {
+ 'AAA': 'node1',
+ 'BBB': 'node2',
+ 'CCC': 'node3'
+ }
+
+ mock = MagicMock(return_value=nodes)
+ with patch.dict(caasp_etcd.__salt__, {'caasp_grains.get': mock}):
+ res = get_endpoints()
+ mock.assert_called_once_with('G@roles:etcd')
+
+ for i in nodes.values():
+ self.assertIn('https://{}:{}'.format(i, ETCD_CLIENT_PORT), res,
+ 'did not get the expected list of etcd endpoints: {}'.format(res))
+
+ mock.reset_mock()
+
+ res = get_endpoints(with_id=True)
+ mock.assert_called_once_with('G@roles:etcd')
+
+ for (j, k) in nodes.items():
+ self.assertIn('{}=https://{}:{}'.format(j, k, ETCD_CLIENT_PORT), res,
+ 'did not get the expected list of etcd endpoints: {}'.format(res))
+
+ mock.reset_mock()
+
+ res = get_endpoints(skip_removed=True)
+ mock.assert_called_once_with('G@roles:etcd and not G@removal_in_progress:true')
+
+ mock.reset_mock()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/_modules/tests/test_caasp_nodes.py new/salt-master/salt/_modules/tests/test_caasp_nodes.py
--- old/salt-master/salt/_modules/tests/test_caasp_nodes.py 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/_modules/tests/test_caasp_nodes.py 2018-04-26 11:56:10.000000000 +0200
@@ -212,6 +212,13 @@
self.assertIn('kube-minion', roles,
'kube-minion role not found in replacement')
+ # check we cannot use an excluded node
+ with self.assertRaises(ExecutionAborted):
+ replacement, roles = get_replacement_for(self.minion_1,
+ replacement=self.minion_3,
+ excluded=[self.minion_3],
+ **self.get_replacement_for_kwargs)
+
def test_invalid_etcd_replacement(self):
'''
Check get_replacement_for() realizes a minion
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/_modules/tests/test_caasp_pillar.py new/salt-master/salt/_modules/tests/test_caasp_pillar.py
--- old/salt-master/salt/_modules/tests/test_caasp_pillar.py 1970-01-01 01:00:00.000000000 +0100
+++ new/salt-master/salt/_modules/tests/test_caasp_pillar.py 2018-04-26 11:56:10.000000000 +0200
@@ -0,0 +1,49 @@
+from __future__ import absolute_import
+
+import unittest
+
+import caasp_pillar
+from caasp_log import ExecutionAborted
+from caasp_pillar import get as get_pillar
+
+try:
+ from mock import patch, MagicMock
+except ImportError:
+ _mocking_lib_available = False
+else:
+ _mocking_lib_available = True
+
+
+caasp_pillar.__salt__ = {}
+
+
+class TestGetPillar(unittest.TestCase):
+ '''
+ Some basic tests for caasp_pillar.get()
+ '''
+
+ def test_get_pillar(self):
+
+ mock = MagicMock()
+ with patch.dict(caasp_pillar.__salt__, {'pillar.get': mock}):
+ # check we get a integer
+ mock.return_value = '123'
+ res = get_pillar('some_int_pillar')
+ self.assertTrue(isinstance(res, int),
+ 'expected to get a integer: {}'.format(res))
+ mock.reset_mock()
+
+ # check we get a boolean
+ for value in ['true', 'on', 'TRUE']:
+ mock.return_value = value
+ res = get_pillar('some_bool_pillar')
+ self.assertTrue(isinstance(res, bool),
+ 'expected to get a bool: {}'.format(res))
+ mock.reset_mock()
+
+ # check we get a string
+ mock.return_value = 'something'
+ res = get_pillar('some_str_pillar')
+ self.assertTrue(isinstance(res, str),
+ 'expected to get a string: {}'.format(res))
+ mock.reset_mock()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/_states/caasp_etcd.py new/salt-master/salt/_states/caasp_etcd.py
--- old/salt-master/salt/_states/caasp_etcd.py 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/_states/caasp_etcd.py 2018-04-26 11:56:10.000000000 +0200
@@ -21,13 +21,22 @@
def etcdctl(name, retry={}, **kwargs):
'''
Run an etcdctl command
+
+ Arguments:
+
+ In addition to all the arguments supported by the `caasp_cmd.run` state.
+
+ * `skip_this`: (optional) skip current node when calculating the list of etcd endpoints.
+
'''
retry_ = {'attempts': DEFAULT_ATTEMPTS,
'interval': DEFAULT_ATTEMPTS_INTERVAL,
'until': None}
retry_.update(retry)
- args = __salt__['caasp_etcd.get_etcdctl_args_str']()
+ skip_this = kwargs.pop('skip_this', False)
+
+ args = __salt__['caasp_etcd.get_etcdctl_args_str'](skip_this=skip_this)
cmd = 'etcdctl {} {}'.format(args, name)
log.debug('CaaS: running etcdctl as: %s', cmd)
@@ -53,7 +62,7 @@
name = 'member add {} {}'.format(this_id, this_peer_url)
log.debug('CaaS: adding etcd member')
- return etcdctl(name=name, **kwargs)
+ return etcdctl(name=name, skip_this=True, **kwargs)
# once the member has been added to the cluster, we
# must make sure etcd joins an "existing" cluster.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/addons/remove-pre-reboot.sls new/salt-master/salt/addons/remove-pre-reboot.sls
--- old/salt-master/salt/addons/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/addons/remove-pre-reboot.sls 1970-01-01 01:00:00.000000000 +0100
@@ -1,3 +0,0 @@
-
-/etc/kubernetes/addons/namespace.yaml:
- file.absent
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/addons/tiller/manifests/20-deployment.yaml new/salt-master/salt/addons/tiller/manifests/20-deployment.yaml
--- old/salt-master/salt/addons/tiller/manifests/20-deployment.yaml 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/addons/tiller/manifests/20-deployment.yaml 2018-04-26 11:56:10.000000000 +0200
@@ -34,7 +34,7 @@
- env:
- name: TILLER_NAMESPACE
value: kube-system
- image: sles12/tiller:2.7.2
+ image: sles12/tiller:2.8.2
imagePullPolicy: IfNotPresent
livenessProbe:
httpGet:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cert/remove-pre-reboot.sls new/salt-master/salt/cert/remove-pre-reboot.sls
--- old/salt-master/salt/cert/remove-pre-reboot.sls 1970-01-01 01:00:00.000000000 +0100
+++ new/salt-master/salt/cert/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200
@@ -0,0 +1,6 @@
+
+{{ pillar['ssl']['crt_file'] }}:
+ file.absent
+
+{{ pillar['ssl']['key_file'] }}:
+ file.absent
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cleanup/remove-post-orchestration.sls new/salt-master/salt/cleanup/remove-post-orchestration.sls
--- old/salt-master/salt/cleanup/remove-post-orchestration.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/cleanup/remove-post-orchestration.sls 2018-04-26 11:56:10.000000000 +0200
@@ -10,7 +10,7 @@
# k8s cluster
###############
-{%- set k8s_nodes = salt['mine.get']('roles:(kube-master|kube-minion)', 'nodename', expr_form='grain_pcre').keys() %}
+{%- set k8s_nodes = salt.caasp_nodes.get_with_expr('G@roles:kube-master', booted=True) %}
{%- if forced or target in k8s_nodes %}
{%- from '_macros/kubectl.jinja' import kubectl with context %}
@@ -24,7 +24,7 @@
# etcd node
###############
-{%- set etcd_members = salt['mine.get']('roles:etcd', 'nodename', expr_form='grain').keys() %}
+{%- set etcd_members = salt.caasp_nodes.get_with_expr('G@roles:etcd', booted=True) %}
{%- if forced or target in etcd_members %}
etcd-remove-member:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cleanup/remove-pre-reboot.sls new/salt-master/salt/cleanup/remove-pre-reboot.sls
--- old/salt-master/salt/cleanup/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/cleanup/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200
@@ -2,24 +2,10 @@
# try to remove some dirs that could contain sensitive
# information, even when they were not directly managed by us
-wipe-etc-kubernetes:
- cmd.run:
- - name: rm -rf /etc/kubernetes/*
-
wipe-certificates:
cmd.run:
- name: rm -rf /var/lib/ca-certificates/*
-# remove some logs that could contain sensitive information
-wipe-var-log:
- cmd.run:
- - name: |-
- for f in apparmor audit containers faillog firewall localmessages pods zypper.log YaST2 ; do
- rm -rf /var/log/$f
- done
- # NOTE: do not try to remove /var/log/salt
- # or the Salt minion will crash...
-
# cleanup all the Salt things we can
# NOTE: we must be careful (or Salt will stop working)
cleanup-salt:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cni/cilium/init.sls new/salt-master/salt/cni/cilium/init.sls
--- old/salt-master/salt/cni/cilium/init.sls 1970-01-01 01:00:00.000000000 +0100
+++ new/salt-master/salt/cni/cilium/init.sls 2018-04-26 11:56:10.000000000 +0200
@@ -0,0 +1,21 @@
+{% set plugin = salt['pillar.get']('cni:plugin', 'cilium').lower() %}
+{% if plugin == "cilium" %}
+
+include:
+ - ca-cert
+ - cert
+ - crypto
+
+{% from '_macros/certs.jinja' import certs with context %}
+{{ certs("cilium",
+ pillar['ssl']['cilium_crt'],
+ pillar['ssl']['cilium_key'],
+ cn = grains['nodename'],
+ o = 'system:nodes') }}
+
+{% else %}
+{# See https://github.com/saltstack/salt/issues/14553 #}
+dummy_step:
+ cmd.run:
+ - name: "echo saltstack bug 14553"
+{% endif %}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cni/cilium-config.yaml.jinja new/salt-master/salt/cni/cilium-config.yaml.jinja
--- old/salt-master/salt/cni/cilium-config.yaml.jinja 1970-01-01 01:00:00.000000000 +0100
+++ new/salt-master/salt/cni/cilium-config.yaml.jinja 2018-04-26 11:56:10.000000000 +0200
@@ -0,0 +1,30 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+ name: cilium-config
+ namespace: kube-system
+data:
+ # This etcd-config contains the etcd endpoints of your cluster. If you use
+ # TLS please make sure you uncomment the ca-file line and add the respective
+ # certificate has a k8s secret, see explanation below in the comment labeled
+ # "ETCD-CERT"
+ etcd-config: |-
+ ---
+ endpoints:
+{%- for endpoints in salt.caasp_etcd.get_endpoints().split(',') %}
+ - {{ endpoints }}
+{%- endfor %}
+ #
+ # In case you want to use TLS in etcd, uncomment the following line
+ # and add the certificate as explained in the comment labeled "ETCD-CERT"
+ ca-file: '/etc/pki/trust/anchors/SUSE_CaaSP_CA.crt'
+ #
+ # In case you want client to server authentication, uncomment the following
+ # lines and add the certificate and key in cilium-etcd-secrets below
+ key-file: '{{ cilium_key }}'
+ cert-file: '{{ cilium_certificate }}'
+
+ # If you want to run cilium in debug mode change this value to true
+ debug: "false"
+ disable-ipv4: "false"
+
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cni/cilium-ds.yaml.jinja new/salt-master/salt/cni/cilium-ds.yaml.jinja
--- old/salt-master/salt/cni/cilium-ds.yaml.jinja 1970-01-01 01:00:00.000000000 +0100
+++ new/salt-master/salt/cni/cilium-ds.yaml.jinja 2018-04-26 11:56:10.000000000 +0200
@@ -0,0 +1,171 @@
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+ name: cilium
+ namespace: kube-system
+spec:
+ updateStrategy:
+ type: "RollingUpdate"
+ rollingUpdate:
+ # Specifies the maximum number of Pods that can be unavailable during the update process.
+ # The current default value is 1 or 100% for daemonsets; Adding an explicit value here
+ # to avoid confusion, as the default value is specific to the type (daemonset/deployment).
+ maxUnavailable: "100%"
+ selector:
+ matchLabels:
+ k8s-app: cilium
+ kubernetes.io/cluster-service: "true"
+ template:
+ metadata:
+ labels:
+ k8s-app: cilium
+ kubernetes.io/cluster-service: "true"
+ annotations:
+ # This annotation plus the CriticalAddonsOnly toleration makes
+ # cilium to be a critical pod in the cluster, which ensures cilium
+ # gets priority scheduling.
+ # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-cr...
+ scheduler.alpha.kubernetes.io/critical-pod: ''
+ scheduler.alpha.kubernetes.io/tolerations: >-
+ [{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]
+ spec:
+ serviceAccountName: cilium
+ initContainers:
+ - name: install-cni-conf
+ image: {{ pillar['cilium']['image'] }}
+ command:
+ - /bin/sh
+ - "-c"
+ - "cp -f /etc/cni/net.d/10-cilium-cni.conf /host/etc/cni/net.d/10-cilium-cni.conf"
+ volumeMounts:
+ - name: host-cni-conf
+ mountPath: /host/etc/cni/net.d
+ - name: install-cni-bin
+ image: {{ pillar['cilium']['image'] }}
+ command:
+ - /bin/sh
+ - "-c"
+ - "cp -f /usr/lib/cni/* /host/opt/cni/bin/"
+ volumeMounts:
+ - name: host-cni-bin
+ mountPath: /host/opt/cni/bin/
+
+ containers:
+ - image: {{ pillar['cilium']['image'] }}
+ imagePullPolicy: IfNotPresent
+ name: cilium-agent
+ command: [ "cilium-agent" ]
+ args:
+ - "--debug=$(CILIUM_DEBUG)"
+ - "--disable-envoy-version-check"
+ - "-t=vxlan"
+ - "--kvstore=etcd"
+ - "--kvstore-opt=etcd.config=/var/lib/etcd-config/etcd.config"
+ - "--disable-ipv4=$(DISABLE_IPV4)"
+ ports:
+ - name: prometheus
+ containerPort: 9090
+ lifecycle:
+ preStop:
+ exec:
+ command:
+ - "rm -f /host/etc/cni/net.d/10-cilium-cni.conf /host/opt/cni/bin/cilium-cni"
+ env:
+ - name: "K8S_NODE_NAME"
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ - name: "CILIUM_DEBUG"
+ valueFrom:
+ configMapKeyRef:
+ name: cilium-config
+ key: debug
+ - name: "DISABLE_IPV4"
+ valueFrom:
+ configMapKeyRef:
+ name: cilium-config
+ key: disable-ipv4
+ livenessProbe:
+ exec:
+ command:
+ - cilium
+ - status
+ # The initial delay for the liveness probe is intentionally large to
+ # avoid an endless kill & restart cycle if in the event that the initial
+ # bootstrapping takes longer than expected.
+ initialDelaySeconds: 120
+ failureThreshold: 10
+ periodSeconds: 10
+ readinessProbe:
+ exec:
+ command:
+ - cilium
+ - status
+ initialDelaySeconds: 5
+ periodSeconds: 5
+ volumeMounts:
+ - name: bpf-maps
+ mountPath: /sys/fs/bpf
+ - name: cilium-run
+ mountPath: /var/run/cilium
+ - name: host-cni-bin
+ mountPath: /host/opt/cni/bin/
+ - name: host-cni-conf
+ mountPath: /host/etc/cni/net.d
+ - name: docker-socket
+ mountPath: /var/run/docker.sock
+ readOnly: true
+ - name: etcd-config-path
+ mountPath: /var/lib/etcd-config
+ readOnly: true
+ - name: etcd-certs
+ mountPath: /etc/pki
+ readOnly: true
+ securityContext:
+ capabilities:
+ add:
+ - "NET_ADMIN"
+ privileged: true
+ hostNetwork: true
+ volumes:
+ # To keep state between restarts / upgrades
+ - name: cilium-run
+ hostPath:
+ path: /var/run/cilium
+ # To keep state between restarts / upgrades
+ - name: bpf-maps
+ hostPath:
+ path: /sys/fs/bpf
+ # To read docker events from the node
+ - name: docker-socket
+ hostPath:
+ path: /var/run/docker.sock
+ # To install cilium cni plugin in the host
+ - name: host-cni-bin
+ hostPath:
+ path: {{ pillar['cni']['dirs']['bin'] }}
+ # To install cilium cni configuration in the host
+ - name: host-cni-conf
+ hostPath:
+ path: {{ pillar['cni']['dirs']['conf'] }}
+ # To read the etcd config stored in config maps
+ - name: etcd-config-path
+ configMap:
+ name: cilium-config
+ items:
+ - key: etcd-config
+ path: etcd.config
+ - name: etcd-certs
+ hostPath:
+ path: /etc/pki
+ restartPolicy: Always
+ tolerations:
+ - effect: NoSchedule
+ key: node-role.kubernetes.io/master
+ - effect: NoSchedule
+ key: node.cloudprovider.kubernetes.io/uninitialized
+ value: "true"
+ # Mark cilium's pod as critical for rescheduling
+ - key: CriticalAddonsOnly
+ operator: "Exists"
+
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cni/cilium-rbac.yaml.jinja new/salt-master/salt/cni/cilium-rbac.yaml.jinja
--- old/salt-master/salt/cni/cilium-rbac.yaml.jinja 1970-01-01 01:00:00.000000000 +0100
+++ new/salt-master/salt/cni/cilium-rbac.yaml.jinja 2018-04-26 11:56:10.000000000 +0200
@@ -0,0 +1,98 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: cilium
+ namespace: kube-system
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: cilium
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: cilium
+subjects:
+- kind: ServiceAccount
+ name: cilium
+ namespace: kube-system
+- kind: Group
+ name: system:nodes
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: suse:caasp:psp:cilium
+roleRef:
+ kind: ClusterRole
+ name: suse:caasp:psp:privileged
+ apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: ServiceAccount
+ name: cilium
+ namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: cilium
+rules:
+- apiGroups:
+ - "networking.k8s.io"
+ resources:
+ - networkpolicies
+ verbs:
+ - get
+ - list
+ - watch
+- apiGroups:
+ - ""
+ resources:
+ - namespaces
+ - services
+ - nodes
+ - endpoints
+ - componentstatuses
+ verbs:
+ - get
+ - list
+ - watch
+- apiGroups:
+ - ""
+ resources:
+ - pods
+ - nodes
+ verbs:
+ - get
+ - list
+ - watch
+ - update
+- apiGroups:
+ - extensions
+ resources:
+ - networkpolicies #FIXME remove this when we drop support for k8s NP-beta GH-1202
+ - thirdpartyresources
+ - ingresses
+ verbs:
+ - create
+ - get
+ - list
+ - watch
+- apiGroups:
+ - "apiextensions.k8s.io"
+ resources:
+ - customresourcedefinitions
+ verbs:
+ - create
+ - get
+ - list
+ - watch
+ - update
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumnetworkpolicies
+ - ciliumendpoints
+ verbs:
+ - "*"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cni/init.sls new/salt-master/salt/cni/init.sls
--- old/salt-master/salt/cni/init.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/cni/init.sls 2018-04-26 11:56:10.000000000 +0200
@@ -3,11 +3,12 @@
- addons
- kubectl-config
+{% set plugin = salt['pillar.get']('cni:plugin', 'flannel').lower() %}
+
#######################
# flannel CNI plugin
#######################
-{% set plugin = salt['pillar.get']('cni:plugin', 'flannel').lower() %}
{% if plugin == "flannel" %}
/etc/kubernetes/addons/kube-flannel-rbac.yaml:
@@ -48,3 +49,66 @@
- file: /etc/kubernetes/addons/kube-flannel-rbac.yaml
{% endif %}
+
+{% if plugin == "cilium" %}
+/etc/kubernetes/addons/cilium-config.yaml:
+ file.managed:
+ - source: salt://cni/cilium-config.yaml.jinja
+ - template: jinja
+ - makedirs: true
+ - require:
+ - file: /etc/kubernetes/addons
+ - defaults:
+ user: 'cluster-admin'
+ cilium_certificate: {{ pillar['ssl']['cilium_crt'] }}
+ cilium_key: {{ pillar['ssl']['cilium_key'] }}
+
+ cmd.run:
+ - name: |
+ kubectl apply --namespace kube-system -f /etc/kubernetes/addons/cilium-config.yaml
+ - env:
+ - KUBECONFIG: {{ pillar['paths']['kubeconfig'] }}
+ - require:
+ - kube-apiserver
+ - file: {{ pillar['paths']['kubeconfig'] }}
+ - watch:
+ - file: /etc/kubernetes/addons/cilium-config.yaml
+
+/etc/kubernetes/addons/cilium-rbac.yaml:
+ file.managed:
+ - source: salt://cni/cilium-rbac.yaml.jinja
+ - template: jinja
+ - makedirs: true
+ - require:
+ - file: /etc/kubernetes/addons
+ cmd.run:
+ - name: |
+ kubectl apply --namespace kube-system -f /etc/kubernetes/addons/cilium-rbac.yaml
+ - env:
+ - KUBECONFIG: {{ pillar['paths']['kubeconfig'] }}
+ - require:
+ - kube-apiserver
+ - file: {{ pillar['paths']['kubeconfig'] }}
+ - watch:
+ - file: /etc/kubernetes/addons/cilium-rbac.yaml
+
+/etc/kubernetes/addons/cilium-ds.yaml:
+ file.managed:
+ - source: salt://cni/cilium-ds.yaml.jinja
+ - template: jinja
+ - makedirs: true
+ - require:
+ - file: /etc/kubernetes/addons
+ cmd.run:
+ - name: |
+ kubectl apply --namespace kube-system -f /etc/kubernetes/addons/cilium-ds.yaml
+ - env:
+ - KUBECONFIG: {{ pillar['paths']['kubeconfig'] }}
+ - require:
+ - kube-apiserver
+ - file: {{ pillar['paths']['kubeconfig'] }}
+ - watch:
+ - /etc/kubernetes/addons/cilium-config.yaml
+ - file: /etc/kubernetes/addons/cilium-config.yaml
+
+{% endif %}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/cni/remove-pre-reboot.sls new/salt-master/salt/cni/remove-pre-reboot.sls
--- old/salt-master/salt/cni/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/cni/remove-pre-reboot.sls 1970-01-01 01:00:00.000000000 +0100
@@ -1,7 +0,0 @@
-# cleanup all the things we have created
-
-/etc/kubernetes/addons/kube-flannel-rbac.yaml:
- file.absent
-
-/etc/kubernetes/addons/kube-flannel.yaml:
- file.absent
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/crio/remove-pre-reboot.sls new/salt-master/salt/crio/remove-pre-reboot.sls
--- old/salt-master/salt/crio/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/crio/remove-pre-reboot.sls 1970-01-01 01:00:00.000000000 +0100
@@ -1,13 +0,0 @@
-# cleanup all the things we have created
-
-/etc/systemd/system/kubelet.service.d/kubelet.conf:
- file.absent
-
-/var/lib/containers/storage:
- cmd.run:
- - name: |-
- for subvolume in {{pillar['cri']['crio']['dirs']['root']}}/btrfs/subvolumes/* ; do
- btrfs subvolume delete $subvolume
- done
- rm -rf {{pillar['cri']['crio']['dirs']['root']}}*
- rm -rf {{pillar['cri']['crio']['dirs']['runroot']}}*
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/docker/remove-pre-reboot.sls new/salt-master/salt/docker/remove-pre-reboot.sls
--- old/salt-master/salt/docker/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/docker/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200
@@ -1,21 +1,4 @@
-# cleanup all the things we have created
-/etc/systemd/system/docker.service.d/proxy.conf:
- file.absent
-
-/etc/docker/daemon.json:
- file.absent
-
-/etc/sysconfig/docker:
- file.absent
/etc/docker/certs.d:
file.absent
-
-/var/lib/docker:
- cmd.run:
- - name: |-
- for subvolume in /var/lib/docker/btrfs/subvolumes/* ; do
- btrfs subvolume delete $subvolume
- done
- rm -rf /var/lib/docker/*
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/etc-hosts/remove-pre-reboot.sls new/salt-master/salt/etc-hosts/remove-pre-reboot.sls
--- old/salt-master/salt/etc-hosts/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/etc-hosts/remove-pre-reboot.sls 1970-01-01 01:00:00.000000000 +0100
@@ -1,4 +0,0 @@
-# cleanup all the things we have created
-
-/etc/hosts:
- file.absent
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/etcd/init.sls new/salt-master/salt/etcd/init.sls
--- old/salt-master/salt/etcd/init.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/etcd/init.sls 2018-04-26 11:56:10.000000000 +0200
@@ -14,6 +14,9 @@
- require:
- file: /etc/zypp/repos.d/containers.repo
caasp_etcd.member_add:
+ - retry:
+ interval: 4
+ attempts: 15
- require:
- {{ pillar['ssl']['crt_file'] }}
- {{ pillar['ssl']['key_file'] }}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/etcd/remove-pre-reboot.sls new/salt-master/salt/etcd/remove-pre-reboot.sls
--- old/salt-master/salt/etcd/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/etcd/remove-pre-reboot.sls 1970-01-01 01:00:00.000000000 +0100
@@ -1,23 +0,0 @@
-
-# cleanup all the things we have created for etcd
-
-/etc/sysconfig/etcd:
- file.absent
-
-/etc/sysconfig/etcdctl:
- file.absent
-
-/etc/systemd/system/etcd.service.d/etcd.conf:
- file.absent
-
-etcd-user-removal:
- user.absent:
- - name: etcd
-
-etcd-group-removal:
- group.absent:
- - name: etcd
-
-etcd-wipe-var-lib:
- cmd.run:
- - name: rm -rf /var/lib/etcd/*
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/etcd/remove-pre-stop-services.sls new/salt-master/salt/etcd/remove-pre-stop-services.sls
--- old/salt-master/salt/etcd/remove-pre-stop-services.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/etcd/remove-pre-stop-services.sls 1970-01-01 01:00:00.000000000 +0100
@@ -1,11 +0,0 @@
-
-# this node is being removed from the cluster,
-# but etcd is still running:
-# we explicitly remove the node from the etcd cluster,
-# so it is not considered a node suffering some
-# transient failure...
-etcd-remove-member:
- caasp_etcd.member_remove:
- - nodename: {{ salt['pillar.get']('nodename', grains['nodename']) }}
- # NOTE: we are not requiring /etc/hosts or the certificates
- # because we are assuming this node was on high state
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kube-apiserver/remove-pre-reboot.sls new/salt-master/salt/kube-apiserver/remove-pre-reboot.sls
--- old/salt-master/salt/kube-apiserver/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/kube-apiserver/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200
@@ -8,6 +8,3 @@
{{ pillar['ssl']['kube_apiserver_key'] }}:
file.absent
-
-/etc/kubernetes/apiserver:
- file.absent
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kube-controller-manager/remove-pre-reboot.sls new/salt-master/salt/kube-controller-manager/remove-pre-reboot.sls
--- old/salt-master/salt/kube-controller-manager/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/kube-controller-manager/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200
@@ -11,6 +11,3 @@
{{ pillar['paths']['service_account_key'] }}:
file.absent
-
-{{ pillar['paths']['kube_controller_mgr_config'] }}:
- file.absent
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kube-proxy/remove-pre-reboot.sls new/salt-master/salt/kube-proxy/remove-pre-reboot.sls
--- old/salt-master/salt/kube-proxy/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/kube-proxy/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200
@@ -8,6 +8,3 @@
{{ pillar['ssl']['kube_proxy_key'] }}:
file.absent
-
-{{ pillar['paths']['kube_proxy_config'] }}:
- file.absent
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kube-scheduler/remove-pre-reboot.sls new/salt-master/salt/kube-scheduler/remove-pre-reboot.sls
--- old/salt-master/salt/kube-scheduler/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/kube-scheduler/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200
@@ -8,6 +8,3 @@
{{ pillar['ssl']['kube_scheduler_key'] }}:
file.absent
-
-{{ pillar['paths']['kube_scheduler_config'] }}:
- file.absent
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kubectl-config/remove-pre-reboot.sls new/salt-master/salt/kubectl-config/remove-pre-reboot.sls
--- old/salt-master/salt/kubectl-config/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/kubectl-config/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200
@@ -5,9 +5,3 @@
{{ pillar['ssl']['kubectl_key'] }}:
file.absent
-
-{{ pillar['paths']['kubeconfig'] }}:
- file.absent
-
-/root/.kube:
- file.absent
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kubelet/init.sls new/salt-master/salt/kubelet/init.sls
--- old/salt-master/salt/kubelet/init.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/kubelet/init.sls 2018-04-26 11:56:10.000000000 +0200
@@ -4,6 +4,7 @@
- cert
- kubernetes-common
- kubectl-config
+ - swap
/etc/kubernetes/kubelet-initial:
file.managed:
@@ -82,6 +83,7 @@
- file: /etc/kubernetes/manifests
- file: /etc/kubernetes/kubelet-initial
- kubelet-config
+ - cmd: unmount-swaps
caasp_retriable.retry:
- name: iptables-kubelet
- target: iptables.append
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kubelet/remove-pre-reboot.sls new/salt-master/salt/kubelet/remove-pre-reboot.sls
--- old/salt-master/salt/kubelet/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/kubelet/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200
@@ -10,20 +10,10 @@
{{ pillar['ssl']['kubelet_key'] }}:
file.absent
-/etc/kubernetes/kubelet-initial:
- file.absent
-
+# this file can contain sensitive information, so it must be removed too
{{ pillar['paths']['kubelet_config'] }}:
file.absent
+# and this one too
/etc/kubernetes/openstack-config:
file.absent
-
-wipe-var-lib-kubelet:
- cmd.run:
- - name: |-
- mnts=`mount | grep kubelet | grep tmpfs | cut -f3 -d" "`
- for i in $mnts ; do
- umount $i
- done
- rm -rf /var/lib/kubelet/*
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kubelet/stop.sls new/salt-master/salt/kubelet/stop.sls
--- old/salt-master/salt/kubelet/stop.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/kubelet/stop.sls 2018-04-26 11:56:10.000000000 +0200
@@ -23,23 +23,6 @@
- force: True
{%- endif %}
-{%- if node_removal_in_progress %}
-
-# we must run the `delete node` when haproxy is still running.
-# * in pre-stop-services, we have not cordoned the node yet
-# * in pre-reboot, haproxy has been stopped
-# so we have to do it here...
-
-delete-node-from-kubernetes:
- cmd.run:
- - name: |-
- kubectl --kubeconfig={{ pillar['paths']['kubeconfig'] }} delete node {{ grains['nodename'] }}
- - require:
- - file: {{ pillar['paths']['kubeconfig'] }}
- - drain-kubelet
-
-{%- endif %}
-
kubelet:
service.dead:
- enable: False
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/kubernetes-common/remove-pre-reboot.sls new/salt-master/salt/kubernetes-common/remove-pre-reboot.sls
--- old/salt-master/salt/kubernetes-common/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/kubernetes-common/remove-pre-reboot.sls 2018-04-26 11:56:10.000000000 +0200
@@ -1,10 +1,8 @@
-# cleanup all the things we have created
+
+# some files that could contain sensitive information:
/etc/kubernetes/config:
file.absent
/etc/kubernetes/openstack-config:
file.absent
-
-/var/lib/kubernetes:
- file.absent
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/motd/remove-pre-reboot.sls new/salt-master/salt/motd/remove-pre-reboot.sls
--- old/salt-master/salt/motd/remove-pre-reboot.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/motd/remove-pre-reboot.sls 1970-01-01 01:00:00.000000000 +0100
@@ -1,4 +0,0 @@
-/etc/motd:
- file.managed:
- - contents: |-
- This machine has been removed from the CaaS cluster.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/orch/removal.sls new/salt-master/salt/orch/removal.sls
--- old/salt-master/salt/orch/removal.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/orch/removal.sls 2018-04-26 11:56:10.000000000 +0200
@@ -1,23 +1,44 @@
-# must provide the node (id) to be removed in the 'target' pillar
+{#- must provide the node (id) to be removed in the 'target' pillar #}
{%- set target = salt['pillar.get']('target') %}
+{#- ... and we can provide an optional replacement node #}
+{%- set replacement = salt['pillar.get']('replacement', '') %}
+
+{#- Get a list of nodes seem to be down or unresponsive #}
+{#- This sends a "are you still there?" message to all #}
+{#- the nodes and wait for a response, so it takes some time. #}
+{#- Hopefully this list will not be too long... #}
+{%- set nodes_down = salt.saltutil.runner('manage.down') %}
+{%- if not nodes_down %}
+ {%- do salt.caasp_log.debug('all nodes seem to be up') %}
+ {%- set all_responsive_nodes_tgt = 'P@roles:(etcd|kube-master|kube-minion)' %}
+{%- else %}
+ {%- do salt.caasp_log.debug('nodes "%s" seem to be down', nodes_down|join(',')) %}
+ {%- set all_responsive_nodes_tgt = 'not L@' + nodes_down|join(',')
+ + ' and P@roles:(etcd|kube-master|kube-minion)' %}
+
+ {%- if target in nodes_down %}
+ {%- do salt.caasp_log.abort('target is unresponsive, forced removal must be used') %}
+ {%- endif %}
+{%- endif %}
+
{%- set etcd_members = salt.saltutil.runner('mine.get', tgt='G@roles:etcd', fun='network.interfaces', tgt_type='compound').keys() %}
{%- set masters = salt.saltutil.runner('mine.get', tgt='G@roles:kube-master', fun='network.interfaces', tgt_type='compound').keys() %}
{%- set minions = salt.saltutil.runner('mine.get', tgt='G@roles:kube-minion', fun='network.interfaces', tgt_type='compound').keys() %}
-{#- ... and we can provide an optional replacement node #}
-{%- set replacement = salt['pillar.get']('replacement', '') %}
+{%- set super_master_tgt = salt.caasp_nodes.get_super_master(masters=masters,
+ excluded=[target] + nodes_down) %}
+{%- if not super_master_tgt %}
+ {%- do salt.caasp_log.abort('(after removing %s) no masters are reachable', target) %}
+{%- endif %}
{#- try to use the user-provided replacement or find a replacement by ourselves #}
{#- if no valid replacement can be used/found, `replacement` will be '' #}
{%- set replacement, replacement_roles = salt.caasp_nodes.get_replacement_for(target, replacement,
masters=masters,
minions=minions,
- etcd_members=etcd_members) %}
-
-{##############################
- # set grains
- #############################}
+ etcd_members=etcd_members,
+ excluded=nodes_down) %}
# Ensure we mark all nodes with the "as node is being removed" grain.
# This will ensure the update-etc-hosts orchestration is not run.
@@ -30,6 +51,23 @@
- removal_in_progress
- true
+# make sure we have a solid ground before starting the removal
+# (ie, expired certs produce really funny errors)
+update-config:
+ salt.state:
+ - tgt: '{{ all_responsive_nodes_tgt }}'
+ - tgt_type: compound
+ - sls:
+ - etc-hosts
+ - ca-cert
+ - cert
+ - require:
+ - set-cluster-wide-removal-grain
+
+{##############################
+ # set grains
+ #############################}
+
assign-removal-grain:
salt.function:
- tgt: {{ target }}
@@ -38,47 +76,47 @@
- node_removal_in_progress
- true
- require:
- - set-cluster-wide-removal-grain
+ - update-config
{%- if replacement %}
assign-addition-grain:
salt.function:
- - tgt: {{ replacement }}
+ - tgt: '{{ replacement }}'
- name: grains.setval
- arg:
- node_addition_in_progress
- true
- require:
- - set-cluster-wide-removal-grain
- - assign-removal-grain
+ - update-config
{#- and then we can assign these (new) roles to the replacement #}
{% for role in replacement_roles %}
assign-{{ role }}-role-to-replacement:
salt.function:
- - tgt: {{ replacement }}
+ - tgt: '{{ replacement }}'
- name: grains.append
- arg:
- roles
- {{ role }}
- require:
- - assign-removal-grain
+ - update-config
- assign-addition-grain
- {%- endfor %}
+ {% endfor %}
{%- endif %} {# replacement #}
sync-all:
salt.function:
- - tgt: '*'
+ - tgt: '{{ all_responsive_nodes_tgt }}'
+ - tgt_type: compound
- names:
- saltutil.refresh_pillar
- saltutil.refresh_grains
- mine.update
- saltutil.sync_all
- require:
- - set-cluster-wide-removal-grain
+ - update-config
- assign-removal-grain
{%- for role in replacement_roles %}
- assign-{{ role }}-role-to-replacement
@@ -92,7 +130,7 @@
highstate-replacement:
salt.state:
- - tgt: {{ replacement }}
+ - tgt: '{{ replacement }}'
- highstate: True
- require:
- sync-all
@@ -108,7 +146,7 @@
set-bootstrap-complete-flag-in-replacement:
salt.function:
- - tgt: {{ replacement }}
+ - tgt: '{{ replacement }}'
- name: grains.setval
- arg:
- bootstrap_complete
@@ -119,7 +157,7 @@
# remove the we-are-adding-this-node grain
remove-addition-grain:
salt.function:
- - tgt: {{ replacement }}
+ - tgt: '{{ replacement }}'
- name: grains.delval
- arg:
- node_addition_in_progress
@@ -138,21 +176,6 @@
# the replacement should be ready at this point:
# we can remove the old node running in {{ target }}
-{%- if target in etcd_members %} {# we are only doing this for etcd at the moment... #}
-prepare-target-removal:
- salt.state:
- - tgt: {{ target }}
- - sls:
- {%- if target in etcd_members %}
- - etcd.remove-pre-stop-services
- {%- endif %}
- - require:
- - sync-all
- {%- if replacement %}
- - set-bootstrap-complete-flag-in-replacement
- {%- endif %}
-{%- endif %}
-
stop-services-in-target:
salt.state:
- tgt: {{ target }}
@@ -171,9 +194,9 @@
{%- endif %}
- require:
- sync-all
- {%- if target in etcd_members %}
- - prepare-target-removal
- {%- endif %}
+ {%- if replacement %}
+ - remove-addition-grain
+ {%- endif %}
# remove any other configuration in the machines
cleanups-in-target-before-rebooting:
@@ -184,7 +207,6 @@
- kube-apiserver.remove-pre-reboot
- kube-controller-manager.remove-pre-reboot
- kube-scheduler.remove-pre-reboot
- - addons.remove-pre-reboot
- addons.dns.remove-pre-reboot
- addons.tiller.remove-pre-reboot
- addons.dex.remove-pre-reboot
@@ -193,12 +215,7 @@
- kubelet.remove-pre-reboot
- kubectl-config.remove-pre-reboot
- cri.remove-pre-reboot
- - cni.remove-pre-reboot
- {%- if target in etcd_members %}
- - etcd.remove-pre-reboot
- {%- endif %}
- - etc-hosts.remove-pre-reboot
- - motd.remove-pre-reboot
+ - cert.remove-pre-reboot
- cleanup.remove-pre-reboot
- require:
- stop-services-in-target
@@ -217,14 +234,30 @@
# (we don't need to wait for the node:
# just forget about it...)
-# remove the Salt key and the mine for the target
+# do any cluster-scope removals in the super_master
+remove-from-cluster-in-super-master:
+ salt.state:
+ - tgt: '{{ super_master_tgt }}'
+ - pillar:
+ target: {{ target }}
+ - sls:
+ - cleanup.remove-post-orchestration
+ - require:
+ - sync-all
+ - shutdown-target
+ {%- if replacement %}
+ - remove-addition-grain
+ {%- endif %}
+
+# remove the Salt key
+# (it will appear as "unaccepted")
remove-target-salt-key:
salt.wheel:
- name: key.reject
- include_accepted: True
- match: {{ target }}
- require:
- - shutdown-target
+ - remove-from-cluster-in-super-master
# remove target's data in the Salt Master's cache
remove-target-mine:
@@ -244,18 +277,18 @@
# the etcd server we have just removed (but they would
# keep working fine as long as we had >1 etcd servers)
-{%- set affected_expr = salt.caasp_nodes.get_expr_affected_by(target,
- excluded=[replacement],
- masters=masters,
- minions=minions,
- etcd_members=etcd_members) %}
-
-{%- do salt.caasp_log.debug('will high-state machines affected by removal: %s', affected_expr) %}
+{%- set affected_tgt = salt.caasp_nodes.get_expr_affected_by(target,
+ excluded=[replacement] + nodes_down,
+ masters=masters,
+ minions=minions,
+ etcd_members=etcd_members) %}
+{%- do salt.caasp_log.debug('will high-state machines affected by removal: "%s"', affected_tgt) %}
# make sure the cluster has up-to-date state
sync-after-removal:
salt.function:
- - tgt: '*'
+ - tgt: '{{ all_responsive_nodes_tgt }}'
+ - tgt_type: compound
- names:
- saltutil.clear_cache
- mine.update
@@ -264,7 +297,7 @@
highstate-affected:
salt.state:
- - tgt: {{ affected_expr }}
+ - tgt: '{{ affected_tgt }}'
- tgt_type: compound
- highstate: True
- batch: 1
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/salt-master/salt/top.sls new/salt-master/salt/top.sls
--- old/salt-master/salt/top.sls 2018-04-23 14:03:55.000000000 +0200
+++ new/salt-master/salt/top.sls 2018-04-26 11:56:10.000000000 +0200
@@ -37,3 +37,4 @@
- cert
- kubelet
- kube-proxy
+ - cni/cilium