Hi Viktor,
I hope that this output is related to lkv-srvapp42 only:
lkv-srvapp39:~ # salt-run state.event pretty=True
salt/auth {
"_stamp": "2021-07-19T13:28:59.586422",
"act": "accept",
"id": "lkv-srvapp42.lkvbay.net",
"pub": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApU/6mKT1NrxhllRpDgaR\ndT+FYKpH0c+oueRiB+bvbSXRMbvwJePDFT2CYFyjFQKU85XBmq3r1HRwP8CNn0Z3\nP12l2F+eAl2DHRWdGLnCeIQX/sau5YLBI7acTNAdpkNY0FJ1t7lapobGCdNfRfsh\nS5MCfxEf7CwK/us64uF55cY50vss6JSL/MMD5YoiHXUY7AhaasVv1q/3zYoyf7AN\n6tIJtqF/XdjocWE8gQTEMNbmAHvtqJ/09nyEBrh0ZlDZ/po1adbTvDH8z95cm2cL\nKa1TdasQRfSv4AGR0USH+iS8pWqGDlfnd7UuXgKHfTIyMu59nRZg5CY9qmRsL8UR\nNwIDAQAB\n-----END PUBLIC KEY-----\n",
"result": true
}
minion/refresh/lkv-srvapp42.lkvbay.net {
"Minion data cache refresh": "lkv-srvapp42.lkvbay.net",
"_stamp": "2021-07-19T13:28:59.712892"
}
salt/minion/lkv-srvapp42.lkvbay.net/start {
"_stamp": "2021-07-19T13:29:00.145784",
"cmd": "_minion_event",
"data": "Minion lkv-srvapp42.lkvbay.net started at Mon Jul 19 15:29:00 2021",
"id": "lkv-srvapp42.lkvbay.net",
"pretag": null,
"tag": "salt/minion/lkv-srvapp42.lkvbay.net/start"
}
20210719132900304963 {
"_stamp": "2021-07-19T13:29:00.305236",
"minions": [
"lkv-srvapp42.lkvbay.net"
]
}
salt/job/20210719132900304963/new {
"_stamp": "2021-07-19T13:29:00.305678",
"arg": [],
"fun": "mgractionchains.resume",
"jid": "20210719132900304963",
"minions": [
"lkv-srvapp42.lkvbay.net"
],
"missing": [],
"tgt": "lkv-srvapp42.lkvbay.net",
"tgt_type": "glob",
"user": "salt"
}
salt/job/20210719132900304963/ret/lkv-srvapp42.lkvbay.net {
"_stamp": "2021-07-19T13:29:00.378168",
"cmd": "_return",
"fun": "mgractionchains.resume",
"fun_args": [],
"id": "lkv-srvapp42.lkvbay.net",
"jid": "20210719132900304963",
"metadata": {
"suma-action-chain": true
},
"retcode": 0,
"return": {},
"success": true
}
20210719132900367742 {
"_stamp": "2021-07-19T13:29:00.392732",
"minions": [
"lkv-srvapp42.lkvbay.net"
]
}
salt/job/20210719132900367742/new {
"_stamp": "2021-07-19T13:29:00.393136",
"arg": [],
"fun": "test.ping",
"jid": "20210719132900367742",
"minions": [
"lkv-srvapp42.lkvbay.net"
],
"missing": [],
"tgt": [
"lkv-srvapp42.lkvbay.net"
],
"tgt_type": "list",
"user": "admin"
}
salt/job/20210719132900367742/ret/lkv-srvapp42.lkvbay.net {
"_stamp": "2021-07-19T13:29:00.459355",
"cmd": "_return",
"fun": "test.ping",
"fun_args": [],
"id": "lkv-srvapp42.lkvbay.net",
"jid": "20210719132900367742",
"metadata": {
"batch-mode": true,
"suma-action-chain": false,
"suma-action-id": 0,
"suma-force-pkg-list-refresh": false,
"suma-minion-startup": true
},
"retcode": 0,
"return": true,
"success": true
}
salt/batch/20210719132900368080/start {
"_stamp": "2021-07-19T13:29:00.463419",
"available_minions": [
"lkv-srvapp42.lkvbay.net"
],
"down_minions": [],
"metadata": {
"batch-mode": true,
"suma-action-chain": false,
"suma-action-id": 0,
"suma-force-pkg-list-refresh": false,
"suma-minion-startup": true
}
}
20210719132900368080 {
"_stamp": "2021-07-19T13:29:00.491610",
"minions": [
"lkv-srvapp42.lkvbay.net"
]
}
salt/job/20210719132900368080/new {
"_stamp": "2021-07-19T13:29:00.492090",
"arg": [
{
"__kwarg__": true,
"mods": [
"util.systeminfo"
],
"queue": true
}
],
"fun": "state.apply",
"jid": "20210719132900368080",
"minions": [
"lkv-srvapp42.lkvbay.net"
],
"missing": [],
"tgt": [
"lkv-srvapp42.lkvbay.net"
],
"tgt_type": "list",
"user": "salt"
}
20210719132900559758 {
"_stamp": "2021-07-19T13:29:00.560013",
"minions": [
"lkv-srvapp42.lkvbay.net"
]
}
salt/job/20210719132900559758/new {
"_stamp": "2021-07-19T13:29:00.560412",
"arg": [],
"fun": "test.ping",
"jid": "20210719132900559758",
"minions": [
"lkv-srvapp42.lkvbay.net"
],
"missing": [],
"tgt": [
"lkv-srvapp42.lkvbay.net"
],
"tgt_type": "list",
"user": "admin"
}
salt/job/20210719132900559758/ret/lkv-srvapp42.lkvbay.net {
"_stamp": "2021-07-19T13:29:00.632379",
"cmd": "_return",
"fun": "test.ping",
"fun_args": [],
"id": "lkv-srvapp42.lkvbay.net",
"jid": "20210719132900559758",
"retcode": 0,
"return": true,
"success": true
}
minion/refresh/lkv-srvapp42.lkvbay.net {
"Minion data cache refresh": "lkv-srvapp42.lkvbay.net",
"_stamp": "2021-07-19T13:29:00.720947"
}
20210719132900746311 {
"_stamp": "2021-07-19T13:29:00.746618",
"minions": [
"lkv-srvapp42.lkvbay.net"
]
}
salt/job/20210719132900746311/new {
"_stamp": "2021-07-19T13:29:00.747039",
"arg": [
"machine_id",
"saltboot_initrd",
"susemanager",
{
"__kwarg__": true,
"sanitize": false
}
],
"fun": "grains.item",
"jid": "20210719132900746311",
"minions": [
"lkv-srvapp42.lkvbay.net"
],
"missing": [],
"tgt": [
"lkv-srvapp42.lkvbay.net"
],
"tgt_type": "list",
"user": "admin"
}
salt/job/20210719132900746311/ret/lkv-srvapp42.lkvbay.net {
"_stamp": "2021-07-19T13:29:00.813641",
"cmd": "_return",
"fun": "grains.item",
"fun_args": [
"machine_id",
"saltboot_initrd",
"susemanager",
{
"sanitize": false
}
],
"id": "lkv-srvapp42.lkvbay.net",
"jid": "20210719132900746311",
"out": "nested",
"retcode": 0,
"return": {
"machine_id": "945e1982bdbf5494212beb1c5ea9a35b",
"saltboot_initrd": "",
"susemanager": {
"activation_key": "1-SLES15"
}
},
"success": true
}
salt/job/20210719132900368080/ret/lkv-srvapp42.lkvbay.net {
"_stamp": "2021-07-19T13:29:01.522183",
"cmd": "_return",
"fun": "state.apply",
"fun_args": [
{
"mods": [
"util.systeminfo"
],
"queue": true
}
],
"id": "lkv-srvapp42.lkvbay.net",
"jid": "20210719132900368080",
"metadata": {
"batch-mode": true,
"suma-action-chain": false,
"suma-action-id": 0,
"suma-force-pkg-list-refresh": false,
"suma-minion-startup": true
},
"out": "highstate",
"retcode": 2,
"return": {
"mgrcompat_|-grains_update_|-grains.item_|-module_run": {
"__id__": "grains_update",
"__run_num__": 5,
"__sls__": "util.systeminfo",
"changes": {},
"comment": "An exception occurred in this state: Traceback (most recent call last):\n File \"/usr/lib/python3.6/site-packages/salt/utils/decorators/__init__.py\", line 352, in _call_function\n return self._function(*args, **kwargs)\nTypeError: _run() missing 1 required positional argument: 'name'\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/usr/lib/python3.6/site-packages/salt/state.py\", line 2176, in call\n *cdata[\"args\"], **cdata[\"kwargs\"]\n File \"/usr/lib/python3.6/site-packages/salt/loader.py\", line 2113, in wrapper\n return f(*args, **kwargs)\n File \"/var/cache/salt/minion/extmods/states/mgrcompat.py\", line 85, in module_run\n ret = module.run(**new_kwargs)\n File \"/usr/lib/python3.6/site-packages/salt/utils/decorators/__init__.py\", line 738, in _decorate\n return self._call_function(kwargs)\n File \"/usr/lib/python3.6/site-packages/salt/utils/decorators/__init__.py\", line 355, in _call_function\n self._function, self._orig_f_name\nTypeError: replace() argument 1 must be str, not function\n",
"duration": 2.997,
"name": "grains.item",
"result": false,
"start_time": "15:29:01.517152"
},
"mgrcompat_|-status_uptime_|-status.uptime_|-module_run": {
"__id__": "status_uptime",
"__run_num__": 4,
"__sls__": "util.systeminfo",
"changes": {},
"comment": "An exception occurred in this state: Traceback (most recent call last):\n File \"/usr/lib/python3.6/site-packages/salt/utils/decorators/__init__.py\", line 352, in _call_function\n return self._function(*args, **kwargs)\nTypeError: _run() missing 1 required positional argument: 'name'\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/usr/lib/python3.6/site-packages/salt/state.py\", line 2176, in call\n *cdata[\"args\"], **cdata[\"kwargs\"]\n File \"/usr/lib/python3.6/site-packages/salt/loader.py\", line 2113, in wrapper\n return f(*args, **kwargs)\n File \"/var/cache/salt/minion/extmods/states/mgrcompat.py\", line 85, in module_run\n ret = module.run(**new_kwargs)\n File \"/usr/lib/python3.6/site-packages/salt/utils/decorators/__init__.py\", line 738, in _decorate\n return self._call_function(kwargs)\n File \"/usr/lib/python3.6/site-packages/salt/utils/decorators/__init__.py\", line 355, in _call_function\n self._function, self._orig_f_name\nTypeError: replace() argument 1 must be str, not function\n",
"duration": 5.542,
"name": "status.uptime",
"result": false,
"start_time": "15:29:01.511454"
},
"saltutil_|-sync_beacons_|-sync_beacons_|-sync_beacons": {
"__id__": "sync_beacons",
"__run_num__": 3,
"__sls__": "util.syncbeacons",
"changes": {},
"comment": "No updates to sync",
"duration": 29.281,
"name": "sync_beacons",
"result": true,
"start_time": "15:29:01.480125"
},
"saltutil_|-sync_grains_|-sync_grains_|-sync_grains": {
"__id__": "sync_grains",
"__run_num__": 2,
"__sls__": "util.syncgrains",
"changes": {},
"comment": "No updates to sync",
"duration": 359.752,
"name": "sync_grains",
"result": true,
"start_time": "15:29:01.120157"
},
"saltutil_|-sync_modules_|-sync_modules_|-sync_modules": {
"__id__": "sync_modules",
"__run_num__": 0,
"__sls__": "util.syncmodules",
"changes": {},
"comment": "No updates to sync",
"duration": 139.887,
"name": "sync_modules",
"result": true,
"start_time": "15:29:00.948134"
},
"saltutil_|-sync_states_|-sync_states_|-sync_states": {
"__id__": "sync_states",
"__run_num__": 1,
"__sls__": "util.syncstates",
"changes": {},
"comment": "No updates to sync",
"duration": 31.873,
"name": "sync_states",
"result": true,
"start_time": "15:29:01.088161"
}
},
"success": false
}
minion/refresh/lkv-srvapp42.lkvbay.net {
"Minion data cache refresh": "lkv-srvapp42.lkvbay.net",
"_stamp": "2021-07-19T13:29:02.265064"
}
salt/batch/20210719132900368080/done {
"_stamp": "2021-07-19T13:29:02.527884",
"available_minions": [
"lkv-srvapp42.lkvbay.net"
],
"done_minions": [
"lkv-srvapp42.lkvbay.net"
],
"down_minions": [],
"metadata": {
"batch-mode": true,
"suma-action-chain": false,
"suma-action-id": 0,
"suma-force-pkg-list-refresh": false,
"suma-minion-startup": true
},
"timedout_minions": []
}
^CException occurred in runner state.event: Traceback (most recent call last):
File "/usr/lib/python3.6/site-packages/salt/client/mixins.py", line 385, in low
data['return'] = func(*args, **kwargs)
File "/usr/lib/python3.6/site-packages/salt/runners/state.py", line 313, in event
node=node)
File "/usr/lib/python3.6/site-packages/salt/modules/state.py", line 2412, in event
ret = sevent.get_event(full=True, auto_reconnect=True)
File "/usr/lib/python3.6/site-packages/salt/utils/event.py", line 638, in get_event
ret = self._get_event(wait, tag, match_func, no_block)
File "/usr/lib/python3.6/site-packages/salt/utils/event.py", line 544, in _get_event
raw = self.subscriber.read_sync(timeout=wait)
File "/usr/lib/python3.6/site-packages/salt/transport/ipc.py", line 693, in read_sync
return self.io_loop.run_sync(lambda: self._read(timeout))
File "/usr/lib/python3.6/site-packages/salt/ext/tornado/ioloop.py", line 454, in run_sync
self.start()
File "/usr/lib/python3.6/site-packages/salt/ext/tornado/ioloop.py", line 865, in start
event_pairs = self._impl.poll(poll_timeout)
File "/usr/lib/python3.6/site-packages/salt/scripts.py", line 78, in _handle_signals
hardcrash, trace=trace)
File "/usr/lib/python3.6/site-packages/salt/scripts.py", line 43, in _handle_interrupt
raise exc
SystemExit:
Exiting gracefully on Ctrl-c
lkv-srvapp39:~ #
Regards,
Tobias.
Von: Victor Zhestkov [mailto:Victor.Zhestkov@suse.com]
Gesendet: Montag, 19. Juli 2021 15:20
Tobias, could you please collect the output of
`salt-run state.event pretty=True` for this minion on restarting it.
1. stop salt-minion on the affected system
2. Run `salt-run state.event pretty=True` on SUSE Manager Server
3. start salt-minion on the affected system
And show the events captured related to this minion
Victor