It's very confusing. Just tried `salt-call state.apply util.systeminfo` on my testing SLE15SP2 system with salt-minion-3002.2-37.1 and it works fine. But in your case it seems `salt-call state.apply util.systeminfo` will fail. Please check if it's failing. If so, please collect the output of `salt-call -l trace state.apply util.systeminfo` and send me as a file. Thanks for advance. Regards, Victor On Mon, 2021-07-19 at 13:35 +0000, Crefeld, Tobias LKV Bayern e.V. wrote: Hi Viktor, I hope that this output is related to lkv-srvapp42 only: lkv-srvapp39:~ # salt-run state.event pretty=True salt/auth { "_stamp": "2021-07-19T13:28:59.586422", "act": "accept", "id": "lkv-srvapp42.lkvbay.net", "pub": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApU/6mKT1NrxhllRpDgaR\ndT+FYKpH0c+oueRiB+bvbSXRMbvwJePDFT2CYFyjFQKU85XBmq3r1HRwP8CNn0Z3\nP12l2F+eAl2DHRWdGLnCeIQX/sau5YLBI7acTNAdpkNY0FJ1t7lapobGCdNfRfsh\nS5MCfxEf7CwK/us64uF55cY50vss6JSL/MMD5YoiHXUY7AhaasVv1q/3zYoyf7AN\n6tIJtqF/XdjocWE8gQTEMNbmAHvtqJ/09nyEBrh0ZlDZ/po1adbTvDH8z95cm2cL\nKa1TdasQRfSv4AGR0USH+iS8pWqGDlfnd7UuXgKHfTIyMu59nRZg5CY9qmRsL8UR\nNwIDAQAB\n-----END PUBLIC KEY-----\n", "result": true } minion/refresh/lkv-srvapp42.lkvbay.net { "Minion data cache refresh": "lkv-srvapp42.lkvbay.net", "_stamp": "2021-07-19T13:28:59.712892" } salt/minion/lkv-srvapp42.lkvbay.net/start { "_stamp": "2021-07-19T13:29:00.145784", "cmd": "_minion_event", "data": "Minion lkv-srvapp42.lkvbay.net started at Mon Jul 19 15:29:00 2021", "id": "lkv-srvapp42.lkvbay.net", "pretag": null, "tag": "salt/minion/lkv-srvapp42.lkvbay.net/start" } 20210719132900304963 { "_stamp": "2021-07-19T13:29:00.305236", "minions": [ "lkv-srvapp42.lkvbay.net" ] } salt/job/20210719132900304963/new { "_stamp": "2021-07-19T13:29:00.305678", "arg": [], "fun": "mgractionchains.resume", "jid": "20210719132900304963", "minions": [ "lkv-srvapp42.lkvbay.net" ], "missing": [], "tgt": "lkv-srvapp42.lkvbay.net", "tgt_type": "glob", "user": "salt" } salt/job/20210719132900304963/ret/lkv-srvapp42.lkvbay.net { "_stamp": "2021-07-19T13:29:00.378168", "cmd": "_return", "fun": "mgractionchains.resume", "fun_args": [], "id": "lkv-srvapp42.lkvbay.net", "jid": "20210719132900304963", "metadata": { "suma-action-chain": true }, "retcode": 0, "return": {}, "success": true } 20210719132900367742 { "_stamp": "2021-07-19T13:29:00.392732", "minions": [ "lkv-srvapp42.lkvbay.net" ] } salt/job/20210719132900367742/new { "_stamp": "2021-07-19T13:29:00.393136", "arg": [], "fun": "test.ping", "jid": "20210719132900367742", "minions": [ "lkv-srvapp42.lkvbay.net" ], "missing": [], "tgt": [ "lkv-srvapp42.lkvbay.net" ], "tgt_type": "list", "user": "admin" } salt/job/20210719132900367742/ret/lkv-srvapp42.lkvbay.net { "_stamp": "2021-07-19T13:29:00.459355", "cmd": "_return", "fun": "test.ping", "fun_args": [], "id": "lkv-srvapp42.lkvbay.net", "jid": "20210719132900367742", "metadata": { "batch-mode": true, "suma-action-chain": false, "suma-action-id": 0, "suma-force-pkg-list-refresh": false, "suma-minion-startup": true }, "retcode": 0, "return": true, "success": true } salt/batch/20210719132900368080/start { "_stamp": "2021-07-19T13:29:00.463419", "available_minions": [ "lkv-srvapp42.lkvbay.net" ], "down_minions": [], "metadata": { "batch-mode": true, "suma-action-chain": false, "suma-action-id": 0, "suma-force-pkg-list-refresh": false, "suma-minion-startup": true } } 20210719132900368080 { "_stamp": "2021-07-19T13:29:00.491610", "minions": [ "lkv-srvapp42.lkvbay.net" ] } salt/job/20210719132900368080/new { "_stamp": "2021-07-19T13:29:00.492090", "arg": [ { "__kwarg__": true, "mods": [ "util.systeminfo" ], "queue": true } ], "fun": "state.apply", "jid": "20210719132900368080", "minions": [ "lkv-srvapp42.lkvbay.net" ], "missing": [], "tgt": [ "lkv-srvapp42.lkvbay.net" ], "tgt_type": "list", "user": "salt" } 20210719132900559758 { "_stamp": "2021-07-19T13:29:00.560013", "minions": [ "lkv-srvapp42.lkvbay.net" ] } salt/job/20210719132900559758/new { "_stamp": "2021-07-19T13:29:00.560412", "arg": [], "fun": "test.ping", "jid": "20210719132900559758", "minions": [ "lkv-srvapp42.lkvbay.net" ], "missing": [], "tgt": [ "lkv-srvapp42.lkvbay.net" ], "tgt_type": "list", "user": "admin" } salt/job/20210719132900559758/ret/lkv-srvapp42.lkvbay.net { "_stamp": "2021-07-19T13:29:00.632379", "cmd": "_return", "fun": "test.ping", "fun_args": [], "id": "lkv-srvapp42.lkvbay.net", "jid": "20210719132900559758", "retcode": 0, "return": true, "success": true } minion/refresh/lkv-srvapp42.lkvbay.net { "Minion data cache refresh": "lkv-srvapp42.lkvbay.net", "_stamp": "2021-07-19T13:29:00.720947" } 20210719132900746311 { "_stamp": "2021-07-19T13:29:00.746618", "minions": [ "lkv-srvapp42.lkvbay.net" ] } salt/job/20210719132900746311/new { "_stamp": "2021-07-19T13:29:00.747039", "arg": [ "machine_id", "saltboot_initrd", "susemanager", { "__kwarg__": true, "sanitize": false } ], "fun": "grains.item", "jid": "20210719132900746311", "minions": [ "lkv-srvapp42.lkvbay.net" ], "missing": [], "tgt": [ "lkv-srvapp42.lkvbay.net" ], "tgt_type": "list", "user": "admin" } salt/job/20210719132900746311/ret/lkv-srvapp42.lkvbay.net { "_stamp": "2021-07-19T13:29:00.813641", "cmd": "_return", "fun": "grains.item", "fun_args": [ "machine_id", "saltboot_initrd", "susemanager", { "sanitize": false } ], "id": "lkv-srvapp42.lkvbay.net", "jid": "20210719132900746311", "out": "nested", "retcode": 0, "return": { "machine_id": "945e1982bdbf5494212beb1c5ea9a35b", "saltboot_initrd": "", "susemanager": { "activation_key": "1-SLES15" } }, "success": true } salt/job/20210719132900368080/ret/lkv-srvapp42.lkvbay.net { "_stamp": "2021-07-19T13:29:01.522183", "cmd": "_return", "fun": "state.apply", "fun_args": [ { "mods": [ "util.systeminfo" ], "queue": true } ], "id": "lkv-srvapp42.lkvbay.net", "jid": "20210719132900368080", "metadata": { "batch-mode": true, "suma-action-chain": false, "suma-action-id": 0, "suma-force-pkg-list-refresh": false, "suma-minion-startup": true }, "out": "highstate", "retcode": 2, "return": { "mgrcompat_|-grains_update_|-grains.item_|-module_run": { "__id__": "grains_update", "__run_num__": 5, "__sls__": "util.systeminfo", "changes": {}, "comment": "An exception occurred in this state: Traceback (most recent call last):\n File \"/usr/lib/python3.6/site-packages/salt/utils/decorators/__init__.py\", line 352, in _call_function\n return self._function(*args, **kwargs)\nTypeError: _run() missing 1 required positional argument: 'name'\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/usr/lib/python3.6/site-packages/salt/state.py\", line 2176, in call\n *cdata[\"args\"], **cdata[\"kwargs\"]\n File \"/usr/lib/python3.6/site-packages/salt/loader.py\", line 2113, in wrapper\n return f(*args, **kwargs)\n File \"/var/cache/salt/minion/extmods/states/mgrcompat.py\", line 85, in module_run\n ret = module.run(**new_kwargs)\n File \"/usr/lib/python3.6/site-packages/salt/utils/decorators/__init__.py\", line 738, in _decorate\n return self._call_function(kwargs)\n File \"/usr/lib/python3.6/site-packages/salt/utils/decorators/__init__.py\", line 355, in _call_function\n self._function, self._orig_f_name\nTypeError: replace() argument 1 must be str, not function\n", "duration": 2.997, "name": "grains.item", "result": false, "start_time": "15:29:01.517152" }, "mgrcompat_|-status_uptime_|-status.uptime_|-module_run": { "__id__": "status_uptime", "__run_num__": 4, "__sls__": "util.systeminfo", "changes": {}, "comment": "An exception occurred in this state: Traceback (most recent call last):\n File \"/usr/lib/python3.6/site-packages/salt/utils/decorators/__init__.py\", line 352, in _call_function\n return self._function(*args, **kwargs)\nTypeError: _run() missing 1 required positional argument: 'name'\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/usr/lib/python3.6/site-packages/salt/state.py\", line 2176, in call\n *cdata[\"args\"], **cdata[\"kwargs\"]\n File \"/usr/lib/python3.6/site-packages/salt/loader.py\", line 2113, in wrapper\n return f(*args, **kwargs)\n File \"/var/cache/salt/minion/extmods/states/mgrcompat.py\", line 85, in module_run\n ret = module.run(**new_kwargs)\n File \"/usr/lib/python3.6/site-packages/salt/utils/decorators/__init__.py\", line 738, in _decorate\n return self._call_function(kwargs)\n File \"/usr/lib/python3.6/site-packages/salt/utils/decorators/__init__.py\", line 355, in _call_function\n self._function, self._orig_f_name\nTypeError: replace() argument 1 must be str, not function\n", "duration": 5.542, "name": "status.uptime", "result": false, "start_time": "15:29:01.511454" }, "saltutil_|-sync_beacons_|-sync_beacons_|-sync_beacons": { "__id__": "sync_beacons", "__run_num__": 3, "__sls__": "util.syncbeacons", "changes": {}, "comment": "No updates to sync", "duration": 29.281, "name": "sync_beacons", "result": true, "start_time": "15:29:01.480125" }, "saltutil_|-sync_grains_|-sync_grains_|-sync_grains": { "__id__": "sync_grains", "__run_num__": 2, "__sls__": "util.syncgrains", "changes": {}, "comment": "No updates to sync", "duration": 359.752, "name": "sync_grains", "result": true, "start_time": "15:29:01.120157" }, "saltutil_|-sync_modules_|-sync_modules_|-sync_modules": { "__id__": "sync_modules", "__run_num__": 0, "__sls__": "util.syncmodules", "changes": {}, "comment": "No updates to sync", "duration": 139.887, "name": "sync_modules", "result": true, "start_time": "15:29:00.948134" }, "saltutil_|-sync_states_|-sync_states_|-sync_states": { "__id__": "sync_states", "__run_num__": 1, "__sls__": "util.syncstates", "changes": {}, "comment": "No updates to sync", "duration": 31.873, "name": "sync_states", "result": true, "start_time": "15:29:01.088161" } }, "success": false } minion/refresh/lkv-srvapp42.lkvbay.net { "Minion data cache refresh": "lkv-srvapp42.lkvbay.net", "_stamp": "2021-07-19T13:29:02.265064" } salt/batch/20210719132900368080/done { "_stamp": "2021-07-19T13:29:02.527884", "available_minions": [ "lkv-srvapp42.lkvbay.net" ], "done_minions": [ "lkv-srvapp42.lkvbay.net" ], "down_minions": [], "metadata": { "batch-mode": true, "suma-action-chain": false, "suma-action-id": 0, "suma-force-pkg-list-refresh": false, "suma-minion-startup": true }, "timedout_minions": [] } ^CException occurred in runner state.event: Traceback (most recent call last): File "/usr/lib/python3.6/site-packages/salt/client/mixins.py", line 385, in low data['return'] = func(*args, **kwargs) File "/usr/lib/python3.6/site-packages/salt/runners/state.py", line 313, in event node=node) File "/usr/lib/python3.6/site-packages/salt/modules/state.py", line 2412, in event ret = sevent.get_event(full=True, auto_reconnect=True) File "/usr/lib/python3.6/site-packages/salt/utils/event.py", line 638, in get_event ret = self._get_event(wait, tag, match_func, no_block) File "/usr/lib/python3.6/site-packages/salt/utils/event.py", line 544, in _get_event raw = self.subscriber.read_sync(timeout=wait) File "/usr/lib/python3.6/site-packages/salt/transport/ipc.py", line 693, in read_sync return self.io_loop.run_sync(lambda: self._read(timeout)) File "/usr/lib/python3.6/site-packages/salt/ext/tornado/ioloop.py", line 454, in run_sync self.start() File "/usr/lib/python3.6/site-packages/salt/ext/tornado/ioloop.py", line 865, in start event_pairs = self._impl.poll(poll_timeout) File "/usr/lib/python3.6/site-packages/salt/scripts.py", line 78, in _handle_signals hardcrash, trace=trace) File "/usr/lib/python3.6/site-packages/salt/scripts.py", line 43, in _handle_interrupt raise exc SystemExit: Exiting gracefully on Ctrl-c lkv-srvapp39:~ # Regards, Tobias. Von: Victor Zhestkov [mailto:Victor.Zhestkov@suse.com] Gesendet: Montag, 19. Juli 2021 15:20 Tobias, could you please collect the output of `salt-run state.event pretty=True` for this minion on restarting it. 1. stop salt-minion on the affected system 2. Run `salt-run state.event pretty=True` on SUSE Manager Server 3. start salt-minion on the affected system And show the events captured related to this minion Victor