Comment # 7 on bug 906589 from grant k

clean a single repo

    zypper clean --all OS13-update

tracing its' refresh

    strace zypper -v ref OS13-update

the grep'ing the output, I note a couple of lines

    grep -i key tmp.txt
        open("/lib64/libkeyutils.so.1", O_RDONLY|O_CLOEXEC) = 3
        lstat("/var/cache/zypp/raw/OS13-update3BEeJF/repodata/repomd.xml.key",
{st_mode=S_IFREG|0644, st_size=988, ...}) = 0
        unlink("/var/cache/zypp/raw/OS13-update3BEeJF/repodata/repomd.xml.key")
= 0
???        openat(AT_FDCWD, "/var/lib/rpm/pubkeys",
O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = -1 ENOENT (No such file or
directory)
???        openat(AT_FDCWD, "/var/lib/rpm/pubkeys",
O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = -1 ENOENT (No such file or
directory)
        stat("/var/cache/zypp/raw/OS13-update7VOndN/repodata/repomd.xml.key",
0x7fff967c53e8) = -1 ENOENT (No such file or directory)
        ...
        stat("/var/cache/zypp/raw/OS13-update/repodata/repomd.xml.key",
0x7fff967c5488) = -1 ENOENT (No such file or directory)
        ...
        open("/var/cache/zypp/raw/repo-update-non-oss/repodata/repomd.xml.key",
O_RDONLY) = 4
       
open("/var/adm/mount/AP_0xwjFA7t/repodata/repomd.xml.key.new.zypp.xaOotT",
O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC, 0600) = 4
        stat("/var/adm/mount/AP_0xwjFA7t/repodata/repomd.xml.key",
0x7fff967c4908) = -1 ENOENT (No such file or directory)
        write(1, "Retrieving: repomd.xml.key [", 28) = 28
        write(4, "-----BEGIN PGP PUBLIC KEY BLOCK-"..., 988) = 988
        pread(4, "-----BEGIN PGP PUBLIC KEY BLOCK-"..., 255, 0) = 255
       
open("/var/adm/mount/AP_0xwjFA7t/repodata/repomd.xml.key.new.zypp.xaOotT",
O_RDONLY|O_CLOEXEC) = 6
        pread(6, "-----BEGIN PGP PUBLIC KEY BLOCK-"..., 255, 0) = 255
       
rename("/var/adm/mount/AP_0xwjFA7t/repodata/repomd.xml.key.new.zypp.xaOotT",
"/var/adm/mount/AP_0xwjFA7t/repodata/repomd.xml.key") = 0
        stat("/var/adm/mount/AP_0xwjFA7t/repodata/repomd.xml.key",
{st_mode=S_IFREG|0644, st_size=988, ...}) = 0
        Retrieving: repomd.xml.key [..done]
        lstat("/var/adm/mount/AP_0xwjFA7t/repodata/repomd.xml.key",
{st_mode=S_IFREG|0644, st_size=988, ...}) = 0
        lstat("/var/cache/zypp/raw/OS13-update7VOndN/repodata/repomd.xml.key",
0x7fff967c5498) = -1 ENOENT (No such file or directory)
???        link("/var/adm/mount/AP_0xwjFA7t/repodata/repomd.xml.key",
"/var/cache/zypp/raw/OS13-update7VOndN/repodata/repomd.xml.key") = -1 EXDEV
(Invalid cross-device link)
        stat("/var/adm/mount/AP_0xwjFA7t/repodata/repomd.xml.key",
{st_mode=S_IFREG|0644, st_size=988, ...}) = 0
        stat("/var/cache/zypp/raw/OS13-update7VOndN/repodata/repomd.xml.key",
0x7fff967c5378) = -1 ENOENT (No such file or directory)
        stat("/var/adm/mount/AP_0xwjFA7t/repodata/repomd.xml.key",
{st_mode=S_IFREG|0644, st_size=988, ...}) = 0
        unlink("/var/adm/mount/AP_0xwjFA7t/repodata/repomd.xml.key") = 0
        stat("/var/cache/zypp/raw/OS13-update7VOndN/repodata/repomd.xml.key",
{st_mode=S_IFREG|0644, st_size=988, ...}) = 0
        stat("/var/cache/zypp/raw/OS13-update7VOndN/repodata/repomd.xml.key",
{st_mode=S_IFREG|0644, st_size=988, ...}) = 0
        stat("/var/cache/zypp/raw/OS13-update7VOndN/repodata/repomd.xml.key",
{st_mode=S_IFREG|0644, st_size=988, ...}) = 0
        stat("/var/cache/zypp/raw/OS13-update7VOndN/repodata/repomd.xml.key",
{st_mode=S_IFREG|0644, st_size=988, ...}) = 0
        open("/var/cache/zypp/raw/OS13-update7VOndN/repodata/repomd.xml.key",
O_RDONLY) = 4
        read(4, "-----BEGIN PGP PUBLIC KEY BLOCK-"..., 8191) = 988
        lstat("/var/cache/zypp/raw/OS13-update7VOndN/repodata/repomd.xml.key",
{st_mode=S_IFREG|0644, st_size=988, ...}) = 0
???       
link("/var/cache/zypp/raw/OS13-update7VOndN/repodata/repomd.xml.key",
"/var/tmp/TmpFile.IxTfvC") = -1 EXDEV (Invalid cross-device link)
        stat("/var/cache/zypp/raw/OS13-update7VOndN/repodata/repomd.xml.key",
{st_mode=S_IFREG|0644, st_size=988, ...}) = 0
        read(4, "-----BEGIN PGP PUBLIC KEY BLOCK-"..., 8191) = 988
        mkdir("/var/tmp/zypp.0sJZDK/fake-keyringGpXehV", 0700) = 0
        lstat("/var/tmp/zypp.0sJZDK/fake-keyringGpXehV", {st_mode=S_IFDIR|0700,
st_size=4096, ...}) = 0
        stat("/var/tmp/zypp.0sJZDK/fake-keyringGpXehV", {st_mode=S_IFDIR|0700,
st_size=4096, ...}) = 0
        openat(AT_FDCWD, "/var/tmp/zypp.0sJZDK/fake-keyringGpXehV",
O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 4
        lstat("/var/tmp/zypp.0sJZDK/fake-keyringGpXehV/pubring.kbx",
{st_mode=S_IFREG|0600, st_size=32, ...}) = 0
        unlink("/var/tmp/zypp.0sJZDK/fake-keyringGpXehV/pubring.kbx") = 0
        rmdir("/var/tmp/zypp.0sJZDK/fake-keyringGpXehV") = 0
        ...

digging around,

    ls -al /var/lib/rpm/pubkeys
        ls: cannot access /var/lib/rpm/pubkeys: No such file or directory

but there IS, noting the case difference,

    ls -al /var/lib/rpm/*ubkeys
        -rw-r--r-- 1 root root 12K Nov 28  2011 /var/lib/rpm/Pubkeys


also,

    link("/var/cache/zypp/raw/OS13-update7VOndN/repodata/repomd.xml.key",
"/var/tmp/TmpFile.IxTfvC") = -1 EXDEV (Invalid cross-device link)

not sure whether that's causal. but on our machines, /var/cache is always on a
separate mount

    mount | egrep "/var/cache"
        /dev/mapper/VG1-VARCACHE on /var/cache type ext4
(rw,noatime,nobarrier,stripe=512,data=ordered)

iiuc, hardlinks cannot be created between different partitions, only symlinks
can

if that link is being created by zypper as a HARD link, then that's a possible
problem.  is it?