[solved] Host system crashes on qemu with DMA remapping

Message

tuner23 · Post by **tuner23** » Sun May 08, 2016 5:37 am

Hi,

as already told my host system crashes completely when trying to start a guest system with -device pci-assign,host=01:00.0 option.
It worked very well before upgrading my system, so either it depends on new missing kernel options, new libraries or maybe missing useflags, or udev..?..

The only output i get is when running with strace (i'll first only post the bottom part of the strace here for better readability):

Code: Select all

brk(0x557c3c138000)                     = 0x557c3c138000
futex(0x557c3a94eb54, FUTEX_CMP_REQUEUE_PRIVATE, 1, 2147483647, 0x557c38b1a6c0, 2) = 1
tgkill(8893, 8898, SIGUSR1)             = 0
futex(0x557c38b1a6c0, FUTEX_WAKE_PRIVATE, 1) = 1
futex(0x557c38b1a5a4, FUTEX_WAIT_PRIVATE, 1, NULL) = 0
futex(0x557c38b1a6c0, FUTEX_WAIT_PRIVATE, 2, NULL) = 0
futex(0x557c38b1a6c0, FUTEX_WAIT_PRIVATE, 2, NULL) = 0
open("/sys/bus/pci/devices/0000:01:00.0/config", O_RDWR) = 14
read(14, "\354\20h\201\3\1\20@\1\0\0\2\10\0\0\0\1\250\0\0\0\0\0\0\4\360\255\373\0\0\0\0"..., 256) = 256
open("/sys/bus/pci/devices/0000:01:00.0/resource", O_RDONLY) = 15
fstat(15, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f03c4e1d000
read(15, "0x000000000000a800 0x00000000000"..., 4096) = 741
open("/sys/bus/pci/devices/0000:01:00.0/resource0", O_RDWR) = 16
open("/sys/bus/pci/devices/0000:01:00.0/resource2", O_RDWR) = 17
close(15)                               = 0
munmap(0x7f03c4e1d000, 4096)            = 0
open("/sys/bus/pci/devices/0000:01:00.0/vendor", O_RDONLY) = 15
fstat(15, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f03c4e1d000
read(15, "0x10ec\n", 4096)              = 7
read(15, "", 4096)                      = 0
close(15)                               = 0
munmap(0x7f03c4e1d000, 4096)            = 0
open("/sys/bus/pci/devices/0000:01:00.0/device", O_RDONLY) = 15
fstat(15, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f03c4e1d000
read(15, "0x8168\n", 4096)              = 7
read(15, "", 4096)                      = 0
close(15)                               = 0
munmap(0x7f03c4e1d000, 4096)            = 0
pread(14, "\20", 1, 6)                  = 1
pread(14, "@", 1, 52)                   = 1
pread(14, "\1", 1, 64)                  = 1
pread(14, "H", 1, 65)                   = 1
pread(14, "\3", 1, 72)                  = 1
pread(14, "P", 1, 73)                   = 1
pread(14, "\5", 1, 80)                  = 1
ioctl(7, KVM_CHECK_EXTENSION, 0x1d)     = 1
pread(14, "\20", 1, 6)                  = 1
pread(14, "@", 1, 52)                   = 1
pread(14, "\1", 1, 64)                  = 1
pread(14, "H", 1, 65)                   = 1
pread(14, "\3", 1, 72)                  = 1
pread(14, "P", 1, 73)                   = 1
pread(14, "\5", 1, 80)                  = 1
pread(14, "`", 1, 81)                   = 1
pread(14, "\20", 1, 96)                 = 1
pread(14, "\204", 1, 97)                = 1
pread(14, "\t", 1, 132)                 = 1
pread(14, "\0", 1, 133)                 = 1
pread(14, "\20", 1, 6)                  = 1
pread(14, "@", 1, 52)                   = 1
pread(14, "\1", 1, 64)                  = 1
pread(14, "\20", 1, 6)                  = 1
pread(14, "@", 1, 52)                   = 1
pread(14, "\1", 1, 64)                  = 1
pread(14, "H", 1, 65)                   = 1
pread(14, "\3", 1, 72)                  = 1
pread(14, "P", 1, 73)                   = 1
pread(14, "\5", 1, 80)                  = 1
pread(14, "`", 1, 81)                   = 1
pread(14, "\20", 1, 96)                 = 1
pread(14, "\20", 1, 6)                  = 1
pread(14, "@", 1, 52)                   = 1
pread(14, "\1", 1, 64)                  = 1
pread(14, "H", 1, 65)                   = 1
pread(14, "\3", 1, 72)                  = 1
pread(14, "P", 1, 73)                   = 1
pread(14, "\5", 1, 80)                  = 1
pread(14, "`", 1, 81)                   = 1
pread(14, "\20", 1, 96)                 = 1
pread(14, "\204", 1, 97)                = 1
pread(14, "\t", 1, 132)                 = 1
pread(14, "\0", 1, 133)                 = 1
pread(14, "\20", 1, 6)                  = 1
pread(14, "@", 1, 52)                   = 1
pread(14, "\1", 1, 64)                  = 1
pread(14, "H", 1, 65)                   = 1
pread(14, "\3", 1, 72)                  = 1
pread(14, "\20", 1, 6)                  = 1
pread(14, "@", 1, 52)                   = 1
pread(14, "\1", 1, 64)                  = 1
pread(14, "H", 1, 65)                   = 1
pread(14, "\3", 1, 72)                  = 1
pread(14, "P", 1, 73)                   = 1
pread(14, "\5", 1, 80)                  = 1
pread(14, "`", 1, 81)                   = 1
pread(14, "\20", 1, 96)                 = 1
pread(14, "\204", 1, 97)                = 1
pread(14, "\t", 1, 132)                 = 1
pread(14, "\20", 1, 6)                  = 1
pread(14, "\0", 1, 133)                 = 1
pread(14, "\20", 1, 6)                  = 1
pread(16, 0x7fff1bf0b918, 3, 0)         = -1 EINVAL (Invalid argument)
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_SHARED, 17, 0) = 0x7f03c4e1d000
madvise(0x7f03c4e1d000, 4096, MADV_HUGEPAGE) = -1 EINVAL (Invalid argument)
madvise(0x7f03c4e1d000, 4096, MADV_DONTFORK) = 0
ioctl(7, KVM_CHECK_EXTENSION, 0x10)     = 1
ioctl(7, KVM_CHECK_EXTENSION, 0x2f)     = 1
ioctl(7, KVM_CHECK_EXTENSION, 0x12)     = 1
ioctl(8, KVM_ASSIGN_PCI_DEVICE, 0x7fff1bf0b860) = 0
pread(14, "\1", 1, 61)                  = 1
ioctl(7, KVM_CHECK_EXTENSION, 0x1d)     = 1
ioctl(8, KVM_ASSIGN_DEV_IRQ, 0x7fff1bf0b800) = 0
stat("/sys/bus/pci/devices/0000:01:00.0/rom", {st_mode=S_IFREG|0600, st_size=131072, ...}) = 0
access("/sys/bus/pci/devices/0000:01:00.0/rom", F_OK) = 0
open("/sys/bus/pci/devices/0000:01:00.0/rom", O_RDWR) = 15
fstat(15, {st_mode=S_IFREG|0600, st_size=131072, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f03c4e1c000
write(15, "\1", 1)                      = 1
lseek(15, 0, SEEK_SET)                  = 0
mmap(NULL, 2224128, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f03bc9e1000
munmap(0x7f03bc9e1000, 126976)          = 0
munmap(0x7f03bca20000, 1966080)         = 0
madvise(0x7f03bca00000, 131072, MADV_MERGEABLE) = 0
madvise(0x7f03bca00000, 131072, MADV_HUGEPAGE) = 0
madvise(0x7f03bca00000, 131072, MADV_DONTFORK) = 0
ioctl(7, KVM_CHECK_EXTENSION, 0x10)     = 1
read(15,

..and here it crashes.

What i do:

Code: Select all

# lspci | grep Eth
01:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8411 PCI Express Gigabit Ethernet Controller (rev 01)

# lspci -n
...
01:00.0 0200: 10ec:8168 (rev 01)

Code: Select all

echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts
echo "10ec 8168" > /sys/bus/pci/drivers/pci-stub/new_id
echo "0000:01:00.0" > /sys/bus/pci/devices/0000:01:00.0/driver/unbind
echo "0000:01:00.0" > /sys/bus/pci/drivers/pci-stub/bind

Code: Select all

# qemu-system-x86_64 -enable-kvm -drive file=/vms/prod/res/res.iso,if=virtio,format=raw -drive file=/vms/prod/res/swap,if=virtio,format=raw -drive file=/vms/prod/res/resdata.iso,if=virtio,format=raw -m 512 -nographic -kernel /data/kernels/vmlinuz-2.6.36-gentoo-r8 -append "root=/dev/vda console=ttyS0 earlyprintk=serial" -net nic,model=virtio,macaddr=DE:AD:BE:EF:2D:AE -net tap,ifname=tapres0,script=/etc/qemu/qemu-ifup -device pci-assign,host=01:00.0

The dmesg output:

Code: Select all

# dmesg | grep -e DMAR -e IOMMU
[    0.000000] ACPI: DMAR 0x00000000BF78E0C0 000090 (v01 AMI    OEMDMAR  00000001 MSFT 00000097)
[    0.035072] dmar: IOMMU 0: reg_base_addr fed90000 ver 1:0 cap c90780106f0462 ecap f020e3
[    0.510972] DMAR: No ATSR found
[    0.511274] IOMMU: dmar0 using Queued invalidation
[    0.511410] IOMMU: Setting RMRR:
[    0.511547] IOMMU: Setting identity map for device 0000:00:1a.0 [0xbf7ec000 - 0xbf7fffff]
[    0.511791] IOMMU: Setting identity map for device 0000:00:1d.0 [0xbf7ec000 - 0xbf7fffff]
[    0.512036] IOMMU: Setting identity map for device 0000:00:1a.0 [0xe4000 - 0xe7fff]
[    0.512271] IOMMU: Setting identity map for device 0000:00:1d.0 [0xe4000 - 0xe7fff]
[    0.512507] IOMMU: Prepare 0-16MiB unity mapping for LPC
[    0.512646] IOMMU: Setting identity map for device 0000:00:1f.0 [0x0 - 0xffffff]
...
[1073216.198983] kworker/dying (28210) used greatest stack depth: 11400 bytes left
[1073505.699646] pci-stub 0000:01:00.0: claimed by stub
[1073511.177631] br0: port 1(tapfw0) entered forwarding state
[1073511.177640] br0: port 1(tapfw0) entered forwarding state
[1073511.227594] kvm_iommu_map_guest: No interrupt remapping support, disallowing device assignment. Re-enble with "allow_unsafe_assigned_interrupts=1" module option.
[1073512.177318] br0: port 1(tapfw0) entered disabled state

Installed software:

Code: Select all

# uname -r
4.1.12-gentoo

# eix qemu
[I] app-emulation/qemu
     Installed versions:  2.4.1-r2(08:19:40 26.04.2016)(aio alsa caps curl fdt filecaps jpeg ncurses nls pin-upstream-blobs png python sasl sdl seccomp threads uuid vde vhost-net vnc xattr -accessibility -bluetooth -debug -glusterfs -gtk -gtk2 -infiniband -iscsi -lzo -nfs -numa -opengl -pulseaudio -rbd -sdl2 -selinux -smartcard -snappy -spice -ssh -static -static-softmmu -static-user -systemtap -tci -test -tls -usb -usbredir -virtfs -vte -xen -xfs KERNEL="linux -FreeBSD" PYTHON_TARGETS="python2_7" QEMU_SOFTMMU_TARGETS="i386 m68k x86_64 -aarch64 -alpha -arm -cris -lm32 -microblaze -microblazeel -mips -mips64 -mips64el -mipsel -moxie -or32 -ppc -ppc64 -ppcemb -s390x -sh4 -sh4eb -sparc -sparc64 -tricore -unicore32 -xtensa -xtensaeb" QEMU_USER_TARGETS="i386 m68k x86_64 -aarch64 -alpha -arm -armeb -cris -microblaze -microblazeel -mips -mips64 -mips64el -mipsel -mipsn32 -mipsn32el -or32 -ppc -ppc64 -ppc64abi32 -ppc64le -s390x -sh4 -sh4eb -sparc -sparc32plus -sparc64 -unicore32")

Kernel options (i think on the last kernel there was more options..):

Code: Select all

# cat .config | grep DMAR
CONFIG_DMAR_TABLE=y

Can someone help me out here? I don't know anymore where to search for that error..

Thanks,
antonios.

tuner23 · Post by **tuner23** » Thu May 26, 2016 9:25 am

Ok, i just found out, where the problem was.

I had to pass intel_iommu=on as boot-param. In my old kernel version this wasn't necessary..

Now the bootup looks better:

Code: Select all

# dmesg|grep -e DMAR -e IOMMU
[    0.000000] ACPI: DMAR 0x00000000BF78E0C0 000090 (v01 AMI    OEMDMAR  00000001 MSFT 00000097)
[    0.000000] Intel-IOMMU: enabled
[    0.035092] dmar: IOMMU 0: reg_base_addr fed90000 ver 1:0 cap c90780106f0462 ecap f020e3
[    0.510844] DMAR: No ATSR found
[    0.511145] IOMMU: dmar0 using Queued invalidation
[    0.511284] IOMMU: Setting RMRR:
[    0.511422] IOMMU: Setting identity map for device 0000:00:1a.0 [0xbf7ec000 - 0xbf7fffff]
[    0.511673] IOMMU: Setting identity map for device 0000:00:1d.0 [0xbf7ec000 - 0xbf7fffff]
[    0.511911] IOMMU: Setting identity map for device 0000:00:1a.0 [0xe4000 - 0xe7fff]
[    0.512147] IOMMU: Setting identity map for device 0000:00:1d.0 [0xe4000 - 0xe7fff]
[    0.512381] IOMMU: Prepare 0-16MiB unity mapping for LPC
[    0.512520] IOMMU: Setting identity map for device 0000:00:1f.0 [0x0 - 0xffffff]

.. but i wonder somehow, why the vm boots up and crashes, when iommu is not activated at all. should be easy to check.

Thanks,
Antonis.