Miguel Di Ciurcio Filho
2009-Nov-10 12:01 UTC
[CentOS] Xen pci passthru problems with kernel -164.6.1
Hi there,
After updating a server from CentOS 5.3 to 5.4 my Xen pci-passthru setup
has some troubles.
The server has two NICs, one used by dom0 and the other assigned to a
domU and one SCSI controler assigned to another domU. It has been
working fine since CentOS 5.1, when I did the initial setup.
After upgrading to 5.4, I get this error when starting the domU with one
of the NICs assigned:
Error: pci: improper device assignment specified: pci: 0000:07:00.0 must
be co-assigned to the same guest with 0000:07:00.1, but it is not owned
by pciback.
On the other hand, the domU with the SCSI controler boots, but the
driver inside the domU prints a lot of awful errors and does not work.
I've booted the domU with the SCSI controller using the -128.7.1 kernel
and everything works fine as before -164.6.1.
Any clues on why I can't assign just one NIC to the domU anymore?
# ls -l /sys/bus/pci/drivers/pciback/
total 0
lrwxrwxrwx 1 root root 0 Nov 10 09:55 0000:07:00.1 ->
../../../../devices/pci0000:00/0000:00:02.0/0000:01:00.0/0000:02:02.0/0000:07:00.1
lrwxrwxrwx 1 root root 0 Nov 10 09:55 0000:08:03.0 ->
../../../../devices/pci0000:00/0000:00:02.0/0000:01:00.3/0000:08:03.0
--w------- 1 root root 4096 Nov 10 09:55 bind
lrwxrwxrwx 1 root root 0 Nov 10 09:55 module ->
../../../../module/pciback
--w------- 1 root root 4096 Nov 10 09:55 new_id
--w------- 1 root root 4096 Nov 10 09:55 new_slot
-rw------- 1 root root 0 Nov 10 08:35 permissive
-rw------- 1 root root 0 Nov 10 08:35 quirks
--w------- 1 root root 4096 Nov 10 09:55 remove_id
--w------- 1 root root 4096 Nov 10 09:55 remove_slot
-r-------- 1 root root 4096 Nov 10 09:55 slots
--w------- 1 root root 4096 Nov 10 09:55 unbind
# Script do create the initrds
KVER="2.6.18-164.6.1.el5xen"
mkinitrd -f --omit-scsi-modules --omit-raid-modules \
--with=e1000e --with=xennet --with=xenblk \
--preload=xenblk /boot/initrd-$KVER-domU.img $KVER
mkinitrd -f --preload=pciback /boot/initrd-$KVER.img $KVER
# cat /etc/modprobe.conf
alias eth0 e1000e
options netloop nloopbacks=0
options pciback hide=(0000:07:00.1)(0000:08:03.0)
alias scsi_hostadapter2 megaraid_sas # this is the RAID controller
alias scsi_hostadapter3 ata_piix
# lspci |egrep "(Ether|SCSI)"
07:00.0 Ethernet controller: Intel Corporation 80003ES2LAN Gigabit
Ethernet Controller (Copper) (rev 01)
07:00.1 Ethernet controller: Intel Corporation 80003ES2LAN Gigabit
Ethernet Controller (Copper) (rev 01)
08:03.0 SCSI storage controller: LSI Logic / Symbios Logic 53c1030 PCI-X
Fusion-MPT Dual Ultra320 SCSI (rev 08)
# uname -a
Linux salvador.ic.unicamp.br 2.6.18-164.6.1.el5xen #1 SMP Tue Nov 3
16:48:13 EST 2009 x86_64 x86_64 x86_64 GNU/Linux
# domU-with-NIC.cfg
kernel = "/boot/vmlinuz-2.6.18-164.6.1.el5xen"
ramdisk = "/boot/initrd-2.6.18-164.6.1.el5xen-domU.img"
memory = "4096"
name = "taquaral"
disk = ['phy:/dev/volumes/taquaral-root,xvda1,w',
'phy:/dev/volumes/taquaral-home,xvda2,w',
'phy:/dev/volumes/taquaral-home-staff,xvda3,w',
'phy:/dev/volumes/taquaral-home-adm,xvda4,w',
'phy:/dev/volumes/taquaral-home-admsis,xvda5,w',
'phy:/dev/volumes/taquaral-home-phd,xvda6,w',
'phy:/dev/volumes/taquaral-home-spec,xvda9,w',
'phy:/dev/volumes/taquaral-swap,xvda7,w',
'phy:/dev/volumes/taquaral-var,xvda8,w']
vif = ['mac=00:16:3e:57:13:a5, bridge=vlan0']
vcpus = 4
extra = "console=xvc0"
root = "/dev/xvda1"
pci = ["0000:07:00.1"]
# domU-with-SCSI.cfg
kernel = "/boot/vmlinuz-2.6.18-128.7.1.el5xen"
ramdisk = "/boot/initrd-2.6.18-128.7.1.el5xen-domU.img"
memory = "2048"
name = "flamboyant"
disk = ['phy:/dev/volumes/flamboyant-root,xvda1,w',
'phy:/dev/volumes/flamboyant-var,xvda2,w',
'phy:/dev/volumes/flamboyant-l,xvda3,w',
'phy:/dev/volumes/flamboyant-swap,xvda4,w']
vif = ['mac=00:16:e3:55:55:55, bridge=br0', 'mac=00:16:3e:55:55:10,
bridge=vlan0']
vcpus = 2
extra = "console=xvc0"
root = "/dev/xvda1"
pci = ["0000:08:03.0"]
# menu.lst
title CentOS (2.6.18-164.6.1.el5xen)
root (hd0,0)
kernel /xen.gz-2.6.18-164.6.1.el5 dom0_mem=1G
module /vmlinuz-2.6.18-164.6.1.el5xen ro root=LABEL=/ enforcing=0
module /initrd-2.6.18-164.6.1.el5xen.img
Pasi Kärkkäinen
2009-Nov-10 12:14 UTC
[CentOS] Xen pci passthru problems with kernel -164.6.1
On Tue, Nov 10, 2009 at 10:01:32AM -0200, Miguel Di Ciurcio Filho wrote:> Hi there, > > After updating a server from CentOS 5.3 to 5.4 my Xen pci-passthru setup > has some troubles. > > The server has two NICs, one used by dom0 and the other assigned to a > domU and one SCSI controler assigned to another domU. It has been > working fine since CentOS 5.1, when I did the initial setup. > > After upgrading to 5.4, I get this error when starting the domU with one > of the NICs assigned: > > Error: pci: improper device assignment specified: pci: 0000:07:00.0 must > be co-assigned to the same guest with 0000:07:00.1, but it is not owned > by pciback. > > On the other hand, the domU with the SCSI controler boots, but the > driver inside the domU prints a lot of awful errors and does not work. > > I've booted the domU with the SCSI controller using the -128.7.1 kernel > and everything works fine as before -164.6.1. > > Any clues on why I can't assign just one NIC to the domU anymore? >I think 5.4 adds Xen VT-d pci passthrough support.. maybe that has caused bugs in the normal non-vt-d passthrough. Have you searched Redhat bugzilla about these problems? -- Pasi