Jan Beulich
2011-Aug-25 14:57 UTC
[Xen-devel] [PATCH, RFC 3/7] PCI multi-seg: adjust domctl interface
Again, a couple of directly related functions at once get adjusted to
account for the segment number.
Can we re-define bit assignments for the four uses of machine_bdf in
the domctl interface?
Signed-off-by: Jan Beulich <jbeulich@novell.com>
--- 2011-08-25.orig/tools/libxl/libxl_pci.c 2011-07-25 09:10:24.000000000 +0200
+++ 2011-08-25/tools/libxl/libxl_pci.c 2011-08-25 15:06:40.000000000 +0200
@@ -45,7 +45,7 @@ static unsigned int pcidev_encode_bdf(li
{
unsigned int value;
- value = 0;
+ value = pcidev->domain << 24; /* XXX losing to 8 bits */
value |= (pcidev->bus & 0xff) << 16;
value |= (pcidev->dev & 0x1f) << (8+3);
value |= (pcidev->func & 0x7) << (8+0);
--- 2011-08-25.orig/tools/python/xen/lowlevel/xc/xc.c 2010-11-05
09:22:58.000000000 +0100
+++ 2011-08-25/tools/python/xen/lowlevel/xc/xc.c 2011-08-25 15:06:40.000000000
+0200
@@ -619,6 +619,7 @@ static PyObject *pyxc_test_assign_device
while ( next_bdf(&pci_str, &seg, &bus, &dev, &func) )
{
+ bdf = seg << 24; /* XXX losing top 8 bits */
bdf |= (bus & 0xff) << 16;
bdf |= (dev & 0x1f) << 11;
bdf |= (func & 0x7) << 8;
@@ -651,6 +652,7 @@ static PyObject *pyxc_assign_device(XcOb
while ( next_bdf(&pci_str, &seg, &bus, &dev, &func) )
{
+ bdf = seg << 24; /* XXX losing top 8 bits */
bdf |= (bus & 0xff) << 16;
bdf |= (dev & 0x1f) << 11;
bdf |= (func & 0x7) << 8;
@@ -683,6 +685,7 @@ static PyObject *pyxc_deassign_device(Xc
while ( next_bdf(&pci_str, &seg, &bus, &dev, &func) )
{
+ bdf = seg << 24; /* XXX losing top 8 bits */
bdf |= (bus & 0xff) << 16;
bdf |= (dev & 0x1f) << 11;
bdf |= (func & 0x7) << 8;
@@ -720,6 +723,7 @@ static PyObject *pyxc_get_device_group(X
if (sdev_array == NULL)
return PyErr_NoMemory();
+ bdf = seg << 24; /* XXX losing top 8 bits */
bdf |= (bus & 0xff) << 16;
bdf |= (dev & 0x1f) << 11;
bdf |= (func & 0x7) << 8;
--- 2011-08-25.orig/xen/arch/x86/domctl.c 2011-06-16 09:21:02.000000000 +0200
+++ 2011-08-25/xen/arch/x86/domctl.c 2011-08-25 15:06:40.000000000 +0200
@@ -746,6 +746,7 @@ long arch_do_domctl(
{
struct domain *d;
u32 max_sdevs;
+ u16 seg;
u8 bus, devfn;
XEN_GUEST_HANDLE_64(uint32) sdevs;
int num_sdevs;
@@ -758,12 +759,14 @@ long arch_do_domctl(
if ( (d = rcu_lock_domain_by_id(domctl->domain)) == NULL )
break;
+ seg = domctl->u.get_device_group.machine_bdf >> 24;
bus = (domctl->u.get_device_group.machine_bdf >> 16) &
0xff;
devfn = (domctl->u.get_device_group.machine_bdf >> 8) &
0xff;
max_sdevs = domctl->u.get_device_group.max_sdevs;
sdevs = domctl->u.get_device_group.sdev_array;
- num_sdevs = iommu_get_device_group(d, bus, devfn, sdevs, max_sdevs);
+ num_sdevs = iommu_get_device_group(d, seg, bus, devfn,
+ sdevs, max_sdevs);
if ( num_sdevs < 0 )
{
dprintk(XENLOG_ERR, "iommu_get_device_group()
failed!\n");
@@ -783,6 +786,7 @@ long arch_do_domctl(
case XEN_DOMCTL_test_assign_device:
{
+ u16 seg;
u8 bus, devfn;
ret = -ENOSYS;
@@ -794,14 +798,15 @@ long arch_do_domctl(
break;
ret = -EINVAL;
+ seg = domctl->u.get_device_group.machine_bdf >> 24;
bus = (domctl->u.assign_device.machine_bdf >> 16) & 0xff;
devfn = (domctl->u.assign_device.machine_bdf >> 8) & 0xff;
- if ( device_assigned(bus, devfn) )
+ if ( device_assigned(seg, bus, devfn) )
{
gdprintk(XENLOG_ERR, "XEN_DOMCTL_test_assign_device: "
- "%x:%x.%x already assigned, or non-existent\n",
- bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
+ "%04x:%02x:%02x.%u already assigned, or
non-existent\n",
+ seg, bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
break;
}
ret = 0;
@@ -811,6 +816,7 @@ long arch_do_domctl(
case XEN_DOMCTL_assign_device:
{
struct domain *d;
+ u16 seg;
u8 bus, devfn;
ret = -ENOSYS;
@@ -829,14 +835,15 @@ long arch_do_domctl(
if ( ret )
goto assign_device_out;
+ seg = domctl->u.get_device_group.machine_bdf >> 24;
bus = (domctl->u.assign_device.machine_bdf >> 16) & 0xff;
devfn = (domctl->u.assign_device.machine_bdf >> 8) & 0xff;
- ret = assign_device(d, bus, devfn);
+ ret = assign_device(d, seg, bus, devfn);
if ( ret )
gdprintk(XENLOG_ERR, "XEN_DOMCTL_assign_device: "
- "assign device (%x:%x.%x) failed\n",
- bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
+ "assign device (%04x:%02x:%02x.%u) failed\n",
+ seg, bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
assign_device_out:
put_domain(d);
@@ -846,6 +853,7 @@ long arch_do_domctl(
case XEN_DOMCTL_deassign_device:
{
struct domain *d;
+ u16 seg;
u8 bus, devfn;
ret = -ENOSYS;
@@ -864,16 +872,17 @@ long arch_do_domctl(
if ( ret )
goto deassign_device_out;
+ seg = domctl->u.get_device_group.machine_bdf >> 24;
bus = (domctl->u.assign_device.machine_bdf >> 16) & 0xff;
devfn = (domctl->u.assign_device.machine_bdf >> 8) & 0xff;
spin_lock(&pcidevs_lock);
- ret = deassign_device(d, bus, devfn);
+ ret = deassign_device(d, seg, bus, devfn);
spin_unlock(&pcidevs_lock);
if ( ret )
gdprintk(XENLOG_ERR, "XEN_DOMCTL_deassign_device: "
- "deassign device (%x:%x.%x) failed\n",
- bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
+ "deassign device (%04x:%02x:%02x.%u) failed\n",
+ seg, bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
deassign_device_out:
put_domain(d);
--- 2011-08-25.orig/xen/drivers/passthrough/amd/pci_amd_iommu.c 2011-08-25
15:06:35.000000000 +0200
+++ 2011-08-25/xen/drivers/passthrough/amd/pci_amd_iommu.c 2011-08-25
15:06:40.000000000 +0200
@@ -288,7 +288,7 @@ static void amd_iommu_disable_domain_dev
}
static int reassign_device( struct domain *source, struct domain *target,
- u8 bus, u8 devfn)
+ u16 seg, u8 bus, u8 devfn)
{
struct pci_dev *pdev;
struct amd_iommu *iommu;
@@ -296,7 +296,7 @@ static int reassign_device( struct domai
struct hvm_iommu *t = domain_hvm_iommu(target);
ASSERT(spin_is_locked(&pcidevs_lock));
- pdev = pci_get_pdev_by_domain(source, 0, bus, devfn);
+ pdev = pci_get_pdev_by_domain(source, seg, bus, devfn);
if ( !pdev )
return -ENODEV;
@@ -329,7 +329,7 @@ static int reassign_device( struct domai
return 0;
}
-static int amd_iommu_assign_device(struct domain *d, u8 bus, u8 devfn)
+static int amd_iommu_assign_device(struct domain *d, u16 seg, u8 bus, u8 devfn)
{
int bdf = (bus << 8) | devfn;
int req_id = get_dma_requestor_id(bdf);
@@ -344,7 +344,7 @@ static int amd_iommu_assign_device(struc
ivrs_mappings[req_id].read_permission);
}
- return reassign_device(dom0, d, bus, devfn);
+ return reassign_device(dom0, d, seg, bus, devfn);
}
static void deallocate_next_page_table(struct page_info* pg, int level)
@@ -409,9 +409,9 @@ static void amd_iommu_domain_destroy(str
}
static int amd_iommu_return_device(
- struct domain *s, struct domain *t, u8 bus, u8 devfn)
+ struct domain *s, struct domain *t, u16 seg, u8 bus, u8 devfn)
{
- return reassign_device(s, t, bus, devfn);
+ return reassign_device(s, t, seg, bus, devfn);
}
static int amd_iommu_add_device(struct pci_dev *pdev)
@@ -458,7 +458,7 @@ static int amd_iommu_remove_device(struc
return 0;
}
-static int amd_iommu_group_id(u8 bus, u8 devfn)
+static int amd_iommu_group_id(u16 seg, u8 bus, u8 devfn)
{
int rt;
int bdf = (bus << 8) | devfn;
--- 2011-08-25.orig/xen/drivers/passthrough/iommu.c 2011-08-25
15:06:35.000000000 +0200
+++ 2011-08-25/xen/drivers/passthrough/iommu.c 2011-08-25 15:06:40.000000000
+0200
@@ -165,7 +165,7 @@ int iommu_remove_device(struct pci_dev *
return hd->platform_ops->remove_device(pdev);
}
-int assign_device(struct domain *d, u8 bus, u8 devfn)
+int assign_device(struct domain *d, u16 seg, u8 bus, u8 devfn)
{
struct hvm_iommu *hd = domain_hvm_iommu(d);
int rc = 0;
@@ -174,7 +174,7 @@ int assign_device(struct domain *d, u8 b
return 0;
spin_lock(&pcidevs_lock);
- if ( (rc = hd->platform_ops->assign_device(d, bus, devfn)) )
+ if ( (rc = hd->platform_ops->assign_device(d, seg, bus, devfn)) )
goto done;
if ( has_arch_pdevs(d) && !need_iommu(d) )
@@ -272,7 +272,7 @@ int iommu_unmap_page(struct domain *d, u
}
/* caller should hold the pcidevs_lock */
-int deassign_device(struct domain *d, u8 bus, u8 devfn)
+int deassign_device(struct domain *d, u16 seg, u8 bus, u8 devfn)
{
struct hvm_iommu *hd = domain_hvm_iommu(d);
struct pci_dev *pdev = NULL;
@@ -282,7 +282,7 @@ int deassign_device(struct domain *d, u8
return -EINVAL;
ASSERT(spin_is_locked(&pcidevs_lock));
- pdev = pci_get_pdev(0, bus, devfn);
+ pdev = pci_get_pdev(seg, bus, devfn);
if ( !pdev )
return -ENODEV;
@@ -293,12 +293,12 @@ int deassign_device(struct domain *d, u8
return -EINVAL;
}
- ret = hd->platform_ops->reassign_device(d, dom0, bus, devfn);
+ ret = hd->platform_ops->reassign_device(d, dom0, seg, bus, devfn);
if ( ret )
{
dprintk(XENLOG_ERR VTDPREFIX,
- "d%d: Deassign device (%x:%x.%x) failed!\n",
- d->domain_id, bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
+ "d%d: Deassign device (%04x:%02x:%02x.%u) failed!\n",
+ d->domain_id, seg, bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
return ret;
}
@@ -347,7 +347,7 @@ int __init iommu_setup(void)
return rc;
}
-int iommu_get_device_group(struct domain *d, u8 bus, u8 devfn,
+int iommu_get_device_group(struct domain *d, u16 seg, u8 bus, u8 devfn,
XEN_GUEST_HANDLE_64(uint32) buf, int max_sdevs)
{
struct hvm_iommu *hd = domain_hvm_iommu(d);
@@ -360,15 +360,16 @@ int iommu_get_device_group(struct domain
if ( !iommu_enabled || !ops || !ops->get_device_group_id )
return 0;
- group_id = ops->get_device_group_id(bus, devfn);
+ group_id = ops->get_device_group_id(seg, bus, devfn);
spin_lock(&pcidevs_lock);
for_each_pdev( d, pdev )
{
- if ( (pdev->bus == bus) && (pdev->devfn == devfn) )
+ if ( (pdev->seg != seg) ||
+ ((pdev->bus == bus) && (pdev->devfn == devfn)) )
continue;
- sdev_id = ops->get_device_group_id(pdev->bus, pdev->devfn);
+ sdev_id = ops->get_device_group_id(seg, pdev->bus,
pdev->devfn);
if ( (sdev_id == group_id) && (i < max_sdevs) )
{
bdf = 0;
--- 2011-08-25.orig/xen/drivers/passthrough/pci.c 2011-08-25 15:06:35.000000000
+0200
+++ 2011-08-25/xen/drivers/passthrough/pci.c 2011-08-25 15:06:40.000000000 +0200
@@ -441,11 +441,12 @@ void pci_release_devices(struct domain *
while ( (pdev = pci_get_pdev_by_domain(d, -1, -1, -1)) )
{
pci_cleanup_msi(pdev);
- bus = pdev->bus; devfn = pdev->devfn;
- if ( deassign_device(d, bus, devfn) )
- printk("domain %d: deassign device (%02x:%02x.%x)
failed!\n",
- d->domain_id, pdev->bus, PCI_SLOT(pdev->devfn),
- PCI_FUNC(pdev->devfn));
+ bus = pdev->bus;
+ devfn = pdev->devfn;
+ if ( deassign_device(d, pdev->seg, bus, devfn) )
+ printk("domain %d: deassign device (%04x:%02x:%02x.%u)
failed!\n",
+ d->domain_id, pdev->seg, bus,
+ PCI_SLOT(devfn), PCI_FUNC(devfn));
}
spin_unlock(&pcidevs_lock);
}
--- 2011-08-25.orig/xen/drivers/passthrough/vtd/iommu.c 2011-08-25
15:06:35.000000000 +0200
+++ 2011-08-25/xen/drivers/passthrough/vtd/iommu.c 2011-08-25 15:06:40.000000000
+0200
@@ -1573,13 +1573,13 @@ out:
static int reassign_device_ownership(
struct domain *source,
struct domain *target,
- u8 bus, u8 devfn)
+ u16 seg, u8 bus, u8 devfn)
{
struct pci_dev *pdev;
int ret;
ASSERT(spin_is_locked(&pcidevs_lock));
- pdev = pci_get_pdev_by_domain(source, 0, bus, devfn);
+ pdev = pci_get_pdev_by_domain(source, seg, bus, devfn);
if (!pdev)
return -ENODEV;
@@ -2117,12 +2117,12 @@ int __init intel_vtd_setup(void)
* If the device isn''t owned by dom0, it means it already
* has been assigned to other domain, or it''s not exist.
*/
-int device_assigned(u8 bus, u8 devfn)
+int device_assigned(u16 seg, u8 bus, u8 devfn)
{
struct pci_dev *pdev;
spin_lock(&pcidevs_lock);
- pdev = pci_get_pdev_by_domain(dom0, 0, bus, devfn);
+ pdev = pci_get_pdev_by_domain(dom0, seg, bus, devfn);
if (!pdev)
{
spin_unlock(&pcidevs_lock);
@@ -2133,7 +2133,8 @@ int device_assigned(u8 bus, u8 devfn)
return 0;
}
-static int intel_iommu_assign_device(struct domain *d, u8 bus, u8 devfn)
+static int intel_iommu_assign_device(
+ struct domain *d, u16 seg, u8 bus, u8 devfn)
{
struct acpi_rmrr_unit *rmrr;
int ret = 0, i;
@@ -2144,7 +2145,7 @@ static int intel_iommu_assign_device(str
return -ENODEV;
ASSERT(spin_is_locked(&pcidevs_lock));
- pdev = pci_get_pdev(0, bus, devfn);
+ pdev = pci_get_pdev(seg, bus, devfn);
if (!pdev)
return -ENODEV;
@@ -2155,7 +2156,7 @@ static int intel_iommu_assign_device(str
return -EBUSY;
}
- ret = reassign_device_ownership(dom0, d, bus, devfn);
+ ret = reassign_device_ownership(dom0, d, seg, bus, devfn);
if ( ret )
goto done;
@@ -2187,7 +2188,7 @@ done:
return ret;
}
-static int intel_iommu_group_id(u8 bus, u8 devfn)
+static int intel_iommu_group_id(u16 seg, u8 bus, u8 devfn)
{
u8 secbus;
if ( find_upstream_bridge(&bus, &devfn, &secbus) < 0 )
--- 2011-08-25.orig/xen/include/xen/iommu.h 2011-08-25 15:06:23.000000000 +0200
+++ 2011-08-25/xen/include/xen/iommu.h 2011-08-25 15:06:40.000000000 +0200
@@ -74,10 +74,10 @@ int iommu_remove_device(struct pci_dev *
int iommu_domain_init(struct domain *d);
void iommu_dom0_init(struct domain *d);
void iommu_domain_destroy(struct domain *d);
-int device_assigned(u8 bus, u8 devfn);
-int assign_device(struct domain *d, u8 bus, u8 devfn);
-int deassign_device(struct domain *d, u8 bus, u8 devfn);
-int iommu_get_device_group(struct domain *d, u8 bus, u8 devfn,
+int device_assigned(u16 seg, u8 bus, u8 devfn);
+int assign_device(struct domain *d, u16 seg, u8 bus, u8 devfn);
+int deassign_device(struct domain *d, u16 seg, u8 bus, u8 devfn);
+int iommu_get_device_group(struct domain *d, u16 seg, u8 bus, u8 devfn,
XEN_GUEST_HANDLE_64(uint32) buf, int max_sdevs);
/* iommu_map_page() takes flags to direct the mapping operation. */
@@ -125,14 +125,14 @@ struct iommu_ops {
void (*dom0_init)(struct domain *d);
int (*add_device)(struct pci_dev *pdev);
int (*remove_device)(struct pci_dev *pdev);
- int (*assign_device)(struct domain *d, u8 bus, u8 devfn);
+ int (*assign_device)(struct domain *d, u16 seg, u8 bus, u8 devfn);
void (*teardown)(struct domain *d);
int (*map_page)(struct domain *d, unsigned long gfn, unsigned long mfn,
unsigned int flags);
int (*unmap_page)(struct domain *d, unsigned long gfn);
int (*reassign_device)(struct domain *s, struct domain *t,
- u8 bus, u8 devfn);
- int (*get_device_group_id)(u8 bus, u8 devfn);
+ u16 seg, u8 bus, u8 devfn);
+ int (*get_device_group_id)(u16 seg, u8 bus, u8 devfn);
void (*update_ire_from_apic)(unsigned int apic, unsigned int reg, unsigned
int value);
void (*update_ire_from_msi)(struct msi_desc *msi_desc, struct msi_msg
*msg);
void (*read_msi_from_ire)(struct msi_desc *msi_desc, struct msi_msg *msg);
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel